From d1eab19d09fb243c074749a189a6371c3ca8873f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 4 Nov 2026 21:41:21 -0800 Subject: [PATCH 1/6] test(evals): replay whether trimming thread history saves money Cache reads dominate list-price spend, but any cut has to repay the uncached rebuild it causes. These scripts replay local transcripts against T3's own pricing code to price cuts, compaction, and seed selection, with no model calls. Co-Authored-By: Claude Opus 5.5 --- evals/README.md | 1 + evals/cache-snip/README.md | 35 ++ evals/cache-snip/compose.ts | 90 +++ evals/cache-snip/decisions.ts | 155 +++++ evals/cache-snip/learn.py | 123 ++++ evals/cache-snip/relevance.py | 61 ++ evals/cache-snip/relevanceRows.ts | 74 +++ evals/cache-snip/relevanceSample.ts | 159 +++++ evals/cache-snip/relevanceScore.py | 66 ++ evals/cache-snip/replay.ts | 599 ++++++++++++++++++ evals/cache-snip/shape.ts | 151 +++++ evals/cache-snip/simulate.ts | 943 ++++++++++++++++++++++++++++ evals/cache-snip/spend.ts | 119 ++++ evals/cache-snip/usageRecords.ts | 150 +++++ evals/package.json | 8 + 15 files changed, 2734 insertions(+) create mode 100644 evals/cache-snip/README.md create mode 100644 evals/cache-snip/compose.ts create mode 100644 evals/cache-snip/decisions.ts create mode 100644 evals/cache-snip/learn.py create mode 100644 evals/cache-snip/relevance.py create mode 100644 evals/cache-snip/relevanceRows.ts create mode 100644 evals/cache-snip/relevanceSample.ts create mode 100644 evals/cache-snip/relevanceScore.py create mode 100644 evals/cache-snip/replay.ts create mode 100644 evals/cache-snip/shape.ts create mode 100644 evals/cache-snip/simulate.ts create mode 100644 evals/cache-snip/spend.ts create mode 100644 evals/cache-snip/usageRecords.ts diff --git a/evals/README.md b/evals/README.md index 7efe8b34d36dc..a9b3c44914abe 100644 --- a/evals/README.md +++ b/evals/README.md @@ -11,6 +11,7 @@ Write or run an eval when a change affects what a model sees or does: agent inst | [thread-split](thread-split/) | Offline local-model evaluation of whether a message can start an independent thread. | | [model-evaluations](model-evaluations/) | Whether blind judges use outcome evidence, respect task scope, ignore injected scoring instructions, and abstain when evidence is missing. | | [test-signal](test-signal/) | Whether a cheap model running the test-signal check flags low-signal tests. | +| [cache-snip](cache-snip/) | Whether trimming stale thread history saves money after the cache rebuild. Replays only, no model. | ## Run an eval diff --git a/evals/cache-snip/README.md b/evals/cache-snip/README.md new file mode 100644 index 0000000000000..41ad84b9fcb97 --- /dev/null +++ b/evals/cache-snip/README.md @@ -0,0 +1,35 @@ +# Cache snip + +Would removing stale history from a thread lower total input cost, once the cache rebuild is charged? This replays real sessions against T3's own pricing code. It calls no model, costs nothing, and writes numbers and hashes only, never chat text. + +Run against copies, and keep the output out of the repo: + +```sh +mkdir -p +cp ~/.t3/userdata/usage-model-rates.json ~/.t3/userdata/usage-scan-cache-v11.json +cd evals +pnpm cache-snip:spend # dollars by category, model, provider +pnpm cache-snip:shape 30 # prompt sizes, cold calls, long-context cost +pnpm cache-snip:replay 30 # transcripts to per-session call and block records +pnpm cache-snip:compose 30 # what prompts are made of +for rate in 1 0.3 0.1; do + CACHE_SNIP_REFETCH=$rate pnpm cache-snip:simulate 30 + cp /simulate-results.json /simulate-results-refetch$rate.json +done +pnpm cache-snip:decisions 30 # dollar net of compacting at each candidate moment +python3 cache-snip/learn.py # model against fixed rules, held out by session and by time +pnpm cache-snip:relevance-rows 30 # what was knowable about each old tool output at each cut +python3 cache-snip/relevance.py # still-needed model, then rerun simulate to rank seeds with it +pnpm cache-snip:relevance-sample # old tool outputs with the message they faced. Holds chat text, keep it local +/bin/python cache-snip/relevanceScore.py # one GPU job at a time +``` + +`learn.py` needs numpy, pandas, and scikit-learn. + +## Reading the numbers + +- A snip at call `t` costs a rebuild of everything after the first removed block, and saves that block's reads on every later call before the next compaction. `simulate.ts` charges both. Its no-snip replay must reproduce T3's priced cost, and it prints that check first. +- `CACHE_SNIP_REFETCH` is the share of wrong snips that make an agent fetch the content again. It decides most outcomes, so report every result across a range. +- `CACHE_SNIP_REEXPLORE=tokens,calls` charges what a fresh chat re-reads per head cut. Set it from real handoff sessions. +- "Later quoted" is how `replay.ts` decides a block was used. It misses silent use, so the relevance ceiling depends on it. +- Replays measure dollars, never answer quality. Lost context that degrades an agent's work without a re-fetch does not show up here. diff --git a/evals/cache-snip/compose.ts b/evals/cache-snip/compose.ts new file mode 100644 index 0000000000000..19b23eeac8af0 --- /dev/null +++ b/evals/cache-snip/compose.ts @@ -0,0 +1,90 @@ +// @effect-diagnostics nodeBuiltinImport:off globalConsole:off +// What is the prompt made of, weighted by how many times each token is re-read? +// usage: node evals/cache-snip/compose.ts [days] +import * as NodeFS from "node:fs"; +import * as NodePath from "node:path"; + +import type { Block, Replay } from "./replay.ts"; + +const dir = process.argv[2]; +if (dir === undefined) throw new Error("usage: compose.ts [days]"); +const days = Number(process.argv[3] ?? 30); +const GRACE_CALLS = 5; + +const replays = NodeFS.readdirSync(NodePath.join(dir, "replay")).map( + (name) => JSON.parse(NodeFS.readFileSync(NodePath.join(dir, "replay", name), "utf8")) as Replay, +); +const latest = Math.max(...replays.flatMap((replay) => replay.calls.map((call) => call.ts))); +const cutoff = latest - days * 86_400_000; + +const ageName = (age: number) => + age <= 10 ? "<=10" : age <= 50 ? "11-50" : age <= 200 ? "51-200" : ">200"; + +for (const provider of ["codex", "claude"]) { + const tally = new Map(); + const bump = (key: string, value: number) => tally.set(key, (tally.get(key) ?? 0) + value); + let negativeBase = 0; + let callCount = 0; + const epochPeaks: number[] = []; + for (const replay of replays.filter((candidate) => candidate.provider === provider)) { + let live: Block[] = []; + const byArrival = new Map(); + for (const block of replay.blocks) { + const list = byArrival.get(block.arrival) ?? []; + list.push(block); + byArrival.set(block.arrival, list); + } + let peak = 0; + replay.calls.forEach((call, index) => { + if (call.reset) { + if (peak > 0) epochPeaks.push(peak); + peak = 0; + live = []; + } + peak = Math.max(peak, call.prompt); + live.push(...(byArrival.get(index) ?? [])); + if (call.ts < cutoff) return; + callCount += 1; + let attributed = 0; + for (const block of live) { + attributed += block.tokens; + if (block.kind !== "tool_output") { + bump(`kind:${block.kind}`, block.tokens); + continue; + } + const age = index - block.arrival; + bump("kind:tool_output", block.tokens); + bump(`age:${ageName(age)}`, block.tokens); + bump(`class:${block.toolClass ?? "none"}`, block.tokens); + if (block.lastUse + GRACE_CALLS < index) bump("dead:strict", block.tokens); + if (block.lastUseLoose + GRACE_CALLS < index) bump("dead:loose", block.tokens); + if (block.supersededAt !== null && block.supersededAt <= index) { + bump("superseded", block.tokens); + } + if ( + (block.supersededAt !== null && block.supersededAt <= index) || + (block.rereadAt !== null && block.rereadAt <= index) + ) { + bump("superseded+reread", block.tokens); + } + } + if (attributed > call.prompt * 1.02) negativeBase += 1; + bump("base", Math.max(0, call.prompt - attributed)); + bump("prompt", call.prompt); + }); + if (peak > 0) epochPeaks.push(peak); + } + const prompt = tally.get("prompt") ?? 1; + const sorted = epochPeaks.toSorted((a, b) => a - b); + const q = (value: number) => sorted[Math.floor(value * sorted.length)] ?? 0; + console.log( + `\n== ${provider}: prompt-weighted composition over ${callCount} calls, ${(prompt / 1e9).toFixed(2)}B prompt tokens`, + ); + for (const [key, value] of [...tally.entries()].toSorted(([a], [b]) => a.localeCompare(b))) { + if (key === "prompt") continue; + console.log(` ${key.padEnd(20)} ${((value / prompt) * 100).toFixed(1).padStart(5)}%`); + } + console.log( + ` calls with attributed > prompt: ${negativeBase}. Epoch peak prompt p10 ${q(0.1)} p50 ${q(0.5)} p90 ${q(0.9)} (${sorted.length} epochs)`, + ); +} diff --git a/evals/cache-snip/decisions.ts b/evals/cache-snip/decisions.ts new file mode 100644 index 0000000000000..2ac944683c758 --- /dev/null +++ b/evals/cache-snip/decisions.ts @@ -0,0 +1,155 @@ +// @effect-diagnostics nodeBuiltinImport:off globalConsole:off +// One row per moment a compaction could fire: what was knowable then, and what compacting there really saved. +// usage: node evals/cache-snip/decisions.ts [days] > decisions.csv +import * as NodeFS from "node:fs"; +import * as NodeCrypto from "node:crypto"; +import * as NodePath from "node:path"; + +import type { Block, Replay } from "./replay.ts"; +import { LONG_LIMIT, makeRateLookup, runEpoch, splitEpochs, type RateMode } from "./simulate.ts"; + +const dir = process.argv[2]; +if (dir === undefined) throw new Error("usage: decisions.ts [days]"); +const days = Number(process.argv[3] ?? 30); +const MIN_PROMPT = 50_000; +const REFETCH_RATES = [1, 0.3, 0.1]; + +const replays = NodeFS.readdirSync(NodePath.join(dir, "replay")).map( + (name) => JSON.parse(NodeFS.readFileSync(NodePath.join(dir, "replay", name), "utf8")) as Replay, +); +const latest = Math.max(...replays.flatMap((replay) => replay.calls.map((call) => call.ts))); +const cutoff = latest - days * 86_400_000; +const mode = (process.env["CACHE_SNIP_RATES"] ?? "trueList") as RateMode; +const ratesFor = makeRateLookup(dir, mode); + +const header = [ + "session", + "provider", + "epoch", + "t", + "ts", + "prompt", + "offset", + "sessionCalls", + "userTurns", + "callsPerTurn", + "gapMinutes", + "isCold", + "isUserTurn", + "prevReadShare", + "missOverRead", + "readUsdPerCall", + "longTier", + "growthPerCall", + "toolShare", + "idle10Share", + "idle30Share", + "priorTurnShare", + "callsThisTurn", + "userEstimate", + "textKey", + ...REFETCH_RATES.map((rate) => `net${rate}`), +]; +const rows: string[] = [header.join(",")]; +const hash = (value: string) => + NodeCrypto.createHash("sha1").update(value).digest("hex").slice(0, 12); + +for (const replay of replays) { + const session = hash(replay.file); + let sessionCalls = 0; + let userTurns = 0; + splitEpochs(replay, ratesFor).forEach((epoch, epochIndex) => { + const byArrival = new Map(); + for (const block of epoch.blocks) { + const list = byArrival.get(block.arrival) ?? []; + list.push(block); + byArrival.set(block.arrival, list); + } + const never = () => []; + const baseline = REFETCH_RATES.map( + (refetchRate) => runEpoch(epoch, never, cutoff, { refetchRate }).usd, + ); + const live: Block[] = []; + let lastUserCall = epoch.firstIndex; + epoch.calls.forEach((call, offset) => { + const t = epoch.firstIndex + offset; + const arriving = byArrival.get(t) ?? []; + live.push(...arriving); + sessionCalls += 1; + const userBlocks = arriving.filter((block) => block.kind === "user"); + const isUserTurn = epoch.turnStarts.has(t); + if (isUserTurn) userTurns += 1; + const callsThisTurn = t - lastUserCall; + if (isUserTurn) lastUserCall = t; + const previous = epoch.calls[offset - 1]; + const gapMinutes = previous === undefined ? 0 : (call.ts - previous.ts) / 60_000; + const isCold = gapMinutes >= 60; + if (offset === 0 || call.prompt < MIN_PROMPT || call.ts < cutoff) return; + if (!isUserTurn && !isCold) return; + + let lastUser = -1; + for (const block of live) { + if (block.kind === "user" && block.arrival < t) + lastUser = Math.max(lastUser, block.arrival); + } + let tool = 0; + let idle10 = 0; + let idle30 = 0; + let prior = 0; + for (const block of live) { + if (block.kind !== "tool_output") continue; + tool += block.tokens; + let last = block.arrival; + for (const use of block.uses) { + if (use >= t) break; + last = use; + } + if (t - last > 10) idle10 += block.tokens; + if (t - last > 30) idle30 += block.tokens; + if (block.arrival < lastUser) prior += block.tokens; + } + const rates = epoch.rates[offset]!; + const net = REFETCH_RATES.map( + (refetchRate, index) => + baseline[index]! - + runEpoch(epoch, never, cutoff, { refetchRate, compactCalls: new Set([t]) }).usd, + ); + const longTier = rates.longInput !== null && call.prompt > LONG_LIMIT; + rows.push( + [ + session, + replay.provider, + epochIndex, + t, + call.ts, + call.prompt, + offset, + sessionCalls, + userTurns, + (sessionCalls / Math.max(1, userTurns)).toFixed(2), + gapMinutes.toFixed(2), + isCold ? 1 : 0, + isUserTurn ? 1 : 0, + previous === undefined ? 0 : (previous.read / Math.max(1, previous.prompt)).toFixed(3), + (rates.miss / Math.max(rates.read, 1e-12)).toFixed(1), + (call.prompt * rates.read * (longTier ? rates.longInput! : 1)).toFixed(5), + longTier ? 1 : 0, + ((call.prompt - epoch.calls[0]!.prompt) / offset).toFixed(1), + (tool / call.prompt).toFixed(3), + (idle10 / call.prompt).toFixed(3), + (idle30 / call.prompt).toFixed(3), + (prior / call.prompt).toFixed(3), + callsThisTurn, + userBlocks.reduce((sum, block) => sum + block.estimate, 0).toFixed(0), + userBlocks[0]?.textKey ?? "", + ...net.map((value) => value.toFixed(5)), + ].join(","), + ); + }); + }); +} +NodeFS.writeFileSync( + NodePath.join(dir, mode === "trueList" ? "decisions.csv" : `decisions-${mode}.csv`), + `${rows.join("\n")}\n`, +); +console.log(`candidates ${rows.length - 1}`); diff --git a/evals/cache-snip/learn.py b/evals/cache-snip/learn.py new file mode 100644 index 0000000000000..6137bd6c2d192 --- /dev/null +++ b/evals/cache-snip/learn.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""Can a model choose when to compact better than a fixed rule? Held-out sessions, real dollars. + +usage: python3 evals/cache-snip/learn.py +Numeric features only, so it runs on CPU and needs no chat text. Run decisions.ts first. +""" +import json +import os +import sys + +import numpy as np +import pandas as pd +from sklearn.ensemble import HistGradientBoostingClassifier +from sklearn.model_selection import GroupKFold + +artifact = sys.argv[1] +frame = pd.read_csv(os.path.join(artifact, "decisions.csv")) +results = json.load(open(os.path.join(artifact, "simulate-results-refetch1.json"))) +BASE = {provider: results[provider]["never"][0] for provider in results} +FEATURES = [ + "prompt", "offset", "sessionCalls", "userTurns", "callsPerTurn", "gapMinutes", "isCold", + "isUserTurn", "prevReadShare", "missOverRead", "readUsdPerCall", "longTier", "growthPerCall", + "toolShare", "idle10Share", "idle30Share", "priorTurnShare", "callsThisTurn", "userEstimate", +] +THRESHOLDS = [100, 125, 150, 175, 200, 225, 250, 260, 270, 300] +REFETCH_RATES = ["1", "0.3", "0.1"] +MOMENTS = { + "any moment": lambda rows: np.ones(len(rows), bool), + "user turn": lambda rows: rows.isUserTurn.to_numpy() == 1, + "cold": lambda rows: rows.isCold.to_numpy() == 1, +} + + +def per_session(rows, chosen, column): + """Compact at the first chosen moment of each epoch. Dollars saved per session.""" + picked = rows[chosen].sort_values("t").groupby(["session", "epoch"], sort=False).head(1) + return picked.groupby("session")[column].sum() + + +def oracle(rows, column): + best = rows.groupby(["session", "epoch"])[column].max().clip(lower=0) + return best.groupby("session").sum() + + +def learned_rule(train, test, column, moment): + allowed = MOMENTS[moment] + def saved(rows, threshold): + return per_session(rows, allowed(rows) & (rows.prompt.to_numpy() >= threshold * 1000), column).sum() + best = max(THRESHOLDS, key=lambda threshold: saved(train, threshold)) + return per_session(test, allowed(test) & (test.prompt.to_numpy() >= best * 1000), column), best + + +def stopping_model(train, test, column): + """Labels a moment positive only when it is near the best moment left in the epoch, since waiting is an option.""" + ordered = train.sort_values(["session", "epoch", "t"]) + remaining_best = ordered.groupby(["session", "epoch"])[column].transform(lambda values: values[::-1].cummax()[::-1]) + labels = ((ordered[column] > 0) & (ordered[column] >= 0.9 * remaining_best)).astype(int) + model = HistGradientBoostingClassifier( + max_depth=3, max_iter=200, learning_rate=0.05, min_samples_leaf=15, random_state=7 + ) + model.fit(ordered[FEATURES], labels, sample_weight=ordered[column].abs().clip(lower=1e-3)) + return per_session(test, model.predict_proba(test[FEATURES])[:, 1] > 0.5, column) + + +def interval(values, sessions, base): + aligned = values.reindex(sessions).fillna(0).to_numpy() + draws = aligned[np.random.default_rng(7).integers(0, len(aligned), (4000, len(aligned)))].sum(axis=1) + low, high = np.percentile(draws, [2.5, 97.5]) + return aligned.sum() / base * 100, low / base * 100, high / base * 100 + + +def compare(rows, title): + print(f"\n== {title}: % of provider spend saved [95% CI], one compaction per epoch") + for provider in ["codex", "claude"]: + subset = rows[rows.provider == provider].reset_index(drop=True) + sessions = subset.session.unique() + for rate in REFETCH_RATES: + column = f"net{rate}" + collected = {f"rule, {moment}": pd.Series(dtype=float) for moment in MOMENTS} + collected["model"] = pd.Series(dtype=float) + chosen_thresholds = {moment: [] for moment in MOMENTS} + for train_index, test_index in GroupKFold(n_splits=5).split(subset, groups=subset.session): + train, test = subset.iloc[train_index], subset.iloc[test_index] + for moment in MOMENTS: + saved, threshold = learned_rule(train, test, column, moment) + chosen_thresholds[moment].append(threshold) + collected[f"rule, {moment}"] = pd.concat([collected[f"rule, {moment}"], saved]) + collected["model"] = pd.concat([collected["model"], stopping_model(train, test, column)]) + base = BASE[provider] + print(f"\n{provider}, baseline ${base:.0f}, refetch {rate}") + print(" {:28s} {:6.1f} [{:6.1f},{:6.1f}]".format("oracle", *interval(oracle(subset, column), sessions, base))) + for name, values in collected.items(): + note = "" + if name.startswith("rule"): + note = f" thresholds {sorted(set(chosen_thresholds[name.split(', ')[1]]))}" + print(" {:28s} {:6.1f} [{:6.1f},{:6.1f}]{}".format(name, *interval(values, sessions, base), note)) + for moment in MOMENTS: + difference = collected["model"].reindex(sessions).fillna(0) - collected[f"rule, {moment}"].reindex(sessions).fillna(0) + print(" {:28s} {:6.1f} [{:6.1f},{:6.1f}]".format(f"model minus rule, {moment}", *interval(difference, sessions, base))) + + +compare(frame, "held out by session") + +latest = frame.ts.max() +split = latest - 7 * 86_400_000 +past, recent = frame[frame.ts < split], frame[frame.ts >= split] +print(f"\n== trained on the past, tested on the last week: {len(past)} training moments, {len(recent)} test moments") +for provider in ["codex", "claude"]: + train, test = past[past.provider == provider], recent[recent.provider == provider] + sessions = test.session.unique() + if len(train) < 50 or len(test) < 20: + print(f"{provider}: too few moments, train {len(train)}, test {len(test)}. Pooling providers instead.") + train = past + for rate in REFETCH_RATES: + column = f"net{rate}" + rule_saved, threshold = learned_rule(train, test, column, "any moment") + model_saved = stopping_model(train, test, column) + base = BASE[provider] + print( + f"{provider} refetch {rate}: oracle {interval(oracle(test, column), sessions, base)[0]:5.1f}, " + f"rule(>{threshold}k) {interval(rule_saved, sessions, base)[0]:5.1f}, model {interval(model_saved, sessions, base)[0]:5.1f}, " + f"model minus rule {interval(model_saved.reindex(sessions).fillna(0) - rule_saved.reindex(sessions).fillna(0), sessions, base)}" + ) diff --git a/evals/cache-snip/relevance.py b/evals/cache-snip/relevance.py new file mode 100644 index 0000000000000..b7285dc3b3e0b --- /dev/null +++ b/evals/cache-snip/relevance.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""How well can numeric features alone predict that an old tool output is still needed? + +usage: python3 evals/cache-snip/relevance.py +Run relevanceRows.ts first. Writes out-of-fold scores that simulate.ts ranks seeds with. +""" +import os +import sys + +import numpy as np +import pandas as pd +from sklearn.ensemble import HistGradientBoostingClassifier +from sklearn.metrics import roc_auc_score +from sklearn.model_selection import GroupKFold + +artifact = sys.argv[1] +rows = pd.read_csv(os.path.join(artifact, "relevance-rows.csv")) +FEATURES = [ + "tokens", "age", "idle", "usesSoFar", "turnsSinceArrival", "turnsSinceUse", "namedNow", + "namedBefore", "reread", "superseded", "toolClass", "prompt", "isCold", +] +print(f"rows {len(rows)}, sessions {rows.session.nunique()}, used later {rows.used.mean():.3f}, by tokens {np.average(rows.used, weights=rows.tokens):.3f}") +print("moments per session p50", int(rows.groupby('session').t.nunique().median()), "tool outputs per moment p50 / p90", *[int(v) for v in np.percentile(rows.groupby(['session','t']).size(), [50, 90])]) + +scores = np.zeros(len(rows)) +for train_index, test_index in GroupKFold(n_splits=5).split(rows, groups=rows.session): + model = HistGradientBoostingClassifier(max_depth=4, max_iter=200, learning_rate=0.07, min_samples_leaf=50, random_state=7) + train = rows.iloc[train_index] + model.fit(train[FEATURES], train.used, sample_weight=np.sqrt(train.tokens)) + scores[test_index] = model.predict_proba(rows.iloc[test_index][FEATURES])[:, 1] +rows["score"] = scores + + +def auc(frame, column, sign=1): + if frame.used.nunique() < 2: + return float("nan") + return roc_auc_score(frame.used, sign * frame[column], sample_weight=frame.tokens) + + +print("\nToken-weighted AUC for 'still used later', held out by session:") +for name, frame in [("all", rows), ("codex", rows[rows.provider == "codex"]), ("claude", rows[rows.provider == "claude"]), ("older than 50 calls", rows[rows.age > 50]), ("idle over 30 calls", rows[rows.idle > 30])]: + print(f" {name:22s} model {auc(frame, 'score'):.3f} recency of last use {auc(frame, 'idle', -1):.3f} age {auc(frame, 'age', -1):.3f} n {len(frame)}") + +named = rows[rows.namedNow == 1] +print(f"\nNamed by the new message: {len(named)} rows, used later {np.average(named.used, weights=named.tokens):.3f}. Not named: {np.average(rows[rows.namedNow == 0].used, weights=rows[rows.namedNow == 0].tokens):.3f}") +for low, high in [(0, 5), (5, 20), (20, 50), (50, 100), (100, 10**9)]: + bucket = rows[(rows.idle >= low) & (rows.idle < high)] + if len(bucket): + print(f" idle {low:>3}-{high if high < 10**9 else 'inf':<4} share of tokens {bucket.tokens.sum() / rows.tokens.sum():.2f}, used later {np.average(bucket.used, weights=bucket.tokens):.3f}") + +latest = rows.ts.max() +split = latest - 7 * 86_400_000 +past, recent = rows[rows.ts < split], rows[rows.ts >= split] +if len(past) > 1000 and len(recent) > 1000: + model = HistGradientBoostingClassifier(max_depth=4, max_iter=200, learning_rate=0.07, min_samples_leaf=50, random_state=7) + model.fit(past[FEATURES], past.used, sample_weight=np.sqrt(past.tokens)) + recent = recent.assign(score=model.predict_proba(recent[FEATURES])[:, 1]) + print(f"\nTrained on the past, tested on the last week: AUC {auc(recent, 'score'):.3f}, recency alone {auc(recent, 'idle', -1):.3f}") + +keys = rows.session + ":" + rows.t.astype(str) + ":" + rows.block.astype(str) +pd.DataFrame({"key": keys, "score": rows.score.round(4)}).to_csv(os.path.join(artifact, "relevance-scores.csv"), index=False, header=False) diff --git a/evals/cache-snip/relevanceRows.ts b/evals/cache-snip/relevanceRows.ts new file mode 100644 index 0000000000000..3e99d294b6b71 --- /dev/null +++ b/evals/cache-snip/relevanceRows.ts @@ -0,0 +1,74 @@ +// @effect-diagnostics nodeBuiltinImport:off globalConsole:off +// One row per tool output at each moment a head cut could fire: what was knowable then, and whether it was used later. +// usage: node evals/cache-snip/relevanceRows.ts [days] +import * as NodeFS from "node:fs"; +import * as NodePath from "node:path"; + +import type { Block, Replay } from "./replay.ts"; +import { makeRateLookup, sessionKey, splitEpochs } from "./simulate.ts"; + +const dir = process.argv[2]; +if (dir === undefined) throw new Error("usage: relevanceRows.ts [days]"); +const days = Number(process.argv[3] ?? 30); +const MIN_PROMPT = 100_000; +const MIN_BLOCK_TOKENS = 50; +const CLASSES = ["read", "search", "shell", "edit", "mcp", "agent", "other"]; + +const replays = NodeFS.readdirSync(NodePath.join(dir, "replay")).map( + (name) => JSON.parse(NodeFS.readFileSync(NodePath.join(dir, "replay", name), "utf8")) as Replay, +); +const latest = Math.max(...replays.flatMap((replay) => replay.calls.map((call) => call.ts))); +const cutoff = latest - days * 86_400_000; +const ratesFor = makeRateLookup(dir, "trueList"); + +const out = NodeFS.createWriteStream(NodePath.join(dir, "relevance-rows.csv")); +out.write( + "session,provider,t,ts,block,tokens,age,idle,usesSoFar,turnsSinceArrival,turnsSinceUse,namedNow,namedBefore,reread,superseded,toolClass,prompt,isCold,used,usedLoose\n", +); +let rows = 0; +let moments = 0; +for (const replay of replays) { + const session = sessionKey(replay.file); + const turnStarts = [...new Set(replay.turnStarts)].toSorted((a, b) => a - b); + for (const epoch of splitEpochs(replay, ratesFor)) { + const live: Block[] = []; + const byArrival = new Map(); + for (const block of epoch.blocks) { + const list = byArrival.get(block.arrival) ?? []; + list.push(block); + byArrival.set(block.arrival, list); + } + epoch.calls.forEach((call, offset) => { + const t = epoch.firstIndex + offset; + live.push(...(byArrival.get(t) ?? [])); + const previous = epoch.calls[offset - 1]; + const isCold = previous !== undefined && (call.ts - previous.ts) / 60_000 >= 60; + if (call.ts < cutoff || call.prompt < MIN_PROMPT || offset === 0) return; + if (!epoch.turnStarts.has(t) && !isCold) return; + moments += 1; + const turnsBefore = (index: number) => + turnStarts.filter((start) => start > index && start <= t).length; + for (const block of live) { + if (block.kind !== "tool_output" || block.tokens < MIN_BLOCK_TOKENS || block.arrival >= t) + continue; + let last = block.arrival; + let usesSoFar = 0; + for (const use of block.uses) { + if (use >= t) break; + last = use; + usesSoFar += 1; + } + const used = block.uses.some((use) => use >= t) ? 1 : 0; + const usedLoose = block.lastUseLoose >= t ? 1 : 0; + out.write( + `${session},${replay.provider},${t},${call.ts},${block.id},${block.tokens},${t - block.arrival},${t - last},${usesSoFar},${turnsBefore(block.arrival)},${turnsBefore(last)},${block.mentions.includes(t) ? 1 : 0},${block.mentions.some((at) => at < t) ? 1 : 0},${block.rereadAt !== null && block.rereadAt <= t ? 1 : 0},${block.supersededAt !== null && block.supersededAt <= t ? 1 : 0},${CLASSES.indexOf(block.toolClass ?? "other")},${call.prompt},${isCold ? 1 : 0},${used},${usedLoose}\n`, + ); + rows += 1; + } + }); + } +} +out.end(); +console.log( + `rows ${rows}, moments ${moments}, tool outputs per moment ${(rows / Math.max(moments, 1)).toFixed(0)}`, +); diff --git a/evals/cache-snip/relevanceSample.ts b/evals/cache-snip/relevanceSample.ts new file mode 100644 index 0000000000000..161e1a27f8d10 --- /dev/null +++ b/evals/cache-snip/relevanceSample.ts @@ -0,0 +1,159 @@ +// @effect-diagnostics nodeBuiltinImport:off globalConsole:off +// Samples old tool outputs with the message they faced, for scoring on a local model. Writes chat text, so keep the output local. +// usage: node evals/cache-snip/relevanceSample.ts [rows per split] +import * as NodeCrypto from "node:crypto"; +import * as NodeFS from "node:fs"; +import * as NodePath from "node:path"; + +import { readClaude, readCodex, type Replay, type TextCapture } from "./replay.ts"; +import { sessionKey } from "./simulate.ts"; +import { loadApiCalls, type ApiCall } from "./usageRecords.ts"; + +const dir = process.argv[2]; +if (dir === undefined) throw new Error("usage: relevanceSample.ts [rows per split]"); +const perSplit = Number(process.argv[3] ?? 300); +const MIN_IDLE_CALLS = 5; +const MIN_TOKENS = 100; +const MAX_ROWS_PER_SESSION = 40; +const POSITIVE_SHARE = 0.4; + +interface Row { + session: string; + provider: string; + t: number; + block: number; + tokens: number; + age: number; + idle: number; + used: number; + split: "calibration" | "sealed"; + key: number; +} + +const stableKey = (value: string) => + NodeCrypto.createHash("sha1").update(value).digest().readUInt32BE(0); + +const lines = NodeFS.readFileSync(NodePath.join(dir, "relevance-rows.csv"), "utf8") + .trim() + .split("\n"); +const names = lines[0]!.split(","); +const column = (name: string) => names.indexOf(name); +const rows: Row[] = lines.slice(1).flatMap((line) => { + const cells = line.split(","); + const session = cells[column("session")]!; + const tokens = Number(cells[column("tokens")]); + const idle = Number(cells[column("idle")]); + if (tokens < MIN_TOKENS || idle <= MIN_IDLE_CALLS) return []; + const t = cells[column("t")]!; + const block = cells[column("block")]!; + return [ + { + session, + provider: cells[column("provider")]!, + t: Number(t), + block: Number(block), + tokens, + age: Number(cells[column("age")]), + idle, + used: Number(cells[column("used")]), + // Whole sessions stay on one side, so a model never scores a thread it was tuned on. + split: stableKey(`split:${session}`) % 10 < 7 ? "calibration" : "sealed", + key: stableKey(`row:${session}:${t}:${block}`), + }, + ]; +}); + +function pick(split: Row["split"], used: number, count: number) { + const held = new Map(); + const chosen: Row[] = []; + for (const row of rows + .filter((candidate) => candidate.split === split && candidate.used === used) + .toSorted((a, b) => a.key - b.key)) { + const taken = held.get(row.session) ?? 0; + if (taken >= MAX_ROWS_PER_SESSION || chosen.length >= count) continue; + held.set(row.session, taken + 1); + chosen.push(row); + } + return chosen; +} + +const sample = (["calibration", "sealed"] as const).flatMap((split) => [ + ...pick(split, 1, Math.round(perSplit * POSITIVE_SHARE)), + ...pick(split, 0, Math.round(perSplit * (1 - POSITIVE_SHARE))), +]); + +const replays = new Map( + NodeFS.readdirSync(NodePath.join(dir, "replay")).map((name) => { + const replay = JSON.parse( + NodeFS.readFileSync(NodePath.join(dir, "replay", name), "utf8"), + ) as Replay; + return [sessionKey(replay.file), replay] as const; + }), +); +const callsByFile = new Map(); +for (const call of loadApiCalls(NodePath.join(dir, "usage-scan-cache-v11.json"))) { + const list = callsByFile.get(call.file) ?? []; + list.push(call); + callsByFile.set(call.file, list); +} + +const clip = (text: string, head: number, tail = 0) => + text.length <= head + tail + ? text + : `${text.slice(0, head)}\n[... ${text.length - head - tail} characters omitted ...]\n${tail > 0 ? text.slice(-tail) : ""}`; + +const out = NodeFS.createWriteStream(NodePath.join(dir, "relevance-sample.jsonl")); +let written = 0; +let mismatched = 0; +const bySession = new Map(); +for (const row of sample) { + const list = bySession.get(row.session) ?? []; + list.push(row); + bySession.set(row.session, list); +} +for (const [session, sessionRows] of bySession) { + const replay = replays.get(session); + if (replay === undefined) continue; + const capture: TextCapture = { + wanted: new Set(sessionRows.map((row) => row.block)), + outputs: new Map(), + calls: new Map(), + users: [], + answers: [], + }; + const calls = callsByFile.get(replay.file) ?? []; + const read = replay.provider === "codex" ? readCodex : readClaude; + const builder = await read(replay.file, calls, () => 0, capture); + for (const row of sessionRows) { + const output = capture.outputs.get(row.block); + if (output === undefined || builder.blocks[row.block]?.tokens !== row.tokens) { + mismatched += 1; + continue; + } + const newest = capture.users.filter((user) => user.call <= row.t).slice(-3); + const newMessage = newest.at(-1)?.text ?? ""; + const answer = capture.answers.filter((item) => item.call < row.t).at(-1)?.text ?? ""; + out.write( + `${JSON.stringify({ + id: `${session}:${row.t}:${row.block}`, + split: row.split, + provider: row.provider, + used: row.used, + tokens: row.tokens, + idle: row.idle, + age: row.age, + state: { + new_user_message: clip(newMessage, 2_000), + earlier_user_messages: newest.slice(0, -1).map((user) => clip(user.text, 500)), + latest_assistant_answer: clip(answer, 1_000), + earlier_tool_call: capture.calls.get(row.block) ?? "", + earlier_tool_output: clip(output, 6_000, 1_500), + calls_since_it_was_last_used: row.idle, + }, + })}\n`, + ); + written += 1; + } +} +out.end(); +console.log(`sampled ${written}, not found ${mismatched}, sessions ${bySession.size}`); diff --git a/evals/cache-snip/relevanceScore.py b/evals/cache-snip/relevanceScore.py new file mode 100644 index 0000000000000..c5b26f3cfb8fe --- /dev/null +++ b/evals/cache-snip/relevanceScore.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Scores the relevance sample on one local decision model. Resumable, one request at a time. + +usage: python relevanceScore.py [split] +Run under the mlx-decision virtualenv, and run one GPU job at a time. +""" +import json +import pathlib +import sys +import time + +import mlx_decision + +artifact = pathlib.Path(sys.argv[1]) +name, path = sys.argv[2], sys.argv[3] +split = sys.argv[4] if len(sys.argv) > 4 else "calibration" +PROMPTS = { + "needed": { + "type": "noul", + "instructions": "An agent is about to handle new_user_message. Would it still need the contents of earlier_tool_output, either to handle that message or to finish work still in progress? Answer yes only if it would need the actual contents, not just the fact that the tool ran.", + "criteria": { + "true": "The agent still needs these contents.", + "false": "These contents are no longer relevant: finished work, superseded results, or unrelated to the new message.", + }, + }, + "stale": { + "type": "noul", + "instructions": "Has earlier_tool_output gone stale for this conversation? Answer yes if it could be removed from the agent's context with no effect on how it handles new_user_message and the work still in progress. If removal could hurt, answer no.", + "criteria": { + "true": "Safe to remove: it no longer matters.", + "false": "Keep it: it may still matter.", + }, + }, +} +rows = [json.loads(line) for line in (artifact / "relevance-sample.jsonl").read_text().splitlines()] +rows = [row for row in rows if row["split"] == split] +out = artifact / f"relevance-scores-{name}.jsonl" +done = set() +if out.exists(): + for line in out.read_text().splitlines(): + row = json.loads(line) + done.add((row["id"], row["prompt"])) +model = mlx_decision.load(path) +print(name, "loaded", model.info().get("max_input_tokens"), flush=True) +with out.open("a") as handle: + for prompt, question in PROMPTS.items(): + for row in rows: + if (row["id"], prompt) in done: + continue + started = time.perf_counter() + result = {"id": row["id"], "prompt": prompt, "model": name, "split": split} + try: + response = model.decide_request({"state": row["state"], "questions": {"q": question}}) + result["score"] = None if response.truncated else float(response.answers["q"].noul) + if response.truncated: + result["error"] = "truncated" + except Exception as error: + if "METAL" in repr(error): + raise + result["score"] = None + result["error"] = repr(error)[:200] + result["seconds"] = time.perf_counter() - started + handle.write(json.dumps(result) + "\n") + handle.flush() + print(name, prompt, "done", flush=True) +print(name, "all-done", flush=True) diff --git a/evals/cache-snip/replay.ts b/evals/cache-snip/replay.ts new file mode 100644 index 0000000000000..d51b3e693f631 --- /dev/null +++ b/evals/cache-snip/replay.ts @@ -0,0 +1,599 @@ +// @effect-diagnostics nodeBuiltinImport:off globalConsole:off +// Rebuilds what each billed request carried, from local transcripts. Output holds numbers and hashes, never chat text. +// usage: node evals/cache-snip/replay.ts [days] +import * as NodeCrypto from "node:crypto"; +import * as NodeFS from "node:fs"; +import * as NodePath from "node:path"; +import * as NodeReadline from "node:readline"; + +import { loadApiCalls, promptTokens, type ApiCall } from "./usageRecords.ts"; + +export type BlockKind = "tool_output" | "tool_call" | "assistant" | "reasoning" | "user"; +export type ToolClass = "read" | "search" | "shell" | "edit" | "mcp" | "agent" | "other"; + +export interface Block { + /** Position in the replay's block list. */ + id: number; + /** Index of the first call whose prompt contains this block. */ + arrival: number; + /** First call whose prompt no longer contains it, after a real compaction or reset. */ + end: number | null; + tokens: number; + /** Raw size before prompt growth pinned `tokens`. Fits the weights in `WEIGHTS`. */ + estimate: number; + kind: BlockKind; + toolClass: ToolClass | null; + /** Responses that quoted something only this block supplied, in order. */ + uses: number[]; + /** User messages that named something only this block supplied. Known before the turn runs. */ + mentions: number[]; + /** Last of `uses`, or arrival. */ + lastUse: number; + /** Same, also counting plain long words. A looser reading of "used". */ + lastUseLoose: number; + /** First later call that made this block out of date: same command re-run, or its file edited. */ + supersededAt: number | null; + /** First later call that read any part of the same file again. Weaker than `supersededAt`. */ + rereadAt: number | null; + commandKey: string | null; + /** Hashed paths this block read. Lets the natural experiment spot re-reads across compactions. */ + fileKeys: string[]; + /** Transcript line, joining earlier blind-judge labels. Null where the source has none. */ + line: number | null; + /** Hash of a user message's text, joining thread-split labels. */ + textKey: string | null; +} + +export interface Call { + ts: number; + model: string; + prompt: number; + read: number; + miss: number; + output: number; + usd: number; + /** Prompt shrank by over 20% since the previous call. */ + reset: boolean; +} + +export interface Replay { + file: string; + provider: string; + calls: Call[]; + blocks: Block[]; + /** Calls that start a real user turn, excluding harness messages that only look like one. */ + turnStarts: number[]; +} + +// Fitted against reported prompt growth (compose.ts's companion regression): Claude R2 0.99, Codex R2 0.50. +const WEIGHTS: Readonly>>> = { + codex: { tool_output: 0.85, tool_call: 1.1, assistant: 1.1, reasoning: 0.25, user: 1 }, + claude: { tool_output: 1.6, tool_call: 1.55, assistant: 1.55, reasoning: 1.4, user: 1.4 }, +}; +const IMAGE_TOKEN_ESTIMATE = 1500; +const MAX_SCAN_CHARS = 200_000; +const RESET_DROP = 0.8; + +const hash = (value: string) => + NodeCrypto.createHash("sha1").update(value).digest("hex").slice(0, 12); +export const hashUserText = (text: string) => hash(text.replace(/\s+/g, " ").trim().slice(0, 200)); + +const CANDIDATE = /[A-Za-z0-9_][A-Za-z0-9_\-./]{3,}/g; +const strictToken = (token: string) => + token.length >= 5 && (/[_\-./]/.test(token) || /\d/.test(token) || /[a-z][A-Z]/.test(token)); +const looseToken = (token: string) => token.length >= 8 && /^[A-Za-z]+$/.test(token); + +function tokensOf(text: string) { + const strict = new Set(); + const loose = new Set(); + for (const match of text.slice(0, MAX_SCAN_CHARS).matchAll(CANDIDATE)) { + const token = match[0].replace(/[./\-_]+$/, ""); + if (strictToken(token)) strict.add(token); + else if (looseToken(token)) loose.add(token.toLowerCase()); + } + return { strict, loose }; +} + +/** Local-only text for building a labelled set. Replays themselves never hold chat text. */ +export interface TextCapture { + readonly wanted: ReadonlySet; + readonly outputs: Map; + readonly calls: Map; + readonly users: { call: number; text: string }[]; + readonly answers: { call: number; text: string }[]; +} + +interface ToolMeta { + readonly label?: string; + readonly toolClass: ToolClass; + readonly commandKey: string | null; + readonly reads: readonly string[]; + readonly edits: readonly string[]; +} + +const READ_COMMAND = + /\b(?:cat|sed\s+-n|nl|head|tail|less|bat)\b[^|;&\n]*?([\w./~-]+\.[A-Za-z0-9]+)/g; +const PATCH_FILE = /\*\*\* (?:Update|Add|Delete) File: ([^\s"\\]+)/g; +const SEARCH_COMMAND = /(^|[\s;&|(])(rg|grep|find|ls|fd|tree|git\s+(log|grep|show|diff|status))\b/; + +/** One classifier per provider shape, fed by the raw command text. */ +function classifyCommand(text: string): ToolMeta { + const reads = [...text.matchAll(READ_COMMAND)].flatMap((match) => match[1] ?? []); + const edits = [...text.matchAll(PATCH_FILE)].flatMap((match) => match[1] ?? []); + const toolClass: ToolClass = + edits.length > 0 + ? "edit" + : reads.length > 0 + ? "read" + : SEARCH_COMMAND.test(text) + ? "search" + : "shell"; + return { + label: text.slice(0, 400), + toolClass, + commandKey: hash(text.replace(/\s+/g, " ").trim()), + reads, + edits, + }; +} + +class ReplayBuilder { + private readonly weights: Readonly>; + private readonly capture: TextCapture | undefined; + constructor(weights: Readonly>, capture?: TextCapture) { + this.weights = weights; + this.capture = capture; + } + readonly calls: Call[] = []; + readonly blocks: Block[] = []; + readonly turnStarts: number[] = []; + private live: Block[] = []; + private pendingArrivals: Block[] = []; + private strictProvider = new Map(); + private looseProvider = new Map(); + private lastByCommand = new Map(); + private readersByFile = new Map(); + /** Codex logs a response's usage before its tool output and again after, so inputs in between belong to the next call. */ + inputOffset = 0; + /** Transcript line being read. */ + line: number | null = null; + + private add(kind: BlockKind, estimate: number, arrival: number, meta: ToolMeta | null): Block { + const block: Block = { + id: this.blocks.length, + arrival, + end: null, + tokens: 0, + estimate, + kind, + toolClass: meta?.toolClass ?? null, + uses: [], + mentions: [], + lastUse: arrival, + lastUseLoose: arrival, + supersededAt: null, + rereadAt: null, + commandKey: meta?.commandKey ?? null, + fileKeys: (meta?.reads ?? []).map(hash), + line: this.line, + textKey: null, + }; + this.blocks.push(block); + this.pendingArrivals.push(block); + return block; + } + + markTurnStart() { + this.turnStarts.push(this.calls.length + this.inputOffset); + } + + userMessage(text: string, images = 0) { + const block = this.add( + "user", + text.length / 4 + images * IMAGE_TOKEN_ESTIMATE, + this.calls.length + this.inputOffset, + null, + ); + block.textKey = hashUserText(text); + this.capture?.users.push({ call: block.arrival, text }); + const { strict } = tokensOf(text); + const mentioned = new Set(); + for (const token of strict) { + const provider = this.strictProvider.get(token); + if (provider !== undefined) mentioned.add(provider); + } + for (const provider of mentioned) provider.mentions.push(block.arrival); + } + + toolOutput(text: string, images: number, meta: ToolMeta | null) { + const block = this.add( + "tool_output", + text.length / 4 + images * IMAGE_TOKEN_ESTIMATE, + this.calls.length + this.inputOffset, + meta, + ); + if (this.capture?.wanted.has(block.id) === true) { + this.capture.outputs.set(block.id, text); + this.capture.calls.set(block.id, meta?.label ?? ""); + } + const { strict, loose } = tokensOf(text); + for (const token of strict) this.strictProvider.set(token, block); + for (const token of loose) this.looseProvider.set(token, block); + if (meta !== null) { + if (meta.commandKey !== null) { + const earlier = this.lastByCommand.get(meta.commandKey); + if (earlier !== undefined && earlier.supersededAt === null) { + earlier.supersededAt = block.arrival; + } + this.lastByCommand.set(meta.commandKey, block); + } + for (const file of meta.reads) { + const readers = this.readersByFile.get(file) ?? []; + for (const reader of readers) reader.rereadAt ??= block.arrival; + readers.push(block); + this.readersByFile.set(file, readers); + } + for (const file of meta.edits) { + for (const reader of this.readersByFile.get(file) ?? []) { + if (reader.supersededAt === null) reader.supersededAt = block.arrival; + } + this.readersByFile.delete(file); + } + } + } + + /** A model emission: enters the prompt after `response`, and proves earlier blocks were used at `response`. */ + emission( + kind: Exclude, + text: string, + estimate: number, + response: number, + ) { + this.add(kind, estimate, response + 1, null); + if (kind === "assistant") this.capture?.answers.push({ call: response, text }); + const { strict, loose } = tokensOf(text); + const strictHits = new Map(); + for (const token of strict) { + const provider = this.strictProvider.get(token); + if (provider === undefined) continue; + const hit = strictHits.get(provider) ?? { count: 0, long: false }; + hit.count += 1; + hit.long ||= token.length >= 12; + strictHits.set(provider, hit); + } + for (const [provider, hit] of strictHits) { + if (hit.count >= 2 || hit.long) { + if (provider.uses.at(-1) !== response) provider.uses.push(response); + provider.lastUse = Math.max(provider.lastUse, response); + provider.lastUseLoose = Math.max(provider.lastUseLoose, response); + } + } + const looseHits = new Map(); + for (const token of loose) { + const provider = this.looseProvider.get(token); + if (provider !== undefined) looseHits.set(provider, (looseHits.get(provider) ?? 0) + 1); + } + for (const [provider, count] of looseHits) { + if (count >= 2) provider.lastUseLoose = Math.max(provider.lastUseLoose, response); + } + } + + apiCall(call: ApiCall, usd: number) { + this.inputOffset = 0; + const index = this.calls.length; + const prompt = promptTokens(call); + const previous = this.calls[index - 1]; + const reset = previous !== undefined && prompt < previous.prompt * RESET_DROP; + if (reset) { + for (const block of this.live) block.end = index; + this.live = []; + this.strictProvider.clear(); + this.looseProvider.clear(); + this.lastByCommand.clear(); + this.readersByFile.clear(); + } + + const arriving = this.pendingArrivals.filter((block) => block.arrival <= index); + this.pendingArrivals = this.pendingArrivals.filter((block) => block.arrival > index); + const weighted = (block: Block) => block.estimate * this.weights[block.kind]; + const estimated = arriving.reduce((sum, block) => sum + weighted(block), 0); + const growth = previous === undefined || reset ? prompt : Math.max(0, prompt - previous.prompt); + // Never attribute more than was billed. Growth left over stays unattributed framing, which no snip can remove. + const scale = estimated > growth && estimated > 0 ? growth / estimated : 1; + for (const block of arriving) { + block.tokens = Math.round(weighted(block) * scale); + this.live.push(block); + } + + this.calls.push({ + ts: call.timestampMs, + model: call.model, + prompt, + read: call.cacheRead, + miss: call.uncached + call.cacheWrite, + output: call.output, + usd, + reset, + }); + } +} + +type Json = Record; +const isObject = (value: unknown): value is Json => + typeof value === "object" && value !== null && !Array.isArray(value); + +function contentText(content: unknown): { text: string; images: number } { + if (typeof content === "string") return { text: content, images: 0 }; + if (!Array.isArray(content)) return { text: "", images: 0 }; + let text = ""; + let images = 0; + for (const part of content) { + if (!isObject(part)) continue; + const type = part["type"]; + if (type === "input_image" || type === "image" || type === "output_image") images += 1; + else if (typeof part["text"] === "string") text += `${part["text"]}\n`; + } + return { text, images }; +} + +async function* lines(file: string) { + const reader = NodeReadline.createInterface({ + input: NodeFS.createReadStream(file), + crlfDelay: Infinity, + }); + for await (const line of reader) { + try { + const parsed = JSON.parse(line) as unknown; + if (isObject(parsed)) yield parsed; + } catch { + // Unterminated tail of a live session. + } + } +} + +function callQueue(calls: readonly ApiCall[]) { + const byTimestamp = new Map(); + for (const call of calls) { + const list = byTimestamp.get(call.timestampMs) ?? []; + list.push(call); + byTimestamp.set(call.timestampMs, list); + } + return (timestampMs: number) => byTimestamp.get(timestampMs)?.shift(); +} + +export async function readCodex( + file: string, + calls: readonly ApiCall[], + priceOf: (call: ApiCall) => number, + capture?: TextCapture, +) { + const builder = new ReplayBuilder(WEIGHTS["codex"]!, capture); + const take = callQueue(calls); + const metaByCall = new Map(); + let lineNumber = 0; + for await (const line of lines(file)) { + lineNumber += 1; + builder.line = lineNumber; + const type = line["type"]; + const payload = isObject(line["payload"]) ? line["payload"] : {}; + const inner = payload["type"]; + const timestamp = + typeof line["timestamp"] === "string" ? Date.parse(line["timestamp"]) : Number.NaN; + if (type === "token_usage_record" || (type === "event_msg" && inner === "token_count")) { + const call = take(timestamp); + if (call !== undefined) builder.apiCall(call, priceOf(call)); + else if (type === "token_usage_record") builder.inputOffset = 1; + continue; + } + if (type === "event_msg" && inner === "task_started") builder.markTurnStart(); + if (type !== "response_item") continue; + const response = builder.calls.length; + if (inner === "message") { + const { text, images } = contentText(payload["content"]); + if (payload["role"] === "assistant") { + builder.emission("assistant", text, text.length / 4, response); + } else if (payload["role"] === "user") { + builder.userMessage(text, images); + } + } else if (inner === "agent_message") { + const { text } = contentText(payload["content"]); + builder.userMessage(text); + } else if (inner === "reasoning") { + const encrypted = payload["encrypted_content"]; + const { text } = contentText(payload["summary"]); + builder.emission( + "reasoning", + text, + (typeof encrypted === "string" ? encrypted.length : 0) * 0.2, + response, + ); + } else if (inner === "custom_tool_call" || inner === "function_call") { + const raw = inner === "custom_tool_call" ? payload["input"] : payload["arguments"]; + const text = typeof raw === "string" ? raw : ""; + const name = typeof payload["name"] === "string" ? payload["name"] : ""; + const namespace = typeof payload["namespace"] === "string" ? payload["namespace"] : ""; + const command = classifyCommand(text); + const meta: ToolMeta = + namespace.startsWith("mcp") || name.startsWith("mcp__") + ? { ...command, toolClass: "mcp" } + : name === "spawn_agent" || name === "wait_agent" + ? { ...command, toolClass: "agent" } + : command; + if (typeof payload["call_id"] === "string") metaByCall.set(payload["call_id"], meta); + builder.emission("tool_call", text, text.length / 4, response); + } else if (inner === "custom_tool_call_output" || inner === "function_call_output") { + const callId = payload["call_id"]; + const { text, images } = contentText(payload["output"]); + builder.toolOutput( + text, + images, + (typeof callId === "string" ? metaByCall.get(callId) : undefined) ?? null, + ); + } + } + return builder; +} + +const CLAUDE_READ_TOOLS = new Set(["Read"]); +const CLAUDE_SEARCH_TOOLS = new Set(["Grep", "Glob", "LS", "ToolSearch"]); +const CLAUDE_EDIT_TOOLS = new Set(["Edit", "Write", "MultiEdit", "NotebookEdit"]); + +function classifyClaudeTool(name: string, input: Json): ToolMeta { + return { + ...claudeToolMeta(name, input), + label: `${name} ${JSON.stringify(input).slice(0, 380)}`, + }; +} + +function claudeToolMeta(name: string, input: Json): ToolMeta { + const key = hash(`${name}:${JSON.stringify(input)}`); + const path = typeof input["file_path"] === "string" ? input["file_path"] : null; + if (CLAUDE_READ_TOOLS.has(name)) { + return { toolClass: "read", commandKey: key, reads: path === null ? [] : [path], edits: [] }; + } + if (CLAUDE_EDIT_TOOLS.has(name)) { + return { toolClass: "edit", commandKey: key, reads: [], edits: path === null ? [] : [path] }; + } + if (CLAUDE_SEARCH_TOOLS.has(name)) { + return { toolClass: "search", commandKey: key, reads: [], edits: [] }; + } + if (name === "Bash" && typeof input["command"] === "string") { + return classifyCommand(input["command"]); + } + if (name === "Agent" || name === "Task") { + return { toolClass: "agent", commandKey: key, reads: [], edits: [] }; + } + return { + toolClass: name.startsWith("mcp__") ? "mcp" : "other", + commandKey: key, + reads: [], + edits: [], + }; +} + +export async function readClaude( + file: string, + calls: readonly ApiCall[], + priceOf: (call: ApiCall) => number, + capture?: TextCapture, +) { + const builder = new ReplayBuilder(WEIGHTS["claude"]!, capture); + const byKey = new Map(calls.map((call) => [call.dedupeKey, call])); + const metaByUse = new Map(); + for await (const line of lines(file)) { + const message = line["message"]; + if (!isObject(message)) continue; + const content = message["content"]; + if (line["type"] === "user") { + if (line["isMeta"] === true) continue; + if (typeof content === "string") { + builder.markTurnStart(); + builder.userMessage(content); + continue; + } + if (!Array.isArray(content)) continue; + for (const part of content) { + if (!isObject(part)) continue; + if (part["type"] === "tool_result") { + const { text, images } = contentText(part["content"]); + const useId = part["tool_use_id"]; + builder.toolOutput( + text, + images, + (typeof useId === "string" ? metaByUse.get(useId) : undefined) ?? null, + ); + } else if (part["type"] === "text" && typeof part["text"] === "string") { + builder.markTurnStart(); + builder.userMessage(part["text"]); + } + } + } else if (line["type"] === "assistant") { + const key = `${String(message["id"])}:${String(line["requestId"])}`; + const call = byKey.get(key); + if (call !== undefined) { + builder.apiCall(call, priceOf(call)); + byKey.delete(key); + } + if (!Array.isArray(content)) continue; + const response = Math.max(0, builder.calls.length - 1); + for (const part of content) { + if (!isObject(part)) continue; + if (part["type"] === "text" && typeof part["text"] === "string") { + builder.emission("assistant", part["text"], part["text"].length / 4, response); + } else if (part["type"] === "thinking") { + const thinking = typeof part["thinking"] === "string" ? part["thinking"] : ""; + const signature = typeof part["signature"] === "string" ? part["signature"] : ""; + builder.emission( + "reasoning", + thinking, + Math.max(thinking.length / 4, signature.length * 0.2), + response, + ); + } else if (part["type"] === "tool_use") { + const input = isObject(part["input"]) ? part["input"] : {}; + const name = typeof part["name"] === "string" ? part["name"] : ""; + if (typeof part["id"] === "string") { + metaByUse.set(part["id"], classifyClaudeTool(name, input)); + } + const text = JSON.stringify(input); + builder.emission("tool_call", text, text.length / 4, response); + } + } + } + } + return builder; +} + +if (import.meta.main) { + const dir = process.argv[2]; + if (dir === undefined) throw new Error("usage: replay.ts [days]"); + const days = Number(process.argv[3] ?? 30); + const { loadRates, price } = await import("./usageRecords.ts"); + const rates = loadRates(NodePath.join(dir, "usage-model-rates.json")); + const all = loadApiCalls(NodePath.join(dir, "usage-scan-cache-v11.json")); + const latest = Math.max(...all.map((call) => call.timestampMs)); + const cutoff = latest - days * 86_400_000; + const byFile = new Map(); + for (const call of all) { + const list = byFile.get(call.file) ?? []; + list.push(call); + byFile.set(call.file, list); + } + const outDir = NodePath.join(dir, "replay"); + NodeFS.mkdirSync(outDir, { recursive: true }); + let written = 0; + let matched = 0; + let expected = 0; + let missingFiles = 0; + for (const [file, calls] of byFile) { + if (!calls.some((call) => call.timestampMs >= cutoff)) continue; + const provider = calls[0]!.provider; + if (provider !== "codex" && provider !== "claude") continue; + expected += calls.length; + const priceOf = (call: ApiCall) => price(rates, call).costUsd; + let builder: ReplayBuilder; + try { + builder = + provider === "codex" + ? await readCodex(file, calls, priceOf) + : await readClaude(file, calls, priceOf); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + missingFiles += 1; + continue; + } + matched += builder.calls.length; + const replay: Replay = { + file, + provider, + calls: builder.calls, + blocks: builder.blocks, + turnStarts: builder.turnStarts, + }; + NodeFS.writeFileSync(NodePath.join(outDir, `${hash(file)}.json`), JSON.stringify(replay)); + written += 1; + } + console.log( + `replays ${written}, missing transcripts ${missingFiles}, calls aligned ${matched} of ${expected} (${((matched / Math.max(expected, 1)) * 100).toFixed(1)}%)`, + ); +} diff --git a/evals/cache-snip/shape.ts b/evals/cache-snip/shape.ts new file mode 100644 index 0000000000000..2fff66de4d752 --- /dev/null +++ b/evals/cache-snip/shape.ts @@ -0,0 +1,151 @@ +// @effect-diagnostics nodeBuiltinImport:off globalConsole:off +// How prompts behave over a session: size, cold misses, and what list price really is above the long-context cliff. +// usage: node evals/cache-snip/shape.ts [days] +import * as NodeFS from "node:fs"; +import * as NodePath from "node:path"; + +import { lookupRate } from "../../apps/server/src/usage/usagePricing.ts"; +import { loadApiCalls, loadRates, price, promptTokens, type ApiCall } from "./usageRecords.ts"; + +const dir = process.argv[2]; +if (dir === undefined) throw new Error("usage: shape.ts [days]"); +const days = Number(process.argv[3] ?? 30); +const rates = loadRates(NodePath.join(dir, "usage-model-rates.json")); +const document = ( + JSON.parse(NodeFS.readFileSync(NodePath.join(dir, "usage-model-rates.json"), "utf8")) as { + document: Record>; + } +).document; +const all = loadApiCalls(NodePath.join(dir, "usage-scan-cache-v11.json")); +const latest = Math.max(...all.map((call) => call.timestampMs)); +const calls = all.filter((call) => call.timestampMs >= latest - days * 86_400_000); + +const LONG_CONTEXT_LIMIT = 272_000; + +/** List price including the long-context tier T3's table leaves out, per OpenAI's pricing page. */ +function listUsd(call: ApiCall): { billed: number; t3: number } { + const t3 = price(rates, call).costUsd; + const entry = document[call.model]; + const long = + entry !== undefined && + promptTokens(call) > LONG_CONTEXT_LIMIT && + entry["input_cost_per_token_above_272k_tokens"] !== undefined; + if (!long) return { billed: t3, t3 }; + const rate = lookupRate(rates, call.model); + if (rate === null) return { billed: t3, t3 }; + const input = entry["input_cost_per_token_above_272k_tokens"] ?? rate.inputCostPerToken; + const read = entry["cache_read_input_token_cost_above_272k_tokens"] ?? rate.cacheReadCostPerToken; + const output = entry["output_cost_per_token_above_272k_tokens"] ?? rate.outputCostPerToken; + return { + billed: call.uncached * input + call.cacheRead * read + call.output * output, + t3, + }; +} + +const bucketEdges = [50_000, 100_000, 150_000, 200_000, 272_000, 400_000, 600_000, Infinity]; +const bucketName = (edge: number) => (edge === Infinity ? ">600k" : `<=${edge / 1000}k`); + +for (const provider of ["codex", "claude"]) { + const subset = calls.filter((call) => call.provider === provider); + const rows = bucketEdges.map(() => ({ calls: 0, t3: 0, billed: 0, read: 0 })); + for (const call of subset) { + const index = bucketEdges.findIndex((edge) => promptTokens(call) <= edge); + const row = rows[index]!; + const { billed, t3 } = listUsd(call); + row.calls += 1; + row.t3 += t3; + row.billed += billed; + row.read += price(rates, call).categoryCostUsd?.cacheRead ?? 0; + } + const t3Total = rows.reduce((sum, row) => sum + row.t3, 0); + const billedTotal = rows.reduce((sum, row) => sum + row.billed, 0); + console.log(`\n== ${provider}, last ${days} days, by prompt size`); + console.log("bucket calls T3$ list$ %ofT3$ read$"); + rows.forEach((row, index) => { + console.log( + `${bucketName(bucketEdges[index]!).padEnd(9)} ${String(row.calls).padStart(6)} ${row.t3.toFixed(0).padStart(6)} ${row.billed.toFixed(0).padStart(7)} ${((row.t3 / Math.max(t3Total, 1e-9)) * 100).toFixed(1).padStart(6)}% ${row.read.toFixed(0).padStart(6)}`, + ); + }); + console.log( + `T3 estimate $${t3Total.toFixed(0)}, with long-context tier $${billedTotal.toFixed(0)} (+${(((billedTotal - t3Total) / t3Total) * 100).toFixed(1)}%)`, + ); +} + +interface Session { + readonly key: string; + readonly provider: string; + readonly calls: readonly ApiCall[]; +} +const sessionMap = new Map(); +for (const call of calls) { + const list = sessionMap.get(call.file) ?? []; + list.push(call); + sessionMap.set(call.file, list); +} +const sessions: Session[] = [...sessionMap.entries()].map(([key, list]) => ({ + key, + provider: list[0]!.provider, + calls: list.toSorted((a, b) => a.timestampMs - b.timestampMs), +})); + +const COLD_MIN_PROMPT = 10_000; +const COLD_READ_SHARE = 0.5; +const quantile = (values: readonly number[], q: number) => { + if (values.length === 0) return 0; + const sorted = values.toSorted((a, b) => a - b); + return sorted[Math.min(sorted.length - 1, Math.floor(q * sorted.length))]!; +}; + +for (const provider of ["codex", "claude"]) { + const subset = sessions.filter((session) => session.provider === provider); + let cold = 0; + let total = 0; + let coldTokens = 0; + let coldAfterGap = 0; + let drops = 0; + const gapsBeforeCold: number[] = []; + const coldPrompts: number[] = []; + const gapBuckets = { "<5m": 0, "5-30m": 0, "30-60m": 0, ">60m": 0 }; + const allGaps = { "<5m": 0, "5-30m": 0, "30-60m": 0, ">60m": 0 }; + const key = (minutes: number) => + minutes < 5 ? "<5m" : minutes < 30 ? "5-30m" : minutes < 60 ? "30-60m" : ">60m"; + for (const session of subset) { + session.calls.forEach((call, index) => { + total += 1; + const prompt = promptTokens(call); + const previous = session.calls[index - 1]; + const gapMinutes = + previous === undefined ? 0 : (call.timestampMs - previous.timestampMs) / 60_000; + if (previous !== undefined) allGaps[key(gapMinutes)] += 1; + if (previous !== undefined && prompt < promptTokens(previous) * 0.6) drops += 1; + if (previous === undefined) return; + if (prompt >= COLD_MIN_PROMPT && call.cacheRead / prompt < COLD_READ_SHARE) { + cold += 1; + coldTokens += call.uncached + call.cacheWrite; + gapsBeforeCold.push(gapMinutes); + coldPrompts.push(prompt); + gapBuckets[key(gapMinutes)] += 1; + if (gapMinutes >= 5) coldAfterGap += 1; + } + }); + } + const callCounts = subset.map((session) => session.calls.length); + const usdBySession = subset + .map((session) => session.calls.reduce((sum, call) => sum + price(rates, call).costUsd, 0)) + .toSorted((a, b) => b - a); + const usdTotal = usdBySession.reduce((sum, value) => sum + value, 0); + const top = (n: number) => + usdBySession.slice(0, n).reduce((sum, value) => sum + value, 0) / Math.max(usdTotal, 1e-9); + console.log(`\n== ${provider}, session shape`); + console.log( + `sessions ${subset.length}, calls ${total}, calls/session p50 ${quantile(callCounts, 0.5)} p90 ${quantile(callCounts, 0.9)} max ${Math.max(0, ...callCounts)}`, + ); + console.log( + `$ concentration: top 10 sessions ${(top(10) * 100).toFixed(0)}%, top 20 ${(top(20) * 100).toFixed(0)}%, top 50 ${(top(50) * 100).toFixed(0)}%`, + ); + console.log( + `cold calls ${cold} (${((cold / Math.max(total, 1)) * 100).toFixed(2)}%), uncached+write tokens in them ${(coldTokens / 1e6).toFixed(1)}M, prompt p50 ${quantile(coldPrompts, 0.5)} p90 ${quantile(coldPrompts, 0.9)}`, + ); + console.log("gap before cold calls:", gapBuckets, " all gaps:", allGaps); + console.log(`prompt drops over 40% (compaction or reset): ${drops}`); +} diff --git a/evals/cache-snip/simulate.ts b/evals/cache-snip/simulate.ts new file mode 100644 index 0000000000000..9fed25674abc9 --- /dev/null +++ b/evals/cache-snip/simulate.ts @@ -0,0 +1,943 @@ +// @effect-diagnostics nodeBuiltinImport:off globalConsole:off +// Replays real sessions under snip policies and reports dollars against billed list price. +// usage: node evals/cache-snip/simulate.ts [days] +import * as NodeCrypto from "node:crypto"; +import * as NodeFS from "node:fs"; +import * as NodePath from "node:path"; + +import { lookupRate } from "../../apps/server/src/usage/usagePricing.ts"; +import type { Block, Call, Replay } from "./replay.ts"; +import { loadRates } from "./usageRecords.ts"; + +interface CallRates { + readonly read: number; + readonly miss: number; + readonly output: number; + /** Multiplies input and output above the long-context limit. Null when the model has no such tier. */ + readonly longInput: number | null; + readonly longOutput: number | null; +} + +const LONG_CONTEXT_LIMIT = 272_000; +const GRACE_CALLS = 5; +const REFETCH_CALL_OUTPUT_TOKENS = 100; +// A fresh chat re-reads for a while. Without a gap, that growth alone would retrigger the cut. +const HEAD_CUT_COOLDOWN_CALLS = 20; +const USER_SEED_TOKENS = 5_000; + +export interface Epoch { + readonly calls: readonly Call[]; + readonly blocks: readonly Block[]; + readonly session: string; + readonly firstIndex: number; + readonly turnStarts: ReadonlySet; + /** The call index each call has in the original replay. Blocks use these. */ + readonly rates: readonly CallRates[]; +} + +export type RateMode = "t3Table" | "trueList" | "apiWrites"; +export type { CallRates }; + +export function makeRateLookup(dir: string, mode: RateMode) { + const table = loadRates(NodePath.join(dir, "usage-model-rates.json")); + const document = ( + JSON.parse(NodeFS.readFileSync(NodePath.join(dir, "usage-model-rates.json"), "utf8")) as { + document: Record>; + } + ).document; + const cache = new Map(); + return (model: string, subagent: boolean): CallRates => { + const key = `${model}:${subagent}`; + const held = cache.get(key); + if (held !== undefined) return held; + const rate = lookupRate(table, model); + const entry = document[model]; + const provider = entry?.["litellm_provider"] as unknown as string | undefined; + const anthropic = provider === "anthropic"; + const oneHourWrite = entry?.["cache_creation_input_token_cost_above_1hr"]; + const longKnown = entry?.["input_cost_per_token_above_272k_tokens"] !== undefined; + const result: CallRates = + rate === null + ? { read: 0, miss: 0, output: 0, longInput: null, longOutput: null } + : { + read: rate.cacheReadCostPerToken, + miss: anthropic + ? // Main Claude sessions write 1h entries; T3's table prices them all at the 5m rate. + mode !== "t3Table" && !subagent && oneHourWrite !== undefined + ? oneHourWrite + : rate.cacheCreationCostPerToken + : // Codex rollouts report no writes. API billing on GPT-5.6+ charges 1.25x input for them. + mode === "apiWrites" + ? rate.cacheCreationCostPerToken + : rate.inputCostPerToken, + output: rate.outputCostPerToken, + longInput: mode !== "t3Table" && longKnown ? 2 : null, + longOutput: mode !== "t3Table" && longKnown ? 1.5 : null, + }; + cache.set(key, result); + return result; + }; +} + +export const sessionKey = (file: string) => + NodeCrypto.createHash("sha1").update(file).digest("hex").slice(0, 12); + +export function splitEpochs( + replay: Replay, + ratesFor: (model: string, subagent: boolean) => CallRates, +): Epoch[] { + const subagent = replay.file.includes("/subagents/"); + const turnStarts = new Set(replay.turnStarts); + const session = sessionKey(replay.file); + const starts = replay.calls.flatMap((call, index) => (index === 0 || call.reset ? [index] : [])); + return starts.map((start, position) => { + const stop = starts[position + 1] ?? replay.calls.length; + const calls = replay.calls.slice(start, stop); + return { + calls, + session, + firstIndex: start, + turnStarts, + rates: calls.map((call) => ratesFor(call.model, subagent)), + blocks: replay.blocks.filter((block) => block.arrival >= start && block.arrival < stop), + }; + }); +} + +interface Tally { + usd: number; + snips: number; + snippedTokens: number; + refetches: number; + refetchTokens: number; + wrongSnips: number; + seedTokens: number; +} +const emptyTally = (): Tally => ({ + usd: 0, + snips: 0, + snippedTokens: 0, + refetches: 0, + refetchTokens: 0, + wrongSnips: 0, + seedTokens: 0, +}); + +/** What a policy may look at before call `t`. Foresight lives only in selectors named oracle. */ +export interface View { + readonly epoch: Epoch; + readonly t: number; + readonly live: readonly Block[]; + readonly coldHint: boolean; + /** A user message entered the prompt at this call: the natural boundary between tasks. */ + readonly userTurn: boolean; + readonly prompt: number; + readonly removedBefore: (arrival: number) => number; +} + +export type Decide = (view: View) => readonly Block[]; + +export const LONG_LIMIT = LONG_CONTEXT_LIMIT; +const nextUse = (block: Block, after: number) => block.uses.find((use) => use > after) ?? null; + +export interface RunOptions { + readonly compactAt?: number; + readonly summaryTokens?: number; + readonly coldHintMinutes?: number; + /** Share of wrong snips that actually make the agent fetch the content again. */ + readonly refetchRate?: number; + /** Share of cuts whose kept prefix still misses, as when a fork lands on a cold cache. */ + readonly cutMissRate?: number; + /** New chat seeded with the newest tokens only. No summary call, and the first request is fully uncached. */ + readonly headCut?: { + readonly trigger: number; + readonly keepTokens: number; + readonly floor: number; + readonly when: "userTurn" | "cold"; + /** Newest tokens always kept. The rest of `keepTokens` is the head budget, filled by `rank`. */ + readonly keepRecent?: number; + /** Drops tool output from the newest window, so the seed can end up smaller than `keepRecent`. */ + readonly seedFilter?: (block: Block, t: number, epoch: Epoch) => boolean; + /** Higher is kept first. Defaults to newest first, as T3's handoff does. */ + readonly rank?: (block: Block, t: number, epoch: Epoch) => number; + }; + /** What a fresh chat re-reads because it lost context: added tokens and extra calls per head cut. */ + readonly reexplore?: { readonly tokens: number; readonly calls: number }; + /** Compact at exactly these calls. Used to price each candidate moment. */ + readonly compactCalls?: ReadonlySet; + /** Compact only at this moment, instead of whenever the prompt is over the limit. */ + readonly compactWhen?: "cold" | "userTurn" | "coldOrUserTurn"; +} + +/** One epoch under a policy. `compactAt` replaces everything but the base with a summary. */ +export function runEpoch( + epoch: Epoch, + decide: Decide, + cutoff: number, + options: RunOptions = {}, +): Tally { + const tally = emptyTally(); + const { calls, blocks, firstIndex, rates } = epoch; + const byArrival = new Map(); + for (const block of blocks) { + const list = byArrival.get(block.arrival) ?? []; + list.push(block); + byArrival.set(block.arrival, list); + } + const base = + calls[0]!.prompt - + blocks.reduce((sum, block) => sum + (block.arrival === firstIndex ? block.tokens : 0), 0); + const removedAt = new Map(); + const pendingRefetch = new Map(); + let live: Block[] = []; + let removedTokens = 0; + let tail = 0; + // After a compaction the prompt is floor + what the real session appended since `anchor`. + let compaction: { floor: number; anchorPrompt: number } | null = null; + let lastCut = Number.NEGATIVE_INFINITY; + + const removedBefore = (arrival: number) => { + let total = 0; + for (const block of removedAt.keys()) if (block.arrival < arrival) total += block.tokens; + return total; + }; + const promptOf = (call: Call) => + compaction === null + ? call.prompt - removedTokens + tail + : compaction.floor + Math.max(0, call.prompt - compaction.anchorPrompt) + tail; + const refetchRate = options.refetchRate ?? 1; + const scheduleRefetch = (block: Block, t: number) => { + const use = nextUse(block, t - 1); + if (use === null) return; + tally.wrongSnips += 1; + // Not every later mention needs the bytes back. Deterministic so policies compare fairly. + if (((block.arrival * 2654435761 + block.tokens) >>> 0) / 4294967296 >= refetchRate) return; + const due = Math.max(use, t + 1); + if (due - firstIndex >= calls.length) return; + const list = pendingRefetch.get(due) ?? []; + list.push(block); + pendingRefetch.set(due, list); + }; + + const coldMinutes = options.coldHintMinutes ?? 60; + calls.forEach((call, offset) => { + const t = firstIndex + offset; + const callRates = rates[offset]!; + const previous = calls[offset - 1]; + live.push(...(byArrival.get(t) ?? [])); + + const tierAt = (tokens: number) => + callRates.longInput !== null && tokens > LONG_CONTEXT_LIMIT + ? { input: callRates.longInput, output: callRates.longOutput ?? 1 } + : { input: 1, output: 1 }; + + let extraUsd = 0; + let tailFresh = 0; + for (const block of pendingRefetch.get(t) ?? []) { + if (!removedAt.has(block)) continue; + removedAt.delete(block); + if (compaction === null) removedTokens -= block.tokens; + // Refetched content sits at the tail, so snipping it again only busts the tail. + const refetched: Block = { ...block, arrival: t }; + const roundTrip = promptOf(call); + const tier = tierAt(roundTrip); + extraUsd += + Math.max(0, roundTrip - call.miss) * callRates.read * tier.input + + REFETCH_CALL_OUTPUT_TOKENS * callRates.output * tier.output; + tail += block.tokens; + tailFresh += block.tokens; + live.push(refetched); + tally.refetches += 1; + tally.refetchTokens += block.tokens; + } + pendingRefetch.delete(t); + + const gapMinutes = previous === undefined ? 0 : (call.ts - previous.ts) / 60_000; + const toRemove = decide({ + epoch, + t, + live, + coldHint: gapMinutes >= coldMinutes, + userTurn: epoch.turnStarts.has(t), + prompt: promptOf(call), + removedBefore, + }); + + let prefixLimit = Number.POSITIVE_INFINITY; + if (toRemove.length > 0) { + const firstArrival = Math.min(...toRemove.map((block) => block.arrival)); + const offsetBefore = firstArrival - firstIndex - 1; + const before = offsetBefore >= 0 ? calls[offsetBefore]!.prompt : base; + prefixLimit = Math.max(0, before - removedBefore(firstArrival)); + if (((t * 2654435761 + call.prompt) >>> 0) / 4294967296 < (options.cutMissRate ?? 0)) { + prefixLimit = 0; + } + const removing = new Set(toRemove); + live = live.filter((block) => !removing.has(block)); + tally.snips += 1; + for (const block of toRemove) { + removedAt.set(block, t); + removedTokens += block.tokens; + tally.snippedTokens += block.tokens; + scheduleRefetch(block, t); + } + } + + const moment = options.compactWhen; + const userTurn = epoch.turnStarts.has(t); + const cold = gapMinutes >= coldMinutes; + const momentOk = + moment === undefined || + (moment === "cold" && cold) || + (moment === "userTurn" && userTurn) || + (moment === "coldOrUserTurn" && (cold || userTurn)); + if ( + options.compactCalls?.has(t) === true || + (options.compactAt !== undefined && momentOk && promptOf(call) >= options.compactAt) + ) { + // Codex's prompt floor rises from 33k on a fresh session to 67k after compaction, so 30k survives. + const summaryTokens = options.summaryTokens ?? 30_000; + const beforeCompaction = promptOf(call); + const tier = tierAt(beforeCompaction); + extraUsd += + Math.max(0, beforeCompaction - call.miss) * callRates.read * tier.input + + summaryTokens * callRates.output * tier.output; + for (const block of live) { + removedAt.set(block, t); + tally.snippedTokens += block.tokens; + scheduleRefetch(block, t); + } + live = []; + tail = 0; + tailFresh = 0; + tally.snips += 1; + compaction = { floor: base + summaryTokens, anchorPrompt: call.prompt }; + prefixLimit = base; + } + + const cut = options.headCut; + if ( + cut !== undefined && + promptOf(call) >= cut.trigger && + t - lastCut >= HEAD_CUT_COOLDOWN_CALLS && + (cut.when === "userTurn" ? userTurn : cold) + ) { + lastCut = t; + const rank = cut.rank ?? ((block: Block) => block.arrival); + const seedable = live.filter( + (block) => block.kind !== "reasoning" && block.kind !== "tool_call", + ); + const keep = new Set(); + let kept = 0; + const take = (block: Block, limit: number) => { + if (keep.has(block) || kept + block.tokens > limit) return; + keep.add(block); + kept += block.tokens; + }; + // Instructions and the latest answer survive whatever the ranking says, as in T3's handoff. + const newestFirst = seedable.toReversed(); + for (const block of newestFirst) if (block.kind === "user") take(block, USER_SEED_TOKENS); + const latestAnswer = newestFirst.find((block) => block.kind === "assistant"); + if (latestAnswer !== undefined) take(latestAnswer, cut.keepTokens); + const recent = cut.keepRecent ?? 0; + let walked = 0; + for (const block of newestFirst) { + if ((cut.seedFilter === undefined ? kept : walked) >= recent) break; + walked += block.tokens; + if (block.kind === "tool_output" && cut.seedFilter?.(block, t, epoch) === false) continue; + take(block, recent); + } + // A filtered seed is meant to end up small, so nothing tops it back up. + const head = + cut.seedFilter !== undefined + ? [] + : seedable.filter( + (block) => block.kind === "tool_output" || cut.keepRecent === undefined, + ); + for (const block of head.toSorted((a, b) => rank(b, t, epoch) - rank(a, t, epoch))) { + take(block, cut.keepTokens); + } + for (const block of live) { + if (keep.has(block)) continue; + removedAt.set(block, t); + tally.snippedTokens += block.tokens; + scheduleRefetch(block, t); + } + live = live.filter((block) => keep.has(block)); + const reexplore = options.reexplore ?? { tokens: 0, calls: 0 }; + tail = reexplore.tokens; + tailFresh = reexplore.tokens; + tally.snips += 1; + tally.seedTokens += kept; + compaction = { floor: cut.floor + kept, anchorPrompt: call.prompt }; + prefixLimit = 0; + const fresh = cut.floor + kept + reexplore.tokens; + const freshTier = tierAt(fresh); + extraUsd += + reexplore.calls * + (fresh * callRates.read * freshTier.input + + REFETCH_CALL_OUTPUT_TOKENS * callRates.output * freshTier.output); + } + + const prompt = promptOf(call); + const tier = tierAt(prompt); + const read = Math.min(Math.max(0, prompt - call.miss - tailFresh), prefixLimit); + const usd = + read * callRates.read * tier.input + + (prompt - read) * callRates.miss * tier.input + + call.output * callRates.output * tier.output + + extraUsd; + if (call.ts >= cutoff) tally.usd += usd; + }); + return tally; +} + +const never: Decide = () => []; + +const isOutput = (block: Block) => block.kind === "tool_output"; + +/** Foresight: the block is never quoted again in this epoch. */ +const deadOracle = (view: View) => + view.live.filter((block) => isOutput(block) && block.lastUse + GRACE_CALLS < view.t); + +const supersededOnly = (view: View) => + view.live.filter( + (block) => + isOutput(block) && + ((block.supersededAt !== null && block.supersededAt <= view.t) || + (block.rereadAt !== null && block.rereadAt <= view.t)), + ); + +/** Least recently used: not quoted for `idle` calls. Uses only what has already happened. */ +const idleFor = (idle: number, minTokens: number) => (view: View) => + view.live.filter((block) => { + if (!isOutput(block) || block.tokens < minTokens) return false; + let last = block.arrival; + for (const use of block.uses) { + if (use >= view.t) break; + last = use; + } + return view.t - last > idle; + }); + +/** Dead outputs plus old tool inputs and reasoning, for the widest ceiling. */ +const deadPlusOld = (view: View) => + view.live.filter( + (block) => + (isOutput(block) && block.lastUse + GRACE_CALLS < view.t) || + ((block.kind === "tool_call" || block.kind === "reasoning") && view.t - block.arrival > 50), + ); + +/** Stand-in for a classifier of given AUC, thresholded at a false positive rate. Noise is fixed per block. */ +const noisyClassifier = (auc: number, falsePositiveRate: number) => { + const separation = Math.SQRT2 * inverseNormal(auc); + const threshold = inverseNormal(1 - falsePositiveRate); + return (view: View) => + view.live.filter((block) => { + if (!isOutput(block)) return false; + const dead = block.lastUse + GRACE_CALLS < view.t; + const noise = blockNoise(block); + return (dead ? separation : 0) + noise > threshold; + }); +}; + +function inverseNormal(p: number) { + // Acklam's rational approximation, central region only: the AUCs and rates used here stay inside it. + const q = p - 0.5; + const r = q * q; + return ( + (q * + (((((-3.969683028665376e1 * r + 2.209460984245205e2) * r - 2.759285104469687e2) * r + + 1.38357751867269e2) * + r - + 3.066479806614716e1) * + r + + 2.506628277459239)) / + (((((-5.447609879822406e1 * r + 1.615858368580409e2) * r - 1.556989798598866e2) * r + + 6.680131188771972e1) * + r - + 1.328068155288572e1) * + r + + 1) + ); +} + +function blockNoise(block: Block) { + const seed = (block.arrival * 2654435761 + block.tokens * 40503) >>> 0; + const u1 = ((seed * 1664525 + 1013904223) >>> 0) / 4294967296 || 1e-9; + const u2 = (((seed ^ 0x9e3779b9) * 22695477 + 1) >>> 0) / 4294967296; + return Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2); +} + +/** Tool output from before the user's last message. The task boundary rule. */ +const beforeLastUserTurn = (minTokens: number) => (view: View) => { + let lastUser = -1; + for (const block of view.live) + if (block.kind === "user") lastUser = Math.max(lastUser, block.arrival); + return view.live.filter( + (block) => isOutput(block) && block.tokens >= minTokens && block.arrival < lastUser, + ); +}; + +const atUserTurn = + (decide: Decide): Decide => + (view) => + view.userTurn ? decide(view) : []; + +/** + * Rewind: drop the finished turn's tool work and keep what the user saw. Everything before that turn is untouched, + * so it stays cached and only the kept answer and the new message are rebuilt. + */ +const finishedTurnWork = (minTokens: number) => (view: View) => { + const users = view.live.filter((block) => block.kind === "user" && block.arrival < view.t); + const start = users.at(-1)?.arrival; + if (start === undefined) return []; + const chosen = view.live.filter( + (block) => + block.arrival >= start && + block.arrival < view.t && + (block.kind === "tool_output" || block.kind === "tool_call" || block.kind === "reasoning"), + ); + return chosen.reduce((sum, block) => sum + block.tokens, 0) >= minTokens ? chosen : []; +}; + +/** Snip only when the gap says the cache is already cold. */ +const whenCold = + (select: (view: View) => readonly Block[], minTokens: number): Decide => + (view) => { + if (!view.coldHint) return []; + const chosen = select(view); + return chosen.reduce((sum, block) => sum + block.tokens, 0) >= minTokens ? chosen : []; + }; + +/** Claude's clear_tool_uses semantics: oldest first, keep N, skip unless enough clears. */ +const keepRecent = + (trigger: number, keep: number, clearAtLeast: number): Decide => + (view) => { + if (view.prompt < trigger) return []; + const outputs = view.live.filter(isOutput); + const candidates = outputs.slice(0, Math.max(0, outputs.length - keep)); + return candidates.reduce((sum, block) => sum + block.tokens, 0) >= clearAtLeast + ? candidates + : []; + }; + +/** + * Estimates remaining calls from room left before the provider compacts, and charges the rebuild honestly. + * Counts the long-context cliff, where a snip that lands under the limit halves every later read. + */ +const whenRoomPays = + ( + select: (view: View) => readonly Block[], + compactLimit: number, + margin: number, + minShare: number, + ): Decide => + (view) => { + const chosen = select(view); + if (chosen.length === 0) return []; + const snipped = chosen.reduce((sum, block) => sum + block.tokens, 0); + if (snipped < minShare * view.prompt) return []; + const offset = view.t - view.epoch.firstIndex; + if (offset < 5) return []; + const rates = view.epoch.rates[offset]!; + const start = view.epoch.calls[0]!.prompt; + const growth = Math.max(200, (view.epoch.calls[offset]!.prompt - start) / offset); + const room = Math.max(0, compactLimit - view.prompt) / growth; + const remaining = Math.min(room, offset); + const firstArrival = Math.min(...chosen.map((block) => block.arrival)); + const offsetBefore = firstArrival - view.epoch.firstIndex - 1; + const before = offsetBefore >= 0 ? view.epoch.calls[offsetBefore]!.prompt : 0; + const kept = Math.max(0, before - view.removedBefore(firstArrival)); + const after = view.prompt - snipped; + const tier = (tokens: number) => + rates.longInput !== null && tokens > LONG_CONTEXT_LIMIT ? rates.longInput : 1; + const rebuild = + Math.max(0, after - kept) * rates.miss * tier(after) + + kept * rates.read * tier(after) - + view.prompt * rates.read * tier(view.prompt); + const perCall = view.prompt * rates.read * tier(view.prompt) - after * rates.read * tier(after); + return remaining * perCall > rebuild * margin ? chosen : []; + }; + +/** Foresight on both relevance and remaining calls: best snip times found by repeated best-insert. */ +function oracleDecide( + epoch: Epoch, + cutoff: number, + select: (view: View) => readonly Block[], +): Decide { + const count = epoch.calls.length; + if (count < 3) return never; + const step = Math.max(1, Math.floor(count / 60)); + const candidates: number[] = []; + for (let offset = step; offset < count; offset += step) + candidates.push(epoch.firstIndex + offset); + const planned = new Set(); + const planDecide: Decide = (view) => (planned.has(view.t) ? select(view) : []); + let best = runEpoch(epoch, planDecide, cutoff).usd; + for (let round = 0; round < 8; round += 1) { + let bestTime: number | null = null; + for (const time of candidates) { + if (planned.has(time)) continue; + planned.add(time); + const usd = runEpoch(epoch, planDecide, cutoff).usd; + planned.delete(time); + if (usd < best - 1e-9) { + best = usd; + bestTime = time; + } + } + if (bestTime === null) break; + planned.add(bestTime); + } + return planDecide; +} + +interface PolicyDefinition { + readonly name: string; + readonly make: (epoch: Epoch, cutoff: number) => Decide; + readonly options?: RunOptions; +} + +const stateless = ( + name: string, + decide: Decide, + options?: PolicyDefinition["options"], +): PolicyDefinition => ({ + name, + make: () => decide, + ...(options === undefined ? {} : { options }), +}); + +/** Out-of-fold scores from relevance.py, keyed session:t:block. */ +let relevanceScores: ReadonlyMap = new Map(); + +const lastSeenUse = (block: Block, t: number) => { + let last = block.arrival; + for (const use of block.uses) { + if (use >= t) break; + last = use; + } + return last; +}; + +const SELECTORS: ReadonlyArray< + readonly [string, NonNullable["rank"]>] +> = [ + ["newest first", (block) => block.arrival], + ["least recently used", (block, t) => lastSeenUse(block, t)], + [ + "named by the new message, then LRU", + (block, t) => (block.mentions.includes(t) ? 1e9 : 0) + lastSeenUse(block, t), + ], + [ + "trained on numeric features", + (block, t, epoch) => + block.kind === "tool_output" + ? (relevanceScores.get(`${epoch.session}:${t}:${block.id}`) ?? 0) + : 1, + ], + ...[0.75, 0.85, 0.95].map( + (auc) => + [ + `stand-in AUC ${auc}`, + (block: Block, t: number) => + (nextUse(block, t - 1) !== null ? Math.SQRT2 * inverseNormal(auc) : 0) + + blockNoise(block), + ] as const, + ), + ["oracle", (block, t) => (nextUse(block, t - 1) !== null ? 1e9 : 0) + block.arrival], +]; +const idleCalls = (block: Block, t: number) => t - lastSeenUse(block, t); +const standInFilter = (auc: number, falsePositiveRate: number) => { + const separation = Math.SQRT2 * inverseNormal(auc); + const threshold = inverseNormal(1 - falsePositiveRate); + return (block: Block, t: number) => + // Here "positive" means still needed, so the rate is how often an unneeded block slips through. + (nextUse(block, t - 1) !== null ? separation : 0) + blockNoise(block) > threshold; +}; +const SEED_FILTERS: ReadonlyArray< + readonly [string, (block: Block, t: number, epoch: Epoch) => boolean] +> = [ + ["no tool output at all", () => false], + ["used in the last 5 calls", (block, t) => idleCalls(block, t) <= 5], + ["used in the last 20 calls", (block, t) => idleCalls(block, t) <= 20], + [ + "named by the message, or used in the last 5", + (block, t) => block.mentions.includes(t) || idleCalls(block, t) <= 5, + ], + [ + "numeric model, p over 0.3", + (block, t, epoch) => (relevanceScores.get(`${epoch.session}:${t}:${block.id}`) ?? 0) > 0.3, + ], + ["stand-in AUC 0.85, lets 30% of unneeded through", standInFilter(0.85, 0.3)], + ["stand-in AUC 0.95, lets 30% of unneeded through", standInFilter(0.95, 0.3)], + ["oracle: only blocks used again", (block, t) => nextUse(block, t - 1) !== null], +]; +const PRUNE_CONFIGS = [ + { trigger: 150_000, keepRecent: 20_000, keepTokens: 20_000, when: "userTurn" }, + { trigger: 150_000, keepRecent: 50_000, keepTokens: 50_000, when: "userTurn" }, + { trigger: 100_000, keepRecent: 20_000, keepTokens: 20_000, when: "cold" }, +] as const; +const SELECTION_CONFIGS = [ + { trigger: 150_000, keepRecent: 20_000, keepTokens: 20_000, when: "userTurn" }, + { trigger: 150_000, keepRecent: 20_000, keepTokens: 30_000, when: "userTurn" }, + { trigger: 150_000, keepRecent: 20_000, keepTokens: 50_000, when: "userTurn" }, + { trigger: 150_000, keepRecent: 20_000, keepTokens: 80_000, when: "userTurn" }, + { trigger: 260_000, keepRecent: 20_000, keepTokens: 50_000, when: "userTurn" }, + { trigger: 100_000, keepRecent: 20_000, keepTokens: 50_000, when: "cold" }, +] as const; + +const REFETCH = Number(process.env["CACHE_SNIP_REFETCH"] ?? 1); +const CUT_MISS = Number(process.env["CACHE_SNIP_CUT_MISS"] ?? 0); +const [REEXPLORE_TOKENS, REEXPLORE_CALLS] = (process.env["CACHE_SNIP_REEXPLORE"] ?? "0,0") + .split(",") + .map(Number); +const LIMITS: Readonly> = { codex: 318_000, claude: 700_000 }; + +export const policiesFor = (provider: string): readonly PolicyDefinition[] => { + const limit = LIMITS[provider] ?? 300_000; + // Median first prompt of a fresh session: system, tools, and instructions. + const floor = provider === "claude" ? 49_000 : 33_000; + const cutAt = (trigger: number, keepTokens: number, when: "userTurn" | "cold") => + rule(`head cut over ${trigger / 1000}k keep ${keepTokens / 1000}k, ${when}`, never, { + headCut: { trigger, keepTokens, floor, when }, + reexplore: { tokens: REEXPLORE_TOKENS ?? 0, calls: REEXPLORE_CALLS ?? 0 }, + }); + const withRefetch = { refetchRate: REFETCH, cutMissRate: CUT_MISS }; + const rule = (name: string, decide: Decide, options: RunOptions = {}) => + stateless(name, decide, { ...withRefetch, ...options }); + return [ + rule("never", never), + { + name: "CEILING dead+old + oracle timing", + make: (epoch, cutoff) => oracleDecide(epoch, cutoff, deadPlusOld), + options: withRefetch, + }, + { + name: "CEILING dead outputs + oracle timing", + make: (epoch, cutoff) => oracleDecide(epoch, cutoff, deadOracle), + options: withRefetch, + }, + rule("dead + room rule m1 s10%", whenRoomPays(deadOracle, limit, 1, 0.1)), + rule("dead + room rule m2 s15%", whenRoomPays(deadOracle, limit, 2, 0.15)), + rule("dead + cold only", whenCold(deadOracle, 5_000)), + { + name: "idle>30 + oracle timing", + make: (epoch, cutoff) => oracleDecide(epoch, cutoff, idleFor(30, 200)), + options: withRefetch, + }, + rule("idle>10 + room rule m2 s15%", whenRoomPays(idleFor(10, 200), limit, 2, 0.15)), + rule("idle>30 + room rule m2 s15%", whenRoomPays(idleFor(30, 200), limit, 2, 0.15)), + rule("idle>60 + room rule m2 s15%", whenRoomPays(idleFor(60, 200), limit, 2, 0.15)), + rule("idle>30 + cold only", whenCold(idleFor(30, 200), 5_000)), + rule("idle>10 + cold only", whenCold(idleFor(10, 200), 5_000)), + rule("superseded + cold only", whenCold(supersededOnly, 2_000)), + ...[100_000, 150_000, 200_000, 260_000].flatMap((trigger) => + [5_000, 20_000, 50_000, 100_000].map((keep) => cutAt(trigger, keep, "userTurn")), + ), + cutAt(100_000, 20_000, "cold"), + cutAt(100_000, 50_000, "cold"), + ...PRUNE_CONFIGS.flatMap((config) => [ + rule( + `prune ${config.trigger / 1000}k/${config.keepRecent / 1000}k/${config.when}: keep all`, + never, + { + headCut: { ...config, floor }, + reexplore: { tokens: REEXPLORE_TOKENS ?? 0, calls: REEXPLORE_CALLS ?? 0 }, + }, + ), + ...SEED_FILTERS.map(([name, seedFilter]) => + rule( + `prune ${config.trigger / 1000}k/${config.keepRecent / 1000}k/${config.when}: ${name}`, + never, + { + headCut: { ...config, floor, seedFilter }, + reexplore: { tokens: REEXPLORE_TOKENS ?? 0, calls: REEXPLORE_CALLS ?? 0 }, + }, + ), + ), + ]), + ...SELECTION_CONFIGS.flatMap((config) => + SELECTORS.map(([name, rank]) => + rule( + `select ${config.trigger / 1000}k/head${(config.keepTokens - config.keepRecent) / 1000}k/${config.when}: ${name}`, + never, + { + headCut: { ...config, floor, rank }, + reexplore: { tokens: REEXPLORE_TOKENS ?? 0, calls: REEXPLORE_CALLS ?? 0 }, + }, + ), + ), + ), + rule("rewind finished turn, over 2k", atUserTurn(finishedTurnWork(2_000))), + rule("rewind finished turn, over 10k", atUserTurn(finishedTurnWork(10_000))), + rule("rewind finished turn, over 30k", atUserTurn(finishedTurnWork(30_000))), + rule("prior turns + cold only", whenCold(beforeLastUserTurn(200), 5_000)), + rule( + "prior turns at user turn, room m2 s15%", + atUserTurn(whenRoomPays(beforeLastUserTurn(200), limit, 2, 0.15)), + ), + rule( + "prior turns at user turn, room m1 s25%", + atUserTurn(whenRoomPays(beforeLastUserTurn(200), limit, 1, 0.25)), + ), + rule("AUC.75 fpr5% + room m2 s15%", whenRoomPays(noisyClassifier(0.75, 0.05), limit, 2, 0.15)), + rule("AUC.85 fpr5% + room m2 s15%", whenRoomPays(noisyClassifier(0.85, 0.05), limit, 2, 0.15)), + rule("AUC.95 fpr5% + room m2 s15%", whenRoomPays(noisyClassifier(0.95, 0.05), limit, 2, 0.15)), + rule("AUC.85 fpr10% + room m2 s15%", whenRoomPays(noisyClassifier(0.85, 0.1), limit, 2, 0.15)), + rule("AUC.95 fpr10% + room m2 s15%", whenRoomPays(noisyClassifier(0.95, 0.1), limit, 2, 0.15)), + rule("AUC.85 fpr30% + room m2 s15%", whenRoomPays(noisyClassifier(0.85, 0.3), limit, 2, 0.15)), + rule("AUC.95 fpr30% + room m2 s15%", whenRoomPays(noisyClassifier(0.95, 0.3), limit, 2, 0.15)), + rule("AUC.85 fpr5% + cold only", whenCold(noisyClassifier(0.85, 0.05), 5_000)), + rule("keep3 trigger100k no-min (docs default)", keepRecent(100_000, 3, 0)), + rule("keep30 trigger200k min75k", keepRecent(200_000, 30, 75_000)), + rule("keep60 trigger250k min100k", keepRecent(250_000, 60, 100_000)), + rule("keep30 trigger400k min150k", keepRecent(400_000, 30, 150_000)), + rule("compact at 200k", never, { compactAt: 200_000 }), + rule("compact at 260k", never, { compactAt: 260_000 }), + rule("compact at 400k", never, { compactAt: 400_000 }), + rule("compact when cold, over 100k", never, { compactAt: 100_000, compactWhen: "cold" }), + rule("compact when cold, over 150k", never, { compactAt: 150_000, compactWhen: "cold" }), + rule("compact at user turn, over 150k", never, { compactAt: 150_000, compactWhen: "userTurn" }), + rule("compact at user turn, over 200k", never, { compactAt: 200_000, compactWhen: "userTurn" }), + rule("compact at user turn, over 260k", never, { compactAt: 260_000, compactWhen: "userTurn" }), + ]; +}; + +if (import.meta.main) { + const dir = process.argv[2]; + if (dir === undefined) throw new Error("usage: simulate.ts [days]"); + const days = Number(process.argv[3] ?? 30); + const scoresPath = NodePath.join(dir, "relevance-scores.csv"); + if (NodeFS.existsSync(scoresPath)) { + relevanceScores = new Map( + NodeFS.readFileSync(scoresPath, "utf8") + .trim() + .split("\n") + .map((line) => { + const [key, value] = line.split(","); + return [key ?? "", Number(value)] as const; + }), + ); + } + const replays = NodeFS.readdirSync(NodePath.join(dir, "replay")).map( + (name) => JSON.parse(NodeFS.readFileSync(NodePath.join(dir, "replay", name), "utf8")) as Replay, + ); + const latest = Math.max(...replays.flatMap((replay) => replay.calls.map((call) => call.ts))); + const cutoff = latest - days * 86_400_000; + + const billed = new Map(); + for (const replay of replays) { + const inWindow = replay.calls.filter((call) => call.ts >= cutoff); + billed.set( + replay.provider, + (billed.get(replay.provider) ?? 0) + inWindow.reduce((sum, call) => sum + call.usd, 0), + ); + } + + // Gate: the no-snip replay at T3's rates must reproduce what T3 priced. + const t3Rates = makeRateLookup(dir, "t3Table"); + for (const provider of ["codex", "claude"]) { + let simulated = 0; + for (const replay of replays.filter((candidate) => candidate.provider === provider)) { + for (const epoch of splitEpochs(replay, t3Rates)) + simulated += runEpoch(epoch, never, cutoff).usd; + } + const target = billed.get(provider) ?? 0; + console.log( + `gate ${provider}: simulated no-snip $${simulated.toFixed(2)} vs T3-priced $${target.toFixed(2)} (${(((simulated - target) / target) * 100).toFixed(2)}%)`, + ); + } + + const ratesFor = makeRateLookup(dir, (process.env["CACHE_SNIP_RATES"] ?? "trueList") as RateMode); + const results: Record> = {}; + for (const provider of ["codex", "claude"]) { + const subset = replays.filter((candidate) => candidate.provider === provider); + const perSession = policiesFor(provider).map((policy) => { + const usd: number[] = []; + let snips = 0; + let snipped = 0; + let refetches = 0; + let refetchTokens = 0; + let wrong = 0; + let seed = 0; + for (const replay of subset) { + let sessionUsd = 0; + for (const epoch of splitEpochs(replay, ratesFor)) { + const tally = runEpoch(epoch, policy.make(epoch, cutoff), cutoff, policy.options ?? {}); + sessionUsd += tally.usd; + snips += tally.snips; + snipped += tally.snippedTokens; + refetches += tally.refetches; + refetchTokens += tally.refetchTokens; + wrong += tally.wrongSnips; + seed += tally.seedTokens; + } + usd.push(sessionUsd); + } + return { name: policy.name, usd, snips, snipped, refetches, refetchTokens, wrong, seed }; + }); + const baseline = perSession[0]!.usd; + const baselineTotal = baseline.reduce((sum, value) => sum + value, 0); + console.log( + `\n== ${provider}, true list baseline $${baselineTotal.toFixed(0)} over ${subset.length} sessions, last ${days} days`, + ); + console.log( + "policy".padEnd(44) + + " usd saved save% [95% CI] snips snipMtok refetch wrong refMtok", + ); + results[provider] = {}; + const random = seeded(7); + for (const row of perSession) { + const total = row.usd.reduce((sum, value) => sum + value, 0); + const samples: number[] = []; + for (let iteration = 0; iteration < 400; iteration += 1) { + let base = 0; + let policy = 0; + for (let draw = 0; draw < baseline.length; draw += 1) { + const pick = Math.floor(random() * baseline.length); + base += baseline[pick]!; + policy += row.usd[pick]!; + } + samples.push((base - policy) / Math.max(base, 1e-9)); + } + samples.sort((a, b) => a - b); + const low = samples[Math.floor(0.025 * samples.length)]!; + const high = samples[Math.floor(0.975 * samples.length)]!; + results[provider]![row.name] = [ + total, + low, + high, + row.refetchTokens, + row.snips, + row.snipped, + row.seed, + ]; + console.log( + row.name.padEnd(44) + + total.toFixed(0).padStart(6) + + (baselineTotal - total).toFixed(0).padStart(8) + + `${(((baselineTotal - total) / baselineTotal) * 100).toFixed(1)}%`.padStart(7) + + ` [${(low * 100).toFixed(1)},${(high * 100).toFixed(1)}]`.padStart(14) + + String(row.snips).padStart(7) + + (row.snipped / 1e6).toFixed(0).padStart(9) + + String(row.refetches).padStart(8) + + String(row.wrong).padStart(6) + + (row.refetchTokens / 1e6).toFixed(1).padStart(7), + ); + } + } + NodeFS.writeFileSync( + NodePath.join(dir, "simulate-results.json"), + JSON.stringify(results, null, 1), + ); +} + +function seeded(seed: number) { + let state = seed; + return () => { + state = (state * 1664525 + 1013904223) >>> 0; + return state / 4294967296; + }; +} diff --git a/evals/cache-snip/spend.ts b/evals/cache-snip/spend.ts new file mode 100644 index 0000000000000..aa11724cd9fb2 --- /dev/null +++ b/evals/cache-snip/spend.ts @@ -0,0 +1,119 @@ +// @effect-diagnostics nodeBuiltinImport:off globalConsole:off globalDate:off +// Prices every API call in a copy of T3's usage scan cache with T3's own pricing code. +// usage: node evals/cache-snip/spend.ts +import * as NodePath from "node:path"; + +import { loadApiCalls, loadRates, price, type ApiCall } from "./usageRecords.ts"; + +const dir = process.argv[2]; +if (dir === undefined) throw new Error("usage: spend.ts "); +const calls = loadApiCalls(NodePath.join(dir, "usage-scan-cache-v11.json")); +const rates = loadRates(NodePath.join(dir, "usage-model-rates.json")); + +interface Row { + calls: number; + usd: number; + input: number; + cacheRead: number; + cacheWrite: number; + output: number; + unsplitUsd: number; + tUncached: number; + tRead: number; + tWrite: number; + tOutput: number; +} +const emptyRow = (): Row => ({ + calls: 0, + usd: 0, + input: 0, + cacheRead: 0, + cacheWrite: 0, + output: 0, + unsplitUsd: 0, + tUncached: 0, + tRead: 0, + tWrite: 0, + tOutput: 0, +}); + +function tally(rows: Map, key: string, call: ApiCall) { + const row = rows.get(key) ?? emptyRow(); + const priced = price(rates, call); + row.calls += 1; + row.usd += priced.costUsd; + if (priced.categoryCostUsd === null) row.unsplitUsd += priced.costUsd; + else { + row.input += priced.categoryCostUsd.input; + row.cacheRead += priced.categoryCostUsd.cacheRead; + row.cacheWrite += priced.categoryCostUsd.cacheWrite; + row.output += priced.categoryCostUsd.output; + } + row.tUncached += call.uncached; + row.tRead += call.cacheRead; + row.tWrite += call.cacheWrite; + row.tOutput += call.output; + rows.set(key, row); +} + +const pct = (part: number, whole: number) => + whole > 0 ? `${((part / whole) * 100).toFixed(1).padStart(5)}%` : " -"; +const usd = (value: number) => `$${value.toFixed(0)}`.padStart(8); +const mtok = (value: number) => `${(value / 1e6).toFixed(0)}M`.padStart(8); + +function report(title: string, keyOf: (call: ApiCall) => string, subset: readonly ApiCall[]) { + const rows = new Map(); + for (const call of subset) tally(rows, keyOf(call), call); + console.log(`\n== ${title}`); + console.log( + "key".padEnd(34) + + "calls".padStart(8) + + "total$".padStart(8) + + " read$% write$% uncach$% output$%" + + " tRead tWrite tUncach tOutput", + ); + const sorted = [...rows.entries()].toSorted(([, a], [, b]) => b.usd - a.usd); + const all = sorted.reduce((sum, [, row]) => { + for (const key of Object.keys(row) as (keyof Row)[]) sum[key] += row[key]; + return sum; + }, emptyRow()); + for (const [key, row] of [...sorted, ["TOTAL", all] as const]) { + console.log( + key.padEnd(34).slice(0, 34) + + String(row.calls).padStart(8) + + usd(row.usd) + + " " + + pct(row.cacheRead, row.usd) + + " " + + pct(row.cacheWrite, row.usd) + + " " + + pct(row.input, row.usd) + + " " + + pct(row.output, row.usd) + + mtok(row.tRead) + + " " + + mtok(row.tWrite) + + " " + + mtok(row.tUncached) + + " " + + mtok(row.tOutput), + ); + } + if (all.unsplitUsd > 0) console.log(` unsplit provider-reported cost: ${usd(all.unsplitUsd)}`); +} + +const latest = Math.max(...calls.map((call) => call.timestampMs)); +const within = (days: number) => + calls.filter((call) => call.timestampMs >= latest - days * 86_400_000); +console.log(`calls ${calls.length}, latest ${new Date(latest).toISOString()}`); +const unpriced = calls.filter((call) => price(rates, call).costSource === "unpriced"); +const unpricedModels = new Map(); +for (const call of unpriced) { + unpricedModels.set(call.model, (unpricedModels.get(call.model) ?? 0) + 1); +} +console.log(`unpriced calls ${unpriced.length}`, Object.fromEntries(unpricedModels)); + +report("all retained, by provider", (call) => call.provider, calls); +report("last 30 days, by provider", (call) => call.provider, within(30)); +report("last 30 days, by model", (call) => `${call.model} ${call.speed}`, within(30)); +report("last 7 days, by model", (call) => `${call.model} ${call.speed}`, within(7)); diff --git a/evals/cache-snip/usageRecords.ts b/evals/cache-snip/usageRecords.ts new file mode 100644 index 0000000000000..d7e46dbc8655a --- /dev/null +++ b/evals/cache-snip/usageRecords.ts @@ -0,0 +1,150 @@ +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeFS from "node:fs"; + +import { + parseRateTable, + priceUsage, + type PricedRecord, + type PricedUsage, + type RateTable, +} from "../../apps/server/src/usage/usagePricing.ts"; + +const SPEEDS = ["standard", "fast", "ultrafast"] as const; +type Speed = (typeof SPEEDS)[number]; + +export interface ApiCall { + readonly provider: string; + readonly file: string; + readonly sessionId: string; + readonly timestampMs: number; + readonly model: string; + readonly speed: Speed; + readonly dedupeKey: string | null; + readonly priced: PricedRecord; + readonly uncached: number; + readonly cacheRead: number; + readonly cacheWrite: number; + readonly output: number; +} + +/** Everything the provider must read for this call. Output is excluded. */ +export const promptTokens = (call: ApiCall) => call.uncached + call.cacheRead + call.cacheWrite; + +type SerializedRecord = readonly [ + timestampMs: number, + modelIndex: number, + sessionIndex: number, + uncachedInputTokens: number, + cachedInputTokens: number, + cacheCreationTokens: number, + outputTokens: number, + reasoningTokens: number, + dedupeKey: string | null, + reportedCostUsd: number | null, + speed: number, + rateModelIndex?: number | null, + unresolvedServiceTierTurnId?: string, +]; + +interface SerializedCache { + readonly models: readonly string[]; + readonly sessions: readonly string[]; + readonly files: Readonly< + Record< + string, + { + readonly p: string; + readonly r: readonly SerializedRecord[]; + readonly t?: readonly SerializedRecord[]; + } + > + >; +} + +// Mirrors UsageService: moved copies of a shared session are dropped, repeated events in one file stay. +const SHARED_SESSION_PROVIDERS = new Set(["codex"]); + +/** Loads a copy of T3's usage scan cache with the Usage page's deduplication applied. */ +export function loadApiCalls(scanCachePath: string): ApiCall[] { + const cache = JSON.parse(NodeFS.readFileSync(scanCachePath, "utf8")) as SerializedCache; + const filesBySession = new Map>(); + for (const [file, entry] of Object.entries(cache.files)) { + for (const row of [...entry.r, ...(entry.t ?? [])]) { + const sessionId = cache.sessions[row[2]] ?? ""; + if (sessionId.length === 0) continue; + const seen = filesBySession.get(sessionId) ?? new Set(); + seen.add(file); + filesBySession.set(sessionId, seen); + } + } + + const seen = new Set(); + const calls: ApiCall[] = []; + for (const [file, entry] of Object.entries(cache.files)) { + const occurrences = new Map(); + for (const row of [...entry.r, ...(entry.t ?? [])]) { + const [ts, modelIndex, sessionIndex, uncached, cacheRead, cacheWrite, output, reasoning] = + row; + const model = cache.models[modelIndex] ?? ""; + const sessionId = cache.sessions[sessionIndex] ?? ""; + let dedupeKey = row[8]; + if (SHARED_SESSION_PROVIDERS.has(entry.p) && (filesBySession.get(sessionId)?.size ?? 0) > 1) { + const key = JSON.stringify([ + entry.p, + sessionId, + ts, + model, + uncached, + cacheRead, + cacheWrite, + ]); + const occurrence = (occurrences.get(key) ?? 0) + 1; + occurrences.set(key, occurrence); + dedupeKey = `${key}:${occurrence}`; + } + if (dedupeKey !== null) { + if (seen.has(dedupeKey)) continue; + seen.add(dedupeKey); + } + const speed = SPEEDS[row[10]] ?? "standard"; + const rateModelIndex = row[11]; + const rateModel = + typeof rateModelIndex === "number" ? cache.models[rateModelIndex] : undefined; + calls.push({ + provider: entry.p, + file, + sessionId, + timestampMs: ts, + model, + speed, + dedupeKey: row[8], + uncached, + cacheRead, + cacheWrite, + output, + priced: { + model, + ...(rateModel === undefined ? {} : { rateModel }), + speed, + reportedCostUsd: row[9], + totals: { + uncachedInputTokens: uncached, + cachedInputTokens: cacheRead, + cacheCreationTokens: cacheWrite, + outputTokens: output, + reasoningTokens: reasoning, + }, + }, + }); + } + } + return calls; +} + +export function loadRates(ratesPath: string): RateTable { + const { document } = JSON.parse(NodeFS.readFileSync(ratesPath, "utf8")) as { document: unknown }; + return parseRateTable(document); +} + +export const price = (rates: RateTable, call: ApiCall): PricedUsage => + priceUsage(rates, call.priced); diff --git a/evals/package.json b/evals/package.json index 94cfc43d5ef31..07a2e0afc44c1 100644 --- a/evals/package.json +++ b/evals/package.json @@ -9,6 +9,14 @@ "mcp-tool-use:regrade": "node mcp-tool-use/regrade.ts", "thread-titles": "node --env-file-if-exists=../.env thread-titles/run.ts", "test-signal": "node --env-file-if-exists=../.env test-signal/run.ts", + "cache-snip:spend": "node cache-snip/spend.ts", + "cache-snip:shape": "node cache-snip/shape.ts", + "cache-snip:replay": "node cache-snip/replay.ts", + "cache-snip:compose": "node cache-snip/compose.ts", + "cache-snip:simulate": "node cache-snip/simulate.ts", + "cache-snip:decisions": "node cache-snip/decisions.ts", + "cache-snip:relevance-rows": "node cache-snip/relevanceRows.ts", + "cache-snip:relevance-sample": "node cache-snip/relevanceSample.ts", "typecheck": "effect-tsgo tsrs --noEmit", "test": "vp test run --config ../vite.config.ts --dir ." }, From a423a431e2387a370d69bef571a250d3c3dec74f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 4 Nov 2026 21:41:22 -0800 Subject: [PATCH 2/6] feat(server): continue long idle threads in a fresh provider session A long thread re-sends its whole history on every call. Once its prompt cache has expired, the next request is uncached anyway, so that is the cheapest moment to drop old history. With the new setting on, a person's message to a thread over 100k tokens, after an hour idle, skips resume and continues through the existing context handoff. Commands, agent messages, and threads with background work are left alone. The setting is off by default. Co-Authored-By: Claude Opus 5.5 --- .../SettingsServerControlsRouteScreen.tsx | 14 ++++ .../src/features/settings/settings-search.ts | 3 + .../ProviderTurnStartService.test.ts | 80 ++++++++++++++++++- .../ProviderTurnStartService.ts | 51 ++++++++++-- .../components/settings/SettingsPanels.tsx | 4 + .../src/components/settings/ZackSettings.tsx | 36 +++++++++ .../src/components/settings/settingsSearch.ts | 6 ++ docs/user/portable-handoffs.md | 6 ++ packages/contracts/src/settings.ts | 2 + 9 files changed, 192 insertions(+), 10 deletions(-) diff --git a/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx index 471634fb265eb..9b0e97c8556e8 100644 --- a/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx @@ -477,6 +477,20 @@ function ServerSettingsDetail(props: { readonly page: SettingsPage }) { ))} ) : null} + + write({ freshSessionOnColdReturn: value })} + /> + { } return `/${parts.join("/")}`; }; -const layerWorktreeLocation = (worktreesDir = managedWorktreesDir) => +const layerWorktreeLocation = ( + worktreesDir = managedWorktreesDir, + settings?: Parameters[0], +) => Layer.mergeAll( Path.layer, Layer.mock(ServerConfig.ServerConfig)({ worktreesDir } as ServerConfig.ServerConfig["Service"]), - ServerSettings.layerTest(), + ServerSettings.layerTest(settings), ); function startWithMissingWorktree( @@ -286,6 +290,7 @@ it.each([ }); function makeLocalCommandHarness(input: { + readonly settings?: Parameters[0]; readonly pstack?: PstackPack.PstackPack["Service"]; readonly text: string; readonly knowledgeStore?: Effect.Success; @@ -1074,7 +1079,7 @@ function makeLocalCommandHarness(input: { }), }), IdAllocator.layer, - layerWorktreeLocation(input.worktreesDir), + layerWorktreeLocation(input.worktreesDir, input.settings), input.fileSystem === undefined ? FileSystem.layerNoop({ exists: () => Effect.succeed(true) }) : Layer.succeed(FileSystem.FileSystem, input.fileSystem), @@ -2380,6 +2385,75 @@ effectIt.effect( }), ); +effectIt.effect.each([ + { name: "a long idle thread", enabled: true, usedTokens: 120_000, idleMinutes: 61, fresh: true }, + { name: "the setting off", enabled: false, usedTokens: 120_000, idleMinutes: 61, fresh: false }, + { name: "a recent turn", enabled: true, usedTokens: 120_000, idleMinutes: 59, fresh: false }, + { name: "a small thread", enabled: true, usedTokens: 99_999, idleMinutes: 61, fresh: false }, + { + name: "a command message", + enabled: true, + usedTokens: 120_000, + idleMinutes: 61, + text: "/compact", + fresh: false, + }, + { + name: "a message not from a person", + enabled: true, + usedTokens: 120_000, + idleMinutes: 61, + createdBy: "system" as const, + fresh: false, + }, +] as const)("starting a turn with $name picks fresh session: $fresh", (scenario) => + Effect.gen(function* () { + const harness = makeLocalCommandHarness({ + settings: { freshSessionOnColdReturn: scenario.enabled }, + text: "text" in scenario ? scenario.text : "Keep going on the migration", + previousTurn: { runStatus: "completed", turnStatus: "completed" }, + recoveryNativeId: "fresh-native-conversation", + }); + const projection = harness.projection(); + const previousAt = "2026-09-04T12:00:00.000Z"; + const previousAttemptId = RunAttemptId.make("previous-failed-attempt"); + harness.setProjection({ + ...projection, + messages: projection.messages.map((message) => ({ + ...message, + ...("createdBy" in scenario ? { createdBy: scenario.createdBy } : {}), + })), + attempts: [ + ...projection.attempts, + { + ...projection.attempts[0]!, + id: previousAttemptId, + runId: RunId.make("previous-failed-run"), + rootNodeId: NodeId.make("previous-root"), + nativeThreadId: "existing-native-conversation", + providerThreadId: projection.providerThreads[0]!.id, + status: "completed", + }, + ], + providerTurns: projection.providerTurns.map((turn) => ({ + ...turn, + tokenUsage: { usedTokens: scenario.usedTokens, updatedAt: previousAt }, + })), + }); + yield* TestClock.setTime(Date.parse(previousAt) + scenario.idleMinutes * 60_000); + + yield* harness.start; + + // Commands run without a root run, which also means no fresh session. + const nativeId = + harness.startRootRun.mock.calls[0]?.[0].providerThread.nativeThreadRef?.nativeId; + expect(nativeId === "fresh-native-conversation").toBe(scenario.fresh); + expect(harness.events.some((event) => event.type === "context-handoff.updated")).toBe( + scenario.fresh, + ); + }), +); + effectIt.effect.each(["prepared", "uncertain"] as const)( "recovers a %s knowledge receipt through the native handoff fallback without resuming the ambiguous conversation", (status) => diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 954cfce326e53..542398091cfca 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -136,6 +136,11 @@ export class ProviderTurnStartServiceV2 extends Context.Service< ProviderTurnStartServiceV2Shape >()("t3/orchestration-v2/ProviderTurnStartService/ProviderTurnStartServiceV2") {} +// Past both limits, a replay of real threads shows a fresh session repaying its uncached first +// request: the cache has already expired, and the smaller prompt is cheaper on every later call. +const COLD_RETURN_IDLE_MS = 60 * 60_000; +const COLD_RETURN_MIN_CONTEXT_TOKENS = 100_000; + export const layer: Layer.Layer< ProviderTurnStartServiceV2, never, @@ -159,6 +164,7 @@ export const layer: Layer.Layer< ProviderTurnStartServiceV2, Effect.gen(function* () { const pstack = yield* PstackPack.PstackPack; + const serverSettings = yield* ServerSettings.ServerSettingsService; const eventSink = yield* EventSink.EventSinkV2; const turnAnalytics = yield* ProviderUsageAnalytics.ProviderTurnAnalytics; const contextHandoffService = yield* ContextHandoffService.ContextHandoffServiceV2; @@ -1039,6 +1045,27 @@ export const layer: Layer.Layer< return retryHandoff; }, ); + const isColdReturn = Effect.fn("ProviderTurnStartService.isColdReturn")(function* () { + const enabled = yield* serverSettings.getSettings.pipe( + Effect.map((settings) => settings.freshSessionOnColdReturn), + Effect.orElseSucceed(() => false), + ); + if (!enabled) return false; + const measured = latestNativeContextUsage(projection, providerThread); + if (measured === undefined) return false; + if (measured.usage.usedTokens < COLD_RETURN_MIN_CONTEXT_TOKENS) return false; + const reportedAt = DateTime.make(measured.reportedAt); + if (Option.isNone(reportedAt)) return false; + const idleMs = + DateTime.toEpochMillis(yield* DateTime.now) - DateTime.toEpochMillis(reportedAt.value); + if (idleMs < COLD_RETURN_IDLE_MS) return false; + const hasPendingWork = yield* ( + session.hasPendingBackgroundWorkForThread?.(providerThread, { includeSubagents: true }) ?? + session.hasPendingBackgroundWork ?? + Effect.succeed(false) + ); + return !hasPendingWork; + }); const prepareRecoveryHandoff = Effect.fn("ProviderTurnStartService.prepareRecoveryHandoff")( function* () { const transferId = yield* idAllocator.allocate.contextTransfer({ @@ -1257,15 +1284,23 @@ export const layer: Layer.Layer< handoff.delivery?.nativeThreadId === providerThread.nativeThreadRef?.nativeId && handoff.delivery?.status === "pending", ); + const coldReturn = + !uncertainDelivery && + message.createdBy === "user" && + !isCompactionCommandMessage(message) && + !goalCommand && + (yield* isColdReturn()); const resumed = yield* Effect.result( - uncertainDelivery + uncertainDelivery || coldReturn ? Effect.fail( new ProviderAdapter.ProviderAdapterTurnStartError({ driver: session.driver, threadId: projection.thread.id, providerThreadId: providerThread.id, runId, - cause: "Uncertain native history injection", + cause: coldReturn + ? "Fresh session after a long idle" + : "Uncertain native history injection", }), ) : session.resumeThread({ @@ -1343,11 +1378,13 @@ export const layer: Layer.Layer< driver: session.driver, providerThreadId: providerThread.id, runId, - reason: uncertainDelivery - ? "uncertain_history_delivery" - : resumed._tag === "Success" - ? "handoff_budget_exhausted" - : "resume_failed", + reason: coldReturn + ? "cold_return" + : uncertainDelivery + ? "uncertain_history_delivery" + : resumed._tag === "Success" + ? "handoff_budget_exhausted" + : "resume_failed", errorTag: resumed._tag === "Failure" ? resumed.failure._tag : "ContextHandoffBudgetError", }); const replacement = yield* loadFromProvider( diff --git a/apps/web/src/components/settings/SettingsPanels.tsx b/apps/web/src/components/settings/SettingsPanels.tsx index 3c65ba84a6e5b..c05aeacf5a772 100644 --- a/apps/web/src/components/settings/SettingsPanels.tsx +++ b/apps/web/src/components/settings/SettingsPanels.tsx @@ -523,6 +523,9 @@ export function useSettingsRestore(onRestored?: () => void) { ? ["Show skills in slash menu"] : []), ...(settings.pstackEnabled !== DEFAULT_UNIFIED_SETTINGS.pstackEnabled ? ["pstack"] : []), + ...(settings.freshSessionOnColdReturn !== DEFAULT_UNIFIED_SETTINGS.freshSessionOnColdReturn + ? ["Fresh session after a long break"] + : []), ...(settings.composerRichTextEnabled !== DEFAULT_UNIFIED_SETTINGS.composerRichTextEnabled ? ["Rich text composer"] : []), @@ -642,6 +645,7 @@ export function useSettingsRestore(onRestored?: () => void) { proactivePanelsEnabled: DEFAULT_UNIFIED_SETTINGS.proactivePanelsEnabled, showSkillsInSlashMenu: DEFAULT_UNIFIED_SETTINGS.showSkillsInSlashMenu, pstackEnabled: DEFAULT_UNIFIED_SETTINGS.pstackEnabled, + freshSessionOnColdReturn: DEFAULT_UNIFIED_SETTINGS.freshSessionOnColdReturn, composerRichTextEnabled: DEFAULT_UNIFIED_SETTINGS.composerRichTextEnabled, sendShortcut: DEFAULT_UNIFIED_SETTINGS.sendShortcut, followUpBehavior: DEFAULT_UNIFIED_SETTINGS.followUpBehavior, diff --git a/apps/web/src/components/settings/ZackSettings.tsx b/apps/web/src/components/settings/ZackSettings.tsx index ed41da6afbfee..1b7bcb4295756 100644 --- a/apps/web/src/components/settings/ZackSettings.tsx +++ b/apps/web/src/components/settings/ZackSettings.tsx @@ -27,6 +27,7 @@ export function ZackSettings() { + @@ -155,6 +156,41 @@ function ExternalHistoryRow() { ); } +function FreshSessionOnColdReturnRow() { + const settings = useScopedSettings(); + const updateSettings = useUpdateScopedSettings(); + return ( + + updateSettings({ + freshSessionOnColdReturn: DEFAULT_UNIFIED_SETTINGS.freshSessionOnColdReturn, + }) + } + /> + ) : null + } + control={ + + updateSettings({ freshSessionOnColdReturn: Boolean(checked) }) + } + aria-label="Fresh session after a long break" + /> + } + /> + ); +} + function PstackRow() { const settings = useScopedSettings(); const updateSettings = useUpdateScopedSettings(); diff --git a/apps/web/src/components/settings/settingsSearch.ts b/apps/web/src/components/settings/settingsSearch.ts index 2c3167b2456ca..f1759744452ef 100644 --- a/apps/web/src/components/settings/settingsSearch.ts +++ b/apps/web/src/components/settings/settingsSearch.ts @@ -615,6 +615,12 @@ export const SETTINGS_SEARCH_ITEMS = [ to: "/settings/labs", searchTerms: ["zack lauren tan skills workflows plugin pack architect setup models"], }, + { + id: "fresh-session-on-cold-return", + title: "Fresh session after a long break", + to: "/settings/labs", + searchTerms: ["zack cached input cost tokens long thread idle context handoff trim"], + }, { id: "composer-rich-text", title: "Rich text composer", diff --git a/docs/user/portable-handoffs.md b/docs/user/portable-handoffs.md index cfefeeab32ea2..856b405537d82 100644 --- a/docs/user/portable-handoffs.md +++ b/docs/user/portable-handoffs.md @@ -21,6 +21,12 @@ to retrieve saved messages and activity, including the remainder of a long item. constraint, you can still repeat it in your next message. Unless the project has conversation summaries turned on, a handoff is a budgeted selection, not a summary. +## Fresh sessions after a long break + +Long threads get more expensive with every message, because the provider re-reads the whole history each time. With **Settings > Labs > Zack > Fresh session after a long break** turned on, a message you send to a thread over about 100,000 tokens, after an hour or more idle, continues in a new provider session. It receives the same handoff described above, and your thread and its history in T3 Code stay as they were. + +The first request costs more because nothing is cached, and the agent may re-read some files, so this suits threads you keep returning to. Runs with background work still in progress are left alone. The setting is off by default. + ## Conversation summaries With conversation summaries on, older history that would not fit in a handoff can arrive as short diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 44bae170d2164..d737f5402863c 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -1301,6 +1301,7 @@ export const ServerSettings = Schema.Struct({ enableAgentDeviceAccess: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), /** Offer the bundled pstack skills in every provider composer. */ pstackEnabled: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), + freshSessionOnColdReturn: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), /** Off by default: turning it on moves skill folders out of the user's Claude and Codex homes. */ skillsLibraryEnabled: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), /** Off by default: each settled run in an opted-in project costs a text generation call. */ @@ -1753,6 +1754,7 @@ export const ServerSettingsPatch = Schema.Struct({ ), enableAgentDeviceAccess: Schema.optionalKey(Schema.Boolean), pstackEnabled: Schema.optionalKey(Schema.Boolean), + freshSessionOnColdReturn: Schema.optionalKey(Schema.Boolean), skillsLibraryEnabled: Schema.optionalKey(Schema.Boolean), enableAssistedLearning: Schema.optionalKey(Schema.Boolean), enableConversationSummaries: Schema.optionalKey(Schema.Boolean), From 5a95650d10dd9f81dfa8021874ca733a187da49e Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 4 Nov 2026 21:41:24 -0800 Subject: [PATCH 3/6] fix(server): preserve native session guards on cold return Resume the existing native thread before deciding whether to start fresh, preserving external-writer locks and background work discovered during resume. Route the environment-wide setting through the correct settings search scope. --- .../ProviderTurnStartService.test.ts | 70 ++++++++++++++++++- .../ProviderTurnStartService.ts | 36 ++++++---- .../settings/settingsSearch.test.ts | 6 ++ .../src/components/settings/settingsSearch.ts | 1 + 4 files changed, 95 insertions(+), 18 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts index e101e5a074e0a..11528d673bc07 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts @@ -311,6 +311,7 @@ function makeLocalCommandHarness(input: { readonly ensureThreadFailure?: unknown; readonly historyReadFailureAfterFallback?: unknown; readonly resumeFailure?: ProviderAdapter.ProviderAdapterV2Error; + readonly pendingBackgroundWorkOnResume?: boolean; readonly interruptOpen?: boolean; readonly interruptRunBeforeOpenFailure?: boolean; readonly writeFailure?: unknown; @@ -789,7 +790,20 @@ function makeLocalCommandHarness(input: { const nativeStart = vi.fn( () => Effect.void, ); - const fallbackEnsureThread = vi.fn(() => Effect.succeed(providerThread)); + const fallbackEnsureThread = vi.fn(() => + Effect.succeed( + input.recoveryNativeId === undefined + ? providerThread + : { + ...providerThread, + nativeThreadRef: { + driver: providerThread.driver, + nativeId: input.recoveryNativeId, + strength: "strong" as const, + }, + }, + ), + ); const resumeFallbackSession = { driver: providerThread.driver, resumeThread: () => @@ -835,13 +849,18 @@ function makeLocalCommandHarness(input: { forkThread, startTurn, }; + let resumedNativeThread = false; const open = vi.fn(() => input.sideChat !== undefined ? Effect.succeed(sideChatSession as never) : input.interruptOpen === true ? Effect.interrupt : "historyReadFailureAfterFallback" in input || "resumeFailure" in input - ? Effect.succeed(resumeFallbackSession as never) + ? Effect.succeed({ + ...resumeFallbackSession, + providerSession, + startTurn: nativeStart, + } as never) : "ensureThreadFailure" in input ? Effect.succeed({ driver: providerThread.driver, ensureThread } as never) : "openFailure" in input @@ -873,7 +892,15 @@ function makeLocalCommandHarness(input: { strength: "strong" as const, }, }), - resumeThread: () => Effect.succeed(projection.providerThreads.at(-1)), + resumeThread: () => + Effect.sync(() => { + resumedNativeThread = true; + return projection.providerThreads.at(-1); + }), + hasPendingBackgroundWorkForThread: () => + Effect.sync( + () => input.pendingBackgroundWorkOnResume === true && resumedNativeThread, + ), } as never) : Effect.die("A local command must not open a native session."), ); @@ -2390,6 +2417,22 @@ effectIt.effect.each([ { name: "the setting off", enabled: false, usedTokens: 120_000, idleMinutes: 61, fresh: false }, { name: "a recent turn", enabled: true, usedTokens: 120_000, idleMinutes: 59, fresh: false }, { name: "a small thread", enabled: true, usedTokens: 99_999, idleMinutes: 61, fresh: false }, + { + name: "a native thread held by another client", + enabled: true, + usedTokens: 120_000, + idleMinutes: 61, + threadInUse: true, + fresh: false, + }, + { + name: "background work discovered while resuming", + enabled: true, + usedTokens: 120_000, + idleMinutes: 61, + pendingBackgroundWorkOnResume: true, + fresh: false, + }, { name: "a command message", enabled: true, @@ -2413,6 +2456,19 @@ effectIt.effect.each([ text: "text" in scenario ? scenario.text : "Keep going on the migration", previousTurn: { runStatus: "completed", turnStatus: "completed" }, recoveryNativeId: "fresh-native-conversation", + ...("threadInUse" in scenario + ? { + resumeFailure: new ProviderAdapter.ProviderAdapterThreadInUseError({ + driver: ProviderDriverKind.make("codex"), + providerThreadId: ProviderThreadId.make("new-provider-thread"), + detail: "This conversation is open in another Codex client.", + cause: new Error("thread existing-native-conversation already has an active writer"), + }), + } + : {}), + ...("pendingBackgroundWorkOnResume" in scenario + ? { pendingBackgroundWorkOnResume: scenario.pendingBackgroundWorkOnResume } + : {}), }); const projection = harness.projection(); const previousAt = "2026-09-04T12:00:00.000Z"; @@ -2444,6 +2500,11 @@ effectIt.effect.each([ yield* harness.start; + if ("threadInUse" in scenario) { + expect(harness.fallbackEnsureThread).not.toHaveBeenCalled(); + expect(harness.startRootRun).not.toHaveBeenCalled(); + expect(harness.projection().runs.at(-1)).toMatchObject({ status: "failed", startedAt: null }); + } // Commands run without a root run, which also means no fresh session. const nativeId = harness.startRootRun.mock.calls[0]?.[0].providerThread.nativeThreadRef?.nativeId; @@ -2451,6 +2512,9 @@ effectIt.effect.each([ expect(harness.events.some((event) => event.type === "context-handoff.updated")).toBe( scenario.fresh, ); + if ("pendingBackgroundWorkOnResume" in scenario) { + expect(nativeId).toBe("existing-native-conversation"); + } }), ); diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 542398091cfca..eaf900af9d128 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -1045,13 +1045,15 @@ export const layer: Layer.Layer< return retryHandoff; }, ); - const isColdReturn = Effect.fn("ProviderTurnStartService.isColdReturn")(function* () { + const isColdReturn = Effect.fn("ProviderTurnStartService.isColdReturn")(function* ( + resumedThread: OrchestrationV2ProviderThread, + ) { const enabled = yield* serverSettings.getSettings.pipe( Effect.map((settings) => settings.freshSessionOnColdReturn), Effect.orElseSucceed(() => false), ); if (!enabled) return false; - const measured = latestNativeContextUsage(projection, providerThread); + const measured = latestNativeContextUsage(projection, resumedThread); if (measured === undefined) return false; if (measured.usage.usedTokens < COLD_RETURN_MIN_CONTEXT_TOKENS) return false; const reportedAt = DateTime.make(measured.reportedAt); @@ -1060,7 +1062,7 @@ export const layer: Layer.Layer< DateTime.toEpochMillis(yield* DateTime.now) - DateTime.toEpochMillis(reportedAt.value); if (idleMs < COLD_RETURN_IDLE_MS) return false; const hasPendingWork = yield* ( - session.hasPendingBackgroundWorkForThread?.(providerThread, { includeSubagents: true }) ?? + session.hasPendingBackgroundWorkForThread?.(resumedThread, { includeSubagents: true }) ?? session.hasPendingBackgroundWork ?? Effect.succeed(false) ); @@ -1284,23 +1286,15 @@ export const layer: Layer.Layer< handoff.delivery?.nativeThreadId === providerThread.nativeThreadRef?.nativeId && handoff.delivery?.status === "pending", ); - const coldReturn = - !uncertainDelivery && - message.createdBy === "user" && - !isCompactionCommandMessage(message) && - !goalCommand && - (yield* isColdReturn()); const resumed = yield* Effect.result( - uncertainDelivery || coldReturn + uncertainDelivery ? Effect.fail( new ProviderAdapter.ProviderAdapterTurnStartError({ driver: session.driver, threadId: projection.thread.id, providerThreadId: providerThread.id, runId, - cause: coldReturn - ? "Fresh session after a long idle" - : "Uncertain native history injection", + cause: "Uncertain native history injection", }), ) : session.resumeThread({ @@ -1310,7 +1304,14 @@ export const layer: Layer.Layer< runtimePolicy: resolvedRuntimePolicy, }), ); - if (resumed._tag === "Success") { + // Resume checks native ownership and refreshes background work before a fresh session. + const coldReturn = + resumed._tag === "Success" && + message.createdBy === "user" && + !isCompactionCommandMessage(message) && + !goalCommand && + (yield* isColdReturn(resumed.success)); + if (resumed._tag === "Success" && !coldReturn) { const context = handoffContextForThread(resumed.success); context.budget = yield* Effect.cached(context.budget); resumedHandoffContext = context; @@ -1385,7 +1386,12 @@ export const layer: Layer.Layer< : resumed._tag === "Success" ? "handoff_budget_exhausted" : "resume_failed", - errorTag: resumed._tag === "Failure" ? resumed.failure._tag : "ContextHandoffBudgetError", + ...(coldReturn + ? {} + : { + errorTag: + resumed._tag === "Failure" ? resumed.failure._tag : "ContextHandoffBudgetError", + }), }); const replacement = yield* loadFromProvider( session.ensureThread({ diff --git a/apps/web/src/components/settings/settingsSearch.test.ts b/apps/web/src/components/settings/settingsSearch.test.ts index 8a9a7e02c55b6..e8b50e154c402 100644 --- a/apps/web/src/components/settings/settingsSearch.test.ts +++ b/apps/web/src/components/settings/settingsSearch.test.ts @@ -234,6 +234,12 @@ describe("settings search targets", () => { expect(isSettingsSearchScopeAvailable(fallback.scope, "environment")).toBe(true); expect(isSettingsSearchScopeAvailable(fallback.scope, "all")).toBe(true); expect(isSettingsSearchScopeAvailable(fallback.scope, "project")).toBe(false); + const coldReturn = getSettingsSearchTargetScope("fresh-session-on-cold-return")!; + expect(coldReturn.scope).toBe("environment-defaults"); + expect(isSettingsSearchScopeAvailable(coldReturn.scope, "environment")).toBe(true); + expect(isSettingsSearchScopeAvailable(coldReturn.scope, "all")).toBe(true); + expect(isSettingsSearchScopeAvailable(coldReturn.scope, "project")).toBe(false); + expect(isSettingsSearchScopeAvailable(coldReturn.scope, "checkout")).toBe(false); const streaming = getSettingsSearchTargetScope("response-streaming")!; expect(streaming.scope).toBe("project-defaults"); expect(isSettingsSearchScopeAvailable(streaming.scope, "project")).toBe(true); diff --git a/apps/web/src/components/settings/settingsSearch.ts b/apps/web/src/components/settings/settingsSearch.ts index f1759744452ef..0a3619c6e08a3 100644 --- a/apps/web/src/components/settings/settingsSearch.ts +++ b/apps/web/src/components/settings/settingsSearch.ts @@ -619,6 +619,7 @@ export const SETTINGS_SEARCH_ITEMS = [ id: "fresh-session-on-cold-return", title: "Fresh session after a long break", to: "/settings/labs", + scope: "environment-defaults", searchTerms: ["zack cached input cost tokens long thread idle context handoff trim"], }, { From b2c4cba61d770179624c5a713f4428503b2cb2a5 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 4 Nov 2026 22:29:39 -0800 Subject: [PATCH 4/6] fix(server): preserve sessions during cold-return preflight --- .../ProviderTurnStartService.test.ts | 62 ++++++++++++++++++- .../ProviderTurnStartService.ts | 27 ++++++-- 2 files changed, 82 insertions(+), 7 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts index 11528d673bc07..4ddccfb1cd111 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts @@ -312,6 +312,7 @@ function makeLocalCommandHarness(input: { readonly historyReadFailureAfterFallback?: unknown; readonly resumeFailure?: ProviderAdapter.ProviderAdapterV2Error; readonly pendingBackgroundWorkOnResume?: boolean; + readonly pendingBackgroundWorkAfterHandoff?: boolean; readonly interruptOpen?: boolean; readonly interruptRunBeforeOpenFailure?: boolean; readonly writeFailure?: unknown; @@ -850,6 +851,7 @@ function makeLocalCommandHarness(input: { startTurn, }; let resumedNativeThread = false; + let preparedRecoveryHandoff = false; const open = vi.fn(() => input.sideChat !== undefined ? Effect.succeed(sideChatSession as never) @@ -899,7 +901,10 @@ function makeLocalCommandHarness(input: { }), hasPendingBackgroundWorkForThread: () => Effect.sync( - () => input.pendingBackgroundWorkOnResume === true && resumedNativeThread, + () => + (input.pendingBackgroundWorkOnResume === true && resumedNativeThread) || + (input.pendingBackgroundWorkAfterHandoff === true && + preparedRecoveryHandoff), ), } as never) : Effect.die("A local command must not open a native session."), @@ -1092,7 +1097,13 @@ function makeLocalCommandHarness(input: { summaryText: input.sideChat ? "PARENT_HISTORY" : "Recovered history", createdByProviderInstanceId: input.sideChat ? request.toProviderInstanceId : null, updatedAt: request.createdAt, - } as OrchestrationV2ContextHandoff), + } as OrchestrationV2ContextHandoff).pipe( + Effect.tap(() => + Effect.sync(() => { + preparedRecoveryHandoff = true; + }), + ), + ), }), Layer.mock(EventSink.EventSinkV2)({ writeIfRunCurrent, @@ -2417,6 +2428,22 @@ effectIt.effect.each([ { name: "the setting off", enabled: false, usedTokens: 120_000, idleMinutes: 61, fresh: false }, { name: "a recent turn", enabled: true, usedTokens: 120_000, idleMinutes: 59, fresh: false }, { name: "a small thread", enabled: true, usedTokens: 99_999, idleMinutes: 61, fresh: false }, + { + name: "recent completion after an old token report", + enabled: true, + usedTokens: 120_000, + idleMinutes: 61, + turnCompletedMinutes: 60, + fresh: false, + }, + { + name: "insufficient room for a fresh handoff", + enabled: true, + usedTokens: 120_000, + idleMinutes: 61, + modelContextWindow: 16_000, + fresh: false, + }, { name: "a native thread held by another client", enabled: true, @@ -2433,6 +2460,14 @@ effectIt.effect.each([ pendingBackgroundWorkOnResume: true, fresh: false, }, + { + name: "background work discovered while preparing the handoff", + enabled: true, + usedTokens: 120_000, + idleMinutes: 61, + pendingBackgroundWorkAfterHandoff: true, + fresh: false, + }, { name: "a command message", enabled: true, @@ -2456,6 +2491,9 @@ effectIt.effect.each([ text: "text" in scenario ? scenario.text : "Keep going on the migration", previousTurn: { runStatus: "completed", turnStatus: "completed" }, recoveryNativeId: "fresh-native-conversation", + ...("modelContextWindow" in scenario + ? { modelContextWindow: scenario.modelContextWindow } + : {}), ...("threadInUse" in scenario ? { resumeFailure: new ProviderAdapter.ProviderAdapterThreadInUseError({ @@ -2469,6 +2507,9 @@ effectIt.effect.each([ ...("pendingBackgroundWorkOnResume" in scenario ? { pendingBackgroundWorkOnResume: scenario.pendingBackgroundWorkOnResume } : {}), + ...("pendingBackgroundWorkAfterHandoff" in scenario + ? { pendingBackgroundWorkAfterHandoff: scenario.pendingBackgroundWorkAfterHandoff } + : {}), }); const projection = harness.projection(); const previousAt = "2026-09-04T12:00:00.000Z"; @@ -2494,6 +2535,13 @@ effectIt.effect.each([ providerTurns: projection.providerTurns.map((turn) => ({ ...turn, tokenUsage: { usedTokens: scenario.usedTokens, updatedAt: previousAt }, + ...("turnCompletedMinutes" in scenario + ? { + completedAt: DateTime.makeUnsafe( + Date.parse(previousAt) + scenario.turnCompletedMinutes * 60_000, + ), + } + : {}), })), }); yield* TestClock.setTime(Date.parse(previousAt) + scenario.idleMinutes * 60_000); @@ -2512,9 +2560,17 @@ effectIt.effect.each([ expect(harness.events.some((event) => event.type === "context-handoff.updated")).toBe( scenario.fresh, ); - if ("pendingBackgroundWorkOnResume" in scenario) { + if ( + "pendingBackgroundWorkOnResume" in scenario || + "pendingBackgroundWorkAfterHandoff" in scenario + ) { expect(nativeId).toBe("existing-native-conversation"); } + if ("modelContextWindow" in scenario) { + yield* dispatchKnowledge(harness.startRootRun.mock.calls[0]![0]); + expect(nativeId).toBe("existing-native-conversation"); + expect(harness.nativeStart).toHaveBeenCalledOnce(); + } }), ); diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index eaf900af9d128..86e3580f13108 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -1058,8 +1058,26 @@ export const layer: Layer.Layer< if (measured.usage.usedTokens < COLD_RETURN_MIN_CONTEXT_TOKENS) return false; const reportedAt = DateTime.make(measured.reportedAt); if (Option.isNone(reportedAt)) return false; - const idleMs = - DateTime.toEpochMillis(yield* DateTime.now) - DateTime.toEpochMillis(reportedAt.value); + // Token telemetry can arrive before a long tool run finishes. Count accepted root + // turn activity on this native conversation, including turns without usage reports. + let lastActivityMs = DateTime.toEpochMillis(reportedAt.value); + const attempts = new Map(projection.attempts.map((attempt) => [attempt.id, attempt])); + for (const turn of projection.providerTurns) { + if (turn.providerThreadId !== resumedThread.id || turn.runAttemptId === null) continue; + const attempt = attempts.get(turn.runAttemptId); + if ( + attempt === undefined || + attempt.nativeThreadId !== resumedThread.nativeThreadRef?.nativeId || + attempt.providerThreadId !== resumedThread.id || + attempt.rootNodeId !== turn.nodeId + ) + continue; + const activityAt = turn.completedAt ?? turn.startedAt; + if (activityAt !== null) { + lastActivityMs = Math.max(lastActivityMs, DateTime.toEpochMillis(activityAt)); + } + } + const idleMs = DateTime.toEpochMillis(yield* DateTime.now) - lastActivityMs; if (idleMs < COLD_RETURN_IDLE_MS) return false; const hasPendingWork = yield* ( session.hasPendingBackgroundWorkForThread?.(resumedThread, { includeSubagents: true }) ?? @@ -1311,7 +1329,7 @@ export const layer: Layer.Layer< !isCompactionCommandMessage(message) && !goalCommand && (yield* isColdReturn(resumed.success)); - if (resumed._tag === "Success" && !coldReturn) { + if (resumed._tag === "Success") { const context = handoffContextForThread(resumed.success); context.budget = yield* Effect.cached(context.budget); resumedHandoffContext = context; @@ -1327,7 +1345,7 @@ export const layer: Layer.Layer< Effect.as(true), Effect.catchTags({ ContextHandoffBudgetError: () => Effect.succeed(false) }), ); - if (fits) return resumed.success; + if (fits && !coldReturn) return resumed.success; const freshThread = { ...resumed.success, nativeThreadRef: null }; const freshContext = handoffContextForThread(freshThread); const { handoff } = yield* recoveryHandoff; @@ -1350,6 +1368,7 @@ export const layer: Layer.Layer< Effect.succeed(false) ); if (hasPendingWork) { + if (fits) return resumed.success; return yield* loadFromProvider( Effect.fail( new ProviderAdapter.ProviderAdapterTurnStartError({ From 207d7a5c136d61ce3fcd047ad1a25df3e7c516b0 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Wed, 4 Nov 2026 23:13:36 -0800 Subject: [PATCH 5/6] test(evals): score local models on whether old tool output is still needed Builds a labelled set from local transcripts, split by session, then scores and fine-tunes decision models on it and compares them with plain recency. The set holds chat text, so it stays local. The fit wrapper works around a one-row batch crash in mlx-decision 1.1.0. Co-Authored-By: Claude Opus 5.5 --- evals/cache-snip/README.md | 2 + evals/cache-snip/relevanceFit.py | 27 +++++ evals/cache-snip/relevanceQuestions.json | 18 +++ evals/cache-snip/relevanceReport.py | 91 +++++++++++++++ evals/cache-snip/relevanceSample.ts | 134 ++++++++++++++++++----- evals/cache-snip/relevanceScore.py | 39 +++---- 6 files changed, 260 insertions(+), 51 deletions(-) create mode 100644 evals/cache-snip/relevanceFit.py create mode 100644 evals/cache-snip/relevanceQuestions.json create mode 100644 evals/cache-snip/relevanceReport.py diff --git a/evals/cache-snip/README.md b/evals/cache-snip/README.md index 41ad84b9fcb97..3f022814aeb6c 100644 --- a/evals/cache-snip/README.md +++ b/evals/cache-snip/README.md @@ -22,6 +22,8 @@ pnpm cache-snip:relevance-rows 30 # what was knowable about each old tool python3 cache-snip/relevance.py # still-needed model, then rerun simulate to rank seeds with it pnpm cache-snip:relevance-sample # old tool outputs with the message they faced. Holds chat text, keep it local /bin/python cache-snip/relevanceScore.py # one GPU job at a time +/bin/python cache-snip/relevanceFit.py # fine-tune, with a one-row batch workaround +python3 cache-snip/relevanceReport.py [split] # models against recency on the same rows ``` `learn.py` needs numpy, pandas, and scikit-learn. diff --git a/evals/cache-snip/relevanceFit.py b/evals/cache-snip/relevanceFit.py new file mode 100644 index 0000000000000..bfbf89751fdb6 --- /dev/null +++ b/evals/cache-snip/relevanceFit.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +"""Runs `mlx-decision training fit`, working around a crash on one-row batches. + +usage: python relevanceFit.py +mlx-decision 1.1.0 sends no attention mask for a one-row batch, and on MLX 0.32.4 that +attention has no gradient path. A shuffle that leaves one row over crashes the last step of +a pass and loses the whole run, so the leftover row joins the batch before it. +""" +import runpy +import sys + +import mlx_decision.training.finetune as finetune + +_batches = finetune.batches + + +def batches(items, rng=None): + result = _batches(items, rng) + if rng is not None and len(result) > 1 and len(result[-1]) == 1: + leftover = result.pop() + result[-1] = result[-1] + leftover + return result + + +finetune.batches = batches +sys.argv = ["mlx-decision", "training", "fit", *sys.argv[1:]] +runpy.run_module("mlx_decision", run_name="__main__", alter_sys=True) diff --git a/evals/cache-snip/relevanceQuestions.json b/evals/cache-snip/relevanceQuestions.json new file mode 100644 index 0000000000000..86434e85a1707 --- /dev/null +++ b/evals/cache-snip/relevanceQuestions.json @@ -0,0 +1,18 @@ +{ + "needed": { + "type": "noul", + "instructions": "An agent is about to handle new_user_message. Would it still need the contents of earlier_tool_output, either to handle that message or to finish work still in progress? Answer yes only if it would need the actual contents, not just the fact that the tool ran.", + "criteria": { + "true": "The agent still needs these contents.", + "false": "These contents are no longer relevant: finished work, superseded results, or unrelated to the new message." + } + }, + "stale": { + "type": "noul", + "instructions": "Has earlier_tool_output gone stale for this conversation? Answer yes if it could be removed from the agent's context with no effect on how it handles new_user_message and the work still in progress. If removal could hurt, answer no.", + "criteria": { + "true": "Safe to remove: it no longer matters.", + "false": "Keep it: it may still matter." + } + } +} diff --git a/evals/cache-snip/relevanceReport.py b/evals/cache-snip/relevanceReport.py new file mode 100644 index 0000000000000..b927d074d3998 --- /dev/null +++ b/evals/cache-snip/relevanceReport.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Compares scored models with plain recency on the same rows, by split. + +usage: python3 evals/cache-snip/relevanceReport.py [split] +A higher score must mean "still needed", so the stale question is inverted. +""" +import glob +import json +import os +import statistics +import sys + +import numpy as np +from sklearn.metrics import roc_auc_score + +artifact = sys.argv[1] +split = sys.argv[2] if len(sys.argv) > 2 else "calibration" +rows = {} +for line in open(os.path.join(artifact, "relevance-compact.jsonl")): + row = json.loads(line) + if row["split"] == split: + rows[row["id"]] = row +numeric = {} +path = os.path.join(artifact, "relevance-scores.csv") +if os.path.exists(path): + for line in open(path): + key, value = line.rstrip("\n").split(",") + numeric[key] = float(value) + + +def session_of(identifier): + return identifier.split(":")[0] + + +def auc(ids, scores): + labels = [rows[i]["used"] for i in ids] + return roc_auc_score(labels, scores) if len(set(labels)) == 2 else float("nan") + + +def interval(ids, scores): + by_session = {} + for identifier, score in zip(ids, scores): + by_session.setdefault(session_of(identifier), []).append((identifier, score)) + sessions = list(by_session) + random = np.random.default_rng(7) + draws = [] + for _ in range(1000): + picked = random.integers(0, len(sessions), len(sessions)) + pairs = [pair for index in picked for pair in by_session[sessions[index]]] + labels = [rows[i]["used"] for i, _ in pairs] + if len(set(labels)) == 2: + draws.append(roc_auc_score(labels, [score for _, score in pairs])) + return np.percentile(draws, [2.5, 97.5]) + + +def line(name, ids, scores, seconds=None): + if len(ids) < 20: + return + low, high = interval(ids, scores) + timing = f" median {statistics.median(seconds):5.2f}s" if seconds else "" + print(f" {name:32s} AUC {auc(ids, scores):.3f} [{low:.3f},{high:.3f}] n {len(ids)}{timing}") + + +def report(title, keep): + ids = [i for i, row in rows.items() if keep(row)] + print(f"\n{title} ({len(ids)} rows, {sum(rows[i]['used'] for i in ids)} used later)") + line("recency of last use", ids, [-rows[i]["idle"] for i in ids]) + known = [i for i in ids if i in numeric] + line("numeric features", known, [numeric[i] for i in known]) + for file in sorted(glob.glob(os.path.join(artifact, "relevance-scores-*.jsonl"))): + name = os.path.basename(file)[len("relevance-scores-") : -len(".jsonl")] + by_prompt = {} + for raw in open(file): + result = json.loads(raw) + if result["id"] in ids and result.get("score") is not None: + by_prompt.setdefault(result["prompt"], []).append(result) + for prompt, results in by_prompt.items(): + sign = -1 if prompt == "stale" else 1 + line( + f"{name} {prompt}", + [result["id"] for result in results], + [sign * result["score"] for result in results], + [result["seconds"] for result in results], + ) + + +print(f"split: {split}") +report("All sampled head blocks", lambda row: True) +report("Idle over 30 calls", lambda row: row["idle"] > 30) +report("Codex", lambda row: row["provider"] == "codex") +report("Claude", lambda row: row["provider"] == "claude") diff --git a/evals/cache-snip/relevanceSample.ts b/evals/cache-snip/relevanceSample.ts index 161e1a27f8d10..083462b726a8c 100644 --- a/evals/cache-snip/relevanceSample.ts +++ b/evals/cache-snip/relevanceSample.ts @@ -1,6 +1,7 @@ // @effect-diagnostics nodeBuiltinImport:off globalConsole:off -// Samples old tool outputs with the message they faced, for scoring on a local model. Writes chat text, so keep the output local. +// Samples old tool outputs with the message they faced, to score and fine-tune local models. Writes chat text, so keep the output local. // usage: node evals/cache-snip/relevanceSample.ts [rows per split] +// Writes relevance-sample.jsonl and relevance-compact.jsonl to score, and relevance-fit-{train,dev}.jsonl to fine-tune. import * as NodeCrypto from "node:crypto"; import * as NodeFS from "node:fs"; import * as NodePath from "node:path"; @@ -16,6 +17,14 @@ const MIN_IDLE_CALLS = 5; const MIN_TOKENS = 100; const MAX_ROWS_PER_SESSION = 40; const POSITIVE_SHARE = 0.4; +const FIT_SESSION_CAP = { used: 400, unused: 600 }; +const FIT_TOTALS = { + train: { used: 4_000, unused: 6_000 }, + dev: { used: 800, unused: 800 }, +}; +const questions = JSON.parse( + NodeFS.readFileSync(new URL("./relevanceQuestions.json", import.meta.url), "utf8"), +) as Record; interface Row { session: string; @@ -63,25 +72,48 @@ const rows: Row[] = lines.slice(1).flatMap((line) => { ]; }); -function pick(split: Row["split"], used: number, count: number) { +function pickFrom(pool: readonly Row[], used: number, count: number, sessionCap: number) { const held = new Map(); const chosen: Row[] = []; - for (const row of rows - .filter((candidate) => candidate.split === split && candidate.used === used) + for (const row of pool + .filter((candidate) => candidate.used === used) .toSorted((a, b) => a.key - b.key)) { const taken = held.get(row.session) ?? 0; - if (taken >= MAX_ROWS_PER_SESSION || chosen.length >= count) continue; + if (taken >= sessionCap || chosen.length >= count) continue; held.set(row.session, taken + 1); chosen.push(row); } return chosen; } +const pick = (split: Row["split"], used: number, count: number) => + pickFrom( + rows.filter((row) => row.split === split), + used, + count, + MAX_ROWS_PER_SESSION, + ); + const sample = (["calibration", "sealed"] as const).flatMap((split) => [ ...pick(split, 1, Math.round(perSplit * POSITIVE_SHARE)), ...pick(split, 0, Math.round(perSplit * (1 - POSITIVE_SHARE))), ]); +const sampled = new Set(sample.map((row) => `${row.session}:${row.t}:${row.block}`)); +// Early stopping uses whole sessions of its own, so it cannot flatter a model on threads it trained on. +const tuning = rows.filter( + (row) => row.split === "calibration" && !sampled.has(`${row.session}:${row.t}:${row.block}`), +); +const isDev = (row: Row) => stableKey(`dev:${row.session}`) % 5 === 0; +const fitRows = (part: keyof typeof FIT_TOTALS) => { + const pool = tuning.filter((row) => (part === "dev") === isDev(row)); + return [ + ...pickFrom(pool, 1, FIT_TOTALS[part].used, FIT_SESSION_CAP.used), + ...pickFrom(pool, 0, FIT_TOTALS[part].unused, FIT_SESSION_CAP.unused), + ]; +}; +const fit = { train: fitRows("train"), dev: fitRows("dev") }; + const replays = new Map( NodeFS.readdirSync(NodePath.join(dir, "replay")).map((name) => { const replay = JSON.parse( @@ -102,15 +134,42 @@ const clip = (text: string, head: number, tail = 0) => ? text : `${text.slice(0, head)}\n[... ${text.length - head - tail} characters omitted ...]\n${tail > 0 ? text.slice(-tail) : ""}`; -const out = NodeFS.createWriteStream(NodePath.join(dir, "relevance-sample.jsonl")); +// Small encoders see about 1,000 tokens, and truncation cuts from the end, so the output goes last. +const compactState = (input: { + newMessage: string; + earlier: readonly string[]; + answer: string; + call: string; + output: string; + idle: number; +}) => + [ + `New message: ${clip(input.newMessage, 700)}`, + ...input.earlier.map((text) => `Earlier message: ${clip(text, 150)}`), + `Latest answer: ${clip(input.answer, 300)}`, + `Tool call: ${clip(input.call, 200)}`, + `Calls since last used: ${input.idle}`, + `Tool output: ${clip(input.output, 1_800, 400)}`, + ].join("\n"); + +const open = (name: string) => NodeFS.createWriteStream(NodePath.join(dir, name)); +const out = open("relevance-sample.jsonl"); +const compactOut = open("relevance-compact.jsonl"); +const fitOut = { train: open("relevance-fit-train.jsonl"), dev: open("relevance-fit-dev.jsonl") }; let written = 0; let mismatched = 0; -const bySession = new Map(); -for (const row of sample) { - const list = bySession.get(row.session) ?? []; - list.push(row); - bySession.set(row.session, list); -} +type Planned = Row & { part: "sample" | "train" | "dev" }; +const bySession = new Map(); +const plan = (part: Planned["part"], list: readonly Row[]) => { + for (const row of list) { + const held = bySession.get(row.session) ?? []; + held.push({ ...row, part }); + bySession.set(row.session, held); + } +}; +plan("sample", sample); +plan("train", fit.train); +plan("dev", fit.dev); for (const [session, sessionRows] of bySession) { const replay = replays.get(session); if (replay === undefined) continue; @@ -132,28 +191,49 @@ for (const [session, sessionRows] of bySession) { } const newest = capture.users.filter((user) => user.call <= row.t).slice(-3); const newMessage = newest.at(-1)?.text ?? ""; + const earlier = newest.slice(0, -1).map((user) => user.text); const answer = capture.answers.filter((item) => item.call < row.t).at(-1)?.text ?? ""; - out.write( - `${JSON.stringify({ - id: `${session}:${row.t}:${row.block}`, + const call = capture.calls.get(row.block) ?? ""; + const id = `${session}:${row.t}:${row.block}`; + const compact = compactState({ newMessage, earlier, answer, call, output, idle: row.idle }); + if (row.part === "sample") { + const base = { + id, split: row.split, provider: row.provider, used: row.used, tokens: row.tokens, idle: row.idle, age: row.age, - state: { - new_user_message: clip(newMessage, 2_000), - earlier_user_messages: newest.slice(0, -1).map((user) => clip(user.text, 500)), - latest_assistant_answer: clip(answer, 1_000), - earlier_tool_call: capture.calls.get(row.block) ?? "", - earlier_tool_output: clip(output, 6_000, 1_500), - calls_since_it_was_last_used: row.idle, - }, - })}\n`, - ); + }; + out.write( + `${JSON.stringify({ + ...base, + state: { + new_user_message: clip(newMessage, 2_000), + earlier_user_messages: earlier.map((text) => clip(text, 500)), + latest_assistant_answer: clip(answer, 1_000), + earlier_tool_call: call, + earlier_tool_output: clip(output, 6_000, 1_500), + calls_since_it_was_last_used: row.idle, + }, + })}\n`, + ); + compactOut.write(`${JSON.stringify({ ...base, state: compact })}\n`); + } else { + fitOut[row.part].write( + `${JSON.stringify({ + id, + state: compact, + questions: { needed: questions["needed"] }, + labels: { needed: row.used === 1 }, + })}\n`, + ); + } written += 1; } } -out.end(); -console.log(`sampled ${written}, not found ${mismatched}, sessions ${bySession.size}`); +for (const stream of [out, compactOut, fitOut.train, fitOut.dev]) stream.end(); +console.log( + `written ${written}, not found ${mismatched}, sessions ${bySession.size}, train ${fit.train.length}, dev ${fit.dev.length}`, +); diff --git a/evals/cache-snip/relevanceScore.py b/evals/cache-snip/relevanceScore.py index c5b26f3cfb8fe..e165007f61512 100644 --- a/evals/cache-snip/relevanceScore.py +++ b/evals/cache-snip/relevanceScore.py @@ -3,8 +3,11 @@ usage: python relevanceScore.py [split] Run under the mlx-decision virtualenv, and run one GPU job at a time. +CACHE_SNIP_SAMPLE picks the sample file (relevance-compact.jsonl for small models). +CACHE_SNIP_PROMPTS limits the questions. """ import json +import os import pathlib import sys import time @@ -14,25 +17,12 @@ artifact = pathlib.Path(sys.argv[1]) name, path = sys.argv[2], sys.argv[3] split = sys.argv[4] if len(sys.argv) > 4 else "calibration" -PROMPTS = { - "needed": { - "type": "noul", - "instructions": "An agent is about to handle new_user_message. Would it still need the contents of earlier_tool_output, either to handle that message or to finish work still in progress? Answer yes only if it would need the actual contents, not just the fact that the tool ran.", - "criteria": { - "true": "The agent still needs these contents.", - "false": "These contents are no longer relevant: finished work, superseded results, or unrelated to the new message.", - }, - }, - "stale": { - "type": "noul", - "instructions": "Has earlier_tool_output gone stale for this conversation? Answer yes if it could be removed from the agent's context with no effect on how it handles new_user_message and the work still in progress. If removal could hurt, answer no.", - "criteria": { - "true": "Safe to remove: it no longer matters.", - "false": "Keep it: it may still matter.", - }, - }, -} -rows = [json.loads(line) for line in (artifact / "relevance-sample.jsonl").read_text().splitlines()] +PROMPTS = json.loads((pathlib.Path(__file__).parent / "relevanceQuestions.json").read_text()) +wanted = os.environ.get("CACHE_SNIP_PROMPTS") +if wanted: + PROMPTS = {key: PROMPTS[key] for key in wanted.split(",")} +SAMPLE = os.environ.get("CACHE_SNIP_SAMPLE", "relevance-sample.jsonl") +rows = [json.loads(line) for line in (artifact / SAMPLE).read_text().splitlines()] rows = [row for row in rows if row["split"] == split] out = artifact / f"relevance-scores-{name}.jsonl" done = set() @@ -40,7 +30,8 @@ for line in out.read_text().splitlines(): row = json.loads(line) done.add((row["id"], row["prompt"])) -model = mlx_decision.load(path) +max_tokens = os.environ.get("CACHE_SNIP_MAX_TOKENS") +model = mlx_decision.load(path, **({"max_input_tokens": int(max_tokens)} if max_tokens else {})) print(name, "loaded", model.info().get("max_input_tokens"), flush=True) with out.open("a") as handle: for prompt, question in PROMPTS.items(): @@ -50,10 +41,10 @@ started = time.perf_counter() result = {"id": row["id"], "prompt": prompt, "model": name, "split": split} try: - response = model.decide_request({"state": row["state"], "questions": {"q": question}}) - result["score"] = None if response.truncated else float(response.answers["q"].noul) - if response.truncated: - result["error"] = "truncated" + response = model.decide_request({"state": row["state"], "questions": {prompt: question}}) + # A small encoder reads the start of a long state, and serving would do the same. + result["score"] = float(response.answers[prompt].noul) + result["truncated"] = bool(response.truncated) except Exception as error: if "METAL" in repr(error): raise From d4eb0943634ac646d57a9a0281dc3d3be802daa6 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Thu, 5 Nov 2026 12:28:50 -0800 Subject: [PATCH 6/6] test(evals): move the thread history replay out of this repo The replay and the relevance scoring read private chats and now run next to that data in a private repo. The cold-return setting does not depend on them. Co-Authored-By: Claude Opus 5.5 --- evals/README.md | 1 - evals/cache-snip/README.md | 37 - evals/cache-snip/compose.ts | 90 --- evals/cache-snip/decisions.ts | 155 ---- evals/cache-snip/learn.py | 123 --- evals/cache-snip/relevance.py | 61 -- evals/cache-snip/relevanceFit.py | 27 - evals/cache-snip/relevanceQuestions.json | 18 - evals/cache-snip/relevanceReport.py | 91 --- evals/cache-snip/relevanceRows.ts | 74 -- evals/cache-snip/relevanceSample.ts | 239 ------ evals/cache-snip/relevanceScore.py | 57 -- evals/cache-snip/replay.ts | 599 -------------- evals/cache-snip/shape.ts | 151 ---- evals/cache-snip/simulate.ts | 943 ----------------------- evals/cache-snip/spend.ts | 119 --- evals/cache-snip/usageRecords.ts | 150 ---- evals/package.json | 8 - 18 files changed, 2943 deletions(-) delete mode 100644 evals/cache-snip/README.md delete mode 100644 evals/cache-snip/compose.ts delete mode 100644 evals/cache-snip/decisions.ts delete mode 100644 evals/cache-snip/learn.py delete mode 100644 evals/cache-snip/relevance.py delete mode 100644 evals/cache-snip/relevanceFit.py delete mode 100644 evals/cache-snip/relevanceQuestions.json delete mode 100644 evals/cache-snip/relevanceReport.py delete mode 100644 evals/cache-snip/relevanceRows.ts delete mode 100644 evals/cache-snip/relevanceSample.ts delete mode 100644 evals/cache-snip/relevanceScore.py delete mode 100644 evals/cache-snip/replay.ts delete mode 100644 evals/cache-snip/shape.ts delete mode 100644 evals/cache-snip/simulate.ts delete mode 100644 evals/cache-snip/spend.ts delete mode 100644 evals/cache-snip/usageRecords.ts diff --git a/evals/README.md b/evals/README.md index a9b3c44914abe..7efe8b34d36dc 100644 --- a/evals/README.md +++ b/evals/README.md @@ -11,7 +11,6 @@ Write or run an eval when a change affects what a model sees or does: agent inst | [thread-split](thread-split/) | Offline local-model evaluation of whether a message can start an independent thread. | | [model-evaluations](model-evaluations/) | Whether blind judges use outcome evidence, respect task scope, ignore injected scoring instructions, and abstain when evidence is missing. | | [test-signal](test-signal/) | Whether a cheap model running the test-signal check flags low-signal tests. | -| [cache-snip](cache-snip/) | Whether trimming stale thread history saves money after the cache rebuild. Replays only, no model. | ## Run an eval diff --git a/evals/cache-snip/README.md b/evals/cache-snip/README.md deleted file mode 100644 index 3f022814aeb6c..0000000000000 --- a/evals/cache-snip/README.md +++ /dev/null @@ -1,37 +0,0 @@ -# Cache snip - -Would removing stale history from a thread lower total input cost, once the cache rebuild is charged? This replays real sessions against T3's own pricing code. It calls no model, costs nothing, and writes numbers and hashes only, never chat text. - -Run against copies, and keep the output out of the repo: - -```sh -mkdir -p -cp ~/.t3/userdata/usage-model-rates.json ~/.t3/userdata/usage-scan-cache-v11.json -cd evals -pnpm cache-snip:spend # dollars by category, model, provider -pnpm cache-snip:shape 30 # prompt sizes, cold calls, long-context cost -pnpm cache-snip:replay 30 # transcripts to per-session call and block records -pnpm cache-snip:compose 30 # what prompts are made of -for rate in 1 0.3 0.1; do - CACHE_SNIP_REFETCH=$rate pnpm cache-snip:simulate 30 - cp /simulate-results.json /simulate-results-refetch$rate.json -done -pnpm cache-snip:decisions 30 # dollar net of compacting at each candidate moment -python3 cache-snip/learn.py # model against fixed rules, held out by session and by time -pnpm cache-snip:relevance-rows 30 # what was knowable about each old tool output at each cut -python3 cache-snip/relevance.py # still-needed model, then rerun simulate to rank seeds with it -pnpm cache-snip:relevance-sample # old tool outputs with the message they faced. Holds chat text, keep it local -/bin/python cache-snip/relevanceScore.py # one GPU job at a time -/bin/python cache-snip/relevanceFit.py # fine-tune, with a one-row batch workaround -python3 cache-snip/relevanceReport.py [split] # models against recency on the same rows -``` - -`learn.py` needs numpy, pandas, and scikit-learn. - -## Reading the numbers - -- A snip at call `t` costs a rebuild of everything after the first removed block, and saves that block's reads on every later call before the next compaction. `simulate.ts` charges both. Its no-snip replay must reproduce T3's priced cost, and it prints that check first. -- `CACHE_SNIP_REFETCH` is the share of wrong snips that make an agent fetch the content again. It decides most outcomes, so report every result across a range. -- `CACHE_SNIP_REEXPLORE=tokens,calls` charges what a fresh chat re-reads per head cut. Set it from real handoff sessions. -- "Later quoted" is how `replay.ts` decides a block was used. It misses silent use, so the relevance ceiling depends on it. -- Replays measure dollars, never answer quality. Lost context that degrades an agent's work without a re-fetch does not show up here. diff --git a/evals/cache-snip/compose.ts b/evals/cache-snip/compose.ts deleted file mode 100644 index 19b23eeac8af0..0000000000000 --- a/evals/cache-snip/compose.ts +++ /dev/null @@ -1,90 +0,0 @@ -// @effect-diagnostics nodeBuiltinImport:off globalConsole:off -// What is the prompt made of, weighted by how many times each token is re-read? -// usage: node evals/cache-snip/compose.ts [days] -import * as NodeFS from "node:fs"; -import * as NodePath from "node:path"; - -import type { Block, Replay } from "./replay.ts"; - -const dir = process.argv[2]; -if (dir === undefined) throw new Error("usage: compose.ts [days]"); -const days = Number(process.argv[3] ?? 30); -const GRACE_CALLS = 5; - -const replays = NodeFS.readdirSync(NodePath.join(dir, "replay")).map( - (name) => JSON.parse(NodeFS.readFileSync(NodePath.join(dir, "replay", name), "utf8")) as Replay, -); -const latest = Math.max(...replays.flatMap((replay) => replay.calls.map((call) => call.ts))); -const cutoff = latest - days * 86_400_000; - -const ageName = (age: number) => - age <= 10 ? "<=10" : age <= 50 ? "11-50" : age <= 200 ? "51-200" : ">200"; - -for (const provider of ["codex", "claude"]) { - const tally = new Map(); - const bump = (key: string, value: number) => tally.set(key, (tally.get(key) ?? 0) + value); - let negativeBase = 0; - let callCount = 0; - const epochPeaks: number[] = []; - for (const replay of replays.filter((candidate) => candidate.provider === provider)) { - let live: Block[] = []; - const byArrival = new Map(); - for (const block of replay.blocks) { - const list = byArrival.get(block.arrival) ?? []; - list.push(block); - byArrival.set(block.arrival, list); - } - let peak = 0; - replay.calls.forEach((call, index) => { - if (call.reset) { - if (peak > 0) epochPeaks.push(peak); - peak = 0; - live = []; - } - peak = Math.max(peak, call.prompt); - live.push(...(byArrival.get(index) ?? [])); - if (call.ts < cutoff) return; - callCount += 1; - let attributed = 0; - for (const block of live) { - attributed += block.tokens; - if (block.kind !== "tool_output") { - bump(`kind:${block.kind}`, block.tokens); - continue; - } - const age = index - block.arrival; - bump("kind:tool_output", block.tokens); - bump(`age:${ageName(age)}`, block.tokens); - bump(`class:${block.toolClass ?? "none"}`, block.tokens); - if (block.lastUse + GRACE_CALLS < index) bump("dead:strict", block.tokens); - if (block.lastUseLoose + GRACE_CALLS < index) bump("dead:loose", block.tokens); - if (block.supersededAt !== null && block.supersededAt <= index) { - bump("superseded", block.tokens); - } - if ( - (block.supersededAt !== null && block.supersededAt <= index) || - (block.rereadAt !== null && block.rereadAt <= index) - ) { - bump("superseded+reread", block.tokens); - } - } - if (attributed > call.prompt * 1.02) negativeBase += 1; - bump("base", Math.max(0, call.prompt - attributed)); - bump("prompt", call.prompt); - }); - if (peak > 0) epochPeaks.push(peak); - } - const prompt = tally.get("prompt") ?? 1; - const sorted = epochPeaks.toSorted((a, b) => a - b); - const q = (value: number) => sorted[Math.floor(value * sorted.length)] ?? 0; - console.log( - `\n== ${provider}: prompt-weighted composition over ${callCount} calls, ${(prompt / 1e9).toFixed(2)}B prompt tokens`, - ); - for (const [key, value] of [...tally.entries()].toSorted(([a], [b]) => a.localeCompare(b))) { - if (key === "prompt") continue; - console.log(` ${key.padEnd(20)} ${((value / prompt) * 100).toFixed(1).padStart(5)}%`); - } - console.log( - ` calls with attributed > prompt: ${negativeBase}. Epoch peak prompt p10 ${q(0.1)} p50 ${q(0.5)} p90 ${q(0.9)} (${sorted.length} epochs)`, - ); -} diff --git a/evals/cache-snip/decisions.ts b/evals/cache-snip/decisions.ts deleted file mode 100644 index 2ac944683c758..0000000000000 --- a/evals/cache-snip/decisions.ts +++ /dev/null @@ -1,155 +0,0 @@ -// @effect-diagnostics nodeBuiltinImport:off globalConsole:off -// One row per moment a compaction could fire: what was knowable then, and what compacting there really saved. -// usage: node evals/cache-snip/decisions.ts [days] > decisions.csv -import * as NodeFS from "node:fs"; -import * as NodeCrypto from "node:crypto"; -import * as NodePath from "node:path"; - -import type { Block, Replay } from "./replay.ts"; -import { LONG_LIMIT, makeRateLookup, runEpoch, splitEpochs, type RateMode } from "./simulate.ts"; - -const dir = process.argv[2]; -if (dir === undefined) throw new Error("usage: decisions.ts [days]"); -const days = Number(process.argv[3] ?? 30); -const MIN_PROMPT = 50_000; -const REFETCH_RATES = [1, 0.3, 0.1]; - -const replays = NodeFS.readdirSync(NodePath.join(dir, "replay")).map( - (name) => JSON.parse(NodeFS.readFileSync(NodePath.join(dir, "replay", name), "utf8")) as Replay, -); -const latest = Math.max(...replays.flatMap((replay) => replay.calls.map((call) => call.ts))); -const cutoff = latest - days * 86_400_000; -const mode = (process.env["CACHE_SNIP_RATES"] ?? "trueList") as RateMode; -const ratesFor = makeRateLookup(dir, mode); - -const header = [ - "session", - "provider", - "epoch", - "t", - "ts", - "prompt", - "offset", - "sessionCalls", - "userTurns", - "callsPerTurn", - "gapMinutes", - "isCold", - "isUserTurn", - "prevReadShare", - "missOverRead", - "readUsdPerCall", - "longTier", - "growthPerCall", - "toolShare", - "idle10Share", - "idle30Share", - "priorTurnShare", - "callsThisTurn", - "userEstimate", - "textKey", - ...REFETCH_RATES.map((rate) => `net${rate}`), -]; -const rows: string[] = [header.join(",")]; -const hash = (value: string) => - NodeCrypto.createHash("sha1").update(value).digest("hex").slice(0, 12); - -for (const replay of replays) { - const session = hash(replay.file); - let sessionCalls = 0; - let userTurns = 0; - splitEpochs(replay, ratesFor).forEach((epoch, epochIndex) => { - const byArrival = new Map(); - for (const block of epoch.blocks) { - const list = byArrival.get(block.arrival) ?? []; - list.push(block); - byArrival.set(block.arrival, list); - } - const never = () => []; - const baseline = REFETCH_RATES.map( - (refetchRate) => runEpoch(epoch, never, cutoff, { refetchRate }).usd, - ); - const live: Block[] = []; - let lastUserCall = epoch.firstIndex; - epoch.calls.forEach((call, offset) => { - const t = epoch.firstIndex + offset; - const arriving = byArrival.get(t) ?? []; - live.push(...arriving); - sessionCalls += 1; - const userBlocks = arriving.filter((block) => block.kind === "user"); - const isUserTurn = epoch.turnStarts.has(t); - if (isUserTurn) userTurns += 1; - const callsThisTurn = t - lastUserCall; - if (isUserTurn) lastUserCall = t; - const previous = epoch.calls[offset - 1]; - const gapMinutes = previous === undefined ? 0 : (call.ts - previous.ts) / 60_000; - const isCold = gapMinutes >= 60; - if (offset === 0 || call.prompt < MIN_PROMPT || call.ts < cutoff) return; - if (!isUserTurn && !isCold) return; - - let lastUser = -1; - for (const block of live) { - if (block.kind === "user" && block.arrival < t) - lastUser = Math.max(lastUser, block.arrival); - } - let tool = 0; - let idle10 = 0; - let idle30 = 0; - let prior = 0; - for (const block of live) { - if (block.kind !== "tool_output") continue; - tool += block.tokens; - let last = block.arrival; - for (const use of block.uses) { - if (use >= t) break; - last = use; - } - if (t - last > 10) idle10 += block.tokens; - if (t - last > 30) idle30 += block.tokens; - if (block.arrival < lastUser) prior += block.tokens; - } - const rates = epoch.rates[offset]!; - const net = REFETCH_RATES.map( - (refetchRate, index) => - baseline[index]! - - runEpoch(epoch, never, cutoff, { refetchRate, compactCalls: new Set([t]) }).usd, - ); - const longTier = rates.longInput !== null && call.prompt > LONG_LIMIT; - rows.push( - [ - session, - replay.provider, - epochIndex, - t, - call.ts, - call.prompt, - offset, - sessionCalls, - userTurns, - (sessionCalls / Math.max(1, userTurns)).toFixed(2), - gapMinutes.toFixed(2), - isCold ? 1 : 0, - isUserTurn ? 1 : 0, - previous === undefined ? 0 : (previous.read / Math.max(1, previous.prompt)).toFixed(3), - (rates.miss / Math.max(rates.read, 1e-12)).toFixed(1), - (call.prompt * rates.read * (longTier ? rates.longInput! : 1)).toFixed(5), - longTier ? 1 : 0, - ((call.prompt - epoch.calls[0]!.prompt) / offset).toFixed(1), - (tool / call.prompt).toFixed(3), - (idle10 / call.prompt).toFixed(3), - (idle30 / call.prompt).toFixed(3), - (prior / call.prompt).toFixed(3), - callsThisTurn, - userBlocks.reduce((sum, block) => sum + block.estimate, 0).toFixed(0), - userBlocks[0]?.textKey ?? "", - ...net.map((value) => value.toFixed(5)), - ].join(","), - ); - }); - }); -} -NodeFS.writeFileSync( - NodePath.join(dir, mode === "trueList" ? "decisions.csv" : `decisions-${mode}.csv`), - `${rows.join("\n")}\n`, -); -console.log(`candidates ${rows.length - 1}`); diff --git a/evals/cache-snip/learn.py b/evals/cache-snip/learn.py deleted file mode 100644 index 6137bd6c2d192..0000000000000 --- a/evals/cache-snip/learn.py +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env python3 -"""Can a model choose when to compact better than a fixed rule? Held-out sessions, real dollars. - -usage: python3 evals/cache-snip/learn.py -Numeric features only, so it runs on CPU and needs no chat text. Run decisions.ts first. -""" -import json -import os -import sys - -import numpy as np -import pandas as pd -from sklearn.ensemble import HistGradientBoostingClassifier -from sklearn.model_selection import GroupKFold - -artifact = sys.argv[1] -frame = pd.read_csv(os.path.join(artifact, "decisions.csv")) -results = json.load(open(os.path.join(artifact, "simulate-results-refetch1.json"))) -BASE = {provider: results[provider]["never"][0] for provider in results} -FEATURES = [ - "prompt", "offset", "sessionCalls", "userTurns", "callsPerTurn", "gapMinutes", "isCold", - "isUserTurn", "prevReadShare", "missOverRead", "readUsdPerCall", "longTier", "growthPerCall", - "toolShare", "idle10Share", "idle30Share", "priorTurnShare", "callsThisTurn", "userEstimate", -] -THRESHOLDS = [100, 125, 150, 175, 200, 225, 250, 260, 270, 300] -REFETCH_RATES = ["1", "0.3", "0.1"] -MOMENTS = { - "any moment": lambda rows: np.ones(len(rows), bool), - "user turn": lambda rows: rows.isUserTurn.to_numpy() == 1, - "cold": lambda rows: rows.isCold.to_numpy() == 1, -} - - -def per_session(rows, chosen, column): - """Compact at the first chosen moment of each epoch. Dollars saved per session.""" - picked = rows[chosen].sort_values("t").groupby(["session", "epoch"], sort=False).head(1) - return picked.groupby("session")[column].sum() - - -def oracle(rows, column): - best = rows.groupby(["session", "epoch"])[column].max().clip(lower=0) - return best.groupby("session").sum() - - -def learned_rule(train, test, column, moment): - allowed = MOMENTS[moment] - def saved(rows, threshold): - return per_session(rows, allowed(rows) & (rows.prompt.to_numpy() >= threshold * 1000), column).sum() - best = max(THRESHOLDS, key=lambda threshold: saved(train, threshold)) - return per_session(test, allowed(test) & (test.prompt.to_numpy() >= best * 1000), column), best - - -def stopping_model(train, test, column): - """Labels a moment positive only when it is near the best moment left in the epoch, since waiting is an option.""" - ordered = train.sort_values(["session", "epoch", "t"]) - remaining_best = ordered.groupby(["session", "epoch"])[column].transform(lambda values: values[::-1].cummax()[::-1]) - labels = ((ordered[column] > 0) & (ordered[column] >= 0.9 * remaining_best)).astype(int) - model = HistGradientBoostingClassifier( - max_depth=3, max_iter=200, learning_rate=0.05, min_samples_leaf=15, random_state=7 - ) - model.fit(ordered[FEATURES], labels, sample_weight=ordered[column].abs().clip(lower=1e-3)) - return per_session(test, model.predict_proba(test[FEATURES])[:, 1] > 0.5, column) - - -def interval(values, sessions, base): - aligned = values.reindex(sessions).fillna(0).to_numpy() - draws = aligned[np.random.default_rng(7).integers(0, len(aligned), (4000, len(aligned)))].sum(axis=1) - low, high = np.percentile(draws, [2.5, 97.5]) - return aligned.sum() / base * 100, low / base * 100, high / base * 100 - - -def compare(rows, title): - print(f"\n== {title}: % of provider spend saved [95% CI], one compaction per epoch") - for provider in ["codex", "claude"]: - subset = rows[rows.provider == provider].reset_index(drop=True) - sessions = subset.session.unique() - for rate in REFETCH_RATES: - column = f"net{rate}" - collected = {f"rule, {moment}": pd.Series(dtype=float) for moment in MOMENTS} - collected["model"] = pd.Series(dtype=float) - chosen_thresholds = {moment: [] for moment in MOMENTS} - for train_index, test_index in GroupKFold(n_splits=5).split(subset, groups=subset.session): - train, test = subset.iloc[train_index], subset.iloc[test_index] - for moment in MOMENTS: - saved, threshold = learned_rule(train, test, column, moment) - chosen_thresholds[moment].append(threshold) - collected[f"rule, {moment}"] = pd.concat([collected[f"rule, {moment}"], saved]) - collected["model"] = pd.concat([collected["model"], stopping_model(train, test, column)]) - base = BASE[provider] - print(f"\n{provider}, baseline ${base:.0f}, refetch {rate}") - print(" {:28s} {:6.1f} [{:6.1f},{:6.1f}]".format("oracle", *interval(oracle(subset, column), sessions, base))) - for name, values in collected.items(): - note = "" - if name.startswith("rule"): - note = f" thresholds {sorted(set(chosen_thresholds[name.split(', ')[1]]))}" - print(" {:28s} {:6.1f} [{:6.1f},{:6.1f}]{}".format(name, *interval(values, sessions, base), note)) - for moment in MOMENTS: - difference = collected["model"].reindex(sessions).fillna(0) - collected[f"rule, {moment}"].reindex(sessions).fillna(0) - print(" {:28s} {:6.1f} [{:6.1f},{:6.1f}]".format(f"model minus rule, {moment}", *interval(difference, sessions, base))) - - -compare(frame, "held out by session") - -latest = frame.ts.max() -split = latest - 7 * 86_400_000 -past, recent = frame[frame.ts < split], frame[frame.ts >= split] -print(f"\n== trained on the past, tested on the last week: {len(past)} training moments, {len(recent)} test moments") -for provider in ["codex", "claude"]: - train, test = past[past.provider == provider], recent[recent.provider == provider] - sessions = test.session.unique() - if len(train) < 50 or len(test) < 20: - print(f"{provider}: too few moments, train {len(train)}, test {len(test)}. Pooling providers instead.") - train = past - for rate in REFETCH_RATES: - column = f"net{rate}" - rule_saved, threshold = learned_rule(train, test, column, "any moment") - model_saved = stopping_model(train, test, column) - base = BASE[provider] - print( - f"{provider} refetch {rate}: oracle {interval(oracle(test, column), sessions, base)[0]:5.1f}, " - f"rule(>{threshold}k) {interval(rule_saved, sessions, base)[0]:5.1f}, model {interval(model_saved, sessions, base)[0]:5.1f}, " - f"model minus rule {interval(model_saved.reindex(sessions).fillna(0) - rule_saved.reindex(sessions).fillna(0), sessions, base)}" - ) diff --git a/evals/cache-snip/relevance.py b/evals/cache-snip/relevance.py deleted file mode 100644 index b7285dc3b3e0b..0000000000000 --- a/evals/cache-snip/relevance.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python3 -"""How well can numeric features alone predict that an old tool output is still needed? - -usage: python3 evals/cache-snip/relevance.py -Run relevanceRows.ts first. Writes out-of-fold scores that simulate.ts ranks seeds with. -""" -import os -import sys - -import numpy as np -import pandas as pd -from sklearn.ensemble import HistGradientBoostingClassifier -from sklearn.metrics import roc_auc_score -from sklearn.model_selection import GroupKFold - -artifact = sys.argv[1] -rows = pd.read_csv(os.path.join(artifact, "relevance-rows.csv")) -FEATURES = [ - "tokens", "age", "idle", "usesSoFar", "turnsSinceArrival", "turnsSinceUse", "namedNow", - "namedBefore", "reread", "superseded", "toolClass", "prompt", "isCold", -] -print(f"rows {len(rows)}, sessions {rows.session.nunique()}, used later {rows.used.mean():.3f}, by tokens {np.average(rows.used, weights=rows.tokens):.3f}") -print("moments per session p50", int(rows.groupby('session').t.nunique().median()), "tool outputs per moment p50 / p90", *[int(v) for v in np.percentile(rows.groupby(['session','t']).size(), [50, 90])]) - -scores = np.zeros(len(rows)) -for train_index, test_index in GroupKFold(n_splits=5).split(rows, groups=rows.session): - model = HistGradientBoostingClassifier(max_depth=4, max_iter=200, learning_rate=0.07, min_samples_leaf=50, random_state=7) - train = rows.iloc[train_index] - model.fit(train[FEATURES], train.used, sample_weight=np.sqrt(train.tokens)) - scores[test_index] = model.predict_proba(rows.iloc[test_index][FEATURES])[:, 1] -rows["score"] = scores - - -def auc(frame, column, sign=1): - if frame.used.nunique() < 2: - return float("nan") - return roc_auc_score(frame.used, sign * frame[column], sample_weight=frame.tokens) - - -print("\nToken-weighted AUC for 'still used later', held out by session:") -for name, frame in [("all", rows), ("codex", rows[rows.provider == "codex"]), ("claude", rows[rows.provider == "claude"]), ("older than 50 calls", rows[rows.age > 50]), ("idle over 30 calls", rows[rows.idle > 30])]: - print(f" {name:22s} model {auc(frame, 'score'):.3f} recency of last use {auc(frame, 'idle', -1):.3f} age {auc(frame, 'age', -1):.3f} n {len(frame)}") - -named = rows[rows.namedNow == 1] -print(f"\nNamed by the new message: {len(named)} rows, used later {np.average(named.used, weights=named.tokens):.3f}. Not named: {np.average(rows[rows.namedNow == 0].used, weights=rows[rows.namedNow == 0].tokens):.3f}") -for low, high in [(0, 5), (5, 20), (20, 50), (50, 100), (100, 10**9)]: - bucket = rows[(rows.idle >= low) & (rows.idle < high)] - if len(bucket): - print(f" idle {low:>3}-{high if high < 10**9 else 'inf':<4} share of tokens {bucket.tokens.sum() / rows.tokens.sum():.2f}, used later {np.average(bucket.used, weights=bucket.tokens):.3f}") - -latest = rows.ts.max() -split = latest - 7 * 86_400_000 -past, recent = rows[rows.ts < split], rows[rows.ts >= split] -if len(past) > 1000 and len(recent) > 1000: - model = HistGradientBoostingClassifier(max_depth=4, max_iter=200, learning_rate=0.07, min_samples_leaf=50, random_state=7) - model.fit(past[FEATURES], past.used, sample_weight=np.sqrt(past.tokens)) - recent = recent.assign(score=model.predict_proba(recent[FEATURES])[:, 1]) - print(f"\nTrained on the past, tested on the last week: AUC {auc(recent, 'score'):.3f}, recency alone {auc(recent, 'idle', -1):.3f}") - -keys = rows.session + ":" + rows.t.astype(str) + ":" + rows.block.astype(str) -pd.DataFrame({"key": keys, "score": rows.score.round(4)}).to_csv(os.path.join(artifact, "relevance-scores.csv"), index=False, header=False) diff --git a/evals/cache-snip/relevanceFit.py b/evals/cache-snip/relevanceFit.py deleted file mode 100644 index bfbf89751fdb6..0000000000000 --- a/evals/cache-snip/relevanceFit.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python3 -"""Runs `mlx-decision training fit`, working around a crash on one-row batches. - -usage: python relevanceFit.py -mlx-decision 1.1.0 sends no attention mask for a one-row batch, and on MLX 0.32.4 that -attention has no gradient path. A shuffle that leaves one row over crashes the last step of -a pass and loses the whole run, so the leftover row joins the batch before it. -""" -import runpy -import sys - -import mlx_decision.training.finetune as finetune - -_batches = finetune.batches - - -def batches(items, rng=None): - result = _batches(items, rng) - if rng is not None and len(result) > 1 and len(result[-1]) == 1: - leftover = result.pop() - result[-1] = result[-1] + leftover - return result - - -finetune.batches = batches -sys.argv = ["mlx-decision", "training", "fit", *sys.argv[1:]] -runpy.run_module("mlx_decision", run_name="__main__", alter_sys=True) diff --git a/evals/cache-snip/relevanceQuestions.json b/evals/cache-snip/relevanceQuestions.json deleted file mode 100644 index 86434e85a1707..0000000000000 --- a/evals/cache-snip/relevanceQuestions.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "needed": { - "type": "noul", - "instructions": "An agent is about to handle new_user_message. Would it still need the contents of earlier_tool_output, either to handle that message or to finish work still in progress? Answer yes only if it would need the actual contents, not just the fact that the tool ran.", - "criteria": { - "true": "The agent still needs these contents.", - "false": "These contents are no longer relevant: finished work, superseded results, or unrelated to the new message." - } - }, - "stale": { - "type": "noul", - "instructions": "Has earlier_tool_output gone stale for this conversation? Answer yes if it could be removed from the agent's context with no effect on how it handles new_user_message and the work still in progress. If removal could hurt, answer no.", - "criteria": { - "true": "Safe to remove: it no longer matters.", - "false": "Keep it: it may still matter." - } - } -} diff --git a/evals/cache-snip/relevanceReport.py b/evals/cache-snip/relevanceReport.py deleted file mode 100644 index b927d074d3998..0000000000000 --- a/evals/cache-snip/relevanceReport.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env python3 -"""Compares scored models with plain recency on the same rows, by split. - -usage: python3 evals/cache-snip/relevanceReport.py [split] -A higher score must mean "still needed", so the stale question is inverted. -""" -import glob -import json -import os -import statistics -import sys - -import numpy as np -from sklearn.metrics import roc_auc_score - -artifact = sys.argv[1] -split = sys.argv[2] if len(sys.argv) > 2 else "calibration" -rows = {} -for line in open(os.path.join(artifact, "relevance-compact.jsonl")): - row = json.loads(line) - if row["split"] == split: - rows[row["id"]] = row -numeric = {} -path = os.path.join(artifact, "relevance-scores.csv") -if os.path.exists(path): - for line in open(path): - key, value = line.rstrip("\n").split(",") - numeric[key] = float(value) - - -def session_of(identifier): - return identifier.split(":")[0] - - -def auc(ids, scores): - labels = [rows[i]["used"] for i in ids] - return roc_auc_score(labels, scores) if len(set(labels)) == 2 else float("nan") - - -def interval(ids, scores): - by_session = {} - for identifier, score in zip(ids, scores): - by_session.setdefault(session_of(identifier), []).append((identifier, score)) - sessions = list(by_session) - random = np.random.default_rng(7) - draws = [] - for _ in range(1000): - picked = random.integers(0, len(sessions), len(sessions)) - pairs = [pair for index in picked for pair in by_session[sessions[index]]] - labels = [rows[i]["used"] for i, _ in pairs] - if len(set(labels)) == 2: - draws.append(roc_auc_score(labels, [score for _, score in pairs])) - return np.percentile(draws, [2.5, 97.5]) - - -def line(name, ids, scores, seconds=None): - if len(ids) < 20: - return - low, high = interval(ids, scores) - timing = f" median {statistics.median(seconds):5.2f}s" if seconds else "" - print(f" {name:32s} AUC {auc(ids, scores):.3f} [{low:.3f},{high:.3f}] n {len(ids)}{timing}") - - -def report(title, keep): - ids = [i for i, row in rows.items() if keep(row)] - print(f"\n{title} ({len(ids)} rows, {sum(rows[i]['used'] for i in ids)} used later)") - line("recency of last use", ids, [-rows[i]["idle"] for i in ids]) - known = [i for i in ids if i in numeric] - line("numeric features", known, [numeric[i] for i in known]) - for file in sorted(glob.glob(os.path.join(artifact, "relevance-scores-*.jsonl"))): - name = os.path.basename(file)[len("relevance-scores-") : -len(".jsonl")] - by_prompt = {} - for raw in open(file): - result = json.loads(raw) - if result["id"] in ids and result.get("score") is not None: - by_prompt.setdefault(result["prompt"], []).append(result) - for prompt, results in by_prompt.items(): - sign = -1 if prompt == "stale" else 1 - line( - f"{name} {prompt}", - [result["id"] for result in results], - [sign * result["score"] for result in results], - [result["seconds"] for result in results], - ) - - -print(f"split: {split}") -report("All sampled head blocks", lambda row: True) -report("Idle over 30 calls", lambda row: row["idle"] > 30) -report("Codex", lambda row: row["provider"] == "codex") -report("Claude", lambda row: row["provider"] == "claude") diff --git a/evals/cache-snip/relevanceRows.ts b/evals/cache-snip/relevanceRows.ts deleted file mode 100644 index 3e99d294b6b71..0000000000000 --- a/evals/cache-snip/relevanceRows.ts +++ /dev/null @@ -1,74 +0,0 @@ -// @effect-diagnostics nodeBuiltinImport:off globalConsole:off -// One row per tool output at each moment a head cut could fire: what was knowable then, and whether it was used later. -// usage: node evals/cache-snip/relevanceRows.ts [days] -import * as NodeFS from "node:fs"; -import * as NodePath from "node:path"; - -import type { Block, Replay } from "./replay.ts"; -import { makeRateLookup, sessionKey, splitEpochs } from "./simulate.ts"; - -const dir = process.argv[2]; -if (dir === undefined) throw new Error("usage: relevanceRows.ts [days]"); -const days = Number(process.argv[3] ?? 30); -const MIN_PROMPT = 100_000; -const MIN_BLOCK_TOKENS = 50; -const CLASSES = ["read", "search", "shell", "edit", "mcp", "agent", "other"]; - -const replays = NodeFS.readdirSync(NodePath.join(dir, "replay")).map( - (name) => JSON.parse(NodeFS.readFileSync(NodePath.join(dir, "replay", name), "utf8")) as Replay, -); -const latest = Math.max(...replays.flatMap((replay) => replay.calls.map((call) => call.ts))); -const cutoff = latest - days * 86_400_000; -const ratesFor = makeRateLookup(dir, "trueList"); - -const out = NodeFS.createWriteStream(NodePath.join(dir, "relevance-rows.csv")); -out.write( - "session,provider,t,ts,block,tokens,age,idle,usesSoFar,turnsSinceArrival,turnsSinceUse,namedNow,namedBefore,reread,superseded,toolClass,prompt,isCold,used,usedLoose\n", -); -let rows = 0; -let moments = 0; -for (const replay of replays) { - const session = sessionKey(replay.file); - const turnStarts = [...new Set(replay.turnStarts)].toSorted((a, b) => a - b); - for (const epoch of splitEpochs(replay, ratesFor)) { - const live: Block[] = []; - const byArrival = new Map(); - for (const block of epoch.blocks) { - const list = byArrival.get(block.arrival) ?? []; - list.push(block); - byArrival.set(block.arrival, list); - } - epoch.calls.forEach((call, offset) => { - const t = epoch.firstIndex + offset; - live.push(...(byArrival.get(t) ?? [])); - const previous = epoch.calls[offset - 1]; - const isCold = previous !== undefined && (call.ts - previous.ts) / 60_000 >= 60; - if (call.ts < cutoff || call.prompt < MIN_PROMPT || offset === 0) return; - if (!epoch.turnStarts.has(t) && !isCold) return; - moments += 1; - const turnsBefore = (index: number) => - turnStarts.filter((start) => start > index && start <= t).length; - for (const block of live) { - if (block.kind !== "tool_output" || block.tokens < MIN_BLOCK_TOKENS || block.arrival >= t) - continue; - let last = block.arrival; - let usesSoFar = 0; - for (const use of block.uses) { - if (use >= t) break; - last = use; - usesSoFar += 1; - } - const used = block.uses.some((use) => use >= t) ? 1 : 0; - const usedLoose = block.lastUseLoose >= t ? 1 : 0; - out.write( - `${session},${replay.provider},${t},${call.ts},${block.id},${block.tokens},${t - block.arrival},${t - last},${usesSoFar},${turnsBefore(block.arrival)},${turnsBefore(last)},${block.mentions.includes(t) ? 1 : 0},${block.mentions.some((at) => at < t) ? 1 : 0},${block.rereadAt !== null && block.rereadAt <= t ? 1 : 0},${block.supersededAt !== null && block.supersededAt <= t ? 1 : 0},${CLASSES.indexOf(block.toolClass ?? "other")},${call.prompt},${isCold ? 1 : 0},${used},${usedLoose}\n`, - ); - rows += 1; - } - }); - } -} -out.end(); -console.log( - `rows ${rows}, moments ${moments}, tool outputs per moment ${(rows / Math.max(moments, 1)).toFixed(0)}`, -); diff --git a/evals/cache-snip/relevanceSample.ts b/evals/cache-snip/relevanceSample.ts deleted file mode 100644 index 083462b726a8c..0000000000000 --- a/evals/cache-snip/relevanceSample.ts +++ /dev/null @@ -1,239 +0,0 @@ -// @effect-diagnostics nodeBuiltinImport:off globalConsole:off -// Samples old tool outputs with the message they faced, to score and fine-tune local models. Writes chat text, so keep the output local. -// usage: node evals/cache-snip/relevanceSample.ts [rows per split] -// Writes relevance-sample.jsonl and relevance-compact.jsonl to score, and relevance-fit-{train,dev}.jsonl to fine-tune. -import * as NodeCrypto from "node:crypto"; -import * as NodeFS from "node:fs"; -import * as NodePath from "node:path"; - -import { readClaude, readCodex, type Replay, type TextCapture } from "./replay.ts"; -import { sessionKey } from "./simulate.ts"; -import { loadApiCalls, type ApiCall } from "./usageRecords.ts"; - -const dir = process.argv[2]; -if (dir === undefined) throw new Error("usage: relevanceSample.ts [rows per split]"); -const perSplit = Number(process.argv[3] ?? 300); -const MIN_IDLE_CALLS = 5; -const MIN_TOKENS = 100; -const MAX_ROWS_PER_SESSION = 40; -const POSITIVE_SHARE = 0.4; -const FIT_SESSION_CAP = { used: 400, unused: 600 }; -const FIT_TOTALS = { - train: { used: 4_000, unused: 6_000 }, - dev: { used: 800, unused: 800 }, -}; -const questions = JSON.parse( - NodeFS.readFileSync(new URL("./relevanceQuestions.json", import.meta.url), "utf8"), -) as Record; - -interface Row { - session: string; - provider: string; - t: number; - block: number; - tokens: number; - age: number; - idle: number; - used: number; - split: "calibration" | "sealed"; - key: number; -} - -const stableKey = (value: string) => - NodeCrypto.createHash("sha1").update(value).digest().readUInt32BE(0); - -const lines = NodeFS.readFileSync(NodePath.join(dir, "relevance-rows.csv"), "utf8") - .trim() - .split("\n"); -const names = lines[0]!.split(","); -const column = (name: string) => names.indexOf(name); -const rows: Row[] = lines.slice(1).flatMap((line) => { - const cells = line.split(","); - const session = cells[column("session")]!; - const tokens = Number(cells[column("tokens")]); - const idle = Number(cells[column("idle")]); - if (tokens < MIN_TOKENS || idle <= MIN_IDLE_CALLS) return []; - const t = cells[column("t")]!; - const block = cells[column("block")]!; - return [ - { - session, - provider: cells[column("provider")]!, - t: Number(t), - block: Number(block), - tokens, - age: Number(cells[column("age")]), - idle, - used: Number(cells[column("used")]), - // Whole sessions stay on one side, so a model never scores a thread it was tuned on. - split: stableKey(`split:${session}`) % 10 < 7 ? "calibration" : "sealed", - key: stableKey(`row:${session}:${t}:${block}`), - }, - ]; -}); - -function pickFrom(pool: readonly Row[], used: number, count: number, sessionCap: number) { - const held = new Map(); - const chosen: Row[] = []; - for (const row of pool - .filter((candidate) => candidate.used === used) - .toSorted((a, b) => a.key - b.key)) { - const taken = held.get(row.session) ?? 0; - if (taken >= sessionCap || chosen.length >= count) continue; - held.set(row.session, taken + 1); - chosen.push(row); - } - return chosen; -} - -const pick = (split: Row["split"], used: number, count: number) => - pickFrom( - rows.filter((row) => row.split === split), - used, - count, - MAX_ROWS_PER_SESSION, - ); - -const sample = (["calibration", "sealed"] as const).flatMap((split) => [ - ...pick(split, 1, Math.round(perSplit * POSITIVE_SHARE)), - ...pick(split, 0, Math.round(perSplit * (1 - POSITIVE_SHARE))), -]); - -const sampled = new Set(sample.map((row) => `${row.session}:${row.t}:${row.block}`)); -// Early stopping uses whole sessions of its own, so it cannot flatter a model on threads it trained on. -const tuning = rows.filter( - (row) => row.split === "calibration" && !sampled.has(`${row.session}:${row.t}:${row.block}`), -); -const isDev = (row: Row) => stableKey(`dev:${row.session}`) % 5 === 0; -const fitRows = (part: keyof typeof FIT_TOTALS) => { - const pool = tuning.filter((row) => (part === "dev") === isDev(row)); - return [ - ...pickFrom(pool, 1, FIT_TOTALS[part].used, FIT_SESSION_CAP.used), - ...pickFrom(pool, 0, FIT_TOTALS[part].unused, FIT_SESSION_CAP.unused), - ]; -}; -const fit = { train: fitRows("train"), dev: fitRows("dev") }; - -const replays = new Map( - NodeFS.readdirSync(NodePath.join(dir, "replay")).map((name) => { - const replay = JSON.parse( - NodeFS.readFileSync(NodePath.join(dir, "replay", name), "utf8"), - ) as Replay; - return [sessionKey(replay.file), replay] as const; - }), -); -const callsByFile = new Map(); -for (const call of loadApiCalls(NodePath.join(dir, "usage-scan-cache-v11.json"))) { - const list = callsByFile.get(call.file) ?? []; - list.push(call); - callsByFile.set(call.file, list); -} - -const clip = (text: string, head: number, tail = 0) => - text.length <= head + tail - ? text - : `${text.slice(0, head)}\n[... ${text.length - head - tail} characters omitted ...]\n${tail > 0 ? text.slice(-tail) : ""}`; - -// Small encoders see about 1,000 tokens, and truncation cuts from the end, so the output goes last. -const compactState = (input: { - newMessage: string; - earlier: readonly string[]; - answer: string; - call: string; - output: string; - idle: number; -}) => - [ - `New message: ${clip(input.newMessage, 700)}`, - ...input.earlier.map((text) => `Earlier message: ${clip(text, 150)}`), - `Latest answer: ${clip(input.answer, 300)}`, - `Tool call: ${clip(input.call, 200)}`, - `Calls since last used: ${input.idle}`, - `Tool output: ${clip(input.output, 1_800, 400)}`, - ].join("\n"); - -const open = (name: string) => NodeFS.createWriteStream(NodePath.join(dir, name)); -const out = open("relevance-sample.jsonl"); -const compactOut = open("relevance-compact.jsonl"); -const fitOut = { train: open("relevance-fit-train.jsonl"), dev: open("relevance-fit-dev.jsonl") }; -let written = 0; -let mismatched = 0; -type Planned = Row & { part: "sample" | "train" | "dev" }; -const bySession = new Map(); -const plan = (part: Planned["part"], list: readonly Row[]) => { - for (const row of list) { - const held = bySession.get(row.session) ?? []; - held.push({ ...row, part }); - bySession.set(row.session, held); - } -}; -plan("sample", sample); -plan("train", fit.train); -plan("dev", fit.dev); -for (const [session, sessionRows] of bySession) { - const replay = replays.get(session); - if (replay === undefined) continue; - const capture: TextCapture = { - wanted: new Set(sessionRows.map((row) => row.block)), - outputs: new Map(), - calls: new Map(), - users: [], - answers: [], - }; - const calls = callsByFile.get(replay.file) ?? []; - const read = replay.provider === "codex" ? readCodex : readClaude; - const builder = await read(replay.file, calls, () => 0, capture); - for (const row of sessionRows) { - const output = capture.outputs.get(row.block); - if (output === undefined || builder.blocks[row.block]?.tokens !== row.tokens) { - mismatched += 1; - continue; - } - const newest = capture.users.filter((user) => user.call <= row.t).slice(-3); - const newMessage = newest.at(-1)?.text ?? ""; - const earlier = newest.slice(0, -1).map((user) => user.text); - const answer = capture.answers.filter((item) => item.call < row.t).at(-1)?.text ?? ""; - const call = capture.calls.get(row.block) ?? ""; - const id = `${session}:${row.t}:${row.block}`; - const compact = compactState({ newMessage, earlier, answer, call, output, idle: row.idle }); - if (row.part === "sample") { - const base = { - id, - split: row.split, - provider: row.provider, - used: row.used, - tokens: row.tokens, - idle: row.idle, - age: row.age, - }; - out.write( - `${JSON.stringify({ - ...base, - state: { - new_user_message: clip(newMessage, 2_000), - earlier_user_messages: earlier.map((text) => clip(text, 500)), - latest_assistant_answer: clip(answer, 1_000), - earlier_tool_call: call, - earlier_tool_output: clip(output, 6_000, 1_500), - calls_since_it_was_last_used: row.idle, - }, - })}\n`, - ); - compactOut.write(`${JSON.stringify({ ...base, state: compact })}\n`); - } else { - fitOut[row.part].write( - `${JSON.stringify({ - id, - state: compact, - questions: { needed: questions["needed"] }, - labels: { needed: row.used === 1 }, - })}\n`, - ); - } - written += 1; - } -} -for (const stream of [out, compactOut, fitOut.train, fitOut.dev]) stream.end(); -console.log( - `written ${written}, not found ${mismatched}, sessions ${bySession.size}, train ${fit.train.length}, dev ${fit.dev.length}`, -); diff --git a/evals/cache-snip/relevanceScore.py b/evals/cache-snip/relevanceScore.py deleted file mode 100644 index e165007f61512..0000000000000 --- a/evals/cache-snip/relevanceScore.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python3 -"""Scores the relevance sample on one local decision model. Resumable, one request at a time. - -usage: python relevanceScore.py [split] -Run under the mlx-decision virtualenv, and run one GPU job at a time. -CACHE_SNIP_SAMPLE picks the sample file (relevance-compact.jsonl for small models). -CACHE_SNIP_PROMPTS limits the questions. -""" -import json -import os -import pathlib -import sys -import time - -import mlx_decision - -artifact = pathlib.Path(sys.argv[1]) -name, path = sys.argv[2], sys.argv[3] -split = sys.argv[4] if len(sys.argv) > 4 else "calibration" -PROMPTS = json.loads((pathlib.Path(__file__).parent / "relevanceQuestions.json").read_text()) -wanted = os.environ.get("CACHE_SNIP_PROMPTS") -if wanted: - PROMPTS = {key: PROMPTS[key] for key in wanted.split(",")} -SAMPLE = os.environ.get("CACHE_SNIP_SAMPLE", "relevance-sample.jsonl") -rows = [json.loads(line) for line in (artifact / SAMPLE).read_text().splitlines()] -rows = [row for row in rows if row["split"] == split] -out = artifact / f"relevance-scores-{name}.jsonl" -done = set() -if out.exists(): - for line in out.read_text().splitlines(): - row = json.loads(line) - done.add((row["id"], row["prompt"])) -max_tokens = os.environ.get("CACHE_SNIP_MAX_TOKENS") -model = mlx_decision.load(path, **({"max_input_tokens": int(max_tokens)} if max_tokens else {})) -print(name, "loaded", model.info().get("max_input_tokens"), flush=True) -with out.open("a") as handle: - for prompt, question in PROMPTS.items(): - for row in rows: - if (row["id"], prompt) in done: - continue - started = time.perf_counter() - result = {"id": row["id"], "prompt": prompt, "model": name, "split": split} - try: - response = model.decide_request({"state": row["state"], "questions": {prompt: question}}) - # A small encoder reads the start of a long state, and serving would do the same. - result["score"] = float(response.answers[prompt].noul) - result["truncated"] = bool(response.truncated) - except Exception as error: - if "METAL" in repr(error): - raise - result["score"] = None - result["error"] = repr(error)[:200] - result["seconds"] = time.perf_counter() - started - handle.write(json.dumps(result) + "\n") - handle.flush() - print(name, prompt, "done", flush=True) -print(name, "all-done", flush=True) diff --git a/evals/cache-snip/replay.ts b/evals/cache-snip/replay.ts deleted file mode 100644 index d51b3e693f631..0000000000000 --- a/evals/cache-snip/replay.ts +++ /dev/null @@ -1,599 +0,0 @@ -// @effect-diagnostics nodeBuiltinImport:off globalConsole:off -// Rebuilds what each billed request carried, from local transcripts. Output holds numbers and hashes, never chat text. -// usage: node evals/cache-snip/replay.ts [days] -import * as NodeCrypto from "node:crypto"; -import * as NodeFS from "node:fs"; -import * as NodePath from "node:path"; -import * as NodeReadline from "node:readline"; - -import { loadApiCalls, promptTokens, type ApiCall } from "./usageRecords.ts"; - -export type BlockKind = "tool_output" | "tool_call" | "assistant" | "reasoning" | "user"; -export type ToolClass = "read" | "search" | "shell" | "edit" | "mcp" | "agent" | "other"; - -export interface Block { - /** Position in the replay's block list. */ - id: number; - /** Index of the first call whose prompt contains this block. */ - arrival: number; - /** First call whose prompt no longer contains it, after a real compaction or reset. */ - end: number | null; - tokens: number; - /** Raw size before prompt growth pinned `tokens`. Fits the weights in `WEIGHTS`. */ - estimate: number; - kind: BlockKind; - toolClass: ToolClass | null; - /** Responses that quoted something only this block supplied, in order. */ - uses: number[]; - /** User messages that named something only this block supplied. Known before the turn runs. */ - mentions: number[]; - /** Last of `uses`, or arrival. */ - lastUse: number; - /** Same, also counting plain long words. A looser reading of "used". */ - lastUseLoose: number; - /** First later call that made this block out of date: same command re-run, or its file edited. */ - supersededAt: number | null; - /** First later call that read any part of the same file again. Weaker than `supersededAt`. */ - rereadAt: number | null; - commandKey: string | null; - /** Hashed paths this block read. Lets the natural experiment spot re-reads across compactions. */ - fileKeys: string[]; - /** Transcript line, joining earlier blind-judge labels. Null where the source has none. */ - line: number | null; - /** Hash of a user message's text, joining thread-split labels. */ - textKey: string | null; -} - -export interface Call { - ts: number; - model: string; - prompt: number; - read: number; - miss: number; - output: number; - usd: number; - /** Prompt shrank by over 20% since the previous call. */ - reset: boolean; -} - -export interface Replay { - file: string; - provider: string; - calls: Call[]; - blocks: Block[]; - /** Calls that start a real user turn, excluding harness messages that only look like one. */ - turnStarts: number[]; -} - -// Fitted against reported prompt growth (compose.ts's companion regression): Claude R2 0.99, Codex R2 0.50. -const WEIGHTS: Readonly>>> = { - codex: { tool_output: 0.85, tool_call: 1.1, assistant: 1.1, reasoning: 0.25, user: 1 }, - claude: { tool_output: 1.6, tool_call: 1.55, assistant: 1.55, reasoning: 1.4, user: 1.4 }, -}; -const IMAGE_TOKEN_ESTIMATE = 1500; -const MAX_SCAN_CHARS = 200_000; -const RESET_DROP = 0.8; - -const hash = (value: string) => - NodeCrypto.createHash("sha1").update(value).digest("hex").slice(0, 12); -export const hashUserText = (text: string) => hash(text.replace(/\s+/g, " ").trim().slice(0, 200)); - -const CANDIDATE = /[A-Za-z0-9_][A-Za-z0-9_\-./]{3,}/g; -const strictToken = (token: string) => - token.length >= 5 && (/[_\-./]/.test(token) || /\d/.test(token) || /[a-z][A-Z]/.test(token)); -const looseToken = (token: string) => token.length >= 8 && /^[A-Za-z]+$/.test(token); - -function tokensOf(text: string) { - const strict = new Set(); - const loose = new Set(); - for (const match of text.slice(0, MAX_SCAN_CHARS).matchAll(CANDIDATE)) { - const token = match[0].replace(/[./\-_]+$/, ""); - if (strictToken(token)) strict.add(token); - else if (looseToken(token)) loose.add(token.toLowerCase()); - } - return { strict, loose }; -} - -/** Local-only text for building a labelled set. Replays themselves never hold chat text. */ -export interface TextCapture { - readonly wanted: ReadonlySet; - readonly outputs: Map; - readonly calls: Map; - readonly users: { call: number; text: string }[]; - readonly answers: { call: number; text: string }[]; -} - -interface ToolMeta { - readonly label?: string; - readonly toolClass: ToolClass; - readonly commandKey: string | null; - readonly reads: readonly string[]; - readonly edits: readonly string[]; -} - -const READ_COMMAND = - /\b(?:cat|sed\s+-n|nl|head|tail|less|bat)\b[^|;&\n]*?([\w./~-]+\.[A-Za-z0-9]+)/g; -const PATCH_FILE = /\*\*\* (?:Update|Add|Delete) File: ([^\s"\\]+)/g; -const SEARCH_COMMAND = /(^|[\s;&|(])(rg|grep|find|ls|fd|tree|git\s+(log|grep|show|diff|status))\b/; - -/** One classifier per provider shape, fed by the raw command text. */ -function classifyCommand(text: string): ToolMeta { - const reads = [...text.matchAll(READ_COMMAND)].flatMap((match) => match[1] ?? []); - const edits = [...text.matchAll(PATCH_FILE)].flatMap((match) => match[1] ?? []); - const toolClass: ToolClass = - edits.length > 0 - ? "edit" - : reads.length > 0 - ? "read" - : SEARCH_COMMAND.test(text) - ? "search" - : "shell"; - return { - label: text.slice(0, 400), - toolClass, - commandKey: hash(text.replace(/\s+/g, " ").trim()), - reads, - edits, - }; -} - -class ReplayBuilder { - private readonly weights: Readonly>; - private readonly capture: TextCapture | undefined; - constructor(weights: Readonly>, capture?: TextCapture) { - this.weights = weights; - this.capture = capture; - } - readonly calls: Call[] = []; - readonly blocks: Block[] = []; - readonly turnStarts: number[] = []; - private live: Block[] = []; - private pendingArrivals: Block[] = []; - private strictProvider = new Map(); - private looseProvider = new Map(); - private lastByCommand = new Map(); - private readersByFile = new Map(); - /** Codex logs a response's usage before its tool output and again after, so inputs in between belong to the next call. */ - inputOffset = 0; - /** Transcript line being read. */ - line: number | null = null; - - private add(kind: BlockKind, estimate: number, arrival: number, meta: ToolMeta | null): Block { - const block: Block = { - id: this.blocks.length, - arrival, - end: null, - tokens: 0, - estimate, - kind, - toolClass: meta?.toolClass ?? null, - uses: [], - mentions: [], - lastUse: arrival, - lastUseLoose: arrival, - supersededAt: null, - rereadAt: null, - commandKey: meta?.commandKey ?? null, - fileKeys: (meta?.reads ?? []).map(hash), - line: this.line, - textKey: null, - }; - this.blocks.push(block); - this.pendingArrivals.push(block); - return block; - } - - markTurnStart() { - this.turnStarts.push(this.calls.length + this.inputOffset); - } - - userMessage(text: string, images = 0) { - const block = this.add( - "user", - text.length / 4 + images * IMAGE_TOKEN_ESTIMATE, - this.calls.length + this.inputOffset, - null, - ); - block.textKey = hashUserText(text); - this.capture?.users.push({ call: block.arrival, text }); - const { strict } = tokensOf(text); - const mentioned = new Set(); - for (const token of strict) { - const provider = this.strictProvider.get(token); - if (provider !== undefined) mentioned.add(provider); - } - for (const provider of mentioned) provider.mentions.push(block.arrival); - } - - toolOutput(text: string, images: number, meta: ToolMeta | null) { - const block = this.add( - "tool_output", - text.length / 4 + images * IMAGE_TOKEN_ESTIMATE, - this.calls.length + this.inputOffset, - meta, - ); - if (this.capture?.wanted.has(block.id) === true) { - this.capture.outputs.set(block.id, text); - this.capture.calls.set(block.id, meta?.label ?? ""); - } - const { strict, loose } = tokensOf(text); - for (const token of strict) this.strictProvider.set(token, block); - for (const token of loose) this.looseProvider.set(token, block); - if (meta !== null) { - if (meta.commandKey !== null) { - const earlier = this.lastByCommand.get(meta.commandKey); - if (earlier !== undefined && earlier.supersededAt === null) { - earlier.supersededAt = block.arrival; - } - this.lastByCommand.set(meta.commandKey, block); - } - for (const file of meta.reads) { - const readers = this.readersByFile.get(file) ?? []; - for (const reader of readers) reader.rereadAt ??= block.arrival; - readers.push(block); - this.readersByFile.set(file, readers); - } - for (const file of meta.edits) { - for (const reader of this.readersByFile.get(file) ?? []) { - if (reader.supersededAt === null) reader.supersededAt = block.arrival; - } - this.readersByFile.delete(file); - } - } - } - - /** A model emission: enters the prompt after `response`, and proves earlier blocks were used at `response`. */ - emission( - kind: Exclude, - text: string, - estimate: number, - response: number, - ) { - this.add(kind, estimate, response + 1, null); - if (kind === "assistant") this.capture?.answers.push({ call: response, text }); - const { strict, loose } = tokensOf(text); - const strictHits = new Map(); - for (const token of strict) { - const provider = this.strictProvider.get(token); - if (provider === undefined) continue; - const hit = strictHits.get(provider) ?? { count: 0, long: false }; - hit.count += 1; - hit.long ||= token.length >= 12; - strictHits.set(provider, hit); - } - for (const [provider, hit] of strictHits) { - if (hit.count >= 2 || hit.long) { - if (provider.uses.at(-1) !== response) provider.uses.push(response); - provider.lastUse = Math.max(provider.lastUse, response); - provider.lastUseLoose = Math.max(provider.lastUseLoose, response); - } - } - const looseHits = new Map(); - for (const token of loose) { - const provider = this.looseProvider.get(token); - if (provider !== undefined) looseHits.set(provider, (looseHits.get(provider) ?? 0) + 1); - } - for (const [provider, count] of looseHits) { - if (count >= 2) provider.lastUseLoose = Math.max(provider.lastUseLoose, response); - } - } - - apiCall(call: ApiCall, usd: number) { - this.inputOffset = 0; - const index = this.calls.length; - const prompt = promptTokens(call); - const previous = this.calls[index - 1]; - const reset = previous !== undefined && prompt < previous.prompt * RESET_DROP; - if (reset) { - for (const block of this.live) block.end = index; - this.live = []; - this.strictProvider.clear(); - this.looseProvider.clear(); - this.lastByCommand.clear(); - this.readersByFile.clear(); - } - - const arriving = this.pendingArrivals.filter((block) => block.arrival <= index); - this.pendingArrivals = this.pendingArrivals.filter((block) => block.arrival > index); - const weighted = (block: Block) => block.estimate * this.weights[block.kind]; - const estimated = arriving.reduce((sum, block) => sum + weighted(block), 0); - const growth = previous === undefined || reset ? prompt : Math.max(0, prompt - previous.prompt); - // Never attribute more than was billed. Growth left over stays unattributed framing, which no snip can remove. - const scale = estimated > growth && estimated > 0 ? growth / estimated : 1; - for (const block of arriving) { - block.tokens = Math.round(weighted(block) * scale); - this.live.push(block); - } - - this.calls.push({ - ts: call.timestampMs, - model: call.model, - prompt, - read: call.cacheRead, - miss: call.uncached + call.cacheWrite, - output: call.output, - usd, - reset, - }); - } -} - -type Json = Record; -const isObject = (value: unknown): value is Json => - typeof value === "object" && value !== null && !Array.isArray(value); - -function contentText(content: unknown): { text: string; images: number } { - if (typeof content === "string") return { text: content, images: 0 }; - if (!Array.isArray(content)) return { text: "", images: 0 }; - let text = ""; - let images = 0; - for (const part of content) { - if (!isObject(part)) continue; - const type = part["type"]; - if (type === "input_image" || type === "image" || type === "output_image") images += 1; - else if (typeof part["text"] === "string") text += `${part["text"]}\n`; - } - return { text, images }; -} - -async function* lines(file: string) { - const reader = NodeReadline.createInterface({ - input: NodeFS.createReadStream(file), - crlfDelay: Infinity, - }); - for await (const line of reader) { - try { - const parsed = JSON.parse(line) as unknown; - if (isObject(parsed)) yield parsed; - } catch { - // Unterminated tail of a live session. - } - } -} - -function callQueue(calls: readonly ApiCall[]) { - const byTimestamp = new Map(); - for (const call of calls) { - const list = byTimestamp.get(call.timestampMs) ?? []; - list.push(call); - byTimestamp.set(call.timestampMs, list); - } - return (timestampMs: number) => byTimestamp.get(timestampMs)?.shift(); -} - -export async function readCodex( - file: string, - calls: readonly ApiCall[], - priceOf: (call: ApiCall) => number, - capture?: TextCapture, -) { - const builder = new ReplayBuilder(WEIGHTS["codex"]!, capture); - const take = callQueue(calls); - const metaByCall = new Map(); - let lineNumber = 0; - for await (const line of lines(file)) { - lineNumber += 1; - builder.line = lineNumber; - const type = line["type"]; - const payload = isObject(line["payload"]) ? line["payload"] : {}; - const inner = payload["type"]; - const timestamp = - typeof line["timestamp"] === "string" ? Date.parse(line["timestamp"]) : Number.NaN; - if (type === "token_usage_record" || (type === "event_msg" && inner === "token_count")) { - const call = take(timestamp); - if (call !== undefined) builder.apiCall(call, priceOf(call)); - else if (type === "token_usage_record") builder.inputOffset = 1; - continue; - } - if (type === "event_msg" && inner === "task_started") builder.markTurnStart(); - if (type !== "response_item") continue; - const response = builder.calls.length; - if (inner === "message") { - const { text, images } = contentText(payload["content"]); - if (payload["role"] === "assistant") { - builder.emission("assistant", text, text.length / 4, response); - } else if (payload["role"] === "user") { - builder.userMessage(text, images); - } - } else if (inner === "agent_message") { - const { text } = contentText(payload["content"]); - builder.userMessage(text); - } else if (inner === "reasoning") { - const encrypted = payload["encrypted_content"]; - const { text } = contentText(payload["summary"]); - builder.emission( - "reasoning", - text, - (typeof encrypted === "string" ? encrypted.length : 0) * 0.2, - response, - ); - } else if (inner === "custom_tool_call" || inner === "function_call") { - const raw = inner === "custom_tool_call" ? payload["input"] : payload["arguments"]; - const text = typeof raw === "string" ? raw : ""; - const name = typeof payload["name"] === "string" ? payload["name"] : ""; - const namespace = typeof payload["namespace"] === "string" ? payload["namespace"] : ""; - const command = classifyCommand(text); - const meta: ToolMeta = - namespace.startsWith("mcp") || name.startsWith("mcp__") - ? { ...command, toolClass: "mcp" } - : name === "spawn_agent" || name === "wait_agent" - ? { ...command, toolClass: "agent" } - : command; - if (typeof payload["call_id"] === "string") metaByCall.set(payload["call_id"], meta); - builder.emission("tool_call", text, text.length / 4, response); - } else if (inner === "custom_tool_call_output" || inner === "function_call_output") { - const callId = payload["call_id"]; - const { text, images } = contentText(payload["output"]); - builder.toolOutput( - text, - images, - (typeof callId === "string" ? metaByCall.get(callId) : undefined) ?? null, - ); - } - } - return builder; -} - -const CLAUDE_READ_TOOLS = new Set(["Read"]); -const CLAUDE_SEARCH_TOOLS = new Set(["Grep", "Glob", "LS", "ToolSearch"]); -const CLAUDE_EDIT_TOOLS = new Set(["Edit", "Write", "MultiEdit", "NotebookEdit"]); - -function classifyClaudeTool(name: string, input: Json): ToolMeta { - return { - ...claudeToolMeta(name, input), - label: `${name} ${JSON.stringify(input).slice(0, 380)}`, - }; -} - -function claudeToolMeta(name: string, input: Json): ToolMeta { - const key = hash(`${name}:${JSON.stringify(input)}`); - const path = typeof input["file_path"] === "string" ? input["file_path"] : null; - if (CLAUDE_READ_TOOLS.has(name)) { - return { toolClass: "read", commandKey: key, reads: path === null ? [] : [path], edits: [] }; - } - if (CLAUDE_EDIT_TOOLS.has(name)) { - return { toolClass: "edit", commandKey: key, reads: [], edits: path === null ? [] : [path] }; - } - if (CLAUDE_SEARCH_TOOLS.has(name)) { - return { toolClass: "search", commandKey: key, reads: [], edits: [] }; - } - if (name === "Bash" && typeof input["command"] === "string") { - return classifyCommand(input["command"]); - } - if (name === "Agent" || name === "Task") { - return { toolClass: "agent", commandKey: key, reads: [], edits: [] }; - } - return { - toolClass: name.startsWith("mcp__") ? "mcp" : "other", - commandKey: key, - reads: [], - edits: [], - }; -} - -export async function readClaude( - file: string, - calls: readonly ApiCall[], - priceOf: (call: ApiCall) => number, - capture?: TextCapture, -) { - const builder = new ReplayBuilder(WEIGHTS["claude"]!, capture); - const byKey = new Map(calls.map((call) => [call.dedupeKey, call])); - const metaByUse = new Map(); - for await (const line of lines(file)) { - const message = line["message"]; - if (!isObject(message)) continue; - const content = message["content"]; - if (line["type"] === "user") { - if (line["isMeta"] === true) continue; - if (typeof content === "string") { - builder.markTurnStart(); - builder.userMessage(content); - continue; - } - if (!Array.isArray(content)) continue; - for (const part of content) { - if (!isObject(part)) continue; - if (part["type"] === "tool_result") { - const { text, images } = contentText(part["content"]); - const useId = part["tool_use_id"]; - builder.toolOutput( - text, - images, - (typeof useId === "string" ? metaByUse.get(useId) : undefined) ?? null, - ); - } else if (part["type"] === "text" && typeof part["text"] === "string") { - builder.markTurnStart(); - builder.userMessage(part["text"]); - } - } - } else if (line["type"] === "assistant") { - const key = `${String(message["id"])}:${String(line["requestId"])}`; - const call = byKey.get(key); - if (call !== undefined) { - builder.apiCall(call, priceOf(call)); - byKey.delete(key); - } - if (!Array.isArray(content)) continue; - const response = Math.max(0, builder.calls.length - 1); - for (const part of content) { - if (!isObject(part)) continue; - if (part["type"] === "text" && typeof part["text"] === "string") { - builder.emission("assistant", part["text"], part["text"].length / 4, response); - } else if (part["type"] === "thinking") { - const thinking = typeof part["thinking"] === "string" ? part["thinking"] : ""; - const signature = typeof part["signature"] === "string" ? part["signature"] : ""; - builder.emission( - "reasoning", - thinking, - Math.max(thinking.length / 4, signature.length * 0.2), - response, - ); - } else if (part["type"] === "tool_use") { - const input = isObject(part["input"]) ? part["input"] : {}; - const name = typeof part["name"] === "string" ? part["name"] : ""; - if (typeof part["id"] === "string") { - metaByUse.set(part["id"], classifyClaudeTool(name, input)); - } - const text = JSON.stringify(input); - builder.emission("tool_call", text, text.length / 4, response); - } - } - } - } - return builder; -} - -if (import.meta.main) { - const dir = process.argv[2]; - if (dir === undefined) throw new Error("usage: replay.ts [days]"); - const days = Number(process.argv[3] ?? 30); - const { loadRates, price } = await import("./usageRecords.ts"); - const rates = loadRates(NodePath.join(dir, "usage-model-rates.json")); - const all = loadApiCalls(NodePath.join(dir, "usage-scan-cache-v11.json")); - const latest = Math.max(...all.map((call) => call.timestampMs)); - const cutoff = latest - days * 86_400_000; - const byFile = new Map(); - for (const call of all) { - const list = byFile.get(call.file) ?? []; - list.push(call); - byFile.set(call.file, list); - } - const outDir = NodePath.join(dir, "replay"); - NodeFS.mkdirSync(outDir, { recursive: true }); - let written = 0; - let matched = 0; - let expected = 0; - let missingFiles = 0; - for (const [file, calls] of byFile) { - if (!calls.some((call) => call.timestampMs >= cutoff)) continue; - const provider = calls[0]!.provider; - if (provider !== "codex" && provider !== "claude") continue; - expected += calls.length; - const priceOf = (call: ApiCall) => price(rates, call).costUsd; - let builder: ReplayBuilder; - try { - builder = - provider === "codex" - ? await readCodex(file, calls, priceOf) - : await readClaude(file, calls, priceOf); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; - missingFiles += 1; - continue; - } - matched += builder.calls.length; - const replay: Replay = { - file, - provider, - calls: builder.calls, - blocks: builder.blocks, - turnStarts: builder.turnStarts, - }; - NodeFS.writeFileSync(NodePath.join(outDir, `${hash(file)}.json`), JSON.stringify(replay)); - written += 1; - } - console.log( - `replays ${written}, missing transcripts ${missingFiles}, calls aligned ${matched} of ${expected} (${((matched / Math.max(expected, 1)) * 100).toFixed(1)}%)`, - ); -} diff --git a/evals/cache-snip/shape.ts b/evals/cache-snip/shape.ts deleted file mode 100644 index 2fff66de4d752..0000000000000 --- a/evals/cache-snip/shape.ts +++ /dev/null @@ -1,151 +0,0 @@ -// @effect-diagnostics nodeBuiltinImport:off globalConsole:off -// How prompts behave over a session: size, cold misses, and what list price really is above the long-context cliff. -// usage: node evals/cache-snip/shape.ts [days] -import * as NodeFS from "node:fs"; -import * as NodePath from "node:path"; - -import { lookupRate } from "../../apps/server/src/usage/usagePricing.ts"; -import { loadApiCalls, loadRates, price, promptTokens, type ApiCall } from "./usageRecords.ts"; - -const dir = process.argv[2]; -if (dir === undefined) throw new Error("usage: shape.ts [days]"); -const days = Number(process.argv[3] ?? 30); -const rates = loadRates(NodePath.join(dir, "usage-model-rates.json")); -const document = ( - JSON.parse(NodeFS.readFileSync(NodePath.join(dir, "usage-model-rates.json"), "utf8")) as { - document: Record>; - } -).document; -const all = loadApiCalls(NodePath.join(dir, "usage-scan-cache-v11.json")); -const latest = Math.max(...all.map((call) => call.timestampMs)); -const calls = all.filter((call) => call.timestampMs >= latest - days * 86_400_000); - -const LONG_CONTEXT_LIMIT = 272_000; - -/** List price including the long-context tier T3's table leaves out, per OpenAI's pricing page. */ -function listUsd(call: ApiCall): { billed: number; t3: number } { - const t3 = price(rates, call).costUsd; - const entry = document[call.model]; - const long = - entry !== undefined && - promptTokens(call) > LONG_CONTEXT_LIMIT && - entry["input_cost_per_token_above_272k_tokens"] !== undefined; - if (!long) return { billed: t3, t3 }; - const rate = lookupRate(rates, call.model); - if (rate === null) return { billed: t3, t3 }; - const input = entry["input_cost_per_token_above_272k_tokens"] ?? rate.inputCostPerToken; - const read = entry["cache_read_input_token_cost_above_272k_tokens"] ?? rate.cacheReadCostPerToken; - const output = entry["output_cost_per_token_above_272k_tokens"] ?? rate.outputCostPerToken; - return { - billed: call.uncached * input + call.cacheRead * read + call.output * output, - t3, - }; -} - -const bucketEdges = [50_000, 100_000, 150_000, 200_000, 272_000, 400_000, 600_000, Infinity]; -const bucketName = (edge: number) => (edge === Infinity ? ">600k" : `<=${edge / 1000}k`); - -for (const provider of ["codex", "claude"]) { - const subset = calls.filter((call) => call.provider === provider); - const rows = bucketEdges.map(() => ({ calls: 0, t3: 0, billed: 0, read: 0 })); - for (const call of subset) { - const index = bucketEdges.findIndex((edge) => promptTokens(call) <= edge); - const row = rows[index]!; - const { billed, t3 } = listUsd(call); - row.calls += 1; - row.t3 += t3; - row.billed += billed; - row.read += price(rates, call).categoryCostUsd?.cacheRead ?? 0; - } - const t3Total = rows.reduce((sum, row) => sum + row.t3, 0); - const billedTotal = rows.reduce((sum, row) => sum + row.billed, 0); - console.log(`\n== ${provider}, last ${days} days, by prompt size`); - console.log("bucket calls T3$ list$ %ofT3$ read$"); - rows.forEach((row, index) => { - console.log( - `${bucketName(bucketEdges[index]!).padEnd(9)} ${String(row.calls).padStart(6)} ${row.t3.toFixed(0).padStart(6)} ${row.billed.toFixed(0).padStart(7)} ${((row.t3 / Math.max(t3Total, 1e-9)) * 100).toFixed(1).padStart(6)}% ${row.read.toFixed(0).padStart(6)}`, - ); - }); - console.log( - `T3 estimate $${t3Total.toFixed(0)}, with long-context tier $${billedTotal.toFixed(0)} (+${(((billedTotal - t3Total) / t3Total) * 100).toFixed(1)}%)`, - ); -} - -interface Session { - readonly key: string; - readonly provider: string; - readonly calls: readonly ApiCall[]; -} -const sessionMap = new Map(); -for (const call of calls) { - const list = sessionMap.get(call.file) ?? []; - list.push(call); - sessionMap.set(call.file, list); -} -const sessions: Session[] = [...sessionMap.entries()].map(([key, list]) => ({ - key, - provider: list[0]!.provider, - calls: list.toSorted((a, b) => a.timestampMs - b.timestampMs), -})); - -const COLD_MIN_PROMPT = 10_000; -const COLD_READ_SHARE = 0.5; -const quantile = (values: readonly number[], q: number) => { - if (values.length === 0) return 0; - const sorted = values.toSorted((a, b) => a - b); - return sorted[Math.min(sorted.length - 1, Math.floor(q * sorted.length))]!; -}; - -for (const provider of ["codex", "claude"]) { - const subset = sessions.filter((session) => session.provider === provider); - let cold = 0; - let total = 0; - let coldTokens = 0; - let coldAfterGap = 0; - let drops = 0; - const gapsBeforeCold: number[] = []; - const coldPrompts: number[] = []; - const gapBuckets = { "<5m": 0, "5-30m": 0, "30-60m": 0, ">60m": 0 }; - const allGaps = { "<5m": 0, "5-30m": 0, "30-60m": 0, ">60m": 0 }; - const key = (minutes: number) => - minutes < 5 ? "<5m" : minutes < 30 ? "5-30m" : minutes < 60 ? "30-60m" : ">60m"; - for (const session of subset) { - session.calls.forEach((call, index) => { - total += 1; - const prompt = promptTokens(call); - const previous = session.calls[index - 1]; - const gapMinutes = - previous === undefined ? 0 : (call.timestampMs - previous.timestampMs) / 60_000; - if (previous !== undefined) allGaps[key(gapMinutes)] += 1; - if (previous !== undefined && prompt < promptTokens(previous) * 0.6) drops += 1; - if (previous === undefined) return; - if (prompt >= COLD_MIN_PROMPT && call.cacheRead / prompt < COLD_READ_SHARE) { - cold += 1; - coldTokens += call.uncached + call.cacheWrite; - gapsBeforeCold.push(gapMinutes); - coldPrompts.push(prompt); - gapBuckets[key(gapMinutes)] += 1; - if (gapMinutes >= 5) coldAfterGap += 1; - } - }); - } - const callCounts = subset.map((session) => session.calls.length); - const usdBySession = subset - .map((session) => session.calls.reduce((sum, call) => sum + price(rates, call).costUsd, 0)) - .toSorted((a, b) => b - a); - const usdTotal = usdBySession.reduce((sum, value) => sum + value, 0); - const top = (n: number) => - usdBySession.slice(0, n).reduce((sum, value) => sum + value, 0) / Math.max(usdTotal, 1e-9); - console.log(`\n== ${provider}, session shape`); - console.log( - `sessions ${subset.length}, calls ${total}, calls/session p50 ${quantile(callCounts, 0.5)} p90 ${quantile(callCounts, 0.9)} max ${Math.max(0, ...callCounts)}`, - ); - console.log( - `$ concentration: top 10 sessions ${(top(10) * 100).toFixed(0)}%, top 20 ${(top(20) * 100).toFixed(0)}%, top 50 ${(top(50) * 100).toFixed(0)}%`, - ); - console.log( - `cold calls ${cold} (${((cold / Math.max(total, 1)) * 100).toFixed(2)}%), uncached+write tokens in them ${(coldTokens / 1e6).toFixed(1)}M, prompt p50 ${quantile(coldPrompts, 0.5)} p90 ${quantile(coldPrompts, 0.9)}`, - ); - console.log("gap before cold calls:", gapBuckets, " all gaps:", allGaps); - console.log(`prompt drops over 40% (compaction or reset): ${drops}`); -} diff --git a/evals/cache-snip/simulate.ts b/evals/cache-snip/simulate.ts deleted file mode 100644 index 9fed25674abc9..0000000000000 --- a/evals/cache-snip/simulate.ts +++ /dev/null @@ -1,943 +0,0 @@ -// @effect-diagnostics nodeBuiltinImport:off globalConsole:off -// Replays real sessions under snip policies and reports dollars against billed list price. -// usage: node evals/cache-snip/simulate.ts [days] -import * as NodeCrypto from "node:crypto"; -import * as NodeFS from "node:fs"; -import * as NodePath from "node:path"; - -import { lookupRate } from "../../apps/server/src/usage/usagePricing.ts"; -import type { Block, Call, Replay } from "./replay.ts"; -import { loadRates } from "./usageRecords.ts"; - -interface CallRates { - readonly read: number; - readonly miss: number; - readonly output: number; - /** Multiplies input and output above the long-context limit. Null when the model has no such tier. */ - readonly longInput: number | null; - readonly longOutput: number | null; -} - -const LONG_CONTEXT_LIMIT = 272_000; -const GRACE_CALLS = 5; -const REFETCH_CALL_OUTPUT_TOKENS = 100; -// A fresh chat re-reads for a while. Without a gap, that growth alone would retrigger the cut. -const HEAD_CUT_COOLDOWN_CALLS = 20; -const USER_SEED_TOKENS = 5_000; - -export interface Epoch { - readonly calls: readonly Call[]; - readonly blocks: readonly Block[]; - readonly session: string; - readonly firstIndex: number; - readonly turnStarts: ReadonlySet; - /** The call index each call has in the original replay. Blocks use these. */ - readonly rates: readonly CallRates[]; -} - -export type RateMode = "t3Table" | "trueList" | "apiWrites"; -export type { CallRates }; - -export function makeRateLookup(dir: string, mode: RateMode) { - const table = loadRates(NodePath.join(dir, "usage-model-rates.json")); - const document = ( - JSON.parse(NodeFS.readFileSync(NodePath.join(dir, "usage-model-rates.json"), "utf8")) as { - document: Record>; - } - ).document; - const cache = new Map(); - return (model: string, subagent: boolean): CallRates => { - const key = `${model}:${subagent}`; - const held = cache.get(key); - if (held !== undefined) return held; - const rate = lookupRate(table, model); - const entry = document[model]; - const provider = entry?.["litellm_provider"] as unknown as string | undefined; - const anthropic = provider === "anthropic"; - const oneHourWrite = entry?.["cache_creation_input_token_cost_above_1hr"]; - const longKnown = entry?.["input_cost_per_token_above_272k_tokens"] !== undefined; - const result: CallRates = - rate === null - ? { read: 0, miss: 0, output: 0, longInput: null, longOutput: null } - : { - read: rate.cacheReadCostPerToken, - miss: anthropic - ? // Main Claude sessions write 1h entries; T3's table prices them all at the 5m rate. - mode !== "t3Table" && !subagent && oneHourWrite !== undefined - ? oneHourWrite - : rate.cacheCreationCostPerToken - : // Codex rollouts report no writes. API billing on GPT-5.6+ charges 1.25x input for them. - mode === "apiWrites" - ? rate.cacheCreationCostPerToken - : rate.inputCostPerToken, - output: rate.outputCostPerToken, - longInput: mode !== "t3Table" && longKnown ? 2 : null, - longOutput: mode !== "t3Table" && longKnown ? 1.5 : null, - }; - cache.set(key, result); - return result; - }; -} - -export const sessionKey = (file: string) => - NodeCrypto.createHash("sha1").update(file).digest("hex").slice(0, 12); - -export function splitEpochs( - replay: Replay, - ratesFor: (model: string, subagent: boolean) => CallRates, -): Epoch[] { - const subagent = replay.file.includes("/subagents/"); - const turnStarts = new Set(replay.turnStarts); - const session = sessionKey(replay.file); - const starts = replay.calls.flatMap((call, index) => (index === 0 || call.reset ? [index] : [])); - return starts.map((start, position) => { - const stop = starts[position + 1] ?? replay.calls.length; - const calls = replay.calls.slice(start, stop); - return { - calls, - session, - firstIndex: start, - turnStarts, - rates: calls.map((call) => ratesFor(call.model, subagent)), - blocks: replay.blocks.filter((block) => block.arrival >= start && block.arrival < stop), - }; - }); -} - -interface Tally { - usd: number; - snips: number; - snippedTokens: number; - refetches: number; - refetchTokens: number; - wrongSnips: number; - seedTokens: number; -} -const emptyTally = (): Tally => ({ - usd: 0, - snips: 0, - snippedTokens: 0, - refetches: 0, - refetchTokens: 0, - wrongSnips: 0, - seedTokens: 0, -}); - -/** What a policy may look at before call `t`. Foresight lives only in selectors named oracle. */ -export interface View { - readonly epoch: Epoch; - readonly t: number; - readonly live: readonly Block[]; - readonly coldHint: boolean; - /** A user message entered the prompt at this call: the natural boundary between tasks. */ - readonly userTurn: boolean; - readonly prompt: number; - readonly removedBefore: (arrival: number) => number; -} - -export type Decide = (view: View) => readonly Block[]; - -export const LONG_LIMIT = LONG_CONTEXT_LIMIT; -const nextUse = (block: Block, after: number) => block.uses.find((use) => use > after) ?? null; - -export interface RunOptions { - readonly compactAt?: number; - readonly summaryTokens?: number; - readonly coldHintMinutes?: number; - /** Share of wrong snips that actually make the agent fetch the content again. */ - readonly refetchRate?: number; - /** Share of cuts whose kept prefix still misses, as when a fork lands on a cold cache. */ - readonly cutMissRate?: number; - /** New chat seeded with the newest tokens only. No summary call, and the first request is fully uncached. */ - readonly headCut?: { - readonly trigger: number; - readonly keepTokens: number; - readonly floor: number; - readonly when: "userTurn" | "cold"; - /** Newest tokens always kept. The rest of `keepTokens` is the head budget, filled by `rank`. */ - readonly keepRecent?: number; - /** Drops tool output from the newest window, so the seed can end up smaller than `keepRecent`. */ - readonly seedFilter?: (block: Block, t: number, epoch: Epoch) => boolean; - /** Higher is kept first. Defaults to newest first, as T3's handoff does. */ - readonly rank?: (block: Block, t: number, epoch: Epoch) => number; - }; - /** What a fresh chat re-reads because it lost context: added tokens and extra calls per head cut. */ - readonly reexplore?: { readonly tokens: number; readonly calls: number }; - /** Compact at exactly these calls. Used to price each candidate moment. */ - readonly compactCalls?: ReadonlySet; - /** Compact only at this moment, instead of whenever the prompt is over the limit. */ - readonly compactWhen?: "cold" | "userTurn" | "coldOrUserTurn"; -} - -/** One epoch under a policy. `compactAt` replaces everything but the base with a summary. */ -export function runEpoch( - epoch: Epoch, - decide: Decide, - cutoff: number, - options: RunOptions = {}, -): Tally { - const tally = emptyTally(); - const { calls, blocks, firstIndex, rates } = epoch; - const byArrival = new Map(); - for (const block of blocks) { - const list = byArrival.get(block.arrival) ?? []; - list.push(block); - byArrival.set(block.arrival, list); - } - const base = - calls[0]!.prompt - - blocks.reduce((sum, block) => sum + (block.arrival === firstIndex ? block.tokens : 0), 0); - const removedAt = new Map(); - const pendingRefetch = new Map(); - let live: Block[] = []; - let removedTokens = 0; - let tail = 0; - // After a compaction the prompt is floor + what the real session appended since `anchor`. - let compaction: { floor: number; anchorPrompt: number } | null = null; - let lastCut = Number.NEGATIVE_INFINITY; - - const removedBefore = (arrival: number) => { - let total = 0; - for (const block of removedAt.keys()) if (block.arrival < arrival) total += block.tokens; - return total; - }; - const promptOf = (call: Call) => - compaction === null - ? call.prompt - removedTokens + tail - : compaction.floor + Math.max(0, call.prompt - compaction.anchorPrompt) + tail; - const refetchRate = options.refetchRate ?? 1; - const scheduleRefetch = (block: Block, t: number) => { - const use = nextUse(block, t - 1); - if (use === null) return; - tally.wrongSnips += 1; - // Not every later mention needs the bytes back. Deterministic so policies compare fairly. - if (((block.arrival * 2654435761 + block.tokens) >>> 0) / 4294967296 >= refetchRate) return; - const due = Math.max(use, t + 1); - if (due - firstIndex >= calls.length) return; - const list = pendingRefetch.get(due) ?? []; - list.push(block); - pendingRefetch.set(due, list); - }; - - const coldMinutes = options.coldHintMinutes ?? 60; - calls.forEach((call, offset) => { - const t = firstIndex + offset; - const callRates = rates[offset]!; - const previous = calls[offset - 1]; - live.push(...(byArrival.get(t) ?? [])); - - const tierAt = (tokens: number) => - callRates.longInput !== null && tokens > LONG_CONTEXT_LIMIT - ? { input: callRates.longInput, output: callRates.longOutput ?? 1 } - : { input: 1, output: 1 }; - - let extraUsd = 0; - let tailFresh = 0; - for (const block of pendingRefetch.get(t) ?? []) { - if (!removedAt.has(block)) continue; - removedAt.delete(block); - if (compaction === null) removedTokens -= block.tokens; - // Refetched content sits at the tail, so snipping it again only busts the tail. - const refetched: Block = { ...block, arrival: t }; - const roundTrip = promptOf(call); - const tier = tierAt(roundTrip); - extraUsd += - Math.max(0, roundTrip - call.miss) * callRates.read * tier.input + - REFETCH_CALL_OUTPUT_TOKENS * callRates.output * tier.output; - tail += block.tokens; - tailFresh += block.tokens; - live.push(refetched); - tally.refetches += 1; - tally.refetchTokens += block.tokens; - } - pendingRefetch.delete(t); - - const gapMinutes = previous === undefined ? 0 : (call.ts - previous.ts) / 60_000; - const toRemove = decide({ - epoch, - t, - live, - coldHint: gapMinutes >= coldMinutes, - userTurn: epoch.turnStarts.has(t), - prompt: promptOf(call), - removedBefore, - }); - - let prefixLimit = Number.POSITIVE_INFINITY; - if (toRemove.length > 0) { - const firstArrival = Math.min(...toRemove.map((block) => block.arrival)); - const offsetBefore = firstArrival - firstIndex - 1; - const before = offsetBefore >= 0 ? calls[offsetBefore]!.prompt : base; - prefixLimit = Math.max(0, before - removedBefore(firstArrival)); - if (((t * 2654435761 + call.prompt) >>> 0) / 4294967296 < (options.cutMissRate ?? 0)) { - prefixLimit = 0; - } - const removing = new Set(toRemove); - live = live.filter((block) => !removing.has(block)); - tally.snips += 1; - for (const block of toRemove) { - removedAt.set(block, t); - removedTokens += block.tokens; - tally.snippedTokens += block.tokens; - scheduleRefetch(block, t); - } - } - - const moment = options.compactWhen; - const userTurn = epoch.turnStarts.has(t); - const cold = gapMinutes >= coldMinutes; - const momentOk = - moment === undefined || - (moment === "cold" && cold) || - (moment === "userTurn" && userTurn) || - (moment === "coldOrUserTurn" && (cold || userTurn)); - if ( - options.compactCalls?.has(t) === true || - (options.compactAt !== undefined && momentOk && promptOf(call) >= options.compactAt) - ) { - // Codex's prompt floor rises from 33k on a fresh session to 67k after compaction, so 30k survives. - const summaryTokens = options.summaryTokens ?? 30_000; - const beforeCompaction = promptOf(call); - const tier = tierAt(beforeCompaction); - extraUsd += - Math.max(0, beforeCompaction - call.miss) * callRates.read * tier.input + - summaryTokens * callRates.output * tier.output; - for (const block of live) { - removedAt.set(block, t); - tally.snippedTokens += block.tokens; - scheduleRefetch(block, t); - } - live = []; - tail = 0; - tailFresh = 0; - tally.snips += 1; - compaction = { floor: base + summaryTokens, anchorPrompt: call.prompt }; - prefixLimit = base; - } - - const cut = options.headCut; - if ( - cut !== undefined && - promptOf(call) >= cut.trigger && - t - lastCut >= HEAD_CUT_COOLDOWN_CALLS && - (cut.when === "userTurn" ? userTurn : cold) - ) { - lastCut = t; - const rank = cut.rank ?? ((block: Block) => block.arrival); - const seedable = live.filter( - (block) => block.kind !== "reasoning" && block.kind !== "tool_call", - ); - const keep = new Set(); - let kept = 0; - const take = (block: Block, limit: number) => { - if (keep.has(block) || kept + block.tokens > limit) return; - keep.add(block); - kept += block.tokens; - }; - // Instructions and the latest answer survive whatever the ranking says, as in T3's handoff. - const newestFirst = seedable.toReversed(); - for (const block of newestFirst) if (block.kind === "user") take(block, USER_SEED_TOKENS); - const latestAnswer = newestFirst.find((block) => block.kind === "assistant"); - if (latestAnswer !== undefined) take(latestAnswer, cut.keepTokens); - const recent = cut.keepRecent ?? 0; - let walked = 0; - for (const block of newestFirst) { - if ((cut.seedFilter === undefined ? kept : walked) >= recent) break; - walked += block.tokens; - if (block.kind === "tool_output" && cut.seedFilter?.(block, t, epoch) === false) continue; - take(block, recent); - } - // A filtered seed is meant to end up small, so nothing tops it back up. - const head = - cut.seedFilter !== undefined - ? [] - : seedable.filter( - (block) => block.kind === "tool_output" || cut.keepRecent === undefined, - ); - for (const block of head.toSorted((a, b) => rank(b, t, epoch) - rank(a, t, epoch))) { - take(block, cut.keepTokens); - } - for (const block of live) { - if (keep.has(block)) continue; - removedAt.set(block, t); - tally.snippedTokens += block.tokens; - scheduleRefetch(block, t); - } - live = live.filter((block) => keep.has(block)); - const reexplore = options.reexplore ?? { tokens: 0, calls: 0 }; - tail = reexplore.tokens; - tailFresh = reexplore.tokens; - tally.snips += 1; - tally.seedTokens += kept; - compaction = { floor: cut.floor + kept, anchorPrompt: call.prompt }; - prefixLimit = 0; - const fresh = cut.floor + kept + reexplore.tokens; - const freshTier = tierAt(fresh); - extraUsd += - reexplore.calls * - (fresh * callRates.read * freshTier.input + - REFETCH_CALL_OUTPUT_TOKENS * callRates.output * freshTier.output); - } - - const prompt = promptOf(call); - const tier = tierAt(prompt); - const read = Math.min(Math.max(0, prompt - call.miss - tailFresh), prefixLimit); - const usd = - read * callRates.read * tier.input + - (prompt - read) * callRates.miss * tier.input + - call.output * callRates.output * tier.output + - extraUsd; - if (call.ts >= cutoff) tally.usd += usd; - }); - return tally; -} - -const never: Decide = () => []; - -const isOutput = (block: Block) => block.kind === "tool_output"; - -/** Foresight: the block is never quoted again in this epoch. */ -const deadOracle = (view: View) => - view.live.filter((block) => isOutput(block) && block.lastUse + GRACE_CALLS < view.t); - -const supersededOnly = (view: View) => - view.live.filter( - (block) => - isOutput(block) && - ((block.supersededAt !== null && block.supersededAt <= view.t) || - (block.rereadAt !== null && block.rereadAt <= view.t)), - ); - -/** Least recently used: not quoted for `idle` calls. Uses only what has already happened. */ -const idleFor = (idle: number, minTokens: number) => (view: View) => - view.live.filter((block) => { - if (!isOutput(block) || block.tokens < minTokens) return false; - let last = block.arrival; - for (const use of block.uses) { - if (use >= view.t) break; - last = use; - } - return view.t - last > idle; - }); - -/** Dead outputs plus old tool inputs and reasoning, for the widest ceiling. */ -const deadPlusOld = (view: View) => - view.live.filter( - (block) => - (isOutput(block) && block.lastUse + GRACE_CALLS < view.t) || - ((block.kind === "tool_call" || block.kind === "reasoning") && view.t - block.arrival > 50), - ); - -/** Stand-in for a classifier of given AUC, thresholded at a false positive rate. Noise is fixed per block. */ -const noisyClassifier = (auc: number, falsePositiveRate: number) => { - const separation = Math.SQRT2 * inverseNormal(auc); - const threshold = inverseNormal(1 - falsePositiveRate); - return (view: View) => - view.live.filter((block) => { - if (!isOutput(block)) return false; - const dead = block.lastUse + GRACE_CALLS < view.t; - const noise = blockNoise(block); - return (dead ? separation : 0) + noise > threshold; - }); -}; - -function inverseNormal(p: number) { - // Acklam's rational approximation, central region only: the AUCs and rates used here stay inside it. - const q = p - 0.5; - const r = q * q; - return ( - (q * - (((((-3.969683028665376e1 * r + 2.209460984245205e2) * r - 2.759285104469687e2) * r + - 1.38357751867269e2) * - r - - 3.066479806614716e1) * - r + - 2.506628277459239)) / - (((((-5.447609879822406e1 * r + 1.615858368580409e2) * r - 1.556989798598866e2) * r + - 6.680131188771972e1) * - r - - 1.328068155288572e1) * - r + - 1) - ); -} - -function blockNoise(block: Block) { - const seed = (block.arrival * 2654435761 + block.tokens * 40503) >>> 0; - const u1 = ((seed * 1664525 + 1013904223) >>> 0) / 4294967296 || 1e-9; - const u2 = (((seed ^ 0x9e3779b9) * 22695477 + 1) >>> 0) / 4294967296; - return Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2); -} - -/** Tool output from before the user's last message. The task boundary rule. */ -const beforeLastUserTurn = (minTokens: number) => (view: View) => { - let lastUser = -1; - for (const block of view.live) - if (block.kind === "user") lastUser = Math.max(lastUser, block.arrival); - return view.live.filter( - (block) => isOutput(block) && block.tokens >= minTokens && block.arrival < lastUser, - ); -}; - -const atUserTurn = - (decide: Decide): Decide => - (view) => - view.userTurn ? decide(view) : []; - -/** - * Rewind: drop the finished turn's tool work and keep what the user saw. Everything before that turn is untouched, - * so it stays cached and only the kept answer and the new message are rebuilt. - */ -const finishedTurnWork = (minTokens: number) => (view: View) => { - const users = view.live.filter((block) => block.kind === "user" && block.arrival < view.t); - const start = users.at(-1)?.arrival; - if (start === undefined) return []; - const chosen = view.live.filter( - (block) => - block.arrival >= start && - block.arrival < view.t && - (block.kind === "tool_output" || block.kind === "tool_call" || block.kind === "reasoning"), - ); - return chosen.reduce((sum, block) => sum + block.tokens, 0) >= minTokens ? chosen : []; -}; - -/** Snip only when the gap says the cache is already cold. */ -const whenCold = - (select: (view: View) => readonly Block[], minTokens: number): Decide => - (view) => { - if (!view.coldHint) return []; - const chosen = select(view); - return chosen.reduce((sum, block) => sum + block.tokens, 0) >= minTokens ? chosen : []; - }; - -/** Claude's clear_tool_uses semantics: oldest first, keep N, skip unless enough clears. */ -const keepRecent = - (trigger: number, keep: number, clearAtLeast: number): Decide => - (view) => { - if (view.prompt < trigger) return []; - const outputs = view.live.filter(isOutput); - const candidates = outputs.slice(0, Math.max(0, outputs.length - keep)); - return candidates.reduce((sum, block) => sum + block.tokens, 0) >= clearAtLeast - ? candidates - : []; - }; - -/** - * Estimates remaining calls from room left before the provider compacts, and charges the rebuild honestly. - * Counts the long-context cliff, where a snip that lands under the limit halves every later read. - */ -const whenRoomPays = - ( - select: (view: View) => readonly Block[], - compactLimit: number, - margin: number, - minShare: number, - ): Decide => - (view) => { - const chosen = select(view); - if (chosen.length === 0) return []; - const snipped = chosen.reduce((sum, block) => sum + block.tokens, 0); - if (snipped < minShare * view.prompt) return []; - const offset = view.t - view.epoch.firstIndex; - if (offset < 5) return []; - const rates = view.epoch.rates[offset]!; - const start = view.epoch.calls[0]!.prompt; - const growth = Math.max(200, (view.epoch.calls[offset]!.prompt - start) / offset); - const room = Math.max(0, compactLimit - view.prompt) / growth; - const remaining = Math.min(room, offset); - const firstArrival = Math.min(...chosen.map((block) => block.arrival)); - const offsetBefore = firstArrival - view.epoch.firstIndex - 1; - const before = offsetBefore >= 0 ? view.epoch.calls[offsetBefore]!.prompt : 0; - const kept = Math.max(0, before - view.removedBefore(firstArrival)); - const after = view.prompt - snipped; - const tier = (tokens: number) => - rates.longInput !== null && tokens > LONG_CONTEXT_LIMIT ? rates.longInput : 1; - const rebuild = - Math.max(0, after - kept) * rates.miss * tier(after) + - kept * rates.read * tier(after) - - view.prompt * rates.read * tier(view.prompt); - const perCall = view.prompt * rates.read * tier(view.prompt) - after * rates.read * tier(after); - return remaining * perCall > rebuild * margin ? chosen : []; - }; - -/** Foresight on both relevance and remaining calls: best snip times found by repeated best-insert. */ -function oracleDecide( - epoch: Epoch, - cutoff: number, - select: (view: View) => readonly Block[], -): Decide { - const count = epoch.calls.length; - if (count < 3) return never; - const step = Math.max(1, Math.floor(count / 60)); - const candidates: number[] = []; - for (let offset = step; offset < count; offset += step) - candidates.push(epoch.firstIndex + offset); - const planned = new Set(); - const planDecide: Decide = (view) => (planned.has(view.t) ? select(view) : []); - let best = runEpoch(epoch, planDecide, cutoff).usd; - for (let round = 0; round < 8; round += 1) { - let bestTime: number | null = null; - for (const time of candidates) { - if (planned.has(time)) continue; - planned.add(time); - const usd = runEpoch(epoch, planDecide, cutoff).usd; - planned.delete(time); - if (usd < best - 1e-9) { - best = usd; - bestTime = time; - } - } - if (bestTime === null) break; - planned.add(bestTime); - } - return planDecide; -} - -interface PolicyDefinition { - readonly name: string; - readonly make: (epoch: Epoch, cutoff: number) => Decide; - readonly options?: RunOptions; -} - -const stateless = ( - name: string, - decide: Decide, - options?: PolicyDefinition["options"], -): PolicyDefinition => ({ - name, - make: () => decide, - ...(options === undefined ? {} : { options }), -}); - -/** Out-of-fold scores from relevance.py, keyed session:t:block. */ -let relevanceScores: ReadonlyMap = new Map(); - -const lastSeenUse = (block: Block, t: number) => { - let last = block.arrival; - for (const use of block.uses) { - if (use >= t) break; - last = use; - } - return last; -}; - -const SELECTORS: ReadonlyArray< - readonly [string, NonNullable["rank"]>] -> = [ - ["newest first", (block) => block.arrival], - ["least recently used", (block, t) => lastSeenUse(block, t)], - [ - "named by the new message, then LRU", - (block, t) => (block.mentions.includes(t) ? 1e9 : 0) + lastSeenUse(block, t), - ], - [ - "trained on numeric features", - (block, t, epoch) => - block.kind === "tool_output" - ? (relevanceScores.get(`${epoch.session}:${t}:${block.id}`) ?? 0) - : 1, - ], - ...[0.75, 0.85, 0.95].map( - (auc) => - [ - `stand-in AUC ${auc}`, - (block: Block, t: number) => - (nextUse(block, t - 1) !== null ? Math.SQRT2 * inverseNormal(auc) : 0) + - blockNoise(block), - ] as const, - ), - ["oracle", (block, t) => (nextUse(block, t - 1) !== null ? 1e9 : 0) + block.arrival], -]; -const idleCalls = (block: Block, t: number) => t - lastSeenUse(block, t); -const standInFilter = (auc: number, falsePositiveRate: number) => { - const separation = Math.SQRT2 * inverseNormal(auc); - const threshold = inverseNormal(1 - falsePositiveRate); - return (block: Block, t: number) => - // Here "positive" means still needed, so the rate is how often an unneeded block slips through. - (nextUse(block, t - 1) !== null ? separation : 0) + blockNoise(block) > threshold; -}; -const SEED_FILTERS: ReadonlyArray< - readonly [string, (block: Block, t: number, epoch: Epoch) => boolean] -> = [ - ["no tool output at all", () => false], - ["used in the last 5 calls", (block, t) => idleCalls(block, t) <= 5], - ["used in the last 20 calls", (block, t) => idleCalls(block, t) <= 20], - [ - "named by the message, or used in the last 5", - (block, t) => block.mentions.includes(t) || idleCalls(block, t) <= 5, - ], - [ - "numeric model, p over 0.3", - (block, t, epoch) => (relevanceScores.get(`${epoch.session}:${t}:${block.id}`) ?? 0) > 0.3, - ], - ["stand-in AUC 0.85, lets 30% of unneeded through", standInFilter(0.85, 0.3)], - ["stand-in AUC 0.95, lets 30% of unneeded through", standInFilter(0.95, 0.3)], - ["oracle: only blocks used again", (block, t) => nextUse(block, t - 1) !== null], -]; -const PRUNE_CONFIGS = [ - { trigger: 150_000, keepRecent: 20_000, keepTokens: 20_000, when: "userTurn" }, - { trigger: 150_000, keepRecent: 50_000, keepTokens: 50_000, when: "userTurn" }, - { trigger: 100_000, keepRecent: 20_000, keepTokens: 20_000, when: "cold" }, -] as const; -const SELECTION_CONFIGS = [ - { trigger: 150_000, keepRecent: 20_000, keepTokens: 20_000, when: "userTurn" }, - { trigger: 150_000, keepRecent: 20_000, keepTokens: 30_000, when: "userTurn" }, - { trigger: 150_000, keepRecent: 20_000, keepTokens: 50_000, when: "userTurn" }, - { trigger: 150_000, keepRecent: 20_000, keepTokens: 80_000, when: "userTurn" }, - { trigger: 260_000, keepRecent: 20_000, keepTokens: 50_000, when: "userTurn" }, - { trigger: 100_000, keepRecent: 20_000, keepTokens: 50_000, when: "cold" }, -] as const; - -const REFETCH = Number(process.env["CACHE_SNIP_REFETCH"] ?? 1); -const CUT_MISS = Number(process.env["CACHE_SNIP_CUT_MISS"] ?? 0); -const [REEXPLORE_TOKENS, REEXPLORE_CALLS] = (process.env["CACHE_SNIP_REEXPLORE"] ?? "0,0") - .split(",") - .map(Number); -const LIMITS: Readonly> = { codex: 318_000, claude: 700_000 }; - -export const policiesFor = (provider: string): readonly PolicyDefinition[] => { - const limit = LIMITS[provider] ?? 300_000; - // Median first prompt of a fresh session: system, tools, and instructions. - const floor = provider === "claude" ? 49_000 : 33_000; - const cutAt = (trigger: number, keepTokens: number, when: "userTurn" | "cold") => - rule(`head cut over ${trigger / 1000}k keep ${keepTokens / 1000}k, ${when}`, never, { - headCut: { trigger, keepTokens, floor, when }, - reexplore: { tokens: REEXPLORE_TOKENS ?? 0, calls: REEXPLORE_CALLS ?? 0 }, - }); - const withRefetch = { refetchRate: REFETCH, cutMissRate: CUT_MISS }; - const rule = (name: string, decide: Decide, options: RunOptions = {}) => - stateless(name, decide, { ...withRefetch, ...options }); - return [ - rule("never", never), - { - name: "CEILING dead+old + oracle timing", - make: (epoch, cutoff) => oracleDecide(epoch, cutoff, deadPlusOld), - options: withRefetch, - }, - { - name: "CEILING dead outputs + oracle timing", - make: (epoch, cutoff) => oracleDecide(epoch, cutoff, deadOracle), - options: withRefetch, - }, - rule("dead + room rule m1 s10%", whenRoomPays(deadOracle, limit, 1, 0.1)), - rule("dead + room rule m2 s15%", whenRoomPays(deadOracle, limit, 2, 0.15)), - rule("dead + cold only", whenCold(deadOracle, 5_000)), - { - name: "idle>30 + oracle timing", - make: (epoch, cutoff) => oracleDecide(epoch, cutoff, idleFor(30, 200)), - options: withRefetch, - }, - rule("idle>10 + room rule m2 s15%", whenRoomPays(idleFor(10, 200), limit, 2, 0.15)), - rule("idle>30 + room rule m2 s15%", whenRoomPays(idleFor(30, 200), limit, 2, 0.15)), - rule("idle>60 + room rule m2 s15%", whenRoomPays(idleFor(60, 200), limit, 2, 0.15)), - rule("idle>30 + cold only", whenCold(idleFor(30, 200), 5_000)), - rule("idle>10 + cold only", whenCold(idleFor(10, 200), 5_000)), - rule("superseded + cold only", whenCold(supersededOnly, 2_000)), - ...[100_000, 150_000, 200_000, 260_000].flatMap((trigger) => - [5_000, 20_000, 50_000, 100_000].map((keep) => cutAt(trigger, keep, "userTurn")), - ), - cutAt(100_000, 20_000, "cold"), - cutAt(100_000, 50_000, "cold"), - ...PRUNE_CONFIGS.flatMap((config) => [ - rule( - `prune ${config.trigger / 1000}k/${config.keepRecent / 1000}k/${config.when}: keep all`, - never, - { - headCut: { ...config, floor }, - reexplore: { tokens: REEXPLORE_TOKENS ?? 0, calls: REEXPLORE_CALLS ?? 0 }, - }, - ), - ...SEED_FILTERS.map(([name, seedFilter]) => - rule( - `prune ${config.trigger / 1000}k/${config.keepRecent / 1000}k/${config.when}: ${name}`, - never, - { - headCut: { ...config, floor, seedFilter }, - reexplore: { tokens: REEXPLORE_TOKENS ?? 0, calls: REEXPLORE_CALLS ?? 0 }, - }, - ), - ), - ]), - ...SELECTION_CONFIGS.flatMap((config) => - SELECTORS.map(([name, rank]) => - rule( - `select ${config.trigger / 1000}k/head${(config.keepTokens - config.keepRecent) / 1000}k/${config.when}: ${name}`, - never, - { - headCut: { ...config, floor, rank }, - reexplore: { tokens: REEXPLORE_TOKENS ?? 0, calls: REEXPLORE_CALLS ?? 0 }, - }, - ), - ), - ), - rule("rewind finished turn, over 2k", atUserTurn(finishedTurnWork(2_000))), - rule("rewind finished turn, over 10k", atUserTurn(finishedTurnWork(10_000))), - rule("rewind finished turn, over 30k", atUserTurn(finishedTurnWork(30_000))), - rule("prior turns + cold only", whenCold(beforeLastUserTurn(200), 5_000)), - rule( - "prior turns at user turn, room m2 s15%", - atUserTurn(whenRoomPays(beforeLastUserTurn(200), limit, 2, 0.15)), - ), - rule( - "prior turns at user turn, room m1 s25%", - atUserTurn(whenRoomPays(beforeLastUserTurn(200), limit, 1, 0.25)), - ), - rule("AUC.75 fpr5% + room m2 s15%", whenRoomPays(noisyClassifier(0.75, 0.05), limit, 2, 0.15)), - rule("AUC.85 fpr5% + room m2 s15%", whenRoomPays(noisyClassifier(0.85, 0.05), limit, 2, 0.15)), - rule("AUC.95 fpr5% + room m2 s15%", whenRoomPays(noisyClassifier(0.95, 0.05), limit, 2, 0.15)), - rule("AUC.85 fpr10% + room m2 s15%", whenRoomPays(noisyClassifier(0.85, 0.1), limit, 2, 0.15)), - rule("AUC.95 fpr10% + room m2 s15%", whenRoomPays(noisyClassifier(0.95, 0.1), limit, 2, 0.15)), - rule("AUC.85 fpr30% + room m2 s15%", whenRoomPays(noisyClassifier(0.85, 0.3), limit, 2, 0.15)), - rule("AUC.95 fpr30% + room m2 s15%", whenRoomPays(noisyClassifier(0.95, 0.3), limit, 2, 0.15)), - rule("AUC.85 fpr5% + cold only", whenCold(noisyClassifier(0.85, 0.05), 5_000)), - rule("keep3 trigger100k no-min (docs default)", keepRecent(100_000, 3, 0)), - rule("keep30 trigger200k min75k", keepRecent(200_000, 30, 75_000)), - rule("keep60 trigger250k min100k", keepRecent(250_000, 60, 100_000)), - rule("keep30 trigger400k min150k", keepRecent(400_000, 30, 150_000)), - rule("compact at 200k", never, { compactAt: 200_000 }), - rule("compact at 260k", never, { compactAt: 260_000 }), - rule("compact at 400k", never, { compactAt: 400_000 }), - rule("compact when cold, over 100k", never, { compactAt: 100_000, compactWhen: "cold" }), - rule("compact when cold, over 150k", never, { compactAt: 150_000, compactWhen: "cold" }), - rule("compact at user turn, over 150k", never, { compactAt: 150_000, compactWhen: "userTurn" }), - rule("compact at user turn, over 200k", never, { compactAt: 200_000, compactWhen: "userTurn" }), - rule("compact at user turn, over 260k", never, { compactAt: 260_000, compactWhen: "userTurn" }), - ]; -}; - -if (import.meta.main) { - const dir = process.argv[2]; - if (dir === undefined) throw new Error("usage: simulate.ts [days]"); - const days = Number(process.argv[3] ?? 30); - const scoresPath = NodePath.join(dir, "relevance-scores.csv"); - if (NodeFS.existsSync(scoresPath)) { - relevanceScores = new Map( - NodeFS.readFileSync(scoresPath, "utf8") - .trim() - .split("\n") - .map((line) => { - const [key, value] = line.split(","); - return [key ?? "", Number(value)] as const; - }), - ); - } - const replays = NodeFS.readdirSync(NodePath.join(dir, "replay")).map( - (name) => JSON.parse(NodeFS.readFileSync(NodePath.join(dir, "replay", name), "utf8")) as Replay, - ); - const latest = Math.max(...replays.flatMap((replay) => replay.calls.map((call) => call.ts))); - const cutoff = latest - days * 86_400_000; - - const billed = new Map(); - for (const replay of replays) { - const inWindow = replay.calls.filter((call) => call.ts >= cutoff); - billed.set( - replay.provider, - (billed.get(replay.provider) ?? 0) + inWindow.reduce((sum, call) => sum + call.usd, 0), - ); - } - - // Gate: the no-snip replay at T3's rates must reproduce what T3 priced. - const t3Rates = makeRateLookup(dir, "t3Table"); - for (const provider of ["codex", "claude"]) { - let simulated = 0; - for (const replay of replays.filter((candidate) => candidate.provider === provider)) { - for (const epoch of splitEpochs(replay, t3Rates)) - simulated += runEpoch(epoch, never, cutoff).usd; - } - const target = billed.get(provider) ?? 0; - console.log( - `gate ${provider}: simulated no-snip $${simulated.toFixed(2)} vs T3-priced $${target.toFixed(2)} (${(((simulated - target) / target) * 100).toFixed(2)}%)`, - ); - } - - const ratesFor = makeRateLookup(dir, (process.env["CACHE_SNIP_RATES"] ?? "trueList") as RateMode); - const results: Record> = {}; - for (const provider of ["codex", "claude"]) { - const subset = replays.filter((candidate) => candidate.provider === provider); - const perSession = policiesFor(provider).map((policy) => { - const usd: number[] = []; - let snips = 0; - let snipped = 0; - let refetches = 0; - let refetchTokens = 0; - let wrong = 0; - let seed = 0; - for (const replay of subset) { - let sessionUsd = 0; - for (const epoch of splitEpochs(replay, ratesFor)) { - const tally = runEpoch(epoch, policy.make(epoch, cutoff), cutoff, policy.options ?? {}); - sessionUsd += tally.usd; - snips += tally.snips; - snipped += tally.snippedTokens; - refetches += tally.refetches; - refetchTokens += tally.refetchTokens; - wrong += tally.wrongSnips; - seed += tally.seedTokens; - } - usd.push(sessionUsd); - } - return { name: policy.name, usd, snips, snipped, refetches, refetchTokens, wrong, seed }; - }); - const baseline = perSession[0]!.usd; - const baselineTotal = baseline.reduce((sum, value) => sum + value, 0); - console.log( - `\n== ${provider}, true list baseline $${baselineTotal.toFixed(0)} over ${subset.length} sessions, last ${days} days`, - ); - console.log( - "policy".padEnd(44) + - " usd saved save% [95% CI] snips snipMtok refetch wrong refMtok", - ); - results[provider] = {}; - const random = seeded(7); - for (const row of perSession) { - const total = row.usd.reduce((sum, value) => sum + value, 0); - const samples: number[] = []; - for (let iteration = 0; iteration < 400; iteration += 1) { - let base = 0; - let policy = 0; - for (let draw = 0; draw < baseline.length; draw += 1) { - const pick = Math.floor(random() * baseline.length); - base += baseline[pick]!; - policy += row.usd[pick]!; - } - samples.push((base - policy) / Math.max(base, 1e-9)); - } - samples.sort((a, b) => a - b); - const low = samples[Math.floor(0.025 * samples.length)]!; - const high = samples[Math.floor(0.975 * samples.length)]!; - results[provider]![row.name] = [ - total, - low, - high, - row.refetchTokens, - row.snips, - row.snipped, - row.seed, - ]; - console.log( - row.name.padEnd(44) + - total.toFixed(0).padStart(6) + - (baselineTotal - total).toFixed(0).padStart(8) + - `${(((baselineTotal - total) / baselineTotal) * 100).toFixed(1)}%`.padStart(7) + - ` [${(low * 100).toFixed(1)},${(high * 100).toFixed(1)}]`.padStart(14) + - String(row.snips).padStart(7) + - (row.snipped / 1e6).toFixed(0).padStart(9) + - String(row.refetches).padStart(8) + - String(row.wrong).padStart(6) + - (row.refetchTokens / 1e6).toFixed(1).padStart(7), - ); - } - } - NodeFS.writeFileSync( - NodePath.join(dir, "simulate-results.json"), - JSON.stringify(results, null, 1), - ); -} - -function seeded(seed: number) { - let state = seed; - return () => { - state = (state * 1664525 + 1013904223) >>> 0; - return state / 4294967296; - }; -} diff --git a/evals/cache-snip/spend.ts b/evals/cache-snip/spend.ts deleted file mode 100644 index aa11724cd9fb2..0000000000000 --- a/evals/cache-snip/spend.ts +++ /dev/null @@ -1,119 +0,0 @@ -// @effect-diagnostics nodeBuiltinImport:off globalConsole:off globalDate:off -// Prices every API call in a copy of T3's usage scan cache with T3's own pricing code. -// usage: node evals/cache-snip/spend.ts -import * as NodePath from "node:path"; - -import { loadApiCalls, loadRates, price, type ApiCall } from "./usageRecords.ts"; - -const dir = process.argv[2]; -if (dir === undefined) throw new Error("usage: spend.ts "); -const calls = loadApiCalls(NodePath.join(dir, "usage-scan-cache-v11.json")); -const rates = loadRates(NodePath.join(dir, "usage-model-rates.json")); - -interface Row { - calls: number; - usd: number; - input: number; - cacheRead: number; - cacheWrite: number; - output: number; - unsplitUsd: number; - tUncached: number; - tRead: number; - tWrite: number; - tOutput: number; -} -const emptyRow = (): Row => ({ - calls: 0, - usd: 0, - input: 0, - cacheRead: 0, - cacheWrite: 0, - output: 0, - unsplitUsd: 0, - tUncached: 0, - tRead: 0, - tWrite: 0, - tOutput: 0, -}); - -function tally(rows: Map, key: string, call: ApiCall) { - const row = rows.get(key) ?? emptyRow(); - const priced = price(rates, call); - row.calls += 1; - row.usd += priced.costUsd; - if (priced.categoryCostUsd === null) row.unsplitUsd += priced.costUsd; - else { - row.input += priced.categoryCostUsd.input; - row.cacheRead += priced.categoryCostUsd.cacheRead; - row.cacheWrite += priced.categoryCostUsd.cacheWrite; - row.output += priced.categoryCostUsd.output; - } - row.tUncached += call.uncached; - row.tRead += call.cacheRead; - row.tWrite += call.cacheWrite; - row.tOutput += call.output; - rows.set(key, row); -} - -const pct = (part: number, whole: number) => - whole > 0 ? `${((part / whole) * 100).toFixed(1).padStart(5)}%` : " -"; -const usd = (value: number) => `$${value.toFixed(0)}`.padStart(8); -const mtok = (value: number) => `${(value / 1e6).toFixed(0)}M`.padStart(8); - -function report(title: string, keyOf: (call: ApiCall) => string, subset: readonly ApiCall[]) { - const rows = new Map(); - for (const call of subset) tally(rows, keyOf(call), call); - console.log(`\n== ${title}`); - console.log( - "key".padEnd(34) + - "calls".padStart(8) + - "total$".padStart(8) + - " read$% write$% uncach$% output$%" + - " tRead tWrite tUncach tOutput", - ); - const sorted = [...rows.entries()].toSorted(([, a], [, b]) => b.usd - a.usd); - const all = sorted.reduce((sum, [, row]) => { - for (const key of Object.keys(row) as (keyof Row)[]) sum[key] += row[key]; - return sum; - }, emptyRow()); - for (const [key, row] of [...sorted, ["TOTAL", all] as const]) { - console.log( - key.padEnd(34).slice(0, 34) + - String(row.calls).padStart(8) + - usd(row.usd) + - " " + - pct(row.cacheRead, row.usd) + - " " + - pct(row.cacheWrite, row.usd) + - " " + - pct(row.input, row.usd) + - " " + - pct(row.output, row.usd) + - mtok(row.tRead) + - " " + - mtok(row.tWrite) + - " " + - mtok(row.tUncached) + - " " + - mtok(row.tOutput), - ); - } - if (all.unsplitUsd > 0) console.log(` unsplit provider-reported cost: ${usd(all.unsplitUsd)}`); -} - -const latest = Math.max(...calls.map((call) => call.timestampMs)); -const within = (days: number) => - calls.filter((call) => call.timestampMs >= latest - days * 86_400_000); -console.log(`calls ${calls.length}, latest ${new Date(latest).toISOString()}`); -const unpriced = calls.filter((call) => price(rates, call).costSource === "unpriced"); -const unpricedModels = new Map(); -for (const call of unpriced) { - unpricedModels.set(call.model, (unpricedModels.get(call.model) ?? 0) + 1); -} -console.log(`unpriced calls ${unpriced.length}`, Object.fromEntries(unpricedModels)); - -report("all retained, by provider", (call) => call.provider, calls); -report("last 30 days, by provider", (call) => call.provider, within(30)); -report("last 30 days, by model", (call) => `${call.model} ${call.speed}`, within(30)); -report("last 7 days, by model", (call) => `${call.model} ${call.speed}`, within(7)); diff --git a/evals/cache-snip/usageRecords.ts b/evals/cache-snip/usageRecords.ts deleted file mode 100644 index d7e46dbc8655a..0000000000000 --- a/evals/cache-snip/usageRecords.ts +++ /dev/null @@ -1,150 +0,0 @@ -// @effect-diagnostics nodeBuiltinImport:off -import * as NodeFS from "node:fs"; - -import { - parseRateTable, - priceUsage, - type PricedRecord, - type PricedUsage, - type RateTable, -} from "../../apps/server/src/usage/usagePricing.ts"; - -const SPEEDS = ["standard", "fast", "ultrafast"] as const; -type Speed = (typeof SPEEDS)[number]; - -export interface ApiCall { - readonly provider: string; - readonly file: string; - readonly sessionId: string; - readonly timestampMs: number; - readonly model: string; - readonly speed: Speed; - readonly dedupeKey: string | null; - readonly priced: PricedRecord; - readonly uncached: number; - readonly cacheRead: number; - readonly cacheWrite: number; - readonly output: number; -} - -/** Everything the provider must read for this call. Output is excluded. */ -export const promptTokens = (call: ApiCall) => call.uncached + call.cacheRead + call.cacheWrite; - -type SerializedRecord = readonly [ - timestampMs: number, - modelIndex: number, - sessionIndex: number, - uncachedInputTokens: number, - cachedInputTokens: number, - cacheCreationTokens: number, - outputTokens: number, - reasoningTokens: number, - dedupeKey: string | null, - reportedCostUsd: number | null, - speed: number, - rateModelIndex?: number | null, - unresolvedServiceTierTurnId?: string, -]; - -interface SerializedCache { - readonly models: readonly string[]; - readonly sessions: readonly string[]; - readonly files: Readonly< - Record< - string, - { - readonly p: string; - readonly r: readonly SerializedRecord[]; - readonly t?: readonly SerializedRecord[]; - } - > - >; -} - -// Mirrors UsageService: moved copies of a shared session are dropped, repeated events in one file stay. -const SHARED_SESSION_PROVIDERS = new Set(["codex"]); - -/** Loads a copy of T3's usage scan cache with the Usage page's deduplication applied. */ -export function loadApiCalls(scanCachePath: string): ApiCall[] { - const cache = JSON.parse(NodeFS.readFileSync(scanCachePath, "utf8")) as SerializedCache; - const filesBySession = new Map>(); - for (const [file, entry] of Object.entries(cache.files)) { - for (const row of [...entry.r, ...(entry.t ?? [])]) { - const sessionId = cache.sessions[row[2]] ?? ""; - if (sessionId.length === 0) continue; - const seen = filesBySession.get(sessionId) ?? new Set(); - seen.add(file); - filesBySession.set(sessionId, seen); - } - } - - const seen = new Set(); - const calls: ApiCall[] = []; - for (const [file, entry] of Object.entries(cache.files)) { - const occurrences = new Map(); - for (const row of [...entry.r, ...(entry.t ?? [])]) { - const [ts, modelIndex, sessionIndex, uncached, cacheRead, cacheWrite, output, reasoning] = - row; - const model = cache.models[modelIndex] ?? ""; - const sessionId = cache.sessions[sessionIndex] ?? ""; - let dedupeKey = row[8]; - if (SHARED_SESSION_PROVIDERS.has(entry.p) && (filesBySession.get(sessionId)?.size ?? 0) > 1) { - const key = JSON.stringify([ - entry.p, - sessionId, - ts, - model, - uncached, - cacheRead, - cacheWrite, - ]); - const occurrence = (occurrences.get(key) ?? 0) + 1; - occurrences.set(key, occurrence); - dedupeKey = `${key}:${occurrence}`; - } - if (dedupeKey !== null) { - if (seen.has(dedupeKey)) continue; - seen.add(dedupeKey); - } - const speed = SPEEDS[row[10]] ?? "standard"; - const rateModelIndex = row[11]; - const rateModel = - typeof rateModelIndex === "number" ? cache.models[rateModelIndex] : undefined; - calls.push({ - provider: entry.p, - file, - sessionId, - timestampMs: ts, - model, - speed, - dedupeKey: row[8], - uncached, - cacheRead, - cacheWrite, - output, - priced: { - model, - ...(rateModel === undefined ? {} : { rateModel }), - speed, - reportedCostUsd: row[9], - totals: { - uncachedInputTokens: uncached, - cachedInputTokens: cacheRead, - cacheCreationTokens: cacheWrite, - outputTokens: output, - reasoningTokens: reasoning, - }, - }, - }); - } - } - return calls; -} - -export function loadRates(ratesPath: string): RateTable { - const { document } = JSON.parse(NodeFS.readFileSync(ratesPath, "utf8")) as { document: unknown }; - return parseRateTable(document); -} - -export const price = (rates: RateTable, call: ApiCall): PricedUsage => - priceUsage(rates, call.priced); diff --git a/evals/package.json b/evals/package.json index 07a2e0afc44c1..94cfc43d5ef31 100644 --- a/evals/package.json +++ b/evals/package.json @@ -9,14 +9,6 @@ "mcp-tool-use:regrade": "node mcp-tool-use/regrade.ts", "thread-titles": "node --env-file-if-exists=../.env thread-titles/run.ts", "test-signal": "node --env-file-if-exists=../.env test-signal/run.ts", - "cache-snip:spend": "node cache-snip/spend.ts", - "cache-snip:shape": "node cache-snip/shape.ts", - "cache-snip:replay": "node cache-snip/replay.ts", - "cache-snip:compose": "node cache-snip/compose.ts", - "cache-snip:simulate": "node cache-snip/simulate.ts", - "cache-snip:decisions": "node cache-snip/decisions.ts", - "cache-snip:relevance-rows": "node cache-snip/relevanceRows.ts", - "cache-snip:relevance-sample": "node cache-snip/relevanceSample.ts", "typecheck": "effect-tsgo tsrs --noEmit", "test": "vp test run --config ../vite.config.ts --dir ." },