diff --git a/AGENTS.md b/AGENTS.md index 7a7d7635..1a6097a0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -59,7 +59,7 @@ There are several places that need reviewing and updating with any new feature o - `src/schema/metadata-contract.ts` — Single source of truth for the `$metadata` contract - `schemas/generated/latest/_structured_context_schema_meta.json` — Generated metaschema for the newest dialect (generated on build or with `bun run generate:schema-meta`). `schemas/generated//` folders are frozen copies, created by the `version` script (`generate:schema-meta --release`) only when a release changes the dialect; never edit them. `src/schema/meta-schemas.ts` loads them at runtime. - `src/schema/gating.ts` — Cascade gating: computes hierarchy depth and evaluates rules with subtree gating (schema/hierarchy/blocking-rule failures suppress downstream rule checks). See [docs/rules.md](docs/rules.md#cascade-gating). -- `src/judge/` — Semantic rule judges: provider interface and types (`types.ts`), verdict resolution (`verdicts.ts`), per-node dispatch (`judge-node.ts`), and providers (`provider/anthropic.ts`, `provider/openai.ts`, `provider/jev.ts`, `provider/fake.ts`), provider hydration and the bundled catalogue (`factory.ts`), and `sctx providers` (`src/commands/providers.ts`). See [docs/semantic-rules.md](docs/semantic-rules.md). +- `src/judge/` — Semantic rule judges: provider interface and types (`types.ts`), verdict resolution (`verdicts.ts`), per-node and grouped dispatch (`judge-node.ts`), and providers (`provider/anthropic.ts`, `provider/openai.ts`, `provider/jev.ts`, `provider/fake.ts`), provider hydration and the bundled catalogue (`factory.ts`), and `sctx providers` (`src/commands/providers.ts`). See [docs/semantic-rules.md](docs/semantic-rules.md). - `src/schema/rule-filter.ts` — `--rule-filter` parsing and rule filtering by mode, category, severity, type or id. ## Testing diff --git a/docs/concepts.md b/docs/concepts.md index f18c7639..738aff07 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -173,6 +173,12 @@ A **semantic rule** (`mode: semantic`) asks a model to judge a node, for example A **judge** is the model that judges semantic rules: a decision model (Jev) or a chat model (Claude, OpenAI-compatible). For each rule, the judge gives one **verdict** from the rule's `verdicts`, for example `customer-need`, `vague` or `solution`. The rule's `severity` says which verdicts report a violation. Semantic rules cost money, so they are judged only when a run asks for it (`--semantic` or `--judge`); a rule's `check` always runs. The judge is a configured **model provider**: a bundled provider or an entry in `modelProviders`. See [docs/semantic-rules.md](semantic-rules.md). +A **node ruleset group** is the set of nodes that share the same semantic rules. It forms within one evaluation step of [cascade gating](#cascade-gating): one hierarchy depth, or the set of non-hierarchy nodes. Nodes are keyed on the rules left after `when` and `check` filtering. A judge on the `openrouter-decisions` or `typesafe-systemone` API judges one group in as few decision requests as it can. + +A **decision request** is one call to a decisions API. It carries the state of one or more nodes and a list of decision questions. + +A **decision question** is one choice question about one node under one semantic rule, inside a decision request. + ### Cascade gating **Cascade gating** is the mechanism by which a schema failure, hierarchy structure violation, or blocking rule violation on a node suppresses rule evaluation for that node's `hierarchy` subtree. It exists so downstream checks — especially costly semantic (model-judged) rules — aren't run against inputs already known to be invalid. diff --git a/docs/config.md b/docs/config.md index 3245a3f3..dea778e9 100644 --- a/docs/config.md +++ b/docs/config.md @@ -79,9 +79,9 @@ A provider's `api` names the **request shape**, potentially supported by multipl | API | Shape | Bundled providers | |---|---|---| -| `openrouter-decisions` | Decisions API, closed-set `choice` questions, answers with confidence and probabilities | `openrouter-jev`, `openrouter-openai-decisions` | -| `typesafe-systemone` | Same decisions shape (currently) | `typesafe-jev` | -| `openai-decisions` | OpenAI's Decisions API: same concept, its own request/response shape | `openai-decisions` | +| `openrouter-decisions` | Decisions API, closed-set `choice` questions, answers with confidence and probabilities; nodes with the same rules share a decision request | `openrouter-jev`, `openrouter-openai-decisions` | +| `typesafe-systemone` | Same decisions shape (currently), including shared decision requests | `typesafe-jev` | +| `openai-decisions` | OpenAI's Decisions API: same concept, its own request/response shape; one node per request | `openai-decisions` | | `anthropic-messages` | Chat: one request per rule, structured JSON verdict | `anthropic`, `anthropic-haiku-5-5`, `anthropic-env` | | `openai-completions` | Chat: OpenAI-compatible; works with Ollama, vLLM and OpenRouter | `openai`, `openai-env`, `openrouter-openai` | diff --git a/docs/semantic-rules.md b/docs/semantic-rules.md index fbe3c8e6..a963ce97 100644 --- a/docs/semantic-rules.md +++ b/docs/semantic-rules.md @@ -108,9 +108,10 @@ Gating uses the severity of the verdict. A verdict with `error` severity gates t ## How judging works -- **Order.** On each node, expression rules run first. Then structured-context applies each semantic rule's [`check`](#preconditions-check) and sends the rules that pass to the judge in one call. +- **Order.** On each node, expression rules run first. Then structured-context applies each semantic rule's [`check`](#preconditions-check) and sends the rules that pass to the judge. +- **Grouping.** A judge on the `openrouter-decisions` or `typesafe-systemone` API judges each node ruleset group (see [concepts](concepts.md#semantic-rules)) in as few decision requests as it can, with one decision question per node and rule. This includes the bundled `openrouter-openai-decisions` provider. A decision request is limited to about 32,000 estimated input tokens and 40 decision questions, so a larger group is split into several decision requests, never within a node. A node that fills a decision request on tokens is sent whole as its own decision request. A node with more than 40 rules has its rules split across decision requests. If a node's questions are split across several requests and one fails, all of that node's rules are reported as errors. Other providers, including `openai-decisions` and chat models, judge one node per request. Within a space, the nodes at one hierarchy depth are judged together, because their verdicts decide whether the next depth is judged. Grouping can shift verdicts and probabilities slightly compared with judging each node alone. - **Gating.** structured-context does not judge a gated node. [Coverage output](rules.md#coverage-output) shows which nodes were gated. -- **Judge errors.** If the judge fails (for example, a network error or output that structured-context cannot read), structured-context reports a violation. A judge error does not gate. +- **Judge errors.** If the judge fails (for example, a network error or output that structured-context cannot read), structured-context reports a violation. A judge error does not gate. If one decision request fails, only the rules on the nodes in that decision request report the error. - **Input.** The judge gets the node's title, type, parent titles and schema data, including frontmatter. - **Output.** A violation from a semantic rule also includes the verdict, the judge's evidence and any guidance. diff --git a/src/judge/judge-node.ts b/src/judge/judge-node.ts index 7ab7edb2..844a0a75 100644 --- a/src/judge/judge-node.ts +++ b/src/judge/judge-node.ts @@ -16,31 +16,23 @@ export type NodeJudgement = { error?: string; }; -/** Judge every given semantic rule on a node in one provider call. */ -export async function judgeNode( - judge: SemanticJudgeProvider, - node: SpaceNode, - rules: Rule[], -): Promise { - if (rules.length === 0) return []; - const resolved = rules.map(resolveSemanticRule); - const specs = resolved.map((r) => r.spec); - let results: Awaited>; - try { - results = await judge.evaluate({ - node, - rules: specs, - }); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - return rules.map((rule) => ({ - rule, - failed: true, - evidence: `Error during evaluation: ${message}`, - error: message, - })); - } +type JudgeResults = Awaited>; +type ResolvedRules = ReturnType; +function resolveAll(rules: Rule[]) { + return rules.map(resolveSemanticRule); +} + +function errorJudgements(rules: Rule[], message: string): NodeJudgement[] { + return rules.map((rule) => ({ + rule, + failed: true, + evidence: `Error during evaluation: ${message}`, + error: message, + })); +} + +function interpret(rules: Rule[], resolved: ResolvedRules, results: JudgeResults): NodeJudgement[] { return rules.map((rule, i) => { const result = results[rule.id]; if (!result || isJudgeError(result)) { @@ -63,3 +55,54 @@ export async function judgeNode( }; }); } + +const errorMessage = (err: unknown) => (err instanceof Error ? err.message : String(err)); + +/** Judge every given semantic rule on a node in one provider call. */ +export async function judgeNode( + judge: SemanticJudgeProvider, + node: SpaceNode, + rules: Rule[], +): Promise { + if (rules.length === 0) return []; + const resolved = resolveAll(rules); + let results: JudgeResults; + try { + results = await judge.evaluate({ node, rules: resolved.map((r) => r.spec) }); + } catch (err) { + return errorJudgements(rules, errorMessage(err)); + } + return interpret(rules, resolved, results); +} + +/** + * Judge several nodes at once. A judge with `evaluateNodes` gets every node in one call and may fold them into + * fewer requests; any other judge is called once per node. Results align with `items`. A failure marks only the + * affected node's rules as errors. + */ +export async function judgeNodes( + judge: SemanticJudgeProvider, + items: Array<{ node: SpaceNode; rules: Rule[] }>, +): Promise { + if (!judge.evaluateNodes) { + const out: NodeJudgement[][] = []; + for (const item of items) out.push(await judgeNode(judge, item.node, item.rules)); + return out; + } + const active = items + .map((item, index) => ({ ...item, index, resolved: resolveAll(item.rules) })) + .filter((a) => a.rules.length > 0); + const out: NodeJudgement[][] = items.map(() => []); + if (active.length === 0) return out; + let perNode: JudgeResults[]; + try { + perNode = await judge.evaluateNodes(active.map((a) => ({ node: a.node, rules: a.resolved.map((r) => r.spec) }))); + } catch (err) { + for (const a of active) out[a.index] = errorJudgements(a.rules, errorMessage(err)); + return out; + } + active.forEach((a, k) => { + out[a.index] = interpret(a.rules, a.resolved, perNode[k] ?? {}); + }); + return out; +} diff --git a/src/judge/provider/jev.ts b/src/judge/provider/jev.ts index 6ab9b105..8681f4fc 100644 --- a/src/judge/provider/jev.ts +++ b/src/judge/provider/jev.ts @@ -1,4 +1,10 @@ -import type { HydratedProvider, SemanticJudgeInput, SemanticJudgeProvider, SemanticJudgeResult } from '../types'; +import type { + HydratedProvider, + SemanticJudgeInput, + SemanticJudgeProvider, + SemanticJudgeResult, + SemanticRuleSpec, +} from '../types'; export interface JevJudgeProviderOptions extends HydratedProvider { apiKey?: string; @@ -8,6 +14,12 @@ export const JEV_LATEST = 'jev-latest'; export const OPENROUTER_DECISIONS_ENDPOINT = 'https://openrouter.ai/api/alpha/decisions'; +/** Estimated input-token cap (characters / 4) for one DecisionRequest's `state` plus `questions`: a conservative cost and latency guard. */ +export const JEV_DECISION_REQUEST_MAX_ESTIMATED_TOKENS = 32_000; + +/** Cap on the number of DecisionQuestions in one DecisionRequest. */ +export const JEV_DECISION_REQUEST_MAX_QUESTIONS = 40; + type JevChoiceAnswer = { type?: string; choice?: string; @@ -19,7 +31,7 @@ type JevChoiceAnswer = { * Judge provider for TypeSafe's Jev decision model via OpenRouter's Decisions API. * * Each node is sent once as `state`, with one `choice` question per semantic rule whose criteria are - * the rule's closed verdict set. Jev returns the chosen verdict with confidence and probabilities, + * the rule's closed verdict set. `evaluateNodes` shares one DecisionRequest between nodes with the same rule set. Jev returns the chosen verdict with confidence and probabilities, * but no prose, so `evidence` is a summary of the distribution. */ export class JevJudgeProvider implements SemanticJudgeProvider { @@ -36,52 +48,149 @@ export class JevJudgeProvider implements SemanticJudgeProvider { async evaluate(input: SemanticJudgeInput): Promise> { const { node, rules } = input; - const questions: Record = {}; - for (const rule of rules) { - questions[rule.id] = { - type: 'choice', - instructions: rule.rubric || rule.description, - criteria: rule.verdicts, - }; + const results: Record = {}; + // A node with more rules than one DecisionRequest allows is split by rule; the state is sent whole each time. + for (let start = 0; start < rules.length; start += JEV_DECISION_REQUEST_MAX_QUESTIONS) { + const part = rules.slice(start, start + JEV_DECISION_REQUEST_MAX_QUESTIONS); + const questions: Record = {}; + for (const rule of part) questions[rule.id] = decisionQuestion(rule, rule.rubric || rule.description); + const data = await this.post(nodeState(node), questions); + for (const rule of part) results[rule.id] = toResult(rule, data.answers?.[rule.id]); + } + return results; + } + + /** + * Judge many nodes with fewer DecisionRequests. Nodes that share the same semantic rule set form a + * NodeRulesetGroup, sent as one DecisionRequest, + * with every node under `state.nodes` and one DecisionQuestion per node and rule. A NodeRulesetGroup is split into DecisionRequests by + * `JEV_DECISION_REQUEST_MAX_ESTIMATED_TOKENS` and `JEV_DECISION_REQUEST_MAX_QUESTIONS`, never within a node. A node alone in its DecisionRequest, even + * one over the token cap, is sent whole as a plain single-node DecisionRequest; one with more rules than the question cap has its rules split + * across DecisionRequests. A failed DecisionRequest marks only its own nodes' rules + * as errors. + */ + async evaluateNodes(inputs: SemanticJudgeInput[]): Promise>> { + const results: Array> = inputs.map(() => ({})); + const nodeRulesetGroups = new Map(); + inputs.forEach((input, i) => { + if (input.rules.length === 0) return; + const key = JSON.stringify(input.rules); + nodeRulesetGroups.set(key, [...(nodeRulesetGroups.get(key) ?? []), i]); + }); + + for (const indices of nodeRulesetGroups.values()) { + for (const requestInputs of planDecisionRequests(indices, inputs)) { + try { + if (requestInputs.length === 1) { + results[requestInputs[0]!] = await this.evaluate(inputs[requestInputs[0]!]!); + } else { + await this.evaluateDecisionRequest(requestInputs, inputs, results); + } + } catch (err) { + const error = err instanceof Error ? err.message : String(err); + for (const i of requestInputs) { + results[i] = Object.fromEntries(inputs[i]!.rules.map((r) => [r.id, { error }])); + } + } + } } + return results; + } + private async evaluateDecisionRequest( + requestInputs: number[], + inputs: SemanticJudgeInput[], + results: Array>, + ): Promise { + const nodes: Record = {}; + const questions: Record = {}; + requestInputs.forEach((inputIndex, k) => { + const input = inputs[inputIndex]!; + nodes[`n${k}`] = nodeState(input.node); + for (const rule of input.rules) { + questions[`${rule.id}__n${k}`] = decisionQuestion( + rule, + `For node \`nodes.n${k}\`: ${rule.rubric || rule.description}`, + ); + } + }); + const data = await this.post({ nodes }, questions); + requestInputs.forEach((inputIndex, k) => { + for (const rule of inputs[inputIndex]!.rules) { + results[inputIndex]![rule.id] = toResult(rule, data.answers?.[`${rule.id}__n${k}`]); + } + }); + } + + private async post(state: unknown, questions: Record): Promise { const res = await fetch(this.endpoint, { method: 'POST', headers: { 'content-type': 'application/json', authorization: `Bearer ${this.apiKey}` }, - body: JSON.stringify({ - model: this.model, - state: { - title: node.title, - type: node.resolvedType, - parents: node.resolvedParents?.map((p) => p.title) ?? [], - ...node.schemaData, - }, - questions, - }), + body: JSON.stringify({ model: this.model, state, questions }), }); - if (!res.ok) { const errorText = await res.text().catch(() => ''); throw new Error(`Jev API error (${res.status}): ${errorText}`); } + return (await res.json()) as JevResponse; + } +} - const data = (await res.json()) as { answers?: Record }; - const results: Record = {}; - for (const rule of rules) { - const answer = data.answers?.[rule.id]; - if (!answer?.choice || !(answer.choice in rule.verdicts)) { - results[rule.id] = { error: `Jev returned no valid verdict for rule "${rule.id}" (choice: ${answer?.choice})` }; - continue; - } - results[rule.id] = { - verdict: answer.choice, - confidence: answer.confidence, - probabilities: answer.probabilities, - evidence: describeDistribution(answer), - }; +type JevResponse = { answers?: Record }; + +function nodeState(node: SemanticJudgeInput['node']) { + return { + title: node.title, + type: node.resolvedType, + parents: node.resolvedParents?.map((p) => p.title) ?? [], + ...node.schemaData, + }; +} + +function decisionQuestion(rule: SemanticRuleSpec, instructions: string) { + return { type: 'choice', instructions, criteria: rule.verdicts }; +} + +function toResult(rule: SemanticRuleSpec, answer: JevChoiceAnswer | undefined): SemanticJudgeResult { + if (!answer?.choice || !(answer.choice in rule.verdicts)) { + return { error: `Jev returned no valid verdict for rule "${rule.id}" (choice: ${answer?.choice})` }; + } + return { + verdict: answer.choice, + confidence: answer.confidence, + probabilities: answer.probabilities, + evidence: describeDistribution(answer), + }; +} + +const estimateTokens = (value: unknown) => Math.ceil(JSON.stringify(value).length / 4); + +/** Plan the DecisionRequests for one NodeRulesetGroup (inputs sharing a rule set): runs of nodes, in order, never splitting a node. */ +function planDecisionRequests(indices: number[], inputs: SemanticJudgeInput[]): number[][] { + const requests: number[][] = []; + let current: number[] = []; + let tokens = 0; + let questions = 0; + for (const i of indices) { + const input = inputs[i]!; + const cost = estimateTokens(nodeState(input.node)) + estimateTokens(input.rules); + const count = input.rules.length; + if ( + current.length > 0 && + (tokens + cost > JEV_DECISION_REQUEST_MAX_ESTIMATED_TOKENS || + questions + count > JEV_DECISION_REQUEST_MAX_QUESTIONS) + ) { + requests.push(current); + current = []; + tokens = 0; + questions = 0; } - return results; + current.push(i); + tokens += cost; + questions += count; } + if (current.length > 0) requests.push(current); + return requests; } function describeDistribution(answer: JevChoiceAnswer): string { diff --git a/src/judge/types.ts b/src/judge/types.ts index 79417d1e..c11ad7be 100644 --- a/src/judge/types.ts +++ b/src/judge/types.ts @@ -34,6 +34,12 @@ export type SemanticJudgeResult = SemanticJudgeVerdict | { error: string }; export interface SemanticJudgeProvider { readonly name: string; evaluate(input: SemanticJudgeInput): Promise>; + /** + * Optional: judge many nodes in one call, so a provider can fold them into fewer model calls. Returns one + * result map per input, in order. A failure that affects only some inputs is reported as `{ error }` + * results for those inputs' rules; throw only when nothing could be judged. + */ + evaluateNodes?(inputs: SemanticJudgeInput[]): Promise>>; } /** Wire APIs a provider can speak. Names the request shape, never the vendor or model. */ diff --git a/src/schema/gating.ts b/src/schema/gating.ts index b09b4e84..70727871 100644 --- a/src/schema/gating.ts +++ b/src/schema/gating.ts @@ -1,4 +1,5 @@ -import { judgeNode } from '../judge/judge-node'; +import type { NodeJudgement } from '../judge/judge-node'; +import { judgeNodes } from '../judge/judge-node'; import type { SemanticJudgeProvider } from '../judge/types'; import type { SpaceGraph } from '../space-graph'; import type { Rule, RuleSeverity, RuleViolation, SpaceNode } from '../types'; @@ -205,10 +206,19 @@ export async function evaluateRulesWithGating( return applicable; } - async function evaluateApplicableRules(node: SpaceNode): Promise<{ blockingRuleId?: string }> { - let blockingRuleId: string | undefined; + /** A node whose deterministic rules have run and whose semantic rules await the judge. */ + type PreparedNode = { + violations: RuleViolation[]; + semantic: Rule[]; + blockingRuleId?: string; + judged(judgements: NodeJudgement[]): void; + }; + + async function prepareNode(node: SpaceNode): Promise { + const local: RuleViolation[] = []; + const prepared: PreparedNode = { violations: local, semantic: [], judged: () => {} }; const applicable = await applicableRules(node, (rule, error) => - violations.push(expressionErrorViolation(rule, node.label, error)), + local.push(expressionErrorViolation(rule, node.label, error)), ); function report( @@ -216,7 +226,7 @@ export async function evaluateRulesWithGating( detail: { severity?: RuleSeverity; verdict?: string; guidance?: string; evidence?: string; error?: string } = {}, ): void { const severity = detail.severity ?? ruleSeverity(rule); - violations.push({ + local.push({ file: node.label, ruleId: rule.id, category: rule.category, @@ -227,13 +237,15 @@ export async function evaluateRulesWithGating( ...(detail.guidance !== undefined && { guidance: detail.guidance }), }); // A judge error says nothing about the node, so it never gates. - if (!detail.error && ruleGatesDescendants(rule, severity) && !blockingRuleId) blockingRuleId = rule.id; + if (!detail.error && ruleGatesDescendants(rule, severity) && !prepared.blockingRuleId) { + prepared.blockingRuleId = rule.id; + } } - // Cheap deterministic rules first; the judge is only consulted afterwards, once per node. + // Cheap deterministic rules first; the judge is only consulted afterwards. for (const rule of applicable.filter((r) => r.mode !== 'semantic')) { const outcome = await tryEvaluateExpression(rule.check as string, buildEvalContext(node, nodes, nodeIndex)); - if ('error' in outcome) violations.push(expressionErrorViolation(rule, node.label, outcome.error)); + if ('error' in outcome) local.push(expressionErrorViolation(rule, node.label, outcome.error)); else if (outcome.value !== true) report(rule); } @@ -244,12 +256,12 @@ export async function evaluateRulesWithGating( nodes, nodeIndex, ); - violations.push(...prechecked.violations); + local.push(...prechecked.violations); const gatingRule = prechecked.failed.find((r) => ruleGatesDescendants(r)); - if (gatingRule && !blockingRuleId) blockingRuleId = gatingRule.id; - const semantic = prechecked.judgeable; - if (judge && semantic.length > 0) { - for (const j of await judgeNode(judge, node, semantic)) { + if (gatingRule && !prepared.blockingRuleId) prepared.blockingRuleId = gatingRule.id; + if (judge) prepared.semantic = prechecked.judgeable; + prepared.judged = (judgements) => { + for (const j of judgements) { if (j.failed) { report(j.rule, { severity: j.severity, @@ -260,52 +272,77 @@ export async function evaluateRulesWithGating( }); } } + }; + return prepared; + } + + /** + * Evaluate one set of nodes together, so their semantic rules go to the judge in one pass and a judge + * with `evaluateNodes` can share model calls. Gating for the set is already decided; violations are + * recorded in node order. + */ + async function evaluateNodeSet(nodes: readonly SpaceNode[]): Promise> { + const prepared: PreparedNode[] = []; + for (const node of nodes) prepared.push(await prepareNode(node)); + if (judge) { + const pending = nodes.map((node, i) => ({ node, rules: prepared[i]!.semantic })); + const judgements = await judgeNodes(judge, pending); + prepared.forEach((p, i) => { + p.judged(judgements[i] ?? []); + }); } - return { blockingRuleId }; + for (const p of prepared) violations.push(...p.violations); + return prepared.map((p) => ({ blockingRuleId: p.blockingRuleId })); } // Non-hierarchy nodes sit outside the subtree cascade — always evaluated, never gated. - for (const node of graph.nonHierarchy) { - await evaluateApplicableRules(node); - } + await evaluateNodeSet(graph.nonHierarchy); const visited = new Set(); - const queue: Array<{ node: SpaceNode; ancestorOrigin?: string }> = []; + // Hierarchy nodes are judged one depth at a time: a node's verdicts decide whether its children are evaluated. + let depthNodes: Array<{ node: SpaceNode; ancestorOrigin?: string }> = []; for (const node of [...graph.hierarchyRoots, ...graph.orphans]) { - queue.push({ node, ancestorOrigin: undefined }); + depthNodes.push({ node, ancestorOrigin: undefined }); } - let qi = 0; - while (qi < queue.length) { - const { node, ancestorOrigin } = queue[qi++]!; - if (visited.has(node.title)) continue; - visited.add(node.title); - - const selfSeed = seedByTitle.get(node.title); - const isGatedByAncestor = ancestorOrigin !== undefined; - const isSelfBlocked = selfSeed !== undefined; - const isGated = isGatedByAncestor || isSelfBlocked; - const shouldEvaluate = !isGated || includeGated; + while (depthNodes.length > 0) { + const current: typeof depthNodes = []; + for (const entry of depthNodes) { + if (visited.has(entry.node.title)) continue; + visited.add(entry.node.title); + current.push(entry); + } + const isGated = (e: (typeof depthNodes)[number]) => e.ancestorOrigin !== undefined || seedByTitle.has(e.node.title); + const toEvaluate = current.filter((e) => !isGated(e) || includeGated); + const evaluated = await evaluateNodeSet(toEvaluate.map((e) => e.node)); + const blockingByTitle = new Map(toEvaluate.map((e, i) => [e.node.title, evaluated[i]!.blockingRuleId])); - const { blockingRuleId } = shouldEvaluate ? await evaluateApplicableRules(node) : {}; + const next: typeof depthNodes = []; + for (const { node, ancestorOrigin } of current) { + const selfSeed = seedByTitle.get(node.title); + const isGatedByAncestor = ancestorOrigin !== undefined; + const isSelfBlocked = selfSeed !== undefined; + const blockingRuleId = blockingByTitle.get(node.title); - let childOrigin: string | undefined = ancestorOrigin; - if (isGatedByAncestor) { - gateOrigins.get(ancestorOrigin!)!.gatedFiles.push(node.label); - } else if (isSelfBlocked) { - const origin = originFor(node.title, { file: selfSeed!.file, reason: selfSeed!.reason }); - origin.gatedFiles.push(node.label); - childOrigin = node.title; - } else if (blockingRuleId) { - originFor(node.title, { file: node.label, reason: 'rule', ruleId: blockingRuleId }); - childOrigin = node.title; - } else { - childOrigin = undefined; - } + let childOrigin: string | undefined = ancestorOrigin; + if (isGatedByAncestor) { + gateOrigins.get(ancestorOrigin!)!.gatedFiles.push(node.label); + } else if (isSelfBlocked) { + const origin = originFor(node.title, { file: selfSeed!.file, reason: selfSeed!.reason }); + origin.gatedFiles.push(node.label); + childOrigin = node.title; + } else if (blockingRuleId) { + originFor(node.title, { file: node.label, reason: 'rule', ruleId: blockingRuleId }); + childOrigin = node.title; + } else { + childOrigin = undefined; + } - for (const child of graph.hierarchyChildren.get(node.title) ?? []) { - queue.push({ node: child, ancestorOrigin: childOrigin }); + for (const child of graph.hierarchyChildren.get(node.title) ?? []) { + next.push({ node: child, ancestorOrigin: childOrigin }); + } } + depthNodes = next; } return { diff --git a/src/schema/validate-rules.ts b/src/schema/validate-rules.ts index 342b7012..232a1bc5 100644 --- a/src/schema/validate-rules.ts +++ b/src/schema/validate-rules.ts @@ -1,4 +1,4 @@ -import { judgeNode } from '../judge/judge-node'; +import { judgeNodes } from '../judge/judge-node'; import type { SemanticJudgeProvider } from '../judge/types'; import type { Rule, RuleViolation, SpaceNode } from '../types'; import { @@ -117,15 +117,19 @@ export async function validateRules( const judge = options.judge; const runnableSemanticRules = judge ? semanticRules : semanticRules.filter((r) => r.check); if (runnableSemanticRules.length > 0) { + const perNode: RuleViolation[][] = []; // kept in node order + const pending: Array<{ node: SpaceNode; rules: Rule[]; found: RuleViolation[] }> = []; for (const node of nodes) { if (options.judgeTarget !== undefined && node.label !== options.judgeTarget) continue; + const found: RuleViolation[] = []; + perNode.push(found); const applicable: Rule[] = []; for (const r of runnableSemanticRules) { if (r.type && r.type !== node.resolvedType) continue; if (r.when) { const when = await tryEvaluateExpression(r.when, buildEvalContext(node, nodes, nodeIndex)); if ('error' in when) { - violations.push(expressionErrorViolation(r, node.label, when.error)); + found.push(expressionErrorViolation(r, node.label, when.error)); continue; } if (when.value !== true) continue; @@ -139,12 +143,16 @@ export async function validateRules( // A semantic rule's `check` fails fast before the judge. const prechecked = await applySemanticChecks(applicable, node, nodes, nodeIndex); - violations.push(...prechecked.violations); - if (!judge) continue; + found.push(...prechecked.violations); + if (judge) pending.push({ node, rules: prechecked.judgeable, found }); + } - for (const j of await judgeNode(judge, node, prechecked.judgeable)) { + // One judge call for every node, so a judge with `evaluateNodes` can share model calls. + const judged = judge ? await judgeNodes(judge, pending) : []; + pending.forEach(({ node, found }, i) => { + for (const j of judged[i] ?? []) { if (!j.failed) continue; - violations.push({ + found.push({ file: node.label, ruleId: j.rule.id, category: j.rule.category, @@ -154,9 +162,9 @@ export async function validateRules( ...(j.verdict !== undefined && { verdict: j.verdict }), ...(j.guidance !== undefined && { guidance: j.guidance }), }); - if (!j.error) failedNodeLabels.add(node.label); } - } + }); + for (const found of perNode) violations.push(...found); } return violations; diff --git a/tests/judge/factory.test.ts b/tests/judge/factory.test.ts index 4742146f..a9993ea0 100644 --- a/tests/judge/factory.test.ts +++ b/tests/judge/factory.test.ts @@ -10,6 +10,7 @@ import { AnthropicJudgeProvider } from '../../src/judge/provider/anthropic'; import { FakeJudgeProvider } from '../../src/judge/provider/fake'; import { JevJudgeProvider } from '../../src/judge/provider/jev'; import { OpenAIJudgeProvider } from '../../src/judge/provider/openai'; +import { passFailRule } from './helpers'; describe('hydrateProvider', () => { it('prefers the explicit value over the env var and leaves unset fields undefined', () => { @@ -64,7 +65,7 @@ describe('createModelProvider', () => { schemaData: {}, resolvedParents: [], }, - rules: [], + rules: [passFailRule], }) .catch(() => {}); expect(calls[0]).toBe('https://openrouter.ai/api/alpha/decisions'); diff --git a/tests/judge/judge-node.test.ts b/tests/judge/judge-node.test.ts index 1948012c..d8ac8615 100644 --- a/tests/judge/judge-node.test.ts +++ b/tests/judge/judge-node.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'bun:test'; -import { judgeNode, type NodeJudgement } from '../../src/judge/judge-node'; +import { judgeNode, judgeNodes, type NodeJudgement } from '../../src/judge/judge-node'; import type { SemanticJudgeProvider, SemanticJudgeResult } from '../../src/judge/types'; import type { Rule } from '../../src/schema/metadata-contract'; import type { SpaceNode } from '../../src/types'; @@ -36,3 +36,70 @@ describe('judgeNode', () => { }); }); }); + +describe('judgeNodes', () => { + const rule = { + id: passFailRule.id, + description: passFailRule.description, + category: passFailRule.category, + mode: 'semantic', + severity: { fail: 'error' }, + verdicts: passFailRule.verdicts, + } as unknown as Rule; + const other = { ...node, title: 'Other', label: 'other.md' } as SpaceNode; + + it('calls a judge without evaluateNodes once per node', async () => { + const seen: string[] = []; + const judge: SemanticJudgeProvider = { + name: 'single', + evaluate: (input) => { + seen.push(input.node.title); + return Promise.resolve({ [passFailRule.id]: { verdict: 'pass' } }); + }, + }; + const out = await judgeNodes(judge, [ + { node: node as SpaceNode, rules: [rule] }, + { node: other, rules: [rule] }, + ]); + expect(seen).toEqual([node.title, 'Other']); + expect(out.map((o) => o.length)).toEqual([1, 1]); + }); + + it('sends every node to evaluateNodes at once and maps results back by position', async () => { + let calls = 0; + const judge: SemanticJudgeProvider = { + name: 'multi-node', + evaluate: () => Promise.reject(new Error('unused')), + evaluateNodes: (inputs) => { + calls++; + return Promise.resolve( + inputs.map((i) => ({ + [passFailRule.id]: i.node.title === 'Other' ? { verdict: 'fail' } : { verdict: 'pass' }, + })), + ); + }, + }; + const out = await judgeNodes(judge, [ + { node: node as SpaceNode, rules: [rule] }, + { node: other, rules: [] }, + { node: other, rules: [rule] }, + ]); + expect(calls).toBe(1); + expect(out[0]?.[0]).toMatchObject({ failed: false, verdict: 'pass' }); + expect(out[1]).toEqual([]); + expect(out[2]?.[0]).toMatchObject({ failed: true, verdict: 'fail' }); + }); + + it('marks every rule as an error, not a pass, when the evaluateNodes call throws', async () => { + const judge: SemanticJudgeProvider = { + name: 'multi-node', + evaluate: () => Promise.reject(new Error('unused')), + evaluateNodes: () => Promise.reject(new Error('boom')), + }; + const out = await judgeNodes(judge, [ + { node: node as SpaceNode, rules: [rule] }, + { node: other, rules: [rule] }, + ]); + expect(out.flat().every((j) => j.failed && j.error === 'boom')).toBe(true); + }); +}); diff --git a/tests/judge/live.test.ts b/tests/judge/live.test.ts index 315a79c3..4c82cb7e 100644 --- a/tests/judge/live.test.ts +++ b/tests/judge/live.test.ts @@ -3,9 +3,9 @@ * SCTX_LIVE_JUDGE_TESTS=1 (`bun run test:judge`), and each provider only when its API key is set. */ import { describe, expect, it } from 'bun:test'; -import { createModelProvider } from '../../src/judge/factory'; -import type { SemanticJudgeInput, SemanticRuleSpec } from '../../src/judge/types'; -import { node, passFailRule } from './helpers'; +import { BUNDLED_PROVIDERS, createModelProvider } from '../../src/judge/factory'; +import type { SemanticJudgeInput, SemanticJudgeResult, SemanticRuleSpec } from '../../src/judge/types'; +import { graded, node, passFailRule } from './helpers'; const CASES = [ { name: 'openrouter-jev', api: 'openrouter-decisions' as const, keyEnv: 'OPENROUTER_API_KEY' }, @@ -38,3 +38,69 @@ describe.skipIf(process.env.SCTX_LIVE_JUDGE_TESTS !== '1')('live judges', () => ); } }); + +/** + * Drift between judging nodes one at a time (`evaluate`) and batched (`evaluateNodes`). Prints per-node, per-rule + * verdicts and probability deltas; asserts only that both paths return a valid verdict, so the numbers are for a + * person to read. Run with `bun test tests/judge/live.test.ts -t drift` and the env below. + */ +describe.skipIf(process.env.SCTX_LIVE_JUDGE_TESTS !== '1')('live judges: batching drift', () => { + const titles = [ + 'I cannot find my receipts when tax time comes', + 'Build a receipt scanner app', + 'Increase filing revenue by 20%', + 'Tax forms are confusing and I never know which one I need', + ]; + const inputs: SemanticJudgeInput[] = titles.map((title, i) => ({ + node: { ...node, label: `n${i}.md`, title, schemaData: { ...node.schemaData, title } }, + rules: [passFailRule, graded], + })); + + for (const name of ['openrouter-jev', 'openrouter-openai-decisions']) { + it.skipIf(!process.env.OPENROUTER_API_KEY)( + `${name}: single versus batched verdicts and probabilities`, + async () => { + const provider = createModelProvider(name, BUNDLED_PROVIDERS[name]!, { + OPENROUTER_API_KEY: process.env.OPENROUTER_API_KEY, + }); + if (!provider.evaluateNodes) throw new Error(`${name} does not batch`); + const single: Array> = []; + for (const input of inputs) single.push(await provider.evaluate(input)); + const batched = await provider.evaluateNodes(inputs); + + const lines: string[] = []; + let flips = 0; + let maxDelta = 0; + inputs.forEach((input, i) => { + for (const rule of input.rules) { + const a = single[i]?.[rule.id]; + const b = batched[i]?.[rule.id]; + if (!a || !b || 'error' in a || 'error' in b) { + expect(a).not.toHaveProperty('error'); + expect(b).not.toHaveProperty('error'); + continue; + } + const deltas = Object.keys(rule.verdicts).map((v) => { + const d = (b.probabilities?.[v] ?? 0) - (a.probabilities?.[v] ?? 0); + maxDelta = Math.max(maxDelta, Math.abs(d)); + return `${v} ${d >= 0 ? '+' : ''}${d.toFixed(3)}`; + }); + if (a.verdict !== b.verdict) flips++; + lines.push( + `${input.node.label} ${rule.id}: single=${a.verdict} batched=${b.verdict}${a.verdict === b.verdict ? '' : ' (FLIP)'}; delta ${deltas.join(', ')}`, + ); + } + }); + console.log( + `[drift ${name}] verdict flips: ${flips}; max |probability delta|: ${maxDelta.toFixed(3)}\n${lines.join('\n')}`, + ); + for (const r of [...single, ...batched]) { + for (const rule of [passFailRule, graded]) { + expect(Object.keys(rule.verdicts)).toContain((r[rule.id] as { verdict: string }).verdict); + } + } + }, + 180_000, + ); + } +}); diff --git a/tests/judge/provider/jev.test.ts b/tests/judge/provider/jev.test.ts index da3ae032..9a649690 100644 --- a/tests/judge/provider/jev.test.ts +++ b/tests/judge/provider/jev.test.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, it } from 'bun:test'; import { createModelProvider } from '../../../src/judge/factory'; -import { JevJudgeProvider } from '../../../src/judge/provider/jev'; +import { JEV_DECISION_REQUEST_MAX_QUESTIONS, JevJudgeProvider } from '../../../src/judge/provider/jev'; import { graded, node, passFailRule } from '../helpers'; const realFetch = globalThis.fetch; @@ -80,4 +80,142 @@ describe('JevJudgeProvider', () => { 'OPENROUTER_API_KEY', ); }); + + describe('evaluateNodes', () => { + const jev = () => new JevJudgeProvider({ api: 'openrouter-decisions', apiKey: 'k' }); + const nodeNamed = (title: string, extra: Record = {}) => ({ + ...node, + title, + schemaData: { ...node.schemaData, title, ...extra }, + }); + /** Answers every question in the request with `pass`, or fails the request for which `fail` returns true. */ + function answering(fail: (body: { state: { nodes?: unknown } }) => boolean = () => false) { + const bodies: Array<{ state: { nodes?: Record }; questions: Record }> = []; + globalThis.fetch = (async (_url: string, init: RequestInit) => { + const body = JSON.parse(init.body as string); + bodies.push(body); + if (fail(body)) return new Response('upstream down', { status: 500 }); + const answers = Object.fromEntries( + Object.keys(body.questions).map((q) => [q, { choice: 'pass', probabilities: { pass: 1, fail: 0 } }]), + ); + return new Response(JSON.stringify({ answers }), { status: 200 }); + }) as unknown as typeof fetch; + return bodies; + } + + it('shares one request between nodes with the same rule set, one question per node and rule', async () => { + const bodies = answering(); + const results = await jev().evaluateNodes([ + { node: nodeNamed('A'), rules: [passFailRule, graded] }, + { node: nodeNamed('B'), rules: [passFailRule, graded] }, + ]); + expect(bodies).toHaveLength(1); + expect(Object.keys(bodies[0]!.state.nodes!)).toEqual(['n0', 'n1']); + expect(Object.keys(bodies[0]!.questions).sort()).toEqual([ + 'customer-voice__n0', + 'customer-voice__n1', + 'framing__n0', + 'framing__n1', + ]); + expect((bodies[0]!.questions.framing__n1 as { instructions: string }).instructions).toStartWith( + 'For node `nodes.n1`:', + ); + expect(results.map((r) => Object.keys(r).sort())).toEqual([ + ['customer-voice', 'framing'], + ['customer-voice', 'framing'], + ]); + }); + + it('sends nodes with different rule sets in separate requests', async () => { + const bodies = answering(); + await jev().evaluateNodes([ + { node: nodeNamed('A'), rules: [passFailRule] }, + { node: nodeNamed('B'), rules: [graded] }, + ]); + expect(bodies).toHaveLength(2); + }); + + it('sends a lone node as a plain single-node request', async () => { + const bodies = answering(); + await jev().evaluateNodes([{ node: nodeNamed('A'), rules: [passFailRule] }]); + expect(bodies[0]!.state.nodes).toBeUndefined(); + expect(Object.keys(bodies[0]!.questions)).toEqual(['customer-voice']); + }); + + it('splits a NodeRulesetGroup by the question cap without splitting a node', async () => { + const bodies = answering(); + const count = JEV_DECISION_REQUEST_MAX_QUESTIONS + 5; + const results = await jev().evaluateNodes( + Array.from({ length: count }, (_, i) => ({ node: nodeNamed(`N${i}`), rules: [passFailRule] })), + ); + expect(bodies.map((b) => Object.keys(b.questions).length)).toEqual([JEV_DECISION_REQUEST_MAX_QUESTIONS, 5]); + expect(results).toHaveLength(count); + expect(results.every((r) => 'verdict' in r['customer-voice']!)).toBe(true); + }); + + it('splits a NodeRulesetGroup by estimated size, and sends an oversized node alone and whole', async () => { + const bodies = answering(); + const big = (title: string) => nodeNamed(title, { body: 'x'.repeat(60_000) }); // ~15k tokens each + const huge = nodeNamed('Huge', { body: 'y'.repeat(200_000) }); // ~50k tokens, over the cap alone + await jev().evaluateNodes([ + { node: big('A'), rules: [passFailRule] }, + { node: big('B'), rules: [passFailRule] }, + { node: big('C'), rules: [passFailRule] }, + { node: huge, rules: [passFailRule] }, + ]); + expect(bodies.map((b) => (b.state.nodes ? Object.keys(b.state.nodes).length : 1))).toEqual([2, 1, 1]); + expect(JSON.stringify(bodies[2]!.state)).toContain('y'.repeat(200_000)); + }); + + it('reports an error on the rules of a node with no answer in a batched response, leaving other nodes intact', async () => { + globalThis.fetch = (async () => + new Response( + JSON.stringify({ + answers: { 'customer-voice__n0': { choice: 'pass' }, 'customer-voice__n2': { choice: 'fail' } }, + }), + { status: 200 }, + )) as unknown as typeof fetch; + const results = await jev().evaluateNodes( + ['A', 'B', 'C'].map((title) => ({ node: nodeNamed(title), rules: [passFailRule] })), + ); + expect(results[0]?.['customer-voice']).toMatchObject({ verdict: 'pass' }); + expect(results[1]?.['customer-voice']).toHaveProperty('error'); + expect(results[2]?.['customer-voice']).toMatchObject({ verdict: 'fail' }); + }); + + it('resolves answers for a rule id that itself contains the node suffix', async () => { + const bodies = answering(); + const tricky = { ...passFailRule, id: 'tricky__n0' }; + const results = await jev().evaluateNodes([ + { node: nodeNamed('A'), rules: [tricky] }, + { node: nodeNamed('B'), rules: [tricky] }, + ]); + expect(Object.keys(bodies[0]!.questions)).toEqual(['tricky__n0__n0', 'tricky__n0__n1']); + expect(results.map((r) => r.tricky__n0)).toEqual([ + expect.objectContaining({ verdict: 'pass' }), + expect.objectContaining({ verdict: 'pass' }), + ]); + }); + + it('splits a single node with more rules than the question cap across requests, sending its state whole each time', async () => { + const bodies = answering(); + const count = JEV_DECISION_REQUEST_MAX_QUESTIONS + 3; + const rules = Array.from({ length: count }, (_, i) => ({ ...passFailRule, id: `r${i}` })); + const results = await jev().evaluateNodes([{ node: nodeNamed('A'), rules }]); + expect(bodies.map((b) => Object.keys(b.questions).length)).toEqual([JEV_DECISION_REQUEST_MAX_QUESTIONS, 3]); + expect(bodies.every((b) => b.state.nodes === undefined)).toBe(true); + expect(Object.keys(results[0]!)).toHaveLength(count); + expect(Object.values(results[0]!).every((r) => 'verdict' in r)).toBe(true); + }); + + it('maps a failed request to errors on its own nodes only', async () => { + answering((body) => JSON.stringify(body).includes('"title":"A"')); + const results = await jev().evaluateNodes([ + { node: nodeNamed('A', { body: 'x'.repeat(100_000) }), rules: [passFailRule] }, + { node: nodeNamed('B', { body: 'x'.repeat(100_000) }), rules: [passFailRule] }, + ]); + expect(results[0]?.['customer-voice']).toMatchObject({ error: expect.stringContaining('Jev API error (500)') }); + expect(results[1]?.['customer-voice']).toMatchObject({ verdict: 'pass' }); + }); + }); }); diff --git a/tests/schema/gating.test.ts b/tests/schema/gating.test.ts index 515e1643..794ce6aa 100644 --- a/tests/schema/gating.test.ts +++ b/tests/schema/gating.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from 'bun:test'; +import type { SemanticJudgeProvider } from '../../src/judge/types'; import { computeHierarchyDepths, evaluateRulesWithGating, @@ -185,4 +186,43 @@ describe('evaluateRulesWithGating', () => { expect(violations.find((v) => v.ruleId === 'global-fail')).toBeDefined(); }); + + describe('judging by depth', () => { + const semanticRule = { + id: 'sem', + category: 'best-practice', + description: 'semantic', + mode: 'semantic', + severity: { fail: 'error' }, + verdicts: { pass: 'ok', fail: 'not ok' }, + } as unknown as Rule; + + it('judges each depth in one evaluateNodes call and still gates children of a failing node', async () => { + const goal = makeSpaceNode('Goal', 'goal'); + const oppA = makeSpaceNode('Opp A', 'opportunity', 'Goal'); + const oppB = makeSpaceNode('Opp B', 'opportunity', 'Goal'); + const solA = makeSpaceNode('Sol A', 'solution', 'Opp A'); + const solB = makeSpaceNode('Sol B', 'solution', 'Opp B'); + const nodes = [goal, oppA, oppB, solA, solB]; + const graph = buildSpaceGraph(nodes, hierarchy); + + const calls: string[][] = []; + const provider: SemanticJudgeProvider = { + name: 'multi-node', + evaluate: () => Promise.reject(new Error('expected evaluateNodes')), + evaluateNodes: async (inputs) => { + calls.push(inputs.map((i) => i.node.title)); + return inputs.map((i) => + Object.fromEntries(i.rules.map((r) => [r.id, { verdict: i.node.title === 'Opp A' ? 'fail' : 'pass' }])), + ); + }, + }; + + const { violations, gates } = await evaluateRulesWithGating(nodes, [semanticRule], graph, [], { provider }); + expect(calls).toEqual([['Goal'], ['Opp A', 'Opp B'], ['Sol B']]); + expect(violations.map((v) => v.file)).toEqual(['Opp A.md']); + expect(gates).toHaveLength(1); + expect(gates[0]?.gatedFiles).toEqual(['Sol A.md']); + }); + }); });