From 7654380982558a165b98be3acc08218537f472fc Mon Sep 17 00:00:00 2001 From: Roger Barnes Date: Fri, 16 Oct 2026 14:43:04 +1100 Subject: [PATCH 1/6] feat(judge): per-node input cap, body trimming and missing-context verdict Cap one node's input (node plus related nodes) at 32k estimated tokens for every provider. Over the cap, cut related bodies by water-filling, then the evaluated node's body, ending each cut at whitespace with a [RULE_EVALUATION_CONTENT_TRIMMED n chars] marker. Input still over the cap reports a semantic evaluation error and skips the provider. Rules with semantic.scope or includeContent offer missing-context (warning by default). The validate summary and JSON report trimmed nodes and rules. Rename the Jev packing constants to request packing limit. --- src/commands/validate.ts | 7 ++ src/judge/judge-node.ts | 46 ++++++++-- src/judge/provider/jev.ts | 25 +++--- src/judge/trim.ts | 88 ++++++++++++++++++++ src/judge/verdicts.ts | 14 ++++ src/schema/gating.ts | 5 +- src/schema/validate-rules.ts | 5 +- src/validate.ts | 18 +++- tests/judge/provider/jev.test.ts | 10 +-- tests/judge/renderers.test.ts | 78 +++++++++++++++++ tests/judge/trim.test.ts | 84 +++++++++++++++++++ tests/judge/trimming.test.ts | 97 ++++++++++++++++++++++ tests/validate/semantic-validation.test.ts | 29 +++++++ 13 files changed, 479 insertions(+), 27 deletions(-) create mode 100644 src/judge/trim.ts create mode 100644 tests/judge/renderers.test.ts create mode 100644 tests/judge/trim.test.ts create mode 100644 tests/judge/trimming.test.ts diff --git a/src/commands/validate.ts b/src/commands/validate.ts index 489ed21c..c9cd79ac 100644 --- a/src/commands/validate.ts +++ b/src/commands/validate.ts @@ -85,6 +85,7 @@ export async function validate( parseIssues: result.parseIssues, unresolvedContentLinks: result.unresolvedContentLinks, coverageGates: result.coverageGates, + contextTrimmed: result.contextTrimmed, skippedByDepth: result.skippedByDepth, depthLimit: result.depthLimit, }, @@ -237,6 +238,12 @@ export async function validate( for (const { file } of result.skippedByDepth) console.log(` ${file}`); } + if (result.contextTrimmed.nodeCount > 0) { + console.log( + `\nContext trimmed for ${result.contextTrimmed.nodeCount} node(s) across rules ${result.contextTrimmed.ruleIds.join(', ')}.`, + ); + } + console.log(`\n`); if ( diff --git a/src/judge/judge-node.ts b/src/judge/judge-node.ts index 63167432..6340aa2d 100644 --- a/src/judge/judge-node.ts +++ b/src/judge/judge-node.ts @@ -1,6 +1,7 @@ import type { SpaceGraph } from '../space-graph'; import type { Rule, RuleSeverity, SpaceNode } from '../types'; import { judgeContext, type RuleContext, ruleContext } from './context'; +import { type TrimReport, trimToCap } from './trim'; import type { SemanticJudgeInput, SemanticJudgeProvider } from './types'; import { isJudgeError, resolveSemanticRule } from './verdicts'; @@ -60,10 +61,27 @@ function interpret(rules: Rule[], resolved: ResolvedRules, results: JudgeResults const errorMessage = (err: unknown) => (err instanceof Error ? err.message : String(err)); -/** One provider input and the rules (by position in their node's rule list) it answers. */ -type Unit = { item: number; positions: number[]; rules: Rule[]; resolved: ResolvedRules; input: SemanticJudgeInput }; +/** Reported for a node whose input is over the per-node input cap even with every body cut. */ +const OVER_CAP_MESSAGE = 'input over the per-node cap after trimming content'; -/** Split each node's rules into inputs by the context they ask for; rules that share a context share an input. */ +/** + * One provider input and the rules (by position in their node's rule list) it answers. `trimmed` is set when + * a body was cut to fit the per-node input cap; `overCap` when that was not enough, so the input is not sent. + */ +type Unit = { + item: number; + positions: number[]; + rules: Rule[]; + resolved: ResolvedRules; + input: SemanticJudgeInput; + trimmed: boolean; + overCap: boolean; +}; + +/** + * Split each node's rules into inputs by the context they ask for; rules that share a context share an input. + * Each input is trimmed to the per-node input cap. + */ function planUnits(graph: SpaceGraph, items: Array<{ node: SpaceNode; rules: Rule[] }>): Unit[] { const units: Unit[] = []; items.forEach(({ node, rules }, item) => { @@ -78,12 +96,15 @@ function planUnits(graph: SpaceGraph, items: Array<{ node: SpaceNode; rules: Rul for (const { context, positions } of groups.values()) { const unitRules = positions.map((p) => rules[p]!); const resolved = resolveAll(unitRules); + const { input, trimmed, overCap } = trimToCap(judgeContext(graph, node, context)); units.push({ item, positions, rules: unitRules, resolved, - input: { ...judgeContext(graph, node, context), rules: resolved.map((r) => r.spec) }, + input: { ...input, rules: resolved.map((r) => r.spec) }, + trimmed, + overCap, }); } }); @@ -93,15 +114,18 @@ function planUnits(graph: SpaceGraph, items: Array<{ node: SpaceNode; rules: Rul /** * Judge several nodes at once. A judge with `evaluateNodes` gets every input in one call and may fold them into * fewer requests; any other judge is called once per input. A node's rules that share a context (related nodes - * and bodies) form one input. Results align with `items`, in rule order. A failure marks only the affected - * rules as errors. + * and bodies) form one input. An input over the per-node input cap has bodies trimmed; one still over is not sent + * and marks its rules as errors. Trimmed nodes and rules are added to `report`. Results align with `items`, in + * rule order. A failure marks only the affected rules as errors. */ export async function judgeNodes( judge: SemanticJudgeProvider, items: Array<{ node: SpaceNode; rules: Rule[] }>, graph: SpaceGraph, + report?: TrimReport, ): Promise { - const units = planUnits(graph, items); + const planned = planUnits(graph, items); + const units = planned.filter((u) => !u.overCap); const out: NodeJudgement[][] = items.map(({ rules }) => new Array(rules.length)); const place = (unit: Unit, judgements: NodeJudgement[]) => { unit.positions.forEach((position, k) => { @@ -109,6 +133,14 @@ export async function judgeNodes( }); }; + for (const unit of planned) { + if (unit.overCap) place(unit, errorJudgements(unit.rules, OVER_CAP_MESSAGE)); + else if (unit.trimmed && report) { + report.nodes.add(items[unit.item]!.node.label); + for (const rule of unit.rules) report.ruleIds.add(rule.id); + } + } + if (!judge.evaluateNodes) { for (const unit of units) { try { diff --git a/src/judge/provider/jev.ts b/src/judge/provider/jev.ts index 23afc890..29b31518 100644 --- a/src/judge/provider/jev.ts +++ b/src/judge/provider/jev.ts @@ -1,3 +1,4 @@ +import { estimateTokens } from '../trim'; import type { HydratedProvider, RelatedNode, @@ -15,11 +16,14 @@ export const JEV_LATEST = 'jev-latest'; export const OPENROUTER_DECISIONS_ENDPOINT = 'https://openrouter.ai/api/alpha/decisions'; -/** Estimated input-token cap (characters / 4) for one DecisionRequest's `state` plus `questions`: a conservative cost and latency guard. */ -export const JEV_DECISION_REQUEST_MAX_ESTIMATED_TOKENS = 32_000; +/** + * Request packing limit: the estimated input tokens (characters / 4) packed into one DecisionRequest's `state` + * plus `questions`. A conservative cost and latency guard. Not the per-node input cap, which bounds one node's input. + */ +export const JEV_REQUEST_PACKING_LIMIT_TOKENS = 32_000; -/** Cap on the number of DecisionQuestions in one DecisionRequest. */ -export const JEV_DECISION_REQUEST_MAX_QUESTIONS = 40; +/** Request packing limit on the number of DecisionQuestions in one DecisionRequest. */ +export const JEV_REQUEST_PACKING_LIMIT_QUESTIONS = 40; type JevChoiceAnswer = { type?: string; @@ -51,8 +55,8 @@ export class JevJudgeProvider implements SemanticJudgeProvider { const { node, rules } = input; const results: Record = {}; // A node with more rules than one DecisionRequest allows is split by rule; the state is sent whole each time. - for (let start = 0; start < rules.length; start += JEV_DECISION_REQUEST_MAX_QUESTIONS) { - const part = rules.slice(start, start + JEV_DECISION_REQUEST_MAX_QUESTIONS); + for (let start = 0; start < rules.length; start += JEV_REQUEST_PACKING_LIMIT_QUESTIONS) { + const part = rules.slice(start, start + JEV_REQUEST_PACKING_LIMIT_QUESTIONS); const questions: Record = {}; for (const rule of part) questions[rule.id] = decisionQuestion(rule, rule.rubric || rule.description); const data = await this.post(nodeState(node, input.related), questions); @@ -65,8 +69,8 @@ export class JevJudgeProvider implements SemanticJudgeProvider { * Judge many nodes with fewer DecisionRequests. Nodes that share the same semantic rule set form a * NodeRulesetGroup, sent as one DecisionRequest, * with every node under `state.nodes` and one DecisionQuestion per node and rule. A NodeRulesetGroup is split into DecisionRequests by - * `JEV_DECISION_REQUEST_MAX_ESTIMATED_TOKENS` and `JEV_DECISION_REQUEST_MAX_QUESTIONS`, never within a node. A node alone in its DecisionRequest, even - * one over the token cap, is sent whole as a plain single-node DecisionRequest; one with more rules than the question cap has its rules split + * `JEV_REQUEST_PACKING_LIMIT_TOKENS` and `JEV_REQUEST_PACKING_LIMIT_QUESTIONS`, never within a node. A node alone in its DecisionRequest, even + * one over the token limit, is sent whole as a plain single-node DecisionRequest; one with more rules than the question limit has its rules split * across DecisionRequests. A failed DecisionRequest marks only its own nodes' rules * as errors. */ @@ -172,8 +176,6 @@ function toResult(rule: SemanticRuleSpec, answer: JevChoiceAnswer | undefined): }; } -const estimateTokens = (value: unknown) => Math.ceil(JSON.stringify(value).length / 4); - /** Plan the DecisionRequests for one NodeRulesetGroup (inputs sharing a rule set): runs of nodes, in order, never splitting a node. */ function planDecisionRequests(indices: number[], inputs: SemanticJudgeInput[]): number[][] { const requests: number[][] = []; @@ -186,8 +188,7 @@ function planDecisionRequests(indices: number[], inputs: SemanticJudgeInput[]): const count = input.rules.length; if ( current.length > 0 && - (tokens + cost > JEV_DECISION_REQUEST_MAX_ESTIMATED_TOKENS || - questions + count > JEV_DECISION_REQUEST_MAX_QUESTIONS) + (tokens + cost > JEV_REQUEST_PACKING_LIMIT_TOKENS || questions + count > JEV_REQUEST_PACKING_LIMIT_QUESTIONS) ) { requests.push(current); current = []; diff --git a/src/judge/trim.ts b/src/judge/trim.ts new file mode 100644 index 00000000..3e2a552d --- /dev/null +++ b/src/judge/trim.ts @@ -0,0 +1,88 @@ +import type { SemanticJudgeInput } from './types'; + +/** The per-node input cap: estimated tokens of one node's input (the node and its related nodes), the same for every provider. */ +export const PER_NODE_INPUT_CAP_TOKENS = 32_000; + +/** Estimated tokens of a value: its JSON length divided by 4. Provider-independent, and roughly right is enough. */ +export const estimateTokens = (value: unknown) => Math.ceil(JSON.stringify(value).length / 4); + +/** The placeholder that ends a body cut to fit the per-node input cap. `removed` is the number of characters cut. */ +export const trimMarker = (removed: number) => `[RULE_EVALUATION_CONTENT_TRIMMED ${removed} chars]`; + +/** The part of a judge input that the per-node input cap measures. */ +export type NodeInput = Pick; + +/** + * Cut `body` to at most `limit` characters, ending at the last whitespace before the limit, and end it with the + * trim marker. A body within the limit is returned whole. + */ +export function cutBody(body: string, limit: number): string { + if (body.length <= limit) return body; + let head = body.slice(0, limit); + if (!/\s/.test(body.charAt(limit))) { + const lastSpace = head.search(/\s\S*$/); + head = lastSpace < 0 ? '' : head.slice(0, lastSpace); + } + head = head.trimEnd(); + const marker = trimMarker(body.length - head.length); + return head ? `${head} ${marker}` : marker; +} + +function limitBodies(input: NodeInput, relatedLimit: number, nodeLimit: number): NodeInput { + const cut = >(data: T, limit: number): T => + typeof data.content === 'string' ? { ...data, content: cutBody(data.content, limit) } : data; + return { + node: { ...input.node, schemaData: cut(input.node.schemaData, nodeLimit) }, + ...(input.related && { related: input.related.map((r) => ({ ...r, data: cut(r.data, relatedLimit) })) }), + }; +} + +const bodyLength = (data: Record) => (typeof data.content === 'string' ? data.content.length : 0); + +/** The largest limit in `[0, max]` for which `fits` holds, or -1 when even 0 does not fit. `fits` shrinks as the limit falls. */ +function largestFitting(max: number, fits: (limit: number) => boolean): number { + if (!fits(0)) return -1; + let low = 0; + let high = max; + while (low < high) { + const mid = Math.ceil((low + high) / 2); + if (fits(mid)) low = mid; + else high = mid - 1; + } + return low; +} + +export interface TrimResult { + input: NodeInput; + /** True when any body was cut. */ + trimmed: boolean; + /** True when the input is still over the cap with every body cut to nothing. */ + overCap: boolean; +} + +/** + * Bring a node's input under the per-node input cap by cutting bodies only: never fields, never a whole node. + * Related bodies are cut first, with one ceiling for all of them (longer bodies are cut to it, shorter ones stay + * whole) set as high as the cap allows. If that reaches zero and the input is still over, the evaluated node's + * body is cut the same way. Every cut body ends with the trim marker. + */ +export function trimToCap(input: NodeInput, cap = PER_NODE_INPUT_CAP_TOKENS): TrimResult { + if (estimateTokens(input) <= cap) return { input, trimmed: false, overCap: false }; + + const nodeLength = bodyLength(input.node.schemaData); + const relatedMax = Math.max(0, ...(input.related ?? []).map((r) => bodyLength(r.data))); + const fitsRelated = (limit: number) => estimateTokens(limitBodies(input, limit, nodeLength)) <= cap; + const relatedLimit = largestFitting(relatedMax, fitsRelated); + if (relatedLimit >= 0) return { input: limitBodies(input, relatedLimit, nodeLength), trimmed: true, overCap: false }; + + const fitsNode = (limit: number) => estimateTokens(limitBodies(input, 0, limit)) <= cap; + const nodeLimit = largestFitting(nodeLength, fitsNode); + if (nodeLimit >= 0) return { input: limitBodies(input, 0, nodeLimit), trimmed: true, overCap: false }; + return { input: limitBodies(input, 0, 0), trimmed: true, overCap: true }; +} + +/** What a run trimmed: the nodes (by label) and the rules whose input had a body cut. */ +export interface TrimReport { + nodes: Set; + ruleIds: Set; +} diff --git a/src/judge/verdicts.ts b/src/judge/verdicts.ts index 2ff0d4f8..1cfd93d4 100644 --- a/src/judge/verdicts.ts +++ b/src/judge/verdicts.ts @@ -1,6 +1,7 @@ import { RESERVED_VERDICTS } from '../schema/metadata-contract'; import { ruleSemantic, ruleSeverity } from '../schema/rule-severity'; import type { Rule, RuleSeverity } from '../types'; +import { ruleContext } from './context'; import type { SemanticJudgeResult, SemanticRuleSpec } from './types'; /** Verdicts for a semantic rule that declares none. */ @@ -9,6 +10,12 @@ export const DEFAULT_VERDICTS: Record = { fail: 'The node does not meet the rubric.', }; +/** The reserved verdict a rule with related nodes or bodies offers, for input a trim marker shows was cut. */ +export const MISSING_CONTEXT_VERDICT = 'missing-context'; + +const MISSING_CONTEXT_DESCRIPTION = + 'Content needed to judge the rule was cut: a [RULE_EVALUATION_CONTENT_TRIMMED n chars] marker shows where. Choose this only when such a marker is present and the cut content is needed. Never choose it when no marker appears.'; + /** A semantic rule resolved for evaluation: what the judge sees, and what each verdict means for the user. */ export interface ResolvedSemanticRule { /** What the judge sees: the closed verdict set. */ @@ -65,6 +72,13 @@ export function resolveSemanticRule(rule: Rule): ResolvedSemanticRule { throw new Error(`Semantic rule "${rule.id}" has no passing verdict: every verdict has a severity.`); } + // A rule that asks for related nodes or bodies offers `missing-context` (a static property of the rule), a warning by default. + const context = ruleContext(rule); + if (context.scope.length > 0 || context.content !== 'none') { + verdicts[MISSING_CONTEXT_VERDICT] = MISSING_CONTEXT_DESCRIPTION; + if (!(MISSING_CONTEXT_VERDICT in mapping)) severities[MISSING_CONTEXT_VERDICT] = 'warning'; + } + return { spec: { id: rule.id, diff --git a/src/schema/gating.ts b/src/schema/gating.ts index fcd730cc..cdfc796b 100644 --- a/src/schema/gating.ts +++ b/src/schema/gating.ts @@ -1,5 +1,6 @@ import type { NodeJudgement } from '../judge/judge-node'; import { judgeNodes } from '../judge/judge-node'; +import type { TrimReport } from '../judge/trim'; import type { SemanticJudgeProvider } from '../judge/types'; import type { SpaceGraph } from '../space-graph'; import type { Rule, RuleSeverity, RuleViolation, SpaceNode } from '../types'; @@ -60,6 +61,8 @@ export type GatedRuleEvaluation = { }; export interface GatingOptions { + /** Collects the nodes and rules whose input had content trimmed to fit the per-node input cap. */ + trimReport?: TrimReport; /** Evaluate rules on gated nodes anyway (gates are still reported). */ includeGated?: boolean; /** Evaluates `semantic` rules. Without it, a rule's semantic part is not evaluated. */ @@ -280,7 +283,7 @@ export async function evaluateRulesWithGating( for (const node of nodes) prepared.push(await prepareNode(node)); if (judge) { const pending = nodes.map((node, i) => ({ node, rules: prepared[i]!.semantic })); - const judgements = await judgeNodes(judge, pending, graph); + const judgements = await judgeNodes(judge, pending, graph, options.trimReport); prepared.forEach((p, i) => { p.judged(judgements[i] ?? []); }); diff --git a/src/schema/validate-rules.ts b/src/schema/validate-rules.ts index c961fd71..a3f6151a 100644 --- a/src/schema/validate-rules.ts +++ b/src/schema/validate-rules.ts @@ -1,4 +1,5 @@ import { judgeNodes } from '../judge/judge-node'; +import type { TrimReport } from '../judge/trim'; import type { SemanticJudgeProvider } from '../judge/types'; import { buildSpaceGraph } from '../space-graph'; import type { Rule, RuleViolation, SpaceNode } from '../types'; @@ -19,6 +20,8 @@ export interface ValidateRulesOptions { judgeTarget?: string; /** Set of node labels that have already failed upstream structural checks. */ gatedNodeLabels?: Set; + /** Collects the nodes and rules whose input had content trimmed to fit the per-node input cap. */ + trimReport?: TrimReport; } /** @@ -110,7 +113,7 @@ export async function validateRules( } // One provider call for every node, so a provider with `evaluateNodes` can share model calls. - const judged = await judgeNodes(judge, pending, buildSpaceGraph(nodes, undefined)); + const judged = await judgeNodes(judge, pending, buildSpaceGraph(nodes, undefined), options.trimReport); pending.forEach(({ node, found }, i) => { for (const j of judged[i] ?? []) { if (!j.failed) continue; diff --git a/src/validate.ts b/src/validate.ts index 547b659f..4a2df783 100644 --- a/src/validate.ts +++ b/src/validate.ts @@ -1,6 +1,7 @@ import { isAbsolute, relative, resolve } from 'node:path'; import type { ErrorObject } from 'ajv'; import type { Config, SpaceConfig } from './config'; +import type { TrimReport } from './judge/trim'; import type { SemanticJudgeProvider } from './judge/types'; import { readSpace } from './read/read-space'; import { computeHierarchyDepths, evaluateRulesWithGating, type GatingSeed } from './schema/gating'; @@ -50,6 +51,8 @@ export interface ValidationResult { * See docs/rules.md for gating semantics. */ coverageGates: CoverageGate[]; + /** Semantic rule input that had content trimmed to fit the per-node input cap: the node count and the rule ids. */ + contextTrimmed: { nodeCount: number; ruleIds: string[] }; /** Nodes excluded from validation entirely by a `depth` limit (breadth-first from hierarchy roots). */ skippedByDepth: Array<{ file: string; title: string }>; /** The depth limit applied, if any (undefined means the full space was validated). */ @@ -185,6 +188,11 @@ type SpaceRuleEvaluation = { gates: CoverageGate[]; }; +const summariseTrim = (report: TrimReport): ValidationResult['contextTrimmed'] => ({ + nodeCount: report.nodes.size, + ruleIds: [...report.ruleIds].sort(), +}); + /** * Evaluate a space's rules. With a main hierarchy, evaluation cascades: schema, hierarchy * and gating-rule failures (the `seeds`) suppress rule evaluation for the failing node's subtree. @@ -200,6 +208,7 @@ async function evaluateSpaceRules( judge?: SemanticJudgeProvider; judgeTarget?: string; ruleFilter?: RuleFilter; + trimReport?: TrimReport; } = {}, ): Promise { const rules = metadata.rules && filterRules(metadata.rules, options.ruleFilter); @@ -209,6 +218,7 @@ async function evaluateSpaceRules( const result = await validateRules(nodes, rules, { judge: provider, judgeTarget: options.judgeTarget, + trimReport: options.trimReport, }); return { violations: result, gates: [] }; } @@ -217,6 +227,7 @@ async function evaluateSpaceRules( includeGated: options.includeGated, provider, judgeTarget: options.judgeTarget, + trimReport: options.trimReport, }); } @@ -239,6 +250,7 @@ interface SpaceValidationResult { orphans: SpaceNode[]; unresolvedContentLinks: ValidationResult['unresolvedContentLinks']; coverageGates: CoverageGate[]; + contextTrimmed: ValidationResult['contextTrimmed']; } /** @@ -274,6 +286,7 @@ async function validateNodes( orphans: [], unresolvedContentLinks: [], coverageGates: [], + contextTrimmed: { nodeCount: 0, ruleIds: [] }, }; for (const node of nodes) { @@ -338,9 +351,11 @@ async function validateNodes( }), ...result.hierarchyViolations.map((v): GatingSeed => ({ file: v.file, title: v.nodeTitle, reason: 'hierarchy' })), ]; - const { violations, gates } = await evaluateSpaceRules(nodes, metadata, seeds, options); + const trimReport: TrimReport = { nodes: new Set(), ruleIds: new Set() }; + const { violations, gates } = await evaluateSpaceRules(nodes, metadata, seeds, { ...options, trimReport }); result.ruleViolations = violations; result.coverageGates = gates; + result.contextTrimmed = summariseTrim(trimReport); } // Collect unresolved content links (warnings — not errors) @@ -414,6 +429,7 @@ export async function validateSpace( parseIssues: readResult.parseIssues, unresolvedContentLinks: result.unresolvedContentLinks, coverageGates: result.coverageGates, + contextTrimmed: result.contextTrimmed, skippedByDepth, depthLimit: options.depth, }; diff --git a/tests/judge/provider/jev.test.ts b/tests/judge/provider/jev.test.ts index 5e8e6fcb..56bb0024 100644 --- a/tests/judge/provider/jev.test.ts +++ b/tests/judge/provider/jev.test.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, it } from 'bun:test'; import { createModelProvider } from '../../../src/judge/factory'; -import { JEV_DECISION_REQUEST_MAX_QUESTIONS, JevJudgeProvider } from '../../../src/judge/provider/jev'; +import { JEV_REQUEST_PACKING_LIMIT_QUESTIONS, JevJudgeProvider } from '../../../src/judge/provider/jev'; import { graded, node, passFailRule } from '../helpers'; const realFetch = globalThis.fetch; @@ -144,11 +144,11 @@ describe('JevJudgeProvider', () => { it('splits a NodeRulesetGroup by the question cap without splitting a node', async () => { const bodies = answering(); - const count = JEV_DECISION_REQUEST_MAX_QUESTIONS + 5; + const count = JEV_REQUEST_PACKING_LIMIT_QUESTIONS + 5; const results = await jev().evaluateNodes( Array.from({ length: count }, (_, i) => ({ node: nodeNamed(`N${i}`), rules: [passFailRule] })), ); - expect(bodies.map((b) => Object.keys(b.questions).length)).toEqual([JEV_DECISION_REQUEST_MAX_QUESTIONS, 5]); + expect(bodies.map((b) => Object.keys(b.questions).length)).toEqual([JEV_REQUEST_PACKING_LIMIT_QUESTIONS, 5]); expect(results).toHaveLength(count); expect(results.every((r) => 'verdict' in r['customer-voice']!)).toBe(true); }); @@ -199,10 +199,10 @@ describe('JevJudgeProvider', () => { it('splits a single node with more rules than the question cap across requests, sending its state whole each time', async () => { const bodies = answering(); - const count = JEV_DECISION_REQUEST_MAX_QUESTIONS + 3; + const count = JEV_REQUEST_PACKING_LIMIT_QUESTIONS + 3; const rules = Array.from({ length: count }, (_, i) => ({ ...passFailRule, id: `r${i}` })); const results = await jev().evaluateNodes([{ node: nodeNamed('A'), rules }]); - expect(bodies.map((b) => Object.keys(b.questions).length)).toEqual([JEV_DECISION_REQUEST_MAX_QUESTIONS, 3]); + expect(bodies.map((b) => Object.keys(b.questions).length)).toEqual([JEV_REQUEST_PACKING_LIMIT_QUESTIONS, 3]); expect(bodies.every((b) => b.state.nodes === undefined)).toBe(true); expect(Object.keys(results[0]!)).toHaveLength(count); expect(Object.values(results[0]!).every((r) => 'verdict' in r)).toBe(true); diff --git a/tests/judge/renderers.test.ts b/tests/judge/renderers.test.ts new file mode 100644 index 00000000..78b82554 --- /dev/null +++ b/tests/judge/renderers.test.ts @@ -0,0 +1,78 @@ +import { afterEach, describe, expect, it } from 'bun:test'; +import { buildJudgeUserPrompt } from '../../src/judge/prompt'; +import { JevJudgeProvider } from '../../src/judge/provider/jev'; +import { OpenAIDecisionsJudgeProvider } from '../../src/judge/provider/openai-decisions'; +import { trimMarker } from '../../src/judge/trim'; +import type { SemanticJudgeInput } from '../../src/judge/types'; +import { resolveSemanticRule } from '../../src/judge/verdicts'; +import type { Rule } from '../../src/types'; +import { node } from './helpers'; + +const realFetch = globalThis.fetch; +afterEach(() => { + globalThis.fetch = realFetch; +}); + +/** Capture the one request a provider sends and return its body as text. */ +function captureRequest(answers: unknown): { body: () => string } { + let sent = ''; + globalThis.fetch = (async (_url: string, init: RequestInit) => { + sent = String(init.body); + return new Response(JSON.stringify({ answers }), { status: 200 }); + }) as unknown as typeof fetch; + return { body: () => sent }; +} + +const scoped = resolveSemanticRule({ + id: 'coherent', + category: 'coherence', + description: 'The node fits its parent', + semantic: { scope: ['parent'], includeContent: 'fullScope' }, +} as unknown as Rule); + +const marker = trimMarker(1234); +const input: SemanticJudgeInput = { + node, + related: [ + { + relations: ['parent'], + title: 'Faster Tax Filing', + type: 'goal', + data: { owner: 'finance', content: `Goal body ${marker}` }, + }, + ], + rules: [scoped.spec], +}; + +/** Every renderer shows the relation, the trim marker inside a body, and the `missing-context` option. */ +describe('provider renderers', () => { + const cases: Array<[string, () => Promise]> = [ + ['chat prompt', async () => buildJudgeUserPrompt(input, input.rules[0]!)], + [ + 'openai-decisions', + async () => { + const request = captureRequest([]); + await new OpenAIDecisionsJudgeProvider({ api: 'openai-decisions', apiKey: 'k' }).evaluate(input); + return request.body(); + }, + ], + [ + 'jev', + async () => { + const request = captureRequest({}); + await new JevJudgeProvider({ api: 'openrouter-decisions', apiKey: 'k' }).evaluate(input); + return request.body(); + }, + ], + ]; + + for (const [name, render] of cases) { + it(`${name} shows the relation, trim marker and missing-context option`, async () => { + const rendered = await render(); + expect(rendered).toContain('parent'); + expect(rendered).toContain('Faster Tax Filing'); + expect(rendered).toContain(marker); + expect(rendered).toContain('missing-context'); + }); + } +}); diff --git a/tests/judge/trim.test.ts b/tests/judge/trim.test.ts new file mode 100644 index 00000000..7bd566d7 --- /dev/null +++ b/tests/judge/trim.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from 'bun:test'; +import { cutBody, estimateTokens, type NodeInput, trimMarker, trimToCap } from '../../src/judge/trim'; +import { node } from './helpers'; + +const words = (n: number) => Array.from({ length: n }, (_, i) => `w${i}`).join(' '); +const input = (nodeBody: string | undefined, relatedBodies: string[]): NodeInput => ({ + node: { ...node, schemaData: { ...node.schemaData, ...(nodeBody !== undefined && { content: nodeBody }) } }, + related: relatedBodies.map((content, i) => ({ + relations: ['child'], + title: `Related ${i}`, + type: 'solution', + data: { owner: 'team', content }, + })), +}); +const relatedBody = (r: NodeInput, i: number) => r.related?.[i]?.data.content as string; + +describe('cutBody', () => { + it('leaves a body within the limit whole', () => { + expect(cutBody('one two', 7)).toBe('one two'); + }); + + it('ends at the last whitespace before the limit and counts the characters removed', () => { + const out = cutBody('alpha beta gamma', 12); + expect(out).toBe(`alpha beta ${trimMarker(6)}`); + }); + + it('keeps a word that ends exactly at the limit', () => { + expect(cutBody('alpha beta gamma', 10)).toBe(`alpha beta ${trimMarker(6)}`); + }); + + it('leaves just the marker when nothing is kept', () => { + expect(cutBody('alpha beta', 0)).toBe(trimMarker(10)); + expect(cutBody('alphabeta', 4)).toBe(trimMarker(9)); + }); +}); + +describe('trimToCap', () => { + it('does nothing under the cap', () => { + const i = input('body', ['rel']); + expect(trimToCap(i, 10_000)).toEqual({ input: i, trimmed: false, overCap: false }); + }); + + it('cuts the longest related bodies to one ceiling and keeps shorter ones whole', () => { + const short = words(10); + const i = input('own', [words(2000), short, words(1500)]); + const cap = Math.ceil(estimateTokens(i) / 2); + const out = trimToCap(i, cap); + expect(out.trimmed).toBe(true); + expect(out.overCap).toBe(false); + expect(estimateTokens(out.input)).toBeLessThanOrEqual(cap); + expect(relatedBody(out.input, 1)).toBe(short); + expect(relatedBody(out.input, 0)).toContain('[RULE_EVALUATION_CONTENT_TRIMMED'); + expect(relatedBody(out.input, 2)).toContain('[RULE_EVALUATION_CONTENT_TRIMMED'); + expect(out.input.node.schemaData.content).toBe('own'); + expect(out.input.related).toHaveLength(3); + }); + + it('cuts the evaluated node body only once related bodies are exhausted', () => { + const own = words(3000); + const i = input(own, [words(50)]); + const withoutBodies = estimateTokens(input('', [''])); + const out = trimToCap(i, withoutBodies + 200); + expect(out.overCap).toBe(false); + expect(relatedBody(out.input, 0)).toBe(trimMarker(words(50).length)); + expect(String(out.input.node.schemaData.content)).toContain('[RULE_EVALUATION_CONTENT_TRIMMED'); + expect(String(out.input.node.schemaData.content).length).toBeLessThan(own.length); + }); + + it('never cuts fields or drops nodes, and reports an input that still does not fit', () => { + const i = input(words(100), [words(100)]); + const out = trimToCap(i, 10); + expect(out.overCap).toBe(true); + expect(out.input.related).toHaveLength(1); + expect(out.input.related?.[0]?.data.owner).toBe('team'); + expect(out.input.node.schemaData.source).toBe(node.schemaData.source); + }); + + it('does not mutate its input', () => { + const i = input(words(100), [words(1000)]); + const before = JSON.stringify(i); + trimToCap(i, 500); + expect(JSON.stringify(i)).toBe(before); + }); +}); diff --git a/tests/judge/trimming.test.ts b/tests/judge/trimming.test.ts new file mode 100644 index 00000000..77877181 --- /dev/null +++ b/tests/judge/trimming.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, it } from 'bun:test'; +import { judgeNodes } from '../../src/judge/judge-node'; +import { FakeJudgeProvider } from '../../src/judge/provider/fake'; +import { PER_NODE_INPUT_CAP_TOKENS, type TrimReport } from '../../src/judge/trim'; +import { resolveSemanticRule } from '../../src/judge/verdicts'; +import { buildSpaceGraph } from '../../src/space-graph'; +import type { Rule } from '../../src/types'; +import { makeHierarchy, makeLevel, makeParentRef, makeSpaceNode } from '../test-helpers'; + +const hierarchy = makeHierarchy([makeLevel('goal'), makeLevel('opportunity')]); +const text = (tokens: number) => 'word '.repeat(tokens * 0.8); +const goal = (body: string) => makeSpaceNode('Goal', 'goal', { content: body }); +const opp = (body: string) => + makeSpaceNode('Opp', 'opportunity', { content: body }, undefined, [makeParentRef('Goal')]); +const rule = (id: string, semantic: object) => + ({ id, category: 'coherence', description: `${id} desc`, semantic }) as unknown as Rule; +const report = (): TrimReport => ({ nodes: new Set(), ruleIds: new Set() }); + +describe('per-node input cap in judgeNodes', () => { + it('trims related bodies, still calls the provider, and records the node and rule', async () => { + const nodes = [goal(text(PER_NODE_INPUT_CAP_TOKENS * 2)), opp('short own body')]; + const fake = new FakeJudgeProvider(); + const trimmed = report(); + const rules = [rule('r', { scope: ['parent'], includeContent: 'fullScope' })]; + const out = await judgeNodes(fake, [{ node: nodes[1]!, rules }], buildSpaceGraph(nodes, hierarchy), trimmed); + expect(out[0]?.[0]?.failed).toBe(false); + expect(fake.callHistory).toHaveLength(1); + expect(String(fake.callHistory[0]?.related?.[0]?.data.content)).toContain('[RULE_EVALUATION_CONTENT_TRIMMED'); + expect(fake.callHistory[0]?.node.schemaData.content).toBe('short own body'); + expect([...trimmed.nodes]).toEqual(['Opp.md']); + expect([...trimmed.ruleIds]).toEqual(['r']); + }); + + it('records nothing when the input fits', async () => { + const nodes = [goal('short'), opp('short')]; + const trimmed = report(); + const rules = [rule('r', { scope: ['parent'], includeContent: 'fullScope' })]; + await judgeNodes(new FakeJudgeProvider(), [{ node: nodes[1]!, rules }], buildSpaceGraph(nodes, hierarchy), trimmed); + expect(trimmed.nodes.size).toBe(0); + }); + + it('reports an error under the rule id, without calling the provider, when fields alone exceed the cap', async () => { + const huge = makeSpaceNode('Huge', 'opportunity', { notes: text(PER_NODE_INPUT_CAP_TOKENS * 2) }); + const fake = new FakeJudgeProvider(); + const out = await judgeNodes(fake, [{ node: huge, rules: [rule('r', {})] }], buildSpaceGraph([huge], hierarchy)); + expect(fake.callHistory).toHaveLength(0); + expect(out[0]?.[0]).toMatchObject({ + failed: true, + error: 'input over the per-node cap after trimming content', + }); + }); + + it('judges the other rules of a node whose scoped input is over the cap', async () => { + const huge = makeSpaceNode('Huge', 'opportunity', { notes: text(PER_NODE_INPUT_CAP_TOKENS * 2) }); + const fake = new FakeJudgeProvider(); + const rules = [rule('a', { scope: ['parent'] }), rule('b', {})]; + const out = await judgeNodes(fake, [{ node: huge, rules }], buildSpaceGraph([huge], hierarchy)); + expect(out[0]?.every((j) => j.error?.includes('per-node cap'))).toBe(true); + }); +}); + +describe('missing-context verdict', () => { + const resolved = (semantic: object) => resolveSemanticRule(rule('r', semantic)); + + it('is offered on a rule with a scope or includeContent, as a warning by default', () => { + for (const semantic of [{ scope: ['parent'] }, { includeContent: 'nodeOnly' }]) { + const r = resolved(semantic); + expect(Object.keys(r.spec.verdicts)).toEqual(['pass', 'fail', 'missing-context']); + expect(r.severities['missing-context']).toBe('warning'); + } + }); + + it('is not offered on a rule with neither', () => { + expect(Object.keys(resolved({}).spec.verdicts)).toEqual(['pass', 'fail']); + }); + + it('takes the severity from semantic.severity, including ignore', () => { + expect(resolved({ scope: ['parent'], severity: { 'missing-context': 'info' } }).severities['missing-context']).toBe( + 'info', + ); + expect( + resolved({ scope: ['parent'], severity: { 'missing-context': 'ignore' } }).severities['missing-context'], + ).toBeUndefined(); + }); + + it('is told to the provider as a verdict about trim markers and resolves to a violation', async () => { + const nodes = [goal('g'), opp('o')]; + const fake = new FakeJudgeProvider({ verdict: 'missing-context', evidence: 'cut' }); + const out = await judgeNodes( + fake, + [{ node: nodes[1]!, rules: [rule('r', { scope: ['parent'] })] }], + buildSpaceGraph(nodes, hierarchy), + ); + expect(fake.callHistory[0]?.rules[0]?.verdicts['missing-context']).toContain('RULE_EVALUATION_CONTENT_TRIMMED'); + expect(out[0]?.[0]).toMatchObject({ failed: true, verdict: 'missing-context', severity: 'warning' }); + }); +}); diff --git a/tests/validate/semantic-validation.test.ts b/tests/validate/semantic-validation.test.ts index e9f14979..4c8d2a6b 100644 --- a/tests/validate/semantic-validation.test.ts +++ b/tests/validate/semantic-validation.test.ts @@ -1,7 +1,9 @@ import { afterAll, beforeAll, describe, expect, it } from 'bun:test'; +import { writeFileSync } from 'node:fs'; import { join } from 'node:path'; import type { Config } from '../../src/config'; import { FakeJudgeProvider } from '../../src/judge/provider/fake'; +import { PER_NODE_INPUT_CAP_TOKENS } from '../../src/judge/trim'; import { bundledSchemasDir } from '../../src/schema/schema'; import { createSpaceContext } from '../../src/space-context'; import type { FileValidationResult } from '../../src/types'; @@ -229,3 +231,30 @@ describe('Semantic rules: verdicts, batching and rule filter', () => { expect(expressionOnly.ruleViolations.some((v) => v.verdict !== undefined)).toBe(false); }); }); + +describe('context trimming in a run', () => { + const scopedRule = { + id: 'task-fits-goal', + category: 'coherence', + description: 'The task serves its goal', + type: 'task', + semantic: { scope: ['parent'], includeContent: 'fullScope' }, + }; + let space: TempSpace; + beforeAll(() => { + space = makeTempSpace([scopedRule]); + writeFileSync(space.file('G.md'), `---\ntype: goal\n---\n${'word '.repeat(PER_NODE_INPUT_CAP_TOKENS * 2)}\n`); + }); + afterAll(() => space.cleanup()); + + it('reports how many nodes were trimmed and for which rules', async () => { + const context = createSpaceContext('ts', space.config, { configDir: space.dir }); + const result = await validateSpace(context, { judge: new FakeJudgeProvider({ pass: true }) }); + expect(result.contextTrimmed).toEqual({ nodeCount: 1, ruleIds: ['task-fits-goal'] }); + }); + + it('reports nothing without a judge', async () => { + const context = createSpaceContext('ts', space.config, { configDir: space.dir }); + expect((await validateSpace(context)).contextTrimmed).toEqual({ nodeCount: 0, ruleIds: [] }); + }); +}); From fc6711afb4dcd4f23e9ce87f21c52a9f9ed2d1e9 Mon Sep 17 00:00:00 2001 From: Roger Barnes Date: Fri, 16 Oct 2026 14:43:06 +1100 Subject: [PATCH 2/6] docs: describe the per-node input cap, trimming and missing-context --- AGENTS.md | 2 +- docs/concepts.md | 4 +++ docs/semantic-rules.md | 33 +++++++++++++++++-- plugin/.claude-plugin/plugin.json | 2 +- .../references/schema-authoring.md | 2 +- 5 files changed, 38 insertions(+), 5 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index a564c6c8..0a187c80 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -59,7 +59,7 @@ There are several places that need reviewing and updating with any new feature o - `src/schema/metadata-contract.ts` — Single source of truth for the `$metadata` contract - `schemas/generated/latest/_structured_context_schema_meta.json` — Generated metaschema for the newest dialect (generated on build or with `bun run generate:schema-meta`). `schemas/generated//` folders are frozen copies, created by the `version` script (`generate:schema-meta --release`) only when a release changes the dialect; never edit them. `src/schema/meta-schemas.ts` loads them at runtime. - `src/schema/gating.ts` — Cascade gating: computes hierarchy depth and evaluates rules with subtree gating (schema/hierarchy/blocking-rule failures suppress downstream rule evaluation). See [docs/rules.md](docs/rules.md#cascade-gating). -- `src/judge/` — Semantic rule evaluation (the "judge" is its internal name): provider interface and types (`types.ts`), verdict resolution (`verdicts.ts`), per-node and grouped dispatch (`judge-node.ts`), related-node selection for `semantic.scope` (`context.ts`), and providers (`provider/anthropic.ts`, `provider/openai.ts`, `provider/jev.ts`, `provider/fake.ts`), provider hydration and the bundled catalogue (`factory.ts`), and `sctx providers` (`src/commands/providers.ts`). See [docs/semantic-rules.md](docs/semantic-rules.md). +- `src/judge/` — Semantic rule evaluation (the "judge" is its internal name): provider interface and types (`types.ts`), verdict resolution (`verdicts.ts`), per-node and grouped dispatch (`judge-node.ts`), related-node selection for `semantic.scope` (`context.ts`), the per-node input cap and body trimming (`trim.ts`), and providers (`provider/anthropic.ts`, `provider/openai.ts`, `provider/jev.ts`, `provider/fake.ts`), provider hydration and the bundled catalogue (`factory.ts`), and `sctx providers` (`src/commands/providers.ts`). See [docs/semantic-rules.md](docs/semantic-rules.md). - `src/schema/rule-filter.ts` — `--rule-filter` parsing and rule filtering by category, severity, type or id. ## Testing diff --git a/docs/concepts.md b/docs/concepts.md index 55d3f3fc..9b60acec 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -183,6 +183,10 @@ A **decision request** is one call to a decisions API. It carries the state of o A **decision question** is one choice question about one node under one semantic rule, inside a decision request. +The **per-node input cap** limits the input for one evaluated node: the node and its related nodes, measured in estimated tokens. It is the same for every model provider. When the input is over the cap, structured-context trims node bodies. See [Input size and trimming](semantic-rules.md#input-size-and-trimming). + +The **request packing limit** limits how much a model provider packs into one decision request: estimated input tokens and decision questions. It decides how a node ruleset group is split into decision requests, never within a node. It does not limit the input for a single node. + ### Cascade gating **Cascade gating** is the mechanism by which a schema failure, hierarchy structure violation, or blocking rule violation on a node suppresses rule evaluation for that node's `hierarchy` subtree. It exists so downstream checks — especially costly semantic (model-judged) rules — aren't run against inputs already known to be invalid. diff --git a/docs/semantic-rules.md b/docs/semantic-rules.md index a3704792..d29ea05b 100644 --- a/docs/semantic-rules.md +++ b/docs/semantic-rules.md @@ -146,13 +146,42 @@ Scope sends node data only, never the verdicts of other rules, and [gating](rule A global rule cannot have a `semantic` part, so it has no scope. +## Input size and trimming + +The input for one evaluated node is its data and its related nodes. It is limited by the **per-node input cap**: 32,000 estimated tokens (characters divided by 4, over the node data as JSON and the bodies). The cap is the same for every model provider and cannot be configured. The estimate is rough on purpose. + +When the input is over the cap, structured-context cuts node bodies and nothing else. It never cuts a field and never drops a node, so the model always knows which related nodes exist. + +1. It cuts related bodies first. One ceiling applies to all of them: bodies longer than the ceiling are cut to it, shorter bodies stay whole. The ceiling is as high as the cap allows. +2. If the related bodies are all cut to nothing and the input is still over the cap, it cuts the evaluated node's body the same way. + +A cut ends at the last whitespace before the limit and appends a marker with the number of characters removed: + +``` +[RULE_EVALUATION_CONTENT_TRIMMED 6120 chars] +``` + +A body cut to nothing keeps the marker, so a trimmed body is always visible. + +If the input is still over the cap after every body is cut, structured-context does not call the model. It reports a violation for each rule that shares the input, with the suffix `(Semantic evaluation error: input over the per-node cap after trimming content)`. Large fields cause this; trimming does not touch them. + +### The `missing-context` verdict + +A rule with `semantic.scope` or `semantic.includeContent` also offers the model the verdict `missing-context`. The model chooses it only when a trim marker shows that content it needs was cut. A rule without either never offers it. + +`missing-context` reports a `warning` by default. Set `semantic.severity` to change it, for example `{ "missing-context": "info" }`, or `ignore` to report nothing. + +### Reporting + +A run that trims content prints one line in its summary, for example `Context trimmed for 3 node(s) across rules solution-addresses-opportunity, solution-fits-goal.` The `--json` output carries the same information as `contextTrimmed`, with `nodeCount` and `ruleIds`. + ## How semantic evaluation works - **Order.** On each node, structured-context runs each rule's [`expression`](#combining-an-expression) first. It sends the semantic parts of the rules that pass to the model provider. -- **Grouping.** A model provider on the `openrouter-decisions` or `typesafe-systemone` API evaluates each node ruleset group (see [concepts](concepts.md#semantic-rules)) in as few decision requests as it can, with one decision question per node and rule. This includes the bundled `openrouter-openai-decisions` provider. A decision request is limited to about 32,000 estimated input tokens and 40 decision questions, so a larger group is split into several decision requests, never within a node. A node that fills a decision request on tokens is sent whole as its own decision request. A node with more than 40 rules has its rules split across decision requests. If a node's questions are split across several requests and one fails, all of that node's rules are reported as errors. Other providers, including `openai-decisions` and chat models, evaluate one node per request. Within a space, the nodes at one hierarchy depth are evaluated together, because their verdicts decide whether the next depth is evaluated. Grouping can shift verdicts and probabilities slightly compared with evaluating each node alone. +- **Grouping.** A model provider on the `openrouter-decisions` or `typesafe-systemone` API evaluates each node ruleset group (see [concepts](concepts.md#semantic-rules)) in as few decision requests as it can, with one decision question per node and rule. This includes the bundled `openrouter-openai-decisions` provider. The request packing limit is about 32,000 estimated input tokens and 40 decision questions per decision request, measured on the input after [trimming](#input-size-and-trimming). A larger group is split into several decision requests, never within a node. A node that fills a decision request on tokens is sent whole as its own decision request. A node with more than 40 rules has its rules split across decision requests. If a node's questions are split across several requests and one fails, all of that node's rules are reported as errors. Other providers, including `openai-decisions` and chat models, evaluate one node per request. Within a space, the nodes at one hierarchy depth are evaluated together, because their verdicts decide whether the next depth is evaluated. Grouping can shift verdicts and probabilities slightly compared with evaluating each node alone. - **Gating.** structured-context does not evaluate the semantic part on a gated node. [Coverage output](rules.md#coverage-output) shows which nodes were gated. - **Evaluation errors.** If the model provider fails (for example, a network error or output that structured-context cannot read), structured-context reports a violation with the suffix `(Semantic evaluation error: ...)`. An evaluation error does not gate. If one decision request fails, only the rules on the nodes in that decision request report the error. -- **Input.** The model provider gets the node's title, type, parent titles and schema data, including frontmatter. It gets the node's body, and any [related nodes](#related-nodes-and-content), only as the rule's `semantic.scope` and `semantic.includeContent` ask. +- **Input.** The model provider gets the node's title, type, parent titles and schema data, including frontmatter. It gets the node's body, and any [related nodes](#related-nodes-and-content), only as the rule's `semantic.scope` and `semantic.includeContent` ask, [trimmed](#input-size-and-trimming) to the per-node input cap. - **Output.** A violation from a semantic verdict also includes the verdict, the model's evidence and any guidance. `sctx validate-file` evaluates the semantic part only for the file and the rules on its ancestors that can gate it. Thus one file costs at most one model call for each ancestor level, plus one. diff --git a/plugin/.claude-plugin/plugin.json b/plugin/.claude-plugin/plugin.json index 2d6afc57..17090296 100644 --- a/plugin/.claude-plugin/plugin.json +++ b/plugin/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "structured-context", - "version": "0.12.17", + "version": "0.12.18", "description": "Structured content authoring for Obsidian — automatic validation hooks, slash commands, and agent skills to keep markdown files compliant with your schema.", "homepage": "https://github.com/mindsocket/structured-context", "repository": "https://github.com/mindsocket/structured-context", diff --git a/plugin/skills/structured-context/references/schema-authoring.md b/plugin/skills/structured-context/references/schema-authoring.md index 58293e58..5d1469a5 100644 --- a/plugin/skills/structured-context/references/schema-authoring.md +++ b/plugin/skills/structured-context/references/schema-authoring.md @@ -66,7 +66,7 @@ Relationships define how sub-entities (nodes inside other files) are parsed and Rules are a flat array. Each rule's `category` (`validation`, `coherence`, `workflow`, `best-practice`) supplies its default severity: `validation` → `error`, `coherence` and `workflow` → `warning`, `best-practice` → `info`. Only `error` violations fail validation; warnings and info are reported only. An optional per-rule `severity` (`error` | `warning` | `info`) overrides the category default — prefer picking the right category over overriding. -A rule has an `expression` (a JSONata test that must return `true`), a `semantic` object (a model evaluates the node against a `rubric`, which falls back to `description`), or both; a rule with neither is rejected. `global: true` runs an `expression` once for the space (no `when`, `type` or `semantic`). `semantic: {}` takes every default; it may declare `verdicts` (verdict → description, or `{ description, guidance }`; default `pass`/`fail`) with a `semantic.severity` map saying which verdicts are `error`/`warning`/`info`/`ignore` (required with custom verdicts). `semantic.scope` (a list of `parent`, `siblings`, `children`, `ancestors`, `descendants`, `branch` (ancestors plus descendants) along the main hierarchy) sends related nodes with the evaluated node; `semantic.includeContent` (`nodeOnly` or `fullScope`/`true`) sends node bodies, and without it no body is sent. The top-level `severity` is that of a failed `expression` and of the default `fail` verdict. A rule's `expression` always runs and, when it fails, the `semantic` part is skipped; the `semantic` part runs only with `--semantic`. Run `sctx docs semantic-rules` for the full semantic rule reference. A `when` JSONata test (the applicability condition, any rule) limits a rule to the nodes where it returns true. An optional `gatesDescendants` controls whether a failure suppresses rules on the node's subtree (default: true for `error` severity). +A rule has an `expression` (a JSONata test that must return `true`), a `semantic` object (a model evaluates the node against a `rubric`, which falls back to `description`), or both; a rule with neither is rejected. `global: true` runs an `expression` once for the space (no `when`, `type` or `semantic`). `semantic: {}` takes every default; it may declare `verdicts` (verdict → description, or `{ description, guidance }`; default `pass`/`fail`) with a `semantic.severity` map saying which verdicts are `error`/`warning`/`info`/`ignore` (required with custom verdicts). `semantic.scope` (a list of `parent`, `siblings`, `children`, `ancestors`, `descendants`, `branch` (ancestors plus descendants) along the main hierarchy) sends related nodes with the evaluated node; `semantic.includeContent` (`nodeOnly` or `fullScope`/`true`) sends node bodies, and without it no body is sent; input over the per-node input cap (32k estimated tokens) has bodies trimmed with a `[RULE_EVALUATION_CONTENT_TRIMMED n chars]` marker, and such a rule also offers the `missing-context` verdict (default `warning`, set in `semantic.severity`). The top-level `severity` is that of a failed `expression` and of the default `fail` verdict. A rule's `expression` always runs and, when it fails, the `semantic` part is skipped; the `semantic` part runs only with `--semantic`. Run `sctx docs semantic-rules` for the full semantic rule reference. A `when` JSONata test (the applicability condition, any rule) limits a rule to the nodes where it returns true. An optional `gatesDescendants` controls whether a failure suppresses rules on the node's subtree (default: true for `error` severity). ## Metadata composition semantics From 0b523f97db324684c656e6624a8472e0dd858d16 Mon Sep 17 00:00:00 2001 From: Roger Barnes Date: Fri, 16 Oct 2026 14:45:15 +1100 Subject: [PATCH 3/6] docs: link the wide-scope recommendation to input trimming --- docs/semantic-rules.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/semantic-rules.md b/docs/semantic-rules.md index d29ea05b..ce7cc85c 100644 --- a/docs/semantic-rules.md +++ b/docs/semantic-rules.md @@ -126,7 +126,7 @@ By default the model sees the evaluated node alone. Use `semantic.scope` to send Entries add together, and each node is sent once. An empty or absent `scope` sends the evaluated node alone. Other relationships are not followed. -A wide scope can send many nodes, and every node sends all its schema data. Prefer to attach a rule at the level where `parent`, `children` or `ancestors` covers what it needs, rather than `descendants` or `branch` on a node near the top of a large tree. +A wide scope can send many nodes, and every node sends all its schema data, which [trimming](#input-size-and-trimming) never cuts. Prefer to attach a rule at the level where `parent`, `children` or `ancestors` covers what it needs, rather than `descendants` or `branch` on a node near the top of a large tree. Scope sends node data only, never the verdicts of other rules, and [gating](rules.md#cascade-gating) does not change which nodes it selects. From ab605336108163e6be00dd7a8ca117ac5f54c150 Mon Sep 17 00:00:00 2001 From: Roger Barnes Date: Fri, 16 Oct 2026 15:24:26 +1100 Subject: [PATCH 4/6] test(docs): check that backticked repo paths in docs and skills exist Adds a test that scans README, AGENTS, CLAUDE, top-level docs and plugin markdown for backticked repo paths and fails on any that do not resolve. Lines containing an example marker (e.g., eg, example) are skipped, as are placeholders and gitignored paths. Fixes the four stale paths it found. --- docs/schema-dialect-update-proposal.md | 8 +-- docs/torres-alignment-research.md | 2 +- plugin/.claude-plugin/plugin.json | 2 +- .../references/schema-authoring.md | 2 +- tests/docs/references.test.ts | 72 +++++++++++++++++++ 5 files changed, 79 insertions(+), 7 deletions(-) create mode 100644 tests/docs/references.test.ts diff --git a/docs/schema-dialect-update-proposal.md b/docs/schema-dialect-update-proposal.md index 5821d933..3caa59f0 100644 --- a/docs/schema-dialect-update-proposal.md +++ b/docs/schema-dialect-update-proposal.md @@ -29,7 +29,7 @@ Issue #28 opened the door to reorganize schema metadata, rules, and composition - Metadata is composed across `$ref` graph (DFS, root last) with deterministic merge semantics. *(Superseded by #121: metadata is composed only through explicit `$metadata.imports`; `$ref` is validation-only. See [schemas.md](schemas.md#composition-and-merge-semantics).)* - Rule conflict policy is error by default; explicit `override: true` enables replacement. - `$metadata.rules` supports `$ref` imports for specific rules and rule sets. *(Removed by #121: import a rules-only schema via `$metadata.imports` instead.)* -- Metadata contract source is `src/metadata-contract.ts` (`json-schema-to-ts` types), with generated metaschema artifact in `schemas/generated/_ost_tools_schema_meta.json`. +- Metadata contract source is `src/schema/metadata-contract.ts` (`json-schema-to-ts` types), with generated metaschema artifact in `schemas/generated/latest/_structured_context_schema_meta.json`. ## Decision Areas and Recommendations @@ -322,10 +322,10 @@ Use TS-authored metadata contract as the source of truth (`as const`) and infer Shipped workflow: -- Author metadata schema contract in `src/metadata-contract.ts`. +- Author metadata schema contract in `src/schema/metadata-contract.ts`. - Infer TS types from that contract via `json-schema-to-ts`. - Validate runtime metadata via AJV using the same contract object. -- Generate `schemas/generated/_ost_tools_schema_meta.json` from the in-code contract (`bun run generate:schema-meta`). +- Generate `schemas/generated/latest/_structured_context_schema_meta.json` from the in-code contract (`bun run generate:schema-meta`). This keeps one authoritative contract while still producing a portable schema artifact for tooling. @@ -485,7 +485,7 @@ Stage D acceptance criteria: - **Breaking**: rule declarations migrated to flat `rules[]` with per-rule `category`. - **New**: metadata composition across `$ref` graph with deterministic merge semantics. - **New**: rule imports via `$metadata.rules` `$ref` entries and explicit rule override (`override: true`) behavior. -- **Internal**: metadata contract source consolidated in `src/metadata-contract.ts` with generated metaschema artifact. +- **Internal**: metadata contract source consolidated in `src/schema/metadata-contract.ts` with generated metaschema artifact. ## Open questions for decision diff --git a/docs/torres-alignment-research.md b/docs/torres-alignment-research.md index ab3fa531..d715f8ed 100644 --- a/docs/torres-alignment-research.md +++ b/docs/torres-alignment-research.md @@ -173,7 +173,7 @@ My thoughts so far: "significant gap if the tool intends to model a complete OST per Torres" * I think we should also explore alternative schemas. We could `mkdir schemas && mv schema.json schemas/general.json` then add a `schemas/strict_ost.json` for the Torres version (limited parentage, opinionated names from Torres' work). We have '--schema' scafolding already. We could promote the schema in config.json to space level, and update the default so - `schemas/general.json` is used - partly for backward compatibility but mainly because I have stronger opinions about structure. Put common `refs` in a shared file to avoid repetition. + `schemas/strategy_general.json` is used - partly for backward compatibility but mainly because I have stronger opinions about structure. Put common `refs` in a shared file to avoid repetition. * Side note - there doesn't appear to be an IP (trademark) issue with "OST". We should be respectful and attribute Torres' work, while calling out these tools' applicability to a broader range of frameworks. The project already wants to explore adjacent and composite models anyway. * Taking that idea one further "general.json" might really be strategy_plus_ost.json or something. diff --git a/plugin/.claude-plugin/plugin.json b/plugin/.claude-plugin/plugin.json index 17090296..76ab923b 100644 --- a/plugin/.claude-plugin/plugin.json +++ b/plugin/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "structured-context", - "version": "0.12.18", + "version": "0.12.19", "description": "Structured content authoring for Obsidian — automatic validation hooks, slash commands, and agent skills to keep markdown files compliant with your schema.", "homepage": "https://github.com/mindsocket/structured-context", "repository": "https://github.com/mindsocket/structured-context", diff --git a/plugin/skills/structured-context/references/schema-authoring.md b/plugin/skills/structured-context/references/schema-authoring.md index 5d1469a5..e19e56e5 100644 --- a/plugin/skills/structured-context/references/schema-authoring.md +++ b/plugin/skills/structured-context/references/schema-authoring.md @@ -1,7 +1,7 @@ # Schema Authoring Reference Schema files use a 2020-12-based dialect with top-level `$metadata`. -See `~/src/structured-context/schemas/` for examples (`strategy_general.json`, `strict_ost.json`, `_ost_strict.json`). +See the bundled schemas shipped with `sctx` for examples (`strategy_general.json`, `strict_ost.json`, `_ost_strict.json`). ## `$metadata` (top-level) diff --git a/tests/docs/references.test.ts b/tests/docs/references.test.ts new file mode 100644 index 00000000..0010ce55 --- /dev/null +++ b/tests/docs/references.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from 'bun:test'; +import { execFileSync } from 'node:child_process'; +import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs'; +import { join, resolve } from 'node:path'; + +/** + * Guards against path rot in the docs and skills that point readers at files in this repo. + * A stale path sends the reader somewhere that no longer exists. + * + * This checks that a path resolves, not that the claim around it is true. + * Historical records (docs/plans) are out of scope: they cite paths that have since moved. + */ + +const ROOT = resolve(import.meta.dir, '../..'); + +/** Docs that describe the current tree: agent instructions, README, top-level docs, and plugin skills. */ +const SCANNED = ['README.md', 'AGENTS.md', 'CLAUDE.md', 'docs', 'plugin']; + +/** Repo-relative paths appear in backticks; these are the roots worth checking. */ +const PATH_PATTERN = /`\/?((?:src|tests|schemas|plugin|docs|scripts|hook-test|smoke)\/[^`\s]*)`/g; + +/** Illustrative paths, not real ones: placeholders, globs, and naming-convention examples. */ +const PLACEHOLDER = /YYYY|<[^>]+>|\[[^\]]+\]|\*|PascalCase|kebab-case|\bNN\b|\.\.\./; + +/** A line that gives an example or aside. Paths on these lines are illustrative, not promises. */ +const EXAMPLE_LINE = /\b(e\.g\.?|eg|i\.e\.?|example|examples|for instance)\b/i; + +function markdownFilesUnder(relPath: string): string[] { + const abs = join(ROOT, relPath); + if (!existsSync(abs)) return []; + if (statSync(abs).isFile()) return relPath.endsWith('.md') ? [relPath] : []; + return readdirSync(abs) + .filter((entry) => entry !== 'node_modules') + .flatMap((entry) => markdownFilesUnder(join(relPath, entry))); +} + +const docs = SCANNED.flatMap(markdownFilesUnder).filter((doc) => !doc.startsWith('docs/plans/')); + +/** Gitignored paths (generated DBs, build output) never exist in a fresh checkout. */ +function isGitignored(relPath: string): boolean { + try { + execFileSync('git', ['check-ignore', '-q', relPath], { cwd: ROOT }); + return true; + } catch { + return false; + } +} + +describe('documentation file references', () => { + it('scans the docs an agent is pointed at', () => { + expect(docs).toContain('AGENTS.md'); + expect(docs).toContain('docs/concepts.md'); + expect(docs.some((doc) => doc.startsWith('plugin/'))).toBe(true); + }); + + it.each(docs)('%s references only paths that exist', (doc) => { + const broken: string[] = []; + + for (const line of readFileSync(join(ROOT, doc), 'utf-8').split('\n')) { + if (EXAMPLE_LINE.test(line)) continue; + for (const match of line.matchAll(PATH_PATTERN)) { + // Trim `file.ts:12, 30` line citations and any trailing prose punctuation. + const path = (match[1] ?? '').replace(/:[\d,\s–-]+$/, '').replace(/[.,;)]+$/, ''); + if (PLACEHOLDER.test(path)) continue; + if (isGitignored(path)) continue; + if (!existsSync(join(ROOT, path))) broken.push(path); + } + } + + expect(broken, `${doc} references paths that no longer exist`).toEqual([]); + }); +}); From 9958ed4d5b544851bf1dc4ed40f45cf16dea9ab2 Mon Sep 17 00:00:00 2001 From: Roger Barnes Date: Fri, 16 Oct 2026 15:35:06 +1100 Subject: [PATCH 5/6] docs: keep semantic evaluation internals out of the user docs Drop request grouping, packing and trimming mechanics from config.md and semantic-rules.md. Keep what a user acts on: that grouping saves cost and can shift verdicts, the per-node input cap, the trim marker and the errors. --- docs/config.md | 8 ++++---- docs/semantic-rules.md | 15 ++++++--------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/docs/config.md b/docs/config.md index 1bfd52a2..7f6cf171 100644 --- a/docs/config.md +++ b/docs/config.md @@ -79,10 +79,10 @@ A provider's `api` names the **request shape**, potentially supported by multipl | API | Shape | Bundled providers | |---|---|---| -| `openrouter-decisions` | Decisions API, closed-set `choice` questions, answers with confidence and probabilities; nodes with the same rules share a decision request | `openrouter-jev`, `openrouter-openai-decisions` | -| `typesafe-systemone` | Same decisions shape (currently), including shared decision requests | `typesafe-jev` | -| `openai-decisions` | OpenAI's Decisions API: same concept, its own request/response shape; one node per request | `openai-decisions` | -| `anthropic-messages` | Chat: one request per rule, structured JSON verdict | `anthropic`, `anthropic-haiku-5-5`, `anthropic-env` | +| `openrouter-decisions` | Decisions API, closed-set `choice` questions, answers with confidence and probabilities | `openrouter-jev`, `openrouter-openai-decisions` | +| `typesafe-systemone` | Same decisions shape (currently) | `typesafe-jev` | +| `openai-decisions` | OpenAI's Decisions API: same concept, its own request/response shape | `openai-decisions` | +| `anthropic-messages` | Chat: structured JSON verdict | `anthropic`, `anthropic-haiku-5-5`, `anthropic-env` | | `openai-completions` | Chat: OpenAI-compatible; works with Ollama, vLLM and OpenRouter | `openai`, `openai-env`, `openrouter-openai` | Bundled provider names follow the pattern: diff --git a/docs/semantic-rules.md b/docs/semantic-rules.md index ce7cc85c..00a3e152 100644 --- a/docs/semantic-rules.md +++ b/docs/semantic-rules.md @@ -148,22 +148,19 @@ A global rule cannot have a `semantic` part, so it has no scope. ## Input size and trimming -The input for one evaluated node is its data and its related nodes. It is limited by the **per-node input cap**: 32,000 estimated tokens (characters divided by 4, over the node data as JSON and the bodies). The cap is the same for every model provider and cannot be configured. The estimate is rough on purpose. +The input for one evaluated node is its data and its related nodes. It is limited by the **per-node input cap** of about 32,000 tokens. The cap cannot be configured. When the input is over the cap, structured-context cuts node bodies and nothing else. It never cuts a field and never drops a node, so the model always knows which related nodes exist. -1. It cuts related bodies first. One ceiling applies to all of them: bodies longer than the ceiling are cut to it, shorter bodies stay whole. The ceiling is as high as the cap allows. -2. If the related bodies are all cut to nothing and the input is still over the cap, it cuts the evaluated node's body the same way. +It cuts the longest related bodies first. It cuts the evaluated node's body only when the related bodies are cut to nothing. -A cut ends at the last whitespace before the limit and appends a marker with the number of characters removed: +Each cut ends with a marker that gives the number of characters removed: ``` [RULE_EVALUATION_CONTENT_TRIMMED 6120 chars] ``` -A body cut to nothing keeps the marker, so a trimmed body is always visible. - -If the input is still over the cap after every body is cut, structured-context does not call the model. It reports a violation for each rule that shares the input, with the suffix `(Semantic evaluation error: input over the per-node cap after trimming content)`. Large fields cause this; trimming does not touch them. +If the input is still over the cap after every body is cut, structured-context does not call the model and reports a violation with the suffix `(Semantic evaluation error: input over the per-node cap after trimming content)`. Large fields cause this. ### The `missing-context` verdict @@ -178,9 +175,9 @@ A run that trims content prints one line in its summary, for example `Context tr ## How semantic evaluation works - **Order.** On each node, structured-context runs each rule's [`expression`](#combining-an-expression) first. It sends the semantic parts of the rules that pass to the model provider. -- **Grouping.** A model provider on the `openrouter-decisions` or `typesafe-systemone` API evaluates each node ruleset group (see [concepts](concepts.md#semantic-rules)) in as few decision requests as it can, with one decision question per node and rule. This includes the bundled `openrouter-openai-decisions` provider. The request packing limit is about 32,000 estimated input tokens and 40 decision questions per decision request, measured on the input after [trimming](#input-size-and-trimming). A larger group is split into several decision requests, never within a node. A node that fills a decision request on tokens is sent whole as its own decision request. A node with more than 40 rules has its rules split across decision requests. If a node's questions are split across several requests and one fails, all of that node's rules are reported as errors. Other providers, including `openai-decisions` and chat models, evaluate one node per request. Within a space, the nodes at one hierarchy depth are evaluated together, because their verdicts decide whether the next depth is evaluated. Grouping can shift verdicts and probabilities slightly compared with evaluating each node alone. +- **Grouping.** A model provider on a decisions API (`openrouter-decisions` or `typesafe-systemone`) evaluates nodes that share the same rules together, in as few requests as it can, to save cost. Grouping can shift verdicts and probabilities slightly compared with evaluating each node alone. - **Gating.** structured-context does not evaluate the semantic part on a gated node. [Coverage output](rules.md#coverage-output) shows which nodes were gated. -- **Evaluation errors.** If the model provider fails (for example, a network error or output that structured-context cannot read), structured-context reports a violation with the suffix `(Semantic evaluation error: ...)`. An evaluation error does not gate. If one decision request fails, only the rules on the nodes in that decision request report the error. +- **Evaluation errors.** If the model provider fails (for example, a network error or output that structured-context cannot read), structured-context reports a violation with the suffix `(Semantic evaluation error: ...)`. An evaluation error does not gate. - **Input.** The model provider gets the node's title, type, parent titles and schema data, including frontmatter. It gets the node's body, and any [related nodes](#related-nodes-and-content), only as the rule's `semantic.scope` and `semantic.includeContent` ask, [trimmed](#input-size-and-trimming) to the per-node input cap. - **Output.** A violation from a semantic verdict also includes the verdict, the model's evidence and any guidance. From 2420e40fdb85efc361df24204d1a8dfda5790da4 Mon Sep 17 00:00:00 2001 From: Roger Barnes Date: Fri, 16 Oct 2026 17:12:45 +1100 Subject: [PATCH 6/6] refactor(judge): simplify trimming plumbing and cut trimming docs to the essentials --- docs/concepts.md | 4 +- docs/semantic-rules.md | 28 ++------- plugin/.claude-plugin/plugin.json | 2 +- .../references/schema-authoring.md | 2 +- src/judge/judge-node.ts | 63 +++++++++---------- src/judge/provider/jev.ts | 5 +- src/judge/trim.ts | 26 +++----- src/judge/verdicts.ts | 4 +- src/schema/gating.ts | 1 - src/schema/validate-rules.ts | 1 - tests/judge/trim.test.ts | 18 ++---- 11 files changed, 53 insertions(+), 101 deletions(-) diff --git a/docs/concepts.md b/docs/concepts.md index 9b60acec..77fab9d0 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -183,9 +183,7 @@ A **decision request** is one call to a decisions API. It carries the state of o A **decision question** is one choice question about one node under one semantic rule, inside a decision request. -The **per-node input cap** limits the input for one evaluated node: the node and its related nodes, measured in estimated tokens. It is the same for every model provider. When the input is over the cap, structured-context trims node bodies. See [Input size and trimming](semantic-rules.md#input-size-and-trimming). - -The **request packing limit** limits how much a model provider packs into one decision request: estimated input tokens and decision questions. It decides how a node ruleset group is split into decision requests, never within a node. It does not limit the input for a single node. +The **per-node input cap** limits the input for one evaluated node and its related nodes. See [Input size and trimming](semantic-rules.md#input-size-and-trimming). ### Cascade gating diff --git a/docs/semantic-rules.md b/docs/semantic-rules.md index 00a3e152..5fbadea6 100644 --- a/docs/semantic-rules.md +++ b/docs/semantic-rules.md @@ -126,7 +126,7 @@ By default the model sees the evaluated node alone. Use `semantic.scope` to send Entries add together, and each node is sent once. An empty or absent `scope` sends the evaluated node alone. Other relationships are not followed. -A wide scope can send many nodes, and every node sends all its schema data, which [trimming](#input-size-and-trimming) never cuts. Prefer to attach a rule at the level where `parent`, `children` or `ancestors` covers what it needs, rather than `descendants` or `branch` on a node near the top of a large tree. +A wide scope can send many nodes, and every node sends all its schema data. Prefer to attach a rule at the level where `parent`, `children` or `ancestors` covers what it needs, rather than `descendants` or `branch` on a node near the top of a large tree. Scope sends node data only, never the verdicts of other rules, and [gating](rules.md#cascade-gating) does not change which nodes it selects. @@ -148,29 +148,9 @@ A global rule cannot have a `semantic` part, so it has no scope. ## Input size and trimming -The input for one evaluated node is its data and its related nodes. It is limited by the **per-node input cap** of about 32,000 tokens. The cap cannot be configured. +The input for one evaluated node is capped at about 32,000 estimated tokens. Over the cap, structured-context cuts node bodies, longest related bodies first, and marks each cut with `[RULE_EVALUATION_CONTENT_TRIMMED n chars]`. If the input still does not fit, the rule reports a semantic evaluation error. A run that trims content says so in its summary. -When the input is over the cap, structured-context cuts node bodies and nothing else. It never cuts a field and never drops a node, so the model always knows which related nodes exist. - -It cuts the longest related bodies first. It cuts the evaluated node's body only when the related bodies are cut to nothing. - -Each cut ends with a marker that gives the number of characters removed: - -``` -[RULE_EVALUATION_CONTENT_TRIMMED 6120 chars] -``` - -If the input is still over the cap after every body is cut, structured-context does not call the model and reports a violation with the suffix `(Semantic evaluation error: input over the per-node cap after trimming content)`. Large fields cause this. - -### The `missing-context` verdict - -A rule with `semantic.scope` or `semantic.includeContent` also offers the model the verdict `missing-context`. The model chooses it only when a trim marker shows that content it needs was cut. A rule without either never offers it. - -`missing-context` reports a `warning` by default. Set `semantic.severity` to change it, for example `{ "missing-context": "info" }`, or `ignore` to report nothing. - -### Reporting - -A run that trims content prints one line in its summary, for example `Context trimmed for 3 node(s) across rules solution-addresses-opportunity, solution-fits-goal.` The `--json` output carries the same information as `contextTrimmed`, with `nodeCount` and `ruleIds`. +A rule with `semantic.scope` or `semantic.includeContent` can return `missing-context` when content it needs was cut. It is a `warning` by default; change it in `semantic.severity`. ## How semantic evaluation works @@ -178,7 +158,7 @@ A run that trims content prints one line in its summary, for example `Context tr - **Grouping.** A model provider on a decisions API (`openrouter-decisions` or `typesafe-systemone`) evaluates nodes that share the same rules together, in as few requests as it can, to save cost. Grouping can shift verdicts and probabilities slightly compared with evaluating each node alone. - **Gating.** structured-context does not evaluate the semantic part on a gated node. [Coverage output](rules.md#coverage-output) shows which nodes were gated. - **Evaluation errors.** If the model provider fails (for example, a network error or output that structured-context cannot read), structured-context reports a violation with the suffix `(Semantic evaluation error: ...)`. An evaluation error does not gate. -- **Input.** The model provider gets the node's title, type, parent titles and schema data, including frontmatter. It gets the node's body, and any [related nodes](#related-nodes-and-content), only as the rule's `semantic.scope` and `semantic.includeContent` ask, [trimmed](#input-size-and-trimming) to the per-node input cap. +- **Input.** The model provider gets the node's title, type, parent titles and schema data, including frontmatter. It gets the node's body, and any [related nodes](#related-nodes-and-content), only as the rule's `semantic.scope` and `semantic.includeContent` ask ([trimmed](#input-size-and-trimming) to fit). - **Output.** A violation from a semantic verdict also includes the verdict, the model's evidence and any guidance. `sctx validate-file` evaluates the semantic part only for the file and the rules on its ancestors that can gate it. Thus one file costs at most one model call for each ancestor level, plus one. diff --git a/plugin/.claude-plugin/plugin.json b/plugin/.claude-plugin/plugin.json index 76ab923b..70791955 100644 --- a/plugin/.claude-plugin/plugin.json +++ b/plugin/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "structured-context", - "version": "0.12.19", + "version": "0.12.20", "description": "Structured content authoring for Obsidian — automatic validation hooks, slash commands, and agent skills to keep markdown files compliant with your schema.", "homepage": "https://github.com/mindsocket/structured-context", "repository": "https://github.com/mindsocket/structured-context", diff --git a/plugin/skills/structured-context/references/schema-authoring.md b/plugin/skills/structured-context/references/schema-authoring.md index e19e56e5..27640cf2 100644 --- a/plugin/skills/structured-context/references/schema-authoring.md +++ b/plugin/skills/structured-context/references/schema-authoring.md @@ -66,7 +66,7 @@ Relationships define how sub-entities (nodes inside other files) are parsed and Rules are a flat array. Each rule's `category` (`validation`, `coherence`, `workflow`, `best-practice`) supplies its default severity: `validation` → `error`, `coherence` and `workflow` → `warning`, `best-practice` → `info`. Only `error` violations fail validation; warnings and info are reported only. An optional per-rule `severity` (`error` | `warning` | `info`) overrides the category default — prefer picking the right category over overriding. -A rule has an `expression` (a JSONata test that must return `true`), a `semantic` object (a model evaluates the node against a `rubric`, which falls back to `description`), or both; a rule with neither is rejected. `global: true` runs an `expression` once for the space (no `when`, `type` or `semantic`). `semantic: {}` takes every default; it may declare `verdicts` (verdict → description, or `{ description, guidance }`; default `pass`/`fail`) with a `semantic.severity` map saying which verdicts are `error`/`warning`/`info`/`ignore` (required with custom verdicts). `semantic.scope` (a list of `parent`, `siblings`, `children`, `ancestors`, `descendants`, `branch` (ancestors plus descendants) along the main hierarchy) sends related nodes with the evaluated node; `semantic.includeContent` (`nodeOnly` or `fullScope`/`true`) sends node bodies, and without it no body is sent; input over the per-node input cap (32k estimated tokens) has bodies trimmed with a `[RULE_EVALUATION_CONTENT_TRIMMED n chars]` marker, and such a rule also offers the `missing-context` verdict (default `warning`, set in `semantic.severity`). The top-level `severity` is that of a failed `expression` and of the default `fail` verdict. A rule's `expression` always runs and, when it fails, the `semantic` part is skipped; the `semantic` part runs only with `--semantic`. Run `sctx docs semantic-rules` for the full semantic rule reference. A `when` JSONata test (the applicability condition, any rule) limits a rule to the nodes where it returns true. An optional `gatesDescendants` controls whether a failure suppresses rules on the node's subtree (default: true for `error` severity). +A rule has an `expression` (a JSONata test that must return `true`), a `semantic` object (a model evaluates the node against a `rubric`, which falls back to `description`), or both; a rule with neither is rejected. `global: true` runs an `expression` once for the space (no `when`, `type` or `semantic`). `semantic: {}` takes every default; it may declare `verdicts` (verdict → description, or `{ description, guidance }`; default `pass`/`fail`) with a `semantic.severity` map saying which verdicts are `error`/`warning`/`info`/`ignore` (required with custom verdicts). `semantic.scope` (a list of `parent`, `siblings`, `children`, `ancestors`, `descendants`, `branch` (ancestors plus descendants) along the main hierarchy) sends related nodes with the evaluated node; `semantic.includeContent` (`nodeOnly` or `fullScope`/`true`) sends node bodies, and without it no body is sent; such a rule may return `missing-context` (default `warning`) when its input was trimmed to fit. The top-level `severity` is that of a failed `expression` and of the default `fail` verdict. A rule's `expression` always runs and, when it fails, the `semantic` part is skipped; the `semantic` part runs only with `--semantic`. Run `sctx docs semantic-rules` for the full semantic rule reference. A `when` JSONata test (the applicability condition, any rule) limits a rule to the nodes where it returns true. An optional `gatesDescendants` controls whether a failure suppresses rules on the node's subtree (default: true for `error` severity). ## Metadata composition semantics diff --git a/src/judge/judge-node.ts b/src/judge/judge-node.ts index 6340aa2d..50ac48e9 100644 --- a/src/judge/judge-node.ts +++ b/src/judge/judge-node.ts @@ -61,29 +61,24 @@ function interpret(rules: Rule[], resolved: ResolvedRules, results: JudgeResults const errorMessage = (err: unknown) => (err instanceof Error ? err.message : String(err)); -/** Reported for a node whose input is over the per-node input cap even with every body cut. */ const OVER_CAP_MESSAGE = 'input over the per-node cap after trimming content'; -/** - * One provider input and the rules (by position in their node's rule list) it answers. `trimmed` is set when - * a body was cut to fit the per-node input cap; `overCap` when that was not enough, so the input is not sent. - */ -type Unit = { - item: number; - positions: number[]; - rules: Rule[]; - resolved: ResolvedRules; - input: SemanticJudgeInput; - trimmed: boolean; - overCap: boolean; -}; +/** Rules (by position in their node's rule list) of one node that share an input. */ +type Slot = { item: number; positions: number[]; rules: Rule[] }; +/** One provider input and the rules it answers. */ +type Unit = Slot & { resolved: ResolvedRules; input: SemanticJudgeInput }; /** - * Split each node's rules into inputs by the context they ask for; rules that share a context share an input. - * Each input is trimmed to the per-node input cap. + * Split each node's rules into inputs by the context they ask for, trimmed to the per-node input cap; rules that + * share a context share an input. Inputs that cannot fit go to `overCap`; trimmed ones are added to `report`. */ -function planUnits(graph: SpaceGraph, items: Array<{ node: SpaceNode; rules: Rule[] }>): Unit[] { +function planUnits( + graph: SpaceGraph, + items: Array<{ node: SpaceNode; rules: Rule[] }>, + report?: TrimReport, +): { units: Unit[]; overCap: Slot[] } { const units: Unit[] = []; + const overCap: Slot[] = []; items.forEach(({ node, rules }, item) => { const groups = new Map(); rules.forEach((rule, position) => { @@ -96,27 +91,32 @@ function planUnits(graph: SpaceGraph, items: Array<{ node: SpaceNode; rules: Rul for (const { context, positions } of groups.values()) { const unitRules = positions.map((p) => rules[p]!); const resolved = resolveAll(unitRules); - const { input, trimmed, overCap } = trimToCap(judgeContext(graph, node, context)); + const fitted = trimToCap(judgeContext(graph, node, context)); + if (!fitted) { + overCap.push({ item, positions, rules: unitRules }); + continue; + } + if (fitted.trimmed && report) { + report.nodes.add(node.label); + for (const rule of unitRules) report.ruleIds.add(rule.id); + } units.push({ item, positions, rules: unitRules, resolved, - input: { ...input, rules: resolved.map((r) => r.spec) }, - trimmed, - overCap, + input: { ...fitted.input, rules: resolved.map((r) => r.spec) }, }); } }); - return units; + return { units, overCap }; } /** * Judge several nodes at once. A judge with `evaluateNodes` gets every input in one call and may fold them into * fewer requests; any other judge is called once per input. A node's rules that share a context (related nodes - * and bodies) form one input. An input over the per-node input cap has bodies trimmed; one still over is not sent - * and marks its rules as errors. Trimmed nodes and rules are added to `report`. Results align with `items`, in - * rule order. A failure marks only the affected rules as errors. + * and bodies) form one input; an input that cannot fit the per-node input cap is not sent. Results align with + * `items`, in rule order. A failure marks only the affected rules as errors. */ export async function judgeNodes( judge: SemanticJudgeProvider, @@ -124,22 +124,15 @@ export async function judgeNodes( graph: SpaceGraph, report?: TrimReport, ): Promise { - const planned = planUnits(graph, items); - const units = planned.filter((u) => !u.overCap); + const { units, overCap } = planUnits(graph, items, report); const out: NodeJudgement[][] = items.map(({ rules }) => new Array(rules.length)); - const place = (unit: Unit, judgements: NodeJudgement[]) => { + const place = (unit: Slot, judgements: NodeJudgement[]) => { unit.positions.forEach((position, k) => { out[unit.item]![position] = judgements[k]!; }); }; - for (const unit of planned) { - if (unit.overCap) place(unit, errorJudgements(unit.rules, OVER_CAP_MESSAGE)); - else if (unit.trimmed && report) { - report.nodes.add(items[unit.item]!.node.label); - for (const rule of unit.rules) report.ruleIds.add(rule.id); - } - } + for (const slot of overCap) place(slot, errorJudgements(slot.rules, OVER_CAP_MESSAGE)); if (!judge.evaluateNodes) { for (const unit of units) { diff --git a/src/judge/provider/jev.ts b/src/judge/provider/jev.ts index 29b31518..cf6560e1 100644 --- a/src/judge/provider/jev.ts +++ b/src/judge/provider/jev.ts @@ -16,10 +16,7 @@ export const JEV_LATEST = 'jev-latest'; export const OPENROUTER_DECISIONS_ENDPOINT = 'https://openrouter.ai/api/alpha/decisions'; -/** - * Request packing limit: the estimated input tokens (characters / 4) packed into one DecisionRequest's `state` - * plus `questions`. A conservative cost and latency guard. Not the per-node input cap, which bounds one node's input. - */ +/** Request packing limit: estimated input tokens (characters / 4) in one DecisionRequest's `state` plus `questions`. */ export const JEV_REQUEST_PACKING_LIMIT_TOKENS = 32_000; /** Request packing limit on the number of DecisionQuestions in one DecisionRequest. */ diff --git a/src/judge/trim.ts b/src/judge/trim.ts index 3e2a552d..c1d342ab 100644 --- a/src/judge/trim.ts +++ b/src/judge/trim.ts @@ -52,33 +52,25 @@ function largestFitting(max: number, fits: (limit: number) => boolean): number { return low; } -export interface TrimResult { - input: NodeInput; - /** True when any body was cut. */ - trimmed: boolean; - /** True when the input is still over the cap with every body cut to nothing. */ - overCap: boolean; -} - /** - * Bring a node's input under the per-node input cap by cutting bodies only: never fields, never a whole node. - * Related bodies are cut first, with one ceiling for all of them (longer bodies are cut to it, shorter ones stay - * whole) set as high as the cap allows. If that reaches zero and the input is still over, the evaluated node's - * body is cut the same way. Every cut body ends with the trim marker. + * Fit a node's input under the per-node input cap by cutting bodies only: related bodies to one shared ceiling + * first, then the evaluated node's body. Undefined when the input does not fit even with every body cut. */ -export function trimToCap(input: NodeInput, cap = PER_NODE_INPUT_CAP_TOKENS): TrimResult { - if (estimateTokens(input) <= cap) return { input, trimmed: false, overCap: false }; +export function trimToCap( + input: NodeInput, + cap = PER_NODE_INPUT_CAP_TOKENS, +): { input: NodeInput; trimmed: boolean } | undefined { + if (estimateTokens(input) <= cap) return { input, trimmed: false }; const nodeLength = bodyLength(input.node.schemaData); const relatedMax = Math.max(0, ...(input.related ?? []).map((r) => bodyLength(r.data))); const fitsRelated = (limit: number) => estimateTokens(limitBodies(input, limit, nodeLength)) <= cap; const relatedLimit = largestFitting(relatedMax, fitsRelated); - if (relatedLimit >= 0) return { input: limitBodies(input, relatedLimit, nodeLength), trimmed: true, overCap: false }; + if (relatedLimit >= 0) return { input: limitBodies(input, relatedLimit, nodeLength), trimmed: true }; const fitsNode = (limit: number) => estimateTokens(limitBodies(input, 0, limit)) <= cap; const nodeLimit = largestFitting(nodeLength, fitsNode); - if (nodeLimit >= 0) return { input: limitBodies(input, 0, nodeLimit), trimmed: true, overCap: false }; - return { input: limitBodies(input, 0, 0), trimmed: true, overCap: true }; + return nodeLimit >= 0 ? { input: limitBodies(input, 0, nodeLimit), trimmed: true } : undefined; } /** What a run trimmed: the nodes (by label) and the rules whose input had a body cut. */ diff --git a/src/judge/verdicts.ts b/src/judge/verdicts.ts index 1cfd93d4..a290e369 100644 --- a/src/judge/verdicts.ts +++ b/src/judge/verdicts.ts @@ -11,7 +11,7 @@ export const DEFAULT_VERDICTS: Record = { }; /** The reserved verdict a rule with related nodes or bodies offers, for input a trim marker shows was cut. */ -export const MISSING_CONTEXT_VERDICT = 'missing-context'; +const MISSING_CONTEXT_VERDICT = 'missing-context'; const MISSING_CONTEXT_DESCRIPTION = 'Content needed to judge the rule was cut: a [RULE_EVALUATION_CONTENT_TRIMMED n chars] marker shows where. Choose this only when such a marker is present and the cut content is needed. Never choose it when no marker appears.'; @@ -72,7 +72,7 @@ export function resolveSemanticRule(rule: Rule): ResolvedSemanticRule { throw new Error(`Semantic rule "${rule.id}" has no passing verdict: every verdict has a severity.`); } - // A rule that asks for related nodes or bodies offers `missing-context` (a static property of the rule), a warning by default. + // A warning by default. const context = ruleContext(rule); if (context.scope.length > 0 || context.content !== 'none') { verdicts[MISSING_CONTEXT_VERDICT] = MISSING_CONTEXT_DESCRIPTION; diff --git a/src/schema/gating.ts b/src/schema/gating.ts index cdfc796b..1e29d75e 100644 --- a/src/schema/gating.ts +++ b/src/schema/gating.ts @@ -61,7 +61,6 @@ export type GatedRuleEvaluation = { }; export interface GatingOptions { - /** Collects the nodes and rules whose input had content trimmed to fit the per-node input cap. */ trimReport?: TrimReport; /** Evaluate rules on gated nodes anyway (gates are still reported). */ includeGated?: boolean; diff --git a/src/schema/validate-rules.ts b/src/schema/validate-rules.ts index a3f6151a..5a5d2596 100644 --- a/src/schema/validate-rules.ts +++ b/src/schema/validate-rules.ts @@ -20,7 +20,6 @@ export interface ValidateRulesOptions { judgeTarget?: string; /** Set of node labels that have already failed upstream structural checks. */ gatedNodeLabels?: Set; - /** Collects the nodes and rules whose input had content trimmed to fit the per-node input cap. */ trimReport?: TrimReport; } diff --git a/tests/judge/trim.test.ts b/tests/judge/trim.test.ts index 7bd566d7..b2285163 100644 --- a/tests/judge/trim.test.ts +++ b/tests/judge/trim.test.ts @@ -37,42 +37,36 @@ describe('cutBody', () => { describe('trimToCap', () => { it('does nothing under the cap', () => { const i = input('body', ['rel']); - expect(trimToCap(i, 10_000)).toEqual({ input: i, trimmed: false, overCap: false }); + expect(trimToCap(i, 10_000)).toEqual({ input: i, trimmed: false }); }); it('cuts the longest related bodies to one ceiling and keeps shorter ones whole', () => { const short = words(10); const i = input('own', [words(2000), short, words(1500)]); const cap = Math.ceil(estimateTokens(i) / 2); - const out = trimToCap(i, cap); + const out = trimToCap(i, cap)!; expect(out.trimmed).toBe(true); - expect(out.overCap).toBe(false); expect(estimateTokens(out.input)).toBeLessThanOrEqual(cap); expect(relatedBody(out.input, 1)).toBe(short); expect(relatedBody(out.input, 0)).toContain('[RULE_EVALUATION_CONTENT_TRIMMED'); expect(relatedBody(out.input, 2)).toContain('[RULE_EVALUATION_CONTENT_TRIMMED'); expect(out.input.node.schemaData.content).toBe('own'); expect(out.input.related).toHaveLength(3); + expect(out.input.related?.[0]?.data.owner).toBe('team'); }); it('cuts the evaluated node body only once related bodies are exhausted', () => { const own = words(3000); const i = input(own, [words(50)]); const withoutBodies = estimateTokens(input('', [''])); - const out = trimToCap(i, withoutBodies + 200); - expect(out.overCap).toBe(false); + const out = trimToCap(i, withoutBodies + 200)!; expect(relatedBody(out.input, 0)).toBe(trimMarker(words(50).length)); expect(String(out.input.node.schemaData.content)).toContain('[RULE_EVALUATION_CONTENT_TRIMMED'); expect(String(out.input.node.schemaData.content).length).toBeLessThan(own.length); }); - it('never cuts fields or drops nodes, and reports an input that still does not fit', () => { - const i = input(words(100), [words(100)]); - const out = trimToCap(i, 10); - expect(out.overCap).toBe(true); - expect(out.input.related).toHaveLength(1); - expect(out.input.related?.[0]?.data.owner).toBe('team'); - expect(out.input.node.schemaData.source).toBe(node.schemaData.source); + it('returns nothing for an input that does not fit with every body cut', () => { + expect(trimToCap(input(words(100), [words(100)]), 10)).toBeUndefined(); }); it('does not mutate its input', () => {