From 3cc4932db230a53f0b67ec615e11b510bfe6fe18 Mon Sep 17 00:00:00 2001 From: Roger Barnes Date: Fri, 16 Oct 2026 14:38:00 +1100 Subject: [PATCH 1/6] feat(judge): semantic.scope and semantic.includeContent A semantic rule can add related nodes (parent, siblings, children, ancestors, descendants; main-hierarchy edges only) and choose which node bodies are sent. Providers receive related nodes as a flat list with their relations. Without includeContent no body is sent, including the evaluated node's own. --- .../_structured_context_schema_meta.json | 17 +++ src/judge/context.ts | 119 +++++++++++++++ src/judge/judge-node.ts | 88 +++++++---- src/judge/prompt.ts | 22 ++- src/judge/provider/anthropic.ts | 16 +- src/judge/provider/jev.ts | 17 ++- src/judge/provider/openai-decisions.ts | 16 +- src/judge/provider/openai.ts | 16 +- src/judge/provider/per-rule.ts | 16 +- src/judge/types.ts | 21 ++- src/schema/gating.ts | 2 +- src/schema/metadata-contract.ts | 17 +++ src/schema/validate-rules.ts | 3 +- tests/judge/context.test.ts | 142 ++++++++++++++++++ tests/judge/judge-node.test.ts | 8 +- tests/judge/prompt.test.ts | 16 ++ tests/judge/provider/jev.test.ts | 16 ++ tests/judge/provider/openai-decisions.test.ts | 17 +++ tests/schema/semantic-rule-contract.test.ts | 17 +++ 19 files changed, 538 insertions(+), 48 deletions(-) create mode 100644 src/judge/context.ts create mode 100644 tests/judge/context.test.ts diff --git a/schemas/generated/latest/_structured_context_schema_meta.json b/schemas/generated/latest/_structured_context_schema_meta.json index 96b9d963..2ef905f3 100644 --- a/schemas/generated/latest/_structured_context_schema_meta.json +++ b/schemas/generated/latest/_structured_context_schema_meta.json @@ -256,6 +256,23 @@ "enum": ["error", "warning", "info", "ignore"] }, "minProperties": 1 + }, + "scope": { + "type": "array", + "items": { + "enum": ["parent", "siblings", "children", "ancestors", "descendants"] + }, + "uniqueItems": true + }, + "includeContent": { + "oneOf": [ + { + "enum": ["nodeOnly", "fullScope"] + }, + { + "const": true + } + ] } }, "additionalProperties": false diff --git a/src/judge/context.ts b/src/judge/context.ts new file mode 100644 index 00000000..35e2a139 --- /dev/null +++ b/src/judge/context.ts @@ -0,0 +1,119 @@ +import { SEMANTIC_SCOPES } from '../schema/metadata-contract'; +import { ruleSemantic } from '../schema/rule-severity'; +import type { SpaceGraph } from '../space-graph'; +import type { Rule, SpaceNode } from '../types'; +import type { RelatedNode, SemanticJudgeInput } from './types'; + +export type SemanticScope = (typeof SEMANTIC_SCOPES)[number]; +export type ContentMode = 'none' | 'nodeOnly' | 'fullScope'; + +/** What a rule asks to send besides the evaluated node's data. */ +export interface RuleContext { + scope: SemanticScope[]; + content: ContentMode; +} + +/** The related nodes and bodies a semantic rule asks for. `includeContent: true` means `fullScope`. */ +export function ruleContext(rule: Rule): RuleContext { + const semantic = ruleSemantic(rule); + const include = semantic?.includeContent; + return { + scope: SEMANTIC_SCOPES.filter((s) => semantic?.scope?.includes(s)), + content: include === undefined ? 'none' : include === true ? 'fullScope' : include, + }; +} + +const bodyOf = (node: SpaceNode): string | undefined => + node.content ?? (typeof node.schemaData.content === 'string' ? node.schemaData.content : undefined); + +/** All schema data except the body, plus the body when asked for. */ +function dataWithBody(node: SpaceNode, withBody: boolean): Record { + const { content: _content, ...data } = node.schemaData; + const body = bodyOf(node); + return withBody && body !== undefined ? { ...data, content: body } : data; +} + +/** The evaluated node as a judge sees it. */ +export function judgeNodeView(node: SpaceNode, withBody: boolean): SemanticJudgeInput['node'] { + return { + label: node.label, + title: node.title, + type: node.type, + resolvedType: node.resolvedType, + schemaData: dataWithBody(node, withBody), + resolvedParents: node.resolvedParents, + }; +} + +function relatedView(node: SpaceNode, relation: string, withBody: boolean): RelatedNode { + const { title: _title, type: _type, ...data } = dataWithBody(node, withBody); + return { relations: [relation], title: node.title, type: node.resolvedType, data }; +} + +const RELATION_ORDER = ['parent', 'sibling', 'child', 'ancestor', 'descendant']; + +/** + * The nodes a scope adds for a node, along main-hierarchy edges only. Selection reads the graph's structure and + * nothing else: not verdicts, not gating. Each node appears once, with every relation it has. + */ +export function relatedNodes(graph: SpaceGraph, node: SpaceNode, context: RuleContext): RelatedNode[] { + const withBody = context.content === 'fullScope'; + const hierarchyName = graph.hierarchy?.name; + const parentsOf = (n: SpaceNode): SpaceNode[] => + graph.hierarchy === undefined + ? [] + : n.resolvedParents + .filter((p) => p.source === 'hierarchy' && p.hierarchy === hierarchyName) + .map((p) => graph.nodes.get(p.title)) + .filter((p): p is SpaceNode => p !== undefined); + const childrenOf = (n: SpaceNode) => graph.hierarchyChildren.get(n.title) ?? []; + + const found = new Map(); + const add = (other: SpaceNode, relation: string) => { + if (other.title === node.title) return; + const existing = found.get(other.title); + if (!existing) found.set(other.title, relatedView(other, relation, withBody)); + else if (!existing.relations.includes(relation)) existing.relations.push(relation); + }; + /** Every node reachable by repeatedly following `next`, each visited once. */ + const reach = (next: (n: SpaceNode) => readonly SpaceNode[], relation: string) => { + const seen = new Set([node.title]); + let level = next(node); + while (level.length > 0) { + const following: SpaceNode[] = []; + for (const n of level) { + if (seen.has(n.title)) continue; + seen.add(n.title); + add(n, relation); + following.push(...next(n)); + } + level = following; + } + }; + + const scope = new Set(context.scope); + if (scope.has('parent')) for (const p of parentsOf(node)) add(p, 'parent'); + if (scope.has('siblings')) for (const p of parentsOf(node)) for (const s of childrenOf(p)) add(s, 'sibling'); + if (scope.has('children')) for (const c of childrenOf(node)) add(c, 'child'); + if (scope.has('ancestors')) reach(parentsOf, 'ancestor'); + if (scope.has('descendants')) reach(childrenOf, 'descendant'); + + const rank = (r: RelatedNode) => Math.min(...r.relations.map((x) => RELATION_ORDER.indexOf(x))); + return [...found.values()].sort((a, b) => rank(a) - rank(b) || a.title.localeCompare(b.title)); +} + +/** Stable key for rules that share a context, so they can share a provider input. */ +export const contextKey = (context: RuleContext) => `${context.content}|${context.scope.join(',')}`; + +/** The judge input context for rules that share `context`: the node view and its related nodes. */ +export function judgeContext( + graph: SpaceGraph, + node: SpaceNode, + context: RuleContext, +): Pick { + const related = relatedNodes(graph, node, context); + return { + node: judgeNodeView(node, context.content !== 'none'), + ...(related.length > 0 && { related }), + }; +} diff --git a/src/judge/judge-node.ts b/src/judge/judge-node.ts index c2d9d308..0910b31a 100644 --- a/src/judge/judge-node.ts +++ b/src/judge/judge-node.ts @@ -1,5 +1,7 @@ +import type { SpaceGraph } from '../space-graph'; import type { Rule, RuleSeverity, SpaceNode } from '../types'; -import type { SemanticJudgeProvider } from './types'; +import { contextKey, judgeContext, type RuleContext, ruleContext } from './context'; +import type { SemanticJudgeInput, SemanticJudgeProvider } from './types'; import { isJudgeError, resolveSemanticRule } from './verdicts'; export type NodeJudgement = { @@ -58,51 +60,85 @@ function interpret(rules: Rule[], resolved: ResolvedRules, results: JudgeResults const errorMessage = (err: unknown) => (err instanceof Error ? err.message : String(err)); -/** Judge every given semantic rule on a node in one provider call. */ +/** One provider input and the rules (by position in their node's rule list) it answers. */ +type Unit = { item: number; positions: number[]; rules: Rule[]; resolved: ResolvedRules; input: SemanticJudgeInput }; + +/** Split each node's rules into inputs by the context they ask for; rules that share a context share an input. */ +function planUnits(graph: SpaceGraph, items: Array<{ node: SpaceNode; rules: Rule[] }>): Unit[] { + const units: Unit[] = []; + items.forEach(({ node, rules }, item) => { + const groups = new Map(); + rules.forEach((rule, position) => { + const context = ruleContext(rule); + const key = contextKey(context); + const group = groups.get(key) ?? { context, positions: [] }; + group.positions.push(position); + groups.set(key, group); + }); + for (const { context, positions } of groups.values()) { + const unitRules = positions.map((p) => rules[p]!); + const resolved = resolveAll(unitRules); + units.push({ + item, + positions, + rules: unitRules, + resolved, + input: { ...judgeContext(graph, node, context), rules: resolved.map((r) => r.spec) }, + }); + } + }); + return units; +} + +/** Judge every given semantic rule on a node. */ export async function judgeNode( judge: SemanticJudgeProvider, node: SpaceNode, rules: Rule[], + graph: SpaceGraph, ): Promise { - if (rules.length === 0) return []; - const resolved = resolveAll(rules); - let results: JudgeResults; - try { - results = await judge.evaluate({ node, rules: resolved.map((r) => r.spec) }); - } catch (err) { - return errorJudgements(rules, errorMessage(err)); - } - return interpret(rules, resolved, results); + return (await judgeNodes(judge, [{ node, rules }], graph))[0] ?? []; } /** - * Judge several nodes at once. A judge with `evaluateNodes` gets every node in one call and may fold them into - * fewer requests; any other judge is called once per node. Results align with `items`. A failure marks only the - * affected node's rules as errors. + * Judge several nodes at once. A judge with `evaluateNodes` gets every input in one call and may fold them into + * fewer requests; any other judge is called once per input. A node's rules that share a context (related nodes + * and bodies) form one input. Results align with `items`, in rule order. A failure marks only the affected + * rules as errors. */ export async function judgeNodes( judge: SemanticJudgeProvider, items: Array<{ node: SpaceNode; rules: Rule[] }>, + graph: SpaceGraph, ): Promise { + const units = planUnits(graph, items); + const out: NodeJudgement[][] = items.map(({ rules }) => new Array(rules.length)); + const place = (unit: Unit, judgements: NodeJudgement[]) => { + unit.positions.forEach((position, k) => { + out[unit.item]![position] = judgements[k]!; + }); + }; + if (!judge.evaluateNodes) { - const out: NodeJudgement[][] = []; - for (const item of items) out.push(await judgeNode(judge, item.node, item.rules)); + for (const unit of units) { + try { + place(unit, interpret(unit.rules, unit.resolved, await judge.evaluate(unit.input))); + } catch (err) { + place(unit, errorJudgements(unit.rules, errorMessage(err))); + } + } return out; } - const active = items - .map((item, index) => ({ ...item, index, resolved: resolveAll(item.rules) })) - .filter((a) => a.rules.length > 0); - const out: NodeJudgement[][] = items.map(() => []); - if (active.length === 0) return out; - let perNode: JudgeResults[]; + if (units.length === 0) return out; + let perInput: JudgeResults[]; try { - perNode = await judge.evaluateNodes(active.map((a) => ({ node: a.node, rules: a.resolved.map((r) => r.spec) }))); + perInput = await judge.evaluateNodes(units.map((u) => u.input)); } catch (err) { - for (const a of active) out[a.index] = errorJudgements(a.rules, errorMessage(err)); + for (const unit of units) place(unit, errorJudgements(unit.rules, errorMessage(err))); return out; } - active.forEach((a, k) => { - out[a.index] = interpret(a.rules, a.resolved, perNode[k] ?? {}); + units.forEach((unit, k) => { + place(unit, interpret(unit.rules, unit.resolved, perInput[k] ?? {})); }); return out; } diff --git a/src/judge/prompt.ts b/src/judge/prompt.ts index 05712de9..94a14b0b 100644 --- a/src/judge/prompt.ts +++ b/src/judge/prompt.ts @@ -1,4 +1,4 @@ -import type { SemanticJudgeInput, SemanticJudgeResult, SemanticRuleSpec } from './types'; +import type { RelatedNode, SemanticJudgeInput, SemanticJudgeResult, SemanticRuleSpec } from './types'; function verdictList(rule: SemanticRuleSpec): string[] { return Object.entries(rule.verdicts).map(([name, description]) => `- "${name}": ${description}`); @@ -26,7 +26,11 @@ export function buildJudgeSystemPrompt(): string { } /** User prompt for one rule on one node. */ -export function buildJudgeUserPrompt(node: SemanticJudgeInput['node'], rule: SemanticRuleSpec): string { +export function buildJudgeUserPrompt( + node: SemanticJudgeInput['node'], + rule: SemanticRuleSpec, + related: RelatedNode[] = [], +): string { const rubric = rule.rubric || rule.description; const lines: string[] = [ '# Rule to Evaluate', @@ -48,6 +52,20 @@ export function buildJudgeUserPrompt(node: SemanticJudgeInput['node'], rule: Sem } lines.push('', '## Node Content & Frontmatter:', '```json', JSON.stringify(node.schemaData, null, 2), '```', ''); + if (related.length > 0) { + lines.push('# Related Nodes', 'Each is listed with its relation to the node under evaluation.', ''); + for (const r of related) { + lines.push( + `## ${r.title}`, + `- Relation: ${r.relations.join(', ')}`, + `- Type: ${r.type}`, + '```json', + JSON.stringify(r.data, null, 2), + '```', + '', + ); + } + } lines.push('Evaluate this node against the rule. Return only JSON with "verdict", "confidence" and "evidence".'); return lines.join('\n'); } diff --git a/src/judge/provider/anthropic.ts b/src/judge/provider/anthropic.ts index 9f3bd6cd..740d6366 100644 --- a/src/judge/provider/anthropic.ts +++ b/src/judge/provider/anthropic.ts @@ -1,5 +1,11 @@ import { buildJudgeSystemPrompt, buildJudgeUserPrompt, parseJudgeResponse } from '../prompt'; -import type { HydratedProvider, SemanticJudgeInput, SemanticJudgeResult, SemanticRuleSpec } from '../types'; +import type { + HydratedProvider, + RelatedNode, + SemanticJudgeInput, + SemanticJudgeResult, + SemanticRuleSpec, +} from '../types'; import { PerRuleJudgeProvider } from './per-rule'; export interface AnthropicJudgeProviderOptions extends HydratedProvider { @@ -24,7 +30,11 @@ export class AnthropicJudgeProvider extends PerRuleJudgeProvider { this.endpoint = options.endpoint || ANTHROPIC_MESSAGES_ENDPOINT; } - protected async judgeRule(node: SemanticJudgeInput['node'], rule: SemanticRuleSpec): Promise { + protected async judgeRule( + node: SemanticJudgeInput['node'], + rule: SemanticRuleSpec, + related: RelatedNode[], + ): Promise { const res = await fetch(this.endpoint, { method: 'POST', headers: { @@ -36,7 +46,7 @@ export class AnthropicJudgeProvider extends PerRuleJudgeProvider { model: this.model, max_tokens: 1024, system: buildJudgeSystemPrompt(), - messages: [{ role: 'user', content: buildJudgeUserPrompt(node, rule) }], + messages: [{ role: 'user', content: buildJudgeUserPrompt(node, rule, related) }], }), }); diff --git a/src/judge/provider/jev.ts b/src/judge/provider/jev.ts index 8681f4fc..23afc890 100644 --- a/src/judge/provider/jev.ts +++ b/src/judge/provider/jev.ts @@ -1,5 +1,6 @@ import type { HydratedProvider, + RelatedNode, SemanticJudgeInput, SemanticJudgeProvider, SemanticJudgeResult, @@ -54,7 +55,7 @@ export class JevJudgeProvider implements SemanticJudgeProvider { const part = rules.slice(start, start + JEV_DECISION_REQUEST_MAX_QUESTIONS); const questions: Record = {}; for (const rule of part) questions[rule.id] = decisionQuestion(rule, rule.rubric || rule.description); - const data = await this.post(nodeState(node), questions); + const data = await this.post(nodeState(node, input.related), questions); for (const rule of part) results[rule.id] = toResult(rule, data.answers?.[rule.id]); } return results; @@ -106,7 +107,7 @@ export class JevJudgeProvider implements SemanticJudgeProvider { const questions: Record = {}; requestInputs.forEach((inputIndex, k) => { const input = inputs[inputIndex]!; - nodes[`n${k}`] = nodeState(input.node); + nodes[`n${k}`] = nodeState(input.node, input.related); for (const rule of input.rules) { questions[`${rule.id}__n${k}`] = decisionQuestion( rule, @@ -138,12 +139,20 @@ export class JevJudgeProvider implements SemanticJudgeProvider { type JevResponse = { answers?: Record }; -function nodeState(node: SemanticJudgeInput['node']) { +function nodeState(node: SemanticJudgeInput['node'], related: RelatedNode[] = []) { return { title: node.title, type: node.resolvedType, parents: node.resolvedParents?.map((p) => p.title) ?? [], ...node.schemaData, + ...(related.length > 0 && { + relatedNodes: related.map((r) => ({ + relation: r.relations.join(', '), + title: r.title, + type: r.type, + ...r.data, + })), + }), }; } @@ -173,7 +182,7 @@ function planDecisionRequests(indices: number[], inputs: SemanticJudgeInput[]): let questions = 0; for (const i of indices) { const input = inputs[i]!; - const cost = estimateTokens(nodeState(input.node)) + estimateTokens(input.rules); + const cost = estimateTokens(nodeState(input.node, input.related)) + estimateTokens(input.rules); const count = input.rules.length; if ( current.length > 0 && diff --git a/src/judge/provider/openai-decisions.ts b/src/judge/provider/openai-decisions.ts index 072c9995..61489e11 100644 --- a/src/judge/provider/openai-decisions.ts +++ b/src/judge/provider/openai-decisions.ts @@ -41,9 +41,13 @@ export class OpenAIDecisionsJudgeProvider implements SemanticJudgeProvider { `Title: ${node.title}`, `Type: ${node.resolvedType}`, ...(node.resolvedParents?.length ? [`Parents: ${node.resolvedParents.map((p) => p.title).join(', ')}`] : []), - ...Object.entries(node.schemaData ?? {}) - .filter(([field]) => field !== 'title' && field !== 'type') - .map(([field, value]) => `${field}: ${JSON.stringify(value) ?? String(value)}`), + ...fieldLines(node.schemaData ?? {}), + ...(input.related ?? []).flatMap((r) => [ + '', + `Related node (${r.relations.join(', ')}): ${r.title}`, + `Type: ${r.type}`, + ...fieldLines(r.data), + ]), ]; const res = await fetch(this.endpoint, { @@ -87,6 +91,12 @@ export class OpenAIDecisionsJudgeProvider implements SemanticJudgeProvider { } } +function fieldLines(data: Record): string[] { + return Object.entries(data) + .filter(([field]) => field !== 'title' && field !== 'type') + .map(([field, value]) => `${field}: ${JSON.stringify(value) ?? String(value)}`); +} + function describeDistribution( choice: string, confidence: number | undefined, diff --git a/src/judge/provider/openai.ts b/src/judge/provider/openai.ts index a6243068..1505bba3 100644 --- a/src/judge/provider/openai.ts +++ b/src/judge/provider/openai.ts @@ -1,5 +1,11 @@ import { buildJudgeSystemPrompt, buildJudgeUserPrompt, parseJudgeResponse } from '../prompt'; -import type { HydratedProvider, SemanticJudgeInput, SemanticJudgeResult, SemanticRuleSpec } from '../types'; +import type { + HydratedProvider, + RelatedNode, + SemanticJudgeInput, + SemanticJudgeResult, + SemanticRuleSpec, +} from '../types'; import { PerRuleJudgeProvider } from './per-rule'; export interface OpenAIJudgeProviderOptions extends HydratedProvider { @@ -29,9 +35,13 @@ export class OpenAIJudgeProvider extends PerRuleJudgeProvider { this.endpoint = base.endsWith('/chat/completions') ? base : `${base.replace(/\/+$/, '')}/chat/completions`; } - protected async judgeRule(node: SemanticJudgeInput['node'], rule: SemanticRuleSpec): Promise { + protected async judgeRule( + node: SemanticJudgeInput['node'], + rule: SemanticRuleSpec, + related: RelatedNode[], + ): Promise { const systemPrompt = buildJudgeSystemPrompt(); - const userPrompt = buildJudgeUserPrompt(node, rule); + const userPrompt = buildJudgeUserPrompt(node, rule, related); const headers: Record = { 'content-type': 'application/json', diff --git a/src/judge/provider/per-rule.ts b/src/judge/provider/per-rule.ts index a309c49d..b9dbf613 100644 --- a/src/judge/provider/per-rule.ts +++ b/src/judge/provider/per-rule.ts @@ -1,4 +1,10 @@ -import type { SemanticJudgeInput, SemanticJudgeProvider, SemanticJudgeResult, SemanticRuleSpec } from '../types'; +import type { + RelatedNode, + SemanticJudgeInput, + SemanticJudgeProvider, + SemanticJudgeResult, + SemanticRuleSpec, +} from '../types'; /** * Base for providers that can only judge one rule per request (chat LLMs). `evaluate` makes one call @@ -7,13 +13,17 @@ import type { SemanticJudgeInput, SemanticJudgeProvider, SemanticJudgeResult, Se export abstract class PerRuleJudgeProvider implements SemanticJudgeProvider { abstract readonly name: string; - protected abstract judgeRule(node: SemanticJudgeInput['node'], rule: SemanticRuleSpec): Promise; + protected abstract judgeRule( + node: SemanticJudgeInput['node'], + rule: SemanticRuleSpec, + related: RelatedNode[], + ): Promise; async evaluate(input: SemanticJudgeInput): Promise> { const results: Record = {}; for (const rule of input.rules) { try { - results[rule.id] = await this.judgeRule(input.node, rule); + results[rule.id] = await this.judgeRule(input.node, rule, input.related ?? []); } catch (err) { results[rule.id] = { error: err instanceof Error ? err.message : String(err) }; } diff --git a/src/judge/types.ts b/src/judge/types.ts index 645d98e9..e3a2b025 100644 --- a/src/judge/types.ts +++ b/src/judge/types.ts @@ -10,9 +10,28 @@ export interface SemanticRuleSpec { verdicts: Record; } -/** One node and every semantic rule to judge on it. Judges may answer all rules in a single call. */ +/** + * A node related to the evaluated node by its rules' `scope`. It carries all the node's schema data; the body + * (`data.content`) is present only when the rules' `includeContent` asks for it. + */ +export interface RelatedNode { + /** How it relates to the evaluated node: `parent`, `sibling`, `child`, `ancestor`, `descendant`. A node can have several. */ + relations: string[]; + title: string; + type: string; + /** Schema data without `title` and `type`. */ + data: Record; +} + +/** + * One node and the semantic rules to judge on it, which share one context: the same related nodes and the same + * bodies. Judges may answer all rules in a single call. + */ export interface SemanticJudgeInput { + /** The evaluated node. Its body is in `schemaData.content` only when the rules' `includeContent` asks for it. */ node: Pick; + /** Nodes the rules' `scope` adds, as a flat list. Absent when the scope is empty. */ + related?: RelatedNode[]; rules: SemanticRuleSpec[]; } diff --git a/src/schema/gating.ts b/src/schema/gating.ts index 77836f32..fcd730cc 100644 --- a/src/schema/gating.ts +++ b/src/schema/gating.ts @@ -280,7 +280,7 @@ export async function evaluateRulesWithGating( for (const node of nodes) prepared.push(await prepareNode(node)); if (judge) { const pending = nodes.map((node, i) => ({ node, rules: prepared[i]!.semantic })); - const judgements = await judgeNodes(judge, pending); + const judgements = await judgeNodes(judge, pending, graph); prepared.forEach((p, i) => { p.judged(judgements[i] ?? []); }); diff --git a/src/schema/metadata-contract.ts b/src/schema/metadata-contract.ts index eb63ad42..a3f96483 100644 --- a/src/schema/metadata-contract.ts +++ b/src/schema/metadata-contract.ts @@ -60,6 +60,9 @@ const SEVERITY_SCHEMA = { enum: ['error', 'warning', 'info'] } as const; /** The verdicts the evaluation reserves besides a rule's own; a verdict map may set a severity for them. */ export const RESERVED_VERDICTS = ['missing-context'] as const; +/** Related-node kinds a semantic rule's `scope` can add: main-hierarchy edges only. */ +export const SEMANTIC_SCOPES = ['parent', 'siblings', 'children', 'ancestors', 'descendants'] as const; + const VERDICT_SEVERITY_SCHEMA = { enum: ['error', 'warning', 'info', 'ignore'] } as const; /** The semantic part of a rule: a model evaluates the node against a rubric and returns a verdict. */ @@ -98,6 +101,20 @@ const SEMANTIC_SCHEMA = { additionalProperties: VERDICT_SEVERITY_SCHEMA, minProperties: 1, }, + /** + * Related nodes, along main-hierarchy edges, sent with the evaluated node. Additive; left out or empty means + * the evaluated node alone. + */ + scope: { + type: 'array', + items: { enum: SEMANTIC_SCOPES }, + uniqueItems: true, + }, + /** + * Which node bodies are sent: `nodeOnly` (the evaluated node's), `fullScope` (it and every related node's), + * or `true` for `fullScope`. Left out: no bodies. + */ + includeContent: { oneOf: [{ enum: ['nodeOnly', 'fullScope'] }, { const: true }] }, }, additionalProperties: false, } as const; diff --git a/src/schema/validate-rules.ts b/src/schema/validate-rules.ts index d9639a1e..c961fd71 100644 --- a/src/schema/validate-rules.ts +++ b/src/schema/validate-rules.ts @@ -1,5 +1,6 @@ import { judgeNodes } from '../judge/judge-node'; import type { SemanticJudgeProvider } from '../judge/types'; +import { buildSpaceGraph } from '../space-graph'; import type { Rule, RuleViolation, SpaceNode } from '../types'; import { buildEvalContext, @@ -109,7 +110,7 @@ export async function validateRules( } // One provider call for every node, so a provider with `evaluateNodes` can share model calls. - const judged = await judgeNodes(judge, pending); + const judged = await judgeNodes(judge, pending, buildSpaceGraph(nodes, undefined)); pending.forEach(({ node, found }, i) => { for (const j of judged[i] ?? []) { if (!j.failed) continue; diff --git a/tests/judge/context.test.ts b/tests/judge/context.test.ts new file mode 100644 index 00000000..c1fd1578 --- /dev/null +++ b/tests/judge/context.test.ts @@ -0,0 +1,142 @@ +import { describe, expect, it } from 'bun:test'; +import { judgeContext, type RuleContext, relatedNodes, ruleContext } from '../../src/judge/context'; +import { judgeNodes } from '../../src/judge/judge-node'; +import { FakeJudgeProvider } from '../../src/judge/provider/fake'; +import type { SemanticJudgeInput, SemanticJudgeProvider } from '../../src/judge/types'; +import { evaluateRulesWithGating } from '../../src/schema/gating'; +import { buildSpaceGraph } from '../../src/space-graph'; +import type { Rule, SpaceNode } from '../../src/types'; +import { makeHierarchy, makeLevel, makeParentRef, makeSpaceNode } from '../test-helpers'; + +const hierarchy = makeHierarchy([makeLevel('goal'), makeLevel('opportunity'), makeLevel('solution')]); +const child = (title: string, type: string, parent: string, extra: Record = {}) => + makeSpaceNode(title, type, extra, undefined, [makeParentRef(parent)]); + +const goal = makeSpaceNode('Goal', 'goal', { content: 'goal body', owner: 'team' }); +const opp = child('Opp', 'opportunity', 'Goal', { content: 'opp body' }); +const sibling = child('Sibling', 'opportunity', 'Goal', { content: 'sibling body' }); +const sol = child('Sol', 'solution', 'Opp', { content: 'sol body' }); +const sol2 = child('Sol2', 'solution', 'Opp'); +const subSol = child('SubSol', 'solution', 'Sol'); +// Related to Opp by a non-hierarchy relationship: never in scope. +const assumption = makeSpaceNode('Assume', 'assumption', {}, undefined, [ + makeParentRef('Opp', { source: 'relationship', hierarchy: undefined }), +]); +const nodes: SpaceNode[] = [goal, opp, sibling, sol, sol2, subSol, assumption]; +const graph = buildSpaceGraph(nodes, hierarchy); + +const ctx = (scope: RuleContext['scope'], content: RuleContext['content'] = 'none'): RuleContext => ({ + scope, + content, +}); +const titles = (scope: RuleContext['scope']) => relatedNodes(graph, opp, ctx(scope)).map((r) => r.title); + +describe('relatedNodes', () => { + it('selects each scope along hierarchy edges', () => { + expect(titles(['parent'])).toEqual(['Goal']); + expect(titles(['siblings'])).toEqual(['Sibling']); + expect(titles(['children'])).toEqual(['Sol', 'Sol2']); + expect(titles(['ancestors'])).toEqual(['Goal']); + expect(titles(['descendants'])).toEqual(['Sol', 'Sol2', 'SubSol']); + }); + + it('adds scopes together, once per node, with every relation it has', () => { + const related = relatedNodes(graph, opp, ctx(['parent', 'ancestors', 'children'])); + expect(related.map((r) => [r.title, r.relations])).toEqual([ + ['Goal', ['parent', 'ancestor']], + ['Sol', ['child']], + ['Sol2', ['child']], + ]); + }); + + it('is empty for an empty scope and ignores non-hierarchy relationships', () => { + expect(titles([])).toEqual([]); + expect(titles(['children', 'descendants', 'siblings'])).not.toContain('Assume'); + }); + + it('sends all schema data but the body, which follows the content mode', () => { + const without = relatedNodes(graph, opp, ctx(['parent'], 'nodeOnly'))[0]!; + expect(without.data).toEqual({ owner: 'team' }); + const withBody = relatedNodes(graph, opp, ctx(['parent'], 'fullScope'))[0]!; + expect(withBody.data).toEqual({ owner: 'team', content: 'goal body' }); + }); + + it('finds no relations in a graph without a hierarchy', () => { + const flat = buildSpaceGraph(nodes, undefined); + expect(relatedNodes(flat, opp, ctx(['parent', 'children', 'siblings', 'ancestors', 'descendants']))).toEqual([]); + }); +}); + +describe('judgeContext', () => { + it('sends no bodies unless includeContent asks for them', () => { + expect(judgeContext(graph, opp, ctx([])).node.schemaData).not.toHaveProperty('content'); + expect(judgeContext(graph, opp, ctx([], 'nodeOnly')).node.schemaData.content).toBe('opp body'); + expect(judgeContext(graph, opp, ctx([], 'fullScope')).node.schemaData.content).toBe('opp body'); + }); + + it('omits related when the scope is empty', () => { + expect(judgeContext(graph, opp, ctx([])).related).toBeUndefined(); + }); +}); + +describe('ruleContext', () => { + const rule = (semantic: object) => + ({ id: 'r', category: 'coherence', description: 'd', semantic }) as unknown as Rule; + + it('defaults to the evaluated node alone, with no bodies', () => { + expect(ruleContext(rule({}))).toEqual({ scope: [], content: 'none' }); + }); + + it('reads scope and includeContent, treating true as fullScope', () => { + expect(ruleContext(rule({ scope: ['children', 'parent'], includeContent: 'nodeOnly' }))).toEqual({ + scope: ['parent', 'children'], + content: 'nodeOnly', + }); + expect(ruleContext(rule({ includeContent: true })).content).toBe('fullScope'); + }); +}); + +describe('judgeNodes with scope', () => { + const scoped = (id: string, semantic: object) => + ({ id, category: 'coherence', description: id, semantic }) as unknown as Rule; + + it('shares one input between rules with the same context and splits the rest, keeping rule order', async () => { + const inputs: SemanticJudgeInput[] = []; + const judge: SemanticJudgeProvider = { + name: 'spy', + evaluate: (input) => { + inputs.push(input); + return Promise.resolve(Object.fromEntries(input.rules.map((r) => [r.id, { verdict: 'pass' }]))); + }, + }; + const rules = [scoped('a', { scope: ['parent'] }), scoped('b', {}), scoped('c', { scope: ['parent'] })]; + const out = await judgeNodes(judge, [{ node: opp, rules }], graph); + expect(inputs.map((i) => i.rules.map((r) => r.id))).toEqual([['a', 'c'], ['b']]); + expect(inputs[0]?.related?.map((r) => r.title)).toEqual(['Goal']); + expect(inputs[1]?.related).toBeUndefined(); + expect(out[0]?.map((j) => j.rule.id)).toEqual(['a', 'b', 'c']); + }); +}); + +describe('scope under gating', () => { + it('selects related nodes whether or not a verdict or gate has touched them', async () => { + const rules = [ + // Fails on Goal with an error, which gates Opp and everything below it. + { id: 'goal-fails', category: 'validation', description: 'g', type: 'goal', expression: 'false' }, + { + id: 'sol-coherent', + category: 'coherence', + description: 's', + type: 'solution', + semantic: { scope: ['parent', 'ancestors'] }, + }, + ] as unknown as Rule[]; + const fake = new FakeJudgeProvider(); + await evaluateRulesWithGating(nodes, rules, graph, [], { provider: fake, includeGated: true }); + const call = fake.callHistory.find((c) => c.node.title === 'Sol'); + expect(call?.related?.map((r) => [r.title, r.relations])).toEqual([ + ['Opp', ['parent', 'ancestor']], + ['Goal', ['ancestor']], + ]); + }); +}); diff --git a/tests/judge/judge-node.test.ts b/tests/judge/judge-node.test.ts index cbc0f082..ff038908 100644 --- a/tests/judge/judge-node.test.ts +++ b/tests/judge/judge-node.test.ts @@ -1,7 +1,8 @@ import { describe, expect, it } from 'bun:test'; -import { judgeNode, judgeNodes, type NodeJudgement } from '../../src/judge/judge-node'; +import { judgeNode as judgeNodeIn, judgeNodes as judgeNodesIn, type NodeJudgement } from '../../src/judge/judge-node'; import type { SemanticJudgeProvider, SemanticJudgeResult } from '../../src/judge/types'; import type { Rule } from '../../src/schema/metadata-contract'; +import { buildSpaceGraph } from '../../src/space-graph'; import type { SpaceNode } from '../../src/types'; import { node, passFailRule } from './helpers'; @@ -14,6 +15,11 @@ function canned(results: Record): SemanticJudgeProv }; } +const graph = buildSpaceGraph([], undefined); +const judgeNode = (j: SemanticJudgeProvider, n: SpaceNode, rules: Rule[]) => judgeNodeIn(j, n, rules, graph); +const judgeNodes = (j: SemanticJudgeProvider, items: Array<{ node: SpaceNode; rules: Rule[] }>) => + judgeNodesIn(j, items, graph); + describe('judgeNode', () => { it('treats an unknown verdict as a judge error, not a pass', async () => { const rule = { diff --git a/tests/judge/prompt.test.ts b/tests/judge/prompt.test.ts index efce8946..97ea440e 100644 --- a/tests/judge/prompt.test.ts +++ b/tests/judge/prompt.test.ts @@ -60,3 +60,19 @@ describe('judge prompt and response parser', () => { expect(parseJudgeResponse('I cannot decide', allowed)).toHaveProperty('error'); }); }); + +describe('related nodes in the chat prompt', () => { + it('lists each related node with its relation', () => { + const prompt = buildJudgeUserPrompt(node, passFailRule, [ + { relations: ['parent', 'ancestor'], title: 'Faster Tax Filing', type: 'goal', data: { owner: 'finance' } }, + ]); + expect(prompt).toContain('Related Nodes'); + expect(prompt).toContain('Faster Tax Filing'); + expect(prompt).toContain('Relation: parent, ancestor'); + expect(prompt).toContain('finance'); + }); + + it('has no related section without related nodes', () => { + expect(buildJudgeUserPrompt(node, passFailRule)).not.toContain('Related Nodes'); + }); +}); diff --git a/tests/judge/provider/jev.test.ts b/tests/judge/provider/jev.test.ts index 9a649690..5e8e6fcb 100644 --- a/tests/judge/provider/jev.test.ts +++ b/tests/judge/provider/jev.test.ts @@ -219,3 +219,19 @@ describe('JevJudgeProvider', () => { }); }); }); + +describe('JevJudgeProvider related nodes', () => { + it('sends related nodes in the state with their relation', async () => { + const calls = mockFetch({ answers: { 'customer-voice': { type: 'choice', choice: 'pass' } } }); + const jev = new JevJudgeProvider({ api: 'openrouter-decisions', apiKey: 'k' }); + await jev.evaluate({ + node, + related: [{ relations: ['parent'], title: 'Faster Tax Filing', type: 'goal', data: { owner: 'finance' } }], + rules: [passFailRule], + }); + const sent = JSON.parse(calls[0]?.init.body as string); + expect(sent.state.relatedNodes).toEqual([ + { relation: 'parent', title: 'Faster Tax Filing', type: 'goal', owner: 'finance' }, + ]); + }); +}); diff --git a/tests/judge/provider/openai-decisions.test.ts b/tests/judge/provider/openai-decisions.test.ts index 2c4e0be4..ba4250f9 100644 --- a/tests/judge/provider/openai-decisions.test.ts +++ b/tests/judge/provider/openai-decisions.test.ts @@ -112,3 +112,20 @@ describe('OpenAIDecisionsJudgeProvider', () => { expect(() => createModelProvider('openai-decisions', { api: 'openai-decisions' }, {})).toThrow('OPENAI_API_KEY'); }); }); + +describe('OpenAIDecisionsJudgeProvider related nodes', () => { + it('lists related nodes with their relation in the input', async () => { + const calls = mockFetch({ + answers: [{ type: 'choice', name: 'customer-voice', choice: 'pass', probabilities: [] }], + }); + const provider = new OpenAIDecisionsJudgeProvider({ api: 'openai-decisions', apiKey: 'k' }); + await provider.evaluate({ + node, + related: [{ relations: ['child'], title: 'Receipt scanner', type: 'solution', data: { owner: 'finance' } }], + rules: [passFailRule], + }); + const sent = JSON.parse(calls[0]?.init.body as string); + expect(sent.input).toContain('Related node (child): Receipt scanner'); + expect(sent.input).toContain('owner: "finance"'); + }); +}); diff --git a/tests/schema/semantic-rule-contract.test.ts b/tests/schema/semantic-rule-contract.test.ts index 526d84aa..5151075e 100644 --- a/tests/schema/semantic-rule-contract.test.ts +++ b/tests/schema/semantic-rule-contract.test.ts @@ -102,4 +102,21 @@ describe('rule contract: expression and semantic', () => { expect(() => load('expr-verdicts', { ...expr, verdicts: { a: 'A', b: 'B' } })).toThrow(); expect(load('expr-when', { ...expr, when: 'true' }).rules).toHaveLength(1); }); + + it('accepts scope and includeContent on a semantic part, and nowhere else', () => { + const scope = ['parent', 'siblings', 'children', 'ancestors', 'descendants']; + expect(load('scope-ok', { ...base, semantic: { scope, includeContent: 'fullScope' } }).rules).toHaveLength(1); + expect(load('scope-empty', { ...base, semantic: { scope: [], includeContent: 'nodeOnly' } }).rules).toHaveLength(1); + expect(load('content-true', { ...base, semantic: { includeContent: true } }).rules).toHaveLength(1); + expect(() => load('scope-unknown', { ...base, semantic: { scope: ['cousins'] } })).toThrow(); + expect(() => load('scope-dup', { ...base, semantic: { scope: ['parent', 'parent'] } })).toThrow(); + expect(() => load('scope-string', { ...base, semantic: { scope: 'parent' } })).toThrow(); + expect(() => load('content-false', { ...base, semantic: { includeContent: false } })).toThrow(); + expect(() => load('content-unknown', { ...base, semantic: { includeContent: 'all' } })).toThrow(); + expect(() => load('scope-top-level', { ...base, expression: 'true', scope: ['parent'] })).toThrow(); + }); + + it('rejects scope on a global rule, which cannot have a semantic part', () => { + expect(() => load('global-scope', { ...base, global: true, semantic: { scope: ['parent'] } })).toThrow(); + }); }); From 7771b1282afd68096ffd3381c3f8b136b7cbee2c Mon Sep 17 00:00:00 2001 From: Roger Barnes Date: Fri, 16 Oct 2026 14:38:25 +1100 Subject: [PATCH 2/6] docs: describe semantic.scope and semantic.includeContent --- AGENTS.md | 2 +- docs/concepts.md | 4 +- docs/semantic-rules.md | 38 ++++++++++++++++++- plugin/.claude-plugin/plugin.json | 2 +- .../references/schema-authoring.md | 2 +- 5 files changed, 43 insertions(+), 5 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b4a25890..a564c6c8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -59,7 +59,7 @@ There are several places that need reviewing and updating with any new feature o - `src/schema/metadata-contract.ts` — Single source of truth for the `$metadata` contract - `schemas/generated/latest/_structured_context_schema_meta.json` — Generated metaschema for the newest dialect (generated on build or with `bun run generate:schema-meta`). `schemas/generated//` folders are frozen copies, created by the `version` script (`generate:schema-meta --release`) only when a release changes the dialect; never edit them. `src/schema/meta-schemas.ts` loads them at runtime. - `src/schema/gating.ts` — Cascade gating: computes hierarchy depth and evaluates rules with subtree gating (schema/hierarchy/blocking-rule failures suppress downstream rule evaluation). See [docs/rules.md](docs/rules.md#cascade-gating). -- `src/judge/` — Semantic rule evaluation (the "judge" is its internal name): provider interface and types (`types.ts`), verdict resolution (`verdicts.ts`), per-node and grouped dispatch (`judge-node.ts`), and providers (`provider/anthropic.ts`, `provider/openai.ts`, `provider/jev.ts`, `provider/fake.ts`), provider hydration and the bundled catalogue (`factory.ts`), and `sctx providers` (`src/commands/providers.ts`). See [docs/semantic-rules.md](docs/semantic-rules.md). +- `src/judge/` — Semantic rule evaluation (the "judge" is its internal name): provider interface and types (`types.ts`), verdict resolution (`verdicts.ts`), per-node and grouped dispatch (`judge-node.ts`), related-node selection for `semantic.scope` (`context.ts`), and providers (`provider/anthropic.ts`, `provider/openai.ts`, `provider/jev.ts`, `provider/fake.ts`), provider hydration and the bundled catalogue (`factory.ts`), and `sctx providers` (`src/commands/providers.ts`). See [docs/semantic-rules.md](docs/semantic-rules.md). - `src/schema/rule-filter.ts` — `--rule-filter` parsing and rule filtering by category, severity, type or id. ## Testing diff --git a/docs/concepts.md b/docs/concepts.md index 2f02fa23..569f2c8c 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -171,10 +171,12 @@ See [docs/rules.md](rules.md) for the rules reference, including JSONata express ### Semantic rules -A **semantic rule** is a rule with a `semantic` part. It asks a model to evaluate a node, for example whether an opportunity is a customer need. The `semantic` object can carry a `rubric`, `verdicts` and a verdict `severity` map. A rule can also have an `expression`, a JSONata test that runs before the model. +A **semantic rule** is a rule with a `semantic` part. It asks a model to evaluate a node, for example whether an opportunity is a customer need. The `semantic` object can carry a `rubric`, `verdicts`, a verdict `severity` map, a `scope` and `includeContent`. A rule can also have an `expression`, a JSONata test that runs before the model. A **model provider** evaluates semantic rules: a decision model (Jev) or a chat model (Claude, OpenAI-compatible). It is a bundled provider or an entry in `modelProviders`. For each rule, the model gives one **verdict** from the rule's `semantic.verdicts`, for example `customer-need`, `vague` or `solution`. The rule's `semantic.severity` says which verdicts report a violation. Semantic rules cost money, so they are evaluated only when a run asks for it (`--semantic`); a rule's `expression` always runs. See [docs/semantic-rules.md](semantic-rules.md). +The **evaluated node** is the node a semantic rule is judged on. A rule's `semantic.scope` adds **related nodes** (parent, siblings, children, ancestors, descendants) along the main hierarchy's edges, and `semantic.includeContent` chooses whose body is sent. See [Related nodes and content](semantic-rules.md#related-nodes-and-content). + A **node ruleset group** is the set of nodes that share the same semantic rules. It forms within one evaluation step of [cascade gating](#cascade-gating): one hierarchy depth, or the set of non-hierarchy nodes. Nodes are keyed on the rules left after `when` and `expression` filtering. A model provider on the `openrouter-decisions` or `typesafe-systemone` API evaluates one group in as few decision requests as it can. A **decision request** is one call to a decisions API. It carries the state of one or more nodes and a list of decision questions. diff --git a/docs/semantic-rules.md b/docs/semantic-rules.md index 3037c218..f1790360 100644 --- a/docs/semantic-rules.md +++ b/docs/semantic-rules.md @@ -46,6 +46,8 @@ A semantic rule has the [common rule fields](rules.md) and a `semantic` object. | `semantic.rubric` | no | Tells the model what to assess. Default: `description` | | `semantic.verdicts` | no | The answers the model can give. Default: `pass` and `fail`. See [Verdicts](#verdicts) | | `semantic.severity` | if `semantic.verdicts` is set | The severity of each verdict. See [Severity](#severity) | +| `semantic.scope` | no | The related nodes to send with the evaluated node. Default: none. See [Related nodes and content](#related-nodes-and-content) | +| `semantic.includeContent` | no | Which node bodies to send. Default: none. See [Related nodes and content](#related-nodes-and-content) | A semantic rule cannot be `global`. A rule can also have an `expression`; see [Combining an expression](#combining-an-expression). @@ -107,13 +109,47 @@ The verdict is a classification. The severity says what the verdict means for th Gating uses the severity of the verdict. A verdict with `error` severity gates the node's subtree, as for any rule. `gatesDescendants` changes this, as for any rule. +## Related nodes and content + +By default the model sees the evaluated node alone. Use `semantic.scope` to send related nodes too, so the model can judge coherence across the tree, for example whether a solution addresses its parent opportunity. + +`semantic.scope` is a list. Each entry adds nodes along the edges of the schema's main hierarchy: + +| Entry | Adds | +|---|---| +| `parent` | The direct parents | +| `siblings` | The other children of the node's parents | +| `children` | The direct children | +| `ancestors` | The parents, their parents, and so on | +| `descendants` | The children, their children, and so on | + +Entries add together. A node that matches several entries is sent once, with every relation it has. An empty or absent `scope` sends the evaluated node alone. Other relationships are not followed. A schema without a hierarchy has no related nodes. + +A related node carries all its schema data, including its title and type. The model provider gets the related nodes as a flat list, each with its relation to the evaluated node. Scope sends node data only, never the verdicts of other rules, and [gating](rules.md#cascade-gating) does not change which nodes it selects. + +`semantic.includeContent` chooses which node bodies (the markdown `content`) go to the model: + +| Value | Sends | +|---|---| +| absent | No bodies, not even the evaluated node's | +| `nodeOnly` | The evaluated node's body | +| `fullScope` | The bodies of the evaluated node and of every related node. `true` means the same | + +```json +{ "id": "solution-addresses-opportunity", "category": "coherence", "type": "solution", + "description": "The solution addresses the opportunity it sits under", + "semantic": { "scope": ["parent"], "includeContent": "fullScope" } } +``` + +A global rule cannot have a `semantic` part, so it has no scope. + ## How semantic evaluation works - **Order.** On each node, structured-context runs each rule's [`expression`](#combining-an-expression) first. It sends the semantic parts of the rules that pass to the model provider. - **Grouping.** A model provider on the `openrouter-decisions` or `typesafe-systemone` API evaluates each node ruleset group (see [concepts](concepts.md#semantic-rules)) in as few decision requests as it can, with one decision question per node and rule. This includes the bundled `openrouter-openai-decisions` provider. A decision request is limited to about 32,000 estimated input tokens and 40 decision questions, so a larger group is split into several decision requests, never within a node. A node that fills a decision request on tokens is sent whole as its own decision request. A node with more than 40 rules has its rules split across decision requests. If a node's questions are split across several requests and one fails, all of that node's rules are reported as errors. Other providers, including `openai-decisions` and chat models, evaluate one node per request. Within a space, the nodes at one hierarchy depth are evaluated together, because their verdicts decide whether the next depth is evaluated. Grouping can shift verdicts and probabilities slightly compared with evaluating each node alone. - **Gating.** structured-context does not evaluate the semantic part on a gated node. [Coverage output](rules.md#coverage-output) shows which nodes were gated. - **Evaluation errors.** If the model provider fails (for example, a network error or output that structured-context cannot read), structured-context reports a violation with the suffix `(Semantic evaluation error: ...)`. An evaluation error does not gate. If one decision request fails, only the rules on the nodes in that decision request report the error. -- **Input.** The model provider gets the node's title, type, parent titles and schema data, including frontmatter. +- **Input.** The model provider gets the node's title, type, parent titles and schema data, including frontmatter. It gets the node's body, and any [related nodes](#related-nodes-and-content), only as the rule's `semantic.scope` and `semantic.includeContent` ask. Rules on one node that ask for the same related nodes and bodies share one input; rules that ask for different ones are evaluated separately. - **Output.** A violation from a semantic verdict also includes the verdict, the model's evidence and any guidance. `sctx validate-file` evaluates the semantic part only for the file and the rules on its ancestors that can gate it. Thus one file costs at most one model call for each ancestor level, plus one. diff --git a/plugin/.claude-plugin/plugin.json b/plugin/.claude-plugin/plugin.json index 4172ef25..1ede9e52 100644 --- a/plugin/.claude-plugin/plugin.json +++ b/plugin/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "structured-context", - "version": "0.12.15", + "version": "0.12.16", "description": "Structured content authoring for Obsidian — automatic validation hooks, slash commands, and agent skills to keep markdown files compliant with your schema.", "homepage": "https://github.com/mindsocket/structured-context", "repository": "https://github.com/mindsocket/structured-context", diff --git a/plugin/skills/structured-context/references/schema-authoring.md b/plugin/skills/structured-context/references/schema-authoring.md index 069beb81..a2bfc8f7 100644 --- a/plugin/skills/structured-context/references/schema-authoring.md +++ b/plugin/skills/structured-context/references/schema-authoring.md @@ -66,7 +66,7 @@ Relationships define how sub-entities (nodes inside other files) are parsed and Rules are a flat array. Each rule's `category` (`validation`, `coherence`, `workflow`, `best-practice`) supplies its default severity: `validation` → `error`, `coherence` and `workflow` → `warning`, `best-practice` → `info`. Only `error` violations fail validation; warnings and info are reported only. An optional per-rule `severity` (`error` | `warning` | `info`) overrides the category default — prefer picking the right category over overriding. -A rule has an `expression` (a JSONata test that must return `true`), a `semantic` object (a model evaluates the node against a `rubric`, which falls back to `description`), or both; a rule with neither is rejected. `global: true` runs an `expression` once for the space (no `when`, `type` or `semantic`). `semantic: {}` takes every default; it may declare `verdicts` (verdict → description, or `{ description, guidance }`; default `pass`/`fail`) with a `semantic.severity` map saying which verdicts are `error`/`warning`/`info`/`ignore` (required with custom verdicts). The top-level `severity` is that of a failed `expression` and of the default `fail` verdict. A rule's `expression` always runs and, when it fails, the `semantic` part is skipped; the `semantic` part runs only with `--semantic`. Run `sctx docs semantic-rules` for the full semantic rule reference. A `when` JSONata test (the applicability condition, any rule) limits a rule to the nodes where it returns true. An optional `gatesDescendants` controls whether a failure suppresses rules on the node's subtree (default: true for `error` severity). +A rule has an `expression` (a JSONata test that must return `true`), a `semantic` object (a model evaluates the node against a `rubric`, which falls back to `description`), or both; a rule with neither is rejected. `global: true` runs an `expression` once for the space (no `when`, `type` or `semantic`). `semantic: {}` takes every default; it may declare `verdicts` (verdict → description, or `{ description, guidance }`; default `pass`/`fail`) with a `semantic.severity` map saying which verdicts are `error`/`warning`/`info`/`ignore` (required with custom verdicts). `semantic.scope` (a list of `parent`, `siblings`, `children`, `ancestors`, `descendants` along the main hierarchy) sends related nodes with the evaluated node; `semantic.includeContent` (`nodeOnly` or `fullScope`/`true`) sends node bodies, and without it no body is sent. The top-level `severity` is that of a failed `expression` and of the default `fail` verdict. A rule's `expression` always runs and, when it fails, the `semantic` part is skipped; the `semantic` part runs only with `--semantic`. Run `sctx docs semantic-rules` for the full semantic rule reference. A `when` JSONata test (the applicability condition, any rule) limits a rule to the nodes where it returns true. An optional `gatesDescendants` controls whether a failure suppresses rules on the node's subtree (default: true for `error` severity). ## Metadata composition semantics From ef90187c4dc90291af9cc1a8d9572cd402259252 Mon Sep 17 00:00:00 2001 From: Roger Barnes Date: Fri, 16 Oct 2026 14:40:39 +1100 Subject: [PATCH 3/6] feat(judge): branch scope, shorthand for ancestors and descendants Also recommend attaching wide-scope rules at a level where parent, children or ancestors covers what they need. --- docs/concepts.md | 2 +- docs/semantic-rules.md | 3 +++ plugin/.claude-plugin/plugin.json | 2 +- .../structured-context/references/schema-authoring.md | 2 +- .../latest/_structured_context_schema_meta.json | 2 +- src/judge/context.ts | 11 ++++++++--- src/schema/metadata-contract.ts | 4 ++-- tests/judge/context.test.ts | 9 +++++++++ tests/schema/semantic-rule-contract.test.ts | 2 +- 9 files changed, 27 insertions(+), 10 deletions(-) diff --git a/docs/concepts.md b/docs/concepts.md index 569f2c8c..55d3f3fc 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -175,7 +175,7 @@ A **semantic rule** is a rule with a `semantic` part. It asks a model to evaluat A **model provider** evaluates semantic rules: a decision model (Jev) or a chat model (Claude, OpenAI-compatible). It is a bundled provider or an entry in `modelProviders`. For each rule, the model gives one **verdict** from the rule's `semantic.verdicts`, for example `customer-need`, `vague` or `solution`. The rule's `semantic.severity` says which verdicts report a violation. Semantic rules cost money, so they are evaluated only when a run asks for it (`--semantic`); a rule's `expression` always runs. See [docs/semantic-rules.md](semantic-rules.md). -The **evaluated node** is the node a semantic rule is judged on. A rule's `semantic.scope` adds **related nodes** (parent, siblings, children, ancestors, descendants) along the main hierarchy's edges, and `semantic.includeContent` chooses whose body is sent. See [Related nodes and content](semantic-rules.md#related-nodes-and-content). +The **evaluated node** is the node a semantic rule is judged on. A rule's `semantic.scope` adds **related nodes** (parent, siblings, children, ancestors, descendants, or the whole branch) along the main hierarchy's edges, and `semantic.includeContent` chooses whose body is sent. See [Related nodes and content](semantic-rules.md#related-nodes-and-content). A **node ruleset group** is the set of nodes that share the same semantic rules. It forms within one evaluation step of [cascade gating](#cascade-gating): one hierarchy depth, or the set of non-hierarchy nodes. Nodes are keyed on the rules left after `when` and `expression` filtering. A model provider on the `openrouter-decisions` or `typesafe-systemone` API evaluates one group in as few decision requests as it can. diff --git a/docs/semantic-rules.md b/docs/semantic-rules.md index f1790360..1e4353a8 100644 --- a/docs/semantic-rules.md +++ b/docs/semantic-rules.md @@ -122,9 +122,12 @@ By default the model sees the evaluated node alone. Use `semantic.scope` to send | `children` | The direct children | | `ancestors` | The parents, their parents, and so on | | `descendants` | The children, their children, and so on | +| `branch` | The ancestors and the descendants | Entries add together. A node that matches several entries is sent once, with every relation it has. An empty or absent `scope` sends the evaluated node alone. Other relationships are not followed. A schema without a hierarchy has no related nodes. +A wide scope can send many nodes, and every node sends all its schema data. Prefer to attach a rule at the level where `parent`, `children` or `ancestors` covers what it needs, rather than `descendants` or `branch` on a node near the top of a large tree. + A related node carries all its schema data, including its title and type. The model provider gets the related nodes as a flat list, each with its relation to the evaluated node. Scope sends node data only, never the verdicts of other rules, and [gating](rules.md#cascade-gating) does not change which nodes it selects. `semantic.includeContent` chooses which node bodies (the markdown `content`) go to the model: diff --git a/plugin/.claude-plugin/plugin.json b/plugin/.claude-plugin/plugin.json index 1ede9e52..2d6afc57 100644 --- a/plugin/.claude-plugin/plugin.json +++ b/plugin/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "structured-context", - "version": "0.12.16", + "version": "0.12.17", "description": "Structured content authoring for Obsidian — automatic validation hooks, slash commands, and agent skills to keep markdown files compliant with your schema.", "homepage": "https://github.com/mindsocket/structured-context", "repository": "https://github.com/mindsocket/structured-context", diff --git a/plugin/skills/structured-context/references/schema-authoring.md b/plugin/skills/structured-context/references/schema-authoring.md index a2bfc8f7..58293e58 100644 --- a/plugin/skills/structured-context/references/schema-authoring.md +++ b/plugin/skills/structured-context/references/schema-authoring.md @@ -66,7 +66,7 @@ Relationships define how sub-entities (nodes inside other files) are parsed and Rules are a flat array. Each rule's `category` (`validation`, `coherence`, `workflow`, `best-practice`) supplies its default severity: `validation` → `error`, `coherence` and `workflow` → `warning`, `best-practice` → `info`. Only `error` violations fail validation; warnings and info are reported only. An optional per-rule `severity` (`error` | `warning` | `info`) overrides the category default — prefer picking the right category over overriding. -A rule has an `expression` (a JSONata test that must return `true`), a `semantic` object (a model evaluates the node against a `rubric`, which falls back to `description`), or both; a rule with neither is rejected. `global: true` runs an `expression` once for the space (no `when`, `type` or `semantic`). `semantic: {}` takes every default; it may declare `verdicts` (verdict → description, or `{ description, guidance }`; default `pass`/`fail`) with a `semantic.severity` map saying which verdicts are `error`/`warning`/`info`/`ignore` (required with custom verdicts). `semantic.scope` (a list of `parent`, `siblings`, `children`, `ancestors`, `descendants` along the main hierarchy) sends related nodes with the evaluated node; `semantic.includeContent` (`nodeOnly` or `fullScope`/`true`) sends node bodies, and without it no body is sent. The top-level `severity` is that of a failed `expression` and of the default `fail` verdict. A rule's `expression` always runs and, when it fails, the `semantic` part is skipped; the `semantic` part runs only with `--semantic`. Run `sctx docs semantic-rules` for the full semantic rule reference. A `when` JSONata test (the applicability condition, any rule) limits a rule to the nodes where it returns true. An optional `gatesDescendants` controls whether a failure suppresses rules on the node's subtree (default: true for `error` severity). +A rule has an `expression` (a JSONata test that must return `true`), a `semantic` object (a model evaluates the node against a `rubric`, which falls back to `description`), or both; a rule with neither is rejected. `global: true` runs an `expression` once for the space (no `when`, `type` or `semantic`). `semantic: {}` takes every default; it may declare `verdicts` (verdict → description, or `{ description, guidance }`; default `pass`/`fail`) with a `semantic.severity` map saying which verdicts are `error`/`warning`/`info`/`ignore` (required with custom verdicts). `semantic.scope` (a list of `parent`, `siblings`, `children`, `ancestors`, `descendants`, `branch` (ancestors plus descendants) along the main hierarchy) sends related nodes with the evaluated node; `semantic.includeContent` (`nodeOnly` or `fullScope`/`true`) sends node bodies, and without it no body is sent. The top-level `severity` is that of a failed `expression` and of the default `fail` verdict. A rule's `expression` always runs and, when it fails, the `semantic` part is skipped; the `semantic` part runs only with `--semantic`. Run `sctx docs semantic-rules` for the full semantic rule reference. A `when` JSONata test (the applicability condition, any rule) limits a rule to the nodes where it returns true. An optional `gatesDescendants` controls whether a failure suppresses rules on the node's subtree (default: true for `error` severity). ## Metadata composition semantics diff --git a/schemas/generated/latest/_structured_context_schema_meta.json b/schemas/generated/latest/_structured_context_schema_meta.json index 2ef905f3..fe8acbd6 100644 --- a/schemas/generated/latest/_structured_context_schema_meta.json +++ b/schemas/generated/latest/_structured_context_schema_meta.json @@ -260,7 +260,7 @@ "scope": { "type": "array", "items": { - "enum": ["parent", "siblings", "children", "ancestors", "descendants"] + "enum": ["parent", "siblings", "children", "ancestors", "descendants", "branch"] }, "uniqueItems": true }, diff --git a/src/judge/context.ts b/src/judge/context.ts index 35e2a139..87ff8e5e 100644 --- a/src/judge/context.ts +++ b/src/judge/context.ts @@ -4,7 +4,7 @@ import type { SpaceGraph } from '../space-graph'; import type { Rule, SpaceNode } from '../types'; import type { RelatedNode, SemanticJudgeInput } from './types'; -export type SemanticScope = (typeof SEMANTIC_SCOPES)[number]; +export type SemanticScope = Exclude<(typeof SEMANTIC_SCOPES)[number], 'branch'>; export type ContentMode = 'none' | 'nodeOnly' | 'fullScope'; /** What a rule asks to send besides the evaluated node's data. */ @@ -13,12 +13,17 @@ export interface RuleContext { content: ContentMode; } -/** The related nodes and bodies a semantic rule asks for. `includeContent: true` means `fullScope`. */ +/** + * The related nodes and bodies a semantic rule asks for. `branch` expands to `ancestors` and `descendants`; + * `includeContent: true` means `fullScope`. + */ export function ruleContext(rule: Rule): RuleContext { const semantic = ruleSemantic(rule); const include = semantic?.includeContent; + const asked = new Set(semantic?.scope ?? []); + if (asked.has('branch')) asked.add('ancestors').add('descendants'); return { - scope: SEMANTIC_SCOPES.filter((s) => semantic?.scope?.includes(s)), + scope: SEMANTIC_SCOPES.filter((s): s is SemanticScope => s !== 'branch' && asked.has(s)), content: include === undefined ? 'none' : include === true ? 'fullScope' : include, }; } diff --git a/src/schema/metadata-contract.ts b/src/schema/metadata-contract.ts index a3f96483..b8ac254e 100644 --- a/src/schema/metadata-contract.ts +++ b/src/schema/metadata-contract.ts @@ -60,8 +60,8 @@ const SEVERITY_SCHEMA = { enum: ['error', 'warning', 'info'] } as const; /** The verdicts the evaluation reserves besides a rule's own; a verdict map may set a severity for them. */ export const RESERVED_VERDICTS = ['missing-context'] as const; -/** Related-node kinds a semantic rule's `scope` can add: main-hierarchy edges only. */ -export const SEMANTIC_SCOPES = ['parent', 'siblings', 'children', 'ancestors', 'descendants'] as const; +/** Related-node kinds a semantic rule's `scope` can add: main-hierarchy edges only. `branch` is ancestors plus descendants. */ +export const SEMANTIC_SCOPES = ['parent', 'siblings', 'children', 'ancestors', 'descendants', 'branch'] as const; const VERDICT_SEVERITY_SCHEMA = { enum: ['error', 'warning', 'info', 'ignore'] } as const; diff --git a/tests/judge/context.test.ts b/tests/judge/context.test.ts index c1fd1578..0d256224 100644 --- a/tests/judge/context.test.ts +++ b/tests/judge/context.test.ts @@ -94,6 +94,15 @@ describe('ruleContext', () => { }); expect(ruleContext(rule({ includeContent: true })).content).toBe('fullScope'); }); + + it('expands branch to ancestors and descendants', () => { + expect(ruleContext(rule({ scope: ['branch'] })).scope).toEqual(['ancestors', 'descendants']); + expect(ruleContext(rule({ scope: ['branch', 'parent', 'ancestors'] })).scope).toEqual([ + 'parent', + 'ancestors', + 'descendants', + ]); + }); }); describe('judgeNodes with scope', () => { diff --git a/tests/schema/semantic-rule-contract.test.ts b/tests/schema/semantic-rule-contract.test.ts index 5151075e..3d9df09f 100644 --- a/tests/schema/semantic-rule-contract.test.ts +++ b/tests/schema/semantic-rule-contract.test.ts @@ -104,7 +104,7 @@ describe('rule contract: expression and semantic', () => { }); it('accepts scope and includeContent on a semantic part, and nowhere else', () => { - const scope = ['parent', 'siblings', 'children', 'ancestors', 'descendants']; + const scope = ['parent', 'siblings', 'children', 'ancestors', 'descendants', 'branch']; expect(load('scope-ok', { ...base, semantic: { scope, includeContent: 'fullScope' } }).rules).toHaveLength(1); expect(load('scope-empty', { ...base, semantic: { scope: [], includeContent: 'nodeOnly' } }).rules).toHaveLength(1); expect(load('content-true', { ...base, semantic: { includeContent: true } }).rules).toHaveLength(1); From 07b8a63a946f5aef76db634fa348930e14920806 Mon Sep 17 00:00:00 2001 From: Roger Barnes Date: Fri, 16 Oct 2026 14:44:24 +1100 Subject: [PATCH 4/6] feat(judge): ancestors and descendants imply parent and children Equal node sets now give equal inputs, and a direct parent or child keeps its relation label. --- docs/semantic-rules.md | 2 +- src/judge/context.ts | 6 +++++- tests/judge/context.test.ts | 13 ++++++------- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/docs/semantic-rules.md b/docs/semantic-rules.md index 1e4353a8..ce5fd336 100644 --- a/docs/semantic-rules.md +++ b/docs/semantic-rules.md @@ -124,7 +124,7 @@ By default the model sees the evaluated node alone. Use `semantic.scope` to send | `descendants` | The children, their children, and so on | | `branch` | The ancestors and the descendants | -Entries add together. A node that matches several entries is sent once, with every relation it has. An empty or absent `scope` sends the evaluated node alone. Other relationships are not followed. A schema without a hierarchy has no related nodes. +Entries add together. A node that matches several entries is sent once, with every relation it has. `ancestors` includes `parent`, and `descendants` includes `children`, so a direct parent or child is always labelled as one. An empty or absent `scope` sends the evaluated node alone. Other relationships are not followed. A schema without a hierarchy has no related nodes. A wide scope can send many nodes, and every node sends all its schema data. Prefer to attach a rule at the level where `parent`, `children` or `ancestors` covers what it needs, rather than `descendants` or `branch` on a node near the top of a large tree. diff --git a/src/judge/context.ts b/src/judge/context.ts index 87ff8e5e..52e67dd5 100644 --- a/src/judge/context.ts +++ b/src/judge/context.ts @@ -14,7 +14,8 @@ export interface RuleContext { } /** - * The related nodes and bodies a semantic rule asks for. `branch` expands to `ancestors` and `descendants`; + * The related nodes and bodies a semantic rule asks for. `branch` expands to `ancestors` and `descendants`, which + * imply `parent` and `children`; * `includeContent: true` means `fullScope`. */ export function ruleContext(rule: Rule): RuleContext { @@ -22,6 +23,9 @@ export function ruleContext(rule: Rule): RuleContext { const include = semantic?.includeContent; const asked = new Set(semantic?.scope ?? []); if (asked.has('branch')) asked.add('ancestors').add('descendants'); + // A wider scope implies the narrower one, so equal node sets give equal inputs and direct relations keep their label. + if (asked.has('ancestors')) asked.add('parent'); + if (asked.has('descendants')) asked.add('children'); return { scope: SEMANTIC_SCOPES.filter((s): s is SemanticScope => s !== 'branch' && asked.has(s)), content: include === undefined ? 'none' : include === true ? 'fullScope' : include, diff --git a/tests/judge/context.test.ts b/tests/judge/context.test.ts index 0d256224..cf740b18 100644 --- a/tests/judge/context.test.ts +++ b/tests/judge/context.test.ts @@ -95,13 +95,12 @@ describe('ruleContext', () => { expect(ruleContext(rule({ includeContent: true })).content).toBe('fullScope'); }); - it('expands branch to ancestors and descendants', () => { - expect(ruleContext(rule({ scope: ['branch'] })).scope).toEqual(['ancestors', 'descendants']); - expect(ruleContext(rule({ scope: ['branch', 'parent', 'ancestors'] })).scope).toEqual([ - 'parent', - 'ancestors', - 'descendants', - ]); + it('expands branch, and wider scopes imply the direct relation', () => { + const all: RuleContext['scope'] = ['parent', 'children', 'ancestors', 'descendants']; + expect(ruleContext(rule({ scope: ['branch'] })).scope).toEqual(all); + expect(ruleContext(rule({ scope: ['branch', 'parent', 'ancestors'] })).scope).toEqual(all); + expect(ruleContext(rule({ scope: ['ancestors'] })).scope).toEqual(['parent', 'ancestors']); + expect(ruleContext(rule({ scope: ['descendants'] })).scope).toEqual(['children', 'descendants']); }); }); From f4dc6ebb69f7d98b17fa3049b332fe92ba7b753b Mon Sep 17 00:00:00 2001 From: Roger Barnes Date: Fri, 16 Oct 2026 16:47:13 +1100 Subject: [PATCH 5/6] refactor(judge): simplify related-node context plumbing Per-rule providers take the whole judge input, so new input fields need no signature change. Related nodes keep nearest-first selection order instead of an alphabetical re-sort. Drop the judgeNode wrapper, the body fallback and the includeContent oneOf. --- .../_structured_context_schema_meta.json | 9 +---- src/judge/context.ts | 37 ++++++------------- src/judge/judge-node.ts | 14 +------ src/judge/prompt.ts | 5 +-- src/judge/provider/anthropic.ts | 16 ++------ src/judge/provider/openai.ts | 16 ++------ src/judge/provider/per-rule.ts | 16 ++------ src/judge/types.ts | 4 +- src/schema/metadata-contract.ts | 2 +- tests/judge/judge-node.test.ts | 5 ++- tests/judge/prompt.test.ts | 20 ++++++---- 11 files changed, 45 insertions(+), 99 deletions(-) diff --git a/schemas/generated/latest/_structured_context_schema_meta.json b/schemas/generated/latest/_structured_context_schema_meta.json index fe8acbd6..45f612fe 100644 --- a/schemas/generated/latest/_structured_context_schema_meta.json +++ b/schemas/generated/latest/_structured_context_schema_meta.json @@ -265,14 +265,7 @@ "uniqueItems": true }, "includeContent": { - "oneOf": [ - { - "enum": ["nodeOnly", "fullScope"] - }, - { - "const": true - } - ] + "enum": ["nodeOnly", "fullScope", true] } }, "additionalProperties": false diff --git a/src/judge/context.ts b/src/judge/context.ts index 52e67dd5..48c874f9 100644 --- a/src/judge/context.ts +++ b/src/judge/context.ts @@ -13,11 +13,7 @@ export interface RuleContext { content: ContentMode; } -/** - * The related nodes and bodies a semantic rule asks for. `branch` expands to `ancestors` and `descendants`, which - * imply `parent` and `children`; - * `includeContent: true` means `fullScope`. - */ +/** The related nodes and bodies a semantic rule asks for. `includeContent: true` means `fullScope`. */ export function ruleContext(rule: Rule): RuleContext { const semantic = ruleSemantic(rule); const include = semantic?.includeContent; @@ -32,18 +28,15 @@ export function ruleContext(rule: Rule): RuleContext { }; } -const bodyOf = (node: SpaceNode): string | undefined => - node.content ?? (typeof node.schemaData.content === 'string' ? node.schemaData.content : undefined); - -/** All schema data except the body, plus the body when asked for. */ +/** All schema data, without the body unless asked for. */ function dataWithBody(node: SpaceNode, withBody: boolean): Record { + if (withBody) return node.schemaData; const { content: _content, ...data } = node.schemaData; - const body = bodyOf(node); - return withBody && body !== undefined ? { ...data, content: body } : data; + return data; } /** The evaluated node as a judge sees it. */ -export function judgeNodeView(node: SpaceNode, withBody: boolean): SemanticJudgeInput['node'] { +function judgeNodeView(node: SpaceNode, withBody: boolean): SemanticJudgeInput['node'] { return { label: node.label, title: node.title, @@ -54,16 +47,16 @@ export function judgeNodeView(node: SpaceNode, withBody: boolean): SemanticJudge }; } -function relatedView(node: SpaceNode, relation: string, withBody: boolean): RelatedNode { +type Relation = RelatedNode['relations'][number]; + +function relatedView(node: SpaceNode, relation: Relation, withBody: boolean): RelatedNode { const { title: _title, type: _type, ...data } = dataWithBody(node, withBody); return { relations: [relation], title: node.title, type: node.resolvedType, data }; } -const RELATION_ORDER = ['parent', 'sibling', 'child', 'ancestor', 'descendant']; - /** * The nodes a scope adds for a node, along main-hierarchy edges only. Selection reads the graph's structure and - * nothing else: not verdicts, not gating. Each node appears once, with every relation it has. + * nothing else: not verdicts, not gating. Each node appears once, with every relation it has, nearest first. */ export function relatedNodes(graph: SpaceGraph, node: SpaceNode, context: RuleContext): RelatedNode[] { const withBody = context.content === 'fullScope'; @@ -78,14 +71,13 @@ export function relatedNodes(graph: SpaceGraph, node: SpaceNode, context: RuleCo const childrenOf = (n: SpaceNode) => graph.hierarchyChildren.get(n.title) ?? []; const found = new Map(); - const add = (other: SpaceNode, relation: string) => { + const add = (other: SpaceNode, relation: Relation) => { if (other.title === node.title) return; const existing = found.get(other.title); if (!existing) found.set(other.title, relatedView(other, relation, withBody)); else if (!existing.relations.includes(relation)) existing.relations.push(relation); }; - /** Every node reachable by repeatedly following `next`, each visited once. */ - const reach = (next: (n: SpaceNode) => readonly SpaceNode[], relation: string) => { + const reach = (next: (n: SpaceNode) => readonly SpaceNode[], relation: Relation) => { const seen = new Set([node.title]); let level = next(node); while (level.length > 0) { @@ -106,14 +98,9 @@ export function relatedNodes(graph: SpaceGraph, node: SpaceNode, context: RuleCo if (scope.has('children')) for (const c of childrenOf(node)) add(c, 'child'); if (scope.has('ancestors')) reach(parentsOf, 'ancestor'); if (scope.has('descendants')) reach(childrenOf, 'descendant'); - - const rank = (r: RelatedNode) => Math.min(...r.relations.map((x) => RELATION_ORDER.indexOf(x))); - return [...found.values()].sort((a, b) => rank(a) - rank(b) || a.title.localeCompare(b.title)); + return [...found.values()]; } -/** Stable key for rules that share a context, so they can share a provider input. */ -export const contextKey = (context: RuleContext) => `${context.content}|${context.scope.join(',')}`; - /** The judge input context for rules that share `context`: the node view and its related nodes. */ export function judgeContext( graph: SpaceGraph, diff --git a/src/judge/judge-node.ts b/src/judge/judge-node.ts index 0910b31a..63167432 100644 --- a/src/judge/judge-node.ts +++ b/src/judge/judge-node.ts @@ -1,6 +1,6 @@ import type { SpaceGraph } from '../space-graph'; import type { Rule, RuleSeverity, SpaceNode } from '../types'; -import { contextKey, judgeContext, type RuleContext, ruleContext } from './context'; +import { judgeContext, type RuleContext, ruleContext } from './context'; import type { SemanticJudgeInput, SemanticJudgeProvider } from './types'; import { isJudgeError, resolveSemanticRule } from './verdicts'; @@ -70,7 +70,7 @@ function planUnits(graph: SpaceGraph, items: Array<{ node: SpaceNode; rules: Rul const groups = new Map(); rules.forEach((rule, position) => { const context = ruleContext(rule); - const key = contextKey(context); + const key = `${context.content}|${context.scope.join(',')}`; const group = groups.get(key) ?? { context, positions: [] }; group.positions.push(position); groups.set(key, group); @@ -90,16 +90,6 @@ function planUnits(graph: SpaceGraph, items: Array<{ node: SpaceNode; rules: Rul return units; } -/** Judge every given semantic rule on a node. */ -export async function judgeNode( - judge: SemanticJudgeProvider, - node: SpaceNode, - rules: Rule[], - graph: SpaceGraph, -): Promise { - return (await judgeNodes(judge, [{ node, rules }], graph))[0] ?? []; -} - /** * Judge several nodes at once. A judge with `evaluateNodes` gets every input in one call and may fold them into * fewer requests; any other judge is called once per input. A node's rules that share a context (related nodes diff --git a/src/judge/prompt.ts b/src/judge/prompt.ts index 94a14b0b..dab1a5a2 100644 --- a/src/judge/prompt.ts +++ b/src/judge/prompt.ts @@ -1,4 +1,4 @@ -import type { RelatedNode, SemanticJudgeInput, SemanticJudgeResult, SemanticRuleSpec } from './types'; +import type { SemanticJudgeInput, SemanticJudgeResult, SemanticRuleSpec } from './types'; function verdictList(rule: SemanticRuleSpec): string[] { return Object.entries(rule.verdicts).map(([name, description]) => `- "${name}": ${description}`); @@ -27,9 +27,8 @@ export function buildJudgeSystemPrompt(): string { /** User prompt for one rule on one node. */ export function buildJudgeUserPrompt( - node: SemanticJudgeInput['node'], + { node, related = [] }: Pick, rule: SemanticRuleSpec, - related: RelatedNode[] = [], ): string { const rubric = rule.rubric || rule.description; const lines: string[] = [ diff --git a/src/judge/provider/anthropic.ts b/src/judge/provider/anthropic.ts index 740d6366..e12b4460 100644 --- a/src/judge/provider/anthropic.ts +++ b/src/judge/provider/anthropic.ts @@ -1,11 +1,5 @@ import { buildJudgeSystemPrompt, buildJudgeUserPrompt, parseJudgeResponse } from '../prompt'; -import type { - HydratedProvider, - RelatedNode, - SemanticJudgeInput, - SemanticJudgeResult, - SemanticRuleSpec, -} from '../types'; +import type { HydratedProvider, SemanticJudgeInput, SemanticJudgeResult, SemanticRuleSpec } from '../types'; import { PerRuleJudgeProvider } from './per-rule'; export interface AnthropicJudgeProviderOptions extends HydratedProvider { @@ -30,11 +24,7 @@ export class AnthropicJudgeProvider extends PerRuleJudgeProvider { this.endpoint = options.endpoint || ANTHROPIC_MESSAGES_ENDPOINT; } - protected async judgeRule( - node: SemanticJudgeInput['node'], - rule: SemanticRuleSpec, - related: RelatedNode[], - ): Promise { + protected async judgeRule(input: SemanticJudgeInput, rule: SemanticRuleSpec): Promise { const res = await fetch(this.endpoint, { method: 'POST', headers: { @@ -46,7 +36,7 @@ export class AnthropicJudgeProvider extends PerRuleJudgeProvider { model: this.model, max_tokens: 1024, system: buildJudgeSystemPrompt(), - messages: [{ role: 'user', content: buildJudgeUserPrompt(node, rule, related) }], + messages: [{ role: 'user', content: buildJudgeUserPrompt(input, rule) }], }), }); diff --git a/src/judge/provider/openai.ts b/src/judge/provider/openai.ts index 1505bba3..d85fb51f 100644 --- a/src/judge/provider/openai.ts +++ b/src/judge/provider/openai.ts @@ -1,11 +1,5 @@ import { buildJudgeSystemPrompt, buildJudgeUserPrompt, parseJudgeResponse } from '../prompt'; -import type { - HydratedProvider, - RelatedNode, - SemanticJudgeInput, - SemanticJudgeResult, - SemanticRuleSpec, -} from '../types'; +import type { HydratedProvider, SemanticJudgeInput, SemanticJudgeResult, SemanticRuleSpec } from '../types'; import { PerRuleJudgeProvider } from './per-rule'; export interface OpenAIJudgeProviderOptions extends HydratedProvider { @@ -35,13 +29,9 @@ export class OpenAIJudgeProvider extends PerRuleJudgeProvider { this.endpoint = base.endsWith('/chat/completions') ? base : `${base.replace(/\/+$/, '')}/chat/completions`; } - protected async judgeRule( - node: SemanticJudgeInput['node'], - rule: SemanticRuleSpec, - related: RelatedNode[], - ): Promise { + protected async judgeRule(input: SemanticJudgeInput, rule: SemanticRuleSpec): Promise { const systemPrompt = buildJudgeSystemPrompt(); - const userPrompt = buildJudgeUserPrompt(node, rule, related); + const userPrompt = buildJudgeUserPrompt(input, rule); const headers: Record = { 'content-type': 'application/json', diff --git a/src/judge/provider/per-rule.ts b/src/judge/provider/per-rule.ts index b9dbf613..b00ad34c 100644 --- a/src/judge/provider/per-rule.ts +++ b/src/judge/provider/per-rule.ts @@ -1,10 +1,4 @@ -import type { - RelatedNode, - SemanticJudgeInput, - SemanticJudgeProvider, - SemanticJudgeResult, - SemanticRuleSpec, -} from '../types'; +import type { SemanticJudgeInput, SemanticJudgeProvider, SemanticJudgeResult, SemanticRuleSpec } from '../types'; /** * Base for providers that can only judge one rule per request (chat LLMs). `evaluate` makes one call @@ -13,17 +7,13 @@ import type { export abstract class PerRuleJudgeProvider implements SemanticJudgeProvider { abstract readonly name: string; - protected abstract judgeRule( - node: SemanticJudgeInput['node'], - rule: SemanticRuleSpec, - related: RelatedNode[], - ): Promise; + protected abstract judgeRule(input: SemanticJudgeInput, rule: SemanticRuleSpec): Promise; async evaluate(input: SemanticJudgeInput): Promise> { const results: Record = {}; for (const rule of input.rules) { try { - results[rule.id] = await this.judgeRule(input.node, rule, input.related ?? []); + results[rule.id] = await this.judgeRule(input, rule); } catch (err) { results[rule.id] = { error: err instanceof Error ? err.message : String(err) }; } diff --git a/src/judge/types.ts b/src/judge/types.ts index e3a2b025..4af7b129 100644 --- a/src/judge/types.ts +++ b/src/judge/types.ts @@ -15,8 +15,8 @@ export interface SemanticRuleSpec { * (`data.content`) is present only when the rules' `includeContent` asks for it. */ export interface RelatedNode { - /** How it relates to the evaluated node: `parent`, `sibling`, `child`, `ancestor`, `descendant`. A node can have several. */ - relations: string[]; + /** How it relates to the evaluated node. A node can have several. */ + relations: Array<'parent' | 'sibling' | 'child' | 'ancestor' | 'descendant'>; title: string; type: string; /** Schema data without `title` and `type`. */ diff --git a/src/schema/metadata-contract.ts b/src/schema/metadata-contract.ts index b8ac254e..b4fee0dc 100644 --- a/src/schema/metadata-contract.ts +++ b/src/schema/metadata-contract.ts @@ -114,7 +114,7 @@ const SEMANTIC_SCHEMA = { * Which node bodies are sent: `nodeOnly` (the evaluated node's), `fullScope` (it and every related node's), * or `true` for `fullScope`. Left out: no bodies. */ - includeContent: { oneOf: [{ enum: ['nodeOnly', 'fullScope'] }, { const: true }] }, + includeContent: { enum: ['nodeOnly', 'fullScope', true] }, }, additionalProperties: false, } as const; diff --git a/tests/judge/judge-node.test.ts b/tests/judge/judge-node.test.ts index ff038908..ee0cfdaa 100644 --- a/tests/judge/judge-node.test.ts +++ b/tests/judge/judge-node.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'bun:test'; -import { judgeNode as judgeNodeIn, judgeNodes as judgeNodesIn, type NodeJudgement } from '../../src/judge/judge-node'; +import { judgeNodes as judgeNodesIn, type NodeJudgement } from '../../src/judge/judge-node'; import type { SemanticJudgeProvider, SemanticJudgeResult } from '../../src/judge/types'; import type { Rule } from '../../src/schema/metadata-contract'; import { buildSpaceGraph } from '../../src/space-graph'; @@ -16,9 +16,10 @@ function canned(results: Record): SemanticJudgeProv } const graph = buildSpaceGraph([], undefined); -const judgeNode = (j: SemanticJudgeProvider, n: SpaceNode, rules: Rule[]) => judgeNodeIn(j, n, rules, graph); const judgeNodes = (j: SemanticJudgeProvider, items: Array<{ node: SpaceNode; rules: Rule[] }>) => judgeNodesIn(j, items, graph); +const judgeNode = async (j: SemanticJudgeProvider, n: SpaceNode, rules: Rule[]) => + (await judgeNodes(j, [{ node: n, rules }]))[0] ?? []; describe('judgeNode', () => { it('treats an unknown verdict as a judge error, not a pass', async () => { diff --git a/tests/judge/prompt.test.ts b/tests/judge/prompt.test.ts index 97ea440e..7e04c09e 100644 --- a/tests/judge/prompt.test.ts +++ b/tests/judge/prompt.test.ts @@ -11,7 +11,7 @@ describe('judge prompt and response parser', () => { }); it('user prompt lists the rule, its allowed verdicts and the node', () => { - const prompt = buildJudgeUserPrompt(node, graded); + const prompt = buildJudgeUserPrompt({ node }, graded); expect(prompt).toContain('framing'); expect(prompt).toContain('"weak": Vague need'); expect(prompt).toContain('"solution": A solution in disguise'); @@ -21,8 +21,8 @@ describe('judge prompt and response parser', () => { }); it('user prompt prefers the rubric over the description', () => { - expect(buildJudgeUserPrompt(node, passFailRule)).toContain('Needs to sound like a customer quote'); - expect(buildJudgeUserPrompt(node, graded)).toContain('Rubric: Framing quality'); + expect(buildJudgeUserPrompt({ node }, passFailRule)).toContain('Needs to sound like a customer quote'); + expect(buildJudgeUserPrompt({ node }, graded)).toContain('Rubric: Framing quality'); }); const allowed = ['pass', 'fail']; @@ -63,9 +63,15 @@ describe('judge prompt and response parser', () => { describe('related nodes in the chat prompt', () => { it('lists each related node with its relation', () => { - const prompt = buildJudgeUserPrompt(node, passFailRule, [ - { relations: ['parent', 'ancestor'], title: 'Faster Tax Filing', type: 'goal', data: { owner: 'finance' } }, - ]); + const prompt = buildJudgeUserPrompt( + { + node, + related: [ + { relations: ['parent', 'ancestor'], title: 'Faster Tax Filing', type: 'goal', data: { owner: 'finance' } }, + ], + }, + passFailRule, + ); expect(prompt).toContain('Related Nodes'); expect(prompt).toContain('Faster Tax Filing'); expect(prompt).toContain('Relation: parent, ancestor'); @@ -73,6 +79,6 @@ describe('related nodes in the chat prompt', () => { }); it('has no related section without related nodes', () => { - expect(buildJudgeUserPrompt(node, passFailRule)).not.toContain('Related Nodes'); + expect(buildJudgeUserPrompt({ node }, passFailRule)).not.toContain('Related Nodes'); }); }); From 61a3ecad678f42629e5b001d7a3762397ce93941 Mon Sep 17 00:00:00 2001 From: Roger Barnes Date: Fri, 16 Oct 2026 16:54:38 +1100 Subject: [PATCH 6/6] docs(semantic-rules): drop scope detail readers do not need --- docs/semantic-rules.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/semantic-rules.md b/docs/semantic-rules.md index ce5fd336..a3704792 100644 --- a/docs/semantic-rules.md +++ b/docs/semantic-rules.md @@ -124,11 +124,11 @@ By default the model sees the evaluated node alone. Use `semantic.scope` to send | `descendants` | The children, their children, and so on | | `branch` | The ancestors and the descendants | -Entries add together. A node that matches several entries is sent once, with every relation it has. `ancestors` includes `parent`, and `descendants` includes `children`, so a direct parent or child is always labelled as one. An empty or absent `scope` sends the evaluated node alone. Other relationships are not followed. A schema without a hierarchy has no related nodes. +Entries add together, and each node is sent once. An empty or absent `scope` sends the evaluated node alone. Other relationships are not followed. A wide scope can send many nodes, and every node sends all its schema data. Prefer to attach a rule at the level where `parent`, `children` or `ancestors` covers what it needs, rather than `descendants` or `branch` on a node near the top of a large tree. -A related node carries all its schema data, including its title and type. The model provider gets the related nodes as a flat list, each with its relation to the evaluated node. Scope sends node data only, never the verdicts of other rules, and [gating](rules.md#cascade-gating) does not change which nodes it selects. +Scope sends node data only, never the verdicts of other rules, and [gating](rules.md#cascade-gating) does not change which nodes it selects. `semantic.includeContent` chooses which node bodies (the markdown `content`) go to the model: @@ -152,7 +152,7 @@ A global rule cannot have a `semantic` part, so it has no scope. - **Grouping.** A model provider on the `openrouter-decisions` or `typesafe-systemone` API evaluates each node ruleset group (see [concepts](concepts.md#semantic-rules)) in as few decision requests as it can, with one decision question per node and rule. This includes the bundled `openrouter-openai-decisions` provider. A decision request is limited to about 32,000 estimated input tokens and 40 decision questions, so a larger group is split into several decision requests, never within a node. A node that fills a decision request on tokens is sent whole as its own decision request. A node with more than 40 rules has its rules split across decision requests. If a node's questions are split across several requests and one fails, all of that node's rules are reported as errors. Other providers, including `openai-decisions` and chat models, evaluate one node per request. Within a space, the nodes at one hierarchy depth are evaluated together, because their verdicts decide whether the next depth is evaluated. Grouping can shift verdicts and probabilities slightly compared with evaluating each node alone. - **Gating.** structured-context does not evaluate the semantic part on a gated node. [Coverage output](rules.md#coverage-output) shows which nodes were gated. - **Evaluation errors.** If the model provider fails (for example, a network error or output that structured-context cannot read), structured-context reports a violation with the suffix `(Semantic evaluation error: ...)`. An evaluation error does not gate. If one decision request fails, only the rules on the nodes in that decision request report the error. -- **Input.** The model provider gets the node's title, type, parent titles and schema data, including frontmatter. It gets the node's body, and any [related nodes](#related-nodes-and-content), only as the rule's `semantic.scope` and `semantic.includeContent` ask. Rules on one node that ask for the same related nodes and bodies share one input; rules that ask for different ones are evaluated separately. +- **Input.** The model provider gets the node's title, type, parent titles and schema data, including frontmatter. It gets the node's body, and any [related nodes](#related-nodes-and-content), only as the rule's `semantic.scope` and `semantic.includeContent` ask. - **Output.** A violation from a semantic verdict also includes the verdict, the model's evidence and any guidance. `sctx validate-file` evaluates the semantic part only for the file and the rules on its ancestors that can gate it. Thus one file costs at most one model call for each ancestor level, plus one.