From c27080e132c9585340d58a899450154c516f6f94 Mon Sep 17 00:00:00 2001 From: Roger Barnes Date: Fri, 18 Sep 2026 18:40:31 +1000 Subject: [PATCH 01/22] feat(rules): semantic rules judged by model or decision model (#114) Semantic rules (mode: semantic) declare a closed verdict set (verdicts, failOn) and are judged per node, all of a node's rules in one provider call. Judges: Anthropic (default claude-haiku-5-5), OpenAI-compatible, Jev via OpenRouter decisions API, fake. Semantic rules run inside cascade gating (#115) after deterministic rules and only with --semantic. --rule-filter narrows rules by mode, category, severity, type or id. --- README.md | 3 +- docs/rules.md | 56 ++++- plugin/.claude-plugin/plugin.json | 2 +- .../structured-context/references/commands.md | 5 +- schemas/_ost_strict.json | 8 + .../_structured_context_schema_meta.json | 26 ++- src/api.ts | 16 +- src/commands/validate-file.ts | 9 +- src/commands/validate.ts | 39 +++- src/config.ts | 25 +++ src/index.ts | 21 +- src/judge/anthropic.ts | 55 +++++ src/judge/factory.ts | 70 +++++++ src/judge/fake.ts | 69 +++++++ src/judge/index.ts | 22 ++ src/judge/jev.ts | 94 +++++++++ src/judge/judge-node.ts | 68 +++++++ src/judge/openai.ts | 68 +++++++ src/judge/per-rule.ts | 23 +++ src/judge/prompt.ts | 92 +++++++++ src/judge/types.ts | 73 +++++++ src/judge/verdicts.ts | 39 ++++ src/schema/gating.ts | 85 ++++++-- src/schema/metadata-contract.ts | 8 +- src/schema/rule-filter.ts | 55 +++++ src/schema/validate-rules.ts | 93 ++++++++- src/types.ts | 2 + src/validate.ts | 105 +++++++++- .../Progressive sign-up form.md | 7 + .../valid-directory/Social login option.md | 7 + tests/judge/factory.test.ts | 45 +++++ tests/judge/fake-provider.test.ts | 45 +++++ tests/judge/helpers.ts | 27 +++ tests/judge/jev.test.ts | 77 +++++++ tests/judge/prompt.test.ts | 62 ++++++ tests/judge/verdicts.test.ts | 47 +++++ tests/schema/rule-filter.test.ts | 39 ++++ tests/schema/validate-rules-semantic.test.ts | 168 +++++++++++++++ tests/validate/gating.test.ts | 5 +- tests/validate/semantic-validation.test.ts | 191 ++++++++++++++++++ tests/validate/strict.test.ts | 4 +- 41 files changed, 1898 insertions(+), 57 deletions(-) create mode 100644 src/judge/anthropic.ts create mode 100644 src/judge/factory.ts create mode 100644 src/judge/fake.ts create mode 100644 src/judge/index.ts create mode 100644 src/judge/jev.ts create mode 100644 src/judge/judge-node.ts create mode 100644 src/judge/openai.ts create mode 100644 src/judge/per-rule.ts create mode 100644 src/judge/prompt.ts create mode 100644 src/judge/types.ts create mode 100644 src/judge/verdicts.ts create mode 100644 src/schema/rule-filter.ts create mode 100644 tests/fixtures/strict_ost/valid-directory/Progressive sign-up form.md create mode 100644 tests/fixtures/strict_ost/valid-directory/Social login option.md create mode 100644 tests/judge/factory.test.ts create mode 100644 tests/judge/fake-provider.test.ts create mode 100644 tests/judge/helpers.ts create mode 100644 tests/judge/jev.test.ts create mode 100644 tests/judge/prompt.test.ts create mode 100644 tests/judge/verdicts.test.ts create mode 100644 tests/schema/rule-filter.test.ts create mode 100644 tests/schema/validate-rules-semantic.test.ts create mode 100644 tests/validate/semantic-validation.test.ts diff --git a/README.md b/README.md index efb867a..927834d 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,7 @@ See [docs/schemas.md](docs/schemas.md) (`sctx docs schema`) for the dialect, com ### Validate nodes ```bash -sctx validate [--watch] [--depth ] [--include-gated] +sctx validate [--watch] [--depth ] [--include-gated] [--semantic] [--rule-filter ] ``` Validates markdown files against the JSON schema: @@ -141,6 +141,7 @@ Validates markdown files against the JSON schema: - Runs executable rules (`$metadata.rules`) with **cascade gating**: a schema, hierarchy, or blocking-rule failure on a node suppresses rule evaluation for that node's hierarchy subtree. Gated checks are reported in a Coverage section so you know what wasn't evaluated and why. See [docs/rules.md](docs/rules.md#cascade-gating) for details. - `--depth ` limits evaluation to hierarchy nodes within `n` breadth-first levels from the top — useful for staged agentic fix loops (fix the top level, then request the next). - `--include-gated` evaluates rules even on gated nodes (still reported as gated) — for surfacing cheap bottom-up checks on request. +- `--semantic` also evaluates `mode: semantic` rules with a model judge (Anthropic, OpenAI-compatible, or Jev); `--rule-filter` narrows which rules run. See [docs/rules.md](docs/rules.md#semantic-rules). ### Render space diff --git a/docs/rules.md b/docs/rules.md index d788963..f2a3e63 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -30,7 +30,11 @@ Rules are a flat array. | `category` | yes | `validation` \| `coherence` \| `workflow` \| `best-practice` | | `severity` | no | `error` \| `warning` \| `info`; overrides the category default (see [Severity](#severity)) | | `description` | yes | Human-readable rule intent | -| `check` | yes | JSONata expression; must evaluate to `true` | +| `check` | yes, unless `mode: semantic` | JSONata expression; must evaluate to `true` | +| `mode` | no | `expression` (default) or `semantic` (judged by a model; see [Semantic rules](#semantic-rules)) | +| `rubric` | semantic only | Detailed prompt for the judge; falls back to `description` | +| `verdicts` | semantic only | Closed set of verdicts the judge must choose from: `{ "": "" }`. Default `pass` / `fail`. | +| `failOn` | semantic only | Verdicts that count as a violation. Default `["fail"]`; required when `verdicts` is declared. At least one verdict must pass. | | `type` | no | Restrict rule to nodes of this `resolvedType` | | `scope` | no | Use `"global"` to evaluate once for the whole space | | `override` | no | Only for merge conflicts: allows later duplicate `id` to replace earlier | @@ -160,7 +164,7 @@ original blocker), rather than each generating a new one. Non-hierarchy nodes an `scope: "global"` rules are never gated — cascade gating only applies within the hierarchy DAG. -This exists so that expensive or semantic downstream checks (see #114) aren't wasted +This exists so that expensive or semantic downstream checks (see [Semantic rules](#semantic-rules)) aren't wasted evaluating nodes whose ancestors are already known-bad, and so noise doesn't drown out the actionable failure. Set `gatesDescendants: false` on an error-severity rule whose violations shouldn't suppress checks further down the tree, or `gatesDescendants: true` on a warning/info rule that should. @@ -188,6 +192,54 @@ the top (roots and orphans are depth 1). This is a manual, coarser control than gating — intended for staged workflows (e.g. an agent fixes the top level, then requests the next). Excluded nodes are reported under `skippedByDepth`, separate from gated nodes. +## Semantic rules + +A rule with `"mode": "semantic"` is judged by a model instead of a JSONata expression. Semantic rules cost money and time, so they never run unless asked: pass `--semantic` (or set `semantic: true` / `judge.autoRun` on a space or in config). + +```json +{ + "id": "opportunity-customer-voice", + "category": "best-practice", + "mode": "semantic", + "type": "opportunity", + "description": "Opportunity is framed in the customer's voice", + "rubric": "The opportunity reads as a customer need or pain point, not a solution in disguise or a business metric.", + "verdicts": { + "customer-need": "Clearly a customer need or pain point", + "vague": "A need, but too vague to act on", + "solution": "A solution or business goal in disguise" + }, + "failOn": ["solution"] +} +``` + +The judge must pick exactly one verdict from `verdicts` (default `pass`/`fail`); a verdict listed in `failOn` is a violation, everything else passes. A closed set maps directly onto decision models, which return a choice with confidence and probabilities rather than prose. + +- **Evaluation order.** For each node, deterministic rules run first, then all of the node's semantic rules go to the judge in one call. Providers that can only answer one question per request (chat LLMs) make one request per rule; decision models such as Jev answer all of a node's rules in a single request. +- **Gating.** Semantic rules follow [cascade gating](#cascade-gating): they are not run on gated nodes (recorded as `gatedRules`), and a failing semantic rule gates its subtree when `gatesDescendants` is true (default for `error` severity). A judge *error* (network, unparseable output) is reported as a violation but never gates. +- **Output.** `--json` includes `semanticEvaluations` (verdict, `pass`, confidence, evidence per node/rule) and `gatedRules`. Violations carry the judge's `evidence`; decision models without prose get a summary of the verdict distribution. +- **Scope.** Semantic rules judge a node in isolation, plus its parent titles. (Wider scopes such as ancestors or siblings are not yet supported.) + +### Judge configuration + +Set `judge` at the root of `config.json` or per space (the space wins): + +```json +{ "judge": { "provider": "anthropic", "model": "claude-haiku-5-5" } } +``` + +| Field | Description | +|---|---| +| `provider` | `anthropic`, `openai` (any OpenAI-compatible endpoint, including Ollama and OpenRouter chat models), `jev`, or `fake` (tests). Omitted: inferred from `ANTHROPIC_API_KEY`, then `OPENAI_API_KEY`/`OPENAI_BASE_URL`. `jev` is never inferred. | +| `model` | Defaults: `claude-haiku-5-5` (anthropic), `gpt-4o-mini` (openai), `typesafe/jev-1.13` (jev). | +| `endpoint` | Override the API URL. Jev defaults to OpenRouter's `https://openrouter.ai/api/alpha/decisions`. | +| `apiKey` | Prefer environment variables: `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `OPENROUTER_API_KEY` (jev). `SCTX_JUDGE_*` variables override provider, model, endpoint and key. | +| `autoRun` | Run semantic rules without `--semantic`. | + +### Filtering rules + +`--rule-filter =[,]` (repeatable) evaluates only matching rules. Dimensions are `mode`, `category`, `severity` (effective severity), `type` and `id`; different dimensions combine with AND, values within one with OR. For example, `--semantic --rule-filter mode=semantic` runs only the judged rules, and `--rule-filter mode=expression` skips them entirely. Gating is computed from the rules that ran. + ## Common patterns ```jsonata diff --git a/plugin/.claude-plugin/plugin.json b/plugin/.claude-plugin/plugin.json index 037f46f..c1f1c21 100644 --- a/plugin/.claude-plugin/plugin.json +++ b/plugin/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "structured-context", - "version": "0.12.6", + "version": "0.12.7", "description": "Structured content authoring for Obsidian — automatic validation hooks, slash commands, and agent skills to keep markdown files compliant with your schema.", "homepage": "https://github.com/mindsocket/structured-context", "repository": "https://github.com/mindsocket/structured-context", diff --git a/plugin/skills/structured-context/references/commands.md b/plugin/skills/structured-context/references/commands.md index f2bd438..4781908 100644 --- a/plugin/skills/structured-context/references/commands.md +++ b/plugin/skills/structured-context/references/commands.md @@ -11,7 +11,7 @@ Then use the `sctx` command for all operations. Always pass `--config ` wh ## validate ```bash -sctx validate [--watch] [--config ] [--depth ] [--include-gated] +sctx validate [--watch] [--config ] [--depth ] [--include-gated] [--semantic] [--rule-filter ] ``` Validates all `.md` files in the space against the JSON schema. For each file: @@ -29,6 +29,9 @@ Validates all `.md` files in the space against the JSON schema. For each file: the top — useful for fixing a space level by level (fix the top, re-run with a deeper `--depth`, repeat). `--include-gated` evaluates rules on gated nodes anyway, still reporting them as gated — use it to surface cheap bottom-up checks even when an upstream blocker fired. +`--semantic` also runs `mode: semantic` rules through a configured model judge (costly; off by +default), and `--rule-filter =` (repeatable; mode, category, severity, type, id) +narrows which rules run. `validate-file` accepts the same two flags. Exit codes: `0` = clean (warning and info rule violations don't fail), `1` = validation errors found. diff --git a/schemas/_ost_strict.json b/schemas/_ost_strict.json index 86b5acb..08fad84 100644 --- a/schemas/_ost_strict.json +++ b/schemas/_ost_strict.json @@ -35,6 +35,14 @@ "description": "Explore multiple candidate solutions (aim for at least three) for the target opportunity", "type": "opportunity", "check": "(current.status != 'exploring' and current.status != 'active') or $count(nodes[resolvedParentTitle=$$.current.title and resolvedType='solution']) >= 3" + }, + { + "id": "opportunity-customer-voice", + "category": "best-practice", + "mode": "semantic", + "type": "opportunity", + "description": "Opportunity must be framed in the customer's voice (something a customer would actually say) as a need or pain point, not a solution or business goal", + "rubric": "Verify that the opportunity is stated from the customer's perspective as a problem, pain point, or desire. It must not be a solution in disguise (e.g. 'build X feature') or a business metric (e.g. 'increase revenue')." } ] }, diff --git a/schemas/generated/latest/_structured_context_schema_meta.json b/schemas/generated/latest/_structured_context_schema_meta.json index 02d47c2..4d64d61 100644 --- a/schemas/generated/latest/_structured_context_schema_meta.json +++ b/schemas/generated/latest/_structured_context_schema_meta.json @@ -207,6 +207,30 @@ "type": "string", "minLength": 1 }, + "mode": { + "enum": ["expression", "semantic"] + }, + "rubric": { + "type": "string", + "minLength": 1 + }, + "verdicts": { + "type": "object", + "additionalProperties": { + "type": "string", + "minLength": 1 + }, + "minProperties": 2 + }, + "failOn": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 1, + "uniqueItems": true + }, "type": { "type": "string", "minLength": 1 @@ -221,7 +245,7 @@ "type": "boolean" } }, - "required": ["id", "category", "description", "check"], + "required": ["id", "category", "description"], "additionalProperties": false } } diff --git a/src/api.ts b/src/api.ts index 9362858..1deedf9 100644 --- a/src/api.ts +++ b/src/api.ts @@ -22,6 +22,20 @@ export type { AnySchemaObject, SchemaObject, ValidateFunction } from 'ajv'; export type { Config, SpaceConfig } from './config'; export { loadConfig, resolveSchema, resolveSchemaPath, setConfigPath } from './config'; export { PLUGIN_PREFIX } from './constants'; +export type { + GatedRuleRecord, + JudgeConfig, + SemanticEvaluationRecord, + SemanticJudgeInput, + SemanticJudgeProvider, + SemanticJudgeVerdict, +} from './judge'; +export { + AnthropicJudgeProvider, + createJudgeProvider, + FakeJudgeProvider, + OpenAIJudgeProvider, +} from './judge'; export { assembleSpaceGraph, loadSpaceGraph } from './load-space-graph'; export type { ParseHook, @@ -79,5 +93,5 @@ export type { UnresolvedRef, ValidateFileOutput, } from './types'; -export type { ValidationResult } from './validate'; +export type { ValidateFileOptions, ValidateSpaceOptions, ValidationResult } from './validate'; export { validateFile, validateSpace } from './validate'; diff --git a/src/commands/validate-file.ts b/src/commands/validate-file.ts index e69eef5..61dc7f3 100644 --- a/src/commands/validate-file.ts +++ b/src/commands/validate-file.ts @@ -1,4 +1,5 @@ import { loadConfig } from '../config'; +import { parseRuleFilter } from '../schema/rule-filter'; import type { FileValidationIssue, FileValidationResult } from '../types'; import { validateFile } from '../validate'; @@ -11,10 +12,14 @@ export { validateFile } from '../validate'; */ export async function validateFileCommand( filePath: string, - options: { json?: boolean; includeGated?: boolean } = {}, + options: { json?: boolean; includeGated?: boolean; semantic?: boolean; ruleFilter?: string[] } = {}, ): Promise { const config = loadConfig(); - const result = await validateFile(filePath, config, { includeGated: options.includeGated }); + const result = await validateFile(filePath, config, { + includeGated: options.includeGated, + semantic: options.semantic, + ruleFilter: options.ruleFilter && parseRuleFilter(options.ruleFilter), + }); if (!result.inSpace) { if (options.json) { diff --git a/src/commands/validate.ts b/src/commands/validate.ts index ac9953a..5f94534 100644 --- a/src/commands/validate.ts +++ b/src/commands/validate.ts @@ -1,17 +1,32 @@ import { dirname } from 'node:path'; import chokidar from 'chokidar'; import { getConfigSourceFiles } from '../config'; +import type { SemanticJudgeProvider } from '../judge/types'; import { resolveSpacePluginSchemas } from '../plugins/util'; +import { parseRuleFilter } from '../schema/rule-filter'; import { bundledSchemasDir } from '../schema/schema'; import type { FileValidationIssue, RuleSeverity, SpaceContext } from '../types'; import { formatErrors, validateSpace } from '../validate'; export async function validate( context: SpaceContext, - options: { json?: boolean; depth?: number; includeGated?: boolean } = {}, + options: { + json?: boolean; + depth?: number; + includeGated?: boolean; + semantic?: boolean; + ruleFilter?: string[]; + judgeProvider?: SemanticJudgeProvider; + } = {}, ): Promise { const { schema, schemaRefRegistry } = context; - const result = await validateSpace(context, { depth: options.depth, includeGated: options.includeGated }); + const result = await validateSpace(context, { + depth: options.depth, + includeGated: options.includeGated, + semantic: options.semantic, + ruleFilter: options.ruleFilter && parseRuleFilter(options.ruleFilter), + judgeProvider: options.judgeProvider, + }); const ruleCounts: Record = { error: 0, warning: 0, info: 0 }; for (const v of result.ruleViolations) ruleCounts[v.severity]++; @@ -46,7 +61,10 @@ export async function validate( } for (const v of result.ruleViolations) { if (v.file && v.severity === 'error') { - addError(v.file, `rule:${v.ruleId}`, 'rule', `[${v.ruleId}] ${v.description}`, v.severity); + const message = v.evidence + ? `[${v.ruleId}] ${v.description} (Evidence: ${v.evidence})` + : `[${v.ruleId}] ${v.description}`; + addError(v.file, `rule:${v.ruleId}`, 'rule', message, v.severity); } } const globalRuleErrorCount = result.ruleViolations.filter((v) => !v.file && v.severity === 'error').length; @@ -74,6 +92,7 @@ export async function validate( coverageGates: result.coverageGates, skippedByDepth: result.skippedByDepth, depthLimit: result.depthLimit, + semanticEvaluations: result.semanticEvaluations, }, null, 2, @@ -193,10 +212,24 @@ export async function validate( console.log(` ${severityColor[severity]}${severity.toUpperCase()}${reset} (${violations.length}):`); for (const v of violations) { console.log(` ${v.file ? `${v.file}: ` : ''}${v.description} [${v.category}: ${v.ruleId}]`); + if (v.evidence) { + console.log(` Evidence: ${v.evidence}`); + } } } } + if (result.semanticEvaluations && result.semanticEvaluations.length > 0) { + const passed = result.semanticEvaluations.filter((e) => e.pass).length; + const failed = result.semanticEvaluations.filter((e) => !e.pass).length; + console.log( + `\nSemantic evaluations: ${result.semanticEvaluations.length} total (${passed} passed, ${failed} failed)`, + ); + } + if (result.gatedRules && result.gatedRules.length > 0) { + console.log(`Gated semantic rules (skipped due to upstream errors): ${result.gatedRules.length}`); + } + if (result.hierarchyViolations.length > 0) { console.log(`\nHierarchy violations:`); for (const v of result.hierarchyViolations) { diff --git a/src/config.ts b/src/config.ts index 07bf9b2..ed21616 100644 --- a/src/config.ts +++ b/src/config.ts @@ -4,6 +4,7 @@ import { dirname, isAbsolute, join, resolve } from 'node:path'; import Ajv2020 from 'ajv/dist/2020'; import JSON5 from 'json5'; import { ENV_CONFIG_VAR, XDG_CONFIG_DIR } from './constants'; +import type { JudgeConfig } from './judge/types'; import { normalizePluginName, resolvePluginSchemasDir, @@ -12,6 +13,18 @@ import { } from './plugins/util'; import { bundledSchemasDir } from './schema/schema'; +const JUDGE_CONFIG_SCHEMA = { + type: 'object', + properties: { + provider: { type: 'string', enum: ['anthropic', 'openai', 'fake'] }, + model: { type: 'string' }, + apiKey: { type: 'string' }, + endpoint: { type: 'string' }, + autoRun: { type: 'boolean' }, + }, + additionalProperties: false, +} as const; + const CONFIG_SCHEMA = { type: 'object', properties: { @@ -33,6 +46,8 @@ const CONFIG_SCHEMA = { additionalProperties: false, }, }, + judge: JUDGE_CONFIG_SCHEMA, + semantic: { type: 'boolean' }, }, required: ['name', 'path'], additionalProperties: false, @@ -40,6 +55,8 @@ const CONFIG_SCHEMA = { }, schema: { type: 'string' }, includeSpacesFrom: { type: 'array', items: { type: 'string' } }, + judge: JUDGE_CONFIG_SCHEMA, + semantic: { type: 'boolean' }, }, anyOf: [{ required: ['spaces'] }, { required: ['includeSpacesFrom'] }], additionalProperties: false, @@ -53,6 +70,10 @@ export type SpaceConfig = { plugins?: Record>; /** Named filter views for this space. Keys are view names; values contain the filter expression. */ views?: Record; + /** LLM judge provider configuration for semantic rule evaluation. */ + judge?: JudgeConfig; + /** Whether to evaluate semantic rules during validation for this space. */ + semantic?: boolean; /** * Directory of the config file that defined this space, used to anchor relative plugin-config * paths. Set by loadConfig; absent for a hand-assembled Config, whose callers must pass an @@ -65,6 +86,10 @@ export type Config = { spaces: SpaceConfig[]; schema?: string; includeSpacesFrom?: string[]; + /** Default LLM judge provider configuration for semantic rule evaluation. */ + judge?: JudgeConfig; + /** Whether to evaluate semantic rules during validation across spaces. */ + semantic?: boolean; }; let _configPathOverride: string | undefined; diff --git a/src/index.ts b/src/index.ts index fbe3229..9b571df 100755 --- a/src/index.ts +++ b/src/index.ts @@ -61,8 +61,19 @@ program '--include-gated', 'Evaluate rules even when the file is gated by an upstream schema/hierarchy/rule failure (still reported as gated)', ) + .option( + '--rule-filter ', + 'Evaluate only matching rules; dimensions: mode, category, severity, type, id (repeatable, e.g. mode=semantic)', + (v: string, prev: string[] = []) => [...prev, v], + ) + .option('--semantic', 'Evaluate semantic rules using built-in LLM judge') .action(async (filePath, options) => { - process.exitCode = await validateFileCommand(filePath, { json: options.json, includeGated: options.includeGated }); + process.exitCode = await validateFileCommand(filePath, { + json: options.json, + includeGated: options.includeGated, + semantic: options.semantic, + ruleFilter: options.ruleFilter, + }); }); program @@ -80,6 +91,12 @@ program '--include-gated', 'Evaluate rules even on nodes gated by an upstream schema/hierarchy/rule failure (still reported as gated)', ) + .option( + '--rule-filter ', + 'Evaluate only matching rules; dimensions: mode, category, severity, type, id (repeatable, e.g. mode=semantic)', + (v: string, prev: string[] = []) => [...prev, v], + ) + .option('--semantic', 'Evaluate semantic rules using built-in LLM judge') .action(async (spaceName, options) => { const context = buildSpaceContext(spaceName); if (options.watch) { @@ -90,6 +107,8 @@ program json: options.json, depth: options.depth, includeGated: options.includeGated, + semantic: options.semantic, + ruleFilter: options.ruleFilter, }); } }); diff --git a/src/judge/anthropic.ts b/src/judge/anthropic.ts new file mode 100644 index 0000000..9c581ad --- /dev/null +++ b/src/judge/anthropic.ts @@ -0,0 +1,55 @@ +import { PerRuleJudgeProvider } from './per-rule'; +import { buildJudgeSystemPrompt, buildJudgeUserPrompt, parseJudgeResponse } from './prompt'; +import type { SemanticJudgeInput, SemanticJudgeResult, SemanticRuleSpec } from './types'; + +export interface AnthropicJudgeProviderOptions { + apiKey: string; + model?: string; + endpoint?: string; +} + +export const DEFAULT_ANTHROPIC_JUDGE_MODEL = 'claude-haiku-5-5'; + +/** Built-in judge provider for the Anthropic Messages API. */ +export class AnthropicJudgeProvider extends PerRuleJudgeProvider { + readonly name = 'anthropic'; + private apiKey: string; + private model: string; + private endpoint: string; + + constructor(options: AnthropicJudgeProviderOptions) { + super(); + this.apiKey = options.apiKey; + this.model = options.model || process.env.ANTHROPIC_MODEL || DEFAULT_ANTHROPIC_JUDGE_MODEL; + this.endpoint = options.endpoint || 'https://api.anthropic.com/v1/messages'; + } + + protected async judgeRule(node: SemanticJudgeInput['node'], rule: SemanticRuleSpec): Promise { + const res = await fetch(this.endpoint, { + method: 'POST', + headers: { + 'content-type': 'application/json', + 'x-api-key': this.apiKey, + 'anthropic-version': '2023-06-01', + }, + body: JSON.stringify({ + model: this.model, + max_tokens: 1024, + system: buildJudgeSystemPrompt(), + messages: [{ role: 'user', content: buildJudgeUserPrompt(node, rule) }], + }), + }); + + if (!res.ok) { + const errorText = await res.text().catch(() => ''); + throw new Error(`Anthropic API error (${res.status}): ${errorText}`); + } + + const data = (await res.json()) as { content?: Array<{ type: string; text?: string }> }; + const text = data.content?.find((c) => c.type === 'text')?.text; + if (!text) { + throw new Error('Anthropic API returned empty or missing text block in response.'); + } + return parseJudgeResponse(text, Object.keys(rule.verdicts)); + } +} diff --git a/src/judge/factory.ts b/src/judge/factory.ts new file mode 100644 index 0000000..7f17a5c --- /dev/null +++ b/src/judge/factory.ts @@ -0,0 +1,70 @@ +import { AnthropicJudgeProvider } from './anthropic'; +import { FakeJudgeProvider } from './fake'; +import { JevJudgeProvider } from './jev'; +import { OpenAIJudgeProvider } from './openai'; +import type { JudgeConfig, SemanticJudgeProvider } from './types'; + +/** + * Instantiate a semantic judge provider based on configuration and environment variables. + * + * Precedence: + * 1. Explicitly passed customProvider (if provided) + * 2. config.provider or SCTX_JUDGE_PROVIDER === 'fake' -> FakeJudgeProvider + * 3. config.provider === 'jev' -> JevJudgeProvider (OPENROUTER_API_KEY; always explicit, never auto-detected) + * 4. config.provider === 'anthropic' or ANTHROPIC_API_KEY present -> AnthropicJudgeProvider + * 5. config.provider === 'openai' or OPENAI_API_KEY/OPENAI_BASE_URL present -> OpenAIJudgeProvider + * + * Throws a clear, actionable error if no provider can be resolved. + */ +export function createJudgeProvider( + config?: JudgeConfig, + env: Record = process.env, + customProvider?: SemanticJudgeProvider, +): SemanticJudgeProvider { + if (customProvider) { + return customProvider; + } + + const requestedProvider = config?.provider || env.SCTX_JUDGE_PROVIDER; + const apiKey = config?.apiKey || env.SCTX_JUDGE_API_KEY; + const endpoint = config?.endpoint || env.SCTX_JUDGE_ENDPOINT; + const model = config?.model || env.SCTX_JUDGE_MODEL; + + if (requestedProvider === 'fake') { + return new FakeJudgeProvider(); + } + + if (requestedProvider === 'jev') { + const key = apiKey || env.OPENROUTER_API_KEY; + if (!key) { + throw new Error( + 'Jev judge provider selected, but no API key was found. Set OPENROUTER_API_KEY in the environment or configure apiKey in config.json.', + ); + } + return new JevJudgeProvider({ apiKey: key, model, endpoint }); + } + + // Anthropic resolution + const hasAnthropicKey = Boolean(apiKey || env.ANTHROPIC_API_KEY); + if (requestedProvider === 'anthropic' || (!requestedProvider && hasAnthropicKey)) { + const resolvedKey = apiKey || env.ANTHROPIC_API_KEY; + if (!resolvedKey) { + throw new Error( + 'Anthropic judge provider selected, but no API key was found. Set ANTHROPIC_API_KEY in the environment or configure apiKey in config.json.', + ); + } + return new AnthropicJudgeProvider({ apiKey: resolvedKey, model, endpoint }); + } + + // OpenAI resolution + const hasOpenAiKey = Boolean(apiKey || env.OPENAI_API_KEY); + const hasOpenAiBaseUrl = Boolean(endpoint || env.OPENAI_BASE_URL || env.OPENAI_ENDPOINT); + if (requestedProvider === 'openai' || (!requestedProvider && (hasOpenAiKey || hasOpenAiBaseUrl))) { + const resolvedKey = apiKey || env.OPENAI_API_KEY; + return new OpenAIJudgeProvider({ apiKey: resolvedKey, model, endpoint }); + } + + throw new Error( + 'No semantic judge provider configured. Set ANTHROPIC_API_KEY or OPENAI_API_KEY in the environment, or configure "judge" in config.json.', + ); +} diff --git a/src/judge/fake.ts b/src/judge/fake.ts new file mode 100644 index 0000000..103abf2 --- /dev/null +++ b/src/judge/fake.ts @@ -0,0 +1,69 @@ +import type { SemanticJudgeInput, SemanticJudgeProvider, SemanticJudgeResult, SemanticRuleSpec } from './types'; + +/** A fake answer: a full verdict, or the `{ pass }` shorthand mapped onto the default `pass`/`fail` verdicts. */ +export type FakeAnswer = SemanticJudgeResult | { pass: boolean; evidence?: string }; + +export type FakeJudgeHandler = (call: { + node: SemanticJudgeInput['node']; + rule: SemanticRuleSpec; +}) => Promise | FakeAnswer; + +type FakeConfig = FakeAnswer | FakeJudgeHandler | Map | Record; + +function isHandler(c: FakeConfig): c is FakeJudgeHandler { + return typeof c === 'function'; +} + +function isAnswer(c: FakeConfig): c is FakeAnswer { + return 'pass' in c || 'verdict' in c || 'error' in c; +} + +function normalise(answer: FakeAnswer): SemanticJudgeResult { + if ('pass' in answer) { + return { verdict: answer.pass ? 'pass' : 'fail', evidence: answer.evidence }; + } + return answer; +} + +/** + * In-memory fake judge for offline, deterministic tests. Configure with a single answer, a handler, + * or answers keyed by `ruleId:title`, `ruleId:label` or `ruleId` (most specific wins). Unmatched calls pass. + */ +export class FakeJudgeProvider implements SemanticJudgeProvider { + readonly name = 'fake'; + public callHistory: SemanticJudgeInput[] = []; + + constructor(private config?: FakeConfig) {} + + async evaluate(input: SemanticJudgeInput): Promise> { + this.callHistory.push(input); + const results: Record = {}; + for (const rule of input.rules) { + results[rule.id] = normalise(await this.answer(input.node, rule)); + } + return results; + } + + private async answer(node: SemanticJudgeInput['node'], rule: SemanticRuleSpec): Promise { + const passing = { verdict: Object.keys(rule.verdicts).find((v) => !rule.failOn.includes(v)) ?? 'pass' }; + const fallback = { ...passing, evidence: `Fake passed evaluation for rule "${rule.id}"` }; + const config = this.config; + if (!config) return fallback; + if (isHandler(config)) return await config({ node, rule }); + if (config instanceof Map) { + return ( + config.get(`${rule.id}:${node.title}`) ?? + config.get(`${rule.id}:${node.label}`) ?? + config.get(rule.id) ?? + fallback + ); + } + if (isAnswer(config)) return config; + const rec = config as Record; + return rec[`${rule.id}:${node.title}`] ?? rec[`${rule.id}:${node.label}`] ?? rec[rule.id] ?? fallback; + } + + clearHistory(): void { + this.callHistory = []; + } +} diff --git a/src/judge/index.ts b/src/judge/index.ts new file mode 100644 index 0000000..58a1088 --- /dev/null +++ b/src/judge/index.ts @@ -0,0 +1,22 @@ +export type { AnthropicJudgeProviderOptions } from './anthropic'; +export { AnthropicJudgeProvider } from './anthropic'; +export { createJudgeProvider } from './factory'; +export type { FakeAnswer, FakeJudgeHandler } from './fake'; +export { FakeJudgeProvider } from './fake'; +export type { JevJudgeProviderOptions } from './jev'; +export { JevJudgeProvider } from './jev'; +export type { OpenAIJudgeProviderOptions } from './openai'; +export { OpenAIJudgeProvider } from './openai'; +export { PerRuleJudgeProvider } from './per-rule'; +export { buildJudgeSystemPrompt, buildJudgeUserPrompt, parseJudgeResponse } from './prompt'; +export type { + GatedRuleRecord, + JudgeConfig, + SemanticEvaluationRecord, + SemanticJudgeInput, + SemanticJudgeProvider, + SemanticJudgeResult, + SemanticJudgeVerdict, + SemanticRuleSpec, +} from './types'; +export { DEFAULT_VERDICTS, isJudgeError, semanticRuleSpec } from './verdicts'; diff --git a/src/judge/jev.ts b/src/judge/jev.ts new file mode 100644 index 0000000..46d63f6 --- /dev/null +++ b/src/judge/jev.ts @@ -0,0 +1,94 @@ +import type { SemanticJudgeInput, SemanticJudgeProvider, SemanticJudgeResult } from './types'; + +export interface JevJudgeProviderOptions { + apiKey: string; + model?: string; + endpoint?: string; +} + +export const DEFAULT_JEV_MODEL = 'typesafe/jev-1.13'; +export const DEFAULT_JEV_ENDPOINT = 'https://openrouter.ai/api/alpha/decisions'; + +type JevChoiceAnswer = { + type?: string; + choice?: string; + confidence?: number; + probabilities?: Record; +}; + +/** + * Judge provider for TypeSafe's Jev decision model via OpenRouter's Decisions API. + * + * Each node is sent once as `state`, with one `choice` question per semantic rule whose criteria are + * the rule's closed verdict set. Jev returns the chosen verdict with confidence and probabilities, + * but no prose, so `evidence` is a summary of the distribution. + */ +export class JevJudgeProvider implements SemanticJudgeProvider { + readonly name = 'jev'; + private apiKey: string; + private model: string; + private endpoint: string; + + constructor(options: JevJudgeProviderOptions) { + this.apiKey = options.apiKey; + this.model = options.model || DEFAULT_JEV_MODEL; + this.endpoint = options.endpoint || DEFAULT_JEV_ENDPOINT; + } + + async evaluate(input: SemanticJudgeInput): Promise> { + const { node, rules } = input; + const questions: Record = {}; + for (const rule of rules) { + questions[rule.id] = { + type: 'choice', + instructions: rule.rubric || rule.description, + criteria: rule.verdicts, + }; + } + + const res = await fetch(this.endpoint, { + method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${this.apiKey}` }, + body: JSON.stringify({ + model: this.model, + state: { + title: node.title, + type: node.resolvedType, + parents: node.resolvedParents?.map((p) => p.title) ?? [], + ...node.schemaData, + }, + questions, + }), + }); + + if (!res.ok) { + const errorText = await res.text().catch(() => ''); + throw new Error(`Jev API error (${res.status}): ${errorText}`); + } + + const data = (await res.json()) as { answers?: Record }; + const results: Record = {}; + for (const rule of rules) { + const answer = data.answers?.[rule.id]; + if (!answer?.choice || !(answer.choice in rule.verdicts)) { + results[rule.id] = { error: `Jev returned no valid verdict for rule "${rule.id}" (choice: ${answer?.choice})` }; + continue; + } + results[rule.id] = { + verdict: answer.choice, + confidence: answer.confidence, + probabilities: answer.probabilities, + evidence: describeDistribution(answer), + }; + } + return results; + } +} + +function describeDistribution(answer: JevChoiceAnswer): string { + const parts = Object.entries(answer.probabilities ?? {}) + .sort(([, a], [, b]) => b - a) + .map(([verdict, p]) => `${verdict} ${p.toFixed(2)}`); + const confidence = answer.confidence !== undefined ? `confidence ${answer.confidence.toFixed(2)}` : undefined; + return `Jev chose "${answer.choice}"${[confidence, ...parts].filter(Boolean).length ? ` (${[confidence, ...parts].filter(Boolean).join('; ')})` : ''}`; +} diff --git a/src/judge/judge-node.ts b/src/judge/judge-node.ts new file mode 100644 index 0000000..8c17af2 --- /dev/null +++ b/src/judge/judge-node.ts @@ -0,0 +1,68 @@ +import type { Rule, SpaceNode } from '../types'; +import type { SemanticEvaluationRecord, SemanticJudgeProvider } from './types'; +import { isJudgeError, semanticRuleSpec } from './verdicts'; + +export type NodeJudgement = { + rule: Rule; + /** True when the verdict is one of the rule's `failOn` verdicts, or the judge errored. */ + failed: boolean; + evidence?: string; + /** Set when the judge could not produce a verdict for this rule. */ + error?: string; + /** Present when the judge produced a verdict. */ + record?: SemanticEvaluationRecord; +}; + +/** Judge every given semantic rule on a node in one provider call. */ +export async function judgeNode( + judge: SemanticJudgeProvider, + node: SpaceNode, + rules: Rule[], +): Promise { + if (rules.length === 0) return []; + const specs = rules.map(semanticRuleSpec); + let results: Awaited>; + try { + results = await judge.evaluate({ + node: { + label: node.label, + title: node.title, + type: node.type, + resolvedType: node.resolvedType, + schemaData: node.schemaData, + resolvedParents: node.resolvedParents.map((p) => ({ title: p.title, field: p.field })), + }, + rules: specs, + }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return rules.map((rule) => ({ + rule, + failed: true, + evidence: `Error during evaluation: ${message}`, + error: message, + })); + } + + return rules.map((rule, i) => { + const result = results[rule.id]; + if (!result || isJudgeError(result)) { + const message = result ? result.error : 'Judge returned no result for this rule'; + return { rule, failed: true, evidence: `Error during evaluation: ${message}`, error: message }; + } + const failed = specs[i]!.failOn.includes(result.verdict); + return { + rule, + failed, + evidence: result.evidence, + record: { + file: node.label, + ruleId: rule.id, + verdict: result.verdict, + pass: !failed, + confidence: result.confidence, + evidence: result.evidence, + }, + }; + }); +} diff --git a/src/judge/openai.ts b/src/judge/openai.ts new file mode 100644 index 0000000..fe044e1 --- /dev/null +++ b/src/judge/openai.ts @@ -0,0 +1,68 @@ +import { PerRuleJudgeProvider } from './per-rule'; +import { buildJudgeSystemPrompt, buildJudgeUserPrompt, parseJudgeResponse } from './prompt'; +import type { SemanticJudgeInput, SemanticJudgeResult, SemanticRuleSpec } from './types'; + +export interface OpenAIJudgeProviderOptions { + apiKey?: string; + model?: string; + endpoint?: string; +} + +/** + * Built-in judge provider for OpenAI-compatible Chat Completions API. + * Supports OpenAI, Ollama, vLLM, Groq, OpenRouter, and other compatible endpoints. + */ +export class OpenAIJudgeProvider extends PerRuleJudgeProvider { + readonly name = 'openai'; + private apiKey?: string; + private model: string; + private endpoint: string; + + constructor(options: OpenAIJudgeProviderOptions) { + super(); + this.apiKey = options.apiKey; + this.model = options.model || process.env.OPENAI_MODEL || 'gpt-4o-mini'; + + const base = + options.endpoint || process.env.OPENAI_BASE_URL || process.env.OPENAI_ENDPOINT || 'https://api.openai.com/v1'; + this.endpoint = base.endsWith('/chat/completions') ? base : `${base.replace(/\/+$/, '')}/chat/completions`; + } + + protected async judgeRule(node: SemanticJudgeInput['node'], rule: SemanticRuleSpec): Promise { + const systemPrompt = buildJudgeSystemPrompt(); + const userPrompt = buildJudgeUserPrompt(node, rule); + + const headers: Record = { + 'content-type': 'application/json', + }; + if (this.apiKey) { + headers.authorization = `Bearer ${this.apiKey}`; + } + + const res = await fetch(this.endpoint, { + method: 'POST', + headers, + body: JSON.stringify({ + model: this.model, + response_format: { type: 'json_object' }, + messages: [ + { role: 'system', content: systemPrompt }, + { role: 'user', content: userPrompt }, + ], + }), + }); + + if (!res.ok) { + const errorText = await res.text().catch(() => ''); + throw new Error(`OpenAI API error (${res.status}): ${errorText}`); + } + + const data = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }; + const content = data.choices?.[0]?.message?.content; + if (!content) { + throw new Error('OpenAI API returned empty or missing message content in response.'); + } + + return parseJudgeResponse(content, Object.keys(rule.verdicts)); + } +} diff --git a/src/judge/per-rule.ts b/src/judge/per-rule.ts new file mode 100644 index 0000000..244d670 --- /dev/null +++ b/src/judge/per-rule.ts @@ -0,0 +1,23 @@ +import type { SemanticJudgeInput, SemanticJudgeProvider, SemanticJudgeResult, SemanticRuleSpec } from './types'; + +/** + * Base for providers that can only judge one rule per request (chat LLMs). `evaluate` makes one call + * per rule; a failure on one rule is reported for that rule and does not affect the others. + */ +export abstract class PerRuleJudgeProvider implements SemanticJudgeProvider { + abstract readonly name: string; + + protected abstract judgeRule(node: SemanticJudgeInput['node'], rule: SemanticRuleSpec): Promise; + + async evaluate(input: SemanticJudgeInput): Promise> { + const results: Record = {}; + for (const rule of input.rules) { + try { + results[rule.id] = await this.judgeRule(input.node, rule); + } catch (err) { + results[rule.id] = { error: err instanceof Error ? err.message : String(err) }; + } + } + return results; + } +} diff --git a/src/judge/prompt.ts b/src/judge/prompt.ts new file mode 100644 index 0000000..1d62876 --- /dev/null +++ b/src/judge/prompt.ts @@ -0,0 +1,92 @@ +import type { SemanticJudgeInput, SemanticJudgeResult, SemanticRuleSpec } from './types'; + +function verdictList(rule: SemanticRuleSpec): string[] { + return Object.entries(rule.verdicts).map(([name, description]) => `- "${name}": ${description}`); +} + +/** System prompt: a strict judge that answers with one verdict from a closed set, as JSON. */ +export function buildJudgeSystemPrompt(): string { + return [ + 'You are a strict, objective evaluation judge for structured planning and knowledge documents.', + 'Your task is to choose the one verdict that best describes whether a node satisfies a rule.', + '', + 'You must respond ONLY with a JSON object in this exact format:', + '{', + ' "verdict": "",', + ' "confidence": ,', + ' "evidence": "concise explanation citing specific quotes, phrases, or absent details from the node"', + '}', + '', + 'Guidelines:', + '1. "verdict" must be one of the allowed verdicts listed for the rule, spelled exactly.', + '2. "confidence" is how sure you are of the verdict.', + '3. "evidence" must directly cite or quote relevant parts of the node, explaining the reasoning concisely.', + '4. Do not include markdown code block formatting, intro text, or outro text. Respond only with the JSON object.', + ].join('\n'); +} + +/** User prompt for one rule on one node. */ +export function buildJudgeUserPrompt(node: SemanticJudgeInput['node'], rule: SemanticRuleSpec): string { + const rubric = rule.rubric || rule.description; + const lines: string[] = [ + '# Rule to Evaluate', + `- ID: ${rule.id}`, + `- Category: ${rule.category}`, + `- Rubric: ${rubric}`, + '', + '## Allowed verdicts', + ...verdictList(rule), + '', + '# Node Under Evaluation', + `- File / Label: ${node.label}`, + `- Title: ${node.title}`, + `- Type: ${node.type} (resolved: ${node.resolvedType})`, + ]; + + if (node.resolvedParents && node.resolvedParents.length > 0) { + lines.push(`- Parents: ${node.resolvedParents.map((p) => p.title).join(', ')}`); + } + + lines.push('', '## Node Content & Frontmatter:', '```json', JSON.stringify(node.schemaData, null, 2), '```', ''); + lines.push('Evaluate this node against the rule. Return only JSON with "verdict", "confidence" and "evidence".'); + return lines.join('\n'); +} + +/** + * Parse judge output into a verdict from the rule's allowed set. + * Handles clean JSON, markdown-wrapped JSON, and a regex fallback for surrounding text. + */ +export function parseJudgeResponse(rawText: string, allowed: string[]): SemanticJudgeResult { + let cleaned = rawText.trim(); + cleaned = cleaned.replace(/^```(?:json)?\s*/, '').replace(/\s*```$/, ''); + + let verdict: string | undefined; + let confidence: number | undefined; + let evidence: string | undefined; + + try { + const parsed = JSON.parse(cleaned); + if (typeof parsed.verdict === 'string') verdict = parsed.verdict.trim(); + if (typeof parsed.confidence === 'number') confidence = parsed.confidence; + if (typeof parsed.evidence === 'string') evidence = parsed.evidence.trim(); + } catch { + const verdictMatch = /"verdict"\s*:\s*"((?:[^"\\]|\\.)*)"/i.exec(cleaned); + const confidenceMatch = /"confidence"\s*:\s*([0-9.]+)/i.exec(cleaned); + const evidenceMatch = /"evidence"\s*:\s*"((?:[^"\\]|\\.)*)"/i.exec(cleaned); + verdict = verdictMatch?.[1]?.trim(); + confidence = confidenceMatch ? Number(confidenceMatch[1]) : undefined; + evidence = evidenceMatch?.[1]?.replace(/\\"/g, '"').replace(/\\n/g, '\n').trim(); + } + + if (verdict === undefined || !allowed.includes(verdict)) { + return { + error: `Judge returned ${verdict === undefined ? 'no verdict' : `unknown verdict "${verdict}"`} (allowed: ${allowed.join(', ')}): ${rawText.slice(0, 200)}`, + }; + } + return { + verdict, + ...(confidence !== undefined && + Number.isFinite(confidence) && { confidence: Math.min(1, Math.max(0, confidence)) }), + ...(evidence && { evidence }), + }; +} diff --git a/src/judge/types.ts b/src/judge/types.ts new file mode 100644 index 0000000..ef62826 --- /dev/null +++ b/src/judge/types.ts @@ -0,0 +1,73 @@ +import type { RuleCategory } from '../types'; + +/** A semantic rule as presented to a judge: the closed verdict set it must choose from. */ +export interface SemanticRuleSpec { + id: string; + description: string; + rubric?: string; + category: RuleCategory | string; + /** Allowed verdicts, each with a description of when it applies. */ + verdicts: Record; + /** Verdicts that count as a violation. */ + failOn: string[]; +} + +/** One node and every semantic rule to judge on it. Judges may answer all rules in a single call. */ +export interface SemanticJudgeInput { + node: { + label: string; + title: string; + type: string; + resolvedType: string; + schemaData: Record; + resolvedParents?: Array<{ title: string; field: string }>; + }; + rules: SemanticRuleSpec[]; +} + +/** A judge's answer for one rule: a verdict from the rule's closed set. */ +export interface SemanticJudgeVerdict { + verdict: string; + /** 0-1, when the judge reports one. */ + confidence?: number; + /** Probability per verdict, when the judge reports a distribution. */ + probabilities?: Record; + /** Free-text reasoning; decision models may not provide it. */ + evidence?: string; +} + +/** A judge's answer for one rule: a verdict, or an error that applies to that rule only. */ +export type SemanticJudgeResult = SemanticJudgeVerdict | { error: string }; + +/** Judge interface. Results are keyed by rule id; a missing key is treated as an error. */ +export interface SemanticJudgeProvider { + readonly name: string; + evaluate(input: SemanticJudgeInput): Promise>; +} + +/** Judge configuration from config file or options. */ +export interface JudgeConfig { + provider?: 'anthropic' | 'openai' | 'jev' | 'fake' | string; + model?: string; + apiKey?: string; + endpoint?: string; + autoRun?: boolean; +} + +/** Record of a completed semantic evaluation. */ +export interface SemanticEvaluationRecord { + file: string; + ruleId: string; + verdict: string; + /** True when the verdict is not one of the rule's `failOn` verdicts. */ + pass: boolean; + confidence?: number; + evidence?: string; +} + +/** Record of a rule gated/suppressed due to upstream failure. */ +export interface GatedRuleRecord { + file: string; + ruleId: string; + reason: string; +} diff --git a/src/judge/verdicts.ts b/src/judge/verdicts.ts new file mode 100644 index 0000000..6e06c76 --- /dev/null +++ b/src/judge/verdicts.ts @@ -0,0 +1,39 @@ +import type { Rule } from '../types'; +import type { SemanticJudgeResult, SemanticRuleSpec } from './types'; + +/** Verdicts for a semantic rule that declares none. */ +export const DEFAULT_VERDICTS: Record = { + pass: 'The node meets the rubric.', + fail: 'The node does not meet the rubric.', +}; + +/** Build the judge-facing spec for a semantic rule, applying verdict defaults and checking `failOn`. */ +export function semanticRuleSpec(rule: Rule): SemanticRuleSpec { + const verdicts = rule.verdicts ?? DEFAULT_VERDICTS; + const failOn = rule.failOn ?? (rule.verdicts ? [] : ['fail']); + if (failOn.length === 0) { + throw new Error( + `Semantic rule "${rule.id}" declares verdicts but no failOn; list the verdicts that count as a violation.`, + ); + } + const unknown = failOn.filter((v) => !(v in verdicts)); + if (unknown.length > 0) { + throw new Error(`Semantic rule "${rule.id}" has failOn verdicts not in its verdicts: ${unknown.join(', ')}.`); + } + if (failOn.length >= Object.keys(verdicts).length) { + throw new Error(`Semantic rule "${rule.id}" has no passing verdict: every verdict is in failOn.`); + } + return { + id: rule.id, + description: rule.description, + rubric: rule.rubric, + category: rule.category, + verdicts, + failOn, + }; +} + +/** Whether a judge result is an error rather than a verdict. */ +export function isJudgeError(result: SemanticJudgeResult): result is { error: string } { + return 'error' in result; +} diff --git a/src/schema/gating.ts b/src/schema/gating.ts index 3bb416a..576fcec 100644 --- a/src/schema/gating.ts +++ b/src/schema/gating.ts @@ -1,3 +1,5 @@ +import { judgeNode } from '../judge/judge-node'; +import type { GatedRuleRecord, SemanticEvaluationRecord, SemanticJudgeProvider } from '../judge/types'; import type { SpaceGraph } from '../space-graph'; import type { Rule, RuleViolation, SpaceNode } from '../types'; import { buildEvalContext, evaluateExpression } from './evaluate-rule'; @@ -41,8 +43,19 @@ export type CoverageGate = { export type GatedRuleEvaluation = { violations: RuleViolation[]; gates: CoverageGate[]; + /** Semantic judgements made (empty unless a judge was supplied). */ + semanticEvaluations: SemanticEvaluationRecord[]; + /** Semantic rule/node pairs skipped because the node was gated (empty unless a judge was supplied). */ + gatedRules: GatedRuleRecord[]; }; +export interface GatingOptions { + /** Evaluate rules on gated nodes anyway (gates are still reported). */ + includeGated?: boolean; + /** Judge for `mode: semantic` rules. Without it, semantic rules are not evaluated. */ + judge?: SemanticJudgeProvider; +} + /** * Compute breadth-first depth (1-based, from hierarchy roots and orphans) for every * hierarchy-typed node in the graph. Non-hierarchy nodes are not included. @@ -81,7 +94,7 @@ export async function evaluateRulesWithGating( rules: Rule[], graph: SpaceGraph, seeds: GatingSeed[], - options: { includeGated?: boolean } = {}, + options: GatingOptions = {}, ): Promise { const includeGated = options.includeGated ?? false; @@ -98,7 +111,10 @@ export async function evaluateRulesWithGating( } } + const judge = options.judge; const violations: RuleViolation[] = []; + const semanticEvaluations: SemanticEvaluationRecord[] = []; + const gatedRules: GatedRuleRecord[] = []; const gateOrigins = new Map(); function originFor( @@ -113,12 +129,12 @@ export async function evaluateRulesWithGating( return origin; } - const globalRules = rules.filter((r) => r.scope === 'global'); + const globalRules = rules.filter((r) => r.scope === 'global' && r.mode !== 'semantic'); const sentinel = nodes[0]; if (sentinel) { for (const rule of globalRules) { const context = buildEvalContext(sentinel, nodes, nodeIndex); - const result = await evaluateExpression(rule.check, context); + const result = await evaluateExpression(rule.check as string, context); if (result !== true) { violations.push({ file: '', @@ -131,30 +147,51 @@ export async function evaluateRulesWithGating( } } - const perNodeRules = rules.filter((r) => r.scope !== 'global'); + const perNodeRules = rules.filter((r) => r.scope !== 'global' && (r.mode !== 'semantic' || judge)); + + function applicableRules(node: SpaceNode): Rule[] { + return perNodeRules.filter((r) => !r.type || r.type === node.resolvedType); + } async function evaluateApplicableRules(node: SpaceNode): Promise<{ blockingRuleId?: string }> { let blockingRuleId: string | undefined; - const applicable = perNodeRules.filter((r) => !r.type || r.type === node.resolvedType); - for (const rule of applicable) { - const context = buildEvalContext(node, nodes, nodeIndex); - const result = await evaluateExpression(rule.check, context); - if (result !== true) { - violations.push({ - file: node.label, - ruleId: rule.id, - category: rule.category, - severity: ruleSeverity(rule), - description: rule.description, - }); - if (ruleGatesDescendants(rule) && !blockingRuleId) { - blockingRuleId = rule.id; - } + const applicable = applicableRules(node); + + function report(rule: Rule, evidence?: string, error?: string): void { + violations.push({ + file: node.label, + ruleId: rule.id, + category: rule.category, + severity: ruleSeverity(rule), + description: error ? `${rule.description} (Judge error: ${error})` : rule.description, + ...(evidence !== undefined && { evidence }), + }); + // A judge error says nothing about the node, so it never gates. + if (!error && ruleGatesDescendants(rule) && !blockingRuleId) blockingRuleId = rule.id; + } + + // Cheap deterministic rules first; the judge is only consulted afterwards, once per node. + for (const rule of applicable.filter((r) => r.mode !== 'semantic')) { + const result = await evaluateExpression(rule.check as string, buildEvalContext(node, nodes, nodeIndex)); + if (result !== true) report(rule); + } + + const semantic = applicable.filter((r) => r.mode === 'semantic'); + if (judge && semantic.length > 0) { + for (const j of await judgeNode(judge, node, semantic)) { + if (j.record) semanticEvaluations.push(j.record); + if (j.failed) report(j.rule, j.evidence, j.error); } } return { blockingRuleId }; } + function recordGatedSemantic(node: SpaceNode, reason: string): void { + for (const rule of applicableRules(node)) { + if (rule.mode === 'semantic') gatedRules.push({ file: node.label, ruleId: rule.id, reason }); + } + } + // Non-hierarchy nodes sit outside the subtree cascade — always evaluated, never gated. for (const node of graph.nonHierarchy) { await evaluateApplicableRules(node); @@ -179,6 +216,9 @@ export async function evaluateRulesWithGating( const shouldEvaluate = !isGated || includeGated; const { blockingRuleId } = shouldEvaluate ? await evaluateApplicableRules(node) : {}; + if (!shouldEvaluate) { + recordGatedSemantic(node, isGatedByAncestor ? 'Gated by an ancestor failure' : 'Gated by failure on node'); + } let childOrigin: string | undefined = ancestorOrigin; if (isGatedByAncestor) { @@ -199,5 +239,10 @@ export async function evaluateRulesWithGating( } } - return { violations, gates: [...gateOrigins.values()].filter((g) => g.gatedFiles.length > 0) }; + return { + violations, + gates: [...gateOrigins.values()].filter((g) => g.gatedFiles.length > 0), + semanticEvaluations, + gatedRules, + }; } diff --git a/src/schema/metadata-contract.ts b/src/schema/metadata-contract.ts index 41cc0db..aa50e8d 100644 --- a/src/schema/metadata-contract.ts +++ b/src/schema/metadata-contract.ts @@ -63,13 +63,19 @@ const RULE_SCHEMA = { severity: { enum: ['error', 'warning', 'info'] }, description: { type: 'string', minLength: 1 }, check: { type: 'string', minLength: 1 }, + mode: { enum: ['expression', 'semantic'] }, + rubric: { type: 'string', minLength: 1 }, + /** Semantic rules: the closed set of verdicts a judge may return, each with a description. Defaults to `pass`/`fail`. */ + verdicts: { type: 'object', additionalProperties: { type: 'string', minLength: 1 }, minProperties: 2 }, + /** Semantic rules: which verdicts count as a violation. Defaults to `fail`; every other verdict passes. */ + failOn: { type: 'array', items: { type: 'string', minLength: 1 }, minItems: 1, uniqueItems: true }, type: { type: 'string', minLength: 1 }, scope: { enum: ['global'] }, override: { type: 'boolean' }, /** Whether a violation of this rule gates (suppresses) downstream checks on the node's hierarchy subtree. Defaults to true for `error` severity rules and false otherwise. */ gatesDescendants: { type: 'boolean' }, }, - required: ['id', 'category', 'description', 'check'], + required: ['id', 'category', 'description'], additionalProperties: false, } as const; diff --git a/src/schema/rule-filter.ts b/src/schema/rule-filter.ts new file mode 100644 index 0000000..7d58423 --- /dev/null +++ b/src/schema/rule-filter.ts @@ -0,0 +1,55 @@ +import type { Rule } from '../types'; +import { ruleSeverity } from './validate-rules'; + +export const RULE_FILTER_DIMENSIONS = ['mode', 'category', 'severity', 'type', 'id'] as const; +export type RuleFilterDimension = (typeof RULE_FILTER_DIMENSIONS)[number]; + +/** Narrows which rules are evaluated. Dimensions combine with AND; values within a dimension with OR. */ +export type RuleFilter = Partial>; + +/** Parse `dimension=value[,value]` specs (e.g. from repeated `--rule-filter` flags); repeats of a dimension merge. */ +export function parseRuleFilter(specs: string[]): RuleFilter { + const filter: RuleFilter = {}; + for (const spec of specs) { + const eq = spec.indexOf('='); + const dimension = spec.slice(0, eq) as RuleFilterDimension; + const values = spec + .slice(eq + 1) + .split(',') + .map((v) => v.trim()) + .filter(Boolean); + if (eq < 0 || values.length === 0 || !RULE_FILTER_DIMENSIONS.includes(dimension)) { + throw new Error(`Invalid rule filter "${spec}". Use <${RULE_FILTER_DIMENSIONS.join('|')}>=[,...].`); + } + filter[dimension] = [...(filter[dimension] ?? []), ...values]; + } + return filter; +} + +function dimensionValue(rule: Rule, dimension: RuleFilterDimension): string | undefined { + switch (dimension) { + case 'mode': + return rule.mode ?? 'expression'; + case 'severity': + return ruleSeverity(rule); + case 'category': + return rule.category; + case 'type': + return rule.type; + case 'id': + return rule.id; + } +} + +/** Keep the rules matching every dimension of the filter. A rule with no `type` never matches a `type` filter. */ +export function filterRules(rules: Rule[], filter?: RuleFilter): Rule[] { + if (!filter) return rules; + return rules.filter((rule) => + RULE_FILTER_DIMENSIONS.every((dim) => { + const wanted = filter[dim]; + if (!wanted) return true; + const value = dimensionValue(rule, dim); + return value !== undefined && wanted.includes(value); + }), + ); +} diff --git a/src/schema/validate-rules.ts b/src/schema/validate-rules.ts index 25c0786..03b8d35 100644 --- a/src/schema/validate-rules.ts +++ b/src/schema/validate-rules.ts @@ -1,3 +1,5 @@ +import { judgeNode } from '../judge/judge-node'; +import type { GatedRuleRecord, SemanticEvaluationRecord, SemanticJudgeProvider } from '../judge/types'; import type { Rule, RuleCategory, RuleSeverity, RuleViolation, SpaceNode } from '../types'; import { buildEvalContext, evaluateExpression } from './evaluate-rule'; @@ -14,16 +16,40 @@ export function ruleSeverity(rule: Rule): RuleSeverity { return rule.severity ?? DEFAULT_SEVERITY_BY_CATEGORY[rule.category]; } +export interface ValidateRulesOptions { + /** Whether to run semantic rules (default: false). */ + semantic?: boolean; + /** The semantic judge provider to use if semantic is true. */ + judgeProvider?: SemanticJudgeProvider; + /** Set of node labels that have already failed upstream structural checks. */ + gatedNodeLabels?: Set; +} + +export type ValidatedRulesResult = RuleViolation[] & { + semanticEvaluations: SemanticEvaluationRecord[]; + gatedRules: GatedRuleRecord[]; +}; + /** * Validate nodes against rules metadata. - * Returns a list of rule violations found. + * Evaluates deterministic expression rules and (when opt-in) semantic judge rules. * * @param nodes - All nodes in the space * @param rules - Rules metadata list - * @returns Array of rule violations + * @param options - Validation options including semantic opt-in and judge provider + * @returns Array of rule violations with semanticEvaluations and gatedRules metadata attached */ -export async function validateRules(nodes: SpaceNode[], rules: Rule[]): Promise { +export async function validateRules( + nodes: SpaceNode[], + rules: Rule[], + options: ValidateRulesOptions = {}, +): Promise { const violations: RuleViolation[] = []; + const semanticEvaluations: SemanticEvaluationRecord[] = []; + const gatedRules: GatedRuleRecord[] = []; + + // Track nodes that have failed checks (for cascading gating) + const failedNodeLabels = new Set(options.gatedNodeLabels || []); // Build node index for efficient lookups const nodeIndex = new Map(); @@ -34,16 +60,30 @@ export async function validateRules(nodes: SpaceNode[], rules: Rule[]): Promise< } } - // Evaluate each rule against applicable nodes + // Separate rules into expression rules and semantic rules + const expressionRules: Rule[] = []; + const semanticRules: Rule[] = []; + for (const rule of rules) { + if (rule.mode === 'semantic') { + semanticRules.push(rule); + } else if (rule.check) { + expressionRules.push(rule); + } + } + + // 1. Evaluate deterministic expression rules first + for (const rule of expressionRules) { + const check = rule.check as string; const severity = ruleSeverity(rule); + if (rule.scope === 'global') { // Global rules are evaluated once against the full node set. // A sentinel node provides the evaluation context (nodes array is what matters). const sentinel = nodes[0]; if (sentinel) { const context = buildEvalContext(sentinel, nodes, nodeIndex); - const result = await evaluateExpression(rule.check, context); + const result = await evaluateExpression(check, context); if (result !== true) { violations.push({ file: '', @@ -58,7 +98,7 @@ export async function validateRules(nodes: SpaceNode[], rules: Rule[]): Promise< const targetNodes = rule.type ? nodes.filter((n) => n.resolvedType === rule.type) : nodes; for (const node of targetNodes) { const context = buildEvalContext(node, nodes, nodeIndex); - const result = await evaluateExpression(rule.check, context); + const result = await evaluateExpression(check, context); if (result !== true) { violations.push({ file: node.label, @@ -67,10 +107,49 @@ export async function validateRules(nodes: SpaceNode[], rules: Rule[]): Promise< severity, description: rule.description, }); + failedNodeLabels.add(node.label); } } } } - return violations; + // 2. Evaluate semantic rules only if explicitly requested + if (options.semantic) { + const judgeProvider = options.judgeProvider; + if (!judgeProvider) { + throw new Error('Semantic evaluation was requested but no judgeProvider was provided to validateRules.'); + } + + for (const node of nodes) { + const applicable = semanticRules.filter((r) => !r.type || r.type === node.resolvedType); + if (applicable.length === 0) continue; + + // Safe cascading default: gate semantic evaluation if node failed upstream checks + if (failedNodeLabels.has(node.label)) { + for (const rule of applicable) { + gatedRules.push({ file: node.label, ruleId: rule.id, reason: 'Gated by upstream error on node' }); + } + continue; + } + + for (const j of await judgeNode(judgeProvider, node, applicable)) { + if (j.record) semanticEvaluations.push(j.record); + if (!j.failed) continue; + violations.push({ + file: node.label, + ruleId: j.rule.id, + category: j.rule.category, + severity: ruleSeverity(j.rule), + description: j.error ? `${j.rule.description} (Judge error: ${j.error})` : j.rule.description, + evidence: j.evidence, + }); + if (!j.error) failedNodeLabels.add(node.label); + } + } + } + + const result = violations as ValidatedRulesResult; + result.semanticEvaluations = semanticEvaluations; + result.gatedRules = gatedRules; + return result; } diff --git a/src/types.ts b/src/types.ts index 2401a6c..523a62f 100644 --- a/src/types.ts +++ b/src/types.ts @@ -201,6 +201,8 @@ export type RuleViolation = { category: RuleCategory; severity: RuleSeverity; description: string; + /** Cited evidence or reasoning from evaluation (for semantic judge rules). */ + evidence?: string; }; export type GraphViolation = { diff --git a/src/validate.ts b/src/validate.ts index 26bdc62..4ac28d3 100644 --- a/src/validate.ts +++ b/src/validate.ts @@ -1,8 +1,11 @@ import { isAbsolute, relative, resolve } from 'node:path'; import type { ErrorObject } from 'ajv'; import type { Config, SpaceConfig } from './config'; +import { createJudgeProvider } from './judge/factory'; +import type { GatedRuleRecord, JudgeConfig, SemanticEvaluationRecord, SemanticJudgeProvider } from './judge/types'; import { readSpace } from './read/read-space'; import { computeHierarchyDepths, evaluateRulesWithGating, type GatingSeed } from './schema/gating'; +import { filterRules, type RuleFilter } from './schema/rule-filter'; import { extractEntityInfo } from './schema/schema'; import { validateGraph } from './schema/validate-graph'; import { validateRules } from './schema/validate-rules'; @@ -52,6 +55,10 @@ export interface ValidationResult { skippedByDepth: Array<{ file: string; title: string }>; /** The depth limit applied, if any (undefined means the full space was validated). */ depthLimit?: number; + /** Details on semantic rule evaluations performed (if semantic mode enabled). */ + semanticEvaluations?: SemanticEvaluationRecord[]; + /** Semantic rule/node pairs skipped because the node was gated by an upstream failure. */ + gatedRules?: GatedRuleRecord[]; } /** @@ -159,21 +166,71 @@ export function formatErrors( return formatted; } +export interface ValidateSpaceOptions { + readResult?: ReadSpaceResult; + /** Limit validation to hierarchy nodes within N breadth-first levels from the top. */ + depth?: number; + /** Evaluate rules on gated nodes anyway (gates are still reported). */ + includeGated?: boolean; + /** Evaluate only rules matching this filter (see `parseRuleFilter`). */ + ruleFilter?: RuleFilter; + /** Whether to evaluate semantic rules (default: false, or true if configured in space/config). */ + semantic?: boolean; + /** Optional custom judge provider (for testing or injection). */ + judgeProvider?: SemanticJudgeProvider; +} + +/** + * Resolve the semantic judge for a run: none unless semantic evaluation is requested (option, else + * space/config `semantic` or `judge.autoRun`). An injected provider wins over the configured one. + */ +function resolveJudge( + requested: boolean | undefined, + injected: SemanticJudgeProvider | undefined, + space: SpaceConfig, + config: Config, +): SemanticJudgeProvider | undefined { + const enabled = + requested ?? space.semantic ?? space.judge?.autoRun ?? config.semantic ?? config.judge?.autoRun ?? false; + if (!enabled) return undefined; + return injected ?? createJudgeProvider((space.judge ?? config.judge) as JudgeConfig | undefined); +} + +type SpaceRuleEvaluation = { + violations: RuleViolation[]; + gates: CoverageGate[]; + semanticEvaluations: SemanticEvaluationRecord[]; + gatedRules: GatedRuleRecord[]; +}; + /** - * Evaluate a space's executable rules. With a main hierarchy, evaluation cascades: schema, hierarchy + * Evaluate a space's rules. With a main hierarchy, evaluation cascades: schema, hierarchy * and gating-rule failures (the `seeds`) suppress rule evaluation for the failing node's subtree. * Shared by `validateSpace` and `validateFile` so both report identical rule results for a node. + * Semantic rules run only when a `judge` is supplied. */ async function evaluateSpaceRules( nodes: SpaceNode[], metadata: SchemaMetadata, seeds: GatingSeed[], - options: { includeGated?: boolean } = {}, -): Promise<{ violations: RuleViolation[]; gates: CoverageGate[] }> { - if (!metadata.rules) return { violations: [], gates: [] }; - if (!metadata.hierarchy) return { violations: await validateRules(nodes, metadata.rules), gates: [] }; + options: { includeGated?: boolean; judge?: SemanticJudgeProvider; ruleFilter?: RuleFilter } = {}, +): Promise { + const rules = metadata.rules && filterRules(metadata.rules, options.ruleFilter); + if (!rules) return { violations: [], gates: [], semanticEvaluations: [], gatedRules: [] }; + if (!metadata.hierarchy) { + const result = await validateRules(nodes, rules, { + semantic: options.judge !== undefined, + judgeProvider: options.judge, + }); + return { + violations: result, + gates: [], + semanticEvaluations: result.semanticEvaluations, + gatedRules: result.gatedRules, + }; + } const graph = buildSpaceGraph(nodes, metadata.hierarchy); - return evaluateRulesWithGating(nodes, metadata.rules, graph, seeds, options); + return evaluateRulesWithGating(nodes, rules, graph, seeds, options); } /** @@ -182,7 +239,7 @@ async function evaluateSpaceRules( */ export async function validateSpace( context: SpaceContext, - options: { readResult?: ReadSpaceResult; depth?: number; includeGated?: boolean } = {}, + options: ValidateSpaceOptions = {}, ): Promise { const { schema, schemaRefRegistry, schemaValidator } = context; const metadata = schema.metadata; @@ -299,11 +356,18 @@ export async function validateSpace( ), ...result.hierarchyViolations.map((v): GatingSeed => ({ file: v.file, title: v.nodeTitle, reason: 'hierarchy' })), ]; - const { violations, gates } = await evaluateSpaceRules(nodes, metadata, seeds, { + const judge = resolveJudge(options.semantic, options.judgeProvider, context.space, context.config); + const { violations, gates, semanticEvaluations, gatedRules } = await evaluateSpaceRules(nodes, metadata, seeds, { includeGated: options.includeGated, + ruleFilter: options.ruleFilter, + judge, }); result.ruleViolations = violations; result.coverageGates = gates; + if (judge) { + result.semanticEvaluations = semanticEvaluations; + result.gatedRules = gatedRules; + } } // Collect unresolved content links (warnings — not errors) @@ -338,6 +402,18 @@ function resolveFileSpace(filePath: string, config: Config): { space: SpaceConfi return null; } +export interface ValidateFileOptions { + configDir?: string; + /** Evaluate rules on gated nodes anyway (gates are still reported). */ + includeGated?: boolean; + /** Evaluate only rules matching this filter (see `parseRuleFilter`). */ + ruleFilter?: RuleFilter; + /** Whether to evaluate semantic rules (default: false, or true if configured in space/config). */ + semantic?: boolean; + /** Optional custom judge provider (for testing or injection). */ + judgeProvider?: SemanticJudgeProvider; +} + /** * Validate a single file within its space. * @@ -350,7 +426,7 @@ function resolveFileSpace(filePath: string, config: Config): { space: SpaceConfi export async function validateFile( filePath: string, config: Config, - options: { configDir?: string; includeGated?: boolean } = {}, + options: ValidateFileOptions = {}, ): Promise { const resolved = resolveFileSpace(filePath, config); @@ -459,13 +535,22 @@ export async function validateFile( (v): GatingSeed => ({ file: v.file, title: v.nodeTitle, reason: 'hierarchy' }), ), ]; + const judge = resolveJudge(options.semantic, options.judgeProvider, space, config); const { violations, gates } = await evaluateSpaceRules(nodes, metadata, seeds, { includeGated: options.includeGated, + ruleFilter: options.ruleFilter, + judge, }); for (const v of violations) { if (v.file === label) { const target = v.severity === 'error' ? errors : warnings; - target[`rule:${v.ruleId}`] = { kind: 'rule', message: `[${v.ruleId}] ${v.description}`, severity: v.severity }; + target[`rule:${v.ruleId}`] = { + kind: 'rule', + message: v.evidence + ? `[${v.ruleId}] ${v.description} (Evidence: ${v.evidence})` + : `[${v.ruleId}] ${v.description}`, + severity: v.severity, + }; } } // A gated file with no violations would otherwise look clean — say that its rules were not checked. diff --git a/tests/fixtures/strict_ost/valid-directory/Progressive sign-up form.md b/tests/fixtures/strict_ost/valid-directory/Progressive sign-up form.md new file mode 100644 index 0000000..b5df6e6 --- /dev/null +++ b/tests/fixtures/strict_ost/valid-directory/Progressive sign-up form.md @@ -0,0 +1,7 @@ +--- +type: solution +status: exploring +parent: "[[Sign-up is too complex]]" +--- + +Break the sign-up into a progressive multi-step form that asks for one thing at a time. diff --git a/tests/fixtures/strict_ost/valid-directory/Social login option.md b/tests/fixtures/strict_ost/valid-directory/Social login option.md new file mode 100644 index 0000000..2b8ccb2 --- /dev/null +++ b/tests/fixtures/strict_ost/valid-directory/Social login option.md @@ -0,0 +1,7 @@ +--- +type: solution +status: exploring +parent: "[[Sign-up is too complex]]" +--- + +Offer social login so new users can skip manual account creation entirely. diff --git a/tests/judge/factory.test.ts b/tests/judge/factory.test.ts new file mode 100644 index 0000000..8283112 --- /dev/null +++ b/tests/judge/factory.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from 'bun:test'; +import { AnthropicJudgeProvider } from '../../src/judge/anthropic'; +import { createJudgeProvider } from '../../src/judge/factory'; +import { FakeJudgeProvider } from '../../src/judge/fake'; +import { OpenAIJudgeProvider } from '../../src/judge/openai'; + +describe('createJudgeProvider factory', () => { + it('returns custom provider when passed directly', () => { + const custom = new FakeJudgeProvider(); + const provider = createJudgeProvider(undefined, {}, custom); + expect(provider).toBe(custom); + }); + + it('creates FakeJudgeProvider when provider is "fake"', () => { + const provider = createJudgeProvider({ provider: 'fake' }, {}); + expect(provider.name).toBe('fake'); + expect(provider instanceof FakeJudgeProvider).toBe(true); + }); + + it('creates AnthropicJudgeProvider when ANTHROPIC_API_KEY is in env', () => { + const provider = createJudgeProvider(undefined, { ANTHROPIC_API_KEY: 'test-key' }); + expect(provider.name).toBe('anthropic'); + expect(provider instanceof AnthropicJudgeProvider).toBe(true); + }); + + it('creates AnthropicJudgeProvider from config apiKey', () => { + const provider = createJudgeProvider({ provider: 'anthropic', apiKey: 'test-key-config' }, {}); + expect(provider.name).toBe('anthropic'); + }); + + it('creates OpenAIJudgeProvider when OPENAI_API_KEY is in env', () => { + const provider = createJudgeProvider(undefined, { OPENAI_API_KEY: 'test-openai-key' }); + expect(provider.name).toBe('openai'); + expect(provider instanceof OpenAIJudgeProvider).toBe(true); + }); + + it('creates OpenAIJudgeProvider from OPENAI_BASE_URL (for local endpoints like Ollama)', () => { + const provider = createJudgeProvider(undefined, { OPENAI_BASE_URL: 'http://localhost:11434/v1' }); + expect(provider.name).toBe('openai'); + }); + + it('throws descriptive error when no provider or key is configured', () => { + expect(() => createJudgeProvider(undefined, {})).toThrow('No semantic judge provider configured'); + }); +}); diff --git a/tests/judge/fake-provider.test.ts b/tests/judge/fake-provider.test.ts new file mode 100644 index 0000000..ab93f16 --- /dev/null +++ b/tests/judge/fake-provider.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from 'bun:test'; +import { FakeJudgeProvider } from '../../src/judge/fake'; +import type { SemanticJudgeInput } from '../../src/judge/types'; +import { graded, node, passFailRule } from './helpers'; + +const input: SemanticJudgeInput = { node, rules: [passFailRule, graded] }; + +describe('FakeJudgeProvider', () => { + it("passes every rule by default using each rule's first non-failing verdict", async () => { + const fake = new FakeJudgeProvider(); + const results = await fake.evaluate(input); + expect(results['customer-voice']).toMatchObject({ verdict: 'pass' }); + expect(results.framing).toMatchObject({ verdict: 'good' }); + expect(fake.callHistory).toHaveLength(1); + }); + + it('maps the { pass } shorthand onto pass/fail', async () => { + const results = await new FakeJudgeProvider({ pass: false, evidence: 'nope' }).evaluate({ + node, + rules: [passFailRule], + }); + expect(results['customer-voice']).toEqual({ verdict: 'fail', evidence: 'nope' }); + }); + + it('returns full verdicts and errors as configured', async () => { + const fake = new FakeJudgeProvider({ + framing: { verdict: 'solution', confidence: 0.8 }, + 'customer-voice': { error: 'boom' }, + }); + const results = await fake.evaluate(input); + expect(results.framing).toEqual({ verdict: 'solution', confidence: 0.8 }); + expect(results['customer-voice']).toEqual({ error: 'boom' }); + }); + + it('prefers rule:title over rule keys, and supports Map and handler configs', async () => { + const map = new Map([ + ['framing', { verdict: 'weak' }], + ['framing:I cannot find my receipts', { verdict: 'solution' }], + ]); + expect((await new FakeJudgeProvider(map).evaluate(input)).framing).toEqual({ verdict: 'solution' }); + + const handler = new FakeJudgeProvider(({ rule }) => ({ verdict: rule.id === 'framing' ? 'weak' : 'pass' })); + expect((await handler.evaluate(input)).framing).toEqual({ verdict: 'weak' }); + }); +}); diff --git a/tests/judge/helpers.ts b/tests/judge/helpers.ts new file mode 100644 index 0000000..0ceb608 --- /dev/null +++ b/tests/judge/helpers.ts @@ -0,0 +1,27 @@ +import type { SemanticJudgeInput, SemanticRuleSpec } from '../../src/judge/types'; + +export const node: SemanticJudgeInput['node'] = { + label: 'opp1.md', + title: 'I cannot find my receipts', + type: 'opportunity', + resolvedType: 'opportunity', + schemaData: { title: 'I cannot find my receipts', type: 'opportunity', source: 'User interview with Alice' }, + resolvedParents: [{ title: 'Faster Tax Filing', field: 'parent' }], +}; + +export const passFailRule: SemanticRuleSpec = { + id: 'customer-voice', + description: 'Must be in customer voice', + rubric: 'Needs to sound like a customer quote, not a solution or business goal', + category: 'best-practice', + verdicts: { pass: 'Customer voice.', fail: 'Not customer voice.' }, + failOn: ['fail'], +}; + +export const graded: SemanticRuleSpec = { + id: 'framing', + description: 'Framing quality', + category: 'best-practice', + verdicts: { good: 'Clear need', weak: 'Vague need', solution: 'A solution in disguise' }, + failOn: ['solution'], +}; diff --git a/tests/judge/jev.test.ts b/tests/judge/jev.test.ts new file mode 100644 index 0000000..c220ec3 --- /dev/null +++ b/tests/judge/jev.test.ts @@ -0,0 +1,77 @@ +import { afterEach, describe, expect, it } from 'bun:test'; +import { createJudgeProvider } from '../../src/judge/factory'; +import { JevJudgeProvider } from '../../src/judge/jev'; +import { graded, node, passFailRule } from './helpers'; + +const realFetch = globalThis.fetch; +afterEach(() => { + globalThis.fetch = realFetch; +}); + +function mockFetch(body: unknown, status = 200) { + const calls: Array<{ url: string; init: RequestInit }> = []; + globalThis.fetch = (async (url: string, init: RequestInit) => { + calls.push({ url, init }); + return new Response(JSON.stringify(body), { status }); + }) as unknown as typeof fetch; + return calls; +} + +describe('JevJudgeProvider', () => { + it('sends one request per node with a choice question per rule', async () => { + const calls = mockFetch({ + answers: { + 'customer-voice': { + type: 'choice', + choice: 'pass', + confidence: 0.9, + probabilities: { pass: 0.95, fail: 0.05 }, + }, + framing: { + type: 'choice', + choice: 'solution', + confidence: 0.6, + probabilities: { good: 0.1, weak: 0.3, solution: 0.6 }, + }, + }, + }); + const jev = new JevJudgeProvider({ apiKey: 'k' }); + const results = await jev.evaluate({ node, rules: [passFailRule, graded] }); + + expect(calls).toHaveLength(1); + expect(calls[0]?.url).toBe('https://openrouter.ai/api/alpha/decisions'); + const sent = JSON.parse(calls[0]?.init.body as string); + expect(sent.model).toBe('typesafe/jev-1.13'); + expect(sent.state).toMatchObject({ title: 'I cannot find my receipts', parents: ['Faster Tax Filing'] }); + expect(sent.questions.framing).toEqual({ + type: 'choice', + instructions: 'Framing quality', + criteria: graded.verdicts, + }); + expect(sent.questions['customer-voice'].instructions).toContain('customer quote'); + expect(((calls[0]?.init.headers ?? {}) as Record).authorization).toBe('Bearer k'); + + expect(results.framing).toMatchObject({ verdict: 'solution', confidence: 0.6 }); + expect((results.framing as { evidence: string }).evidence).toContain('solution 0.60'); + }); + + it('reports an error for a rule with a missing or unknown choice, leaving others intact', async () => { + mockFetch({ answers: { framing: { choice: 'nonsense' }, 'customer-voice': { choice: 'fail' } } }); + const results = await new JevJudgeProvider({ apiKey: 'k' }).evaluate({ node, rules: [passFailRule, graded] }); + expect(results.framing).toHaveProperty('error'); + expect(results['customer-voice']).toMatchObject({ verdict: 'fail' }); + }); + + it('throws on an HTTP error', async () => { + mockFetch({ error: 'bad' }, 401); + await expect(new JevJudgeProvider({ apiKey: 'k' }).evaluate({ node, rules: [passFailRule] })).rejects.toThrow( + 'Jev API error (401)', + ); + }); + + it('is created explicitly via provider "jev" with OPENROUTER_API_KEY, never auto-detected', () => { + expect(createJudgeProvider({ provider: 'jev' }, { OPENROUTER_API_KEY: 'k' }).name).toBe('jev'); + expect(() => createJudgeProvider({ provider: 'jev' }, {})).toThrow('OPENROUTER_API_KEY'); + expect(() => createJudgeProvider(undefined, { OPENROUTER_API_KEY: 'k' })).toThrow('No semantic judge provider'); + }); +}); diff --git a/tests/judge/prompt.test.ts b/tests/judge/prompt.test.ts new file mode 100644 index 0000000..efce894 --- /dev/null +++ b/tests/judge/prompt.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from 'bun:test'; +import { buildJudgeSystemPrompt, buildJudgeUserPrompt, parseJudgeResponse } from '../../src/judge/prompt'; +import { graded, node, passFailRule } from './helpers'; + +describe('judge prompt and response parser', () => { + it('system prompt asks for a JSON verdict, confidence and evidence', () => { + const prompt = buildJudgeSystemPrompt(); + expect(prompt).toContain('"verdict"'); + expect(prompt).toContain('"confidence"'); + expect(prompt).toContain('"evidence"'); + }); + + it('user prompt lists the rule, its allowed verdicts and the node', () => { + const prompt = buildJudgeUserPrompt(node, graded); + expect(prompt).toContain('framing'); + expect(prompt).toContain('"weak": Vague need'); + expect(prompt).toContain('"solution": A solution in disguise'); + expect(prompt).toContain('I cannot find my receipts'); + expect(prompt).toContain('Faster Tax Filing'); + expect(prompt).toContain('User interview with Alice'); + }); + + it('user prompt prefers the rubric over the description', () => { + expect(buildJudgeUserPrompt(node, passFailRule)).toContain('Needs to sound like a customer quote'); + expect(buildJudgeUserPrompt(node, graded)).toContain('Rubric: Framing quality'); + }); + + const allowed = ['pass', 'fail']; + + it('parses clean JSON', () => { + expect(parseJudgeResponse('{"verdict":"pass","confidence":0.9,"evidence":"clear"}', allowed)).toEqual({ + verdict: 'pass', + confidence: 0.9, + evidence: 'clear', + }); + }); + + it('parses fenced JSON and clamps confidence', () => { + const raw = '```json\n{"verdict":"fail","confidence":4,"evidence":"solution"}\n```'; + expect(parseJudgeResponse(raw, allowed)).toEqual({ verdict: 'fail', confidence: 1, evidence: 'solution' }); + }); + + it('allows confidence and evidence to be absent', () => { + expect(parseJudgeResponse('{"verdict":"pass"}', allowed)).toEqual({ verdict: 'pass' }); + }); + + it('extracts fields from text surrounding malformed JSON', () => { + const raw = 'Sure! Here you go: {"verdict": "fail", "confidence": 0.4, "evidence": "says \\"build X\\"" trailing'; + const result = parseJudgeResponse(raw, allowed); + expect(result).toMatchObject({ verdict: 'fail', confidence: 0.4 }); + }); + + it('returns an error for a verdict outside the allowed set', () => { + const result = parseJudgeResponse('{"verdict":"maybe"}', allowed); + expect(result).toHaveProperty('error'); + expect((result as { error: string }).error).toContain('unknown verdict "maybe"'); + }); + + it('returns an error when nothing parseable is present', () => { + expect(parseJudgeResponse('I cannot decide', allowed)).toHaveProperty('error'); + }); +}); diff --git a/tests/judge/verdicts.test.ts b/tests/judge/verdicts.test.ts new file mode 100644 index 0000000..4578f0c --- /dev/null +++ b/tests/judge/verdicts.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from 'bun:test'; +import { PerRuleJudgeProvider } from '../../src/judge/per-rule'; +import type { SemanticJudgeResult } from '../../src/judge/types'; +import { semanticRuleSpec } from '../../src/judge/verdicts'; +import type { Rule } from '../../src/types'; +import { graded, node, passFailRule } from './helpers'; + +const base: Rule = { id: 'r', category: 'best-practice', description: 'd', mode: 'semantic' }; + +describe('semanticRuleSpec', () => { + it('defaults to pass/fail with fail as the violation', () => { + const spec = semanticRuleSpec(base); + expect(Object.keys(spec.verdicts)).toEqual(['pass', 'fail']); + expect(spec.failOn).toEqual(['fail']); + }); + + it('uses declared verdicts and failOn', () => { + const spec = semanticRuleSpec({ ...base, verdicts: { a: 'A', b: 'B', c: 'C' }, failOn: ['b', 'c'] }); + expect(spec.failOn).toEqual(['b', 'c']); + }); + + it('rejects declared verdicts without failOn, unknown failOn, and all-failing sets', () => { + expect(() => semanticRuleSpec({ ...base, verdicts: { a: 'A', b: 'B' } })).toThrow('no failOn'); + expect(() => semanticRuleSpec({ ...base, verdicts: { a: 'A', b: 'B' }, failOn: ['z'] })).toThrow( + 'not in its verdicts', + ); + expect(() => semanticRuleSpec({ ...base, verdicts: { a: 'A', b: 'B' }, failOn: ['a', 'b'] })).toThrow( + 'no passing verdict', + ); + }); +}); + +describe('PerRuleJudgeProvider', () => { + class Stub extends PerRuleJudgeProvider { + readonly name = 'stub'; + protected async judgeRule(_n: unknown, rule: { id: string }): Promise { + if (rule.id === 'framing') throw new Error('network down'); + return { verdict: 'pass' }; + } + } + + it('isolates a failing rule from the others', async () => { + const results = await new Stub().evaluate({ node, rules: [passFailRule, graded] }); + expect(results['customer-voice']).toEqual({ verdict: 'pass' }); + expect(results.framing).toEqual({ error: 'network down' }); + }); +}); diff --git a/tests/schema/rule-filter.test.ts b/tests/schema/rule-filter.test.ts new file mode 100644 index 0000000..37054f9 --- /dev/null +++ b/tests/schema/rule-filter.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from 'bun:test'; +import { filterRules, parseRuleFilter } from '../../src/schema/rule-filter'; +import type { Rule } from '../../src/types'; + +const rules: Rule[] = [ + { id: 'a', category: 'validation', description: 'a', check: 'true', type: 'goal' }, + { id: 'b', category: 'best-practice', description: 'b', mode: 'semantic', type: 'opportunity' }, + { id: 'c', category: 'coherence', description: 'c', check: 'true', severity: 'error' }, +]; +const ids = (r: Rule[]) => r.map((x) => x.id); + +describe('rule filter', () => { + it('parses repeated specs and merges values per dimension', () => { + expect(parseRuleFilter(['mode=semantic', 'category=coherence,validation', 'category=workflow'])).toEqual({ + mode: ['semantic'], + category: ['coherence', 'validation', 'workflow'], + }); + }); + + it('rejects malformed specs and unknown dimensions', () => { + expect(() => parseRuleFilter(['semantic'])).toThrow('Invalid rule filter'); + expect(() => parseRuleFilter(['colour=red'])).toThrow('Invalid rule filter'); + expect(() => parseRuleFilter(['mode='])).toThrow('Invalid rule filter'); + }); + + it('filters by mode (defaulting to expression), severity (effective), category, type and id', () => { + expect(ids(filterRules(rules, { mode: ['semantic'] }))).toEqual(['b']); + expect(ids(filterRules(rules, { mode: ['expression'] }))).toEqual(['a', 'c']); + expect(ids(filterRules(rules, { severity: ['error'] }))).toEqual(['a', 'c']); + expect(ids(filterRules(rules, { category: ['coherence'] }))).toEqual(['c']); + expect(ids(filterRules(rules, { type: ['goal', 'opportunity'] }))).toEqual(['a', 'b']); + expect(ids(filterRules(rules, { id: ['c'] }))).toEqual(['c']); + }); + + it('ANDs dimensions and treats an undefined filter as everything', () => { + expect(ids(filterRules(rules, { severity: ['error'], mode: ['semantic'] }))).toEqual([]); + expect(filterRules(rules, undefined)).toBe(rules); + }); +}); diff --git a/tests/schema/validate-rules-semantic.test.ts b/tests/schema/validate-rules-semantic.test.ts new file mode 100644 index 0000000..6ed1cac --- /dev/null +++ b/tests/schema/validate-rules-semantic.test.ts @@ -0,0 +1,168 @@ +import { describe, expect, it } from 'bun:test'; +import { FakeJudgeProvider } from '../../src/judge/fake'; +import { validateRules } from '../../src/schema/validate-rules'; +import type { Rule, SpaceNode } from '../../src/types'; +import { makeParentRef } from '../test-helpers'; + +describe('validateRules - semantic evaluation', () => { + const sampleNodes: SpaceNode[] = [ + { + label: 'outcome.md', + title: 'Outcome Metric', + type: 'outcome', + resolvedType: 'outcome', + schemaData: { title: 'Outcome Metric', type: 'outcome', status: 'active', metric: 'Increase conversions' }, + linkTargets: ['Outcome Metric'], + resolvedParents: [], + resolvedLinks: [], + }, + { + label: 'valid-opp.md', + title: 'I struggle to finish signup on mobile', + type: 'opportunity', + resolvedType: 'opportunity', + schemaData: { + title: 'I struggle to finish signup on mobile', + type: 'opportunity', + status: 'active', + parent: '[[Outcome Metric]]', + source: 'User interview with Jane', + }, + linkTargets: ['I struggle to finish signup on mobile'], + resolvedParents: [makeParentRef('Outcome Metric')], + resolvedLinks: [], + }, + { + label: 'bad-opp.md', + title: 'Build social login integration', + type: 'opportunity', + resolvedType: 'opportunity', + schemaData: { + title: 'Build social login integration', + type: 'opportunity', + status: 'active', + parent: '[[Outcome Metric]]', + source: 'Engineering suggestion', + }, + linkTargets: ['Build social login integration'], + resolvedParents: [makeParentRef('Outcome Metric')], + resolvedLinks: [], + }, + ]; + + const rules: Rule[] = [ + { + id: 'active-opportunity-count', + category: 'workflow', + description: 'At most 2 active opportunities', + check: '$count(nodes[resolvedType="opportunity" and status="active"]) <= 2', + }, + { + id: 'opportunity-customer-voice', + category: 'best-practice', + mode: 'semantic', + type: 'opportunity', + description: 'Opportunity must be framed in customer voice, not a solution or business goal', + rubric: 'Must describe a customer pain point or need in customer language, not a solution to build.', + }, + ]; + + it('skips semantic rules when semantic option is false / omitted (default validate)', async () => { + // Default validate should only run expression rules and remain deterministic & free + const violations = await validateRules(sampleNodes, rules); + expect(violations).toHaveLength(0); + expect(violations.semanticEvaluations).toHaveLength(0); + expect(violations.gatedRules).toHaveLength(0); + }); + + it('evaluates semantic rules when semantic option is true', async () => { + const fake = new FakeJudgeProvider( + new Map([ + [ + 'opportunity-customer-voice:I struggle to finish signup on mobile', + { pass: true, evidence: 'Clearly stated from customer perspective as a pain point' }, + ], + [ + 'opportunity-customer-voice:Build social login integration', + { pass: false, evidence: 'Stated as a solution feature ("Build social login") rather than a customer need' }, + ], + ]), + ); + + const violations = await validateRules(sampleNodes, rules, { + semantic: true, + judgeProvider: fake, + }); + + expect(violations).toHaveLength(1); + expect(violations[0]?.ruleId).toBe('opportunity-customer-voice'); + expect(violations[0]?.file).toBe('bad-opp.md'); + expect(violations[0]?.category).toBe('best-practice'); + expect(violations[0]?.evidence).toBe( + 'Stated as a solution feature ("Build social login") rather than a customer need', + ); + + expect(violations.semanticEvaluations).toHaveLength(2); + expect(violations.semanticEvaluations.find((e) => e.file === 'valid-opp.md')?.pass).toBe(true); + expect(violations.semanticEvaluations.find((e) => e.file === 'bad-opp.md')?.pass).toBe(false); + }); + + it('gates semantic rules when node has upstream structural error (cascading)', async () => { + const fake = new FakeJudgeProvider(); + + // Mark bad-opp.md as having an upstream structural error via gatedNodeLabels + const gatedNodeLabels = new Set(['bad-opp.md']); + + const violations = await validateRules(sampleNodes, rules, { + semantic: true, + judgeProvider: fake, + gatedNodeLabels, + }); + + // bad-opp.md was gated; only valid-opp.md was sent to the judge + expect(fake.callHistory).toHaveLength(1); + expect(fake.callHistory[0]?.node.label).toBe('valid-opp.md'); + + expect(violations.gatedRules).toHaveLength(1); + expect(violations.gatedRules[0]?.file).toBe('bad-opp.md'); + expect(violations.gatedRules[0]?.ruleId).toBe('opportunity-customer-voice'); + }); + + it('gates semantic rules when node fails an upstream expression rule', async () => { + const strictRules: Rule[] = [ + { + id: 'require-interviews-source', + category: 'validation', + type: 'opportunity', + check: '$contains(current.source, "interview") = true', + description: 'Source must mention interview', + }, + { + id: 'opportunity-customer-voice', + category: 'best-practice', + mode: 'semantic', + type: 'opportunity', + description: 'Must be in customer voice', + }, + ]; + + const fake = new FakeJudgeProvider(); + + const violations = await validateRules(sampleNodes, strictRules, { + semantic: true, + judgeProvider: fake, + }); + + // bad-opp.md source is "Engineering suggestion", so it failed require-interviews-source + // Therefore it should be gated from opportunity-customer-voice + const badOppViolations = violations.filter((v) => v.file === 'bad-opp.md'); + expect(badOppViolations).toHaveLength(1); + expect(badOppViolations[0]?.ruleId).toBe('require-interviews-source'); + + // Only valid-opp.md was evaluated by the semantic judge + expect(fake.callHistory).toHaveLength(1); + expect(fake.callHistory[0]?.node.label).toBe('valid-opp.md'); + + expect(violations.gatedRules.some((g) => g.file === 'bad-opp.md')).toBe(true); + }); +}); diff --git a/tests/validate/gating.test.ts b/tests/validate/gating.test.ts index 01a48e2..be8113a 100644 --- a/tests/validate/gating.test.ts +++ b/tests/validate/gating.test.ts @@ -43,14 +43,11 @@ describe('validateSpace cascade gating', () => { expect(gate!.gatedFiles).toContain('Child Opportunity.md'); }); - it('does not gate on a non-error rule violation by default (solution-quantity is best-practice)', async () => { - // The fixture opportunity has only one solution below it, violating solution-quantity. - // That rule is info severity, so it doesn't block and the solution's own rules still run. + it('a clean space with only non-blocking rules reports no gates', async () => { const context = makeSpaceContext(VALID_DIR, SCHEMA_PATH); const result = await validateSpace(context); expect(result.nodeErrorCount).toBe(0); - expect(result.ruleViolations.some((v) => v.ruleId === 'solution-quantity')).toBe(true); expect(result.coverageGates).toEqual([]); }); }); diff --git a/tests/validate/semantic-validation.test.ts b/tests/validate/semantic-validation.test.ts new file mode 100644 index 0000000..3c75dee --- /dev/null +++ b/tests/validate/semantic-validation.test.ts @@ -0,0 +1,191 @@ +import { describe, expect, it } from 'bun:test'; +import { join } from 'node:path'; +import type { Config } from '../../src/config'; +import { FakeJudgeProvider } from '../../src/judge/fake'; +import { bundledSchemasDir } from '../../src/schema/schema'; +import { createSpaceContext } from '../../src/space-context'; +import type { FileValidationResult } from '../../src/types'; +import { validateFile, validateSpace } from '../../src/validate'; + +const STRICT_SCHEMA_PATH = join(bundledSchemasDir, 'strict_ost.json'); +const VALID_DIR = join(import.meta.dir, '../fixtures/strict_ost/valid-directory'); +const INVALID_DIR = join(import.meta.dir, '../fixtures/strict_ost/invalid'); + +describe('Semantic rule evaluation integration (validateSpace & validateFile)', () => { + const validConfig: Config = { + spaces: [ + { + name: 'strict-valid', + path: VALID_DIR, + schema: STRICT_SCHEMA_PATH, + }, + ], + }; + + const validContext = createSpaceContext('strict-valid', validConfig, { configDir: import.meta.dir }); + + describe('validateSpace', () => { + it('skips semantic rules by default (deterministic & free)', async () => { + const result = await validateSpace(validContext); + expect(result.semanticEvaluations ?? []).toHaveLength(0); + expect(result.gatedRules ?? []).toHaveLength(0); + expect(result.ruleViolations.some((v) => v.ruleId === 'opportunity-customer-voice')).toBe(false); + }); + + it('evaluates semantic rules when opt-in flag semantic: true is passed', async () => { + const fakeProvider = new FakeJudgeProvider({ + pass: true, + evidence: 'Opportunity is framed in clear customer language describing difficulty with sign-up.', + }); + + const result = await validateSpace(validContext, { + semantic: true, + judgeProvider: fakeProvider, + }); + + expect(result.semanticEvaluations).toBeDefined(); + expect(result.semanticEvaluations!.length).toBeGreaterThan(0); + + const oppEval = result.semanticEvaluations!.find((e) => e.ruleId === 'opportunity-customer-voice'); + expect(oppEval).toBeDefined(); + expect(oppEval?.pass).toBe(true); + expect(oppEval?.evidence).toContain('framed in clear customer language'); + + // No violation when pass is true + expect(result.ruleViolations.some((v) => v.ruleId === 'opportunity-customer-voice')).toBe(false); + }); + + it('records violation with evidence when judge returns pass: false', async () => { + const fakeProvider = new FakeJudgeProvider({ + pass: false, + evidence: 'The opportunity reads like a feature request rather than a problem statement.', + }); + + const result = await validateSpace(validContext, { + semantic: true, + judgeProvider: fakeProvider, + }); + + const violation = result.ruleViolations.find((v) => v.ruleId === 'opportunity-customer-voice'); + expect(violation).toBeDefined(); + expect(violation?.category).toBe('best-practice'); + expect(violation?.evidence).toBe('The opportunity reads like a feature request rather than a problem statement.'); + expect(violation?.file).toBe('Sign-up is too complex.md'); + }); + + it('gates semantic rules when node has upstream structural errors (cascading)', async () => { + const invalidConfig: Config = { + spaces: [ + { + name: 'strict-invalid', + path: INVALID_DIR, + schema: STRICT_SCHEMA_PATH, + }, + ], + }; + const invalidContext = createSpaceContext('strict-invalid', invalidConfig, { configDir: import.meta.dir }); + const fakeProvider = new FakeJudgeProvider(); + + const result = await validateSpace(invalidContext, { + semantic: true, + judgeProvider: fakeProvider, + }); + + // opportunity-no-source.md fails schema validation (missing required 'source' field) + expect(result.nodeErrors.some((e) => e.file === 'opportunity-no-source.md')).toBe(true); + + // Therefore it must be gated and never evaluated by the judge + expect(result.gatedRules).toBeDefined(); + const gated = result.gatedRules!.find( + (g) => g.file === 'opportunity-no-source.md' && g.ruleId === 'opportunity-customer-voice', + ); + expect(gated).toBeDefined(); + expect(gated?.reason).toMatch(/^Gated by/); + + // Judge was never called for opportunity-no-source.md + expect(fakeProvider.callHistory.some((c) => c.node.label === 'opportunity-no-source.md')).toBe(false); + }); + }); + + describe('validateFile', () => { + const targetFile = join(VALID_DIR, 'Sign-up is too complex.md'); + + it('skips semantic rule evaluation for single file by default', async () => { + const result = (await validateFile(targetFile, validConfig, { + configDir: import.meta.dir, + })) as FileValidationResult; + + expect(result.inSpace).toBe(true); + expect(result.errors['rule:opportunity-customer-voice']).toBeUndefined(); + }); + + it('evaluates semantic rule and reports evidence on violation when semantic: true', async () => { + const failingProvider = new FakeJudgeProvider({ + pass: false, + evidence: 'Opportunity is framed in solution space.', + }); + + const result = (await validateFile(targetFile, validConfig, { + semantic: true, + judgeProvider: failingProvider, + configDir: import.meta.dir, + })) as FileValidationResult; + + expect(result.inSpace).toBe(true); + // best-practice rules default to non-error severity, so they land in warnings + const ruleError = result.warnings['rule:opportunity-customer-voice']; + expect(ruleError).toBeDefined(); + expect(ruleError?.kind).toBe('rule'); + expect(ruleError?.message).toContain('Evidence: Opportunity is framed in solution space.'); + }); + }); +}); + +describe('Semantic rules: verdicts, batching and rule filter', () => { + const config: Config = { + spaces: [{ name: 'strict-valid', path: VALID_DIR, schema: STRICT_SCHEMA_PATH }], + }; + const context = createSpaceContext('strict-valid', config, { configDir: import.meta.dir }); + + it('judges each node once with all of its semantic rules', async () => { + const fake = new FakeJudgeProvider(); + await validateSpace(context, { semantic: true, judgeProvider: fake }); + const labels = fake.callHistory.map((c) => c.node.label); + expect(new Set(labels).size).toBe(labels.length); + expect(fake.callHistory.every((c) => c.rules.length >= 1)).toBe(true); + }); + + it('records the verdict and confidence on evaluations and fails only on failOn verdicts', async () => { + const fake = new FakeJudgeProvider({ verdict: 'fail', confidence: 0.7, evidence: 'solution in disguise' }); + const result = await validateSpace(context, { semantic: true, judgeProvider: fake }); + const record = result.semanticEvaluations?.find((e) => e.ruleId === 'opportunity-customer-voice'); + expect(record).toMatchObject({ verdict: 'fail', pass: false, confidence: 0.7 }); + expect(result.ruleViolations.some((v) => v.ruleId === 'opportunity-customer-voice')).toBe(true); + }); + + it('reports a per-rule judge error as a violation without gating the subtree', async () => { + const fake = new FakeJudgeProvider({ error: 'rate limited' }); + const result = await validateSpace(context, { semantic: true, judgeProvider: fake }); + const violation = result.ruleViolations.find((v) => v.ruleId === 'opportunity-customer-voice'); + expect(violation?.description).toContain('Judge error: rate limited'); + expect(result.coverageGates.filter((g) => g.ruleId === 'opportunity-customer-voice')).toEqual([]); + }); + + it('--rule-filter narrows evaluation: semantic-only runs no expression rules, expression-only runs no judge', async () => { + const semanticOnly = await validateSpace(context, { + semantic: true, + judgeProvider: new FakeJudgeProvider({ pass: false }), + ruleFilter: { mode: ['semantic'] }, + }); + expect(semanticOnly.ruleViolations.every((v) => v.ruleId === 'opportunity-customer-voice')).toBe(true); + + const fake = new FakeJudgeProvider(); + const expressionOnly = await validateSpace(context, { + semantic: true, + judgeProvider: fake, + ruleFilter: { mode: ['expression'] }, + }); + expect(fake.callHistory).toHaveLength(0); + expect(expressionOnly.semanticEvaluations ?? []).toHaveLength(0); + }); +}); diff --git a/tests/validate/strict.test.ts b/tests/validate/strict.test.ts index 753ef52..3f0e2ff 100644 --- a/tests/validate/strict.test.ts +++ b/tests/validate/strict.test.ts @@ -198,8 +198,8 @@ describe('Strict OST schema validation', () => { ({ nodes } = await readSpaceDirectory(makePluginContext(VALID_DIR, STRICT_SCHEMA_PATH))); }); - it('reads all 4 nodes from valid-directory', () => { - expect(nodes).toHaveLength(4); + it('reads all 6 nodes from valid-directory', () => { + expect(nodes).toHaveLength(6); }); it('all nodes pass strict_ost schema validation', () => { From c93143205b889729295f3634e792a186143fbd94 Mon Sep 17 00:00:00 2001 From: Roger Barnes Date: Tue, 13 Oct 2026 20:51:49 +1100 Subject: [PATCH 02/22] fix(rules): require check unless semantic, reject global semantic rules, accept jev in config Addresses review on the semantic rules PR. --- .../_structured_context_schema_meta.json | 23 ++++++++++++++++++ src/config.ts | 2 +- src/schema/metadata-contract.ts | 5 ++++ .../semantic-rule-contract/no-check.json | 6 +++++ .../semantic-check.json | 8 +++++++ .../semantic-global.json | 8 +++++++ .../semantic-rule-contract/semantic-ok.json | 6 +++++ tests/schema/semantic-rule-contract.test.ts | 24 +++++++++++++++++++ 8 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures/semantic-rule-contract/no-check.json create mode 100644 tests/fixtures/semantic-rule-contract/semantic-check.json create mode 100644 tests/fixtures/semantic-rule-contract/semantic-global.json create mode 100644 tests/fixtures/semantic-rule-contract/semantic-ok.json create mode 100644 tests/schema/semantic-rule-contract.test.ts diff --git a/schemas/generated/latest/_structured_context_schema_meta.json b/schemas/generated/latest/_structured_context_schema_meta.json index 4d64d61..8517107 100644 --- a/schemas/generated/latest/_structured_context_schema_meta.json +++ b/schemas/generated/latest/_structured_context_schema_meta.json @@ -246,6 +246,29 @@ } }, "required": ["id", "category", "description"], + "if": { + "properties": { + "mode": { + "const": "semantic" + } + }, + "required": ["mode"] + }, + "then": { + "not": { + "anyOf": [ + { + "required": ["check"] + }, + { + "required": ["scope"] + } + ] + } + }, + "else": { + "required": ["check"] + }, "additionalProperties": false } } diff --git a/src/config.ts b/src/config.ts index ed21616..8cec75d 100644 --- a/src/config.ts +++ b/src/config.ts @@ -16,7 +16,7 @@ import { bundledSchemasDir } from './schema/schema'; const JUDGE_CONFIG_SCHEMA = { type: 'object', properties: { - provider: { type: 'string', enum: ['anthropic', 'openai', 'fake'] }, + provider: { type: 'string', enum: ['anthropic', 'openai', 'jev', 'fake'] }, model: { type: 'string' }, apiKey: { type: 'string' }, endpoint: { type: 'string' }, diff --git a/src/schema/metadata-contract.ts b/src/schema/metadata-contract.ts index aa50e8d..b22625c 100644 --- a/src/schema/metadata-contract.ts +++ b/src/schema/metadata-contract.ts @@ -76,6 +76,11 @@ const RULE_SCHEMA = { gatesDescendants: { type: 'boolean' }, }, required: ['id', 'category', 'description'], + // Semantic rules are judged per node, so they take no `check` and cannot be global; every other rule needs a `check`. + if: { properties: { mode: { const: 'semantic' } }, required: ['mode'] }, + // biome-ignore lint/suspicious/noThenProperty: JSON Schema `if`/`then`/`else` keyword, not a thenable + then: { not: { anyOf: [{ required: ['check'] }, { required: ['scope'] }] } }, + else: { required: ['check'] }, additionalProperties: false, } as const; diff --git a/tests/fixtures/semantic-rule-contract/no-check.json b/tests/fixtures/semantic-rule-contract/no-check.json new file mode 100644 index 0000000..b0678d8 --- /dev/null +++ b/tests/fixtures/semantic-rule-contract/no-check.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://raw.githubusercontent.com/mindsocket/structured-context/main/schemas/generated/latest/_structured_context_schema_meta.json", + "$id": "sctx://semantic-rule-contract.no-check", + "$metadata": { "rules": [{ "id": "r", "category": "validation", "description": "d" }] }, + "type": "object" +} diff --git a/tests/fixtures/semantic-rule-contract/semantic-check.json b/tests/fixtures/semantic-rule-contract/semantic-check.json new file mode 100644 index 0000000..e348ae4 --- /dev/null +++ b/tests/fixtures/semantic-rule-contract/semantic-check.json @@ -0,0 +1,8 @@ +{ + "$schema": "https://raw.githubusercontent.com/mindsocket/structured-context/main/schemas/generated/latest/_structured_context_schema_meta.json", + "$id": "sctx://semantic-rule-contract.semantic-check", + "$metadata": { + "rules": [{ "id": "r", "category": "validation", "description": "d", "mode": "semantic", "check": "true" }] + }, + "type": "object" +} diff --git a/tests/fixtures/semantic-rule-contract/semantic-global.json b/tests/fixtures/semantic-rule-contract/semantic-global.json new file mode 100644 index 0000000..a7bb0bd --- /dev/null +++ b/tests/fixtures/semantic-rule-contract/semantic-global.json @@ -0,0 +1,8 @@ +{ + "$schema": "https://raw.githubusercontent.com/mindsocket/structured-context/main/schemas/generated/latest/_structured_context_schema_meta.json", + "$id": "sctx://semantic-rule-contract.semantic-global", + "$metadata": { + "rules": [{ "id": "r", "category": "validation", "description": "d", "mode": "semantic", "scope": "global" }] + }, + "type": "object" +} diff --git a/tests/fixtures/semantic-rule-contract/semantic-ok.json b/tests/fixtures/semantic-rule-contract/semantic-ok.json new file mode 100644 index 0000000..e1f0ad7 --- /dev/null +++ b/tests/fixtures/semantic-rule-contract/semantic-ok.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://raw.githubusercontent.com/mindsocket/structured-context/main/schemas/generated/latest/_structured_context_schema_meta.json", + "$id": "sctx://semantic-rule-contract.semantic-ok", + "$metadata": { "rules": [{ "id": "r", "category": "validation", "description": "d", "mode": "semantic" }] }, + "type": "object" +} diff --git a/tests/schema/semantic-rule-contract.test.ts b/tests/schema/semantic-rule-contract.test.ts new file mode 100644 index 0000000..d03b6da --- /dev/null +++ b/tests/schema/semantic-rule-contract.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, it } from 'bun:test'; +import { join } from 'node:path'; +import { loadMetadata } from '../../src/schema/schema'; + +const dir = join(import.meta.dir, '..', 'fixtures/semantic-rule-contract'); +const load = (name: string) => loadMetadata(join(dir, `${name}.json`)); + +describe('rule contract: check and semantic mode', () => { + it('requires check on expression rules', () => { + expect(() => load('no-check')).toThrow(); + }); + + it('accepts a semantic rule without check', () => { + expect(load('semantic-ok').rules).toHaveLength(1); + }); + + it('rejects a semantic rule that also declares check', () => { + expect(() => load('semantic-check')).toThrow(); + }); + + it('rejects a global semantic rule', () => { + expect(() => load('semantic-global')).toThrow(); + }); +}); From 0f28d3dbe4fa613d2ffc2ea3514d3c0ec955658e Mon Sep 17 00:00:00 2001 From: Roger Barnes Date: Tue, 13 Oct 2026 22:47:18 +1100 Subject: [PATCH 03/22] docs: document judge config, semantic rule terms and API exports Per the definition of done: config.md (judge/semantic keys), concepts.md (semantic rule, judge, verdict; gating no longer 'future'), api.md (judging section, option lists), AGENTS.md (src/judge, rule-filter), config example, and the remaining src/api.ts exports. Adds a config test for provider jev. --- AGENTS.md | 2 ++ config.example.json | 6 +++++- docs/api.md | 18 ++++++++++++++++-- docs/concepts.md | 8 +++++++- docs/config.md | 23 +++++++++++++++++++++++ docs/rules.md | 14 +------------- src/api.ts | 6 ++++++ tests/config.test.ts | 18 ++++++++++++++++++ 8 files changed, 78 insertions(+), 17 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 2b9f439..6512600 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -46,6 +46,8 @@ There are several places that need reviewing and updating with any new feature o - `src/schema/metadata-contract.ts` — Single source of truth for the `$metadata` contract - `schemas/generated/latest/_structured_context_schema_meta.json` — Generated metaschema for the newest dialect (generated on build or with `bun run generate:schema-meta`). `schemas/generated//` folders are frozen copies, created by the `version` script (`generate:schema-meta --release`) only when a release changes the dialect; never edit them. `src/schema/meta-schemas.ts` loads them at runtime. - `src/schema/gating.ts` — Cascade gating: computes hierarchy depth and evaluates rules with subtree gating (schema/hierarchy/blocking-rule failures suppress downstream rule checks). See [docs/rules.md](docs/rules.md#cascade-gating). +- `src/judge/` — Semantic rule judges: provider interface and types (`types.ts`), verdict resolution (`verdicts.ts`), per-node dispatch (`judge-node.ts`), and providers (`anthropic.ts`, `openai.ts`, `jev.ts`, `fake.ts`). See [docs/rules.md](docs/rules.md#semantic-rules). +- `src/schema/rule-filter.ts` — `--rule-filter` parsing and rule filtering by mode, category, severity, type or id. ## Testing For most development only the main unit tests need re-running regularly. diff --git a/config.example.json b/config.example.json index 77796c3..40c62e4 100644 --- a/config.example.json +++ b/config.example.json @@ -17,5 +17,9 @@ } } ], - "schema": "/path/to/custom/schemas/my-general-schema.json" + "schema": "/path/to/custom/schemas/my-general-schema.json", + "judge": { + "provider": "anthropic", + "model": "claude-haiku-5-5" + } } diff --git a/docs/api.md b/docs/api.md index c24a881..80b66d1 100644 --- a/docs/api.md +++ b/docs/api.md @@ -18,8 +18,8 @@ see [architecture.md](architecture.md#information-flow) for that flow. | `loadSpaceGraph(spaceName, config, options?)` | Read, assemble, and optionally filter a space into a navigable `SpaceGraph`. `options.hierarchy` names the hierarchy that structures the graph (default: the main hierarchy). | | `assembleSpaceGraph(context, options?)` | Assemble (and optionally filter) a graph from a context you already hold — reuses the context instead of rebuilding it. | | `buildSpaceGraph(nodes, hierarchy?)` | Build a navigable `SpaceGraph` directly from an array of `SpaceNode`s and a `Hierarchy` definition. | -| `validateSpace(context, options?)` | Run full validation (schema, references, rules, duplicates) over a space. | -| `validateFile(filePath, config)` | Validate a single file within its space. | +| `validateSpace(context, options?)` | Run full validation (schema, references, rules, duplicates) over a space. Options: `depth`, `includeGated`, `semantic`, `judgeProvider`, `ruleFilter`. | +| `validateFile(filePath, config, options?)` | Validate a single file within its space. Options: `includeGated`, `semantic`, `judgeProvider`, `ruleFilter`. | | `formatWikilink(target, alias?)` | Format a target into an Obsidian wikilink (`[[target]]` or `[[target|alias]]`). | | `parseWikilink(link)` | Parse a wikilink or target string into `{ target, anchor?, displayText? }`. | | `wikilinkToTarget(link)` | Extract the plain target lookup key from a wikilink string. | @@ -27,6 +27,20 @@ see [architecture.md](architecture.md#information-flow) for that flow. `ValidationResult.ruleViolations` carries each violation's effective `severity` (`error`, `warning` or `info`). Only `error` severity violations should be treated as failures; `validateFile` reports `warning` and `info` rule violations under `warnings` rather than `errors`. +## Semantic judging + +Semantic rules are evaluated only when `semantic: true` is passed (or configured; see [config.md](config.md#judge-semantic-rules)). The judge defaults to the configured provider; pass `judgeProvider` to inject your own. + +| Export | Purpose | +| --- | --- | +| `createJudgeProvider(config?, env?, custom?)` | Resolve a `SemanticJudgeProvider` from `JudgeConfig` and the environment. | +| `AnthropicJudgeProvider`, `OpenAIJudgeProvider`, `JevJudgeProvider` | Built-in providers. | +| `PerRuleJudgeProvider` | Base class for providers that can only answer one rule per request; implement `judgeRule`. | +| `FakeJudgeProvider` | Deterministic provider for tests. | +| `parseRuleFilter(specs)`, `filterRules(rules, filter)` | Build and apply a `RuleFilter` (`mode`, `category`, `severity`, `type`, `id`). | + +A `SemanticJudgeProvider` implements `evaluate({ node, rules })` and returns a result per rule id: a `SemanticJudgeVerdict` (`verdict` from the rule's closed set, optional `confidence`, `probabilities`, `evidence`) or `{ error }`. All of a node's semantic rules arrive in one call, so providers that support it can batch. `ValidationResult` adds `semanticEvaluations` and `gatedRules`. + ## Testing utilities Test fixture builders are available via `structured-context/testing` for plugin and integration tests: diff --git a/docs/concepts.md b/docs/concepts.md index 3eaaec3..cb943fa 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -151,9 +151,15 @@ Each executable rule has a **category** (`validation`, `coherence`, `workflow`, See [docs/rules.md](rules.md) for the rules reference, including JSONata expression syntax and the full `$metadata` field reference. +### Semantic rules + +A **semantic rule** (`mode: semantic`) checks meaning that an expression cannot, such as whether an opportunity is framed in the customer's voice. Instead of a `check` expression it carries a `rubric` and a closed set of **verdicts**. + +A **judge** is the model that evaluates semantic rules: a chat LLM (Anthropic, OpenAI-compatible) or a decision model (Jev). It receives a node and its rules and must answer each rule with exactly one **verdict** from that rule's `verdicts`; the rule's `failOn` verdicts are violations and all others pass. A judge may also report a confidence and probabilities. Semantic rules are opt-in (`--semantic`) because they cost money and time, and they run inside [cascade gating](#cascade-gating). See [docs/rules.md](rules.md#semantic-rules) for the rule fields and [docs/config.md](config.md#judge-semantic-rules) for judge configuration. + ### Cascade gating -**Cascade gating** is the mechanism by which a schema failure, hierarchy structure violation, or blocking rule violation on a node suppresses rule evaluation for that node's `hierarchy` subtree. It exists so downstream checks — especially future evaluative or LLM-judged layers — aren't run against inputs already known to be invalid. +**Cascade gating** is the mechanism by which a schema failure, hierarchy structure violation, or blocking rule violation on a node suppresses rule evaluation for that node's `hierarchy` subtree. It exists so downstream checks — especially costly semantic (model-judged) rules — aren't run against inputs already known to be invalid. A **coverage gate** records one instance of this: the gating node, what kind of failure triggered it (`schema`, `hierarchy`, or a specific rule `id`), and the set of nodes whose rule checks were suppressed as a result. `sctx validate` reports coverage gates so both humans and agents can see the validation "edge" — where evaluation stopped and why. See [docs/rules.md](rules.md#cascade-gating) for the full model, including the per-rule `gatesDescendants` field and the `--include-gated` / `--depth` CLI options. diff --git a/docs/config.md b/docs/config.md index 664dd9d..f484ab8 100644 --- a/docs/config.md +++ b/docs/config.md @@ -65,6 +65,29 @@ All plugin names must start with `sctx-` (the prefix is optional in config and n Plugins can also contribute schemas and partials via a `/schemas/` directory (or `"sctx": { "schemas": "..." }` in `package.json`). If a space specifies a plugin-prefixed schema (e.g. `"sctx-wardley-mapping/wardley_map.json"`), schemas are resolved statically without executing plugin code, and the plugin's code hooks are loaded when parsing or rendering. +## Judge (semantic rules) + +`judge` configures the model that evaluates `mode: semantic` [rules](rules.md#semantic-rules). Set it at the top level of the config (all spaces) or on a space (the space wins). `semantic` turns semantic evaluation on by default, at the same two levels; otherwise it runs only with `--semantic`. + +```json +{ + "judge": { "provider": "anthropic", "model": "claude-haiku-5-5" }, + "spaces": [ + { "name": "ProductX", "path": "/path/to/space", "semantic": true, "judge": { "provider": "jev" } } + ] +} +``` + +| Field | Description | +|---|---| +| `provider` | `anthropic`, `openai` (any OpenAI-compatible endpoint: OpenAI, Ollama, vLLM, OpenRouter chat models), `jev` (TypeSafe decision model via OpenRouter's decisions API), or `fake` (tests). If omitted, inferred from `ANTHROPIC_API_KEY`, then `OPENAI_API_KEY` / `OPENAI_BASE_URL`. `jev` is never inferred. | +| `model` | Defaults: `claude-haiku-5-5` (anthropic), `gpt-4o-mini` (openai), `typesafe/jev-1.13` (jev). | +| `endpoint` | Override the API URL. Jev defaults to `https://openrouter.ai/api/alpha/decisions`. | +| `apiKey` | Prefer environment variables over storing keys in config: `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `OPENROUTER_API_KEY` (jev). | +| `autoRun` | Evaluate semantic rules without `--semantic`. | + +`SCTX_JUDGE_PROVIDER`, `SCTX_JUDGE_MODEL`, `SCTX_JUDGE_ENDPOINT` and `SCTX_JUDGE_API_KEY` override the corresponding config fields. Semantic evaluation sends node content to the configured provider. + ## Markdown plugin config Set under `plugins.markdown` per space. diff --git a/docs/rules.md b/docs/rules.md index f2a3e63..9a69200 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -222,19 +222,7 @@ The judge must pick exactly one verdict from `verdicts` (default `pass`/`fail`); ### Judge configuration -Set `judge` at the root of `config.json` or per space (the space wins): - -```json -{ "judge": { "provider": "anthropic", "model": "claude-haiku-5-5" } } -``` - -| Field | Description | -|---|---| -| `provider` | `anthropic`, `openai` (any OpenAI-compatible endpoint, including Ollama and OpenRouter chat models), `jev`, or `fake` (tests). Omitted: inferred from `ANTHROPIC_API_KEY`, then `OPENAI_API_KEY`/`OPENAI_BASE_URL`. `jev` is never inferred. | -| `model` | Defaults: `claude-haiku-5-5` (anthropic), `gpt-4o-mini` (openai), `typesafe/jev-1.13` (jev). | -| `endpoint` | Override the API URL. Jev defaults to OpenRouter's `https://openrouter.ai/api/alpha/decisions`. | -| `apiKey` | Prefer environment variables: `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `OPENROUTER_API_KEY` (jev). `SCTX_JUDGE_*` variables override provider, model, endpoint and key. | -| `autoRun` | Run semantic rules without `--semantic`. | +The `judge` provider, model, endpoint and key are set in `config.json`; see [Judge configuration](config.md#judge-semantic-rules) for the fields, defaults and environment variables. Semantic evaluation is off unless `--semantic` is passed or `semantic` / `judge.autoRun` is set. ### Filtering rules diff --git a/src/api.ts b/src/api.ts index 1deedf9..5f5d501 100644 --- a/src/api.ts +++ b/src/api.ts @@ -28,13 +28,17 @@ export type { SemanticEvaluationRecord, SemanticJudgeInput, SemanticJudgeProvider, + SemanticJudgeResult, SemanticJudgeVerdict, + SemanticRuleSpec, } from './judge'; export { AnthropicJudgeProvider, createJudgeProvider, FakeJudgeProvider, + JevJudgeProvider, OpenAIJudgeProvider, + PerRuleJudgeProvider, } from './judge'; export { assembleSpaceGraph, loadSpaceGraph } from './load-space-graph'; export type { @@ -66,6 +70,8 @@ export { wikilinkToTarget, } from './read/wikilink-utils'; export type { SharedEmbeddingFields } from './schema/metadata-contract'; +export type { RuleFilter } from './schema/rule-filter'; +export { filterRules, parseRuleFilter } from './schema/rule-filter'; export { bundledSchemasDir, loadSchema, type PluginSchemaSource, setBundledSchemasDir } from './schema/schema'; export { createSpaceContext, SpaceNotFoundError } from './space-context'; export { buildSpaceGraph, type SpaceGraph } from './space-graph'; diff --git a/tests/config.test.ts b/tests/config.test.ts index b2eb463..f9e2d33 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -386,4 +386,22 @@ describe('loadConfig with includeSpacesFrom', () => { const mainConfig = JSON5.parse(readFileSync(mainConfigPath, 'utf-8')) as Config; expect(mainConfig.spaces[0]!.name).toBe('main-space'); }); + + it('accepts judge config with the jev provider at root and per space', () => { + const configPath = join(testDir, 'judge-config.json'); + writeFileSync( + configPath, + JSON.stringify({ + judge: { provider: 'jev' }, + semantic: true, + spaces: [ + { name: 's', path: '/p', judge: { provider: 'anthropic', model: 'claude-haiku-5-5' }, semantic: false }, + ], + }), + ); + setConfigPath(configPath); + const config = loadConfig(); + expect(config.judge?.provider).toBe('jev'); + expect(config.spaces?.[0]?.judge?.model).toBe('claude-haiku-5-5'); + }); }); From 1c6141f869ebf505f22c460aa5f3602e5a4af502 Mon Sep 17 00:00:00 2001 From: Roger Barnes Date: Tue, 13 Oct 2026 22:53:48 +1100 Subject: [PATCH 04/22] docs: point schema docs and plugin authoring reference at semantic rules --- docs/schemas.md | 2 +- plugin/.claude-plugin/plugin.json | 2 +- plugin/skills/structured-context/references/schema-authoring.md | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/schemas.md b/docs/schemas.md index b4ebc05..3fce5d6 100644 --- a/docs/schemas.md +++ b/docs/schemas.md @@ -159,7 +159,7 @@ Top-level metadata shape: | `hierarchy.allowSkipLevels` | `boolean` | Optional; allows parent to be any ancestor level | | `relationships` | `Relationship[]` | Optional; defines related node links outside the primary hierarchy. A relationship may set `override: true` | | `aliases` | `Record` | Optional type alias map. Use `{ "type": "goal", "override": true }` to replace an imported alias | -| `rules` | `Rule[]` | Optional flat rule array | +| `rules` | `Rule[]` | Optional flat rule array. Rules are `expression` (JSONata `check`, the default) or `semantic` (judged against a rubric); see [Rules](rules.md) | ### Relationships diff --git a/plugin/.claude-plugin/plugin.json b/plugin/.claude-plugin/plugin.json index c1f1c21..471e605 100644 --- a/plugin/.claude-plugin/plugin.json +++ b/plugin/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "structured-context", - "version": "0.12.7", + "version": "0.12.8", "description": "Structured content authoring for Obsidian — automatic validation hooks, slash commands, and agent skills to keep markdown files compliant with your schema.", "homepage": "https://github.com/mindsocket/structured-context", "repository": "https://github.com/mindsocket/structured-context", diff --git a/plugin/skills/structured-context/references/schema-authoring.md b/plugin/skills/structured-context/references/schema-authoring.md index 2909ea7..0207e84 100644 --- a/plugin/skills/structured-context/references/schema-authoring.md +++ b/plugin/skills/structured-context/references/schema-authoring.md @@ -66,6 +66,8 @@ Relationships define how sub-entities (nodes inside other files) are parsed and Rules are a flat array. Each rule's `category` (`validation`, `coherence`, `workflow`, `best-practice`) supplies its default severity: `validation` → `error`, `coherence` and `workflow` → `warning`, `best-practice` → `info`. Only `error` violations fail validation; warnings and info are reported only. An optional per-rule `severity` (`error` | `warning` | `info`) overrides the category default — prefer picking the right category over overriding. +Rules default to `mode: "expression"` (a JSONata `check`). Set `mode: "semantic"` for rules an LLM or decision model judges against a `rubric` (falls back to `description`): they take no `check` or `scope`, may declare `verdicts` (verdict → description, default `pass`/`fail`) with a required `failOn` list, and only run with `--semantic`. Run `sctx docs rules` for the full semantic rule reference. An optional `gatesDescendants` controls whether a violation suppresses rule checks on the node's subtree (default: true for `error` severity). + ## Metadata composition semantics `$ref`/`$defs` are validation-only: a `$ref` brings no metadata. Metadata travels only via `$metadata.imports` (schema `$id`s): From f12511f05fafd850fc8510dce2caad2f5dd87269 Mon Sep 17 00:00:00 2001 From: Roger Barnes Date: Wed, 14 Oct 2026 10:01:57 +1100 Subject: [PATCH 05/22] feat(validate-file): judge only the target and its gate-capable ancestor rules --- docs/rules.md | 2 +- src/schema/gating.ts | 32 +++++++- src/schema/validate-rules.ts | 3 + src/validate.ts | 9 ++- tests/validate/semantic-validation.test.ts | 89 +++++++++++++++++++++- 5 files changed, 131 insertions(+), 4 deletions(-) diff --git a/docs/rules.md b/docs/rules.md index 9a69200..3844582 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -183,7 +183,7 @@ don't depend on an ancestor being valid. ### Single-file validation -`sctx validate-file ` applies the same gating as `sctx validate`, so a file's rule results match its entry in a full run. Rules are evaluated for the whole space and the output is restricted to the file. If the file is gated, its rule checks are skipped and a `gated:` entry (kind `gated`, severity `info`) under `warnings` names the blocker, so a gated file doesn't read as clean. `--include-gated` evaluates the file's rules anyway. `--depth` is not supported. +`sctx validate-file ` applies the same gating as `sctx validate`, so a file's rule results match its entry in a full run. Expression rules are evaluated for the whole space and the output is restricted to the file. The judge is not: it is consulted only for the file's own semantic rules and for its hierarchy ancestors' semantic rules that can gate (top-down, so a failing ancestor stops the walk), so one file costs at most one call per ancestor level plus one for the file. If the file is gated, its rule checks are skipped and a `gated:` entry (kind `gated`, severity `info`) under `warnings` names the blocker, so a gated file doesn't read as clean. `--include-gated` evaluates the file's rules anyway. `--depth` is not supported. ### Depth-limited evaluation diff --git a/src/schema/gating.ts b/src/schema/gating.ts index 576fcec..fd7a0f5 100644 --- a/src/schema/gating.ts +++ b/src/schema/gating.ts @@ -54,6 +54,12 @@ export interface GatingOptions { includeGated?: boolean; /** Judge for `mode: semantic` rules. Without it, semantic rules are not evaluated. */ judge?: SemanticJudgeProvider; + /** + * Label of the one node whose result is wanted (`validate-file`). Gating is still computed over the + * whole space, but the judge is only consulted for that node and for its hierarchy ancestors' gate-capable + * semantic rules (top-down, so a gated ancestor stops the walk). Other nodes' semantic rules are not judged. + */ + judgeTarget?: string; } /** @@ -149,8 +155,32 @@ export async function evaluateRulesWithGating( const perNodeRules = rules.filter((r) => r.scope !== 'global' && (r.mode !== 'semantic' || judge)); + // With a judge target, only the target and its hierarchy ancestors can need judging. + const targetAncestors = new Set(); + if (options.judgeTarget !== undefined) { + const parentsOf = new Map(); + for (const [parent, children] of graph.hierarchyChildren) { + for (const child of children) parentsOf.set(child.title, [...(parentsOf.get(child.title) ?? []), parent]); + } + const target = nodes.find((n) => n.label === options.judgeTarget); + const pending = target ? [target.title] : []; + while (pending.length > 0) { + for (const parent of parentsOf.get(pending.pop()!) ?? []) { + if (!targetAncestors.has(parent)) { + targetAncestors.add(parent); + pending.push(parent); + } + } + } + } + function applicableRules(node: SpaceNode): Rule[] { - return perNodeRules.filter((r) => !r.type || r.type === node.resolvedType); + return perNodeRules.filter((r) => { + if (r.type && r.type !== node.resolvedType) return false; + if (r.mode !== 'semantic' || options.judgeTarget === undefined || node.label === options.judgeTarget) return true; + // Ancestors only matter through the gates they could raise. + return targetAncestors.has(node.title) && ruleGatesDescendants(r); + }); } async function evaluateApplicableRules(node: SpaceNode): Promise<{ blockingRuleId?: string }> { diff --git a/src/schema/validate-rules.ts b/src/schema/validate-rules.ts index 03b8d35..cf504dc 100644 --- a/src/schema/validate-rules.ts +++ b/src/schema/validate-rules.ts @@ -21,6 +21,8 @@ export interface ValidateRulesOptions { semantic?: boolean; /** The semantic judge provider to use if semantic is true. */ judgeProvider?: SemanticJudgeProvider; + /** Label of the only node to judge (`validate-file`); other nodes' semantic rules are skipped. */ + judgeTarget?: string; /** Set of node labels that have already failed upstream structural checks. */ gatedNodeLabels?: Set; } @@ -121,6 +123,7 @@ export async function validateRules( } for (const node of nodes) { + if (options.judgeTarget !== undefined && node.label !== options.judgeTarget) continue; const applicable = semanticRules.filter((r) => !r.type || r.type === node.resolvedType); if (applicable.length === 0) continue; diff --git a/src/validate.ts b/src/validate.ts index 4ac28d3..803a64c 100644 --- a/src/validate.ts +++ b/src/validate.ts @@ -213,7 +213,12 @@ async function evaluateSpaceRules( nodes: SpaceNode[], metadata: SchemaMetadata, seeds: GatingSeed[], - options: { includeGated?: boolean; judge?: SemanticJudgeProvider; ruleFilter?: RuleFilter } = {}, + options: { + includeGated?: boolean; + judge?: SemanticJudgeProvider; + judgeTarget?: string; + ruleFilter?: RuleFilter; + } = {}, ): Promise { const rules = metadata.rules && filterRules(metadata.rules, options.ruleFilter); if (!rules) return { violations: [], gates: [], semanticEvaluations: [], gatedRules: [] }; @@ -221,6 +226,7 @@ async function evaluateSpaceRules( const result = await validateRules(nodes, rules, { semantic: options.judge !== undefined, judgeProvider: options.judge, + judgeTarget: options.judgeTarget, }); return { violations: result, @@ -540,6 +546,7 @@ export async function validateFile( includeGated: options.includeGated, ruleFilter: options.ruleFilter, judge, + judgeTarget: label, }); for (const v of violations) { if (v.file === label) { diff --git a/tests/validate/semantic-validation.test.ts b/tests/validate/semantic-validation.test.ts index 3c75dee..2913cb7 100644 --- a/tests/validate/semantic-validation.test.ts +++ b/tests/validate/semantic-validation.test.ts @@ -1,4 +1,6 @@ -import { describe, expect, it } from 'bun:test'; +import { afterAll, beforeAll, describe, expect, it } from 'bun:test'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; import { join } from 'node:path'; import type { Config } from '../../src/config'; import { FakeJudgeProvider } from '../../src/judge/fake'; @@ -141,6 +143,91 @@ describe('Semantic rule evaluation integration (validateSpace & validateFile)', }); }); +describe('validateFile judges only the target and its gate-capable ancestor rules', () => { + let dir: string; + let config: Config; + const typeSchema = (type: string, parent?: string) => ({ + type: 'object', + properties: { type: { const: type }, title: { type: 'string' }, ...(parent && { parent: { type: 'string' } }) }, + required: ['type', 'title'], + }); + + beforeAll(() => { + dir = mkdtempSync(join(tmpdir(), 'sctx-judge-target-')); + const schemaPath = join(dir, 'schema.json'); + const rule = (id: string, type: string, severity: string) => ({ + id, + type, + mode: 'semantic', + category: 'validation', + severity, + description: id, + }); + writeFileSync( + schemaPath, + JSON.stringify({ + $schema: + 'https://raw.githubusercontent.com/mindsocket/structured-context/main/schemas/generated/latest/_structured_context_schema_meta.json', + $id: 'sctx://judge-target.test', + $metadata: { + hierarchy: { levels: ['goal', 'task', 'step'] }, + rules: [ + rule('goal-sound', 'goal', 'error'), // gates descendants + rule('task-sound', 'task', 'warning'), // does not gate + rule('step-sound', 'step', 'warning'), + ], + }, + oneOf: [typeSchema('goal'), typeSchema('task', 'p'), typeSchema('step', 'p')], + }), + ); + const space = join(dir, 'space'); + mkdirSync(space); + writeFileSync(join(space, 'G.md'), '---\ntype: goal\n---\n'); + writeFileSync(join(space, 'T.md'), '---\ntype: task\nparent: "[[G]]"\n---\n'); + writeFileSync(join(space, 'S.md'), '---\ntype: step\nparent: "[[T]]"\n---\n'); + writeFileSync(join(space, 'Other.md'), '---\ntype: goal\n---\n'); + config = { spaces: [{ name: 'jt', path: space, schema: schemaPath }] }; + }); + + afterAll(() => rmSync(dir, { recursive: true, force: true })); + + const run = (judge: FakeJudgeProvider, file: string, extra = {}) => + validateFile(join(dir, 'space', file), config, { + semantic: true, + judgeProvider: judge, + configDir: dir, + ...extra, + }) as Promise; + + it('judges the target plus only the gate-capable rules of its ancestors', async () => { + const judge = new FakeJudgeProvider({ pass: true }); + await run(judge, 'S.md'); + const judged = judge.callHistory.map((c) => `${c.node.label}:${c.rules.map((r) => r.id).join(',')}`).sort(); + // G's rule can gate; T's cannot and Other is unrelated. + expect(judged).toEqual(['G.md:goal-sound', 'S.md:step-sound']); + }); + + it('still lets a failing ancestor rule gate the target', async () => { + const judge = new FakeJudgeProvider({ 'goal-sound': { verdict: 'fail' } }); + const result = await run(judge, 'S.md'); + expect(result.warnings['rule:step-sound']).toBeUndefined(); + expect(Object.keys(result.warnings).some((k) => k.startsWith('gated:'))).toBe(true); + expect(judge.callHistory.some((c) => c.node.label === 'S.md')).toBe(false); + }); + + it('with includeGated judges the target even when an ancestor fails', async () => { + const judge = new FakeJudgeProvider({ 'goal-sound': { verdict: 'fail' }, 'step-sound': { verdict: 'fail' } }); + const result = await run(judge, 'S.md', { includeGated: true }); + expect(result.warnings['rule:step-sound']).toBeDefined(); + }); + + it('judges nothing when the filter excludes every semantic rule', async () => { + const judge = new FakeJudgeProvider({ pass: true }); + await run(judge, 'S.md', { ruleFilter: { id: ['no-such-rule'] } }); + expect(judge.callHistory).toHaveLength(0); + }); +}); + describe('Semantic rules: verdicts, batching and rule filter', () => { const config: Config = { spaces: [{ name: 'strict-valid', path: VALID_DIR, schema: STRICT_SCHEMA_PATH }], From a96732ba81b0110180e8bbbdf94fb1e770c9b589 Mon Sep 17 00:00:00 2001 From: Roger Barnes Date: Wed, 14 Oct 2026 10:32:21 +1100 Subject: [PATCH 06/22] refactor(judge): require an explicit provider, move providers to judge/provider, split semantic docs, trim test fixtures --- AGENTS.md | 2 +- README.md | 2 +- docs/api.md | 2 +- docs/concepts.md | 2 +- docs/config.md | 6 +- docs/rules.md | 40 +------- docs/semantic-rules.md | 78 +++++++++++++++ plugin/.claude-plugin/plugin.json | 2 +- plugin/skills/structured-context/SKILL.md | 2 +- .../references/schema-authoring.md | 2 +- src/commands/docs.ts | 1 + src/index.ts | 2 +- src/judge/factory.ts | 96 +++++++++---------- src/judge/index.ts | 18 ++-- src/judge/{ => provider}/anthropic.ts | 4 +- src/judge/{ => provider}/fake.ts | 2 +- src/judge/{ => provider}/jev.ts | 2 +- src/judge/{ => provider}/openai.ts | 4 +- src/judge/{ => provider}/per-rule.ts | 2 +- .../semantic-rule-contract/no-check.json | 6 -- .../semantic-check.json | 8 -- .../semantic-global.json | 8 -- .../semantic-rule-contract/semantic-ok.json | 6 -- .../Progressive sign-up form.md | 7 -- .../valid-directory/Social login option.md | 7 -- tests/judge/factory.test.ts | 55 ++++++----- .../fake.test.ts} | 6 +- tests/judge/{ => provider}/jev.test.ts | 6 +- tests/judge/verdicts.test.ts | 2 +- tests/schema/semantic-rule-contract.test.ts | 37 +++++-- tests/schema/validate-rules-semantic.test.ts | 2 +- tests/validate/gating.test.ts | 5 +- tests/validate/semantic-validation.test.ts | 2 +- tests/validate/strict.test.ts | 4 +- 34 files changed, 234 insertions(+), 196 deletions(-) create mode 100644 docs/semantic-rules.md rename src/judge/{ => provider}/anthropic.ts (97%) rename src/judge/{ => provider}/fake.ts (98%) rename src/judge/{ => provider}/jev.ts (99%) rename src/judge/{ => provider}/openai.ts (97%) rename src/judge/{ => provider}/per-rule.ts (94%) delete mode 100644 tests/fixtures/semantic-rule-contract/no-check.json delete mode 100644 tests/fixtures/semantic-rule-contract/semantic-check.json delete mode 100644 tests/fixtures/semantic-rule-contract/semantic-global.json delete mode 100644 tests/fixtures/semantic-rule-contract/semantic-ok.json delete mode 100644 tests/fixtures/strict_ost/valid-directory/Progressive sign-up form.md delete mode 100644 tests/fixtures/strict_ost/valid-directory/Social login option.md rename tests/judge/{fake-provider.test.ts => provider/fake.test.ts} (90%) rename tests/judge/{ => provider}/jev.test.ts (94%) diff --git a/AGENTS.md b/AGENTS.md index 6512600..66978ea 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -46,7 +46,7 @@ There are several places that need reviewing and updating with any new feature o - `src/schema/metadata-contract.ts` — Single source of truth for the `$metadata` contract - `schemas/generated/latest/_structured_context_schema_meta.json` — Generated metaschema for the newest dialect (generated on build or with `bun run generate:schema-meta`). `schemas/generated//` folders are frozen copies, created by the `version` script (`generate:schema-meta --release`) only when a release changes the dialect; never edit them. `src/schema/meta-schemas.ts` loads them at runtime. - `src/schema/gating.ts` — Cascade gating: computes hierarchy depth and evaluates rules with subtree gating (schema/hierarchy/blocking-rule failures suppress downstream rule checks). See [docs/rules.md](docs/rules.md#cascade-gating). -- `src/judge/` — Semantic rule judges: provider interface and types (`types.ts`), verdict resolution (`verdicts.ts`), per-node dispatch (`judge-node.ts`), and providers (`anthropic.ts`, `openai.ts`, `jev.ts`, `fake.ts`). See [docs/rules.md](docs/rules.md#semantic-rules). +- `src/judge/` — Semantic rule judges: provider interface and types (`types.ts`), verdict resolution (`verdicts.ts`), per-node dispatch (`judge-node.ts`), and providers (`provider/anthropic.ts`, `provider/openai.ts`, `provider/jev.ts`, `provider/fake.ts`). See [docs/semantic-rules.md](docs/semantic-rules.md). - `src/schema/rule-filter.ts` — `--rule-filter` parsing and rule filtering by mode, category, severity, type or id. ## Testing diff --git a/README.md b/README.md index 927834d..7a9e69a 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,7 @@ Validates markdown files against the JSON schema: - Runs executable rules (`$metadata.rules`) with **cascade gating**: a schema, hierarchy, or blocking-rule failure on a node suppresses rule evaluation for that node's hierarchy subtree. Gated checks are reported in a Coverage section so you know what wasn't evaluated and why. See [docs/rules.md](docs/rules.md#cascade-gating) for details. - `--depth ` limits evaluation to hierarchy nodes within `n` breadth-first levels from the top — useful for staged agentic fix loops (fix the top level, then request the next). - `--include-gated` evaluates rules even on gated nodes (still reported as gated) — for surfacing cheap bottom-up checks on request. -- `--semantic` also evaluates `mode: semantic` rules with a model judge (Anthropic, OpenAI-compatible, or Jev); `--rule-filter` narrows which rules run. See [docs/rules.md](docs/rules.md#semantic-rules). +- `--semantic` also evaluates `mode: semantic` rules with a model judge (Anthropic, OpenAI-compatible, or Jev); `--rule-filter` narrows which rules run. See [docs/semantic-rules.md](docs/semantic-rules.md) (`sctx docs semantic`). ### Render space diff --git a/docs/api.md b/docs/api.md index 80b66d1..58d642c 100644 --- a/docs/api.md +++ b/docs/api.md @@ -29,7 +29,7 @@ see [architecture.md](architecture.md#information-flow) for that flow. ## Semantic judging -Semantic rules are evaluated only when `semantic: true` is passed (or configured; see [config.md](config.md#judge-semantic-rules)). The judge defaults to the configured provider; pass `judgeProvider` to inject your own. +Semantic rules are evaluated only when `semantic: true` is passed (or configured; see [config.md](config.md#judge-semantic-rules)); the provider must be named explicitly, never inferred from the environment. The judge defaults to the configured provider; pass `judgeProvider` to inject your own. | Export | Purpose | | --- | --- | diff --git a/docs/concepts.md b/docs/concepts.md index cb943fa..80e85e6 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -155,7 +155,7 @@ See [docs/rules.md](rules.md) for the rules reference, including JSONata express A **semantic rule** (`mode: semantic`) checks meaning that an expression cannot, such as whether an opportunity is framed in the customer's voice. Instead of a `check` expression it carries a `rubric` and a closed set of **verdicts**. -A **judge** is the model that evaluates semantic rules: a chat LLM (Anthropic, OpenAI-compatible) or a decision model (Jev). It receives a node and its rules and must answer each rule with exactly one **verdict** from that rule's `verdicts`; the rule's `failOn` verdicts are violations and all others pass. A judge may also report a confidence and probabilities. Semantic rules are opt-in (`--semantic`) because they cost money and time, and they run inside [cascade gating](#cascade-gating). See [docs/rules.md](rules.md#semantic-rules) for the rule fields and [docs/config.md](config.md#judge-semantic-rules) for judge configuration. +A **judge** is the model that evaluates semantic rules: a chat LLM (Anthropic, OpenAI-compatible) or a decision model (Jev). It receives a node and its rules and must answer each rule with exactly one **verdict** from that rule's `verdicts`; the rule's `failOn` verdicts are violations and all others pass. A judge may also report a confidence and probabilities. Semantic rules are opt-in (`--semantic`) because they cost money and time, and they run inside [cascade gating](#cascade-gating). See [docs/semantic-rules.md](semantic-rules.md) for the rule fields and [docs/config.md](config.md#judge-semantic-rules) for judge configuration. ### Cascade gating diff --git a/docs/config.md b/docs/config.md index f484ab8..fffeb7d 100644 --- a/docs/config.md +++ b/docs/config.md @@ -67,7 +67,7 @@ Plugins can also contribute schemas and partials via a `/schemas/` d ## Judge (semantic rules) -`judge` configures the model that evaluates `mode: semantic` [rules](rules.md#semantic-rules). Set it at the top level of the config (all spaces) or on a space (the space wins). `semantic` turns semantic evaluation on by default, at the same two levels; otherwise it runs only with `--semantic`. +`judge` configures the model that evaluates `mode: semantic` [rules](semantic-rules.md). Set it at the top level of the config (all spaces) or on a space (the space wins). `semantic` turns semantic evaluation on by default, at the same two levels; otherwise it runs only with `--semantic`. ```json { @@ -80,10 +80,10 @@ Plugins can also contribute schemas and partials via a `/schemas/` d | Field | Description | |---|---| -| `provider` | `anthropic`, `openai` (any OpenAI-compatible endpoint: OpenAI, Ollama, vLLM, OpenRouter chat models), `jev` (TypeSafe decision model via OpenRouter's decisions API), or `fake` (tests). If omitted, inferred from `ANTHROPIC_API_KEY`, then `OPENAI_API_KEY` / `OPENAI_BASE_URL`. `jev` is never inferred. | +| `provider` | **Required** to judge (or set `SCTX_JUDGE_PROVIDER`). One of `anthropic`, `openai` (any OpenAI-compatible endpoint: OpenAI, Ollama, vLLM, OpenRouter chat models), `jev` (TypeSafe decision model via OpenRouter's decisions API), or `fake` (tests). Never inferred from API keys in the environment. | | `model` | Defaults: `claude-haiku-5-5` (anthropic), `gpt-4o-mini` (openai), `typesafe/jev-1.13` (jev). | | `endpoint` | Override the API URL. Jev defaults to `https://openrouter.ai/api/alpha/decisions`. | -| `apiKey` | Prefer environment variables over storing keys in config: `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `OPENROUTER_API_KEY` (jev). | +| `apiKey` | Prefer environment variables over storing keys in config. Once the provider is chosen, its key is read from `ANTHROPIC_API_KEY`, `OPENAI_API_KEY` or `OPENROUTER_API_KEY` (jev). | | `autoRun` | Evaluate semantic rules without `--semantic`. | `SCTX_JUDGE_PROVIDER`, `SCTX_JUDGE_MODEL`, `SCTX_JUDGE_ENDPOINT` and `SCTX_JUDGE_API_KEY` override the corresponding config fields. Semantic evaluation sends node content to the configured provider. diff --git a/docs/rules.md b/docs/rules.md index 3844582..2cf1c19 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -31,7 +31,7 @@ Rules are a flat array. | `severity` | no | `error` \| `warning` \| `info`; overrides the category default (see [Severity](#severity)) | | `description` | yes | Human-readable rule intent | | `check` | yes, unless `mode: semantic` | JSONata expression; must evaluate to `true` | -| `mode` | no | `expression` (default) or `semantic` (judged by a model; see [Semantic rules](#semantic-rules)) | +| `mode` | no | `expression` (default) or `semantic` (judged by a model; see [Semantic rules](semantic-rules.md)) | | `rubric` | semantic only | Detailed prompt for the judge; falls back to `description` | | `verdicts` | semantic only | Closed set of verdicts the judge must choose from: `{ "": "" }`. Default `pass` / `fail`. | | `failOn` | semantic only | Verdicts that count as a violation. Default `["fail"]`; required when `verdicts` is declared. At least one verdict must pass. | @@ -164,7 +164,7 @@ original blocker), rather than each generating a new one. Non-hierarchy nodes an `scope: "global"` rules are never gated — cascade gating only applies within the hierarchy DAG. -This exists so that expensive or semantic downstream checks (see [Semantic rules](#semantic-rules)) aren't wasted +This exists so that expensive or semantic downstream checks (see [Semantic rules](semantic-rules.md)) aren't wasted evaluating nodes whose ancestors are already known-bad, and so noise doesn't drown out the actionable failure. Set `gatesDescendants: false` on an error-severity rule whose violations shouldn't suppress checks further down the tree, or `gatesDescendants: true` on a warning/info rule that should. @@ -183,7 +183,7 @@ don't depend on an ancestor being valid. ### Single-file validation -`sctx validate-file ` applies the same gating as `sctx validate`, so a file's rule results match its entry in a full run. Expression rules are evaluated for the whole space and the output is restricted to the file. The judge is not: it is consulted only for the file's own semantic rules and for its hierarchy ancestors' semantic rules that can gate (top-down, so a failing ancestor stops the walk), so one file costs at most one call per ancestor level plus one for the file. If the file is gated, its rule checks are skipped and a `gated:` entry (kind `gated`, severity `info`) under `warnings` names the blocker, so a gated file doesn't read as clean. `--include-gated` evaluates the file's rules anyway. `--depth` is not supported. +`sctx validate-file ` applies the same gating as `sctx validate`, so a file's rule results match its entry in a full run. Expression rules are evaluated for the whole space and the output is restricted to the file. Semantic rules are judged only for the file and its gating ancestors (see [Semantic rules](semantic-rules.md#single-file-validation)). If the file is gated, its rule checks are skipped and a `gated:` entry (kind `gated`, severity `info`) under `warnings` names the blocker, so a gated file doesn't read as clean. `--include-gated` evaluates the file's rules anyway. `--depth` is not supported. ### Depth-limited evaluation @@ -194,39 +194,7 @@ the next). Excluded nodes are reported under `skippedByDepth`, separate from gat ## Semantic rules -A rule with `"mode": "semantic"` is judged by a model instead of a JSONata expression. Semantic rules cost money and time, so they never run unless asked: pass `--semantic` (or set `semantic: true` / `judge.autoRun` on a space or in config). - -```json -{ - "id": "opportunity-customer-voice", - "category": "best-practice", - "mode": "semantic", - "type": "opportunity", - "description": "Opportunity is framed in the customer's voice", - "rubric": "The opportunity reads as a customer need or pain point, not a solution in disguise or a business metric.", - "verdicts": { - "customer-need": "Clearly a customer need or pain point", - "vague": "A need, but too vague to act on", - "solution": "A solution or business goal in disguise" - }, - "failOn": ["solution"] -} -``` - -The judge must pick exactly one verdict from `verdicts` (default `pass`/`fail`); a verdict listed in `failOn` is a violation, everything else passes. A closed set maps directly onto decision models, which return a choice with confidence and probabilities rather than prose. - -- **Evaluation order.** For each node, deterministic rules run first, then all of the node's semantic rules go to the judge in one call. Providers that can only answer one question per request (chat LLMs) make one request per rule; decision models such as Jev answer all of a node's rules in a single request. -- **Gating.** Semantic rules follow [cascade gating](#cascade-gating): they are not run on gated nodes (recorded as `gatedRules`), and a failing semantic rule gates its subtree when `gatesDescendants` is true (default for `error` severity). A judge *error* (network, unparseable output) is reported as a violation but never gates. -- **Output.** `--json` includes `semanticEvaluations` (verdict, `pass`, confidence, evidence per node/rule) and `gatedRules`. Violations carry the judge's `evidence`; decision models without prose get a summary of the verdict distribution. -- **Scope.** Semantic rules judge a node in isolation, plus its parent titles. (Wider scopes such as ancestors or siblings are not yet supported.) - -### Judge configuration - -The `judge` provider, model, endpoint and key are set in `config.json`; see [Judge configuration](config.md#judge-semantic-rules) for the fields, defaults and environment variables. Semantic evaluation is off unless `--semantic` is passed or `semantic` / `judge.autoRun` is set. - -### Filtering rules - -`--rule-filter =[,]` (repeatable) evaluates only matching rules. Dimensions are `mode`, `category`, `severity` (effective severity), `type` and `id`; different dimensions combine with AND, values within one with OR. For example, `--semantic --rule-filter mode=semantic` runs only the judged rules, and `--rule-filter mode=expression` skips them entirely. Gating is computed from the rules that ran. +A rule with `"mode": "semantic"` is judged by a model (an LLM or a decision model) instead of a JSONata expression. They are opt-in (`--semantic`) because they cost money and time, and they run inside [cascade gating](#cascade-gating). See [Semantic rules](semantic-rules.md) (`sctx docs semantic`) for the rule fields, judge setup and filtering. ## Common patterns diff --git a/docs/semantic-rules.md b/docs/semantic-rules.md new file mode 100644 index 0000000..990b9e9 --- /dev/null +++ b/docs/semantic-rules.md @@ -0,0 +1,78 @@ +# Semantic Rules + +A semantic rule is a rule a model judges instead of a JSONata expression: "is this opportunity framed in the customer's voice?" rather than "does this field exist?". Semantic rules are **opt-in**. Nothing is sent to a model unless you pass `--semantic` (or enable it in config), and a space with no semantic rules is unaffected. + +For expression rules, severity, gating and merge behaviour, see [Rules](rules.md). + +## Quick start + +1. **Declare a rule** in the schema's `$metadata.rules`: + + ```json + { + "id": "opportunity-customer-voice", + "category": "best-practice", + "mode": "semantic", + "type": "opportunity", + "description": "Opportunity is framed in the customer's voice", + "rubric": "The opportunity reads as a customer need or pain point, not a solution in disguise or a business metric.", + "verdicts": { + "customer-need": "Clearly a customer need or pain point", + "vague": "A need, but too vague to act on", + "solution": "A solution or business goal in disguise" + }, + "failOn": ["solution"] + } + ``` + +2. **Choose a judge** in `config.json`. The provider is always explicit; keys in your environment are never picked up unless you name the provider: + + ```json + { "judge": { "provider": "anthropic" } } + ``` + + with `ANTHROPIC_API_KEY` set. See [Judge configuration](config.md#judge-semantic-rules) for every field. + +3. **Run it**: `sctx validate --semantic`. + +## Rule fields + +Semantic rules share `id`, `category`, `severity`, `type`, `description` and `gatesDescendants` with [expression rules](rules.md#rule-shape). They differ in these fields: + +| Field | Required | Description | +|---|---|---| +| `mode` | yes | `"semantic"` (the default is `"expression"`) | +| `rubric` | no | What the judge should assess. Falls back to `description` | +| `verdicts` | no | Map of verdict to description. At least two entries. Default: `pass` / `fail` | +| `failOn` | if `verdicts` is set | Verdicts that count as a violation. Default `["fail"]`. At least one verdict must pass | + +A semantic rule has no `check` and cannot be `scope: "global"`; it judges one node at a time. + +The judge must choose exactly one verdict. A closed set keeps results comparable between runs and maps directly onto decision models, which return a choice with confidence and probabilities rather than prose. + +## How judging works + +- **Order.** For each node, expression rules run first, then all of the node's semantic rules go to the judge in one call. Chat LLMs make one request per rule; decision models answer all of a node's rules in a single request. +- **Gating.** Semantic rules follow [cascade gating](rules.md#cascade-gating). They are not judged on gated nodes (recorded as `gatedRules`), and a failing semantic rule gates its subtree when `gatesDescendants` is true (the default for `error` severity). A judge *error* (network failure, unparseable output) is reported as a violation but never gates. +- **Input.** The judge sees the node's title, type, parent titles and its schema data (the fields the parser extracted for validation, including frontmatter). Wider scopes (ancestors, siblings) are not supported yet. +- **Output.** `--json` includes `semanticEvaluations` (verdict, `pass`, confidence and evidence per node and rule) and `gatedRules`. Violations carry the judge's `evidence`; decision models give a summary of the verdict distribution instead of prose. + +### Single-file validation + +`sctx validate-file --semantic` computes gating over the whole space but only calls the judge for the file itself and for its hierarchy ancestors' semantic rules that can gate (top-down, so a failing ancestor stops the walk). A file costs at most one call per ancestor level plus one for the file, however large the space. + +### Filtering rules + +`--rule-filter =[,]` (repeatable) evaluates only matching rules. Dimensions are `mode`, `category`, `severity` (effective severity), `type` and `id`; different dimensions combine with AND, values within one with OR. For example, `--semantic --rule-filter mode=semantic` runs only the judged rules, and `--rule-filter mode=expression` skips them entirely. Gating is computed from the rules that ran. + +## Choosing a judge + +| Provider | Use for | Notes | +|---|---|---| +| `anthropic` | Claude models | Default model `claude-haiku-5-5` | +| `openai` | Any OpenAI-compatible chat endpoint: OpenAI, Ollama, vLLM, Groq, OpenRouter chat models | Set `endpoint` for non-OpenAI servers; the key is optional for local ones | +| `jev` | TypeSafe's Jev decision model | Closed-set answers with calibrated probabilities, one request per node. Uses an OpenRouter key | + +Prefer a decision model when you can: results are repeatable, cheap, and come with a confidence. Use a chat LLM when the rubric needs open reasoning or evidence text. + +Semantic evaluation sends node content to the configured provider. Check that is acceptable for the space before enabling it. diff --git a/plugin/.claude-plugin/plugin.json b/plugin/.claude-plugin/plugin.json index 471e605..e39d96c 100644 --- a/plugin/.claude-plugin/plugin.json +++ b/plugin/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "structured-context", - "version": "0.12.8", + "version": "0.12.9", "description": "Structured content authoring for Obsidian — automatic validation hooks, slash commands, and agent skills to keep markdown files compliant with your schema.", "homepage": "https://github.com/mindsocket/structured-context", "repository": "https://github.com/mindsocket/structured-context", diff --git a/plugin/skills/structured-context/SKILL.md b/plugin/skills/structured-context/SKILL.md index 1cb308d..3987348 100644 --- a/plugin/skills/structured-context/SKILL.md +++ b/plugin/skills/structured-context/SKILL.md @@ -134,4 +134,4 @@ sctx docs config - **`references/schema-authoring.md`** — schema file structure, `$metadata`, JSONata rules (run `sctx docs schema` for schema dialect reference) - **`references/schema-design.md`** — process for designing a schema from existing content -For CLI and config reference, use `sctx docs ` (topics: `concepts`, `config`, `schema`, `rules`). +For CLI and config reference, use `sctx docs ` (topics: `concepts`, `config`, `schema`, `rules`, `semantic`). diff --git a/plugin/skills/structured-context/references/schema-authoring.md b/plugin/skills/structured-context/references/schema-authoring.md index 0207e84..dc81393 100644 --- a/plugin/skills/structured-context/references/schema-authoring.md +++ b/plugin/skills/structured-context/references/schema-authoring.md @@ -66,7 +66,7 @@ Relationships define how sub-entities (nodes inside other files) are parsed and Rules are a flat array. Each rule's `category` (`validation`, `coherence`, `workflow`, `best-practice`) supplies its default severity: `validation` → `error`, `coherence` and `workflow` → `warning`, `best-practice` → `info`. Only `error` violations fail validation; warnings and info are reported only. An optional per-rule `severity` (`error` | `warning` | `info`) overrides the category default — prefer picking the right category over overriding. -Rules default to `mode: "expression"` (a JSONata `check`). Set `mode: "semantic"` for rules an LLM or decision model judges against a `rubric` (falls back to `description`): they take no `check` or `scope`, may declare `verdicts` (verdict → description, default `pass`/`fail`) with a required `failOn` list, and only run with `--semantic`. Run `sctx docs rules` for the full semantic rule reference. An optional `gatesDescendants` controls whether a violation suppresses rule checks on the node's subtree (default: true for `error` severity). +Rules default to `mode: "expression"` (a JSONata `check`). Set `mode: "semantic"` for rules an LLM or decision model judges against a `rubric` (falls back to `description`): they take no `check` or `scope`, may declare `verdicts` (verdict → description, default `pass`/`fail`) with a required `failOn` list, and only run with `--semantic`. Run `sctx docs semantic` for the full semantic rule reference. An optional `gatesDescendants` controls whether a violation suppresses rule checks on the node's subtree (default: true for `error` severity). ## Metadata composition semantics diff --git a/src/commands/docs.ts b/src/commands/docs.ts index a4a85a5..793b6ed 100644 --- a/src/commands/docs.ts +++ b/src/commands/docs.ts @@ -6,6 +6,7 @@ const TOPICS: Record = { config: 'config.md', schema: 'schemas.md', rules: 'rules.md', + semantic: 'semantic-rules.md', }; export function docs(topic?: string): void { diff --git a/src/index.ts b/src/index.ts index 9b571df..06a13b4 100755 --- a/src/index.ts +++ b/src/index.ts @@ -195,7 +195,7 @@ program.addCommand(schemasCmd); program .command('docs [topic]') - .description('Show documentation (no arg: README; topics: concepts, config, schema, rules)') + .description('Show documentation (no arg: README; topics: concepts, config, schema, rules, semantic)') .action((topic?: string) => docs(topic)); const renderCmd = new Command('render').description('Render a space in a given format'); diff --git a/src/judge/factory.ts b/src/judge/factory.ts index 7f17a5c..3a38386 100644 --- a/src/judge/factory.ts +++ b/src/judge/factory.ts @@ -1,70 +1,66 @@ -import { AnthropicJudgeProvider } from './anthropic'; -import { FakeJudgeProvider } from './fake'; -import { JevJudgeProvider } from './jev'; -import { OpenAIJudgeProvider } from './openai'; +import { AnthropicJudgeProvider } from './provider/anthropic'; +import { FakeJudgeProvider } from './provider/fake'; +import { JevJudgeProvider } from './provider/jev'; +import { OpenAIJudgeProvider } from './provider/openai'; import type { JudgeConfig, SemanticJudgeProvider } from './types'; +/** Environment variable each provider reads its API key from when none is configured. */ +const PROVIDER_KEY_ENV = { + anthropic: 'ANTHROPIC_API_KEY', + openai: 'OPENAI_API_KEY', + jev: 'OPENROUTER_API_KEY', +} as const; + +const PROVIDERS = ['anthropic', 'openai', 'jev', 'fake'] as const; + /** - * Instantiate a semantic judge provider based on configuration and environment variables. + * Instantiate a semantic judge provider. * - * Precedence: - * 1. Explicitly passed customProvider (if provided) - * 2. config.provider or SCTX_JUDGE_PROVIDER === 'fake' -> FakeJudgeProvider - * 3. config.provider === 'jev' -> JevJudgeProvider (OPENROUTER_API_KEY; always explicit, never auto-detected) - * 4. config.provider === 'anthropic' or ANTHROPIC_API_KEY present -> AnthropicJudgeProvider - * 5. config.provider === 'openai' or OPENAI_API_KEY/OPENAI_BASE_URL present -> OpenAIJudgeProvider + * The provider is always explicit: `judge.provider` in config or `SCTX_JUDGE_PROVIDER`. It is never + * inferred from whichever API keys happen to be in the environment, so a key set for another tool is + * never spent on judging unless the user chose that provider. * - * Throws a clear, actionable error if no provider can be resolved. + * Once a provider is chosen, its API key comes from `judge.apiKey`, `SCTX_JUDGE_API_KEY`, or the + * provider's usual variable (`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `OPENROUTER_API_KEY`). + * An explicitly passed `customProvider` wins over everything. */ export function createJudgeProvider( config?: JudgeConfig, env: Record = process.env, customProvider?: SemanticJudgeProvider, ): SemanticJudgeProvider { - if (customProvider) { - return customProvider; + if (customProvider) return customProvider; + + const provider = config?.provider || env.SCTX_JUDGE_PROVIDER; + if (!provider) { + throw new Error( + `No semantic judge provider configured. Set "judge.provider" in config.json (one of: ${PROVIDERS.join(', ')}) or SCTX_JUDGE_PROVIDER.`, + ); + } + if (!(PROVIDERS as readonly string[]).includes(provider)) { + throw new Error(`Unknown semantic judge provider "${provider}". Use one of: ${PROVIDERS.join(', ')}.`); } - const requestedProvider = config?.provider || env.SCTX_JUDGE_PROVIDER; - const apiKey = config?.apiKey || env.SCTX_JUDGE_API_KEY; - const endpoint = config?.endpoint || env.SCTX_JUDGE_ENDPOINT; const model = config?.model || env.SCTX_JUDGE_MODEL; + const endpoint = config?.endpoint || env.SCTX_JUDGE_ENDPOINT; + if (provider === 'fake') return new FakeJudgeProvider(); - if (requestedProvider === 'fake') { - return new FakeJudgeProvider(); - } + const keyEnv = PROVIDER_KEY_ENV[provider as keyof typeof PROVIDER_KEY_ENV]; + const apiKey = config?.apiKey || env.SCTX_JUDGE_API_KEY || env[keyEnv]; - if (requestedProvider === 'jev') { - const key = apiKey || env.OPENROUTER_API_KEY; - if (!key) { - throw new Error( - 'Jev judge provider selected, but no API key was found. Set OPENROUTER_API_KEY in the environment or configure apiKey in config.json.', - ); - } - return new JevJudgeProvider({ apiKey: key, model, endpoint }); + // OpenAI-compatible endpoints (Ollama, vLLM, ...) may need no key. + if (!apiKey && provider !== 'openai') { + throw new Error( + `Judge provider "${provider}" selected, but no API key was found. Set ${keyEnv} in the environment or "judge.apiKey" in config.json.`, + ); } - // Anthropic resolution - const hasAnthropicKey = Boolean(apiKey || env.ANTHROPIC_API_KEY); - if (requestedProvider === 'anthropic' || (!requestedProvider && hasAnthropicKey)) { - const resolvedKey = apiKey || env.ANTHROPIC_API_KEY; - if (!resolvedKey) { - throw new Error( - 'Anthropic judge provider selected, but no API key was found. Set ANTHROPIC_API_KEY in the environment or configure apiKey in config.json.', - ); - } - return new AnthropicJudgeProvider({ apiKey: resolvedKey, model, endpoint }); + switch (provider) { + case 'anthropic': + return new AnthropicJudgeProvider({ apiKey: apiKey as string, model, endpoint }); + case 'jev': + return new JevJudgeProvider({ apiKey: apiKey as string, model, endpoint }); + default: + return new OpenAIJudgeProvider({ apiKey, model, endpoint }); } - - // OpenAI resolution - const hasOpenAiKey = Boolean(apiKey || env.OPENAI_API_KEY); - const hasOpenAiBaseUrl = Boolean(endpoint || env.OPENAI_BASE_URL || env.OPENAI_ENDPOINT); - if (requestedProvider === 'openai' || (!requestedProvider && (hasOpenAiKey || hasOpenAiBaseUrl))) { - const resolvedKey = apiKey || env.OPENAI_API_KEY; - return new OpenAIJudgeProvider({ apiKey: resolvedKey, model, endpoint }); - } - - throw new Error( - 'No semantic judge provider configured. Set ANTHROPIC_API_KEY or OPENAI_API_KEY in the environment, or configure "judge" in config.json.', - ); } diff --git a/src/judge/index.ts b/src/judge/index.ts index 58a1088..4200bee 100644 --- a/src/judge/index.ts +++ b/src/judge/index.ts @@ -1,14 +1,14 @@ -export type { AnthropicJudgeProviderOptions } from './anthropic'; -export { AnthropicJudgeProvider } from './anthropic'; export { createJudgeProvider } from './factory'; -export type { FakeAnswer, FakeJudgeHandler } from './fake'; -export { FakeJudgeProvider } from './fake'; -export type { JevJudgeProviderOptions } from './jev'; -export { JevJudgeProvider } from './jev'; -export type { OpenAIJudgeProviderOptions } from './openai'; -export { OpenAIJudgeProvider } from './openai'; -export { PerRuleJudgeProvider } from './per-rule'; export { buildJudgeSystemPrompt, buildJudgeUserPrompt, parseJudgeResponse } from './prompt'; +export type { AnthropicJudgeProviderOptions } from './provider/anthropic'; +export { AnthropicJudgeProvider } from './provider/anthropic'; +export type { FakeAnswer, FakeJudgeHandler } from './provider/fake'; +export { FakeJudgeProvider } from './provider/fake'; +export type { JevJudgeProviderOptions } from './provider/jev'; +export { JevJudgeProvider } from './provider/jev'; +export type { OpenAIJudgeProviderOptions } from './provider/openai'; +export { OpenAIJudgeProvider } from './provider/openai'; +export { PerRuleJudgeProvider } from './provider/per-rule'; export type { GatedRuleRecord, JudgeConfig, diff --git a/src/judge/anthropic.ts b/src/judge/provider/anthropic.ts similarity index 97% rename from src/judge/anthropic.ts rename to src/judge/provider/anthropic.ts index 9c581ad..1c3778b 100644 --- a/src/judge/anthropic.ts +++ b/src/judge/provider/anthropic.ts @@ -1,6 +1,6 @@ +import { buildJudgeSystemPrompt, buildJudgeUserPrompt, parseJudgeResponse } from '../prompt'; +import type { SemanticJudgeInput, SemanticJudgeResult, SemanticRuleSpec } from '../types'; import { PerRuleJudgeProvider } from './per-rule'; -import { buildJudgeSystemPrompt, buildJudgeUserPrompt, parseJudgeResponse } from './prompt'; -import type { SemanticJudgeInput, SemanticJudgeResult, SemanticRuleSpec } from './types'; export interface AnthropicJudgeProviderOptions { apiKey: string; diff --git a/src/judge/fake.ts b/src/judge/provider/fake.ts similarity index 98% rename from src/judge/fake.ts rename to src/judge/provider/fake.ts index 103abf2..9a3e217 100644 --- a/src/judge/fake.ts +++ b/src/judge/provider/fake.ts @@ -1,4 +1,4 @@ -import type { SemanticJudgeInput, SemanticJudgeProvider, SemanticJudgeResult, SemanticRuleSpec } from './types'; +import type { SemanticJudgeInput, SemanticJudgeProvider, SemanticJudgeResult, SemanticRuleSpec } from '../types'; /** A fake answer: a full verdict, or the `{ pass }` shorthand mapped onto the default `pass`/`fail` verdicts. */ export type FakeAnswer = SemanticJudgeResult | { pass: boolean; evidence?: string }; diff --git a/src/judge/jev.ts b/src/judge/provider/jev.ts similarity index 99% rename from src/judge/jev.ts rename to src/judge/provider/jev.ts index 46d63f6..1af476c 100644 --- a/src/judge/jev.ts +++ b/src/judge/provider/jev.ts @@ -1,4 +1,4 @@ -import type { SemanticJudgeInput, SemanticJudgeProvider, SemanticJudgeResult } from './types'; +import type { SemanticJudgeInput, SemanticJudgeProvider, SemanticJudgeResult } from '../types'; export interface JevJudgeProviderOptions { apiKey: string; diff --git a/src/judge/openai.ts b/src/judge/provider/openai.ts similarity index 97% rename from src/judge/openai.ts rename to src/judge/provider/openai.ts index fe044e1..fedd9f0 100644 --- a/src/judge/openai.ts +++ b/src/judge/provider/openai.ts @@ -1,6 +1,6 @@ +import { buildJudgeSystemPrompt, buildJudgeUserPrompt, parseJudgeResponse } from '../prompt'; +import type { SemanticJudgeInput, SemanticJudgeResult, SemanticRuleSpec } from '../types'; import { PerRuleJudgeProvider } from './per-rule'; -import { buildJudgeSystemPrompt, buildJudgeUserPrompt, parseJudgeResponse } from './prompt'; -import type { SemanticJudgeInput, SemanticJudgeResult, SemanticRuleSpec } from './types'; export interface OpenAIJudgeProviderOptions { apiKey?: string; diff --git a/src/judge/per-rule.ts b/src/judge/provider/per-rule.ts similarity index 94% rename from src/judge/per-rule.ts rename to src/judge/provider/per-rule.ts index 244d670..a309c49 100644 --- a/src/judge/per-rule.ts +++ b/src/judge/provider/per-rule.ts @@ -1,4 +1,4 @@ -import type { SemanticJudgeInput, SemanticJudgeProvider, SemanticJudgeResult, SemanticRuleSpec } from './types'; +import type { SemanticJudgeInput, SemanticJudgeProvider, SemanticJudgeResult, SemanticRuleSpec } from '../types'; /** * Base for providers that can only judge one rule per request (chat LLMs). `evaluate` makes one call diff --git a/tests/fixtures/semantic-rule-contract/no-check.json b/tests/fixtures/semantic-rule-contract/no-check.json deleted file mode 100644 index b0678d8..0000000 --- a/tests/fixtures/semantic-rule-contract/no-check.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "$schema": "https://raw.githubusercontent.com/mindsocket/structured-context/main/schemas/generated/latest/_structured_context_schema_meta.json", - "$id": "sctx://semantic-rule-contract.no-check", - "$metadata": { "rules": [{ "id": "r", "category": "validation", "description": "d" }] }, - "type": "object" -} diff --git a/tests/fixtures/semantic-rule-contract/semantic-check.json b/tests/fixtures/semantic-rule-contract/semantic-check.json deleted file mode 100644 index e348ae4..0000000 --- a/tests/fixtures/semantic-rule-contract/semantic-check.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "$schema": "https://raw.githubusercontent.com/mindsocket/structured-context/main/schemas/generated/latest/_structured_context_schema_meta.json", - "$id": "sctx://semantic-rule-contract.semantic-check", - "$metadata": { - "rules": [{ "id": "r", "category": "validation", "description": "d", "mode": "semantic", "check": "true" }] - }, - "type": "object" -} diff --git a/tests/fixtures/semantic-rule-contract/semantic-global.json b/tests/fixtures/semantic-rule-contract/semantic-global.json deleted file mode 100644 index a7bb0bd..0000000 --- a/tests/fixtures/semantic-rule-contract/semantic-global.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "$schema": "https://raw.githubusercontent.com/mindsocket/structured-context/main/schemas/generated/latest/_structured_context_schema_meta.json", - "$id": "sctx://semantic-rule-contract.semantic-global", - "$metadata": { - "rules": [{ "id": "r", "category": "validation", "description": "d", "mode": "semantic", "scope": "global" }] - }, - "type": "object" -} diff --git a/tests/fixtures/semantic-rule-contract/semantic-ok.json b/tests/fixtures/semantic-rule-contract/semantic-ok.json deleted file mode 100644 index e1f0ad7..0000000 --- a/tests/fixtures/semantic-rule-contract/semantic-ok.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "$schema": "https://raw.githubusercontent.com/mindsocket/structured-context/main/schemas/generated/latest/_structured_context_schema_meta.json", - "$id": "sctx://semantic-rule-contract.semantic-ok", - "$metadata": { "rules": [{ "id": "r", "category": "validation", "description": "d", "mode": "semantic" }] }, - "type": "object" -} diff --git a/tests/fixtures/strict_ost/valid-directory/Progressive sign-up form.md b/tests/fixtures/strict_ost/valid-directory/Progressive sign-up form.md deleted file mode 100644 index b5df6e6..0000000 --- a/tests/fixtures/strict_ost/valid-directory/Progressive sign-up form.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -type: solution -status: exploring -parent: "[[Sign-up is too complex]]" ---- - -Break the sign-up into a progressive multi-step form that asks for one thing at a time. diff --git a/tests/fixtures/strict_ost/valid-directory/Social login option.md b/tests/fixtures/strict_ost/valid-directory/Social login option.md deleted file mode 100644 index 2b8ccb2..0000000 --- a/tests/fixtures/strict_ost/valid-directory/Social login option.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -type: solution -status: exploring -parent: "[[Sign-up is too complex]]" ---- - -Offer social login so new users can skip manual account creation entirely. diff --git a/tests/judge/factory.test.ts b/tests/judge/factory.test.ts index 8283112..f610081 100644 --- a/tests/judge/factory.test.ts +++ b/tests/judge/factory.test.ts @@ -1,45 +1,56 @@ import { describe, expect, it } from 'bun:test'; -import { AnthropicJudgeProvider } from '../../src/judge/anthropic'; import { createJudgeProvider } from '../../src/judge/factory'; -import { FakeJudgeProvider } from '../../src/judge/fake'; -import { OpenAIJudgeProvider } from '../../src/judge/openai'; +import { AnthropicJudgeProvider } from '../../src/judge/provider/anthropic'; +import { FakeJudgeProvider } from '../../src/judge/provider/fake'; +import { JevJudgeProvider } from '../../src/judge/provider/jev'; +import { OpenAIJudgeProvider } from '../../src/judge/provider/openai'; describe('createJudgeProvider factory', () => { - it('returns custom provider when passed directly', () => { + it('returns a custom provider when passed directly', () => { const custom = new FakeJudgeProvider(); - const provider = createJudgeProvider(undefined, {}, custom); - expect(provider).toBe(custom); + expect(createJudgeProvider(undefined, {}, custom)).toBe(custom); }); it('creates FakeJudgeProvider when provider is "fake"', () => { const provider = createJudgeProvider({ provider: 'fake' }, {}); - expect(provider.name).toBe('fake'); expect(provider instanceof FakeJudgeProvider).toBe(true); }); - it('creates AnthropicJudgeProvider when ANTHROPIC_API_KEY is in env', () => { - const provider = createJudgeProvider(undefined, { ANTHROPIC_API_KEY: 'test-key' }); - expect(provider.name).toBe('anthropic'); - expect(provider instanceof AnthropicJudgeProvider).toBe(true); + it.each([ + ['anthropic', 'ANTHROPIC_API_KEY', AnthropicJudgeProvider], + ['openai', 'OPENAI_API_KEY', OpenAIJudgeProvider], + ['jev', 'OPENROUTER_API_KEY', JevJudgeProvider], + ] as const)('creates %s from config provider with its usual env key (%s)', (provider, keyEnv, Class) => { + expect(createJudgeProvider({ provider }, { [keyEnv]: 'k' }) instanceof Class).toBe(true); }); - it('creates AnthropicJudgeProvider from config apiKey', () => { - const provider = createJudgeProvider({ provider: 'anthropic', apiKey: 'test-key-config' }, {}); + it('takes the provider from SCTX_JUDGE_PROVIDER and the key from config', () => { + const provider = createJudgeProvider({ apiKey: 'k' }, { SCTX_JUDGE_PROVIDER: 'anthropic' }); expect(provider.name).toBe('anthropic'); }); - it('creates OpenAIJudgeProvider when OPENAI_API_KEY is in env', () => { - const provider = createJudgeProvider(undefined, { OPENAI_API_KEY: 'test-openai-key' }); - expect(provider.name).toBe('openai'); - expect(provider instanceof OpenAIJudgeProvider).toBe(true); + it('never infers a provider from API keys in the environment', () => { + const env = { + ANTHROPIC_API_KEY: 'a', + OPENAI_API_KEY: 'o', + OPENROUTER_API_KEY: 'r', + OPENAI_BASE_URL: 'http://x/v1', + }; + expect(() => createJudgeProvider(undefined, env)).toThrow('No semantic judge provider configured'); + expect(() => createJudgeProvider({ model: 'm' }, env)).toThrow('No semantic judge provider configured'); }); - it('creates OpenAIJudgeProvider from OPENAI_BASE_URL (for local endpoints like Ollama)', () => { - const provider = createJudgeProvider(undefined, { OPENAI_BASE_URL: 'http://localhost:11434/v1' }); - expect(provider.name).toBe('openai'); + it("does not borrow another provider's key", () => { + expect(() => createJudgeProvider({ provider: 'anthropic' }, { OPENAI_API_KEY: 'o' })).toThrow('ANTHROPIC_API_KEY'); + expect(() => createJudgeProvider({ provider: 'jev' }, { ANTHROPIC_API_KEY: 'a' })).toThrow('OPENROUTER_API_KEY'); + }); + + it('lets openai run without a key for local endpoints', () => { + const provider = createJudgeProvider({ provider: 'openai', endpoint: 'http://localhost:11434/v1' }, {}); + expect(provider instanceof OpenAIJudgeProvider).toBe(true); }); - it('throws descriptive error when no provider or key is configured', () => { - expect(() => createJudgeProvider(undefined, {})).toThrow('No semantic judge provider configured'); + it('rejects an unknown provider', () => { + expect(() => createJudgeProvider({ provider: 'gemini' as never }, {})).toThrow('Unknown semantic judge provider'); }); }); diff --git a/tests/judge/fake-provider.test.ts b/tests/judge/provider/fake.test.ts similarity index 90% rename from tests/judge/fake-provider.test.ts rename to tests/judge/provider/fake.test.ts index ab93f16..5bcfd13 100644 --- a/tests/judge/fake-provider.test.ts +++ b/tests/judge/provider/fake.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'bun:test'; -import { FakeJudgeProvider } from '../../src/judge/fake'; -import type { SemanticJudgeInput } from '../../src/judge/types'; -import { graded, node, passFailRule } from './helpers'; +import { FakeJudgeProvider } from '../../../src/judge/provider/fake'; +import type { SemanticJudgeInput } from '../../../src/judge/types'; +import { graded, node, passFailRule } from '../helpers'; const input: SemanticJudgeInput = { node, rules: [passFailRule, graded] }; diff --git a/tests/judge/jev.test.ts b/tests/judge/provider/jev.test.ts similarity index 94% rename from tests/judge/jev.test.ts rename to tests/judge/provider/jev.test.ts index c220ec3..844712e 100644 --- a/tests/judge/jev.test.ts +++ b/tests/judge/provider/jev.test.ts @@ -1,7 +1,7 @@ import { afterEach, describe, expect, it } from 'bun:test'; -import { createJudgeProvider } from '../../src/judge/factory'; -import { JevJudgeProvider } from '../../src/judge/jev'; -import { graded, node, passFailRule } from './helpers'; +import { createJudgeProvider } from '../../../src/judge/factory'; +import { JevJudgeProvider } from '../../../src/judge/provider/jev'; +import { graded, node, passFailRule } from '../helpers'; const realFetch = globalThis.fetch; afterEach(() => { diff --git a/tests/judge/verdicts.test.ts b/tests/judge/verdicts.test.ts index 4578f0c..e9e5331 100644 --- a/tests/judge/verdicts.test.ts +++ b/tests/judge/verdicts.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'bun:test'; -import { PerRuleJudgeProvider } from '../../src/judge/per-rule'; +import { PerRuleJudgeProvider } from '../../src/judge/provider/per-rule'; import type { SemanticJudgeResult } from '../../src/judge/types'; import { semanticRuleSpec } from '../../src/judge/verdicts'; import type { Rule } from '../../src/types'; diff --git a/tests/schema/semantic-rule-contract.test.ts b/tests/schema/semantic-rule-contract.test.ts index d03b6da..847ffe4 100644 --- a/tests/schema/semantic-rule-contract.test.ts +++ b/tests/schema/semantic-rule-contract.test.ts @@ -1,24 +1,47 @@ -import { describe, expect, it } from 'bun:test'; +import { afterAll, beforeAll, describe, expect, it } from 'bun:test'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { loadMetadata } from '../../src/schema/schema'; -const dir = join(import.meta.dir, '..', 'fixtures/semantic-rule-contract'); -const load = (name: string) => loadMetadata(join(dir, `${name}.json`)); +const META = + 'https://raw.githubusercontent.com/mindsocket/structured-context/main/schemas/generated/latest/_structured_context_schema_meta.json'; +const base = { id: 'r', category: 'validation', description: 'd' }; describe('rule contract: check and semantic mode', () => { + let dir: string; + beforeAll(() => { + dir = mkdtempSync(join(tmpdir(), 'sctx-rule-contract-')); + }); + afterAll(() => rmSync(dir, { recursive: true, force: true })); + + const load = (name: string, rule: object) => { + const path = join(dir, `${name}.json`); + writeFileSync( + path, + JSON.stringify({ + $schema: META, + $id: `sctx://rule-contract.${name}`, + $metadata: { rules: [rule] }, + type: 'object', + }), + ); + return loadMetadata(path); + }; + it('requires check on expression rules', () => { - expect(() => load('no-check')).toThrow(); + expect(() => load('no-check', base)).toThrow(); }); it('accepts a semantic rule without check', () => { - expect(load('semantic-ok').rules).toHaveLength(1); + expect(load('semantic-ok', { ...base, mode: 'semantic' }).rules).toHaveLength(1); }); it('rejects a semantic rule that also declares check', () => { - expect(() => load('semantic-check')).toThrow(); + expect(() => load('semantic-check', { ...base, mode: 'semantic', check: 'true' })).toThrow(); }); it('rejects a global semantic rule', () => { - expect(() => load('semantic-global')).toThrow(); + expect(() => load('semantic-global', { ...base, mode: 'semantic', scope: 'global' })).toThrow(); }); }); diff --git a/tests/schema/validate-rules-semantic.test.ts b/tests/schema/validate-rules-semantic.test.ts index 6ed1cac..0a649c1 100644 --- a/tests/schema/validate-rules-semantic.test.ts +++ b/tests/schema/validate-rules-semantic.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'bun:test'; -import { FakeJudgeProvider } from '../../src/judge/fake'; +import { FakeJudgeProvider } from '../../src/judge/provider/fake'; import { validateRules } from '../../src/schema/validate-rules'; import type { Rule, SpaceNode } from '../../src/types'; import { makeParentRef } from '../test-helpers'; diff --git a/tests/validate/gating.test.ts b/tests/validate/gating.test.ts index be8113a..01a48e2 100644 --- a/tests/validate/gating.test.ts +++ b/tests/validate/gating.test.ts @@ -43,11 +43,14 @@ describe('validateSpace cascade gating', () => { expect(gate!.gatedFiles).toContain('Child Opportunity.md'); }); - it('a clean space with only non-blocking rules reports no gates', async () => { + it('does not gate on a non-error rule violation by default (solution-quantity is best-practice)', async () => { + // The fixture opportunity has only one solution below it, violating solution-quantity. + // That rule is info severity, so it doesn't block and the solution's own rules still run. const context = makeSpaceContext(VALID_DIR, SCHEMA_PATH); const result = await validateSpace(context); expect(result.nodeErrorCount).toBe(0); + expect(result.ruleViolations.some((v) => v.ruleId === 'solution-quantity')).toBe(true); expect(result.coverageGates).toEqual([]); }); }); diff --git a/tests/validate/semantic-validation.test.ts b/tests/validate/semantic-validation.test.ts index 2913cb7..a0dc09f 100644 --- a/tests/validate/semantic-validation.test.ts +++ b/tests/validate/semantic-validation.test.ts @@ -3,7 +3,7 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import type { Config } from '../../src/config'; -import { FakeJudgeProvider } from '../../src/judge/fake'; +import { FakeJudgeProvider } from '../../src/judge/provider/fake'; import { bundledSchemasDir } from '../../src/schema/schema'; import { createSpaceContext } from '../../src/space-context'; import type { FileValidationResult } from '../../src/types'; diff --git a/tests/validate/strict.test.ts b/tests/validate/strict.test.ts index 3f0e2ff..753ef52 100644 --- a/tests/validate/strict.test.ts +++ b/tests/validate/strict.test.ts @@ -198,8 +198,8 @@ describe('Strict OST schema validation', () => { ({ nodes } = await readSpaceDirectory(makePluginContext(VALID_DIR, STRICT_SCHEMA_PATH))); }); - it('reads all 6 nodes from valid-directory', () => { - expect(nodes).toHaveLength(6); + it('reads all 4 nodes from valid-directory', () => { + expect(nodes).toHaveLength(4); }); it('all nodes pass strict_ost schema validation', () => { From b93f7e550f2dde773d38f1672389e461c6a80665 Mon Sep 17 00:00:00 2001 From: Roger Barnes Date: Wed, 14 Oct 2026 10:36:01 +1100 Subject: [PATCH 07/22] feat(cli): --no-semantic override, plain error for misconfigured judge, hooks never judge --- README.md | 4 +- docs/semantic-rules.md | 2 +- plugin/.claude-plugin/plugin.json | 2 +- plugin/scripts/state.ts | 2 +- .../structured-context/references/commands.md | 4 +- src/index.ts | 48 +++++++++++++------ src/judge/factory.ts | 9 ++-- src/judge/index.ts | 2 +- tests/judge/factory.test.ts | 4 +- 9 files changed, 49 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 7a9e69a..14d0653 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,7 @@ See [docs/schemas.md](docs/schemas.md) (`sctx docs schema`) for the dialect, com ### Validate nodes ```bash -sctx validate [--watch] [--depth ] [--include-gated] [--semantic] [--rule-filter ] +sctx validate [--watch] [--depth ] [--include-gated] [--semantic|--no-semantic] [--rule-filter ] ``` Validates markdown files against the JSON schema: @@ -141,7 +141,7 @@ Validates markdown files against the JSON schema: - Runs executable rules (`$metadata.rules`) with **cascade gating**: a schema, hierarchy, or blocking-rule failure on a node suppresses rule evaluation for that node's hierarchy subtree. Gated checks are reported in a Coverage section so you know what wasn't evaluated and why. See [docs/rules.md](docs/rules.md#cascade-gating) for details. - `--depth ` limits evaluation to hierarchy nodes within `n` breadth-first levels from the top — useful for staged agentic fix loops (fix the top level, then request the next). - `--include-gated` evaluates rules even on gated nodes (still reported as gated) — for surfacing cheap bottom-up checks on request. -- `--semantic` also evaluates `mode: semantic` rules with a model judge (Anthropic, OpenAI-compatible, or Jev); `--rule-filter` narrows which rules run. See [docs/semantic-rules.md](docs/semantic-rules.md) (`sctx docs semantic`). +- `--semantic` also evaluates `mode: semantic` rules with a model judge (Anthropic, OpenAI-compatible, or Jev); `--no-semantic` overrides `semantic`/`judge.autoRun` in config; `--rule-filter` narrows which rules run. See [docs/semantic-rules.md](docs/semantic-rules.md) (`sctx docs semantic`). ### Render space diff --git a/docs/semantic-rules.md b/docs/semantic-rules.md index 990b9e9..c8d29c7 100644 --- a/docs/semantic-rules.md +++ b/docs/semantic-rules.md @@ -33,7 +33,7 @@ For expression rules, severity, gating and merge behaviour, see [Rules](rules.md with `ANTHROPIC_API_KEY` set. See [Judge configuration](config.md#judge-semantic-rules) for every field. -3. **Run it**: `sctx validate --semantic`. +3. **Run it**: `sctx validate --semantic`. To turn it on by default, set `semantic: true` (or `judge.autoRun`) in config; `--no-semantic` overrides that for a single run. ## Rule fields diff --git a/plugin/.claude-plugin/plugin.json b/plugin/.claude-plugin/plugin.json index e39d96c..b113f3b 100644 --- a/plugin/.claude-plugin/plugin.json +++ b/plugin/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "structured-context", - "version": "0.12.9", + "version": "0.12.10", "description": "Structured content authoring for Obsidian — automatic validation hooks, slash commands, and agent skills to keep markdown files compliant with your schema.", "homepage": "https://github.com/mindsocket/structured-context", "repository": "https://github.com/mindsocket/structured-context", diff --git a/plugin/scripts/state.ts b/plugin/scripts/state.ts index 945f319..6807644 100644 --- a/plugin/scripts/state.ts +++ b/plugin/scripts/state.ts @@ -132,7 +132,7 @@ export function hashSpace(space: SpaceInfo, configFiles: string[]): string { * Info-level rule violations are left out: they are not worth another turn from Claude. */ export async function validateSpace(name: string, options?: HookOptions): Promise> { - const text = await sctx(['validate', name, '--json'], options); + const text = await sctx(['validate', name, '--json', '--no-semantic'], options); if (!text) return {}; const output = JSON.parse(text) as ValidateOutput; diff --git a/plugin/skills/structured-context/references/commands.md b/plugin/skills/structured-context/references/commands.md index 4781908..2d29864 100644 --- a/plugin/skills/structured-context/references/commands.md +++ b/plugin/skills/structured-context/references/commands.md @@ -11,7 +11,7 @@ Then use the `sctx` command for all operations. Always pass `--config ` wh ## validate ```bash -sctx validate [--watch] [--config ] [--depth ] [--include-gated] [--semantic] [--rule-filter ] +sctx validate [--watch] [--config ] [--depth ] [--include-gated] [--semantic|--no-semantic] [--rule-filter ] ``` Validates all `.md` files in the space against the JSON schema. For each file: @@ -30,7 +30,7 @@ the top — useful for fixing a space level by level (fix the top, re-run with a `--depth`, repeat). `--include-gated` evaluates rules on gated nodes anyway, still reporting them as gated — use it to surface cheap bottom-up checks even when an upstream blocker fired. `--semantic` also runs `mode: semantic` rules through a configured model judge (costly; off by -default), and `--rule-filter =` (repeatable; mode, category, severity, type, id) +default; `--no-semantic` forces it off even if config enables it), and `--rule-filter =` (repeatable; mode, category, severity, type, id) narrows which rules run. `validate-file` accepts the same two flags. Exit codes: `0` = clean (warning and info rule violations don't fail), `1` = validation errors found. diff --git a/src/index.ts b/src/index.ts index 06a13b4..0140374 100755 --- a/src/index.ts +++ b/src/index.ts @@ -16,6 +16,7 @@ import { validate, watchValidate } from './commands/validate'; import { validateFileCommand } from './commands/validate-file'; import { loadConfig, setConfigPath } from './config'; import { CLI_NAME } from './constants'; +import { JudgeConfigError } from './judge/factory'; import { createSpaceContext, SpaceNotFoundError } from './space-context'; import type { SpaceContext } from './types'; @@ -37,6 +38,17 @@ export function buildSpaceContext(spaceName: string): SpaceContext { } } +/** Run a validate command, reporting a misconfigured judge as a plain error rather than a stack trace. */ +async function reportJudgeConfigErrors(run: () => Promise): Promise { + try { + return await run(); + } catch (err) { + if (!(err instanceof JudgeConfigError)) throw err; + console.error(`Error: ${err.message}`); + return 1; + } +} + const require = createRequire(import.meta.url); const packageJson = require('../package.json'); @@ -66,14 +78,17 @@ program 'Evaluate only matching rules; dimensions: mode, category, severity, type, id (repeatable, e.g. mode=semantic)', (v: string, prev: string[] = []) => [...prev, v], ) - .option('--semantic', 'Evaluate semantic rules using built-in LLM judge') + .option('--semantic', 'Evaluate semantic rules with the configured judge') + .option('--no-semantic', 'Skip semantic rules even if enabled in config') .action(async (filePath, options) => { - process.exitCode = await validateFileCommand(filePath, { - json: options.json, - includeGated: options.includeGated, - semantic: options.semantic, - ruleFilter: options.ruleFilter, - }); + process.exitCode = await reportJudgeConfigErrors(() => + validateFileCommand(filePath, { + json: options.json, + includeGated: options.includeGated, + semantic: options.semantic, + ruleFilter: options.ruleFilter, + }), + ); }); program @@ -96,20 +111,23 @@ program 'Evaluate only matching rules; dimensions: mode, category, severity, type, id (repeatable, e.g. mode=semantic)', (v: string, prev: string[] = []) => [...prev, v], ) - .option('--semantic', 'Evaluate semantic rules using built-in LLM judge') + .option('--semantic', 'Evaluate semantic rules with the configured judge') + .option('--no-semantic', 'Skip semantic rules even if enabled in config') .action(async (spaceName, options) => { const context = buildSpaceContext(spaceName); if (options.watch) { await watchValidate(context); } else { // exitCode rather than process.exit(): exiting straight away can cut off large piped output - process.exitCode = await validate(context, { - json: options.json, - depth: options.depth, - includeGated: options.includeGated, - semantic: options.semantic, - ruleFilter: options.ruleFilter, - }); + process.exitCode = await reportJudgeConfigErrors(() => + validate(context, { + json: options.json, + depth: options.depth, + includeGated: options.includeGated, + semantic: options.semantic, + ruleFilter: options.ruleFilter, + }), + ); } }); diff --git a/src/judge/factory.ts b/src/judge/factory.ts index 3a38386..b602d8f 100644 --- a/src/judge/factory.ts +++ b/src/judge/factory.ts @@ -11,6 +11,9 @@ const PROVIDER_KEY_ENV = { jev: 'OPENROUTER_API_KEY', } as const; +/** The judge is misconfigured (no provider, unknown provider, missing key). Reported as a plain CLI error. */ +export class JudgeConfigError extends Error {} + const PROVIDERS = ['anthropic', 'openai', 'jev', 'fake'] as const; /** @@ -33,12 +36,12 @@ export function createJudgeProvider( const provider = config?.provider || env.SCTX_JUDGE_PROVIDER; if (!provider) { - throw new Error( + throw new JudgeConfigError( `No semantic judge provider configured. Set "judge.provider" in config.json (one of: ${PROVIDERS.join(', ')}) or SCTX_JUDGE_PROVIDER.`, ); } if (!(PROVIDERS as readonly string[]).includes(provider)) { - throw new Error(`Unknown semantic judge provider "${provider}". Use one of: ${PROVIDERS.join(', ')}.`); + throw new JudgeConfigError(`Unknown semantic judge provider "${provider}". Use one of: ${PROVIDERS.join(', ')}.`); } const model = config?.model || env.SCTX_JUDGE_MODEL; @@ -50,7 +53,7 @@ export function createJudgeProvider( // OpenAI-compatible endpoints (Ollama, vLLM, ...) may need no key. if (!apiKey && provider !== 'openai') { - throw new Error( + throw new JudgeConfigError( `Judge provider "${provider}" selected, but no API key was found. Set ${keyEnv} in the environment or "judge.apiKey" in config.json.`, ); } diff --git a/src/judge/index.ts b/src/judge/index.ts index 4200bee..c4670a8 100644 --- a/src/judge/index.ts +++ b/src/judge/index.ts @@ -1,4 +1,4 @@ -export { createJudgeProvider } from './factory'; +export { createJudgeProvider, JudgeConfigError } from './factory'; export { buildJudgeSystemPrompt, buildJudgeUserPrompt, parseJudgeResponse } from './prompt'; export type { AnthropicJudgeProviderOptions } from './provider/anthropic'; export { AnthropicJudgeProvider } from './provider/anthropic'; diff --git a/tests/judge/factory.test.ts b/tests/judge/factory.test.ts index f610081..fb9b4c1 100644 --- a/tests/judge/factory.test.ts +++ b/tests/judge/factory.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'bun:test'; -import { createJudgeProvider } from '../../src/judge/factory'; +import { createJudgeProvider, JudgeConfigError } from '../../src/judge/factory'; import { AnthropicJudgeProvider } from '../../src/judge/provider/anthropic'; import { FakeJudgeProvider } from '../../src/judge/provider/fake'; import { JevJudgeProvider } from '../../src/judge/provider/jev'; @@ -36,7 +36,7 @@ describe('createJudgeProvider factory', () => { OPENROUTER_API_KEY: 'r', OPENAI_BASE_URL: 'http://x/v1', }; - expect(() => createJudgeProvider(undefined, env)).toThrow('No semantic judge provider configured'); + expect(() => createJudgeProvider(undefined, env)).toThrow(JudgeConfigError); expect(() => createJudgeProvider({ model: 'm' }, env)).toThrow('No semantic judge provider configured'); }); From 403bfbc7b9e275461a6f54b42aafbccfe5438d07 Mon Sep 17 00:00:00 2001 From: Roger Barnes Date: Wed, 14 Oct 2026 12:09:10 +1100 Subject: [PATCH 08/22] feat(rules): verdict severity map with ignore and guidance, when precondition, named judge providers and --judge --- README.md | 4 +- config.example.json | 5 +- docs/api.md | 9 +- docs/concepts.md | 4 +- docs/config.md | 25 ++- docs/rules.md | 19 +- docs/semantic-rules.md | 45 ++++- plugin/.claude-plugin/plugin.json | 2 +- .../structured-context/references/commands.md | 4 +- .../references/schema-authoring.md | 2 +- .../_structured_context_schema_meta.json | 99 ++++++++-- src/api.ts | 4 + src/commands/schemas.ts | 21 +- src/commands/validate-file.ts | 3 +- src/commands/validate.ts | 12 +- src/config.ts | 20 +- src/index.ts | 3 + src/judge/factory.ts | 183 ++++++++++++++---- src/judge/index.ts | 3 +- src/judge/judge-node.ts | 20 +- src/judge/types.ts | 30 ++- src/judge/verdicts.ts | 78 ++++++-- src/schema/gating.ts | 72 ++++--- src/schema/metadata-contract.ts | 64 +++++- src/schema/rule-filter.ts | 22 ++- src/schema/rule-severity.ts | 24 +++ src/schema/validate-rules.ts | 28 ++- src/types.ts | 4 + src/validate.ts | 31 ++- tests/helpers/temp-space.ts | 58 ++++++ tests/judge/factory.test.ts | 80 +++++++- tests/judge/provider/jev.test.ts | 2 +- tests/judge/verdicts.test.ts | 56 ++++-- tests/schema/semantic-rule-contract.test.ts | 31 +++ tests/validate/semantic-rule-shape.test.ts | 151 +++++++++++++++ tests/validate/semantic-validation.test.ts | 65 ++----- 36 files changed, 1037 insertions(+), 246 deletions(-) create mode 100644 src/schema/rule-severity.ts create mode 100644 tests/helpers/temp-space.ts create mode 100644 tests/validate/semantic-rule-shape.test.ts diff --git a/README.md b/README.md index 14d0653..b47d702 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,7 @@ See [docs/schemas.md](docs/schemas.md) (`sctx docs schema`) for the dialect, com ### Validate nodes ```bash -sctx validate [--watch] [--depth ] [--include-gated] [--semantic|--no-semantic] [--rule-filter ] +sctx validate [--watch] [--depth ] [--include-gated] [--semantic|--no-semantic] [--judge ] [--rule-filter ] ``` Validates markdown files against the JSON schema: @@ -141,7 +141,7 @@ Validates markdown files against the JSON schema: - Runs executable rules (`$metadata.rules`) with **cascade gating**: a schema, hierarchy, or blocking-rule failure on a node suppresses rule evaluation for that node's hierarchy subtree. Gated checks are reported in a Coverage section so you know what wasn't evaluated and why. See [docs/rules.md](docs/rules.md#cascade-gating) for details. - `--depth ` limits evaluation to hierarchy nodes within `n` breadth-first levels from the top — useful for staged agentic fix loops (fix the top level, then request the next). - `--include-gated` evaluates rules even on gated nodes (still reported as gated) — for surfacing cheap bottom-up checks on request. -- `--semantic` also evaluates `mode: semantic` rules with a model judge (Anthropic, OpenAI-compatible, or Jev); `--no-semantic` overrides `semantic`/`judge.autoRun` in config; `--rule-filter` narrows which rules run. See [docs/semantic-rules.md](docs/semantic-rules.md) (`sctx docs semantic`). +- `--semantic` also evaluates `mode: semantic` rules with a model judge (Anthropic, OpenAI-compatible, or Jev); `--no-semantic` overrides `semantic`/`judge.autoRun` in config, and `--judge ` picks a configured judge provider for every rule; `--rule-filter` narrows which rules run. See [docs/semantic-rules.md](docs/semantic-rules.md) (`sctx docs semantic`). ### Render space diff --git a/config.example.json b/config.example.json index 40c62e4..2e7d69f 100644 --- a/config.example.json +++ b/config.example.json @@ -18,8 +18,5 @@ } ], "schema": "/path/to/custom/schemas/my-general-schema.json", - "judge": { - "provider": "anthropic", - "model": "claude-haiku-5-5" - } + "judge": { "provider": "none" } } diff --git a/docs/api.md b/docs/api.md index 58d642c..ccde51b 100644 --- a/docs/api.md +++ b/docs/api.md @@ -18,8 +18,8 @@ see [architecture.md](architecture.md#information-flow) for that flow. | `loadSpaceGraph(spaceName, config, options?)` | Read, assemble, and optionally filter a space into a navigable `SpaceGraph`. `options.hierarchy` names the hierarchy that structures the graph (default: the main hierarchy). | | `assembleSpaceGraph(context, options?)` | Assemble (and optionally filter) a graph from a context you already hold — reuses the context instead of rebuilding it. | | `buildSpaceGraph(nodes, hierarchy?)` | Build a navigable `SpaceGraph` directly from an array of `SpaceNode`s and a `Hierarchy` definition. | -| `validateSpace(context, options?)` | Run full validation (schema, references, rules, duplicates) over a space. Options: `depth`, `includeGated`, `semantic`, `judgeProvider`, `ruleFilter`. | -| `validateFile(filePath, config, options?)` | Validate a single file within its space. Options: `includeGated`, `semantic`, `judgeProvider`, `ruleFilter`. | +| `validateSpace(context, options?)` | Run full validation (schema, references, rules, duplicates) over a space. Options: `depth`, `includeGated`, `semantic`, `judge`, `judgeProvider`, `ruleFilter`. | +| `validateFile(filePath, config, options?)` | Validate a single file within its space. Options: `includeGated`, `semantic`, `judge`, `judgeProvider`, `ruleFilter`. | | `formatWikilink(target, alias?)` | Format a target into an Obsidian wikilink (`[[target]]` or `[[target|alias]]`). | | `parseWikilink(link)` | Parse a wikilink or target string into `{ target, anchor?, displayText? }`. | | `wikilinkToTarget(link)` | Extract the plain target lookup key from a wikilink string. | @@ -29,11 +29,12 @@ see [architecture.md](architecture.md#information-flow) for that flow. ## Semantic judging -Semantic rules are evaluated only when `semantic: true` is passed (or configured; see [config.md](config.md#judge-semantic-rules)); the provider must be named explicitly, never inferred from the environment. The judge defaults to the configured provider; pass `judgeProvider` to inject your own. +Semantic rules are evaluated only when `semantic: true` is passed (or configured; see [config.md](config.md#judge-semantic-rules)); the provider must be named explicitly, never inferred from the environment. The judge is built from the configured providers, with `judge` naming one to use for every rule; pass `judgeProvider` to inject your own. | Export | Purpose | | --- | --- | -| `createJudgeProvider(config?, env?, custom?)` | Resolve a `SemanticJudgeProvider` from `JudgeConfig` and the environment. | +| `createJudge(config?, env?, custom?, override?)` | Build the `RoutedJudgeProvider` for a run: routes each rule to the provider it names (`judge`), else the default; `override` names one for every rule. | +| `createJudgeProvider(config?, env?, custom?, name?)` | Resolve a single `SemanticJudgeProvider` from `JudgeConfig` and the environment. Throws `JudgeConfigError` when misconfigured. | | `AnthropicJudgeProvider`, `OpenAIJudgeProvider`, `JevJudgeProvider` | Built-in providers. | | `PerRuleJudgeProvider` | Base class for providers that can only answer one rule per request; implement `judgeRule`. | | `FakeJudgeProvider` | Deterministic provider for tests. | diff --git a/docs/concepts.md b/docs/concepts.md index 80e85e6..aa6f805 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -153,9 +153,9 @@ See [docs/rules.md](rules.md) for the rules reference, including JSONata express ### Semantic rules -A **semantic rule** (`mode: semantic`) checks meaning that an expression cannot, such as whether an opportunity is framed in the customer's voice. Instead of a `check` expression it carries a `rubric` and a closed set of **verdicts**. +A **semantic rule** (`mode: semantic`) checks meaning that an expression cannot, such as whether an opportunity is framed in the customer's voice. Instead of a `check` expression it carries a `rubric` and a closed set of **verdicts**. A verdict is the judge's classification ("customer-need", "vague", "solution"); the rule's `severity` map says what each verdict means for the user (`error`, `warning`, `info`, or `ignore`), and a verdict can carry user-facing `guidance`. -A **judge** is the model that evaluates semantic rules: a chat LLM (Anthropic, OpenAI-compatible) or a decision model (Jev). It receives a node and its rules and must answer each rule with exactly one **verdict** from that rule's `verdicts`; the rule's `failOn` verdicts are violations and all others pass. A judge may also report a confidence and probabilities. Semantic rules are opt-in (`--semantic`) because they cost money and time, and they run inside [cascade gating](#cascade-gating). See [docs/semantic-rules.md](semantic-rules.md) for the rule fields and [docs/config.md](config.md#judge-semantic-rules) for judge configuration. +A **judge** is the model that evaluates semantic rules: a chat LLM (Anthropic, OpenAI-compatible) or a decision model (Jev). It receives a node and its rules and must answer each rule with exactly one **verdict** from that rule's `verdicts`; verdicts with a severity are violations and all others pass. A judge may also report a confidence and probabilities. Semantic rules are opt-in (`--semantic`) because they cost money and time, and they run inside [cascade gating](#cascade-gating). See [docs/semantic-rules.md](semantic-rules.md) for the rule fields and [docs/config.md](config.md#judge-semantic-rules) for judge configuration. ### Cascade gating diff --git a/docs/config.md b/docs/config.md index fffeb7d..acf1c6d 100644 --- a/docs/config.md +++ b/docs/config.md @@ -69,6 +69,8 @@ Plugins can also contribute schemas and partials via a `/schemas/` d `judge` configures the model that evaluates `mode: semantic` [rules](semantic-rules.md). Set it at the top level of the config (all spaces) or on a space (the space wins). `semantic` turns semantic evaluation on by default, at the same two levels; otherwise it runs only with `--semantic`. +The shorthand defines one provider: + ```json { "judge": { "provider": "anthropic", "model": "claude-haiku-5-5" }, @@ -80,13 +82,30 @@ Plugins can also contribute schemas and partials via a `/schemas/` d | Field | Description | |---|---| -| `provider` | **Required** to judge (or set `SCTX_JUDGE_PROVIDER`). One of `anthropic`, `openai` (any OpenAI-compatible endpoint: OpenAI, Ollama, vLLM, OpenRouter chat models), `jev` (TypeSafe decision model via OpenRouter's decisions API), or `fake` (tests). Never inferred from API keys in the environment. | +| `provider` | **Required** to judge (or set `SCTX_JUDGE_PROVIDER`). One of `anthropic`, `openai` (any OpenAI-compatible endpoint: OpenAI, Ollama, vLLM, OpenRouter chat models), `jev` (TypeSafe decision model via OpenRouter's decisions API), `fake` (tests), or `none` (an inert placeholder for templates; judging with it is an error). Never inferred from API keys in the environment. | | `model` | Defaults: `claude-haiku-5-5` (anthropic), `gpt-4o-mini` (openai), `typesafe/jev-1.13` (jev). | | `endpoint` | Override the API URL. Jev defaults to `https://openrouter.ai/api/alpha/decisions`. | -| `apiKey` | Prefer environment variables over storing keys in config. Once the provider is chosen, its key is read from `ANTHROPIC_API_KEY`, `OPENAI_API_KEY` or `OPENROUTER_API_KEY` (jev). | +| `apiKeyEnv` | Name of the environment variable holding the key. Defaults to the provider's usual one: `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `OPENROUTER_API_KEY` (jev). | +| `apiKey` | The key itself. Prefer `apiKeyEnv`; keys do not belong in config files. | | `autoRun` | Evaluate semantic rules without `--semantic`. | -`SCTX_JUDGE_PROVIDER`, `SCTX_JUDGE_MODEL`, `SCTX_JUDGE_ENDPOINT` and `SCTX_JUDGE_API_KEY` override the corresponding config fields. Semantic evaluation sends node content to the configured provider. +To use several providers, name them under `providers` (each takes `type` plus `model`, `endpoint`, `apiKeyEnv`, `apiKey`) and pick the default with `use`: + +```json +{ + "judge": { + "providers": { + "decision": { "type": "jev" }, + "llm": { "type": "anthropic", "model": "claude-haiku-5-5" } + }, + "use": "llm" + } +} +``` + +A rule names one with `judge: "decision"`, and `--judge ` overrides every rule for a run. `use` is only needed when more than one provider is configured. Providers are created when first used, so one no rule reaches needs no credentials. The shorthand's provider is named `default`. Using the names `decision` (a decision model) and `llm` (a chat model) keeps shared schemas and configs portable. + +`SCTX_JUDGE_PROVIDER`, `SCTX_JUDGE_MODEL`, `SCTX_JUDGE_ENDPOINT` and `SCTX_JUDGE_API_KEY` override the shorthand (`default`) provider. Semantic evaluation sends node content to the configured provider. ## Markdown plugin config diff --git a/docs/rules.md b/docs/rules.md index 2cf1c19..9fef316 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -28,13 +28,14 @@ Rules are a flat array. |---|---|---| | `id` | yes | Unique rule identifier | | `category` | yes | `validation` \| `coherence` \| `workflow` \| `best-practice` | -| `severity` | no | `error` \| `warning` \| `info`; overrides the category default (see [Severity](#severity)) | +| `severity` | no | `error` \| `warning` \| `info`; overrides the category default (see [Severity](#severity)). Semantic rules may instead map verdicts to severities | | `description` | yes | Human-readable rule intent | | `check` | yes, unless `mode: semantic` | JSONata expression; must evaluate to `true` | +| `when` | no | JSONata applicability test, evaluated per node before the rule. The rule runs only where it returns `true`; elsewhere it is skipped and not reported. Not allowed with `scope: "global"` | | `mode` | no | `expression` (default) or `semantic` (judged by a model; see [Semantic rules](semantic-rules.md)) | | `rubric` | semantic only | Detailed prompt for the judge; falls back to `description` | -| `verdicts` | semantic only | Closed set of verdicts the judge must choose from: `{ "": "" }`. Default `pass` / `fail`. | -| `failOn` | semantic only | Verdicts that count as a violation. Default `["fail"]`; required when `verdicts` is declared. At least one verdict must pass. | +| `verdicts` | semantic only | Closed set of verdicts the judge must choose from. Default `pass` / `fail`. See [Semantic rules](semantic-rules.md#rule-fields) | +| `judge` | semantic only | Name of the configured judge provider to use; a `--judge` run override wins | | `type` | no | Restrict rule to nodes of this `resolvedType` | | `scope` | no | Use `"global"` to evaluate once for the whole space | | `override` | no | Only for merge conflicts: allows later duplicate `id` to replace earlier | @@ -116,6 +117,18 @@ Inside `nodes[...]`, bare names refer to each candidate node. Use `$$` to refere $count(nodes[resolvedParentTitle=$$.current.title and resolvedType='solution']) ``` +## Conditional rules (`when`) + +`when` is a JSONata test evaluated on each node before the rule, with the same context as `check`. The rule runs only where it returns `true`; on every other node it is skipped and not reported (it is neither passed nor gated). Use it when a rule applies to a subset of nodes that `type` cannot express: + +```json +{ "id": "ready-has-owner", "category": "workflow", "type": "task", + "description": "Ready tasks have an owner", + "when": "current.status = 'ready'", "check": "$exists(current.owner)" } +``` + +A `when` that errors or returns anything other than `true` means the rule does not apply. `when` needs a per-node rule, so it cannot be combined with `scope: "global"`. + ## Reusing rules To share rules, put them in a schema's `$metadata.rules` and list that schema's `$id` in `$metadata.imports`. An import brings the whole schema's metadata, including its own imports: diff --git a/docs/semantic-rules.md b/docs/semantic-rules.md index c8d29c7..ee87ce3 100644 --- a/docs/semantic-rules.md +++ b/docs/semantic-rules.md @@ -18,10 +18,13 @@ For expression rules, severity, gating and merge behaviour, see [Rules](rules.md "rubric": "The opportunity reads as a customer need or pain point, not a solution in disguise or a business metric.", "verdicts": { "customer-need": "Clearly a customer need or pain point", - "vague": "A need, but too vague to act on", + "vague": { + "description": "A need, but too vague to act on", + "guidance": "Say who has the need and what they are trying to do." + }, "solution": "A solution or business goal in disguise" }, - "failOn": ["solution"] + "severity": { "vague": "info", "solution": "warning" } } ``` @@ -37,25 +40,45 @@ For expression rules, severity, gating and merge behaviour, see [Rules](rules.md ## Rule fields -Semantic rules share `id`, `category`, `severity`, `type`, `description` and `gatesDescendants` with [expression rules](rules.md#rule-shape). They differ in these fields: +Semantic rules share `id`, `category`, `type`, `description`, `when` and `gatesDescendants` with [expression rules](rules.md#rule-shape). They differ in these fields: | Field | Required | Description | |---|---|---| | `mode` | yes | `"semantic"` (the default is `"expression"`) | | `rubric` | no | What the judge should assess. Falls back to `description` | -| `verdicts` | no | Map of verdict to description. At least two entries. Default: `pass` / `fail` | -| `failOn` | if `verdicts` is set | Verdicts that count as a violation. Default `["fail"]`. At least one verdict must pass | +| `verdicts` | no | The closed set of answers: `{ "": "" }`, at least two. Default `pass` / `fail` | +| `severity` | with `verdicts` | A severity for the default `fail` verdict, or a map from verdict to severity (see below) | +| `judge` | no | Name of the [judge provider](config.md#judge-semantic-rules) to use. Default: the configured default. A `--judge` run override replaces it | A semantic rule has no `check` and cannot be `scope: "global"`; it judges one node at a time. +### Verdicts, severity and guidance + The judge must choose exactly one verdict. A closed set keeps results comparable between runs and maps directly onto decision models, which return a choice with confidence and probabilities rather than prose. +- **A verdict is a classification, severity is the consequence.** `severity` as a map decides what each verdict means for the user: `error`, `warning` or `info`. A verdict left out of the map passes, and `ignore` states that explicitly: the rule ran and the verdict was recorded, but nothing is reported. At least one verdict must pass. +- **With the default `pass` / `fail` verdicts**, `severity` may be a single value (applied to `fail`) or omitted (the category default applies), like an expression rule. +- **A verdict's value may be a string or `{ "description", "guidance" }`.** The description tells the judge what the verdict means. The guidance is shown to the user when that verdict fires, so a failure arrives with what to do about it; it is never sent to the judge. +- **Gating follows the verdict.** A verdict mapped to `error` gates the node's subtree (unless `gatesDescendants: false`); one mapped to `warning` or `info` does not (unless `gatesDescendants: true`). + +### Running a rule only where it applies + +`when` is a JSONata test evaluated on each node before the rule. The rule runs only where it returns `true`, and a node where it does not is skipped silently (no judge call, nothing reported): + +```json +{ "id": "due-soon-coherent", "category": "coherence", "mode": "semantic", "type": "task", + "description": "A task due within a week has a believable plan", + "when": "current.status = 'ready' and $toMillis(current.due) - $millis() < 604800000" } +``` + +`when` works on expression rules too. It saves money on semantic rules because the cheap test runs first. + ## How judging works - **Order.** For each node, expression rules run first, then all of the node's semantic rules go to the judge in one call. Chat LLMs make one request per rule; decision models answer all of a node's rules in a single request. -- **Gating.** Semantic rules follow [cascade gating](rules.md#cascade-gating). They are not judged on gated nodes (recorded as `gatedRules`), and a failing semantic rule gates its subtree when `gatesDescendants` is true (the default for `error` severity). A judge *error* (network failure, unparseable output) is reported as a violation but never gates. -- **Input.** The judge sees the node's title, type, parent titles and its schema data (the fields the parser extracted for validation, including frontmatter). Wider scopes (ancestors, siblings) are not supported yet. -- **Output.** `--json` includes `semanticEvaluations` (verdict, `pass`, confidence and evidence per node and rule) and `gatedRules`. Violations carry the judge's `evidence`; decision models give a summary of the verdict distribution instead of prose. +- **Gating.** Semantic rules follow [cascade gating](rules.md#cascade-gating). They are not judged on gated nodes (recorded as `gatedRules`). A judge *error* (network failure, unparseable output) is reported as a violation but never gates. +- **Input.** The judge sees the node's title, type, parent titles and its schema data (the fields the parser extracted for validation, including frontmatter). +- **Output.** `--json` includes `semanticEvaluations` (verdict, `pass`, confidence and evidence per node and rule) and `gatedRules`. Violations carry the judge's `evidence`, the `verdict` and any `guidance`. ### Single-file validation @@ -63,11 +86,11 @@ The judge must choose exactly one verdict. A closed set keeps results comparable ### Filtering rules -`--rule-filter =[,]` (repeatable) evaluates only matching rules. Dimensions are `mode`, `category`, `severity` (effective severity), `type` and `id`; different dimensions combine with AND, values within one with OR. For example, `--semantic --rule-filter mode=semantic` runs only the judged rules, and `--rule-filter mode=expression` skips them entirely. Gating is computed from the rules that ran. +`--rule-filter =[,]` (repeatable) evaluates only matching rules. Dimensions are `mode`, `category`, `severity`, `type` and `id`; different dimensions combine with AND, values within one with OR. A rule with a verdict map matches a `severity` filter on any severity it can report. For example, `--semantic --rule-filter mode=semantic` runs only the judged rules, and `--rule-filter mode=expression` skips them entirely. Gating is computed from the rules that ran. ## Choosing a judge -| Provider | Use for | Notes | +| Provider type | Use for | Notes | |---|---|---| | `anthropic` | Claude models | Default model `claude-haiku-5-5` | | `openai` | Any OpenAI-compatible chat endpoint: OpenAI, Ollama, vLLM, Groq, OpenRouter chat models | Set `endpoint` for non-OpenAI servers; the key is optional for local ones | @@ -75,4 +98,6 @@ The judge must choose exactly one verdict. A closed set keeps results comparable Prefer a decision model when you can: results are repeatable, cheap, and come with a confidence. Use a chat LLM when the rubric needs open reasoning or evidence text. +Several providers can be configured under names, for example `decision` and `llm`. A rule picks one with `judge`, and a run can override every rule with `--judge ` (for instance a deeper second opinion on one file: `sctx validate-file F --rule-filter id=r --judge llm`). If a rule's choice and the run override disagree, the run wins. + Semantic evaluation sends node content to the configured provider. Check that is acceptable for the space before enabling it. diff --git a/plugin/.claude-plugin/plugin.json b/plugin/.claude-plugin/plugin.json index b113f3b..7af5727 100644 --- a/plugin/.claude-plugin/plugin.json +++ b/plugin/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "structured-context", - "version": "0.12.10", + "version": "0.12.11", "description": "Structured content authoring for Obsidian — automatic validation hooks, slash commands, and agent skills to keep markdown files compliant with your schema.", "homepage": "https://github.com/mindsocket/structured-context", "repository": "https://github.com/mindsocket/structured-context", diff --git a/plugin/skills/structured-context/references/commands.md b/plugin/skills/structured-context/references/commands.md index 2d29864..0cab9a9 100644 --- a/plugin/skills/structured-context/references/commands.md +++ b/plugin/skills/structured-context/references/commands.md @@ -11,7 +11,7 @@ Then use the `sctx` command for all operations. Always pass `--config ` wh ## validate ```bash -sctx validate [--watch] [--config ] [--depth ] [--include-gated] [--semantic|--no-semantic] [--rule-filter ] +sctx validate [--watch] [--config ] [--depth ] [--include-gated] [--semantic|--no-semantic] [--judge ] [--rule-filter ] ``` Validates all `.md` files in the space against the JSON schema. For each file: @@ -30,7 +30,7 @@ the top — useful for fixing a space level by level (fix the top, re-run with a `--depth`, repeat). `--include-gated` evaluates rules on gated nodes anyway, still reporting them as gated — use it to surface cheap bottom-up checks even when an upstream blocker fired. `--semantic` also runs `mode: semantic` rules through a configured model judge (costly; off by -default; `--no-semantic` forces it off even if config enables it), and `--rule-filter =` (repeatable; mode, category, severity, type, id) +default; `--no-semantic` forces it off even if config enables it; `--judge ` picks a configured judge provider), and `--rule-filter =` (repeatable; mode, category, severity, type, id) narrows which rules run. `validate-file` accepts the same two flags. Exit codes: `0` = clean (warning and info rule violations don't fail), `1` = validation errors found. diff --git a/plugin/skills/structured-context/references/schema-authoring.md b/plugin/skills/structured-context/references/schema-authoring.md index dc81393..ba2753f 100644 --- a/plugin/skills/structured-context/references/schema-authoring.md +++ b/plugin/skills/structured-context/references/schema-authoring.md @@ -66,7 +66,7 @@ Relationships define how sub-entities (nodes inside other files) are parsed and Rules are a flat array. Each rule's `category` (`validation`, `coherence`, `workflow`, `best-practice`) supplies its default severity: `validation` → `error`, `coherence` and `workflow` → `warning`, `best-practice` → `info`. Only `error` violations fail validation; warnings and info are reported only. An optional per-rule `severity` (`error` | `warning` | `info`) overrides the category default — prefer picking the right category over overriding. -Rules default to `mode: "expression"` (a JSONata `check`). Set `mode: "semantic"` for rules an LLM or decision model judges against a `rubric` (falls back to `description`): they take no `check` or `scope`, may declare `verdicts` (verdict → description, default `pass`/`fail`) with a required `failOn` list, and only run with `--semantic`. Run `sctx docs semantic` for the full semantic rule reference. An optional `gatesDescendants` controls whether a violation suppresses rule checks on the node's subtree (default: true for `error` severity). +Rules default to `mode: "expression"` (a JSONata `check`). Set `mode: "semantic"` for rules an LLM or decision model judges against a `rubric` (falls back to `description`): they take no `check` or `scope`, may declare `verdicts` (verdict → description, or `{ description, guidance }`; default `pass`/`fail`) with a `severity` map saying which verdicts are `error`/`warning`/`info`/`ignore`, and only run with `--semantic`. Run `sctx docs semantic` for the full semantic rule reference. A `when` JSONata test (any rule) limits a rule to the nodes where it returns true. An optional `gatesDescendants` controls whether a violation suppresses rule checks on the node's subtree (default: true for `error` severity). ## Metadata composition semantics diff --git a/schemas/generated/latest/_structured_context_schema_meta.json b/schemas/generated/latest/_structured_context_schema_meta.json index 8517107..7062ca5 100644 --- a/schemas/generated/latest/_structured_context_schema_meta.json +++ b/schemas/generated/latest/_structured_context_schema_meta.json @@ -197,7 +197,18 @@ "enum": ["validation", "coherence", "workflow", "best-practice"] }, "severity": { - "enum": ["error", "warning", "info"] + "oneOf": [ + { + "enum": ["error", "warning", "info"] + }, + { + "type": "object", + "additionalProperties": { + "enum": ["error", "warning", "info", "ignore"] + }, + "minProperties": 1 + } + ] }, "description": { "type": "string", @@ -207,6 +218,10 @@ "type": "string", "minLength": 1 }, + "when": { + "type": "string", + "minLength": 1 + }, "mode": { "enum": ["expression", "semantic"] }, @@ -217,19 +232,33 @@ "verdicts": { "type": "object", "additionalProperties": { - "type": "string", - "minLength": 1 + "oneOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "object", + "properties": { + "description": { + "type": "string", + "minLength": 1 + }, + "guidance": { + "type": "string", + "minLength": 1 + } + }, + "required": ["description"], + "additionalProperties": false + } + ] }, "minProperties": 2 }, - "failOn": { - "type": "array", - "items": { - "type": "string", - "minLength": 1 - }, - "minItems": 1, - "uniqueItems": true + "judge": { + "type": "string", + "minLength": 1 }, "type": { "type": "string", @@ -246,6 +275,9 @@ } }, "required": ["id", "category", "description"], + "not": { + "required": ["when", "scope"] + }, "if": { "properties": { "mode": { @@ -264,10 +296,53 @@ "required": ["scope"] } ] + }, + "if": { + "required": ["verdicts"] + }, + "then": { + "required": ["severity"], + "properties": { + "severity": { + "type": "object" + } + } + }, + "else": { + "properties": { + "severity": { + "oneOf": [ + { + "enum": ["error", "warning", "info"] + }, + { + "type": "object" + } + ] + } + } } }, "else": { - "required": ["check"] + "required": ["check"], + "not": { + "anyOf": [ + { + "required": ["verdicts"] + }, + { + "required": ["rubric"] + }, + { + "required": ["judge"] + } + ] + }, + "properties": { + "severity": { + "enum": ["error", "warning", "info"] + } + } }, "additionalProperties": false } diff --git a/src/api.ts b/src/api.ts index 5f5d501..b8dfe20 100644 --- a/src/api.ts +++ b/src/api.ts @@ -25,6 +25,7 @@ export { PLUGIN_PREFIX } from './constants'; export type { GatedRuleRecord, JudgeConfig, + JudgeProviderDefinition, SemanticEvaluationRecord, SemanticJudgeInput, SemanticJudgeProvider, @@ -34,11 +35,14 @@ export type { } from './judge'; export { AnthropicJudgeProvider, + createJudge, createJudgeProvider, FakeJudgeProvider, JevJudgeProvider, + JudgeConfigError, OpenAIJudgeProvider, PerRuleJudgeProvider, + RoutedJudgeProvider, } from './judge'; export { assembleSpaceGraph, loadSpaceGraph } from './load-space-graph'; export type { diff --git a/src/commands/schemas.ts b/src/commands/schemas.ts index 7fca269..8b44113 100644 --- a/src/commands/schemas.ts +++ b/src/commands/schemas.ts @@ -136,8 +136,7 @@ function showMetadata(metadata: SchemaMetadata): void { for (const [category, rules] of groups) { console.log(` ${category} (default severity: ${DEFAULT_SEVERITY_BY_CATEGORY[category]}):`); for (const rule of rules) { - const override = rule.severity ? ` [severity: ${rule.severity}]` : ''; - console.log(` ${rule.id}: ${rule.description}${override}`); + console.log(` ${rule.id}: ${rule.description}${ruleAnnotations(rule)}`); } } } @@ -227,6 +226,24 @@ function generateMermaidErd(metadata: SchemaMetadata, entities: EntityInfo[]): s return mmd; } +/** Bracketed notes on a rule: mode, severity (or verdict severities), judge and `when`. Empty for a plain rule. */ +function ruleAnnotations(rule: Rule): string { + const notes: string[] = []; + if (rule.mode === 'semantic') notes.push('semantic'); + if (typeof rule.severity === 'string') { + notes.push(`severity: ${rule.severity}`); + } else if (rule.severity) { + notes.push( + `severity: ${Object.entries(rule.severity) + .map(([verdict, sev]) => `${verdict}=${sev}`) + .join(', ')}`, + ); + } + if (rule.judge) notes.push(`judge: ${rule.judge}`); + if (rule.when) notes.push(`when: ${rule.when}`); + return notes.length > 0 ? ` [${notes.join('] [')}]` : ''; +} + export async function listSchemas(): Promise { const config = loadConfig(); diff --git a/src/commands/validate-file.ts b/src/commands/validate-file.ts index 61dc7f3..8c69c75 100644 --- a/src/commands/validate-file.ts +++ b/src/commands/validate-file.ts @@ -12,12 +12,13 @@ export { validateFile } from '../validate'; */ export async function validateFileCommand( filePath: string, - options: { json?: boolean; includeGated?: boolean; semantic?: boolean; ruleFilter?: string[] } = {}, + options: { json?: boolean; includeGated?: boolean; semantic?: boolean; judge?: string; ruleFilter?: string[] } = {}, ): Promise { const config = loadConfig(); const result = await validateFile(filePath, config, { includeGated: options.includeGated, semantic: options.semantic, + judge: options.judge, ruleFilter: options.ruleFilter && parseRuleFilter(options.ruleFilter), }); diff --git a/src/commands/validate.ts b/src/commands/validate.ts index 5f94534..3a71f82 100644 --- a/src/commands/validate.ts +++ b/src/commands/validate.ts @@ -6,7 +6,7 @@ import { resolveSpacePluginSchemas } from '../plugins/util'; import { parseRuleFilter } from '../schema/rule-filter'; import { bundledSchemasDir } from '../schema/schema'; import type { FileValidationIssue, RuleSeverity, SpaceContext } from '../types'; -import { formatErrors, validateSpace } from '../validate'; +import { formatErrors, ruleViolationMessage, validateSpace } from '../validate'; export async function validate( context: SpaceContext, @@ -15,6 +15,7 @@ export async function validate( depth?: number; includeGated?: boolean; semantic?: boolean; + judge?: string; ruleFilter?: string[]; judgeProvider?: SemanticJudgeProvider; } = {}, @@ -24,6 +25,7 @@ export async function validate( depth: options.depth, includeGated: options.includeGated, semantic: options.semantic, + judge: options.judge, ruleFilter: options.ruleFilter && parseRuleFilter(options.ruleFilter), judgeProvider: options.judgeProvider, }); @@ -61,10 +63,7 @@ export async function validate( } for (const v of result.ruleViolations) { if (v.file && v.severity === 'error') { - const message = v.evidence - ? `[${v.ruleId}] ${v.description} (Evidence: ${v.evidence})` - : `[${v.ruleId}] ${v.description}`; - addError(v.file, `rule:${v.ruleId}`, 'rule', message, v.severity); + addError(v.file, `rule:${v.ruleId}`, 'rule', ruleViolationMessage(v), v.severity); } } const globalRuleErrorCount = result.ruleViolations.filter((v) => !v.file && v.severity === 'error').length; @@ -215,6 +214,9 @@ export async function validate( if (v.evidence) { console.log(` Evidence: ${v.evidence}`); } + if (v.guidance) { + console.log(` Guidance: ${v.guidance}`); + } } } } diff --git a/src/config.ts b/src/config.ts index 8cec75d..daf75f9 100644 --- a/src/config.ts +++ b/src/config.ts @@ -13,13 +13,31 @@ import { } from './plugins/util'; import { bundledSchemasDir } from './schema/schema'; +const JUDGE_PROVIDER_TYPES = ['anthropic', 'openai', 'jev', 'fake', 'none']; + +const JUDGE_PROVIDER_SCHEMA = { + type: 'object', + properties: { + type: { type: 'string', enum: JUDGE_PROVIDER_TYPES }, + model: { type: 'string' }, + endpoint: { type: 'string' }, + apiKey: { type: 'string' }, + apiKeyEnv: { type: 'string' }, + }, + required: ['type'], + additionalProperties: false, +} as const; + const JUDGE_CONFIG_SCHEMA = { type: 'object', properties: { - provider: { type: 'string', enum: ['anthropic', 'openai', 'jev', 'fake'] }, + provider: { type: 'string', enum: JUDGE_PROVIDER_TYPES }, model: { type: 'string' }, apiKey: { type: 'string' }, + apiKeyEnv: { type: 'string' }, endpoint: { type: 'string' }, + providers: { type: 'object', additionalProperties: JUDGE_PROVIDER_SCHEMA }, + use: { type: 'string' }, autoRun: { type: 'boolean' }, }, additionalProperties: false, diff --git a/src/index.ts b/src/index.ts index 0140374..f84a17d 100755 --- a/src/index.ts +++ b/src/index.ts @@ -80,12 +80,14 @@ program ) .option('--semantic', 'Evaluate semantic rules with the configured judge') .option('--no-semantic', 'Skip semantic rules even if enabled in config') + .option('--judge ', 'Judge every semantic rule with this configured provider (implies --semantic)') .action(async (filePath, options) => { process.exitCode = await reportJudgeConfigErrors(() => validateFileCommand(filePath, { json: options.json, includeGated: options.includeGated, semantic: options.semantic, + judge: options.judge, ruleFilter: options.ruleFilter, }), ); @@ -113,6 +115,7 @@ program ) .option('--semantic', 'Evaluate semantic rules with the configured judge') .option('--no-semantic', 'Skip semantic rules even if enabled in config') + .option('--judge ', 'Judge every semantic rule with this configured provider (implies --semantic)') .action(async (spaceName, options) => { const context = buildSpaceContext(spaceName); if (options.watch) { diff --git a/src/judge/factory.ts b/src/judge/factory.ts index b602d8f..44eca1f 100644 --- a/src/judge/factory.ts +++ b/src/judge/factory.ts @@ -1,64 +1,88 @@ +import type { Rule } from '../types'; import { AnthropicJudgeProvider } from './provider/anthropic'; import { FakeJudgeProvider } from './provider/fake'; import { JevJudgeProvider } from './provider/jev'; import { OpenAIJudgeProvider } from './provider/openai'; -import type { JudgeConfig, SemanticJudgeProvider } from './types'; +import type { + JudgeConfig, + JudgeProviderDefinition, + SemanticJudgeInput, + SemanticJudgeProvider, + SemanticJudgeResult, +} from './types'; -/** Environment variable each provider reads its API key from when none is configured. */ -const PROVIDER_KEY_ENV = { +type Env = Record; + +/** Environment variable each provider type reads its API key from unless `apiKeyEnv` says otherwise. */ +const DEFAULT_KEY_ENV = { anthropic: 'ANTHROPIC_API_KEY', openai: 'OPENAI_API_KEY', jev: 'OPENROUTER_API_KEY', } as const; +const PROVIDER_TYPES = ['anthropic', 'openai', 'jev', 'fake'] as const; + +/** Name of the provider defined by the flat `judge: { provider, ... }` shorthand. */ +export const DEFAULT_JUDGE_NAME = 'default'; + /** The judge is misconfigured (no provider, unknown provider, missing key). Reported as a plain CLI error. */ export class JudgeConfigError extends Error {} -const PROVIDERS = ['anthropic', 'openai', 'jev', 'fake'] as const; +/** Every provider the config defines, by name, and the one used for rules that don't name any. */ +function providerDefinitions( + config: JudgeConfig | undefined, + env: Env, +): { definitions: Record; defaultName?: string } { + const definitions: Record = { ...config?.providers }; + + // The flat shorthand (and SCTX_JUDGE_* overrides) define one provider named "default". + const type = config?.provider || env.SCTX_JUDGE_PROVIDER; + if (type) { + definitions[DEFAULT_JUDGE_NAME] = { + type, + model: config?.model || env.SCTX_JUDGE_MODEL, + endpoint: config?.endpoint || env.SCTX_JUDGE_ENDPOINT, + apiKey: config?.apiKey || env.SCTX_JUDGE_API_KEY, + apiKeyEnv: config?.apiKeyEnv, + }; + } + + const names = Object.keys(definitions); + const defaultName = + config?.use ?? (DEFAULT_JUDGE_NAME in definitions ? DEFAULT_JUDGE_NAME : names.length === 1 ? names[0] : undefined); + return { definitions, defaultName }; +} /** - * Instantiate a semantic judge provider. - * - * The provider is always explicit: `judge.provider` in config or `SCTX_JUDGE_PROVIDER`. It is never - * inferred from whichever API keys happen to be in the environment, so a key set for another tool is - * never spent on judging unless the user chose that provider. - * - * Once a provider is chosen, its API key comes from `judge.apiKey`, `SCTX_JUDGE_API_KEY`, or the - * provider's usual variable (`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `OPENROUTER_API_KEY`). - * An explicitly passed `customProvider` wins over everything. + * Instantiate one provider from its definition. The type is always explicit and is never inferred from API keys + * in the environment, so a key set for another tool is never spent on judging. Once the type is chosen, its key + * comes from `apiKey`, else the environment variable named by `apiKeyEnv` (default: the type's usual variable). */ -export function createJudgeProvider( - config?: JudgeConfig, - env: Record = process.env, - customProvider?: SemanticJudgeProvider, -): SemanticJudgeProvider { - if (customProvider) return customProvider; - - const provider = config?.provider || env.SCTX_JUDGE_PROVIDER; - if (!provider) { +function createProvider(name: string, def: JudgeProviderDefinition, env: Env): SemanticJudgeProvider { + const type = def.type; + if (!type || type === 'none') { throw new JudgeConfigError( - `No semantic judge provider configured. Set "judge.provider" in config.json (one of: ${PROVIDERS.join(', ')}) or SCTX_JUDGE_PROVIDER.`, + `Judge "${name}" has no provider type. Set it to one of: ${PROVIDER_TYPES.join(', ')} (config.example.json shows a template).`, ); } - if (!(PROVIDERS as readonly string[]).includes(provider)) { - throw new JudgeConfigError(`Unknown semantic judge provider "${provider}". Use one of: ${PROVIDERS.join(', ')}.`); + if (!(PROVIDER_TYPES as readonly string[]).includes(type)) { + throw new JudgeConfigError( + `Unknown judge provider "${type}" for "${name}". Use one of: ${PROVIDER_TYPES.join(', ')}.`, + ); } + if (type === 'fake') return new FakeJudgeProvider(); - const model = config?.model || env.SCTX_JUDGE_MODEL; - const endpoint = config?.endpoint || env.SCTX_JUDGE_ENDPOINT; - if (provider === 'fake') return new FakeJudgeProvider(); - - const keyEnv = PROVIDER_KEY_ENV[provider as keyof typeof PROVIDER_KEY_ENV]; - const apiKey = config?.apiKey || env.SCTX_JUDGE_API_KEY || env[keyEnv]; + const keyEnv = def.apiKeyEnv ?? DEFAULT_KEY_ENV[type as keyof typeof DEFAULT_KEY_ENV]; + const apiKey = def.apiKey || env[keyEnv]; + const { model, endpoint } = def; // OpenAI-compatible endpoints (Ollama, vLLM, ...) may need no key. - if (!apiKey && provider !== 'openai') { + if (!apiKey && type !== 'openai') { throw new JudgeConfigError( - `Judge provider "${provider}" selected, but no API key was found. Set ${keyEnv} in the environment or "judge.apiKey" in config.json.`, + `Judge "${name}" (${type}) has no API key. Set ${keyEnv} in the environment, or "apiKey" / "apiKeyEnv" on the provider.`, ); } - - switch (provider) { + switch (type) { case 'anthropic': return new AnthropicJudgeProvider({ apiKey: apiKey as string, model, endpoint }); case 'jev': @@ -67,3 +91,92 @@ export function createJudgeProvider( return new OpenAIJudgeProvider({ apiKey, model, endpoint }); } } + +/** + * Routes each semantic rule to the judge provider it names (`judge` on the rule), else the configured default. + * A run-level override replaces every rule's choice. Providers are created on first use, so one that no rule + * reaches needs no credentials. + */ +export class RoutedJudgeProvider implements SemanticJudgeProvider { + readonly name = 'routed'; + private readonly cache = new Map(); + + constructor( + private readonly definitions: Record, + private readonly defaultName: string | undefined, + private readonly env: Env, + private readonly override?: string, + private readonly injected?: SemanticJudgeProvider, + ) {} + + /** The provider that judges a rule asking for `requested` (undefined: the default). */ + resolve(requested?: string): SemanticJudgeProvider { + if (this.injected) return this.injected; + const name = this.override ?? requested ?? this.defaultName; + if (!name) { + const known = Object.keys(this.definitions); + throw new JudgeConfigError( + known.length === 0 + ? `No semantic judge configured. Set "judge.provider" in config.json (one of: ${PROVIDER_TYPES.join(', ')}) or SCTX_JUDGE_PROVIDER.` + : `Several judge providers are configured (${known.join(', ')}). Set "judge.use" for the default, or name one on the rule ("judge") or with --judge.`, + ); + } + const def = this.definitions[name]; + if (!def) { + const known = Object.keys(this.definitions); + throw new JudgeConfigError( + `Judge "${name}" is not configured.${known.length ? ` Configured: ${known.join(', ')}.` : ' Add it under "judge.providers" in config.json.'}`, + ); + } + let provider = this.cache.get(name); + if (!provider) { + provider = createProvider(name, def, this.env); + this.cache.set(name, provider); + } + return provider; + } + + /** Fail now, before any request is made, if a rule's judge is not usable. */ + prepare(rules: Rule[]): void { + for (const rule of rules) { + if (rule.mode === 'semantic') this.resolve(rule.judge); + } + } + + async evaluate(input: SemanticJudgeInput): Promise> { + const groups = new Map(); + for (const rule of input.rules) { + const provider = this.resolve(rule.judge); + groups.set(provider, [...(groups.get(provider) ?? []), rule]); + } + const results: Record = {}; + for (const [provider, rules] of groups) { + Object.assign(results, await provider.evaluate({ node: input.node, rules })); + } + return results; + } +} + +/** + * Build the judge for a run from config and environment. `override` (the `--judge` flag) names the provider to use + * for every rule; `customProvider` (tests, injection) replaces configured providers altogether. + */ +export function createJudge( + config?: JudgeConfig, + env: Env = process.env, + customProvider?: SemanticJudgeProvider, + override?: string, +): RoutedJudgeProvider { + const { definitions, defaultName } = providerDefinitions(config, env); + return new RoutedJudgeProvider(definitions, defaultName, env, override, customProvider); +} + +/** The single provider for a config: the default one, or the one named by `name`. */ +export function createJudgeProvider( + config?: JudgeConfig, + env: Env = process.env, + customProvider?: SemanticJudgeProvider, + name?: string, +): SemanticJudgeProvider { + return createJudge(config, env, customProvider).resolve(name); +} diff --git a/src/judge/index.ts b/src/judge/index.ts index c4670a8..aae3c8a 100644 --- a/src/judge/index.ts +++ b/src/judge/index.ts @@ -1,4 +1,4 @@ -export { createJudgeProvider, JudgeConfigError } from './factory'; +export { createJudge, createJudgeProvider, JudgeConfigError, RoutedJudgeProvider } from './factory'; export { buildJudgeSystemPrompt, buildJudgeUserPrompt, parseJudgeResponse } from './prompt'; export type { AnthropicJudgeProviderOptions } from './provider/anthropic'; export { AnthropicJudgeProvider } from './provider/anthropic'; @@ -12,6 +12,7 @@ export { PerRuleJudgeProvider } from './provider/per-rule'; export type { GatedRuleRecord, JudgeConfig, + JudgeProviderDefinition, SemanticEvaluationRecord, SemanticJudgeInput, SemanticJudgeProvider, diff --git a/src/judge/judge-node.ts b/src/judge/judge-node.ts index 8c17af2..b1e69bc 100644 --- a/src/judge/judge-node.ts +++ b/src/judge/judge-node.ts @@ -1,11 +1,16 @@ -import type { Rule, SpaceNode } from '../types'; +import type { Rule, RuleSeverity, SpaceNode } from '../types'; import type { SemanticEvaluationRecord, SemanticJudgeProvider } from './types'; -import { isJudgeError, semanticRuleSpec } from './verdicts'; +import { isJudgeError, resolveSemanticRule } from './verdicts'; export type NodeJudgement = { rule: Rule; - /** True when the verdict is one of the rule's `failOn` verdicts, or the judge errored. */ + /** True when the verdict fails the rule (it has a severity), or the judge errored. */ failed: boolean; + /** Severity of the failing verdict; unset for a judge error, which takes the rule's own severity. */ + severity?: RuleSeverity; + verdict?: string; + /** User-facing guidance declared for the failing verdict. */ + guidance?: string; evidence?: string; /** Set when the judge could not produce a verdict for this rule. */ error?: string; @@ -20,7 +25,8 @@ export async function judgeNode( rules: Rule[], ): Promise { if (rules.length === 0) return []; - const specs = rules.map(semanticRuleSpec); + const resolved = rules.map(resolveSemanticRule); + const specs = resolved.map((r) => r.spec); let results: Awaited>; try { results = await judge.evaluate({ @@ -50,10 +56,14 @@ export async function judgeNode( const message = result ? result.error : 'Judge returned no result for this rule'; return { rule, failed: true, evidence: `Error during evaluation: ${message}`, error: message }; } - const failed = specs[i]!.failOn.includes(result.verdict); + const severity = resolved[i]!.severities[result.verdict]; + const failed = severity !== undefined; return { rule, failed, + severity, + verdict: result.verdict, + guidance: failed ? resolved[i]!.guidance[result.verdict] : undefined, evidence: result.evidence, record: { file: node.label, diff --git a/src/judge/types.ts b/src/judge/types.ts index ef62826..9abed8b 100644 --- a/src/judge/types.ts +++ b/src/judge/types.ts @@ -8,8 +8,10 @@ export interface SemanticRuleSpec { category: RuleCategory | string; /** Allowed verdicts, each with a description of when it applies. */ verdicts: Record; - /** Verdicts that count as a violation. */ + /** Verdicts that count as a violation (those with a severity; `ignore`d verdicts pass). */ failOn: string[]; + /** Name of the judge provider the rule asks for, if any. */ + judge?: string; } /** One node and every semantic rule to judge on it. Judges may answer all rules in a single call. */ @@ -45,12 +47,32 @@ export interface SemanticJudgeProvider { evaluate(input: SemanticJudgeInput): Promise>; } -/** Judge configuration from config file or options. */ +/** One judge provider: a backend type plus how to reach it. */ +export interface JudgeProviderDefinition { + /** `none` is an inert placeholder for templates; judging with it is an error. */ + type?: 'anthropic' | 'openai' | 'jev' | 'fake' | 'none' | string; + model?: string; + endpoint?: string; + /** Prefer `apiKeyEnv`: keys do not belong in config files. */ + apiKey?: string; + /** Environment variable holding the key. Defaults to the provider type's usual one (e.g. ANTHROPIC_API_KEY). */ + apiKeyEnv?: string; +} + +/** + * Judge configuration from config file or options. Either the flat shorthand (`provider`, `model`, ...), which + * defines one provider named "default", or several named `providers`. + */ export interface JudgeConfig { - provider?: 'anthropic' | 'openai' | 'jev' | 'fake' | string; + provider?: 'anthropic' | 'openai' | 'jev' | 'fake' | 'none' | string; model?: string; apiKey?: string; + apiKeyEnv?: string; endpoint?: string; + /** Named providers, for rules (or runs) that pick one by name. */ + providers?: Record; + /** Provider for rules that name none. Needed when several providers are configured. */ + use?: string; autoRun?: boolean; } @@ -59,7 +81,7 @@ export interface SemanticEvaluationRecord { file: string; ruleId: string; verdict: string; - /** True when the verdict is not one of the rule's `failOn` verdicts. */ + /** True when the verdict does not fail the rule (it has no severity, or is `ignore`d). */ pass: boolean; confidence?: number; evidence?: string; diff --git a/src/judge/verdicts.ts b/src/judge/verdicts.ts index 6e06c76..6df6122 100644 --- a/src/judge/verdicts.ts +++ b/src/judge/verdicts.ts @@ -1,4 +1,5 @@ -import type { Rule } from '../types'; +import { ruleSeverity } from '../schema/rule-severity'; +import type { Rule, RuleSeverity } from '../types'; import type { SemanticJudgeResult, SemanticRuleSpec } from './types'; /** Verdicts for a semantic rule that declares none. */ @@ -7,32 +8,77 @@ export const DEFAULT_VERDICTS: Record = { fail: 'The node does not meet the rubric.', }; -/** Build the judge-facing spec for a semantic rule, applying verdict defaults and checking `failOn`. */ -export function semanticRuleSpec(rule: Rule): SemanticRuleSpec { - const verdicts = rule.verdicts ?? DEFAULT_VERDICTS; - const failOn = rule.failOn ?? (rule.verdicts ? [] : ['fail']); - if (failOn.length === 0) { +/** A semantic rule resolved for evaluation: what the judge sees, and what each verdict means for the user. */ +export interface ResolvedSemanticRule { + /** What the judge sees: the closed verdict set. */ + spec: SemanticRuleSpec; + /** Severity of each verdict that fails the rule. A verdict absent here passes (including `ignore`d ones). */ + severities: Record; + /** User-facing guidance for verdicts that declare it. */ + guidance: Record; +} + +/** + * Resolve a semantic rule: apply verdict defaults, split each verdict into its judge description and user guidance, + * and work out which verdicts fail at what severity. Throws on an inconsistent rule. + */ +export function resolveSemanticRule(rule: Rule): ResolvedSemanticRule { + const declared = rule.verdicts ?? DEFAULT_VERDICTS; + const verdicts: Record = {}; + const guidance: Record = {}; + for (const [name, value] of Object.entries(declared)) { + if (typeof value === 'string') { + verdicts[name] = value; + } else { + verdicts[name] = value.description; + if (value.guidance) guidance[name] = value.guidance; + } + } + + let mapping: Record; + if (typeof rule.severity === 'object') { + mapping = rule.severity; + } else if (rule.verdicts) { throw new Error( - `Semantic rule "${rule.id}" declares verdicts but no failOn; list the verdicts that count as a violation.`, + `Semantic rule "${rule.id}" declares verdicts, so its severity must map the failing verdicts, e.g. { "${Object.keys(verdicts)[0]}": "warning" }.`, ); + } else { + mapping = { fail: ruleSeverity(rule) }; } - const unknown = failOn.filter((v) => !(v in verdicts)); + + const unknown = Object.keys(mapping).filter((v) => !(v in verdicts)); if (unknown.length > 0) { - throw new Error(`Semantic rule "${rule.id}" has failOn verdicts not in its verdicts: ${unknown.join(', ')}.`); + throw new Error(`Semantic rule "${rule.id}" has severity for verdicts not in its verdicts: ${unknown.join(', ')}.`); } + const severities: Record = {}; + for (const [verdict, severity] of Object.entries(mapping)) { + if (severity !== 'ignore') severities[verdict] = severity; + } + const failOn = Object.keys(severities); if (failOn.length >= Object.keys(verdicts).length) { - throw new Error(`Semantic rule "${rule.id}" has no passing verdict: every verdict is in failOn.`); + throw new Error(`Semantic rule "${rule.id}" has no passing verdict: every verdict has a severity.`); } + return { - id: rule.id, - description: rule.description, - rubric: rule.rubric, - category: rule.category, - verdicts, - failOn, + spec: { + id: rule.id, + description: rule.description, + rubric: rule.rubric, + category: rule.category, + judge: rule.judge, + verdicts, + failOn, + }, + severities, + guidance, }; } +/** The judge-facing spec for a semantic rule. */ +export function semanticRuleSpec(rule: Rule): SemanticRuleSpec { + return resolveSemanticRule(rule).spec; +} + /** Whether a judge result is an error rather than a verdict. */ export function isJudgeError(result: SemanticJudgeResult): result is { error: string } { return 'error' in result; diff --git a/src/schema/gating.ts b/src/schema/gating.ts index fd7a0f5..48b74d3 100644 --- a/src/schema/gating.ts +++ b/src/schema/gating.ts @@ -1,13 +1,21 @@ import { judgeNode } from '../judge/judge-node'; import type { GatedRuleRecord, SemanticEvaluationRecord, SemanticJudgeProvider } from '../judge/types'; import type { SpaceGraph } from '../space-graph'; -import type { Rule, RuleViolation, SpaceNode } from '../types'; +import type { Rule, RuleSeverity, RuleViolation, SpaceNode } from '../types'; import { buildEvalContext, evaluateExpression } from './evaluate-rule'; -import { ruleSeverity } from './validate-rules'; +import { ruleSeverities, ruleSeverity } from './rule-severity'; -/** Whether a violation of this rule gates its subtree: explicit `gatesDescendants`, else true only for `error` severity. */ -export function ruleGatesDescendants(rule: Rule): boolean { - return rule.gatesDescendants ?? ruleSeverity(rule) === 'error'; +/** + * Whether a violation of this rule gates its subtree: explicit `gatesDescendants`, else true only for `error` severity. + * `severity` is that of the violation; a semantic rule's verdicts can differ. + */ +export function ruleGatesDescendants(rule: Rule, severity: RuleSeverity = ruleSeverity(rule)): boolean { + return rule.gatesDescendants ?? severity === 'error'; +} + +/** Whether any violation of this rule could gate its subtree. */ +function ruleCanGate(rule: Rule): boolean { + return rule.gatesDescendants ?? ruleSeverities(rule).includes('error'); } /** @@ -174,30 +182,42 @@ export async function evaluateRulesWithGating( } } - function applicableRules(node: SpaceNode): Rule[] { - return perNodeRules.filter((r) => { - if (r.type && r.type !== node.resolvedType) return false; - if (r.mode !== 'semantic' || options.judgeTarget === undefined || node.label === options.judgeTarget) return true; - // Ancestors only matter through the gates they could raise. - return targetAncestors.has(node.title) && ruleGatesDescendants(r); - }); + /** Rules that apply to this node: matching `type`, passing their `when` test, and (for a judge target) worth judging. */ + async function applicableRules(node: SpaceNode): Promise { + const applicable: Rule[] = []; + for (const r of perNodeRules) { + if (r.type && r.type !== node.resolvedType) continue; + if (r.mode === 'semantic' && options.judgeTarget !== undefined && node.label !== options.judgeTarget) { + // Ancestors only matter through the gates they could raise. + if (!(targetAncestors.has(node.title) && ruleCanGate(r))) continue; + } + if (r.when && (await evaluateExpression(r.when, buildEvalContext(node, nodes, nodeIndex))) !== true) continue; + applicable.push(r); + } + return applicable; } async function evaluateApplicableRules(node: SpaceNode): Promise<{ blockingRuleId?: string }> { let blockingRuleId: string | undefined; - const applicable = applicableRules(node); + const applicable = await applicableRules(node); - function report(rule: Rule, evidence?: string, error?: string): void { + function report( + rule: Rule, + detail: { severity?: RuleSeverity; verdict?: string; guidance?: string; evidence?: string; error?: string } = {}, + ): void { + const severity = detail.severity ?? ruleSeverity(rule); violations.push({ file: node.label, ruleId: rule.id, category: rule.category, - severity: ruleSeverity(rule), - description: error ? `${rule.description} (Judge error: ${error})` : rule.description, - ...(evidence !== undefined && { evidence }), + severity, + description: detail.error ? `${rule.description} (Judge error: ${detail.error})` : rule.description, + ...(detail.evidence !== undefined && { evidence: detail.evidence }), + ...(detail.verdict !== undefined && { verdict: detail.verdict }), + ...(detail.guidance !== undefined && { guidance: detail.guidance }), }); // A judge error says nothing about the node, so it never gates. - if (!error && ruleGatesDescendants(rule) && !blockingRuleId) blockingRuleId = rule.id; + if (!detail.error && ruleGatesDescendants(rule, severity) && !blockingRuleId) blockingRuleId = rule.id; } // Cheap deterministic rules first; the judge is only consulted afterwards, once per node. @@ -210,14 +230,22 @@ export async function evaluateRulesWithGating( if (judge && semantic.length > 0) { for (const j of await judgeNode(judge, node, semantic)) { if (j.record) semanticEvaluations.push(j.record); - if (j.failed) report(j.rule, j.evidence, j.error); + if (j.failed) { + report(j.rule, { + severity: j.severity, + verdict: j.verdict, + guidance: j.guidance, + evidence: j.evidence, + error: j.error, + }); + } } } return { blockingRuleId }; } - function recordGatedSemantic(node: SpaceNode, reason: string): void { - for (const rule of applicableRules(node)) { + async function recordGatedSemantic(node: SpaceNode, reason: string): Promise { + for (const rule of await applicableRules(node)) { if (rule.mode === 'semantic') gatedRules.push({ file: node.label, ruleId: rule.id, reason }); } } @@ -247,7 +275,7 @@ export async function evaluateRulesWithGating( const { blockingRuleId } = shouldEvaluate ? await evaluateApplicableRules(node) : {}; if (!shouldEvaluate) { - recordGatedSemantic(node, isGatedByAncestor ? 'Gated by an ancestor failure' : 'Gated by failure on node'); + await recordGatedSemantic(node, isGatedByAncestor ? 'Gated by an ancestor failure' : 'Gated by failure on node'); } let childOrigin: string | undefined = ancestorOrigin; diff --git a/src/schema/metadata-contract.ts b/src/schema/metadata-contract.ts index b22625c..ebb7030 100644 --- a/src/schema/metadata-contract.ts +++ b/src/schema/metadata-contract.ts @@ -55,20 +55,55 @@ const HIERARCHY_LEVEL_SCHEMA = { additionalProperties: false, } as const; +const SEVERITY_SCHEMA = { enum: ['error', 'warning', 'info'] } as const; + const RULE_SCHEMA = { type: 'object', properties: { id: { type: 'string', minLength: 1 }, category: { enum: ['validation', 'coherence', 'workflow', 'best-practice'] }, - severity: { enum: ['error', 'warning', 'info'] }, + /** + * A severity, or (semantic rules only) a map of verdict to severity. In a map, a verdict mapped to + * `ignore` is judged but not reported, and a verdict left out passes. Defaults from the category. + */ + severity: { + oneOf: [ + SEVERITY_SCHEMA, + { + type: 'object', + additionalProperties: { enum: ['error', 'warning', 'info', 'ignore'] }, + minProperties: 1, + }, + ], + }, description: { type: 'string', minLength: 1 }, check: { type: 'string', minLength: 1 }, + /** Optional JSONata applicability test, evaluated per node before the rule runs. The rule applies only when it returns true. */ + when: { type: 'string', minLength: 1 }, mode: { enum: ['expression', 'semantic'] }, rubric: { type: 'string', minLength: 1 }, - /** Semantic rules: the closed set of verdicts a judge may return, each with a description. Defaults to `pass`/`fail`. */ - verdicts: { type: 'object', additionalProperties: { type: 'string', minLength: 1 }, minProperties: 2 }, - /** Semantic rules: which verdicts count as a violation. Defaults to `fail`; every other verdict passes. */ - failOn: { type: 'array', items: { type: 'string', minLength: 1 }, minItems: 1, uniqueItems: true }, + /** + * Semantic rules: the closed set of verdicts a judge may return. Each is a description (what the verdict + * means, for the judge) or `{ description, guidance }`, where `guidance` is shown to the user when it fires. + * Defaults to `pass`/`fail`. + */ + verdicts: { + type: 'object', + additionalProperties: { + oneOf: [ + { type: 'string', minLength: 1 }, + { + type: 'object', + properties: { description: { type: 'string', minLength: 1 }, guidance: { type: 'string', minLength: 1 } }, + required: ['description'], + additionalProperties: false, + }, + ], + }, + minProperties: 2, + }, + /** Semantic rules: name of the configured judge provider to use. A run-level override takes precedence. */ + judge: { type: 'string', minLength: 1 }, type: { type: 'string', minLength: 1 }, scope: { enum: ['global'] }, override: { type: 'boolean' }, @@ -76,11 +111,24 @@ const RULE_SCHEMA = { gatesDescendants: { type: 'boolean' }, }, required: ['id', 'category', 'description'], - // Semantic rules are judged per node, so they take no `check` and cannot be global; every other rule needs a `check`. + // A global rule has no node to test, so `when` needs a per-node rule. + not: { required: ['when', 'scope'] }, + // Semantic rules are judged per node (no `check`, not global) and with custom verdicts must map each failing + // verdict to a severity. Every other rule needs a `check` and takes none of the semantic-only fields. if: { properties: { mode: { const: 'semantic' } }, required: ['mode'] }, // biome-ignore lint/suspicious/noThenProperty: JSON Schema `if`/`then`/`else` keyword, not a thenable - then: { not: { anyOf: [{ required: ['check'] }, { required: ['scope'] }] } }, - else: { required: ['check'] }, + then: { + not: { anyOf: [{ required: ['check'] }, { required: ['scope'] }] }, + if: { required: ['verdicts'] }, + // biome-ignore lint/suspicious/noThenProperty: JSON Schema `if`/`then`/`else` keyword, not a thenable + then: { required: ['severity'], properties: { severity: { type: 'object' } } }, + else: { properties: { severity: { oneOf: [SEVERITY_SCHEMA, { type: 'object' }] } } }, + }, + else: { + required: ['check'], + not: { anyOf: [{ required: ['verdicts'] }, { required: ['rubric'] }, { required: ['judge'] }] }, + properties: { severity: SEVERITY_SCHEMA }, + }, additionalProperties: false, } as const; diff --git a/src/schema/rule-filter.ts b/src/schema/rule-filter.ts index 7d58423..4393e8a 100644 --- a/src/schema/rule-filter.ts +++ b/src/schema/rule-filter.ts @@ -1,5 +1,5 @@ import type { Rule } from '../types'; -import { ruleSeverity } from './validate-rules'; +import { ruleSeverities } from './rule-severity'; export const RULE_FILTER_DIMENSIONS = ['mode', 'category', 'severity', 'type', 'id'] as const; export type RuleFilterDimension = (typeof RULE_FILTER_DIMENSIONS)[number]; @@ -26,30 +26,32 @@ export function parseRuleFilter(specs: string[]): RuleFilter { return filter; } -function dimensionValue(rule: Rule, dimension: RuleFilterDimension): string | undefined { +function dimensionValues(rule: Rule, dimension: RuleFilterDimension): string[] { switch (dimension) { case 'mode': - return rule.mode ?? 'expression'; + return [rule.mode ?? 'expression']; case 'severity': - return ruleSeverity(rule); + return ruleSeverities(rule); case 'category': - return rule.category; + return [rule.category]; case 'type': - return rule.type; + return rule.type ? [rule.type] : []; case 'id': - return rule.id; + return [rule.id]; } } -/** Keep the rules matching every dimension of the filter. A rule with no `type` never matches a `type` filter. */ +/** + * Keep the rules matching every dimension of the filter. A rule with no `type` never matches a `type` filter, + * and a rule with a verdict map matches a `severity` filter on any severity it can report. + */ export function filterRules(rules: Rule[], filter?: RuleFilter): Rule[] { if (!filter) return rules; return rules.filter((rule) => RULE_FILTER_DIMENSIONS.every((dim) => { const wanted = filter[dim]; if (!wanted) return true; - const value = dimensionValue(rule, dim); - return value !== undefined && wanted.includes(value); + return dimensionValues(rule, dim).some((value) => wanted.includes(value)); }), ); } diff --git a/src/schema/rule-severity.ts b/src/schema/rule-severity.ts new file mode 100644 index 0000000..969fd30 --- /dev/null +++ b/src/schema/rule-severity.ts @@ -0,0 +1,24 @@ +import type { Rule, RuleCategory, RuleSeverity } from '../types'; + +/** Default severity for each rule category. A rule's own `severity` field overrides this. */ +export const DEFAULT_SEVERITY_BY_CATEGORY: Record = { + validation: 'error', + coherence: 'warning', + workflow: 'warning', + 'best-practice': 'info', +}; + +const SEVERITY_ORDER: RuleSeverity[] = ['error', 'warning', 'info']; + +/** Every severity this rule can report. A verdict map lists several; `ignore` entries report nothing. */ +export function ruleSeverities(rule: Rule): RuleSeverity[] { + if (rule.severity === undefined) return [DEFAULT_SEVERITY_BY_CATEGORY[rule.category]]; + if (typeof rule.severity === 'string') return [rule.severity]; + const mapped = new Set(Object.values(rule.severity).filter((s): s is RuleSeverity => s !== 'ignore')); + return SEVERITY_ORDER.filter((s) => mapped.has(s)); +} + +/** Effective severity of a rule: its explicit `severity`, else its category default. With a verdict map, the most severe verdict. */ +export function ruleSeverity(rule: Rule): RuleSeverity { + return ruleSeverities(rule)[0] ?? 'info'; +} diff --git a/src/schema/validate-rules.ts b/src/schema/validate-rules.ts index cf504dc..586cc02 100644 --- a/src/schema/validate-rules.ts +++ b/src/schema/validate-rules.ts @@ -1,20 +1,10 @@ import { judgeNode } from '../judge/judge-node'; import type { GatedRuleRecord, SemanticEvaluationRecord, SemanticJudgeProvider } from '../judge/types'; -import type { Rule, RuleCategory, RuleSeverity, RuleViolation, SpaceNode } from '../types'; +import type { Rule, RuleViolation, SpaceNode } from '../types'; import { buildEvalContext, evaluateExpression } from './evaluate-rule'; +import { ruleSeverity } from './rule-severity'; -/** Default severity for each rule category. A rule's own `severity` field overrides this. */ -export const DEFAULT_SEVERITY_BY_CATEGORY: Record = { - validation: 'error', - coherence: 'warning', - workflow: 'warning', - 'best-practice': 'info', -}; - -/** Effective severity of a rule: its explicit `severity`, else its category default. */ -export function ruleSeverity(rule: Rule): RuleSeverity { - return rule.severity ?? DEFAULT_SEVERITY_BY_CATEGORY[rule.category]; -} +export { DEFAULT_SEVERITY_BY_CATEGORY, ruleSeverity } from './rule-severity'; export interface ValidateRulesOptions { /** Whether to run semantic rules (default: false). */ @@ -100,6 +90,7 @@ export async function validateRules( const targetNodes = rule.type ? nodes.filter((n) => n.resolvedType === rule.type) : nodes; for (const node of targetNodes) { const context = buildEvalContext(node, nodes, nodeIndex); + if (rule.when && (await evaluateExpression(rule.when, context)) !== true) continue; const result = await evaluateExpression(check, context); if (result !== true) { violations.push({ @@ -124,7 +115,12 @@ export async function validateRules( for (const node of nodes) { if (options.judgeTarget !== undefined && node.label !== options.judgeTarget) continue; - const applicable = semanticRules.filter((r) => !r.type || r.type === node.resolvedType); + const applicable: Rule[] = []; + for (const r of semanticRules) { + if (r.type && r.type !== node.resolvedType) continue; + if (r.when && (await evaluateExpression(r.when, buildEvalContext(node, nodes, nodeIndex))) !== true) continue; + applicable.push(r); + } if (applicable.length === 0) continue; // Safe cascading default: gate semantic evaluation if node failed upstream checks @@ -142,9 +138,11 @@ export async function validateRules( file: node.label, ruleId: j.rule.id, category: j.rule.category, - severity: ruleSeverity(j.rule), + severity: j.severity ?? ruleSeverity(j.rule), description: j.error ? `${j.rule.description} (Judge error: ${j.error})` : j.rule.description, evidence: j.evidence, + ...(j.verdict !== undefined && { verdict: j.verdict }), + ...(j.guidance !== undefined && { guidance: j.guidance }), }); if (!j.error) failedNodeLabels.add(node.label); } diff --git a/src/types.ts b/src/types.ts index 523a62f..03fc923 100644 --- a/src/types.ts +++ b/src/types.ts @@ -203,6 +203,10 @@ export type RuleViolation = { description: string; /** Cited evidence or reasoning from evaluation (for semantic judge rules). */ evidence?: string; + /** The verdict that failed the rule (semantic rules). */ + verdict?: string; + /** User-facing guidance declared for that verdict (semantic rules). */ + guidance?: string; }; export type GraphViolation = { diff --git a/src/validate.ts b/src/validate.ts index 803a64c..3ca3e55 100644 --- a/src/validate.ts +++ b/src/validate.ts @@ -1,7 +1,7 @@ import { isAbsolute, relative, resolve } from 'node:path'; import type { ErrorObject } from 'ajv'; import type { Config, SpaceConfig } from './config'; -import { createJudgeProvider } from './judge/factory'; +import { createJudge, RoutedJudgeProvider } from './judge/factory'; import type { GatedRuleRecord, JudgeConfig, SemanticEvaluationRecord, SemanticJudgeProvider } from './judge/types'; import { readSpace } from './read/read-space'; import { computeHierarchyDepths, evaluateRulesWithGating, type GatingSeed } from './schema/gating'; @@ -176,6 +176,8 @@ export interface ValidateSpaceOptions { ruleFilter?: RuleFilter; /** Whether to evaluate semantic rules (default: false, or true if configured in space/config). */ semantic?: boolean; + /** Name of the configured judge provider to use for every semantic rule, overriding the rules' own. Implies `semantic`. */ + judge?: string; /** Optional custom judge provider (for testing or injection). */ judgeProvider?: SemanticJudgeProvider; } @@ -189,11 +191,22 @@ function resolveJudge( injected: SemanticJudgeProvider | undefined, space: SpaceConfig, config: Config, + judgeName?: string, ): SemanticJudgeProvider | undefined { + // Naming a judge on the command line asks for semantic evaluation unless it is switched off explicitly. const enabled = - requested ?? space.semantic ?? space.judge?.autoRun ?? config.semantic ?? config.judge?.autoRun ?? false; + requested ?? + (judgeName !== undefined || + (space.semantic ?? space.judge?.autoRun ?? config.semantic ?? config.judge?.autoRun ?? false)); if (!enabled) return undefined; - return injected ?? createJudgeProvider((space.judge ?? config.judge) as JudgeConfig | undefined); + return createJudge((space.judge ?? config.judge) as JudgeConfig | undefined, process.env, injected, judgeName); +} + +/** One-line text for a rule violation: rule, description, and any judge evidence and verdict guidance. */ +export function ruleViolationMessage(v: RuleViolation): string { + const evidence = v.evidence ? ` (Evidence: ${v.evidence})` : ''; + const guidance = v.guidance ? ` Guidance: ${v.guidance}` : ''; + return `[${v.ruleId}] ${v.description}${evidence}${guidance}`; } type SpaceRuleEvaluation = { @@ -222,6 +235,8 @@ async function evaluateSpaceRules( ): Promise { const rules = metadata.rules && filterRules(metadata.rules, options.ruleFilter); if (!rules) return { violations: [], gates: [], semanticEvaluations: [], gatedRules: [] }; + // Misconfigured judges fail before any request is made. + if (options.judge instanceof RoutedJudgeProvider) options.judge.prepare(rules); if (!metadata.hierarchy) { const result = await validateRules(nodes, rules, { semantic: options.judge !== undefined, @@ -362,7 +377,7 @@ export async function validateSpace( ), ...result.hierarchyViolations.map((v): GatingSeed => ({ file: v.file, title: v.nodeTitle, reason: 'hierarchy' })), ]; - const judge = resolveJudge(options.semantic, options.judgeProvider, context.space, context.config); + const judge = resolveJudge(options.semantic, options.judgeProvider, context.space, context.config, options.judge); const { violations, gates, semanticEvaluations, gatedRules } = await evaluateSpaceRules(nodes, metadata, seeds, { includeGated: options.includeGated, ruleFilter: options.ruleFilter, @@ -416,6 +431,8 @@ export interface ValidateFileOptions { ruleFilter?: RuleFilter; /** Whether to evaluate semantic rules (default: false, or true if configured in space/config). */ semantic?: boolean; + /** Name of the configured judge provider to use for every semantic rule. Implies `semantic`. */ + judge?: string; /** Optional custom judge provider (for testing or injection). */ judgeProvider?: SemanticJudgeProvider; } @@ -541,7 +558,7 @@ export async function validateFile( (v): GatingSeed => ({ file: v.file, title: v.nodeTitle, reason: 'hierarchy' }), ), ]; - const judge = resolveJudge(options.semantic, options.judgeProvider, space, config); + const judge = resolveJudge(options.semantic, options.judgeProvider, space, config, options.judge); const { violations, gates } = await evaluateSpaceRules(nodes, metadata, seeds, { includeGated: options.includeGated, ruleFilter: options.ruleFilter, @@ -553,9 +570,7 @@ export async function validateFile( const target = v.severity === 'error' ? errors : warnings; target[`rule:${v.ruleId}`] = { kind: 'rule', - message: v.evidence - ? `[${v.ruleId}] ${v.description} (Evidence: ${v.evidence})` - : `[${v.ruleId}] ${v.description}`, + message: ruleViolationMessage(v), severity: v.severity, }; } diff --git a/tests/helpers/temp-space.ts b/tests/helpers/temp-space.ts new file mode 100644 index 0000000..263b105 --- /dev/null +++ b/tests/helpers/temp-space.ts @@ -0,0 +1,58 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import type { Config } from '../../src/config'; + +const META = + 'https://raw.githubusercontent.com/mindsocket/structured-context/main/schemas/generated/latest/_structured_context_schema_meta.json'; + +const typeSchema = (type: string, parent: boolean) => ({ + type: 'object', + properties: { type: { const: type }, title: { type: 'string' }, ...(parent && { parent: { type: 'string' } }) }, + required: ['type', 'title'], +}); + +export interface TempSpace { + dir: string; + config: Config; + /** Absolute path of a file in the space. */ + file: (name: string) => string; + cleanup: () => void; +} + +/** + * A throwaway goal -> task -> step space (files G, T, S, plus an unrelated goal Other) with the given rules. + * Each file may add frontmatter via `frontmatter`, e.g. `{ T: 'status: ready' }`. + */ +export function makeTempSpace( + rules: object[], + options: { frontmatter?: Record; judge?: Config['judge'] } = {}, +): TempSpace { + const dir = mkdtempSync(join(tmpdir(), 'sctx-temp-space-')); + const schemaPath = join(dir, 'schema.json'); + writeFileSync( + schemaPath, + JSON.stringify({ + $schema: META, + $id: 'sctx://temp-space.test', + $metadata: { hierarchy: { levels: ['goal', 'task', 'step'] }, rules }, + oneOf: [typeSchema('goal', false), typeSchema('task', true), typeSchema('step', true)], + }), + ); + const space = join(dir, 'space'); + mkdirSync(space); + const extra = (name: string) => (options.frontmatter?.[name] ? `${options.frontmatter[name]}\n` : ''); + writeFileSync(join(space, 'G.md'), `---\ntype: goal\n${extra('G')}---\n`); + writeFileSync(join(space, 'T.md'), `---\ntype: task\nparent: "[[G]]"\n${extra('T')}---\n`); + writeFileSync(join(space, 'S.md'), `---\ntype: step\nparent: "[[T]]"\n${extra('S')}---\n`); + writeFileSync(join(space, 'Other.md'), `---\ntype: goal\n${extra('Other')}---\n`); + return { + dir, + config: { + spaces: [{ name: 'ts', path: space, schema: schemaPath }], + ...(options.judge && { judge: options.judge }), + }, + file: (name) => join(space, name), + cleanup: () => rmSync(dir, { recursive: true, force: true }), + }; +} diff --git a/tests/judge/factory.test.ts b/tests/judge/factory.test.ts index fb9b4c1..5a5816c 100644 --- a/tests/judge/factory.test.ts +++ b/tests/judge/factory.test.ts @@ -1,9 +1,11 @@ import { describe, expect, it } from 'bun:test'; -import { createJudgeProvider, JudgeConfigError } from '../../src/judge/factory'; +import { createJudge, createJudgeProvider, JudgeConfigError } from '../../src/judge/factory'; import { AnthropicJudgeProvider } from '../../src/judge/provider/anthropic'; import { FakeJudgeProvider } from '../../src/judge/provider/fake'; import { JevJudgeProvider } from '../../src/judge/provider/jev'; import { OpenAIJudgeProvider } from '../../src/judge/provider/openai'; +import type { Rule } from '../../src/types'; +import { graded, node, passFailRule } from './helpers'; describe('createJudgeProvider factory', () => { it('returns a custom provider when passed directly', () => { @@ -12,8 +14,7 @@ describe('createJudgeProvider factory', () => { }); it('creates FakeJudgeProvider when provider is "fake"', () => { - const provider = createJudgeProvider({ provider: 'fake' }, {}); - expect(provider instanceof FakeJudgeProvider).toBe(true); + expect(createJudgeProvider({ provider: 'fake' }, {}) instanceof FakeJudgeProvider).toBe(true); }); it.each([ @@ -29,6 +30,12 @@ describe('createJudgeProvider factory', () => { expect(provider.name).toBe('anthropic'); }); + it('reads the key from the variable named by apiKeyEnv', () => { + const config = { provider: 'jev', apiKeyEnv: 'MY_JEV_KEY' }; + expect(createJudgeProvider(config, { MY_JEV_KEY: 'k' }) instanceof JevJudgeProvider).toBe(true); + expect(() => createJudgeProvider(config, { OPENROUTER_API_KEY: 'k' })).toThrow('MY_JEV_KEY'); + }); + it('never infers a provider from API keys in the environment', () => { const env = { ANTHROPIC_API_KEY: 'a', @@ -37,7 +44,7 @@ describe('createJudgeProvider factory', () => { OPENAI_BASE_URL: 'http://x/v1', }; expect(() => createJudgeProvider(undefined, env)).toThrow(JudgeConfigError); - expect(() => createJudgeProvider({ model: 'm' }, env)).toThrow('No semantic judge provider configured'); + expect(() => createJudgeProvider({ model: 'm' }, env)).toThrow('No semantic judge configured'); }); it("does not borrow another provider's key", () => { @@ -50,7 +57,68 @@ describe('createJudgeProvider factory', () => { expect(provider instanceof OpenAIJudgeProvider).toBe(true); }); - it('rejects an unknown provider', () => { - expect(() => createJudgeProvider({ provider: 'gemini' as never }, {})).toThrow('Unknown semantic judge provider'); + it('treats provider "none" as an inert placeholder and rejects unknown providers', () => { + expect(() => createJudgeProvider({ provider: 'none' }, {})).toThrow('no provider type'); + expect(() => createJudgeProvider({ provider: 'gemini' as never }, {})).toThrow('Unknown judge provider'); + }); +}); + +describe('named providers and routing', () => { + const config = { + providers: { decision: { type: 'jev' }, llm: { type: 'anthropic', model: 'claude-haiku-5-5' } }, + use: 'llm', + }; + const env = { OPENROUTER_API_KEY: 'r', ANTHROPIC_API_KEY: 'a' }; + + it('uses the default provider unless the rule names one', () => { + const judge = createJudge(config, env); + expect(judge.resolve().name).toBe('anthropic'); + expect(judge.resolve('decision').name).toBe('jev'); + }); + + it('lets a run-level override replace every rule choice', () => { + expect(createJudge(config, env, undefined, 'decision').resolve('llm').name).toBe('jev'); + }); + + it('needs `use` when several providers are configured and a rule names none', () => { + expect(() => createJudge({ providers: config.providers }, env).resolve()).toThrow('judge.use'); + }); + + it('uses the only configured provider as the default', () => { + expect(createJudge({ providers: { llm: { type: 'anthropic' } } }, env).resolve().name).toBe('anthropic'); + }); + + it('rejects a name that is not configured, listing what is', () => { + expect(() => createJudge(config, env).resolve('nope')).toThrow('Configured: decision, llm'); + }); + + it('creates providers lazily, so an unused one needs no key', () => { + const judge = createJudge(config, { ANTHROPIC_API_KEY: 'a' }); + expect(judge.resolve().name).toBe('anthropic'); + expect(() => judge.resolve('decision')).toThrow('OPENROUTER_API_KEY'); + }); + + it('prepare() fails before any request when a rule names an unusable judge', () => { + const rule = { id: 'r', category: 'best-practice', description: 'd', mode: 'semantic', judge: 'decision' } as Rule; + expect(() => createJudge(config, { ANTHROPIC_API_KEY: 'a' }).prepare([rule])).toThrow(JudgeConfigError); + expect(() => createJudge(config, env).prepare([rule])).not.toThrow(); + }); + + it('sends each rule to the provider it names, in one call per provider', async () => { + const a = new FakeJudgeProvider({ 'customer-voice': { verdict: 'pass' } }); + const b = new FakeJudgeProvider({ framing: { verdict: 'good' } }); + const judge = createJudge({ provider: 'fake' }, {}); + // Swap in recording providers for the two names. + const routed = Object.assign(judge, { + resolve: (name?: string) => (name === 'b' ? b : a), + }); + const results = await routed.evaluate({ + node, + rules: [{ ...passFailRule }, { ...graded, judge: 'b' }], + }); + expect(results['customer-voice']).toEqual({ verdict: 'pass' }); + expect(results.framing).toEqual({ verdict: 'good' }); + expect(a.callHistory.map((c) => c.rules.map((r) => r.id))).toEqual([['customer-voice']]); + expect(b.callHistory.map((c) => c.rules.map((r) => r.id))).toEqual([['framing']]); }); }); diff --git a/tests/judge/provider/jev.test.ts b/tests/judge/provider/jev.test.ts index 844712e..bc450ae 100644 --- a/tests/judge/provider/jev.test.ts +++ b/tests/judge/provider/jev.test.ts @@ -72,6 +72,6 @@ describe('JevJudgeProvider', () => { it('is created explicitly via provider "jev" with OPENROUTER_API_KEY, never auto-detected', () => { expect(createJudgeProvider({ provider: 'jev' }, { OPENROUTER_API_KEY: 'k' }).name).toBe('jev'); expect(() => createJudgeProvider({ provider: 'jev' }, {})).toThrow('OPENROUTER_API_KEY'); - expect(() => createJudgeProvider(undefined, { OPENROUTER_API_KEY: 'k' })).toThrow('No semantic judge provider'); + expect(() => createJudgeProvider(undefined, { OPENROUTER_API_KEY: 'k' })).toThrow('No semantic judge configured'); }); }); diff --git a/tests/judge/verdicts.test.ts b/tests/judge/verdicts.test.ts index e9e5331..4ca1559 100644 --- a/tests/judge/verdicts.test.ts +++ b/tests/judge/verdicts.test.ts @@ -1,32 +1,58 @@ import { describe, expect, it } from 'bun:test'; import { PerRuleJudgeProvider } from '../../src/judge/provider/per-rule'; import type { SemanticJudgeResult } from '../../src/judge/types'; -import { semanticRuleSpec } from '../../src/judge/verdicts'; +import { resolveSemanticRule } from '../../src/judge/verdicts'; import type { Rule } from '../../src/types'; import { graded, node, passFailRule } from './helpers'; const base: Rule = { id: 'r', category: 'best-practice', description: 'd', mode: 'semantic' }; -describe('semanticRuleSpec', () => { - it('defaults to pass/fail with fail as the violation', () => { - const spec = semanticRuleSpec(base); - expect(Object.keys(spec.verdicts)).toEqual(['pass', 'fail']); - expect(spec.failOn).toEqual(['fail']); +describe('resolveSemanticRule', () => { + it('defaults to pass/fail with fail at the rule severity', () => { + const resolved = resolveSemanticRule(base); + expect(Object.keys(resolved.spec.verdicts)).toEqual(['pass', 'fail']); + expect(resolved.spec.failOn).toEqual(['fail']); + expect(resolved.severities).toEqual({ fail: 'info' }); // best-practice default + expect(resolveSemanticRule({ ...base, severity: 'error' }).severities).toEqual({ fail: 'error' }); }); - it('uses declared verdicts and failOn', () => { - const spec = semanticRuleSpec({ ...base, verdicts: { a: 'A', b: 'B', c: 'C' }, failOn: ['b', 'c'] }); - expect(spec.failOn).toEqual(['b', 'c']); + it('maps declared verdicts to severities; unmapped and ignored verdicts pass', () => { + const resolved = resolveSemanticRule({ + ...base, + verdicts: { ok: 'O', minor: 'M', major: 'J', skip: 'S' }, + severity: { minor: 'warning', major: 'error', skip: 'ignore' }, + }); + expect(resolved.severities).toEqual({ minor: 'warning', major: 'error' }); + expect(resolved.spec.failOn).toEqual(['minor', 'major']); }); - it('rejects declared verdicts without failOn, unknown failOn, and all-failing sets', () => { - expect(() => semanticRuleSpec({ ...base, verdicts: { a: 'A', b: 'B' } })).toThrow('no failOn'); - expect(() => semanticRuleSpec({ ...base, verdicts: { a: 'A', b: 'B' }, failOn: ['z'] })).toThrow( - 'not in its verdicts', + it('splits a verdict into the judge description and user guidance', () => { + const resolved = resolveSemanticRule({ + ...base, + verdicts: { + ok: 'Fine', + vague: { description: 'Too vague to act on', guidance: 'Name the customer and the pain.' }, + }, + severity: { vague: 'warning' }, + }); + expect(resolved.spec.verdicts).toEqual({ ok: 'Fine', vague: 'Too vague to act on' }); + expect(resolved.guidance).toEqual({ vague: 'Name the customer and the pain.' }); + }); + + it("carries the rule's judge name to the spec", () => { + expect(resolveSemanticRule({ ...base, judge: 'llm' }).spec.judge).toBe('llm'); + }); + + it('rejects custom verdicts without a severity map, unknown verdicts, and sets with no passing verdict', () => { + expect(() => resolveSemanticRule({ ...base, verdicts: { a: 'A', b: 'B' } })).toThrow( + 'must map the failing verdicts', ); - expect(() => semanticRuleSpec({ ...base, verdicts: { a: 'A', b: 'B' }, failOn: ['a', 'b'] })).toThrow( - 'no passing verdict', + expect(() => resolveSemanticRule({ ...base, verdicts: { a: 'A', b: 'B' }, severity: { z: 'error' } })).toThrow( + 'not in its verdicts', ); + expect(() => + resolveSemanticRule({ ...base, verdicts: { a: 'A', b: 'B' }, severity: { a: 'error', b: 'warning' } }), + ).toThrow('no passing verdict'); }); }); diff --git a/tests/schema/semantic-rule-contract.test.ts b/tests/schema/semantic-rule-contract.test.ts index 847ffe4..3a5cc70 100644 --- a/tests/schema/semantic-rule-contract.test.ts +++ b/tests/schema/semantic-rule-contract.test.ts @@ -44,4 +44,35 @@ describe('rule contract: check and semantic mode', () => { it('rejects a global semantic rule', () => { expect(() => load('semantic-global', { ...base, mode: 'semantic', scope: 'global' })).toThrow(); }); + + it('accepts a verdict map with guidance, a severity map (including ignore), a judge and a when', () => { + const rule = { + ...base, + mode: 'semantic', + judge: 'llm', + when: "current.status = 'ready'", + verdicts: { ok: 'Fine', vague: { description: 'Too vague', guidance: 'Be specific.' } }, + severity: { vague: 'warning' }, + }; + expect(load('shape-ok', rule).rules).toHaveLength(1); + expect(load('ignore-ok', { ...rule, severity: { vague: 'ignore' } }).rules).toHaveLength(1); + }); + + it('requires a severity map when custom verdicts are declared', () => { + const verdicts = { ok: 'Fine', bad: 'Bad' }; + expect(() => load('verdicts-no-severity', { ...base, mode: 'semantic', verdicts })).toThrow(); + expect(() => load('verdicts-scalar', { ...base, mode: 'semantic', verdicts, severity: 'error' })).toThrow(); + }); + + it('keeps semantic-only fields off expression rules', () => { + const expr = { ...base, check: 'true' }; + expect(() => load('expr-map', { ...expr, severity: { fail: 'error' } })).toThrow(); + expect(() => load('expr-judge', { ...expr, judge: 'llm' })).toThrow(); + expect(() => load('expr-verdicts', { ...expr, verdicts: { a: 'A', b: 'B' } })).toThrow(); + expect(load('expr-when', { ...expr, when: 'true' }).rules).toHaveLength(1); + }); + + it('rejects `when` on a global rule', () => { + expect(() => load('global-when', { ...base, check: 'true', scope: 'global', when: 'true' })).toThrow(); + }); }); diff --git a/tests/validate/semantic-rule-shape.test.ts b/tests/validate/semantic-rule-shape.test.ts new file mode 100644 index 0000000..8eab347 --- /dev/null +++ b/tests/validate/semantic-rule-shape.test.ts @@ -0,0 +1,151 @@ +import { afterAll, describe, expect, it } from 'bun:test'; +import { FakeJudgeProvider } from '../../src/judge/provider/fake'; +import { createSpaceContext } from '../../src/space-context'; +import type { FileValidationResult } from '../../src/types'; +import { validateFile, validateSpace } from '../../src/validate'; +import { makeTempSpace } from '../helpers/temp-space'; + +const spaces: ReturnType[] = []; +afterAll(() => { + for (const s of spaces) s.cleanup(); +}); + +/** A semantic rule on steps with minor/major verdicts mapped to warning/error and one ignored verdict. */ +const stepRule = { + id: 'step-fit', + type: 'step', + mode: 'semantic', + category: 'coherence', + description: 'Step fits its task', + verdicts: { + fits: 'The step fits', + minor: { description: 'Slightly off', guidance: 'Tighten the wording.' }, + major: { description: 'Unrelated', guidance: 'Move it under a better task.' }, + unsure: 'Cannot tell', + }, + severity: { minor: 'warning', major: 'error', unsure: 'ignore' }, +}; + +function setup(rules: object[], options?: Parameters[1]) { + const space = makeTempSpace(rules, options); + spaces.push(space); + return { + space, + context: () => createSpaceContext('ts', space.config, { configDir: space.dir }), + }; +} + +describe('semantic rules: verdict severity, ignore and guidance', () => { + it('reports each verdict at its mapped severity with its guidance', async () => { + const { context } = setup([stepRule]); + const minor = await validateSpace(context(), { + semantic: true, + judgeProvider: new FakeJudgeProvider({ 'step-fit': { verdict: 'minor' } }), + }); + expect(minor.ruleViolations).toEqual([ + expect.objectContaining({ + ruleId: 'step-fit', + severity: 'warning', + verdict: 'minor', + guidance: 'Tighten the wording.', + }), + ]); + + const major = await validateSpace(context(), { + semantic: true, + judgeProvider: new FakeJudgeProvider({ 'step-fit': { verdict: 'major' } }), + }); + expect(major.ruleViolations[0]).toMatchObject({ severity: 'error', verdict: 'major' }); + }); + + it('judges an ignored verdict without reporting it', async () => { + const { context } = setup([stepRule]); + const result = await validateSpace(context(), { + semantic: true, + judgeProvider: new FakeJudgeProvider({ 'step-fit': { verdict: 'unsure' } }), + }); + expect(result.ruleViolations).toEqual([]); + expect(result.semanticEvaluations).toEqual([expect.objectContaining({ verdict: 'unsure', pass: true })]); + }); + + it('gates the subtree per verdict: an error-mapped verdict gates, a warning-mapped one does not', async () => { + const taskRule = { ...stepRule, id: 'task-fit', type: 'task' }; + const { context } = setup([taskRule, { ...stepRule, id: 'step-fit' }]); + + const minor = new FakeJudgeProvider({ 'task-fit': { verdict: 'minor' } }); + const ungated = await validateSpace(context(), { semantic: true, judgeProvider: minor }); + expect(ungated.coverageGates).toEqual([]); + expect(minor.callHistory.some((c) => c.node.label === 'S.md')).toBe(true); + + const major = new FakeJudgeProvider({ 'task-fit': { verdict: 'major' } }); + const gated = await validateSpace(context(), { semantic: true, judgeProvider: major }); + expect(gated.coverageGates[0]).toMatchObject({ reason: 'rule', ruleId: 'task-fit' }); + expect(major.callHistory.some((c) => c.node.label === 'S.md')).toBe(false); + }); + + it('shows guidance in validate-file output', async () => { + const { space } = setup([stepRule]); + const result = (await validateFile(space.file('S.md'), space.config, { + semantic: true, + judgeProvider: new FakeJudgeProvider({ 'step-fit': { verdict: 'minor' } }), + configDir: space.dir, + })) as FileValidationResult; + expect(result.warnings['rule:step-fit']?.message).toContain('Guidance: Tighten the wording.'); + }); + + it('--rule-filter severity matches any severity a verdict map can report', async () => { + const { context } = setup([stepRule]); + const judge = new FakeJudgeProvider({ 'step-fit': { verdict: 'minor' } }); + await validateSpace(context(), { semantic: true, judgeProvider: judge, ruleFilter: { severity: ['warning'] } }); + expect(judge.callHistory.length).toBeGreaterThan(0); + const none = new FakeJudgeProvider(); + await validateSpace(context(), { semantic: true, judgeProvider: none, ruleFilter: { severity: ['info'] } }); + expect(none.callHistory).toHaveLength(0); + }); +}); + +describe('rule `when` precondition', () => { + const when = "current.status = 'ready'"; + + it('skips an expression rule on nodes where `when` is not true', async () => { + const rule = { + id: 'needs-owner', + category: 'validation', + description: 'owner', + check: '$exists(current.owner)', + when, + }; + const { context } = setup([rule], { frontmatter: { T: 'status: ready', S: 'status: draft' } }); + const result = await validateSpace(context()); + expect(result.ruleViolations.map((v) => v.file).sort()).toEqual(['T.md']); + }); + + it('does not call the judge for nodes where `when` is not true', async () => { + const rule = { ...stepRule, id: 'task-fit', type: 'task', when }; + const { context } = setup([rule], { frontmatter: { T: 'status: ready' } }); + const judge = new FakeJudgeProvider(); + await validateSpace(context(), { semantic: true, judgeProvider: judge }); + expect(judge.callHistory.map((c) => c.node.label)).toEqual(['T.md']); + + const none = setup([rule]); + const idle = new FakeJudgeProvider(); + const result = await validateSpace(none.context(), { semantic: true, judgeProvider: idle }); + expect(idle.callHistory).toHaveLength(0); + expect(result.gatedRules).toEqual([]); // not applicable is not gated + }); +}); + +describe('run-level judge override', () => { + it('fails before any request when the named judge is not configured', async () => { + const { context } = setup([stepRule], { judge: { providers: { llm: { type: 'fake' } } } }); + await expect(validateSpace(context(), { judge: 'nope' })).rejects.toThrow('Judge "nope" is not configured'); + }); + + it('--judge implies semantic evaluation, and semantic: false still wins', async () => { + const { context } = setup([stepRule], { judge: { providers: { llm: { type: 'fake' } } } }); + const on = await validateSpace(context(), { judge: 'llm' }); + expect(on.semanticEvaluations?.length).toBeGreaterThan(0); + const off = await validateSpace(context(), { judge: 'llm', semantic: false }); + expect(off.semanticEvaluations ?? []).toHaveLength(0); + }); +}); diff --git a/tests/validate/semantic-validation.test.ts b/tests/validate/semantic-validation.test.ts index a0dc09f..e48cb79 100644 --- a/tests/validate/semantic-validation.test.ts +++ b/tests/validate/semantic-validation.test.ts @@ -1,6 +1,4 @@ import { afterAll, beforeAll, describe, expect, it } from 'bun:test'; -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; -import { tmpdir } from 'node:os'; import { join } from 'node:path'; import type { Config } from '../../src/config'; import { FakeJudgeProvider } from '../../src/judge/provider/fake'; @@ -8,6 +6,7 @@ import { bundledSchemasDir } from '../../src/schema/schema'; import { createSpaceContext } from '../../src/space-context'; import type { FileValidationResult } from '../../src/types'; import { validateFile, validateSpace } from '../../src/validate'; +import { makeTempSpace, type TempSpace } from '../helpers/temp-space'; const STRICT_SCHEMA_PATH = join(bundledSchemasDir, 'strict_ost.json'); const VALID_DIR = join(import.meta.dir, '../fixtures/strict_ost/valid-directory'); @@ -144,58 +143,30 @@ describe('Semantic rule evaluation integration (validateSpace & validateFile)', }); describe('validateFile judges only the target and its gate-capable ancestor rules', () => { - let dir: string; - let config: Config; - const typeSchema = (type: string, parent?: string) => ({ - type: 'object', - properties: { type: { const: type }, title: { type: 'string' }, ...(parent && { parent: { type: 'string' } }) }, - required: ['type', 'title'], + let space: TempSpace; + const rule = (id: string, type: string, severity: string) => ({ + id, + type, + mode: 'semantic', + category: 'validation', + severity, + description: id, }); beforeAll(() => { - dir = mkdtempSync(join(tmpdir(), 'sctx-judge-target-')); - const schemaPath = join(dir, 'schema.json'); - const rule = (id: string, type: string, severity: string) => ({ - id, - type, - mode: 'semantic', - category: 'validation', - severity, - description: id, - }); - writeFileSync( - schemaPath, - JSON.stringify({ - $schema: - 'https://raw.githubusercontent.com/mindsocket/structured-context/main/schemas/generated/latest/_structured_context_schema_meta.json', - $id: 'sctx://judge-target.test', - $metadata: { - hierarchy: { levels: ['goal', 'task', 'step'] }, - rules: [ - rule('goal-sound', 'goal', 'error'), // gates descendants - rule('task-sound', 'task', 'warning'), // does not gate - rule('step-sound', 'step', 'warning'), - ], - }, - oneOf: [typeSchema('goal'), typeSchema('task', 'p'), typeSchema('step', 'p')], - }), - ); - const space = join(dir, 'space'); - mkdirSync(space); - writeFileSync(join(space, 'G.md'), '---\ntype: goal\n---\n'); - writeFileSync(join(space, 'T.md'), '---\ntype: task\nparent: "[[G]]"\n---\n'); - writeFileSync(join(space, 'S.md'), '---\ntype: step\nparent: "[[T]]"\n---\n'); - writeFileSync(join(space, 'Other.md'), '---\ntype: goal\n---\n'); - config = { spaces: [{ name: 'jt', path: space, schema: schemaPath }] }; + space = makeTempSpace([ + rule('goal-sound', 'goal', 'error'), // gates descendants + rule('task-sound', 'task', 'warning'), // does not gate + rule('step-sound', 'step', 'warning'), + ]); }); - - afterAll(() => rmSync(dir, { recursive: true, force: true })); + afterAll(() => space.cleanup()); const run = (judge: FakeJudgeProvider, file: string, extra = {}) => - validateFile(join(dir, 'space', file), config, { + validateFile(space.file(file), space.config, { semantic: true, judgeProvider: judge, - configDir: dir, + configDir: space.dir, ...extra, }) as Promise; @@ -242,7 +213,7 @@ describe('Semantic rules: verdicts, batching and rule filter', () => { expect(fake.callHistory.every((c) => c.rules.length >= 1)).toBe(true); }); - it('records the verdict and confidence on evaluations and fails only on failOn verdicts', async () => { + it('records the verdict and confidence on evaluations and fails only on verdicts that have a severity', async () => { const fake = new FakeJudgeProvider({ verdict: 'fail', confidence: 0.7, evidence: 'solution in disguise' }); const result = await validateSpace(context, { semantic: true, judgeProvider: fake }); const record = result.semanticEvaluations?.find((e) => e.ruleId === 'opportunity-customer-voice'); From d7cbe7a447494f765d2017b5249d670326bce705 Mon Sep 17 00:00:00 2001 From: Roger Barnes Date: Wed, 14 Oct 2026 13:13:27 +1100 Subject: [PATCH 09/22] feat(judge): type/default config keys, drop SCTX_JUDGE_* env, report expression errors as violations - judge config uses `type` in the flat and named forms; `use` becomes `default` - remove SCTX_JUDGE_* overrides (config, flags and apiKeyEnv cover them); document where a setting lives - a check or when that cannot be evaluated is an error violation, not a silent pass - okf: stale_after ordering is a validation rule; drop redundant explicit severities --- AGENTS.md | 2 + config.example.json | 2 +- docs/config.md | 14 +++---- docs/rules.md | 2 +- docs/semantic-rules.md | 4 +- schemas/_okf.json | 4 +- src/config.ts | 4 +- src/judge/factory.ts | 32 ++++++++-------- src/judge/types.ts | 6 +-- src/schema/evaluate-rule.ts | 43 +++++++++++++++++++--- src/schema/gating.ts | 28 ++++++++++---- src/schema/validate-rules.ts | 32 ++++++++++++---- tests/config.test.ts | 8 ++-- tests/judge/factory.test.ts | 31 ++++++++-------- tests/judge/provider/jev.test.ts | 4 +- tests/schema/evaluate-rule.test.ts | 23 +++++------- tests/validate/semantic-rule-shape.test.ts | 37 +++++++++++++++++++ 17 files changed, 186 insertions(+), 90 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 66978ea..d6bc782 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -38,6 +38,8 @@ There are several places that need reviewing and updating with any new feature o - **Simplicity first:** Strive for the simplest design that completely solves the problem. Prefer conventions and established filesystem patterns over adding new configuration, programmatic APIs, or abstractions. - **Explicitly state alternatives:** When planning or making architectural decisions, do not anchor to the first familiar mechanism. Formulate and state at least one alternative approach, weighing trade-offs, complexity, and ergonomic impact. - **Minimise blast radius:** Protect established boundaries and interfaces. Treat unnecessary ripple effects (such as making synchronous APIs asynchronous or adding dual APIs) as a design smell indicating the approach should be re-evaluated. +- **Where a setting lives:** Config says *what* to use, CLI flags override it for one run, and the environment holds secrets and the config location only (`apiKeyEnv` names a secret; `SCTX_CONFIG` locates the config). Do not add env vars that set config values, and never infer a choice (such as a judge provider) from a variable that happens to be set: a key present for another tool must not be spent silently. +- **Failures are visible:** A rule or expression that cannot be evaluated is reported as a violation, never skipped or treated as a pass. Likewise a misconfiguration fails before any work or spend. ## Key Files diff --git a/config.example.json b/config.example.json index 2e7d69f..3fa5e8e 100644 --- a/config.example.json +++ b/config.example.json @@ -18,5 +18,5 @@ } ], "schema": "/path/to/custom/schemas/my-general-schema.json", - "judge": { "provider": "none" } + "judge": { "type": "none" } } diff --git a/docs/config.md b/docs/config.md index acf1c6d..7810048 100644 --- a/docs/config.md +++ b/docs/config.md @@ -73,23 +73,23 @@ The shorthand defines one provider: ```json { - "judge": { "provider": "anthropic", "model": "claude-haiku-5-5" }, + "judge": { "type": "anthropic", "model": "claude-haiku-5-5" }, "spaces": [ - { "name": "ProductX", "path": "/path/to/space", "semantic": true, "judge": { "provider": "jev" } } + { "name": "ProductX", "path": "/path/to/space", "semantic": true, "judge": { "type": "jev" } } ] } ``` | Field | Description | |---|---| -| `provider` | **Required** to judge (or set `SCTX_JUDGE_PROVIDER`). One of `anthropic`, `openai` (any OpenAI-compatible endpoint: OpenAI, Ollama, vLLM, OpenRouter chat models), `jev` (TypeSafe decision model via OpenRouter's decisions API), `fake` (tests), or `none` (an inert placeholder for templates; judging with it is an error). Never inferred from API keys in the environment. | +| `type` | **Required** to judge. One of `anthropic`, `openai` (any OpenAI-compatible endpoint: OpenAI, Ollama, vLLM, OpenRouter chat models), `jev` (TypeSafe decision model via OpenRouter's decisions API), `fake` (tests), or `none` (an inert placeholder for templates; judging with it is an error). Never inferred from API keys in the environment. | | `model` | Defaults: `claude-haiku-5-5` (anthropic), `gpt-4o-mini` (openai), `typesafe/jev-1.13` (jev). | | `endpoint` | Override the API URL. Jev defaults to `https://openrouter.ai/api/alpha/decisions`. | | `apiKeyEnv` | Name of the environment variable holding the key. Defaults to the provider's usual one: `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `OPENROUTER_API_KEY` (jev). | | `apiKey` | The key itself. Prefer `apiKeyEnv`; keys do not belong in config files. | | `autoRun` | Evaluate semantic rules without `--semantic`. | -To use several providers, name them under `providers` (each takes `type` plus `model`, `endpoint`, `apiKeyEnv`, `apiKey`) and pick the default with `use`: +To use several providers, name them under `providers` (each takes `type` plus `model`, `endpoint`, `apiKeyEnv`, `apiKey`) and pick the default with `default`: ```json { @@ -98,14 +98,14 @@ To use several providers, name them under `providers` (each takes `type` plus `m "decision": { "type": "jev" }, "llm": { "type": "anthropic", "model": "claude-haiku-5-5" } }, - "use": "llm" + "default": "llm" } } ``` -A rule names one with `judge: "decision"`, and `--judge ` overrides every rule for a run. `use` is only needed when more than one provider is configured. Providers are created when first used, so one no rule reaches needs no credentials. The shorthand's provider is named `default`. Using the names `decision` (a decision model) and `llm` (a chat model) keeps shared schemas and configs portable. +A rule names one with `judge: "decision"`, and `--judge ` overrides every rule for a run. `default` is only needed when more than one provider is configured (a lone provider is the default). Providers, not profiles: a rule's `judge` names a provider, and there is no separate layer between them. Providers are created when first used, so one no rule reaches needs no credentials. The shorthand's provider is named `default`. Using the names `decision` (a decision model) and `llm` (a chat model) keeps shared schemas and configs portable. -`SCTX_JUDGE_PROVIDER`, `SCTX_JUDGE_MODEL`, `SCTX_JUDGE_ENDPOINT` and `SCTX_JUDGE_API_KEY` override the shorthand (`default`) provider. Semantic evaluation sends node content to the configured provider. +There are deliberately no `SCTX_JUDGE_*` environment overrides: config says what to use, flags override per run, and the environment holds only secrets (named by `apiKeyEnv`). Semantic evaluation sends node content to the configured provider. ## Markdown plugin config diff --git a/docs/rules.md b/docs/rules.md index 9fef316..723bd1b 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -127,7 +127,7 @@ $count(nodes[resolvedParentTitle=$$.current.title and resolvedType='solution']) "when": "current.status = 'ready'", "check": "$exists(current.owner)" } ``` -A `when` that errors or returns anything other than `true` means the rule does not apply. `when` needs a per-node rule, so it cannot be combined with `scope: "global"`. +A `when` that returns anything other than `true` means the rule does not apply. A `when` (or `check`) that cannot be evaluated, for example a syntax error, is not silently skipped: it is reported as an `error` violation on that node, with `Rule error:` in its description, and does not gate descendants. `when` needs a per-node rule, so it cannot be combined with `scope: "global"`. ## Reusing rules diff --git a/docs/semantic-rules.md b/docs/semantic-rules.md index ee87ce3..9c2b03c 100644 --- a/docs/semantic-rules.md +++ b/docs/semantic-rules.md @@ -28,10 +28,10 @@ For expression rules, severity, gating and merge behaviour, see [Rules](rules.md } ``` -2. **Choose a judge** in `config.json`. The provider is always explicit; keys in your environment are never picked up unless you name the provider: +2. **Choose a judge** in `config.json`. The judge `type` is always explicit; keys in your environment are never picked up unless you name the type: ```json - { "judge": { "provider": "anthropic" } } + { "judge": { "type": "anthropic" } } ``` with `ANTHROPIC_API_KEY` set. See [Judge configuration](config.md#judge-semantic-rules) for every field. diff --git a/schemas/_okf.json b/schemas/_okf.json index ea2049f..cd5410e 100644 --- a/schemas/_okf.json +++ b/schemas/_okf.json @@ -17,8 +17,7 @@ }, { "id": "okf-stale-after-generated", - "category": "coherence", - "severity": "error", + "category": "validation", "description": "stale_after must be later than generated.at", "check": "$not($exists(current.stale_after)) or $not($exists(current.generated.at)) or $toMillis(current.stale_after) > $toMillis(current.generated.at)" }, @@ -34,7 +33,6 @@ // from other body text: any body counts. "id": "okf-computation-present", "category": "validation", - "severity": "error", "type": "Attested Computation", "description": "An Attested Computation needs its computation: a `computation` path or a body `# Computation` section", "check": "$exists(current.computation) or $exists(current.content)" diff --git a/src/config.ts b/src/config.ts index daf75f9..3655812 100644 --- a/src/config.ts +++ b/src/config.ts @@ -31,13 +31,13 @@ const JUDGE_PROVIDER_SCHEMA = { const JUDGE_CONFIG_SCHEMA = { type: 'object', properties: { - provider: { type: 'string', enum: JUDGE_PROVIDER_TYPES }, + type: { type: 'string', enum: JUDGE_PROVIDER_TYPES }, model: { type: 'string' }, apiKey: { type: 'string' }, apiKeyEnv: { type: 'string' }, endpoint: { type: 'string' }, providers: { type: 'object', additionalProperties: JUDGE_PROVIDER_SCHEMA }, - use: { type: 'string' }, + default: { type: 'string' }, autoRun: { type: 'boolean' }, }, additionalProperties: false, diff --git a/src/judge/factory.ts b/src/judge/factory.ts index 44eca1f..45e4d63 100644 --- a/src/judge/factory.ts +++ b/src/judge/factory.ts @@ -29,27 +29,27 @@ export const DEFAULT_JUDGE_NAME = 'default'; export class JudgeConfigError extends Error {} /** Every provider the config defines, by name, and the one used for rules that don't name any. */ -function providerDefinitions( - config: JudgeConfig | undefined, - env: Env, -): { definitions: Record; defaultName?: string } { +function providerDefinitions(config: JudgeConfig | undefined): { + definitions: Record; + defaultName?: string; +} { const definitions: Record = { ...config?.providers }; - // The flat shorthand (and SCTX_JUDGE_* overrides) define one provider named "default". - const type = config?.provider || env.SCTX_JUDGE_PROVIDER; - if (type) { + // The flat shorthand defines one provider named "default". + if (config?.type) { definitions[DEFAULT_JUDGE_NAME] = { - type, - model: config?.model || env.SCTX_JUDGE_MODEL, - endpoint: config?.endpoint || env.SCTX_JUDGE_ENDPOINT, - apiKey: config?.apiKey || env.SCTX_JUDGE_API_KEY, - apiKeyEnv: config?.apiKeyEnv, + type: config.type, + model: config.model, + endpoint: config.endpoint, + apiKey: config.apiKey, + apiKeyEnv: config.apiKeyEnv, }; } const names = Object.keys(definitions); const defaultName = - config?.use ?? (DEFAULT_JUDGE_NAME in definitions ? DEFAULT_JUDGE_NAME : names.length === 1 ? names[0] : undefined); + config?.default ?? + (DEFAULT_JUDGE_NAME in definitions ? DEFAULT_JUDGE_NAME : names.length === 1 ? names[0] : undefined); return { definitions, defaultName }; } @@ -117,8 +117,8 @@ export class RoutedJudgeProvider implements SemanticJudgeProvider { const known = Object.keys(this.definitions); throw new JudgeConfigError( known.length === 0 - ? `No semantic judge configured. Set "judge.provider" in config.json (one of: ${PROVIDER_TYPES.join(', ')}) or SCTX_JUDGE_PROVIDER.` - : `Several judge providers are configured (${known.join(', ')}). Set "judge.use" for the default, or name one on the rule ("judge") or with --judge.`, + ? `No semantic judge configured. Set "judge.type" in config.json (one of: ${PROVIDER_TYPES.join(', ')}).` + : `Several judge providers are configured (${known.join(', ')}). Set "judge.default" for the default, or name one on the rule ("judge") or with --judge.`, ); } const def = this.definitions[name]; @@ -167,7 +167,7 @@ export function createJudge( customProvider?: SemanticJudgeProvider, override?: string, ): RoutedJudgeProvider { - const { definitions, defaultName } = providerDefinitions(config, env); + const { definitions, defaultName } = providerDefinitions(config); return new RoutedJudgeProvider(definitions, defaultName, env, override, customProvider); } diff --git a/src/judge/types.ts b/src/judge/types.ts index 9abed8b..1f29f05 100644 --- a/src/judge/types.ts +++ b/src/judge/types.ts @@ -60,11 +60,11 @@ export interface JudgeProviderDefinition { } /** - * Judge configuration from config file or options. Either the flat shorthand (`provider`, `model`, ...), which + * Judge configuration from config file or options. Either the flat shorthand (`type`, `model`, ...), which * defines one provider named "default", or several named `providers`. */ export interface JudgeConfig { - provider?: 'anthropic' | 'openai' | 'jev' | 'fake' | 'none' | string; + type?: 'anthropic' | 'openai' | 'jev' | 'fake' | 'none' | string; model?: string; apiKey?: string; apiKeyEnv?: string; @@ -72,7 +72,7 @@ export interface JudgeConfig { /** Named providers, for rules (or runs) that pick one by name. */ providers?: Record; /** Provider for rules that name none. Needed when several providers are configured. */ - use?: string; + default?: string; autoRun?: boolean; } diff --git a/src/schema/evaluate-rule.ts b/src/schema/evaluate-rule.ts index dc2dc49..4db5554 100644 --- a/src/schema/evaluate-rule.ts +++ b/src/schema/evaluate-rule.ts @@ -1,5 +1,5 @@ import jsonata from 'jsonata'; -import type { SpaceNode } from '../types'; +import type { Rule, RuleViolation, SpaceNode } from '../types'; const expressionCache = new Map>(); @@ -17,7 +17,7 @@ export interface EvalContext { /** * Evaluate a JSONata expression against a context. - * Returns the result of the expression (boolean for rule checks). + * Returns the result of the expression (boolean for rule checks). Throws {@link ExpressionError} if it fails. * * @param expr - JSONata expression string * @param context - Evaluation context with $, $$, parent, $lookup @@ -51,9 +51,31 @@ export async function evaluateExpression(expr: string, context: EvalContext): Pr const result = await expression.evaluate(input); return result as boolean | string | number; } catch (error) { - // Log warning and return false (fail safe) - console.warn(`Warning: Error evaluating expression "${expr}":`, error); - return false; + throw new ExpressionError(expr, error); + } +} + +/** A JSONata expression could not be compiled or evaluated. */ +export class ExpressionError extends Error { + constructor( + readonly expression: string, + cause: unknown, + ) { + // JSONata throws plain objects ({ code, message, position }), not Error instances. + const reason = (cause as { message?: unknown } | null)?.message ?? cause; + super(`Expression "${expression}" failed: ${String(reason)}`); + } +} + +/** Result of evaluating a rule's `check` or `when`: a value, or the error that stopped it (never thrown). */ +export type ExpressionOutcome = { value: boolean | string | number } | { error: string }; + +/** Like {@link evaluateExpression} but reports a failure as data, so callers can turn it into a violation. */ +export async function tryEvaluateExpression(expr: string, context: EvalContext): Promise { + try { + return { value: await evaluateExpression(expr, context) }; + } catch (error) { + return { error: error instanceof ExpressionError ? error.message : String(error) }; } } @@ -106,3 +128,14 @@ export function buildEvalContext( parents: flattenedParents.length > 0 ? flattenedParents : undefined, }; } + +/** The violation reported when a rule's `check` or `when` cannot be evaluated: an authoring error, so always `error`. */ +export function expressionErrorViolation(rule: Rule, file: string, error: string): RuleViolation { + return { + file, + ruleId: rule.id, + category: rule.category, + severity: 'error', + description: `${rule.description} (Rule error: ${error})`, + }; +} diff --git a/src/schema/gating.ts b/src/schema/gating.ts index 48b74d3..66dd170 100644 --- a/src/schema/gating.ts +++ b/src/schema/gating.ts @@ -2,7 +2,7 @@ import { judgeNode } from '../judge/judge-node'; import type { GatedRuleRecord, SemanticEvaluationRecord, SemanticJudgeProvider } from '../judge/types'; import type { SpaceGraph } from '../space-graph'; import type { Rule, RuleSeverity, RuleViolation, SpaceNode } from '../types'; -import { buildEvalContext, evaluateExpression } from './evaluate-rule'; +import { buildEvalContext, expressionErrorViolation, tryEvaluateExpression } from './evaluate-rule'; import { ruleSeverities, ruleSeverity } from './rule-severity'; /** @@ -148,8 +148,10 @@ export async function evaluateRulesWithGating( if (sentinel) { for (const rule of globalRules) { const context = buildEvalContext(sentinel, nodes, nodeIndex); - const result = await evaluateExpression(rule.check as string, context); - if (result !== true) { + const outcome = await tryEvaluateExpression(rule.check as string, context); + if ('error' in outcome) { + violations.push(expressionErrorViolation(rule, '', outcome.error)); + } else if (outcome.value !== true) { violations.push({ file: '', ruleId: rule.id, @@ -183,7 +185,7 @@ export async function evaluateRulesWithGating( } /** Rules that apply to this node: matching `type`, passing their `when` test, and (for a judge target) worth judging. */ - async function applicableRules(node: SpaceNode): Promise { + async function applicableRules(node: SpaceNode, onWhenError?: (rule: Rule, error: string) => void): Promise { const applicable: Rule[] = []; for (const r of perNodeRules) { if (r.type && r.type !== node.resolvedType) continue; @@ -191,7 +193,14 @@ export async function evaluateRulesWithGating( // Ancestors only matter through the gates they could raise. if (!(targetAncestors.has(node.title) && ruleCanGate(r))) continue; } - if (r.when && (await evaluateExpression(r.when, buildEvalContext(node, nodes, nodeIndex))) !== true) continue; + if (r.when) { + const outcome = await tryEvaluateExpression(r.when, buildEvalContext(node, nodes, nodeIndex)); + if ('error' in outcome) { + onWhenError?.(r, outcome.error); + continue; + } + if (outcome.value !== true) continue; + } applicable.push(r); } return applicable; @@ -199,7 +208,9 @@ export async function evaluateRulesWithGating( async function evaluateApplicableRules(node: SpaceNode): Promise<{ blockingRuleId?: string }> { let blockingRuleId: string | undefined; - const applicable = await applicableRules(node); + const applicable = await applicableRules(node, (rule, error) => + violations.push(expressionErrorViolation(rule, node.label, error)), + ); function report( rule: Rule, @@ -222,8 +233,9 @@ export async function evaluateRulesWithGating( // Cheap deterministic rules first; the judge is only consulted afterwards, once per node. for (const rule of applicable.filter((r) => r.mode !== 'semantic')) { - const result = await evaluateExpression(rule.check as string, buildEvalContext(node, nodes, nodeIndex)); - if (result !== true) report(rule); + const outcome = await tryEvaluateExpression(rule.check as string, buildEvalContext(node, nodes, nodeIndex)); + if ('error' in outcome) violations.push(expressionErrorViolation(rule, node.label, outcome.error)); + else if (outcome.value !== true) report(rule); } const semantic = applicable.filter((r) => r.mode === 'semantic'); diff --git a/src/schema/validate-rules.ts b/src/schema/validate-rules.ts index 586cc02..372b16c 100644 --- a/src/schema/validate-rules.ts +++ b/src/schema/validate-rules.ts @@ -1,7 +1,7 @@ import { judgeNode } from '../judge/judge-node'; import type { GatedRuleRecord, SemanticEvaluationRecord, SemanticJudgeProvider } from '../judge/types'; import type { Rule, RuleViolation, SpaceNode } from '../types'; -import { buildEvalContext, evaluateExpression } from './evaluate-rule'; +import { buildEvalContext, expressionErrorViolation, tryEvaluateExpression } from './evaluate-rule'; import { ruleSeverity } from './rule-severity'; export { DEFAULT_SEVERITY_BY_CATEGORY, ruleSeverity } from './rule-severity'; @@ -75,8 +75,10 @@ export async function validateRules( const sentinel = nodes[0]; if (sentinel) { const context = buildEvalContext(sentinel, nodes, nodeIndex); - const result = await evaluateExpression(check, context); - if (result !== true) { + const outcome = await tryEvaluateExpression(check, context); + if ('error' in outcome) { + violations.push(expressionErrorViolation(rule, '', outcome.error)); + } else if (outcome.value !== true) { violations.push({ file: '', ruleId: rule.id, @@ -90,9 +92,18 @@ export async function validateRules( const targetNodes = rule.type ? nodes.filter((n) => n.resolvedType === rule.type) : nodes; for (const node of targetNodes) { const context = buildEvalContext(node, nodes, nodeIndex); - if (rule.when && (await evaluateExpression(rule.when, context)) !== true) continue; - const result = await evaluateExpression(check, context); - if (result !== true) { + if (rule.when) { + const when = await tryEvaluateExpression(rule.when, context); + if ('error' in when) { + violations.push(expressionErrorViolation(rule, node.label, when.error)); + continue; + } + if (when.value !== true) continue; + } + const outcome = await tryEvaluateExpression(check, context); + if ('error' in outcome) { + violations.push(expressionErrorViolation(rule, node.label, outcome.error)); + } else if (outcome.value !== true) { violations.push({ file: node.label, ruleId: rule.id, @@ -118,7 +129,14 @@ export async function validateRules( const applicable: Rule[] = []; for (const r of semanticRules) { if (r.type && r.type !== node.resolvedType) continue; - if (r.when && (await evaluateExpression(r.when, buildEvalContext(node, nodes, nodeIndex))) !== true) continue; + if (r.when) { + const when = await tryEvaluateExpression(r.when, buildEvalContext(node, nodes, nodeIndex)); + if ('error' in when) { + violations.push(expressionErrorViolation(r, node.label, when.error)); + continue; + } + if (when.value !== true) continue; + } applicable.push(r); } if (applicable.length === 0) continue; diff --git a/tests/config.test.ts b/tests/config.test.ts index f9e2d33..fb0e481 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -392,16 +392,14 @@ describe('loadConfig with includeSpacesFrom', () => { writeFileSync( configPath, JSON.stringify({ - judge: { provider: 'jev' }, + judge: { type: 'jev' }, semantic: true, - spaces: [ - { name: 's', path: '/p', judge: { provider: 'anthropic', model: 'claude-haiku-5-5' }, semantic: false }, - ], + spaces: [{ name: 's', path: '/p', judge: { type: 'anthropic', model: 'claude-haiku-5-5' }, semantic: false }], }), ); setConfigPath(configPath); const config = loadConfig(); - expect(config.judge?.provider).toBe('jev'); + expect(config.judge?.type).toBe('jev'); expect(config.spaces?.[0]?.judge?.model).toBe('claude-haiku-5-5'); }); }); diff --git a/tests/judge/factory.test.ts b/tests/judge/factory.test.ts index 5a5816c..00dcf91 100644 --- a/tests/judge/factory.test.ts +++ b/tests/judge/factory.test.ts @@ -14,7 +14,7 @@ describe('createJudgeProvider factory', () => { }); it('creates FakeJudgeProvider when provider is "fake"', () => { - expect(createJudgeProvider({ provider: 'fake' }, {}) instanceof FakeJudgeProvider).toBe(true); + expect(createJudgeProvider({ type: 'fake' }, {}) instanceof FakeJudgeProvider).toBe(true); }); it.each([ @@ -22,16 +22,17 @@ describe('createJudgeProvider factory', () => { ['openai', 'OPENAI_API_KEY', OpenAIJudgeProvider], ['jev', 'OPENROUTER_API_KEY', JevJudgeProvider], ] as const)('creates %s from config provider with its usual env key (%s)', (provider, keyEnv, Class) => { - expect(createJudgeProvider({ provider }, { [keyEnv]: 'k' }) instanceof Class).toBe(true); + expect(createJudgeProvider({ type: provider }, { [keyEnv]: 'k' }) instanceof Class).toBe(true); }); - it('takes the provider from SCTX_JUDGE_PROVIDER and the key from config', () => { - const provider = createJudgeProvider({ apiKey: 'k' }, { SCTX_JUDGE_PROVIDER: 'anthropic' }); - expect(provider.name).toBe('anthropic'); + it('never takes the provider type from the environment', () => { + expect(() => createJudgeProvider({ apiKey: 'k' }, { SCTX_JUDGE_PROVIDER: 'anthropic' })).toThrow( + 'No semantic judge configured', + ); }); it('reads the key from the variable named by apiKeyEnv', () => { - const config = { provider: 'jev', apiKeyEnv: 'MY_JEV_KEY' }; + const config = { type: 'jev', apiKeyEnv: 'MY_JEV_KEY' }; expect(createJudgeProvider(config, { MY_JEV_KEY: 'k' }) instanceof JevJudgeProvider).toBe(true); expect(() => createJudgeProvider(config, { OPENROUTER_API_KEY: 'k' })).toThrow('MY_JEV_KEY'); }); @@ -48,25 +49,25 @@ describe('createJudgeProvider factory', () => { }); it("does not borrow another provider's key", () => { - expect(() => createJudgeProvider({ provider: 'anthropic' }, { OPENAI_API_KEY: 'o' })).toThrow('ANTHROPIC_API_KEY'); - expect(() => createJudgeProvider({ provider: 'jev' }, { ANTHROPIC_API_KEY: 'a' })).toThrow('OPENROUTER_API_KEY'); + expect(() => createJudgeProvider({ type: 'anthropic' }, { OPENAI_API_KEY: 'o' })).toThrow('ANTHROPIC_API_KEY'); + expect(() => createJudgeProvider({ type: 'jev' }, { ANTHROPIC_API_KEY: 'a' })).toThrow('OPENROUTER_API_KEY'); }); it('lets openai run without a key for local endpoints', () => { - const provider = createJudgeProvider({ provider: 'openai', endpoint: 'http://localhost:11434/v1' }, {}); + const provider = createJudgeProvider({ type: 'openai', endpoint: 'http://localhost:11434/v1' }, {}); expect(provider instanceof OpenAIJudgeProvider).toBe(true); }); it('treats provider "none" as an inert placeholder and rejects unknown providers', () => { - expect(() => createJudgeProvider({ provider: 'none' }, {})).toThrow('no provider type'); - expect(() => createJudgeProvider({ provider: 'gemini' as never }, {})).toThrow('Unknown judge provider'); + expect(() => createJudgeProvider({ type: 'none' }, {})).toThrow('no provider type'); + expect(() => createJudgeProvider({ type: 'gemini' as never }, {})).toThrow('Unknown judge provider'); }); }); describe('named providers and routing', () => { const config = { providers: { decision: { type: 'jev' }, llm: { type: 'anthropic', model: 'claude-haiku-5-5' } }, - use: 'llm', + default: 'llm', }; const env = { OPENROUTER_API_KEY: 'r', ANTHROPIC_API_KEY: 'a' }; @@ -80,8 +81,8 @@ describe('named providers and routing', () => { expect(createJudge(config, env, undefined, 'decision').resolve('llm').name).toBe('jev'); }); - it('needs `use` when several providers are configured and a rule names none', () => { - expect(() => createJudge({ providers: config.providers }, env).resolve()).toThrow('judge.use'); + it('needs `default` when several providers are configured and a rule names none', () => { + expect(() => createJudge({ providers: config.providers }, env).resolve()).toThrow('judge.default'); }); it('uses the only configured provider as the default', () => { @@ -107,7 +108,7 @@ describe('named providers and routing', () => { it('sends each rule to the provider it names, in one call per provider', async () => { const a = new FakeJudgeProvider({ 'customer-voice': { verdict: 'pass' } }); const b = new FakeJudgeProvider({ framing: { verdict: 'good' } }); - const judge = createJudge({ provider: 'fake' }, {}); + const judge = createJudge({ type: 'fake' }, {}); // Swap in recording providers for the two names. const routed = Object.assign(judge, { resolve: (name?: string) => (name === 'b' ? b : a), diff --git a/tests/judge/provider/jev.test.ts b/tests/judge/provider/jev.test.ts index bc450ae..f2fabb2 100644 --- a/tests/judge/provider/jev.test.ts +++ b/tests/judge/provider/jev.test.ts @@ -70,8 +70,8 @@ describe('JevJudgeProvider', () => { }); it('is created explicitly via provider "jev" with OPENROUTER_API_KEY, never auto-detected', () => { - expect(createJudgeProvider({ provider: 'jev' }, { OPENROUTER_API_KEY: 'k' }).name).toBe('jev'); - expect(() => createJudgeProvider({ provider: 'jev' }, {})).toThrow('OPENROUTER_API_KEY'); + expect(createJudgeProvider({ type: 'jev' }, { OPENROUTER_API_KEY: 'k' }).name).toBe('jev'); + expect(() => createJudgeProvider({ type: 'jev' }, {})).toThrow('OPENROUTER_API_KEY'); expect(() => createJudgeProvider(undefined, { OPENROUTER_API_KEY: 'k' })).toThrow('No semantic judge configured'); }); }); diff --git a/tests/schema/evaluate-rule.test.ts b/tests/schema/evaluate-rule.test.ts index 20e8124..1e09bc4 100644 --- a/tests/schema/evaluate-rule.test.ts +++ b/tests/schema/evaluate-rule.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it } from 'bun:test'; -import { buildEvalContext, evaluateExpression } from '../../src/schema/evaluate-rule'; +import { + buildEvalContext, + ExpressionError, + evaluateExpression, + tryEvaluateExpression, +} from '../../src/schema/evaluate-rule'; import type { SpaceNode } from '../../src/types'; import { makeParentRef } from '../test-helpers'; @@ -111,18 +116,10 @@ describe('evaluate-rule', () => { expect(result).toBe(1); }); - it('returns false on expression evaluation error', async () => { - // Suppress expected warning for invalid syntax - const originalWarn = console.warn; - console.warn = () => {}; - - try { - const context = buildEvalContext(mockNode, allNodes, mockNodeIndex); - const result = await evaluateExpression('invalid.syntax.('!, context); - expect(result).toBe(false); - } finally { - console.warn = originalWarn; - } + it('throws on an expression evaluation error, which tryEvaluateExpression reports as data', async () => { + const context = buildEvalContext(mockNode, allNodes, mockNodeIndex); + await expect(evaluateExpression('invalid.syntax.('!, context)).rejects.toThrow(ExpressionError); + expect(await tryEvaluateExpression('invalid.syntax.('!, context)).toHaveProperty('error'); }); }); diff --git a/tests/validate/semantic-rule-shape.test.ts b/tests/validate/semantic-rule-shape.test.ts index 8eab347..1017ccf 100644 --- a/tests/validate/semantic-rule-shape.test.ts +++ b/tests/validate/semantic-rule-shape.test.ts @@ -149,3 +149,40 @@ describe('run-level judge override', () => { expect(off.semanticEvaluations ?? []).toHaveLength(0); }); }); + +describe('judge type "none"', () => { + it('throws, not skips, when semantic evaluation is attempted', async () => { + for (const judge of [{ type: 'none' }, { providers: { llm: { type: 'none' } }, default: 'llm' }]) { + const { context } = setup([stepRule], { judge }); + await expect(validateSpace(context(), { semantic: true })).rejects.toThrow('no provider type'); + await expect(validateSpace(context(), { judge: 'default' })).rejects.toThrow(); + } + }); + + it('is harmless while semantic evaluation is not requested', async () => { + const { context } = setup([stepRule], { judge: { type: 'none' } }); + await expect(validateSpace(context(), {})).resolves.toBeDefined(); + }); +}); + +describe('rule expression errors', () => { + const broken = { id: 'broken', category: 'validation', description: 'Broken check', check: '$notAFunction(' }; + const brokenWhen = { + id: 'bad-when', + category: 'coherence', + description: 'Task rule', + type: 'task', + when: '1 +', + check: 'true', + }; + + it('reports a failing check or when as an error violation instead of passing silently', async () => { + const { context } = setup([broken, brokenWhen]); + const result = await validateSpace(context(), {}); + const byId = (id: string) => result.ruleViolations.filter((v) => v.ruleId === id); + expect(byId('broken').length).toBeGreaterThan(0); + expect(byId('broken')[0]).toMatchObject({ severity: 'error' }); + expect(byId('broken')[0]?.description).toContain('Rule error'); + expect(byId('bad-when')).toEqual([expect.objectContaining({ file: 'T.md', severity: 'error' })]); + }); +}); From a5fade22339951c9adf8da4e0b4bdaedf06f27f5 Mon Sep 17 00:00:00 2001 From: Roger Barnes Date: Wed, 14 Oct 2026 16:51:42 +1100 Subject: [PATCH 10/22] docs(rules): restructure rule and semantic rule docs; drop judge.autoRun; rename docs topic to semantic-rules rules.md now owns the shared ideas (violation, when, rule filtering) and semantic-rules.md covers only what differs, in shorter sentences. Quick start leads with Jev. judge.autoRun duplicated the semantic config key, so semantic is now the only config switch. sctx docs semantic becomes sctx docs semantic-rules. --- README.md | 2 +- docs/concepts.md | 4 +- docs/config.md | 20 +-- docs/rules.md | 45 ++++--- docs/semantic-rules.md | 125 ++++++++++-------- plugin/.claude-plugin/plugin.json | 2 +- plugin/skills/structured-context/SKILL.md | 2 +- .../references/schema-authoring.md | 2 +- src/commands/docs.ts | 2 +- src/config.ts | 1 - src/judge/types.ts | 1 - src/validate.ts | 7 +- 12 files changed, 117 insertions(+), 96 deletions(-) diff --git a/README.md b/README.md index b47d702..370d370 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,7 @@ Validates markdown files against the JSON schema: - Runs executable rules (`$metadata.rules`) with **cascade gating**: a schema, hierarchy, or blocking-rule failure on a node suppresses rule evaluation for that node's hierarchy subtree. Gated checks are reported in a Coverage section so you know what wasn't evaluated and why. See [docs/rules.md](docs/rules.md#cascade-gating) for details. - `--depth ` limits evaluation to hierarchy nodes within `n` breadth-first levels from the top — useful for staged agentic fix loops (fix the top level, then request the next). - `--include-gated` evaluates rules even on gated nodes (still reported as gated) — for surfacing cheap bottom-up checks on request. -- `--semantic` also evaluates `mode: semantic` rules with a model judge (Anthropic, OpenAI-compatible, or Jev); `--no-semantic` overrides `semantic`/`judge.autoRun` in config, and `--judge ` picks a configured judge provider for every rule; `--rule-filter` narrows which rules run. See [docs/semantic-rules.md](docs/semantic-rules.md) (`sctx docs semantic`). +- `--semantic` also evaluates `mode: semantic` rules with a model judge (Anthropic, OpenAI-compatible, or Jev); `--no-semantic` overrides `semantic` in config, and `--judge ` picks a configured judge provider for every rule; `--rule-filter` narrows which rules run. See [docs/semantic-rules.md](docs/semantic-rules.md) (`sctx docs semantic-rules`). ### Render space diff --git a/docs/concepts.md b/docs/concepts.md index aa6f805..158f298 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -153,9 +153,9 @@ See [docs/rules.md](rules.md) for the rules reference, including JSONata express ### Semantic rules -A **semantic rule** (`mode: semantic`) checks meaning that an expression cannot, such as whether an opportunity is framed in the customer's voice. Instead of a `check` expression it carries a `rubric` and a closed set of **verdicts**. A verdict is the judge's classification ("customer-need", "vague", "solution"); the rule's `severity` map says what each verdict means for the user (`error`, `warning`, `info`, or `ignore`), and a verdict can carry user-facing `guidance`. +A **semantic rule** (`mode: semantic`) asks a model to judge a node, for example whether an opportunity is a customer need. It has a `rubric` instead of a `check`. -A **judge** is the model that evaluates semantic rules: a chat LLM (Anthropic, OpenAI-compatible) or a decision model (Jev). It receives a node and its rules and must answer each rule with exactly one **verdict** from that rule's `verdicts`; verdicts with a severity are violations and all others pass. A judge may also report a confidence and probabilities. Semantic rules are opt-in (`--semantic`) because they cost money and time, and they run inside [cascade gating](#cascade-gating). See [docs/semantic-rules.md](semantic-rules.md) for the rule fields and [docs/config.md](config.md#judge-semantic-rules) for judge configuration. +A **judge** is the model that judges semantic rules: a decision model (Jev) or a chat model (Claude, OpenAI-compatible). For each rule, the judge gives one **verdict** from the rule's `verdicts`, for example `customer-need`, `vague` or `solution`. The rule's `severity` says which verdicts report a violation. Semantic rules cost money, so they run only when you turn them on (`--semantic`, or `semantic: true` in config). See [docs/semantic-rules.md](semantic-rules.md). ### Cascade gating diff --git a/docs/config.md b/docs/config.md index 7810048..373070f 100644 --- a/docs/config.md +++ b/docs/config.md @@ -67,9 +67,9 @@ Plugins can also contribute schemas and partials via a `/schemas/` d ## Judge (semantic rules) -`judge` configures the model that evaluates `mode: semantic` [rules](semantic-rules.md). Set it at the top level of the config (all spaces) or on a space (the space wins). `semantic` turns semantic evaluation on by default, at the same two levels; otherwise it runs only with `--semantic`. +`judge` configures the model that judges [semantic rules](semantic-rules.md). `semantic: true` runs semantic rules without `--semantic`. Set either key at the top level (all spaces) or on a space. A space's value replaces the top-level value. -The shorthand defines one provider: +To use one provider, set its fields directly on `judge`: ```json { @@ -82,14 +82,13 @@ The shorthand defines one provider: | Field | Description | |---|---| -| `type` | **Required** to judge. One of `anthropic`, `openai` (any OpenAI-compatible endpoint: OpenAI, Ollama, vLLM, OpenRouter chat models), `jev` (TypeSafe decision model via OpenRouter's decisions API), `fake` (tests), or `none` (an inert placeholder for templates; judging with it is an error). Never inferred from API keys in the environment. | +| `type` | **Required.** `jev`, `anthropic`, `openai` (see [Choosing a judge](semantic-rules.md#choosing-a-judge)), `fake` (for tests) or `none` (a placeholder for templates; judging with it is an error). | | `model` | Defaults: `claude-haiku-5-5` (anthropic), `gpt-4o-mini` (openai), `typesafe/jev-1.13` (jev). | | `endpoint` | Override the API URL. Jev defaults to `https://openrouter.ai/api/alpha/decisions`. | -| `apiKeyEnv` | Name of the environment variable holding the key. Defaults to the provider's usual one: `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `OPENROUTER_API_KEY` (jev). | -| `apiKey` | The key itself. Prefer `apiKeyEnv`; keys do not belong in config files. | -| `autoRun` | Evaluate semantic rules without `--semantic`. | +| `apiKeyEnv` | The environment variable that holds the key. Defaults: `OPENROUTER_API_KEY` (jev), `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`. | +| `apiKey` | The key itself. Use `apiKeyEnv` instead, so that keys stay out of config files. | -To use several providers, name them under `providers` (each takes `type` plus `model`, `endpoint`, `apiKeyEnv`, `apiKey`) and pick the default with `default`: +To use several providers, give each a name under `providers`. Each provider takes the fields above. `default` names the provider for rules that do not name one: ```json { @@ -103,9 +102,12 @@ To use several providers, name them under `providers` (each takes `type` plus `m } ``` -A rule names one with `judge: "decision"`, and `--judge ` overrides every rule for a run. `default` is only needed when more than one provider is configured (a lone provider is the default). Providers, not profiles: a rule's `judge` names a provider, and there is no separate layer between them. Providers are created when first used, so one no rule reaches needs no credentials. The shorthand's provider is named `default`. Using the names `decision` (a decision model) and `llm` (a chat model) keeps shared schemas and configs portable. +- A rule selects a provider with `judge: "decision"`. `--judge ` selects one for every rule in a run. +- `default` is necessary only with more than one provider. +- A provider needs a key only when a rule uses it. +- Use the names `decision` (a decision model) and `llm` (a chat model), so that shared schemas work with other configs. -There are deliberately no `SCTX_JUDGE_*` environment overrides: config says what to use, flags override per run, and the environment holds only secrets (named by `apiKeyEnv`). Semantic evaluation sends node content to the configured provider. +The environment holds only keys. There are no environment variables that change the judge. ## Markdown plugin config diff --git a/docs/rules.md b/docs/rules.md index 723bd1b..8ea4ce3 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -4,7 +4,12 @@ title: "Executable Rules" # Executable Rules -Rules are JSONata expressions in schema metadata (`$metadata.rules`). They run after structural JSON Schema validation and let you enforce cross-node checks and workflow constraints. +Rules check content after JSON Schema validation, for example cross-node checks and workflow constraints. A schema declares rules in `$metadata.rules`. There are two kinds of rule: + +- An **expression rule** (the default) is a JSONata `check` that must return `true`. +- A **semantic rule** asks a model to judge a node. See [Semantic rules](semantic-rules.md). + +This page applies to both kinds, unless it says otherwise. For metadata structure and composition behavior, see [docs/schemas.md](schemas.md). @@ -28,14 +33,12 @@ Rules are a flat array. |---|---|---| | `id` | yes | Unique rule identifier | | `category` | yes | `validation` \| `coherence` \| `workflow` \| `best-practice` | -| `severity` | no | `error` \| `warning` \| `info`; overrides the category default (see [Severity](#severity)). Semantic rules may instead map verdicts to severities | +| `severity` | no | `error` \| `warning` \| `info`; overrides the category default (see [Severity](#severity)). A semantic rule can map verdicts to severities | | `description` | yes | Human-readable rule intent | | `check` | yes, unless `mode: semantic` | JSONata expression; must evaluate to `true` | -| `when` | no | JSONata applicability test, evaluated per node before the rule. The rule runs only where it returns `true`; elsewhere it is skipped and not reported. Not allowed with `scope: "global"` | -| `mode` | no | `expression` (default) or `semantic` (judged by a model; see [Semantic rules](semantic-rules.md)) | -| `rubric` | semantic only | Detailed prompt for the judge; falls back to `description` | -| `verdicts` | semantic only | Closed set of verdicts the judge must choose from. Default `pass` / `fail`. See [Semantic rules](semantic-rules.md#rule-fields) | -| `judge` | semantic only | Name of the configured judge provider to use; a `--judge` run override wins | +| `when` | no | JSONata test. The rule runs only on nodes where it returns `true`. See [Conditional rules](#conditional-rules-when) | +| `mode` | no | `expression` (default) or `semantic` | +| `rubric`, `verdicts`, `judge` | semantic only | See [Semantic rules](semantic-rules.md#rule-fields) | | `type` | no | Restrict rule to nodes of this `resolvedType` | | `scope` | no | Use `"global"` to evaluate once for the whole space | | `override` | no | Only for merge conflicts: allows later duplicate `id` to replace earlier | @@ -54,7 +57,7 @@ Categories describe a rule's intent and supply its default severity. They do not ### Severity -Every rule violation is reported with a severity: +When a rule finds a problem on a node, it reports a **violation**. Otherwise it reports nothing. Each violation has a severity: | Severity | Meaning | Fails validation | |---|---|---| @@ -85,11 +88,12 @@ Prefer choosing the right category and relying on its default. Set `severity` on - **`sctx validate --json`** — `errors` includes only `error` severity rule violations (keyed `rule:`, with a `severity` field). A `ruleViolations` array lists every violation with `file`, `ruleId`, `category`, `severity` and `description` (`file` is empty for global rules). - **`sctx validate-file`** — `error` severity violations appear under `errors`; `warning` and `info` violations appear under `warnings`. Rule entries carry a `severity` field. -## Evaluation model +## Which nodes a rule runs on -- Rules with `scope: "global"` run once. -- Other rules run per applicable node. -- If `type` is set, only nodes with matching `resolvedType` are evaluated. +- A rule with `scope: "global"` runs once for the space. +- Other rules run once for each node. +- `type` limits a rule to nodes with that `resolvedType`. +- `when` limits a rule to nodes where a test returns `true`. See [Conditional rules](#conditional-rules-when). ## Expression context @@ -119,7 +123,7 @@ $count(nodes[resolvedParentTitle=$$.current.title and resolvedType='solution']) ## Conditional rules (`when`) -`when` is a JSONata test evaluated on each node before the rule, with the same context as `check`. The rule runs only where it returns `true`; on every other node it is skipped and not reported (it is neither passed nor gated). Use it when a rule applies to a subset of nodes that `type` cannot express: +`when` is a JSONata test with the same context as `check`. sctx evaluates it on each node before the rule. The rule runs only where `when` returns `true`. On other nodes the rule does not run: it reports nothing and does not gate. Use `when` when `type` cannot select the nodes a rule applies to: ```json { "id": "ready-has-owner", "category": "workflow", "type": "task", @@ -127,7 +131,9 @@ $count(nodes[resolvedParentTitle=$$.current.title and resolvedType='solution']) "when": "current.status = 'ready'", "check": "$exists(current.owner)" } ``` -A `when` that returns anything other than `true` means the rule does not apply. A `when` (or `check`) that cannot be evaluated, for example a syntax error, is not silently skipped: it is reported as an `error` violation on that node, with `Rule error:` in its description, and does not gate descendants. `when` needs a per-node rule, so it cannot be combined with `scope: "global"`. +`when` costs almost nothing, so use it to stop an expensive rule from running where it does not apply. For a semantic rule, this saves a model call on each node that `when` excludes. + +If sctx cannot evaluate a `when` or a `check` (for example, because of a syntax error), it reports an `error` violation on the node, with `Rule error:` in its description. This violation does not gate descendants. A rule with `scope: "global"` cannot have `when`. ## Reusing rules @@ -205,9 +211,16 @@ the top (roots and orphans are depth 1). This is a manual, coarser control than gating — intended for staged workflows (e.g. an agent fixes the top level, then requests the next). Excluded nodes are reported under `skippedByDepth`, separate from gated nodes. -## Semantic rules +## Filtering rules + +`--rule-filter =[,]` runs only the rules that match. It works on `sctx validate` and `sctx validate-file`. + +- Dimensions: `mode`, `category`, `severity`, `type` and `id`. +- Values in one dimension combine with OR. Repeat the option to add a dimension; dimensions combine with AND. +- A semantic rule with a severity map matches each severity that it can report. +- Gating comes only from the rules that ran. -A rule with `"mode": "semantic"` is judged by a model (an LLM or a decision model) instead of a JSONata expression. They are opt-in (`--semantic`) because they cost money and time, and they run inside [cascade gating](#cascade-gating). See [Semantic rules](semantic-rules.md) (`sctx docs semantic`) for the rule fields, judge setup and filtering. +For example, `--rule-filter mode=expression` skips semantic rules, and `--rule-filter category=workflow,coherence --rule-filter severity=error` runs only workflow and coherence rules that can report an error. ## Common patterns diff --git a/docs/semantic-rules.md b/docs/semantic-rules.md index 9c2b03c..921a4b5 100644 --- a/docs/semantic-rules.md +++ b/docs/semantic-rules.md @@ -1,103 +1,114 @@ # Semantic Rules -A semantic rule is a rule a model judges instead of a JSONata expression: "is this opportunity framed in the customer's voice?" rather than "does this field exist?". Semantic rules are **opt-in**. Nothing is sent to a model unless you pass `--semantic` (or enable it in config), and a space with no semantic rules is unaffected. +A semantic rule asks a model to judge a node, for example "Is this opportunity a customer need, or a solution in disguise?". Use a semantic rule for a check that an expression cannot do. -For expression rules, severity, gating and merge behaviour, see [Rules](rules.md). +This page assumes you know [Rules](rules.md). Semantic rules use the same fields, severities, [`when`](rules.md#conditional-rules-when) and [cascade gating](rules.md#cascade-gating) as expression rules. This page gives only the differences. ## Quick start -1. **Declare a rule** in the schema's `$metadata.rules`: +1. Set `OPENROUTER_API_KEY` in your environment. +2. Add a judge to `config.json`: + + ```json + { "judge": { "type": "jev" } } + ``` + + Jev is a decision model and the recommended judge (see [Choosing a judge](#choosing-a-judge)). To use Claude instead, set `"type": "anthropic"` and `ANTHROPIC_API_KEY`. +3. Add a rule to the schema's `$metadata.rules`: ```json { - "id": "opportunity-customer-voice", + "id": "opportunity-customer-need", "category": "best-practice", "mode": "semantic", "type": "opportunity", - "description": "Opportunity is framed in the customer's voice", - "rubric": "The opportunity reads as a customer need or pain point, not a solution in disguise or a business metric.", - "verdicts": { - "customer-need": "Clearly a customer need or pain point", - "vague": { - "description": "A need, but too vague to act on", - "guidance": "Say who has the need and what they are trying to do." - }, - "solution": "A solution or business goal in disguise" - }, - "severity": { "vague": "info", "solution": "warning" } + "description": "The opportunity is a customer need or pain point, not a solution or a business metric" } ``` -2. **Choose a judge** in `config.json`. The judge `type` is always explicit; keys in your environment are never picked up unless you name the type: +4. Run `sctx validate --semantic`. - ```json - { "judge": { "type": "anthropic" } } - ``` +## Turning semantic rules on - with `ANTHROPIC_API_KEY` set. See [Judge configuration](config.md#judge-semantic-rules) for every field. +Semantic rules cost money and send node content to the judge's provider. Thus they run only when you ask: -3. **Run it**: `sctx validate --semantic`. To turn it on by default, set `semantic: true` (or `judge.autoRun`) in config; `--no-semantic` overrides that for a single run. +- `--semantic` runs them for one run of `sctx validate` or `sctx validate-file`. +- `"semantic": true` in [config](config.md#judge-semantic-rules) runs them on every run. Set it at the top level (all spaces) or on a space. +- `--no-semantic` stops them for one run, and overrides config. ## Rule fields -Semantic rules share `id`, `category`, `type`, `description`, `when` and `gatesDescendants` with [expression rules](rules.md#rule-shape). They differ in these fields: - | Field | Required | Description | |---|---|---| -| `mode` | yes | `"semantic"` (the default is `"expression"`) | -| `rubric` | no | What the judge should assess. Falls back to `description` | -| `verdicts` | no | The closed set of answers: `{ "": "" }`, at least two. Default `pass` / `fail` | -| `severity` | with `verdicts` | A severity for the default `fail` verdict, or a map from verdict to severity (see below) | -| `judge` | no | Name of the [judge provider](config.md#judge-semantic-rules) to use. Default: the configured default. A `--judge` run override replaces it | +| `mode` | yes | `"semantic"` | +| `rubric` | no | Tells the judge what to assess. Default: `description` | +| `verdicts` | no | The answers the judge can give. Default: `pass` and `fail`. See [Verdicts](#verdicts) | +| `severity` | if `verdicts` is set | The severity of each verdict. See [Severity](#severity) | +| `judge` | no | The name of a [judge provider](#several-judges). Default: the configured default provider | -A semantic rule has no `check` and cannot be `scope: "global"`; it judges one node at a time. +A semantic rule cannot have `check` or `scope: "global"`. -### Verdicts, severity and guidance +## Verdicts -The judge must choose exactly one verdict. A closed set keeps results comparable between runs and maps directly onto decision models, which return a choice with confidence and probabilities rather than prose. +For each node, the judge gives exactly one verdict for each rule. The verdict must come from the rule's `verdicts`. A fixed set of verdicts makes results comparable between runs. -- **A verdict is a classification, severity is the consequence.** `severity` as a map decides what each verdict means for the user: `error`, `warning` or `info`. A verdict left out of the map passes, and `ignore` states that explicitly: the rule ran and the verdict was recorded, but nothing is reported. At least one verdict must pass. -- **With the default `pass` / `fail` verdicts**, `severity` may be a single value (applied to `fail`) or omitted (the category default applies), like an expression rule. -- **A verdict's value may be a string or `{ "description", "guidance" }`.** The description tells the judge what the verdict means. The guidance is shown to the user when that verdict fires, so a failure arrives with what to do about it; it is never sent to the judge. -- **Gating follows the verdict.** A verdict mapped to `error` gates the node's subtree (unless `gatesDescendants: false`); one mapped to `warning` or `info` does not (unless `gatesDescendants: true`). +Each verdict has a purpose for one audience: -### Running a rule only where it applies +- The **description** tells the judge when to choose that verdict. +- The **guidance** (optional) tells the user what to do when the judge chooses that verdict. sctx shows it with the violation. sctx does not send it to the judge. -`when` is a JSONata test evaluated on each node before the rule. The rule runs only where it returns `true`, and a node where it does not is skipped silently (no judge call, nothing reported): +A verdict is a description string, or an object with `description` and `guidance`: ```json -{ "id": "due-soon-coherent", "category": "coherence", "mode": "semantic", "type": "task", - "description": "A task due within a week has a believable plan", - "when": "current.status = 'ready' and $toMillis(current.due) - $millis() < 604800000" } +"verdicts": { + "customer-need": "A customer need or pain point", + "vague": { + "description": "A need, but too vague to act on", + "guidance": "Say who has the need and what they are trying to do." + }, + "solution": "A solution or business goal in disguise" +}, +"severity": { "vague": "info", "solution": "warning" } ``` -`when` works on expression rules too. It saves money on semantic rules because the cheap test runs first. +## Severity -## How judging works +The verdict is a classification. The severity says what the verdict means for the user. -- **Order.** For each node, expression rules run first, then all of the node's semantic rules go to the judge in one call. Chat LLMs make one request per rule; decision models answer all of a node's rules in a single request. -- **Gating.** Semantic rules follow [cascade gating](rules.md#cascade-gating). They are not judged on gated nodes (recorded as `gatedRules`). A judge *error* (network failure, unparseable output) is reported as a violation but never gates. -- **Input.** The judge sees the node's title, type, parent titles and its schema data (the fields the parser extracted for validation, including frontmatter). -- **Output.** `--json` includes `semanticEvaluations` (verdict, `pass`, confidence and evidence per node and rule) and `gatedRules`. Violations carry the judge's `evidence`, the `verdict` and any `guidance`. +- With custom verdicts, `severity` is a map from verdict to `error`, `warning`, `info` or `ignore`. +- A verdict with `error`, `warning` or `info` reports a violation at that severity. +- A verdict that is not in the map reports no violation. At least one verdict must report no violation. +- `ignore` also reports no violation. Use it to show that you chose to ignore a verdict. The verdict is still recorded in `--json` output. +- With the default verdicts, `severity` is one value for `fail`, the same as for an expression rule. -### Single-file validation +Gating uses the severity of the verdict. A verdict with `error` severity gates the node's subtree, as for any rule. `gatesDescendants` changes this, as for any rule. -`sctx validate-file --semantic` computes gating over the whole space but only calls the judge for the file itself and for its hierarchy ancestors' semantic rules that can gate (top-down, so a failing ancestor stops the walk). A file costs at most one call per ancestor level plus one for the file, however large the space. +## How judging works -### Filtering rules +- **Order.** On each node, expression rules run first. Then sctx sends all of the node's semantic rules to the judge in one call. +- **Gating.** sctx does not judge a gated node. The JSON output lists the rules it did not judge under `gatedRules`. +- **Judge errors.** If the judge fails (for example, a network error or output that sctx cannot read), sctx reports a violation. A judge error does not gate. +- **Input.** The judge gets the node's title, type, parent titles and schema data, including frontmatter. +- **Output.** Violations include the verdict, the judge's evidence and any guidance. `--json` adds `semanticEvaluations`: the verdict, confidence and evidence for each node and rule. -`--rule-filter =[,]` (repeatable) evaluates only matching rules. Dimensions are `mode`, `category`, `severity`, `type` and `id`; different dimensions combine with AND, values within one with OR. A rule with a verdict map matches a `severity` filter on any severity it can report. For example, `--semantic --rule-filter mode=semantic` runs only the judged rules, and `--rule-filter mode=expression` skips them entirely. Gating is computed from the rules that ran. +`sctx validate-file` judges only the file and the rules on its ancestors that can gate it. Thus one file costs at most one judge call for each ancestor level, plus one. ## Choosing a judge -| Provider type | Use for | Notes | +Use a decision model when you can. Its results are repeatable and cheap, and they include a confidence. Use a chat model when the rubric needs open reasoning or a written explanation. + +| Provider type | Model | Default key variable | |---|---|---| -| `anthropic` | Claude models | Default model `claude-haiku-5-5` | -| `openai` | Any OpenAI-compatible chat endpoint: OpenAI, Ollama, vLLM, Groq, OpenRouter chat models | Set `endpoint` for non-OpenAI servers; the key is optional for local ones | -| `jev` | TypeSafe's Jev decision model | Closed-set answers with calibrated probabilities, one request per node. Uses an OpenRouter key | +| `jev` | TypeSafe's Jev decision model, through OpenRouter. One request for each node | `OPENROUTER_API_KEY` | +| `anthropic` | Claude. Default model: `claude-haiku-5-5` | `ANTHROPIC_API_KEY` | +| `openai` | Any OpenAI-compatible chat endpoint, for example OpenAI, Ollama or vLLM. Set `endpoint` for servers other than OpenAI. Local servers do not need a key | `OPENAI_API_KEY` | + +sctx uses only the provider type that you set. It does not choose a provider from the keys in your environment. See [Judge configuration](config.md#judge-semantic-rules) for all fields. -Prefer a decision model when you can: results are repeatable, cheap, and come with a confidence. Use a chat LLM when the rubric needs open reasoning or evidence text. +### Several judges -Several providers can be configured under names, for example `decision` and `llm`. A rule picks one with `judge`, and a run can override every rule with `--judge ` (for instance a deeper second opinion on one file: `sctx validate-file F --rule-filter id=r --judge llm`). If a rule's choice and the run override disagree, the run wins. +You can configure several providers by name, for example `decision` and `llm`. A rule selects one with `judge`. `--judge ` uses one provider for every rule in a run, and implies `--semantic`. For example, to get a second opinion from a chat model on one rule and one file: -Semantic evaluation sends node content to the configured provider. Check that is acceptable for the space before enabling it. +```bash +sctx validate-file --rule-filter id= --judge llm +``` diff --git a/plugin/.claude-plugin/plugin.json b/plugin/.claude-plugin/plugin.json index 7af5727..d8dd888 100644 --- a/plugin/.claude-plugin/plugin.json +++ b/plugin/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "structured-context", - "version": "0.12.11", + "version": "0.12.12", "description": "Structured content authoring for Obsidian — automatic validation hooks, slash commands, and agent skills to keep markdown files compliant with your schema.", "homepage": "https://github.com/mindsocket/structured-context", "repository": "https://github.com/mindsocket/structured-context", diff --git a/plugin/skills/structured-context/SKILL.md b/plugin/skills/structured-context/SKILL.md index 3987348..8e049b7 100644 --- a/plugin/skills/structured-context/SKILL.md +++ b/plugin/skills/structured-context/SKILL.md @@ -134,4 +134,4 @@ sctx docs config - **`references/schema-authoring.md`** — schema file structure, `$metadata`, JSONata rules (run `sctx docs schema` for schema dialect reference) - **`references/schema-design.md`** — process for designing a schema from existing content -For CLI and config reference, use `sctx docs ` (topics: `concepts`, `config`, `schema`, `rules`, `semantic`). +For CLI and config reference, use `sctx docs ` (topics: `concepts`, `config`, `schema`, `rules`, `semantic-rules`). diff --git a/plugin/skills/structured-context/references/schema-authoring.md b/plugin/skills/structured-context/references/schema-authoring.md index ba2753f..5a6095b 100644 --- a/plugin/skills/structured-context/references/schema-authoring.md +++ b/plugin/skills/structured-context/references/schema-authoring.md @@ -66,7 +66,7 @@ Relationships define how sub-entities (nodes inside other files) are parsed and Rules are a flat array. Each rule's `category` (`validation`, `coherence`, `workflow`, `best-practice`) supplies its default severity: `validation` → `error`, `coherence` and `workflow` → `warning`, `best-practice` → `info`. Only `error` violations fail validation; warnings and info are reported only. An optional per-rule `severity` (`error` | `warning` | `info`) overrides the category default — prefer picking the right category over overriding. -Rules default to `mode: "expression"` (a JSONata `check`). Set `mode: "semantic"` for rules an LLM or decision model judges against a `rubric` (falls back to `description`): they take no `check` or `scope`, may declare `verdicts` (verdict → description, or `{ description, guidance }`; default `pass`/`fail`) with a `severity` map saying which verdicts are `error`/`warning`/`info`/`ignore`, and only run with `--semantic`. Run `sctx docs semantic` for the full semantic rule reference. A `when` JSONata test (any rule) limits a rule to the nodes where it returns true. An optional `gatesDescendants` controls whether a violation suppresses rule checks on the node's subtree (default: true for `error` severity). +Rules default to `mode: "expression"` (a JSONata `check`). Set `mode: "semantic"` for rules an LLM or decision model judges against a `rubric` (falls back to `description`): they take no `check` or `scope`, may declare `verdicts` (verdict → description, or `{ description, guidance }`; default `pass`/`fail`) with a `severity` map saying which verdicts are `error`/`warning`/`info`/`ignore`, and only run with `--semantic`. Run `sctx docs semantic-rules` for the full semantic rule reference. A `when` JSONata test (any rule) limits a rule to the nodes where it returns true. An optional `gatesDescendants` controls whether a violation suppresses rule checks on the node's subtree (default: true for `error` severity). ## Metadata composition semantics diff --git a/src/commands/docs.ts b/src/commands/docs.ts index 793b6ed..339579c 100644 --- a/src/commands/docs.ts +++ b/src/commands/docs.ts @@ -6,7 +6,7 @@ const TOPICS: Record = { config: 'config.md', schema: 'schemas.md', rules: 'rules.md', - semantic: 'semantic-rules.md', + 'semantic-rules': 'semantic-rules.md', }; export function docs(topic?: string): void { diff --git a/src/config.ts b/src/config.ts index 3655812..6f781e5 100644 --- a/src/config.ts +++ b/src/config.ts @@ -38,7 +38,6 @@ const JUDGE_CONFIG_SCHEMA = { endpoint: { type: 'string' }, providers: { type: 'object', additionalProperties: JUDGE_PROVIDER_SCHEMA }, default: { type: 'string' }, - autoRun: { type: 'boolean' }, }, additionalProperties: false, } as const; diff --git a/src/judge/types.ts b/src/judge/types.ts index 1f29f05..6a78a8b 100644 --- a/src/judge/types.ts +++ b/src/judge/types.ts @@ -73,7 +73,6 @@ export interface JudgeConfig { providers?: Record; /** Provider for rules that name none. Needed when several providers are configured. */ default?: string; - autoRun?: boolean; } /** Record of a completed semantic evaluation. */ diff --git a/src/validate.ts b/src/validate.ts index 3ca3e55..fbde912 100644 --- a/src/validate.ts +++ b/src/validate.ts @@ -184,7 +184,7 @@ export interface ValidateSpaceOptions { /** * Resolve the semantic judge for a run: none unless semantic evaluation is requested (option, else - * space/config `semantic` or `judge.autoRun`). An injected provider wins over the configured one. + * space/config `semantic`). An injected provider wins over the configured one. */ function resolveJudge( requested: boolean | undefined, @@ -194,10 +194,7 @@ function resolveJudge( judgeName?: string, ): SemanticJudgeProvider | undefined { // Naming a judge on the command line asks for semantic evaluation unless it is switched off explicitly. - const enabled = - requested ?? - (judgeName !== undefined || - (space.semantic ?? space.judge?.autoRun ?? config.semantic ?? config.judge?.autoRun ?? false)); + const enabled = requested ?? (judgeName !== undefined || (space.semantic ?? config.semantic ?? false)); if (!enabled) return undefined; return createJudge((space.judge ?? config.judge) as JudgeConfig | undefined, process.env, injected, judgeName); } From 69d4fb58ea165bff535fe449dbaa5a56a42af5db Mon Sep 17 00:00:00 2001 From: Roger Barnes Date: Wed, 14 Oct 2026 17:00:04 +1100 Subject: [PATCH 11/22] test(judge): opt-in live judge tests; fix --judge ignored by validate and gatedRules missing from JSON bun run test:judge sets SCTX_LIVE_JUDGE_TESTS=1 and runs judge-test/ against a fixture space with each provider whose API key is set. Without the variable the tests skip, so bun test stays offline. The validate action did not pass --judge through, so naming a judge there ran no semantic rules. validate --json also left out gatedRules, which the docs list. --- AGENTS.md | 1 + judge-test/fixtures/schema.json | 52 ++++++++++++++ .../space/Build an AI receipt scanner.md | 5 ++ .../fixtures/space/Faster tax filing.md | 4 ++ .../I cannot find my receipts at tax time.md | 5 ++ .../fixtures/space/Receipt inbox by email.md | 5 ++ .../space/Use a third-party OCR API.md | 5 ++ judge-test/judges.test.ts | 69 +++++++++++++++++++ package.json | 1 + src/commands/validate.ts | 1 + src/index.ts | 3 +- tests/commands/validate-judge-flag.test.ts | 47 +++++++++++++ 12 files changed, 197 insertions(+), 1 deletion(-) create mode 100644 judge-test/fixtures/schema.json create mode 100644 judge-test/fixtures/space/Build an AI receipt scanner.md create mode 100644 judge-test/fixtures/space/Faster tax filing.md create mode 100644 judge-test/fixtures/space/I cannot find my receipts at tax time.md create mode 100644 judge-test/fixtures/space/Receipt inbox by email.md create mode 100644 judge-test/fixtures/space/Use a third-party OCR API.md create mode 100644 judge-test/judges.test.ts create mode 100644 tests/commands/validate-judge-flag.test.ts diff --git a/AGENTS.md b/AGENTS.md index d6bc782..58293c7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -57,6 +57,7 @@ For most development only the main unit tests need re-running regularly. - `bun run test:hook` — unit test plugin hooks (`hook-test/unit/`) - hook development only - `bun run test:hook:e2e` — test plugin hooks in Claude Code (`hook-test/`) - hook development only - `bun run test:smoke` — smoke tests run against locally configured spaces - only use when changes could affect compatibility. +- `bun run test:judge` — live judge tests (`judge-test/`): real provider calls on a fixture space, for each provider whose API key is set (`OPENROUTER_API_KEY`, `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`). Costs money; only use when changing judges, prompts or semantic rule behaviour. ## Debugging diff --git a/judge-test/fixtures/schema.json b/judge-test/fixtures/schema.json new file mode 100644 index 0000000..a12843d --- /dev/null +++ b/judge-test/fixtures/schema.json @@ -0,0 +1,52 @@ +{ + "$schema": "https://raw.githubusercontent.com/mindsocket/structured-context/main/schemas/generated/latest/_structured_context_schema_meta.json", + "$id": "sctx://judge-test", + "$metadata": { + "hierarchy": { "levels": ["outcome", "opportunity", "solution"] }, + "rules": [ + { + "id": "opportunity-framing", + "category": "coherence", + "mode": "semantic", + "type": "opportunity", + "description": "The opportunity is a customer need or pain point, not a solution or a business goal", + "verdicts": { + "customer-need": "A need or pain point, in terms of what the customer experiences", + "solution": { + "description": "A feature, product or solution the business could build", + "guidance": "Restate it as the customer problem this solution would solve." + } + }, + "severity": { "solution": "error" } + }, + { + "id": "solution-addresses-parent", + "category": "best-practice", + "mode": "semantic", + "type": "solution", + "description": "The solution plausibly addresses its parent opportunity" + } + ] + }, + "oneOf": [ + { + "type": "object", + "properties": { "type": { "const": "outcome" }, "title": { "type": "string" } }, + "required": ["type", "title"] + }, + { + "type": "object", + "properties": { + "type": { "const": "opportunity" }, + "title": { "type": "string" }, + "parent": { "type": "string" } + }, + "required": ["type", "title"] + }, + { + "type": "object", + "properties": { "type": { "const": "solution" }, "title": { "type": "string" }, "parent": { "type": "string" } }, + "required": ["type", "title"] + } + ] +} diff --git a/judge-test/fixtures/space/Build an AI receipt scanner.md b/judge-test/fixtures/space/Build an AI receipt scanner.md new file mode 100644 index 0000000..09c381c --- /dev/null +++ b/judge-test/fixtures/space/Build an AI receipt scanner.md @@ -0,0 +1,5 @@ +--- +type: opportunity +parent: "[[Faster tax filing]]" +--- +Add an AI-powered receipt scanner to the mobile app with OCR and automatic categorisation. diff --git a/judge-test/fixtures/space/Faster tax filing.md b/judge-test/fixtures/space/Faster tax filing.md new file mode 100644 index 0000000..8b92684 --- /dev/null +++ b/judge-test/fixtures/space/Faster tax filing.md @@ -0,0 +1,4 @@ +--- +type: outcome +--- +Customers file their tax return in under an hour. diff --git a/judge-test/fixtures/space/I cannot find my receipts at tax time.md b/judge-test/fixtures/space/I cannot find my receipts at tax time.md new file mode 100644 index 0000000..820fe85 --- /dev/null +++ b/judge-test/fixtures/space/I cannot find my receipts at tax time.md @@ -0,0 +1,5 @@ +--- +type: opportunity +parent: "[[Faster tax filing]]" +--- +In interviews, customers said they spend hours at tax time hunting through email and drawers for receipts, and often give up on deductions they are owed. diff --git a/judge-test/fixtures/space/Receipt inbox by email.md b/judge-test/fixtures/space/Receipt inbox by email.md new file mode 100644 index 0000000..ad6d40c --- /dev/null +++ b/judge-test/fixtures/space/Receipt inbox by email.md @@ -0,0 +1,5 @@ +--- +type: solution +parent: "[[I cannot find my receipts at tax time]]" +--- +Forward receipts to a personal email address; we store and tag them by tax category. diff --git a/judge-test/fixtures/space/Use a third-party OCR API.md b/judge-test/fixtures/space/Use a third-party OCR API.md new file mode 100644 index 0000000..c5dcec8 --- /dev/null +++ b/judge-test/fixtures/space/Use a third-party OCR API.md @@ -0,0 +1,5 @@ +--- +type: solution +parent: "[[Build an AI receipt scanner]]" +--- +Use a third-party OCR API. diff --git a/judge-test/judges.test.ts b/judge-test/judges.test.ts new file mode 100644 index 0000000..facc32e --- /dev/null +++ b/judge-test/judges.test.ts @@ -0,0 +1,69 @@ +/** + * Live judge tests: call real providers on a small fixture space. They cost money, so they run only with + * SCTX_LIVE_JUDGE_TESTS=1 (`bun run test:judge`), and each provider only when its API key is set. + */ +import { describe, expect, it } from 'bun:test'; +import { join } from 'node:path'; + +const ROOT = join(import.meta.dir, '..'); +const CONFIG = join(import.meta.dir, 'fixtures', 'config.json'); +const LIVE = process.env.SCTX_LIVE_JUDGE_TESTS === '1'; + +const PROVIDERS = [ + { name: 'jev', keyEnv: 'OPENROUTER_API_KEY' }, + { name: 'anthropic', keyEnv: 'ANTHROPIC_API_KEY' }, + { name: 'openai', keyEnv: 'OPENAI_API_KEY' }, +]; + +const NEED = 'I cannot find my receipts at tax time.md'; +const SOLUTION = 'Build an AI receipt scanner.md'; +const GATED = 'Use a third-party OCR API.md'; + +type Output = { + ruleViolations: Array<{ file: string; ruleId: string; severity: string; verdict?: string; guidance?: string }>; + semanticEvaluations: Array<{ file: string; ruleId: string; verdict: string; pass: boolean }>; + gatedRules: Array<{ file: string; ruleId: string }>; +}; + +function validate(...args: string[]): { exitCode: number; output: Output } { + const result = Bun.spawnSync(['bun', 'run', 'src/index.ts', 'validate', 'judge-test', '--json', ...args], { + cwd: ROOT, + env: { ...process.env, SCTX_CONFIG: CONFIG }, + stdout: 'pipe', + stderr: 'pipe', + }); + const stdout = new TextDecoder().decode(result.stdout); + if (!stdout.trim().startsWith('{')) throw new Error(new TextDecoder().decode(result.stderr) || stdout); + return { exitCode: result.exitCode, output: JSON.parse(stdout) }; +} + +describe.skipIf(!LIVE)('live judges', () => { + for (const { name, keyEnv } of PROVIDERS) { + it.skipIf(!process.env[keyEnv])( + `${name}: classifies clear-cut opportunities and gates below an error verdict`, + () => { + const { exitCode, output } = validate('--judge', name); + const verdict = (file: string) => + output.semanticEvaluations.find((e) => e.file === file && e.ruleId === 'opportunity-framing')?.verdict; + + expect(verdict(NEED)).toBe('customer-need'); + expect(verdict(SOLUTION)).toBe('solution'); + + const violation = output.ruleViolations.find((v) => v.file === SOLUTION); + expect(violation).toMatchObject({ severity: 'error', verdict: 'solution' }); + expect(violation?.guidance).toContain('customer problem'); + expect(exitCode).toBe(1); + + // The error verdict gates the solution beneath it, so it is not judged. + expect(output.gatedRules).toContainEqual( + expect.objectContaining({ file: GATED, ruleId: 'solution-addresses-parent' }), + ); + expect(output.semanticEvaluations.some((e) => e.file === GATED)).toBe(false); + + // No judge errors: every evaluation has a real verdict. + expect(output.ruleViolations.filter((v) => !v.verdict)).toEqual([]); + }, + 60_000, + ); + } +}); diff --git a/package.json b/package.json index 9915c6a..bce31bd 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "diagram": "bun run src/index.ts diagram", "test": "bun test tests/", "test:smoke": "bun test smoke/", + "test:judge": "SCTX_LIVE_JUDGE_TESTS=1 bun test judge-test/", "test:hook": "bun test hook-test/unit/", "test:hook:e2e": "bun test hook-test/hooks.test.ts", "test:all": "bun test", diff --git a/src/commands/validate.ts b/src/commands/validate.ts index 3a71f82..63d9669 100644 --- a/src/commands/validate.ts +++ b/src/commands/validate.ts @@ -92,6 +92,7 @@ export async function validate( skippedByDepth: result.skippedByDepth, depthLimit: result.depthLimit, semanticEvaluations: result.semanticEvaluations, + gatedRules: result.gatedRules, }, null, 2, diff --git a/src/index.ts b/src/index.ts index f84a17d..eb56e68 100755 --- a/src/index.ts +++ b/src/index.ts @@ -128,6 +128,7 @@ program depth: options.depth, includeGated: options.includeGated, semantic: options.semantic, + judge: options.judge, ruleFilter: options.ruleFilter, }), ); @@ -216,7 +217,7 @@ program.addCommand(schemasCmd); program .command('docs [topic]') - .description('Show documentation (no arg: README; topics: concepts, config, schema, rules, semantic)') + .description('Show documentation (no arg: README; topics: concepts, config, schema, rules, semantic-rules)') .action((topic?: string) => docs(topic)); const renderCmd = new Command('render').description('Render a space in a given format'); diff --git a/tests/commands/validate-judge-flag.test.ts b/tests/commands/validate-judge-flag.test.ts new file mode 100644 index 0000000..5c7390d --- /dev/null +++ b/tests/commands/validate-judge-flag.test.ts @@ -0,0 +1,47 @@ +import { afterAll, describe, expect, it } from 'bun:test'; +import { writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { makeTempSpace } from '../helpers/temp-space'; + +const ROOT = join(import.meta.dir, '..', '..'); +const space = makeTempSpace( + [{ id: 'step-fit', type: 'step', mode: 'semantic', category: 'coherence', description: 'Step fits its task' }], + { judge: { providers: { f: { type: 'fake' } } } }, +); +const configPath = join(space.dir, 'config.json'); +writeFileSync(configPath, JSON.stringify(space.config)); +afterAll(() => space.cleanup()); + +function spawn(...args: string[]) { + const result = Bun.spawnSync(['bun', 'run', 'src/index.ts', ...args, '--json'], { + cwd: ROOT, + env: { ...process.env, SCTX_CONFIG: configPath }, + stdout: 'pipe', + stderr: 'pipe', + }); + return { ...result, stderr: new TextDecoder().decode(result.stderr) }; +} + +const run = (...args: string[]) => JSON.parse(new TextDecoder().decode(spawn(...args).stdout)); + +describe('--judge on the CLI', () => { + it('validate: naming a judge runs semantic rules without --semantic', () => { + expect(run('validate', 'ts', '--judge', 'f').semanticEvaluations).toHaveLength(1); + }); + + // An unknown name fails only if the flag reaches the judge. + for (const args of [ + ['validate', 'ts'], + ['validate-file', space.file('S.md')], + ]) { + it(`${args[0]} passes --judge to the judge`, () => { + const result = spawn(...args, '--judge', 'missing'); + expect(result.exitCode).toBe(1); + expect(result.stderr).toContain('missing'); + }); + } + + it('--no-semantic wins over --judge', () => { + expect(run('validate', 'ts', '--judge', 'f', '--no-semantic').semanticEvaluations).toBeUndefined(); + }); +}); From b2571b3a6b6f20149a6f1cd7dd324e7d8e4813bb Mon Sep 17 00:00:00 2001 From: Roger Barnes Date: Wed, 14 Oct 2026 17:28:58 +1100 Subject: [PATCH 12/22] refactor(api): keep judge and rule-filter internals out of the public API; guard API surface in AGENTS.md No consumer needs the judge providers, factory, judge types or rule filter, so src/api.ts and docs/api.md return to their previous scope. AGENTS.md now requires explicit agreement before any export is added. --- AGENTS.md | 3 ++- docs/api.md | 19 ++----------------- src/api.ts | 26 +------------------------- 3 files changed, 5 insertions(+), 43 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 58293c7..5193f6c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -38,12 +38,13 @@ There are several places that need reviewing and updating with any new feature o - **Simplicity first:** Strive for the simplest design that completely solves the problem. Prefer conventions and established filesystem patterns over adding new configuration, programmatic APIs, or abstractions. - **Explicitly state alternatives:** When planning or making architectural decisions, do not anchor to the first familiar mechanism. Formulate and state at least one alternative approach, weighing trade-offs, complexity, and ergonomic impact. - **Minimise blast radius:** Protect established boundaries and interfaces. Treat unnecessary ripple effects (such as making synchronous APIs asynchronous or adding dual APIs) as a design smell indicating the approach should be re-evaluated. +- **Public API surface:** Do not add exports to `src/api.ts`, `src/types.ts` or `src/testing.ts` (the `structured-context/api`, `/types` and `/testing` entry points), or add types or hooks to the plugin interface, without the maintainer's explicit agreement on that specific surface. Exporting something to make docs complete, to tidy up, or because a feature has a public-looking class is not a reason. If a change seems to need a new export, stop and propose it, with the consumer that needs it. Internal modules can change freely. - **Where a setting lives:** Config says *what* to use, CLI flags override it for one run, and the environment holds secrets and the config location only (`apiKeyEnv` names a secret; `SCTX_CONFIG` locates the config). Do not add env vars that set config values, and never infer a choice (such as a judge provider) from a variable that happens to be set: a key present for another tool must not be spent silently. - **Failures are visible:** A rule or expression that cannot be evaluated is reported as a violation, never skipped or treated as a pass. Likewise a misconfiguration fails before any work or spend. ## Key Files -- `src/api.ts` — Public library entry point (`structured-context/api`). Re-exports the supported programmatic surface. Keep CLI-only concerns out of it. +- `src/api.ts` — Public library entry point (`structured-context/api`). Re-exports the supported programmatic surface. Keep CLI-only concerns out of it. Do not extend it without agreement (see **Public API surface** above). - `schemas/` — Bundled default schema files (JSON5) using the structured-context schema dialect and top-level `$metadata`. Files starting with `_` are "partials" (fragments for `$ref`). - `src/schema/metadata-contract.ts` — Single source of truth for the `$metadata` contract - `schemas/generated/latest/_structured_context_schema_meta.json` — Generated metaschema for the newest dialect (generated on build or with `bun run generate:schema-meta`). `schemas/generated//` folders are frozen copies, created by the `version` script (`generate:schema-meta --release`) only when a release changes the dialect; never edit them. `src/schema/meta-schemas.ts` loads them at runtime. diff --git a/docs/api.md b/docs/api.md index ccde51b..c24a881 100644 --- a/docs/api.md +++ b/docs/api.md @@ -18,8 +18,8 @@ see [architecture.md](architecture.md#information-flow) for that flow. | `loadSpaceGraph(spaceName, config, options?)` | Read, assemble, and optionally filter a space into a navigable `SpaceGraph`. `options.hierarchy` names the hierarchy that structures the graph (default: the main hierarchy). | | `assembleSpaceGraph(context, options?)` | Assemble (and optionally filter) a graph from a context you already hold — reuses the context instead of rebuilding it. | | `buildSpaceGraph(nodes, hierarchy?)` | Build a navigable `SpaceGraph` directly from an array of `SpaceNode`s and a `Hierarchy` definition. | -| `validateSpace(context, options?)` | Run full validation (schema, references, rules, duplicates) over a space. Options: `depth`, `includeGated`, `semantic`, `judge`, `judgeProvider`, `ruleFilter`. | -| `validateFile(filePath, config, options?)` | Validate a single file within its space. Options: `includeGated`, `semantic`, `judge`, `judgeProvider`, `ruleFilter`. | +| `validateSpace(context, options?)` | Run full validation (schema, references, rules, duplicates) over a space. | +| `validateFile(filePath, config)` | Validate a single file within its space. | | `formatWikilink(target, alias?)` | Format a target into an Obsidian wikilink (`[[target]]` or `[[target|alias]]`). | | `parseWikilink(link)` | Parse a wikilink or target string into `{ target, anchor?, displayText? }`. | | `wikilinkToTarget(link)` | Extract the plain target lookup key from a wikilink string. | @@ -27,21 +27,6 @@ see [architecture.md](architecture.md#information-flow) for that flow. `ValidationResult.ruleViolations` carries each violation's effective `severity` (`error`, `warning` or `info`). Only `error` severity violations should be treated as failures; `validateFile` reports `warning` and `info` rule violations under `warnings` rather than `errors`. -## Semantic judging - -Semantic rules are evaluated only when `semantic: true` is passed (or configured; see [config.md](config.md#judge-semantic-rules)); the provider must be named explicitly, never inferred from the environment. The judge is built from the configured providers, with `judge` naming one to use for every rule; pass `judgeProvider` to inject your own. - -| Export | Purpose | -| --- | --- | -| `createJudge(config?, env?, custom?, override?)` | Build the `RoutedJudgeProvider` for a run: routes each rule to the provider it names (`judge`), else the default; `override` names one for every rule. | -| `createJudgeProvider(config?, env?, custom?, name?)` | Resolve a single `SemanticJudgeProvider` from `JudgeConfig` and the environment. Throws `JudgeConfigError` when misconfigured. | -| `AnthropicJudgeProvider`, `OpenAIJudgeProvider`, `JevJudgeProvider` | Built-in providers. | -| `PerRuleJudgeProvider` | Base class for providers that can only answer one rule per request; implement `judgeRule`. | -| `FakeJudgeProvider` | Deterministic provider for tests. | -| `parseRuleFilter(specs)`, `filterRules(rules, filter)` | Build and apply a `RuleFilter` (`mode`, `category`, `severity`, `type`, `id`). | - -A `SemanticJudgeProvider` implements `evaluate({ node, rules })` and returns a result per rule id: a `SemanticJudgeVerdict` (`verdict` from the rule's closed set, optional `confidence`, `probabilities`, `evidence`) or `{ error }`. All of a node's semantic rules arrive in one call, so providers that support it can batch. `ValidationResult` adds `semanticEvaluations` and `gatedRules`. - ## Testing utilities Test fixture builders are available via `structured-context/testing` for plugin and integration tests: diff --git a/src/api.ts b/src/api.ts index b8dfe20..9362858 100644 --- a/src/api.ts +++ b/src/api.ts @@ -22,28 +22,6 @@ export type { AnySchemaObject, SchemaObject, ValidateFunction } from 'ajv'; export type { Config, SpaceConfig } from './config'; export { loadConfig, resolveSchema, resolveSchemaPath, setConfigPath } from './config'; export { PLUGIN_PREFIX } from './constants'; -export type { - GatedRuleRecord, - JudgeConfig, - JudgeProviderDefinition, - SemanticEvaluationRecord, - SemanticJudgeInput, - SemanticJudgeProvider, - SemanticJudgeResult, - SemanticJudgeVerdict, - SemanticRuleSpec, -} from './judge'; -export { - AnthropicJudgeProvider, - createJudge, - createJudgeProvider, - FakeJudgeProvider, - JevJudgeProvider, - JudgeConfigError, - OpenAIJudgeProvider, - PerRuleJudgeProvider, - RoutedJudgeProvider, -} from './judge'; export { assembleSpaceGraph, loadSpaceGraph } from './load-space-graph'; export type { ParseHook, @@ -74,8 +52,6 @@ export { wikilinkToTarget, } from './read/wikilink-utils'; export type { SharedEmbeddingFields } from './schema/metadata-contract'; -export type { RuleFilter } from './schema/rule-filter'; -export { filterRules, parseRuleFilter } from './schema/rule-filter'; export { bundledSchemasDir, loadSchema, type PluginSchemaSource, setBundledSchemasDir } from './schema/schema'; export { createSpaceContext, SpaceNotFoundError } from './space-context'; export { buildSpaceGraph, type SpaceGraph } from './space-graph'; @@ -103,5 +79,5 @@ export type { UnresolvedRef, ValidateFileOutput, } from './types'; -export type { ValidateFileOptions, ValidateSpaceOptions, ValidationResult } from './validate'; +export type { ValidationResult } from './validate'; export { validateFile, validateSpace } from './validate'; From 595e7a9559cbc64346c88c8192176090e66a5b6e Mon Sep 17 00:00:00 2001 From: Roger Barnes Date: Wed, 14 Oct 2026 17:35:09 +1100 Subject: [PATCH 13/22] refactor(judge): drop semanticEvaluations and gatedRules, slim judge types Violations and coverage gates already report what users act on, so the per-evaluation and per-gated-rule records go, with their JSON fields and CLI summary lines. The judge input node is a Pick of SpaceNode, the rule spec no longer carries failOn (severity stays out of the judge's input), and JudgeConfig extends the provider definition with a real type union. --- docs/semantic-rules.md | 6 +-- judge-test/judges.test.ts | 42 +++++++++-------- src/commands/validate.ts | 13 ------ src/config.ts | 4 +- src/judge/factory.ts | 2 +- src/judge/index.ts | 2 - src/judge/judge-node.ts | 21 +-------- src/judge/provider/fake.ts | 6 ++- src/judge/types.ts | 49 ++++---------------- src/judge/verdicts.ts | 4 +- src/schema/gating.ts | 20 +------- src/schema/validate-rules.ts | 26 ++--------- src/validate.ts | 23 ++------- tests/commands/validate-judge-flag.test.ts | 8 +--- tests/judge/factory.test.ts | 5 +- tests/judge/helpers.ts | 6 +-- tests/judge/verdicts.test.ts | 2 - tests/schema/validate-rules-semantic.test.ts | 18 ++----- tests/validate/semantic-rule-shape.test.ts | 24 +++++----- tests/validate/semantic-validation.test.ts | 35 ++++++-------- 20 files changed, 93 insertions(+), 223 deletions(-) diff --git a/docs/semantic-rules.md b/docs/semantic-rules.md index 921a4b5..0ecb925 100644 --- a/docs/semantic-rules.md +++ b/docs/semantic-rules.md @@ -78,7 +78,7 @@ The verdict is a classification. The severity says what the verdict means for th - With custom verdicts, `severity` is a map from verdict to `error`, `warning`, `info` or `ignore`. - A verdict with `error`, `warning` or `info` reports a violation at that severity. - A verdict that is not in the map reports no violation. At least one verdict must report no violation. -- `ignore` also reports no violation. Use it to show that you chose to ignore a verdict. The verdict is still recorded in `--json` output. +- `ignore` also reports no violation. Use it to show that you chose to ignore a verdict. - With the default verdicts, `severity` is one value for `fail`, the same as for an expression rule. Gating uses the severity of the verdict. A verdict with `error` severity gates the node's subtree, as for any rule. `gatesDescendants` changes this, as for any rule. @@ -86,10 +86,10 @@ Gating uses the severity of the verdict. A verdict with `error` severity gates t ## How judging works - **Order.** On each node, expression rules run first. Then sctx sends all of the node's semantic rules to the judge in one call. -- **Gating.** sctx does not judge a gated node. The JSON output lists the rules it did not judge under `gatedRules`. +- **Gating.** sctx does not judge a gated node. [Coverage output](rules.md#coverage-output) shows which nodes were gated. - **Judge errors.** If the judge fails (for example, a network error or output that sctx cannot read), sctx reports a violation. A judge error does not gate. - **Input.** The judge gets the node's title, type, parent titles and schema data, including frontmatter. -- **Output.** Violations include the verdict, the judge's evidence and any guidance. `--json` adds `semanticEvaluations`: the verdict, confidence and evidence for each node and rule. +- **Output.** A violation from a semantic rule also includes the verdict, the judge's evidence and any guidance. `sctx validate-file` judges only the file and the rules on its ancestors that can gate it. Thus one file costs at most one judge call for each ancestor level, plus one. diff --git a/judge-test/judges.test.ts b/judge-test/judges.test.ts index facc32e..cd3e821 100644 --- a/judge-test/judges.test.ts +++ b/judge-test/judges.test.ts @@ -15,14 +15,19 @@ const PROVIDERS = [ { name: 'openai', keyEnv: 'OPENAI_API_KEY' }, ]; -const NEED = 'I cannot find my receipts at tax time.md'; const SOLUTION = 'Build an AI receipt scanner.md'; const GATED = 'Use a third-party OCR API.md'; type Output = { - ruleViolations: Array<{ file: string; ruleId: string; severity: string; verdict?: string; guidance?: string }>; - semanticEvaluations: Array<{ file: string; ruleId: string; verdict: string; pass: boolean }>; - gatedRules: Array<{ file: string; ruleId: string }>; + ruleViolations: Array<{ + file: string; + ruleId: string; + severity: string; + description: string; + verdict?: string; + guidance?: string; + }>; + coverageGates: Array<{ file: string; ruleId?: string; gatedFiles: string[] }>; }; function validate(...args: string[]): { exitCode: number; output: Output } { @@ -43,25 +48,24 @@ describe.skipIf(!LIVE)('live judges', () => { `${name}: classifies clear-cut opportunities and gates below an error verdict`, () => { const { exitCode, output } = validate('--judge', name); - const verdict = (file: string) => - output.semanticEvaluations.find((e) => e.file === file && e.ruleId === 'opportunity-framing')?.verdict; - expect(verdict(NEED)).toBe('customer-need'); - expect(verdict(SOLUTION)).toBe('solution'); - - const violation = output.ruleViolations.find((v) => v.file === SOLUTION); - expect(violation).toMatchObject({ severity: 'error', verdict: 'solution' }); + // No judge errors, and the only violation is the solution in disguise. + expect(output.ruleViolations.filter((v) => v.description.includes('Judge error'))).toEqual([]); + expect(output.ruleViolations).toHaveLength(1); + const violation = output.ruleViolations[0]; + expect(violation).toMatchObject({ + file: SOLUTION, + ruleId: 'opportunity-framing', + severity: 'error', + verdict: 'solution', + }); expect(violation?.guidance).toContain('customer problem'); expect(exitCode).toBe(1); - // The error verdict gates the solution beneath it, so it is not judged. - expect(output.gatedRules).toContainEqual( - expect.objectContaining({ file: GATED, ruleId: 'solution-addresses-parent' }), - ); - expect(output.semanticEvaluations.some((e) => e.file === GATED)).toBe(false); - - // No judge errors: every evaluation has a real verdict. - expect(output.ruleViolations.filter((v) => !v.verdict)).toEqual([]); + // The error verdict gates the solution beneath it, so that solution is not judged. + expect(output.coverageGates).toEqual([ + expect.objectContaining({ file: SOLUTION, ruleId: 'opportunity-framing', gatedFiles: [GATED] }), + ]); }, 60_000, ); diff --git a/src/commands/validate.ts b/src/commands/validate.ts index 63d9669..b02f78e 100644 --- a/src/commands/validate.ts +++ b/src/commands/validate.ts @@ -91,8 +91,6 @@ export async function validate( coverageGates: result.coverageGates, skippedByDepth: result.skippedByDepth, depthLimit: result.depthLimit, - semanticEvaluations: result.semanticEvaluations, - gatedRules: result.gatedRules, }, null, 2, @@ -222,17 +220,6 @@ export async function validate( } } - if (result.semanticEvaluations && result.semanticEvaluations.length > 0) { - const passed = result.semanticEvaluations.filter((e) => e.pass).length; - const failed = result.semanticEvaluations.filter((e) => !e.pass).length; - console.log( - `\nSemantic evaluations: ${result.semanticEvaluations.length} total (${passed} passed, ${failed} failed)`, - ); - } - if (result.gatedRules && result.gatedRules.length > 0) { - console.log(`Gated semantic rules (skipped due to upstream errors): ${result.gatedRules.length}`); - } - if (result.hierarchyViolations.length > 0) { console.log(`\nHierarchy violations:`); for (const v of result.hierarchyViolations) { diff --git a/src/config.ts b/src/config.ts index 6f781e5..776127a 100644 --- a/src/config.ts +++ b/src/config.ts @@ -4,7 +4,7 @@ import { dirname, isAbsolute, join, resolve } from 'node:path'; import Ajv2020 from 'ajv/dist/2020'; import JSON5 from 'json5'; import { ENV_CONFIG_VAR, XDG_CONFIG_DIR } from './constants'; -import type { JudgeConfig } from './judge/types'; +import { JUDGE_PROVIDER_TYPES, type JudgeConfig } from './judge/types'; import { normalizePluginName, resolvePluginSchemasDir, @@ -13,8 +13,6 @@ import { } from './plugins/util'; import { bundledSchemasDir } from './schema/schema'; -const JUDGE_PROVIDER_TYPES = ['anthropic', 'openai', 'jev', 'fake', 'none']; - const JUDGE_PROVIDER_SCHEMA = { type: 'object', properties: { diff --git a/src/judge/factory.ts b/src/judge/factory.ts index 45e4d63..885d9c2 100644 --- a/src/judge/factory.ts +++ b/src/judge/factory.ts @@ -22,7 +22,7 @@ const DEFAULT_KEY_ENV = { const PROVIDER_TYPES = ['anthropic', 'openai', 'jev', 'fake'] as const; -/** Name of the provider defined by the flat `judge: { provider, ... }` shorthand. */ +/** Name of the provider defined by the flat `judge: { type, ... }` shorthand. */ export const DEFAULT_JUDGE_NAME = 'default'; /** The judge is misconfigured (no provider, unknown provider, missing key). Reported as a plain CLI error. */ diff --git a/src/judge/index.ts b/src/judge/index.ts index aae3c8a..da74efd 100644 --- a/src/judge/index.ts +++ b/src/judge/index.ts @@ -10,10 +10,8 @@ export type { OpenAIJudgeProviderOptions } from './provider/openai'; export { OpenAIJudgeProvider } from './provider/openai'; export { PerRuleJudgeProvider } from './provider/per-rule'; export type { - GatedRuleRecord, JudgeConfig, JudgeProviderDefinition, - SemanticEvaluationRecord, SemanticJudgeInput, SemanticJudgeProvider, SemanticJudgeResult, diff --git a/src/judge/judge-node.ts b/src/judge/judge-node.ts index b1e69bc..4e7ff51 100644 --- a/src/judge/judge-node.ts +++ b/src/judge/judge-node.ts @@ -1,5 +1,5 @@ import type { Rule, RuleSeverity, SpaceNode } from '../types'; -import type { SemanticEvaluationRecord, SemanticJudgeProvider } from './types'; +import type { SemanticJudgeProvider } from './types'; import { isJudgeError, resolveSemanticRule } from './verdicts'; export type NodeJudgement = { @@ -14,8 +14,6 @@ export type NodeJudgement = { evidence?: string; /** Set when the judge could not produce a verdict for this rule. */ error?: string; - /** Present when the judge produced a verdict. */ - record?: SemanticEvaluationRecord; }; /** Judge every given semantic rule on a node in one provider call. */ @@ -30,14 +28,7 @@ export async function judgeNode( let results: Awaited>; try { results = await judge.evaluate({ - node: { - label: node.label, - title: node.title, - type: node.type, - resolvedType: node.resolvedType, - schemaData: node.schemaData, - resolvedParents: node.resolvedParents.map((p) => ({ title: p.title, field: p.field })), - }, + node, rules: specs, }); } catch (err) { @@ -65,14 +56,6 @@ export async function judgeNode( verdict: result.verdict, guidance: failed ? resolved[i]!.guidance[result.verdict] : undefined, evidence: result.evidence, - record: { - file: node.label, - ruleId: rule.id, - verdict: result.verdict, - pass: !failed, - confidence: result.confidence, - evidence: result.evidence, - }, }; }); } diff --git a/src/judge/provider/fake.ts b/src/judge/provider/fake.ts index 9a3e217..863c4fc 100644 --- a/src/judge/provider/fake.ts +++ b/src/judge/provider/fake.ts @@ -27,7 +27,8 @@ function normalise(answer: FakeAnswer): SemanticJudgeResult { /** * In-memory fake judge for offline, deterministic tests. Configure with a single answer, a handler, - * or answers keyed by `ruleId:title`, `ruleId:label` or `ruleId` (most specific wins). Unmatched calls pass. + * or answers keyed by `ruleId:title`, `ruleId:label` or `ruleId` (most specific wins). Unmatched calls get `pass`, + * or the rule's first declared verdict. */ export class FakeJudgeProvider implements SemanticJudgeProvider { readonly name = 'fake'; @@ -45,7 +46,8 @@ export class FakeJudgeProvider implements SemanticJudgeProvider { } private async answer(node: SemanticJudgeInput['node'], rule: SemanticRuleSpec): Promise { - const passing = { verdict: Object.keys(rule.verdicts).find((v) => !rule.failOn.includes(v)) ?? 'pass' }; + // The fake does not know severities: default to `pass`, else the first declared verdict. + const passing = { verdict: 'pass' in rule.verdicts ? 'pass' : (Object.keys(rule.verdicts)[0] ?? 'pass') }; const fallback = { ...passing, evidence: `Fake passed evaluation for rule "${rule.id}"` }; const config = this.config; if (!config) return fallback; diff --git a/src/judge/types.ts b/src/judge/types.ts index 6a78a8b..c22594f 100644 --- a/src/judge/types.ts +++ b/src/judge/types.ts @@ -1,4 +1,4 @@ -import type { RuleCategory } from '../types'; +import type { RuleCategory, SpaceNode } from '../types'; /** A semantic rule as presented to a judge: the closed verdict set it must choose from. */ export interface SemanticRuleSpec { @@ -8,22 +8,13 @@ export interface SemanticRuleSpec { category: RuleCategory | string; /** Allowed verdicts, each with a description of when it applies. */ verdicts: Record; - /** Verdicts that count as a violation (those with a severity; `ignore`d verdicts pass). */ - failOn: string[]; /** Name of the judge provider the rule asks for, if any. */ judge?: string; } /** One node and every semantic rule to judge on it. Judges may answer all rules in a single call. */ export interface SemanticJudgeInput { - node: { - label: string; - title: string; - type: string; - resolvedType: string; - schemaData: Record; - resolvedParents?: Array<{ title: string; field: string }>; - }; + node: Pick; rules: SemanticRuleSpec[]; } @@ -47,10 +38,13 @@ export interface SemanticJudgeProvider { evaluate(input: SemanticJudgeInput): Promise>; } +/** Judge provider types. `none` is an inert placeholder for templates; judging with it is an error. */ +export const JUDGE_PROVIDER_TYPES = ['anthropic', 'openai', 'jev', 'fake', 'none'] as const; +export type JudgeProviderType = (typeof JUDGE_PROVIDER_TYPES)[number]; + /** One judge provider: a backend type plus how to reach it. */ export interface JudgeProviderDefinition { - /** `none` is an inert placeholder for templates; judging with it is an error. */ - type?: 'anthropic' | 'openai' | 'jev' | 'fake' | 'none' | string; + type: JudgeProviderType; model?: string; endpoint?: string; /** Prefer `apiKeyEnv`: keys do not belong in config files. */ @@ -60,35 +54,12 @@ export interface JudgeProviderDefinition { } /** - * Judge configuration from config file or options. Either the flat shorthand (`type`, `model`, ...), which - * defines one provider named "default", or several named `providers`. + * Judge configuration: the flat shorthand (provider fields directly on `judge`), which defines one provider + * named "default", and/or several named `providers`. */ -export interface JudgeConfig { - type?: 'anthropic' | 'openai' | 'jev' | 'fake' | 'none' | string; - model?: string; - apiKey?: string; - apiKeyEnv?: string; - endpoint?: string; +export interface JudgeConfig extends Partial { /** Named providers, for rules (or runs) that pick one by name. */ providers?: Record; /** Provider for rules that name none. Needed when several providers are configured. */ default?: string; } - -/** Record of a completed semantic evaluation. */ -export interface SemanticEvaluationRecord { - file: string; - ruleId: string; - verdict: string; - /** True when the verdict does not fail the rule (it has no severity, or is `ignore`d). */ - pass: boolean; - confidence?: number; - evidence?: string; -} - -/** Record of a rule gated/suppressed due to upstream failure. */ -export interface GatedRuleRecord { - file: string; - ruleId: string; - reason: string; -} diff --git a/src/judge/verdicts.ts b/src/judge/verdicts.ts index 6df6122..67646ea 100644 --- a/src/judge/verdicts.ts +++ b/src/judge/verdicts.ts @@ -54,8 +54,7 @@ export function resolveSemanticRule(rule: Rule): ResolvedSemanticRule { for (const [verdict, severity] of Object.entries(mapping)) { if (severity !== 'ignore') severities[verdict] = severity; } - const failOn = Object.keys(severities); - if (failOn.length >= Object.keys(verdicts).length) { + if (Object.keys(severities).length >= Object.keys(verdicts).length) { throw new Error(`Semantic rule "${rule.id}" has no passing verdict: every verdict has a severity.`); } @@ -67,7 +66,6 @@ export function resolveSemanticRule(rule: Rule): ResolvedSemanticRule { category: rule.category, judge: rule.judge, verdicts, - failOn, }, severities, guidance, diff --git a/src/schema/gating.ts b/src/schema/gating.ts index 66dd170..0d3f3cd 100644 --- a/src/schema/gating.ts +++ b/src/schema/gating.ts @@ -1,5 +1,5 @@ import { judgeNode } from '../judge/judge-node'; -import type { GatedRuleRecord, SemanticEvaluationRecord, SemanticJudgeProvider } from '../judge/types'; +import type { SemanticJudgeProvider } from '../judge/types'; import type { SpaceGraph } from '../space-graph'; import type { Rule, RuleSeverity, RuleViolation, SpaceNode } from '../types'; import { buildEvalContext, expressionErrorViolation, tryEvaluateExpression } from './evaluate-rule'; @@ -51,10 +51,6 @@ export type CoverageGate = { export type GatedRuleEvaluation = { violations: RuleViolation[]; gates: CoverageGate[]; - /** Semantic judgements made (empty unless a judge was supplied). */ - semanticEvaluations: SemanticEvaluationRecord[]; - /** Semantic rule/node pairs skipped because the node was gated (empty unless a judge was supplied). */ - gatedRules: GatedRuleRecord[]; }; export interface GatingOptions { @@ -127,8 +123,6 @@ export async function evaluateRulesWithGating( const judge = options.judge; const violations: RuleViolation[] = []; - const semanticEvaluations: SemanticEvaluationRecord[] = []; - const gatedRules: GatedRuleRecord[] = []; const gateOrigins = new Map(); function originFor( @@ -241,7 +235,6 @@ export async function evaluateRulesWithGating( const semantic = applicable.filter((r) => r.mode === 'semantic'); if (judge && semantic.length > 0) { for (const j of await judgeNode(judge, node, semantic)) { - if (j.record) semanticEvaluations.push(j.record); if (j.failed) { report(j.rule, { severity: j.severity, @@ -256,12 +249,6 @@ export async function evaluateRulesWithGating( return { blockingRuleId }; } - async function recordGatedSemantic(node: SpaceNode, reason: string): Promise { - for (const rule of await applicableRules(node)) { - if (rule.mode === 'semantic') gatedRules.push({ file: node.label, ruleId: rule.id, reason }); - } - } - // Non-hierarchy nodes sit outside the subtree cascade — always evaluated, never gated. for (const node of graph.nonHierarchy) { await evaluateApplicableRules(node); @@ -286,9 +273,6 @@ export async function evaluateRulesWithGating( const shouldEvaluate = !isGated || includeGated; const { blockingRuleId } = shouldEvaluate ? await evaluateApplicableRules(node) : {}; - if (!shouldEvaluate) { - await recordGatedSemantic(node, isGatedByAncestor ? 'Gated by an ancestor failure' : 'Gated by failure on node'); - } let childOrigin: string | undefined = ancestorOrigin; if (isGatedByAncestor) { @@ -312,7 +296,5 @@ export async function evaluateRulesWithGating( return { violations, gates: [...gateOrigins.values()].filter((g) => g.gatedFiles.length > 0), - semanticEvaluations, - gatedRules, }; } diff --git a/src/schema/validate-rules.ts b/src/schema/validate-rules.ts index 372b16c..83b6c74 100644 --- a/src/schema/validate-rules.ts +++ b/src/schema/validate-rules.ts @@ -1,5 +1,5 @@ import { judgeNode } from '../judge/judge-node'; -import type { GatedRuleRecord, SemanticEvaluationRecord, SemanticJudgeProvider } from '../judge/types'; +import type { SemanticJudgeProvider } from '../judge/types'; import type { Rule, RuleViolation, SpaceNode } from '../types'; import { buildEvalContext, expressionErrorViolation, tryEvaluateExpression } from './evaluate-rule'; import { ruleSeverity } from './rule-severity'; @@ -17,11 +17,6 @@ export interface ValidateRulesOptions { gatedNodeLabels?: Set; } -export type ValidatedRulesResult = RuleViolation[] & { - semanticEvaluations: SemanticEvaluationRecord[]; - gatedRules: GatedRuleRecord[]; -}; - /** * Validate nodes against rules metadata. * Evaluates deterministic expression rules and (when opt-in) semantic judge rules. @@ -29,16 +24,14 @@ export type ValidatedRulesResult = RuleViolation[] & { * @param nodes - All nodes in the space * @param rules - Rules metadata list * @param options - Validation options including semantic opt-in and judge provider - * @returns Array of rule violations with semanticEvaluations and gatedRules metadata attached + * @returns Rule violations */ export async function validateRules( nodes: SpaceNode[], rules: Rule[], options: ValidateRulesOptions = {}, -): Promise { +): Promise { const violations: RuleViolation[] = []; - const semanticEvaluations: SemanticEvaluationRecord[] = []; - const gatedRules: GatedRuleRecord[] = []; // Track nodes that have failed checks (for cascading gating) const failedNodeLabels = new Set(options.gatedNodeLabels || []); @@ -142,15 +135,9 @@ export async function validateRules( if (applicable.length === 0) continue; // Safe cascading default: gate semantic evaluation if node failed upstream checks - if (failedNodeLabels.has(node.label)) { - for (const rule of applicable) { - gatedRules.push({ file: node.label, ruleId: rule.id, reason: 'Gated by upstream error on node' }); - } - continue; - } + if (failedNodeLabels.has(node.label)) continue; for (const j of await judgeNode(judgeProvider, node, applicable)) { - if (j.record) semanticEvaluations.push(j.record); if (!j.failed) continue; violations.push({ file: node.label, @@ -167,8 +154,5 @@ export async function validateRules( } } - const result = violations as ValidatedRulesResult; - result.semanticEvaluations = semanticEvaluations; - result.gatedRules = gatedRules; - return result; + return violations; } diff --git a/src/validate.ts b/src/validate.ts index fbde912..f33eab1 100644 --- a/src/validate.ts +++ b/src/validate.ts @@ -2,7 +2,7 @@ import { isAbsolute, relative, resolve } from 'node:path'; import type { ErrorObject } from 'ajv'; import type { Config, SpaceConfig } from './config'; import { createJudge, RoutedJudgeProvider } from './judge/factory'; -import type { GatedRuleRecord, JudgeConfig, SemanticEvaluationRecord, SemanticJudgeProvider } from './judge/types'; +import type { JudgeConfig, SemanticJudgeProvider } from './judge/types'; import { readSpace } from './read/read-space'; import { computeHierarchyDepths, evaluateRulesWithGating, type GatingSeed } from './schema/gating'; import { filterRules, type RuleFilter } from './schema/rule-filter'; @@ -55,10 +55,6 @@ export interface ValidationResult { skippedByDepth: Array<{ file: string; title: string }>; /** The depth limit applied, if any (undefined means the full space was validated). */ depthLimit?: number; - /** Details on semantic rule evaluations performed (if semantic mode enabled). */ - semanticEvaluations?: SemanticEvaluationRecord[]; - /** Semantic rule/node pairs skipped because the node was gated by an upstream failure. */ - gatedRules?: GatedRuleRecord[]; } /** @@ -209,8 +205,6 @@ export function ruleViolationMessage(v: RuleViolation): string { type SpaceRuleEvaluation = { violations: RuleViolation[]; gates: CoverageGate[]; - semanticEvaluations: SemanticEvaluationRecord[]; - gatedRules: GatedRuleRecord[]; }; /** @@ -231,7 +225,7 @@ async function evaluateSpaceRules( } = {}, ): Promise { const rules = metadata.rules && filterRules(metadata.rules, options.ruleFilter); - if (!rules) return { violations: [], gates: [], semanticEvaluations: [], gatedRules: [] }; + if (!rules) return { violations: [], gates: [] }; // Misconfigured judges fail before any request is made. if (options.judge instanceof RoutedJudgeProvider) options.judge.prepare(rules); if (!metadata.hierarchy) { @@ -240,12 +234,7 @@ async function evaluateSpaceRules( judgeProvider: options.judge, judgeTarget: options.judgeTarget, }); - return { - violations: result, - gates: [], - semanticEvaluations: result.semanticEvaluations, - gatedRules: result.gatedRules, - }; + return { violations: result, gates: [] }; } const graph = buildSpaceGraph(nodes, metadata.hierarchy); return evaluateRulesWithGating(nodes, rules, graph, seeds, options); @@ -375,17 +364,13 @@ export async function validateSpace( ...result.hierarchyViolations.map((v): GatingSeed => ({ file: v.file, title: v.nodeTitle, reason: 'hierarchy' })), ]; const judge = resolveJudge(options.semantic, options.judgeProvider, context.space, context.config, options.judge); - const { violations, gates, semanticEvaluations, gatedRules } = await evaluateSpaceRules(nodes, metadata, seeds, { + const { violations, gates } = await evaluateSpaceRules(nodes, metadata, seeds, { includeGated: options.includeGated, ruleFilter: options.ruleFilter, judge, }); result.ruleViolations = violations; result.coverageGates = gates; - if (judge) { - result.semanticEvaluations = semanticEvaluations; - result.gatedRules = gatedRules; - } } // Collect unresolved content links (warnings — not errors) diff --git a/tests/commands/validate-judge-flag.test.ts b/tests/commands/validate-judge-flag.test.ts index 5c7390d..e9c3453 100644 --- a/tests/commands/validate-judge-flag.test.ts +++ b/tests/commands/validate-judge-flag.test.ts @@ -22,13 +22,7 @@ function spawn(...args: string[]) { return { ...result, stderr: new TextDecoder().decode(result.stderr) }; } -const run = (...args: string[]) => JSON.parse(new TextDecoder().decode(spawn(...args).stdout)); - describe('--judge on the CLI', () => { - it('validate: naming a judge runs semantic rules without --semantic', () => { - expect(run('validate', 'ts', '--judge', 'f').semanticEvaluations).toHaveLength(1); - }); - // An unknown name fails only if the flag reaches the judge. for (const args of [ ['validate', 'ts'], @@ -42,6 +36,6 @@ describe('--judge on the CLI', () => { } it('--no-semantic wins over --judge', () => { - expect(run('validate', 'ts', '--judge', 'f', '--no-semantic').semanticEvaluations).toBeUndefined(); + expect(spawn('validate', 'ts', '--judge', 'missing', '--no-semantic').exitCode).toBe(0); }); }); diff --git a/tests/judge/factory.test.ts b/tests/judge/factory.test.ts index 00dcf91..bdeaca4 100644 --- a/tests/judge/factory.test.ts +++ b/tests/judge/factory.test.ts @@ -4,6 +4,7 @@ import { AnthropicJudgeProvider } from '../../src/judge/provider/anthropic'; import { FakeJudgeProvider } from '../../src/judge/provider/fake'; import { JevJudgeProvider } from '../../src/judge/provider/jev'; import { OpenAIJudgeProvider } from '../../src/judge/provider/openai'; +import type { JudgeConfig } from '../../src/judge/types'; import type { Rule } from '../../src/types'; import { graded, node, passFailRule } from './helpers'; @@ -32,7 +33,7 @@ describe('createJudgeProvider factory', () => { }); it('reads the key from the variable named by apiKeyEnv', () => { - const config = { type: 'jev', apiKeyEnv: 'MY_JEV_KEY' }; + const config: JudgeConfig = { type: 'jev', apiKeyEnv: 'MY_JEV_KEY' }; expect(createJudgeProvider(config, { MY_JEV_KEY: 'k' }) instanceof JevJudgeProvider).toBe(true); expect(() => createJudgeProvider(config, { OPENROUTER_API_KEY: 'k' })).toThrow('MY_JEV_KEY'); }); @@ -65,7 +66,7 @@ describe('createJudgeProvider factory', () => { }); describe('named providers and routing', () => { - const config = { + const config: JudgeConfig = { providers: { decision: { type: 'jev' }, llm: { type: 'anthropic', model: 'claude-haiku-5-5' } }, default: 'llm', }; diff --git a/tests/judge/helpers.ts b/tests/judge/helpers.ts index 0ceb608..5e9d054 100644 --- a/tests/judge/helpers.ts +++ b/tests/judge/helpers.ts @@ -6,7 +6,9 @@ export const node: SemanticJudgeInput['node'] = { type: 'opportunity', resolvedType: 'opportunity', schemaData: { title: 'I cannot find my receipts', type: 'opportunity', source: 'User interview with Alice' }, - resolvedParents: [{ title: 'Faster Tax Filing', field: 'parent' }], + resolvedParents: [ + { title: 'Faster Tax Filing', field: 'parent', source: 'hierarchy', selfRef: false, fieldOn: 'child' }, + ], }; export const passFailRule: SemanticRuleSpec = { @@ -15,7 +17,6 @@ export const passFailRule: SemanticRuleSpec = { rubric: 'Needs to sound like a customer quote, not a solution or business goal', category: 'best-practice', verdicts: { pass: 'Customer voice.', fail: 'Not customer voice.' }, - failOn: ['fail'], }; export const graded: SemanticRuleSpec = { @@ -23,5 +24,4 @@ export const graded: SemanticRuleSpec = { description: 'Framing quality', category: 'best-practice', verdicts: { good: 'Clear need', weak: 'Vague need', solution: 'A solution in disguise' }, - failOn: ['solution'], }; diff --git a/tests/judge/verdicts.test.ts b/tests/judge/verdicts.test.ts index 4ca1559..de924fa 100644 --- a/tests/judge/verdicts.test.ts +++ b/tests/judge/verdicts.test.ts @@ -11,7 +11,6 @@ describe('resolveSemanticRule', () => { it('defaults to pass/fail with fail at the rule severity', () => { const resolved = resolveSemanticRule(base); expect(Object.keys(resolved.spec.verdicts)).toEqual(['pass', 'fail']); - expect(resolved.spec.failOn).toEqual(['fail']); expect(resolved.severities).toEqual({ fail: 'info' }); // best-practice default expect(resolveSemanticRule({ ...base, severity: 'error' }).severities).toEqual({ fail: 'error' }); }); @@ -23,7 +22,6 @@ describe('resolveSemanticRule', () => { severity: { minor: 'warning', major: 'error', skip: 'ignore' }, }); expect(resolved.severities).toEqual({ minor: 'warning', major: 'error' }); - expect(resolved.spec.failOn).toEqual(['minor', 'major']); }); it('splits a verdict into the judge description and user guidance', () => { diff --git a/tests/schema/validate-rules-semantic.test.ts b/tests/schema/validate-rules-semantic.test.ts index 0a649c1..e5679b4 100644 --- a/tests/schema/validate-rules-semantic.test.ts +++ b/tests/schema/validate-rules-semantic.test.ts @@ -69,10 +69,10 @@ describe('validateRules - semantic evaluation', () => { it('skips semantic rules when semantic option is false / omitted (default validate)', async () => { // Default validate should only run expression rules and remain deterministic & free - const violations = await validateRules(sampleNodes, rules); + const fake = new FakeJudgeProvider(); + const violations = await validateRules(sampleNodes, rules, { judgeProvider: fake }); expect(violations).toHaveLength(0); - expect(violations.semanticEvaluations).toHaveLength(0); - expect(violations.gatedRules).toHaveLength(0); + expect(fake.callHistory).toHaveLength(0); }); it('evaluates semantic rules when semantic option is true', async () => { @@ -101,10 +101,7 @@ describe('validateRules - semantic evaluation', () => { expect(violations[0]?.evidence).toBe( 'Stated as a solution feature ("Build social login") rather than a customer need', ); - - expect(violations.semanticEvaluations).toHaveLength(2); - expect(violations.semanticEvaluations.find((e) => e.file === 'valid-opp.md')?.pass).toBe(true); - expect(violations.semanticEvaluations.find((e) => e.file === 'bad-opp.md')?.pass).toBe(false); + expect(fake.callHistory.map((c) => c.node.label).sort()).toEqual(['bad-opp.md', 'valid-opp.md']); }); it('gates semantic rules when node has upstream structural error (cascading)', async () => { @@ -122,10 +119,7 @@ describe('validateRules - semantic evaluation', () => { // bad-opp.md was gated; only valid-opp.md was sent to the judge expect(fake.callHistory).toHaveLength(1); expect(fake.callHistory[0]?.node.label).toBe('valid-opp.md'); - - expect(violations.gatedRules).toHaveLength(1); - expect(violations.gatedRules[0]?.file).toBe('bad-opp.md'); - expect(violations.gatedRules[0]?.ruleId).toBe('opportunity-customer-voice'); + expect(violations).toHaveLength(0); }); it('gates semantic rules when node fails an upstream expression rule', async () => { @@ -162,7 +156,5 @@ describe('validateRules - semantic evaluation', () => { // Only valid-opp.md was evaluated by the semantic judge expect(fake.callHistory).toHaveLength(1); expect(fake.callHistory[0]?.node.label).toBe('valid-opp.md'); - - expect(violations.gatedRules.some((g) => g.file === 'bad-opp.md')).toBe(true); }); }); diff --git a/tests/validate/semantic-rule-shape.test.ts b/tests/validate/semantic-rule-shape.test.ts index 1017ccf..3f60f56 100644 --- a/tests/validate/semantic-rule-shape.test.ts +++ b/tests/validate/semantic-rule-shape.test.ts @@ -1,5 +1,6 @@ import { afterAll, describe, expect, it } from 'bun:test'; import { FakeJudgeProvider } from '../../src/judge/provider/fake'; +import type { JudgeConfig } from '../../src/judge/types'; import { createSpaceContext } from '../../src/space-context'; import type { FileValidationResult } from '../../src/types'; import { validateFile, validateSpace } from '../../src/validate'; @@ -60,12 +61,10 @@ describe('semantic rules: verdict severity, ignore and guidance', () => { it('judges an ignored verdict without reporting it', async () => { const { context } = setup([stepRule]); - const result = await validateSpace(context(), { - semantic: true, - judgeProvider: new FakeJudgeProvider({ 'step-fit': { verdict: 'unsure' } }), - }); + const fake = new FakeJudgeProvider({ 'step-fit': { verdict: 'unsure' } }); + const result = await validateSpace(context(), { semantic: true, judgeProvider: fake }); + expect(fake.callHistory).toHaveLength(1); expect(result.ruleViolations).toEqual([]); - expect(result.semanticEvaluations).toEqual([expect.objectContaining({ verdict: 'unsure', pass: true })]); }); it('gates the subtree per verdict: an error-mapped verdict gates, a warning-mapped one does not', async () => { @@ -131,7 +130,7 @@ describe('rule `when` precondition', () => { const idle = new FakeJudgeProvider(); const result = await validateSpace(none.context(), { semantic: true, judgeProvider: idle }); expect(idle.callHistory).toHaveLength(0); - expect(result.gatedRules).toEqual([]); // not applicable is not gated + expect(result.coverageGates).toEqual([]); // not applicable is not gated }); }); @@ -143,16 +142,19 @@ describe('run-level judge override', () => { it('--judge implies semantic evaluation, and semantic: false still wins', async () => { const { context } = setup([stepRule], { judge: { providers: { llm: { type: 'fake' } } } }); - const on = await validateSpace(context(), { judge: 'llm' }); - expect(on.semanticEvaluations?.length).toBeGreaterThan(0); - const off = await validateSpace(context(), { judge: 'llm', semantic: false }); - expect(off.semanticEvaluations ?? []).toHaveLength(0); + const on = new FakeJudgeProvider(); + await validateSpace(context(), { judge: 'llm', judgeProvider: on }); + expect(on.callHistory.length).toBeGreaterThan(0); + const off = new FakeJudgeProvider(); + await validateSpace(context(), { judge: 'llm', semantic: false, judgeProvider: off }); + expect(off.callHistory).toHaveLength(0); }); }); describe('judge type "none"', () => { it('throws, not skips, when semantic evaluation is attempted', async () => { - for (const judge of [{ type: 'none' }, { providers: { llm: { type: 'none' } }, default: 'llm' }]) { + const judges: JudgeConfig[] = [{ type: 'none' }, { providers: { llm: { type: 'none' } }, default: 'llm' }]; + for (const judge of judges) { const { context } = setup([stepRule], { judge }); await expect(validateSpace(context(), { semantic: true })).rejects.toThrow('no provider type'); await expect(validateSpace(context(), { judge: 'default' })).rejects.toThrow(); diff --git a/tests/validate/semantic-validation.test.ts b/tests/validate/semantic-validation.test.ts index e48cb79..a322585 100644 --- a/tests/validate/semantic-validation.test.ts +++ b/tests/validate/semantic-validation.test.ts @@ -27,9 +27,9 @@ describe('Semantic rule evaluation integration (validateSpace & validateFile)', describe('validateSpace', () => { it('skips semantic rules by default (deterministic & free)', async () => { - const result = await validateSpace(validContext); - expect(result.semanticEvaluations ?? []).toHaveLength(0); - expect(result.gatedRules ?? []).toHaveLength(0); + const fake = new FakeJudgeProvider({ pass: false }); + const result = await validateSpace(validContext, { judgeProvider: fake }); + expect(fake.callHistory).toHaveLength(0); expect(result.ruleViolations.some((v) => v.ruleId === 'opportunity-customer-voice')).toBe(false); }); @@ -44,15 +44,11 @@ describe('Semantic rule evaluation integration (validateSpace & validateFile)', judgeProvider: fakeProvider, }); - expect(result.semanticEvaluations).toBeDefined(); - expect(result.semanticEvaluations!.length).toBeGreaterThan(0); - - const oppEval = result.semanticEvaluations!.find((e) => e.ruleId === 'opportunity-customer-voice'); - expect(oppEval).toBeDefined(); - expect(oppEval?.pass).toBe(true); - expect(oppEval?.evidence).toContain('framed in clear customer language'); + expect(fakeProvider.callHistory.some((c) => c.rules.some((r) => r.id === 'opportunity-customer-voice'))).toBe( + true, + ); - // No violation when pass is true + // No violation when the verdict has no severity expect(result.ruleViolations.some((v) => v.ruleId === 'opportunity-customer-voice')).toBe(false); }); @@ -96,13 +92,9 @@ describe('Semantic rule evaluation integration (validateSpace & validateFile)', expect(result.nodeErrors.some((e) => e.file === 'opportunity-no-source.md')).toBe(true); // Therefore it must be gated and never evaluated by the judge - expect(result.gatedRules).toBeDefined(); - const gated = result.gatedRules!.find( - (g) => g.file === 'opportunity-no-source.md' && g.ruleId === 'opportunity-customer-voice', + expect(result.coverageGates.some((g) => g.file === 'opportunity-no-source.md' && g.reason === 'schema')).toBe( + true, ); - expect(gated).toBeDefined(); - expect(gated?.reason).toMatch(/^Gated by/); - // Judge was never called for opportunity-no-source.md expect(fakeProvider.callHistory.some((c) => c.node.label === 'opportunity-no-source.md')).toBe(false); }); @@ -213,12 +205,11 @@ describe('Semantic rules: verdicts, batching and rule filter', () => { expect(fake.callHistory.every((c) => c.rules.length >= 1)).toBe(true); }); - it('records the verdict and confidence on evaluations and fails only on verdicts that have a severity', async () => { + it('reports a verdict that has a severity as a violation carrying the verdict and evidence', async () => { const fake = new FakeJudgeProvider({ verdict: 'fail', confidence: 0.7, evidence: 'solution in disguise' }); const result = await validateSpace(context, { semantic: true, judgeProvider: fake }); - const record = result.semanticEvaluations?.find((e) => e.ruleId === 'opportunity-customer-voice'); - expect(record).toMatchObject({ verdict: 'fail', pass: false, confidence: 0.7 }); - expect(result.ruleViolations.some((v) => v.ruleId === 'opportunity-customer-voice')).toBe(true); + const violation = result.ruleViolations.find((v) => v.ruleId === 'opportunity-customer-voice'); + expect(violation).toMatchObject({ verdict: 'fail', evidence: 'solution in disguise' }); }); it('reports a per-rule judge error as a violation without gating the subtree', async () => { @@ -244,6 +235,6 @@ describe('Semantic rules: verdicts, batching and rule filter', () => { ruleFilter: { mode: ['expression'] }, }); expect(fake.callHistory).toHaveLength(0); - expect(expressionOnly.semanticEvaluations ?? []).toHaveLength(0); + expect(expressionOnly.ruleViolations.some((v) => v.verdict !== undefined)).toBe(false); }); }); From d7549df5e4e8f642a13fd4a46cc3f80d17b0d459 Mon Sep 17 00:00:00 2001 From: Roger Barnes Date: Wed, 14 Oct 2026 18:10:27 +1100 Subject: [PATCH 14/22] refactor(judge): model providers in config, judge picks one; runs ask for semantic rules Config now separates how to reach a model (top-level modelProviders) from which one judges (judge.provider, top level, with a space override). Config no longer turns semantic rules on: --semantic or --judge (or the API options) ask on each run, so --no-semantic and the hooks' pin go. Rules no longer name a judge: that is config, not schema. The judge is built once per run, and only when a semantic rule is to be judged. --- AGENTS.md | 2 +- README.md | 4 +- config.example.json | 5 +- docs/concepts.md | 2 +- docs/config.md | 45 +++-- docs/rules.md | 2 +- docs/semantic-rules.md | 27 +-- plugin/.claude-plugin/plugin.json | 2 +- plugin/scripts/state.ts | 2 +- .../structured-context/references/commands.md | 4 +- .../_structured_context_schema_meta.json | 7 - src/commands/schemas.ts | 3 +- src/config.ts | 29 +--- src/index.ts | 6 +- src/judge/factory.ts | 157 ++++-------------- src/judge/index.ts | 4 +- src/judge/types.ts | 27 ++- src/judge/verdicts.ts | 1 - src/schema/gating.ts | 4 +- src/schema/metadata-contract.ts | 4 +- src/validate.ts | 50 +++--- tests/commands/validate-judge-flag.test.ts | 6 +- tests/config.test.ts | 13 +- tests/helpers/temp-space.ts | 4 +- tests/judge/factory.test.ts | 120 ++++--------- tests/judge/provider/jev.test.ts | 7 +- tests/judge/verdicts.test.ts | 4 - tests/schema/semantic-rule-contract.test.ts | 5 +- tests/validate/semantic-rule-shape.test.ts | 50 +++--- 29 files changed, 207 insertions(+), 389 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 5193f6c..da8eece 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,7 +39,7 @@ There are several places that need reviewing and updating with any new feature o - **Explicitly state alternatives:** When planning or making architectural decisions, do not anchor to the first familiar mechanism. Formulate and state at least one alternative approach, weighing trade-offs, complexity, and ergonomic impact. - **Minimise blast radius:** Protect established boundaries and interfaces. Treat unnecessary ripple effects (such as making synchronous APIs asynchronous or adding dual APIs) as a design smell indicating the approach should be re-evaluated. - **Public API surface:** Do not add exports to `src/api.ts`, `src/types.ts` or `src/testing.ts` (the `structured-context/api`, `/types` and `/testing` entry points), or add types or hooks to the plugin interface, without the maintainer's explicit agreement on that specific surface. Exporting something to make docs complete, to tidy up, or because a feature has a public-looking class is not a reason. If a change seems to need a new export, stop and propose it, with the consumer that needs it. Internal modules can change freely. -- **Where a setting lives:** Config says *what* to use, CLI flags override it for one run, and the environment holds secrets and the config location only (`apiKeyEnv` names a secret; `SCTX_CONFIG` locates the config). Do not add env vars that set config values, and never infer a choice (such as a judge provider) from a variable that happens to be set: a key present for another tool must not be spent silently. +- **Where a setting lives:** Config says *what* to use, CLI flags override it for one run, and the environment holds secrets and the config location only (`apiKeyEnv` names a secret; `SCTX_CONFIG` locates the config). Do not add env vars that set config values, and never infer a choice (such as a judge provider) from a variable that happens to be set: a key present for another tool must not be spent silently. Config also never switches on costly or outward-facing behaviour (such as semantic judging): each run asks for it, through a flag or an API option. - **Failures are visible:** A rule or expression that cannot be evaluated is reported as a violation, never skipped or treated as a pass. Likewise a misconfiguration fails before any work or spend. ## Key Files diff --git a/README.md b/README.md index 370d370..8501255 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,7 @@ See [docs/schemas.md](docs/schemas.md) (`sctx docs schema`) for the dialect, com ### Validate nodes ```bash -sctx validate [--watch] [--depth ] [--include-gated] [--semantic|--no-semantic] [--judge ] [--rule-filter ] +sctx validate [--watch] [--depth ] [--include-gated] [--semantic] [--judge ] [--rule-filter ] ``` Validates markdown files against the JSON schema: @@ -141,7 +141,7 @@ Validates markdown files against the JSON schema: - Runs executable rules (`$metadata.rules`) with **cascade gating**: a schema, hierarchy, or blocking-rule failure on a node suppresses rule evaluation for that node's hierarchy subtree. Gated checks are reported in a Coverage section so you know what wasn't evaluated and why. See [docs/rules.md](docs/rules.md#cascade-gating) for details. - `--depth ` limits evaluation to hierarchy nodes within `n` breadth-first levels from the top — useful for staged agentic fix loops (fix the top level, then request the next). - `--include-gated` evaluates rules even on gated nodes (still reported as gated) — for surfacing cheap bottom-up checks on request. -- `--semantic` also evaluates `mode: semantic` rules with a model judge (Anthropic, OpenAI-compatible, or Jev); `--no-semantic` overrides `semantic` in config, and `--judge ` picks a configured judge provider for every rule; `--rule-filter` narrows which rules run. See [docs/semantic-rules.md](docs/semantic-rules.md) (`sctx docs semantic-rules`). +- `--semantic` also evaluates `mode: semantic` rules with the configured judge (Jev, Anthropic or OpenAI-compatible); `--judge ` uses that configured model provider instead and implies `--semantic`; `--rule-filter` narrows which rules run. See [docs/semantic-rules.md](docs/semantic-rules.md) (`sctx docs semantic-rules`). ### Render space diff --git a/config.example.json b/config.example.json index 3fa5e8e..ff78093 100644 --- a/config.example.json +++ b/config.example.json @@ -18,5 +18,8 @@ } ], "schema": "/path/to/custom/schemas/my-general-schema.json", - "judge": { "type": "none" } + "modelProviders": { + "jev": { "type": "jev" } + }, + "judge": { "provider": "jev" } } diff --git a/docs/concepts.md b/docs/concepts.md index 158f298..2584146 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -155,7 +155,7 @@ See [docs/rules.md](rules.md) for the rules reference, including JSONata express A **semantic rule** (`mode: semantic`) asks a model to judge a node, for example whether an opportunity is a customer need. It has a `rubric` instead of a `check`. -A **judge** is the model that judges semantic rules: a decision model (Jev) or a chat model (Claude, OpenAI-compatible). For each rule, the judge gives one **verdict** from the rule's `verdicts`, for example `customer-need`, `vague` or `solution`. The rule's `severity` says which verdicts report a violation. Semantic rules cost money, so they run only when you turn them on (`--semantic`, or `semantic: true` in config). See [docs/semantic-rules.md](semantic-rules.md). +A **judge** is the model that judges semantic rules: a decision model (Jev) or a chat model (Claude, OpenAI-compatible). For each rule, the judge gives one **verdict** from the rule's `verdicts`, for example `customer-need`, `vague` or `solution`. The rule's `severity` says which verdicts report a violation. Semantic rules cost money, so they run only when a run asks for them (`--semantic` or `--judge`). The judge is a configured **model provider**. See [docs/semantic-rules.md](semantic-rules.md). ### Cascade gating diff --git a/docs/config.md b/docs/config.md index 373070f..e0bde76 100644 --- a/docs/config.md +++ b/docs/config.md @@ -65,49 +65,44 @@ All plugin names must start with `sctx-` (the prefix is optional in config and n Plugins can also contribute schemas and partials via a `/schemas/` directory (or `"sctx": { "schemas": "..." }` in `package.json`). If a space specifies a plugin-prefixed schema (e.g. `"sctx-wardley-mapping/wardley_map.json"`), schemas are resolved statically without executing plugin code, and the plugin's code hooks are loaded when parsing or rendering. -## Judge (semantic rules) - -`judge` configures the model that judges [semantic rules](semantic-rules.md). `semantic: true` runs semantic rules without `--semantic`. Set either key at the top level (all spaces) or on a space. A space's value replaces the top-level value. +## Model providers -To use one provider, set its fields directly on `judge`: +`modelProviders` names the models that sctx can use, and says how to reach each one. Set it at the top level only. ```json { - "judge": { "type": "anthropic", "model": "claude-haiku-5-5" }, - "spaces": [ - { "name": "ProductX", "path": "/path/to/space", "semantic": true, "judge": { "type": "jev" } } - ] + "modelProviders": { + "jev": { "type": "jev" }, + "claude": { "type": "anthropic", "model": "claude-haiku-5-5" }, + "local": { "type": "openai", "endpoint": "http://localhost:11434/v1" } + } } ``` | Field | Description | |---|---| -| `type` | **Required.** `jev`, `anthropic`, `openai` (see [Choosing a judge](semantic-rules.md#choosing-a-judge)), `fake` (for tests) or `none` (a placeholder for templates; judging with it is an error). | -| `model` | Defaults: `claude-haiku-5-5` (anthropic), `gpt-4o-mini` (openai), `typesafe/jev-1.13` (jev). | +| `type` | **Required.** `jev`, `anthropic` or `openai` (see [Choosing a judge](semantic-rules.md#choosing-a-judge)), or `fake` (for tests). | +| `model` | Defaults: `typesafe/jev-1.13` (jev), `claude-haiku-5-5` (anthropic), `gpt-4o-mini` (openai). | | `endpoint` | Override the API URL. Jev defaults to `https://openrouter.ai/api/alpha/decisions`. | -| `apiKeyEnv` | The environment variable that holds the key. Defaults: `OPENROUTER_API_KEY` (jev), `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`. | +| `apiKeyEnv` | The environment variable that holds the key. Defaults: `OPENROUTER_API_KEY` (jev), `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`. A local `openai` endpoint can need no key. | | `apiKey` | The key itself. Use `apiKeyEnv` instead, so that keys stay out of config files. | -To use several providers, give each a name under `providers`. Each provider takes the fields above. `default` names the provider for rules that do not name one: +sctx creates a provider only when it uses it, so a provider that is not used needs no key. The environment holds only keys: no environment variable selects or configures a provider. + +## Judge (semantic rules) + +`judge.provider` names the model provider that judges [semantic rules](semantic-rules.md). Set it at the top level. A space can set its own `judge`, which replaces the top-level one for that space, for example to keep a private space on a local model: ```json { - "judge": { - "providers": { - "decision": { "type": "jev" }, - "llm": { "type": "anthropic", "model": "claude-haiku-5-5" } - }, - "default": "llm" - } + "judge": { "provider": "jev" }, + "spaces": [ + { "name": "private-notes", "path": "/path/to/notes", "judge": { "provider": "local" } } + ] } ``` -- A rule selects a provider with `judge: "decision"`. `--judge ` selects one for every rule in a run. -- `default` is necessary only with more than one provider. -- A provider needs a key only when a rule uses it. -- Use the names `decision` (a decision model) and `llm` (a chat model), so that shared schemas work with other configs. - -The environment holds only keys. There are no environment variables that change the judge. +Config does not turn semantic rules on. Each run asks for them with `--semantic`, or with `--judge `, which also selects a different provider for that run. ## Markdown plugin config diff --git a/docs/rules.md b/docs/rules.md index 8ea4ce3..bd9c1d9 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -38,7 +38,7 @@ Rules are a flat array. | `check` | yes, unless `mode: semantic` | JSONata expression; must evaluate to `true` | | `when` | no | JSONata test. The rule runs only on nodes where it returns `true`. See [Conditional rules](#conditional-rules-when) | | `mode` | no | `expression` (default) or `semantic` | -| `rubric`, `verdicts`, `judge` | semantic only | See [Semantic rules](semantic-rules.md#rule-fields) | +| `rubric`, `verdicts` | semantic only | See [Semantic rules](semantic-rules.md#rule-fields) | | `type` | no | Restrict rule to nodes of this `resolvedType` | | `scope` | no | Use `"global"` to evaluate once for the whole space | | `override` | no | Only for merge conflicts: allows later duplicate `id` to replace earlier | diff --git a/docs/semantic-rules.md b/docs/semantic-rules.md index 0ecb925..1193e4b 100644 --- a/docs/semantic-rules.md +++ b/docs/semantic-rules.md @@ -7,13 +7,16 @@ This page assumes you know [Rules](rules.md). Semantic rules use the same fields ## Quick start 1. Set `OPENROUTER_API_KEY` in your environment. -2. Add a judge to `config.json`: +2. Add a model provider and choose it as the judge in `config.json`: ```json - { "judge": { "type": "jev" } } + { + "modelProviders": { "jev": { "type": "jev" } }, + "judge": { "provider": "jev" } + } ``` - Jev is a decision model and the recommended judge (see [Choosing a judge](#choosing-a-judge)). To use Claude instead, set `"type": "anthropic"` and `ANTHROPIC_API_KEY`. + Jev is a decision model and the recommended judge (see [Choosing a judge](#choosing-a-judge)). To use Claude instead, add a provider with `"type": "anthropic"` and set `ANTHROPIC_API_KEY`. 3. Add a rule to the schema's `$metadata.rules`: ```json @@ -30,11 +33,12 @@ This page assumes you know [Rules](rules.md). Semantic rules use the same fields ## Turning semantic rules on -Semantic rules cost money and send node content to the judge's provider. Thus they run only when you ask: +Semantic rules cost money and send node content to the judge's provider. Thus they run only when you ask, on each run: -- `--semantic` runs them for one run of `sctx validate` or `sctx validate-file`. -- `"semantic": true` in [config](config.md#judge-semantic-rules) runs them on every run. Set it at the top level (all spaces) or on a space. -- `--no-semantic` stops them for one run, and overrides config. +- `--semantic` runs them with the configured judge. +- `--judge ` runs them with that model provider instead. + +Both work on `sctx validate` and `sctx validate-file`. Config cannot turn semantic rules on. The API is the same: pass `semantic` or `judge` to `validateSpace` or `validateFile`. ## Rule fields @@ -44,7 +48,6 @@ Semantic rules cost money and send node content to the judge's provider. Thus th | `rubric` | no | Tells the judge what to assess. Default: `description` | | `verdicts` | no | The answers the judge can give. Default: `pass` and `fail`. See [Verdicts](#verdicts) | | `severity` | if `verdicts` is set | The severity of each verdict. See [Severity](#severity) | -| `judge` | no | The name of a [judge provider](#several-judges). Default: the configured default provider | A semantic rule cannot have `check` or `scope: "global"`. @@ -103,12 +106,10 @@ Use a decision model when you can. Its results are repeatable and cheap, and the | `anthropic` | Claude. Default model: `claude-haiku-5-5` | `ANTHROPIC_API_KEY` | | `openai` | Any OpenAI-compatible chat endpoint, for example OpenAI, Ollama or vLLM. Set `endpoint` for servers other than OpenAI. Local servers do not need a key | `OPENAI_API_KEY` | -sctx uses only the provider type that you set. It does not choose a provider from the keys in your environment. See [Judge configuration](config.md#judge-semantic-rules) for all fields. - -### Several judges +sctx uses only the provider that you configure. It does not choose a provider from the keys in your environment. See [Model providers](config.md#model-providers) and [Judge](config.md#judge-semantic-rules) for all fields. -You can configure several providers by name, for example `decision` and `llm`. A rule selects one with `judge`. `--judge ` uses one provider for every rule in a run, and implies `--semantic`. For example, to get a second opinion from a chat model on one rule and one file: +A rule does not choose its judge: that is configuration, not schema. To use a different provider for one run, name it with `--judge`. For example, to get a second opinion from a chat model on one rule and one file: ```bash -sctx validate-file --rule-filter id= --judge llm +sctx validate-file --rule-filter id= --judge claude ``` diff --git a/plugin/.claude-plugin/plugin.json b/plugin/.claude-plugin/plugin.json index d8dd888..09abbb9 100644 --- a/plugin/.claude-plugin/plugin.json +++ b/plugin/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "structured-context", - "version": "0.12.12", + "version": "0.12.13", "description": "Structured content authoring for Obsidian — automatic validation hooks, slash commands, and agent skills to keep markdown files compliant with your schema.", "homepage": "https://github.com/mindsocket/structured-context", "repository": "https://github.com/mindsocket/structured-context", diff --git a/plugin/scripts/state.ts b/plugin/scripts/state.ts index 6807644..945f319 100644 --- a/plugin/scripts/state.ts +++ b/plugin/scripts/state.ts @@ -132,7 +132,7 @@ export function hashSpace(space: SpaceInfo, configFiles: string[]): string { * Info-level rule violations are left out: they are not worth another turn from Claude. */ export async function validateSpace(name: string, options?: HookOptions): Promise> { - const text = await sctx(['validate', name, '--json', '--no-semantic'], options); + const text = await sctx(['validate', name, '--json'], options); if (!text) return {}; const output = JSON.parse(text) as ValidateOutput; diff --git a/plugin/skills/structured-context/references/commands.md b/plugin/skills/structured-context/references/commands.md index 0cab9a9..0e160a6 100644 --- a/plugin/skills/structured-context/references/commands.md +++ b/plugin/skills/structured-context/references/commands.md @@ -11,7 +11,7 @@ Then use the `sctx` command for all operations. Always pass `--config ` wh ## validate ```bash -sctx validate [--watch] [--config ] [--depth ] [--include-gated] [--semantic|--no-semantic] [--judge ] [--rule-filter ] +sctx validate [--watch] [--config ] [--depth ] [--include-gated] [--semantic] [--judge ] [--rule-filter ] ``` Validates all `.md` files in the space against the JSON schema. For each file: @@ -30,7 +30,7 @@ the top — useful for fixing a space level by level (fix the top, re-run with a `--depth`, repeat). `--include-gated` evaluates rules on gated nodes anyway, still reporting them as gated — use it to surface cheap bottom-up checks even when an upstream blocker fired. `--semantic` also runs `mode: semantic` rules through a configured model judge (costly; off by -default; `--no-semantic` forces it off even if config enables it; `--judge ` picks a configured judge provider), and `--rule-filter =` (repeatable; mode, category, severity, type, id) +default; `--judge ` judges with that configured model provider and implies `--semantic`), and `--rule-filter =` (repeatable; mode, category, severity, type, id) narrows which rules run. `validate-file` accepts the same two flags. Exit codes: `0` = clean (warning and info rule violations don't fail), `1` = validation errors found. diff --git a/schemas/generated/latest/_structured_context_schema_meta.json b/schemas/generated/latest/_structured_context_schema_meta.json index 7062ca5..feb8506 100644 --- a/schemas/generated/latest/_structured_context_schema_meta.json +++ b/schemas/generated/latest/_structured_context_schema_meta.json @@ -256,10 +256,6 @@ }, "minProperties": 2 }, - "judge": { - "type": "string", - "minLength": 1 - }, "type": { "type": "string", "minLength": 1 @@ -332,9 +328,6 @@ }, { "required": ["rubric"] - }, - { - "required": ["judge"] } ] }, diff --git a/src/commands/schemas.ts b/src/commands/schemas.ts index 8b44113..d6cb99f 100644 --- a/src/commands/schemas.ts +++ b/src/commands/schemas.ts @@ -226,7 +226,7 @@ function generateMermaidErd(metadata: SchemaMetadata, entities: EntityInfo[]): s return mmd; } -/** Bracketed notes on a rule: mode, severity (or verdict severities), judge and `when`. Empty for a plain rule. */ +/** Bracketed notes on a rule: mode, severity (or verdict severities) and `when`. Empty for a plain rule. */ function ruleAnnotations(rule: Rule): string { const notes: string[] = []; if (rule.mode === 'semantic') notes.push('semantic'); @@ -239,7 +239,6 @@ function ruleAnnotations(rule: Rule): string { .join(', ')}`, ); } - if (rule.judge) notes.push(`judge: ${rule.judge}`); if (rule.when) notes.push(`when: ${rule.when}`); return notes.length > 0 ? ` [${notes.join('] [')}]` : ''; } diff --git a/src/config.ts b/src/config.ts index 776127a..c50cb16 100644 --- a/src/config.ts +++ b/src/config.ts @@ -4,7 +4,7 @@ import { dirname, isAbsolute, join, resolve } from 'node:path'; import Ajv2020 from 'ajv/dist/2020'; import JSON5 from 'json5'; import { ENV_CONFIG_VAR, XDG_CONFIG_DIR } from './constants'; -import { JUDGE_PROVIDER_TYPES, type JudgeConfig } from './judge/types'; +import { type JudgeConfig, MODEL_PROVIDER_TYPES, type ModelProviderConfig } from './judge/types'; import { normalizePluginName, resolvePluginSchemasDir, @@ -13,10 +13,10 @@ import { } from './plugins/util'; import { bundledSchemasDir } from './schema/schema'; -const JUDGE_PROVIDER_SCHEMA = { +const MODEL_PROVIDER_SCHEMA = { type: 'object', properties: { - type: { type: 'string', enum: JUDGE_PROVIDER_TYPES }, + type: { type: 'string', enum: MODEL_PROVIDER_TYPES }, model: { type: 'string' }, endpoint: { type: 'string' }, apiKey: { type: 'string' }, @@ -28,15 +28,7 @@ const JUDGE_PROVIDER_SCHEMA = { const JUDGE_CONFIG_SCHEMA = { type: 'object', - properties: { - type: { type: 'string', enum: JUDGE_PROVIDER_TYPES }, - model: { type: 'string' }, - apiKey: { type: 'string' }, - apiKeyEnv: { type: 'string' }, - endpoint: { type: 'string' }, - providers: { type: 'object', additionalProperties: JUDGE_PROVIDER_SCHEMA }, - default: { type: 'string' }, - }, + properties: { provider: { type: 'string', minLength: 1 } }, additionalProperties: false, } as const; @@ -62,7 +54,6 @@ const CONFIG_SCHEMA = { }, }, judge: JUDGE_CONFIG_SCHEMA, - semantic: { type: 'boolean' }, }, required: ['name', 'path'], additionalProperties: false, @@ -70,8 +61,8 @@ const CONFIG_SCHEMA = { }, schema: { type: 'string' }, includeSpacesFrom: { type: 'array', items: { type: 'string' } }, + modelProviders: { type: 'object', additionalProperties: MODEL_PROVIDER_SCHEMA }, judge: JUDGE_CONFIG_SCHEMA, - semantic: { type: 'boolean' }, }, anyOf: [{ required: ['spaces'] }, { required: ['includeSpacesFrom'] }], additionalProperties: false, @@ -85,10 +76,8 @@ export type SpaceConfig = { plugins?: Record>; /** Named filter views for this space. Keys are view names; values contain the filter expression. */ views?: Record; - /** LLM judge provider configuration for semantic rule evaluation. */ + /** Judge for this space's semantic rules; overrides the top-level `judge`. */ judge?: JudgeConfig; - /** Whether to evaluate semantic rules during validation for this space. */ - semantic?: boolean; /** * Directory of the config file that defined this space, used to anchor relative plugin-config * paths. Set by loadConfig; absent for a hand-assembled Config, whose callers must pass an @@ -101,10 +90,10 @@ export type Config = { spaces: SpaceConfig[]; schema?: string; includeSpacesFrom?: string[]; - /** Default LLM judge provider configuration for semantic rule evaluation. */ + /** Model providers by name: how to reach each model. */ + modelProviders?: Record; + /** Judge for semantic rules (a model provider name). A space's `judge` overrides it. */ judge?: JudgeConfig; - /** Whether to evaluate semantic rules during validation across spaces. */ - semantic?: boolean; }; let _configPathOverride: string | undefined; diff --git a/src/index.ts b/src/index.ts index eb56e68..2dd3b06 100755 --- a/src/index.ts +++ b/src/index.ts @@ -79,8 +79,7 @@ program (v: string, prev: string[] = []) => [...prev, v], ) .option('--semantic', 'Evaluate semantic rules with the configured judge') - .option('--no-semantic', 'Skip semantic rules even if enabled in config') - .option('--judge ', 'Judge every semantic rule with this configured provider (implies --semantic)') + .option('--judge ', 'Judge semantic rules with this model provider (implies --semantic)') .action(async (filePath, options) => { process.exitCode = await reportJudgeConfigErrors(() => validateFileCommand(filePath, { @@ -114,8 +113,7 @@ program (v: string, prev: string[] = []) => [...prev, v], ) .option('--semantic', 'Evaluate semantic rules with the configured judge') - .option('--no-semantic', 'Skip semantic rules even if enabled in config') - .option('--judge ', 'Judge every semantic rule with this configured provider (implies --semantic)') + .option('--judge ', 'Judge semantic rules with this model provider (implies --semantic)') .action(async (spaceName, options) => { const context = buildSpaceContext(spaceName); if (options.watch) { diff --git a/src/judge/factory.ts b/src/judge/factory.ts index 885d9c2..edab056 100644 --- a/src/judge/factory.ts +++ b/src/judge/factory.ts @@ -1,15 +1,8 @@ -import type { Rule } from '../types'; import { AnthropicJudgeProvider } from './provider/anthropic'; import { FakeJudgeProvider } from './provider/fake'; import { JevJudgeProvider } from './provider/jev'; import { OpenAIJudgeProvider } from './provider/openai'; -import type { - JudgeConfig, - JudgeProviderDefinition, - SemanticJudgeInput, - SemanticJudgeProvider, - SemanticJudgeResult, -} from './types'; +import { MODEL_PROVIDER_TYPES, type ModelProviderConfig, type SemanticJudgeProvider } from './types'; type Env = Record; @@ -20,54 +13,23 @@ const DEFAULT_KEY_ENV = { jev: 'OPENROUTER_API_KEY', } as const; -const PROVIDER_TYPES = ['anthropic', 'openai', 'jev', 'fake'] as const; - -/** Name of the provider defined by the flat `judge: { type, ... }` shorthand. */ -export const DEFAULT_JUDGE_NAME = 'default'; - /** The judge is misconfigured (no provider, unknown provider, missing key). Reported as a plain CLI error. */ export class JudgeConfigError extends Error {} -/** Every provider the config defines, by name, and the one used for rules that don't name any. */ -function providerDefinitions(config: JudgeConfig | undefined): { - definitions: Record; - defaultName?: string; -} { - const definitions: Record = { ...config?.providers }; - - // The flat shorthand defines one provider named "default". - if (config?.type) { - definitions[DEFAULT_JUDGE_NAME] = { - type: config.type, - model: config.model, - endpoint: config.endpoint, - apiKey: config.apiKey, - apiKeyEnv: config.apiKeyEnv, - }; - } - - const names = Object.keys(definitions); - const defaultName = - config?.default ?? - (DEFAULT_JUDGE_NAME in definitions ? DEFAULT_JUDGE_NAME : names.length === 1 ? names[0] : undefined); - return { definitions, defaultName }; -} - /** * Instantiate one provider from its definition. The type is always explicit and is never inferred from API keys * in the environment, so a key set for another tool is never spent on judging. Once the type is chosen, its key * comes from `apiKey`, else the environment variable named by `apiKeyEnv` (default: the type's usual variable). */ -function createProvider(name: string, def: JudgeProviderDefinition, env: Env): SemanticJudgeProvider { +export function createModelProvider( + name: string, + def: ModelProviderConfig, + env: Env = process.env, +): SemanticJudgeProvider { const type = def.type; - if (!type || type === 'none') { + if (!(MODEL_PROVIDER_TYPES as readonly string[]).includes(type)) { throw new JudgeConfigError( - `Judge "${name}" has no provider type. Set it to one of: ${PROVIDER_TYPES.join(', ')} (config.example.json shows a template).`, - ); - } - if (!(PROVIDER_TYPES as readonly string[]).includes(type)) { - throw new JudgeConfigError( - `Unknown judge provider "${type}" for "${name}". Use one of: ${PROVIDER_TYPES.join(', ')}.`, + `Model provider "${name}" has unknown type "${type}". Use one of: ${MODEL_PROVIDER_TYPES.join(', ')}.`, ); } if (type === 'fake') return new FakeJudgeProvider(); @@ -79,7 +41,7 @@ function createProvider(name: string, def: JudgeProviderDefinition, env: Env): S // OpenAI-compatible endpoints (Ollama, vLLM, ...) may need no key. if (!apiKey && type !== 'openai') { throw new JudgeConfigError( - `Judge "${name}" (${type}) has no API key. Set ${keyEnv} in the environment, or "apiKey" / "apiKeyEnv" on the provider.`, + `Model provider "${name}" (${type}) has no API key. Set ${keyEnv} in the environment, or "apiKey" / "apiKeyEnv" on the provider.`, ); } switch (type) { @@ -93,90 +55,27 @@ function createProvider(name: string, def: JudgeProviderDefinition, env: Env): S } /** - * Routes each semantic rule to the judge provider it names (`judge` on the rule), else the configured default. - * A run-level override replaces every rule's choice. Providers are created on first use, so one that no rule - * reaches needs no credentials. - */ -export class RoutedJudgeProvider implements SemanticJudgeProvider { - readonly name = 'routed'; - private readonly cache = new Map(); - - constructor( - private readonly definitions: Record, - private readonly defaultName: string | undefined, - private readonly env: Env, - private readonly override?: string, - private readonly injected?: SemanticJudgeProvider, - ) {} - - /** The provider that judges a rule asking for `requested` (undefined: the default). */ - resolve(requested?: string): SemanticJudgeProvider { - if (this.injected) return this.injected; - const name = this.override ?? requested ?? this.defaultName; - if (!name) { - const known = Object.keys(this.definitions); - throw new JudgeConfigError( - known.length === 0 - ? `No semantic judge configured. Set "judge.type" in config.json (one of: ${PROVIDER_TYPES.join(', ')}).` - : `Several judge providers are configured (${known.join(', ')}). Set "judge.default" for the default, or name one on the rule ("judge") or with --judge.`, - ); - } - const def = this.definitions[name]; - if (!def) { - const known = Object.keys(this.definitions); - throw new JudgeConfigError( - `Judge "${name}" is not configured.${known.length ? ` Configured: ${known.join(', ')}.` : ' Add it under "judge.providers" in config.json.'}`, - ); - } - let provider = this.cache.get(name); - if (!provider) { - provider = createProvider(name, def, this.env); - this.cache.set(name, provider); - } - return provider; - } - - /** Fail now, before any request is made, if a rule's judge is not usable. */ - prepare(rules: Rule[]): void { - for (const rule of rules) { - if (rule.mode === 'semantic') this.resolve(rule.judge); - } - } - - async evaluate(input: SemanticJudgeInput): Promise> { - const groups = new Map(); - for (const rule of input.rules) { - const provider = this.resolve(rule.judge); - groups.set(provider, [...(groups.get(provider) ?? []), rule]); - } - const results: Record = {}; - for (const [provider, rules] of groups) { - Object.assign(results, await provider.evaluate({ node: input.node, rules })); - } - return results; - } -} - -/** - * Build the judge for a run from config and environment. `override` (the `--judge` flag) names the provider to use - * for every rule; `customProvider` (tests, injection) replaces configured providers altogether. + * The judge for a run: the model provider named by `name` (the `--judge` override), else the space's + * `judge.provider`, else the top-level one. Throws {@link JudgeConfigError} when it cannot be built. */ export function createJudge( - config?: JudgeConfig, - env: Env = process.env, - customProvider?: SemanticJudgeProvider, - override?: string, -): RoutedJudgeProvider { - const { definitions, defaultName } = providerDefinitions(config); - return new RoutedJudgeProvider(definitions, defaultName, env, override, customProvider); -} - -/** The single provider for a config: the default one, or the one named by `name`. */ -export function createJudgeProvider( - config?: JudgeConfig, - env: Env = process.env, - customProvider?: SemanticJudgeProvider, + config: { modelProviders?: Record; judge?: { provider?: string } }, + space: { judge?: { provider?: string } } = {}, name?: string, + env: Env = process.env, ): SemanticJudgeProvider { - return createJudge(config, env, customProvider).resolve(name); + const chosen = name ?? space.judge?.provider ?? config.judge?.provider; + const known = Object.keys(config.modelProviders ?? {}); + if (!chosen) { + throw new JudgeConfigError( + 'No judge configured. Set "judge.provider" in config.json to the name of an entry in "modelProviders".', + ); + } + const def = config.modelProviders?.[chosen]; + if (!def) { + throw new JudgeConfigError( + `Model provider "${chosen}" is not configured.${known.length ? ` Configured: ${known.join(', ')}.` : ' Add it under "modelProviders" in config.json.'}`, + ); + } + return createModelProvider(chosen, def, env); } diff --git a/src/judge/index.ts b/src/judge/index.ts index da74efd..2068eb3 100644 --- a/src/judge/index.ts +++ b/src/judge/index.ts @@ -1,4 +1,4 @@ -export { createJudge, createJudgeProvider, JudgeConfigError, RoutedJudgeProvider } from './factory'; +export { createJudge, createModelProvider, JudgeConfigError } from './factory'; export { buildJudgeSystemPrompt, buildJudgeUserPrompt, parseJudgeResponse } from './prompt'; export type { AnthropicJudgeProviderOptions } from './provider/anthropic'; export { AnthropicJudgeProvider } from './provider/anthropic'; @@ -11,7 +11,7 @@ export { OpenAIJudgeProvider } from './provider/openai'; export { PerRuleJudgeProvider } from './provider/per-rule'; export type { JudgeConfig, - JudgeProviderDefinition, + ModelProviderConfig, SemanticJudgeInput, SemanticJudgeProvider, SemanticJudgeResult, diff --git a/src/judge/types.ts b/src/judge/types.ts index c22594f..a190a4b 100644 --- a/src/judge/types.ts +++ b/src/judge/types.ts @@ -8,8 +8,6 @@ export interface SemanticRuleSpec { category: RuleCategory | string; /** Allowed verdicts, each with a description of when it applies. */ verdicts: Record; - /** Name of the judge provider the rule asks for, if any. */ - judge?: string; } /** One node and every semantic rule to judge on it. Judges may answer all rules in a single call. */ @@ -38,13 +36,13 @@ export interface SemanticJudgeProvider { evaluate(input: SemanticJudgeInput): Promise>; } -/** Judge provider types. `none` is an inert placeholder for templates; judging with it is an error. */ -export const JUDGE_PROVIDER_TYPES = ['anthropic', 'openai', 'jev', 'fake', 'none'] as const; -export type JudgeProviderType = (typeof JUDGE_PROVIDER_TYPES)[number]; +/** Model provider types. `fake` is for tests. */ +export const MODEL_PROVIDER_TYPES = ['anthropic', 'openai', 'jev', 'fake'] as const; +export type ModelProviderType = (typeof MODEL_PROVIDER_TYPES)[number]; -/** One judge provider: a backend type plus how to reach it. */ -export interface JudgeProviderDefinition { - type: JudgeProviderType; +/** How to reach a model: config `modelProviders.`. */ +export interface ModelProviderConfig { + type: ModelProviderType; model?: string; endpoint?: string; /** Prefer `apiKeyEnv`: keys do not belong in config files. */ @@ -53,13 +51,8 @@ export interface JudgeProviderDefinition { apiKeyEnv?: string; } -/** - * Judge configuration: the flat shorthand (provider fields directly on `judge`), which defines one provider - * named "default", and/or several named `providers`. - */ -export interface JudgeConfig extends Partial { - /** Named providers, for rules (or runs) that pick one by name. */ - providers?: Record; - /** Provider for rules that name none. Needed when several providers are configured. */ - default?: string; +/** Which model provider judges semantic rules: config `judge`, at the top level or on a space. */ +export interface JudgeConfig { + /** Name of an entry in `modelProviders`. */ + provider?: string; } diff --git a/src/judge/verdicts.ts b/src/judge/verdicts.ts index 67646ea..1f593c8 100644 --- a/src/judge/verdicts.ts +++ b/src/judge/verdicts.ts @@ -64,7 +64,6 @@ export function resolveSemanticRule(rule: Rule): ResolvedSemanticRule { description: rule.description, rubric: rule.rubric, category: rule.category, - judge: rule.judge, verdicts, }, severities, diff --git a/src/schema/gating.ts b/src/schema/gating.ts index 0d3f3cd..0c9588f 100644 --- a/src/schema/gating.ts +++ b/src/schema/gating.ts @@ -57,7 +57,7 @@ export interface GatingOptions { /** Evaluate rules on gated nodes anyway (gates are still reported). */ includeGated?: boolean; /** Judge for `mode: semantic` rules. Without it, semantic rules are not evaluated. */ - judge?: SemanticJudgeProvider; + provider?: SemanticJudgeProvider; /** * Label of the one node whose result is wanted (`validate-file`). Gating is still computed over the * whole space, but the judge is only consulted for that node and for its hierarchy ancestors' gate-capable @@ -121,7 +121,7 @@ export async function evaluateRulesWithGating( } } - const judge = options.judge; + const judge = options.provider; const violations: RuleViolation[] = []; const gateOrigins = new Map(); diff --git a/src/schema/metadata-contract.ts b/src/schema/metadata-contract.ts index ebb7030..08ac537 100644 --- a/src/schema/metadata-contract.ts +++ b/src/schema/metadata-contract.ts @@ -102,8 +102,6 @@ const RULE_SCHEMA = { }, minProperties: 2, }, - /** Semantic rules: name of the configured judge provider to use. A run-level override takes precedence. */ - judge: { type: 'string', minLength: 1 }, type: { type: 'string', minLength: 1 }, scope: { enum: ['global'] }, override: { type: 'boolean' }, @@ -126,7 +124,7 @@ const RULE_SCHEMA = { }, else: { required: ['check'], - not: { anyOf: [{ required: ['verdicts'] }, { required: ['rubric'] }, { required: ['judge'] }] }, + not: { anyOf: [{ required: ['verdicts'] }, { required: ['rubric'] }] }, properties: { severity: SEVERITY_SCHEMA }, }, additionalProperties: false, diff --git a/src/validate.ts b/src/validate.ts index f33eab1..93040b2 100644 --- a/src/validate.ts +++ b/src/validate.ts @@ -1,8 +1,8 @@ import { isAbsolute, relative, resolve } from 'node:path'; import type { ErrorObject } from 'ajv'; import type { Config, SpaceConfig } from './config'; -import { createJudge, RoutedJudgeProvider } from './judge/factory'; -import type { JudgeConfig, SemanticJudgeProvider } from './judge/types'; +import { createJudge } from './judge/factory'; +import type { SemanticJudgeProvider } from './judge/types'; import { readSpace } from './read/read-space'; import { computeHierarchyDepths, evaluateRulesWithGating, type GatingSeed } from './schema/gating'; import { filterRules, type RuleFilter } from './schema/rule-filter'; @@ -170,7 +170,7 @@ export interface ValidateSpaceOptions { includeGated?: boolean; /** Evaluate only rules matching this filter (see `parseRuleFilter`). */ ruleFilter?: RuleFilter; - /** Whether to evaluate semantic rules (default: false, or true if configured in space/config). */ + /** Evaluate semantic rules with the configured judge (default: false). */ semantic?: boolean; /** Name of the configured judge provider to use for every semantic rule, overriding the rules' own. Implies `semantic`. */ judge?: string; @@ -179,20 +179,18 @@ export interface ValidateSpaceOptions { } /** - * Resolve the semantic judge for a run: none unless semantic evaluation is requested (option, else - * space/config `semantic`). An injected provider wins over the configured one. + * How a run gets its judge: none unless the caller asks (`semantic`, or a `judge` name, which implies it; + * `semantic: false` wins). The judge is built only if a semantic rule is to be judged, so a misconfigured + * judge fails before any request and only when it is needed. An injected provider replaces the configured one. */ -function resolveJudge( - requested: boolean | undefined, - injected: SemanticJudgeProvider | undefined, +function judgeForRun( + options: { semantic?: boolean; judge?: string; judgeProvider?: SemanticJudgeProvider }, space: SpaceConfig, config: Config, - judgeName?: string, -): SemanticJudgeProvider | undefined { - // Naming a judge on the command line asks for semantic evaluation unless it is switched off explicitly. - const enabled = requested ?? (judgeName !== undefined || (space.semantic ?? config.semantic ?? false)); - if (!enabled) return undefined; - return createJudge((space.judge ?? config.judge) as JudgeConfig | undefined, process.env, injected, judgeName); +): (() => SemanticJudgeProvider) | undefined { + const requested = options.semantic ?? options.judge !== undefined; + if (!requested) return undefined; + return () => options.judgeProvider ?? createJudge(config, space, options.judge); } /** One-line text for a rule violation: rule, description, and any judge evidence and verdict guidance. */ @@ -211,7 +209,7 @@ type SpaceRuleEvaluation = { * Evaluate a space's rules. With a main hierarchy, evaluation cascades: schema, hierarchy * and gating-rule failures (the `seeds`) suppress rule evaluation for the failing node's subtree. * Shared by `validateSpace` and `validateFile` so both report identical rule results for a node. - * Semantic rules run only when a `judge` is supplied. + * Semantic rules run only when `judge` is supplied; it is called once, and only if a semantic rule survives the filter. */ async function evaluateSpaceRules( nodes: SpaceNode[], @@ -219,25 +217,29 @@ async function evaluateSpaceRules( seeds: GatingSeed[], options: { includeGated?: boolean; - judge?: SemanticJudgeProvider; + judge?: () => SemanticJudgeProvider; judgeTarget?: string; ruleFilter?: RuleFilter; } = {}, ): Promise { const rules = metadata.rules && filterRules(metadata.rules, options.ruleFilter); if (!rules) return { violations: [], gates: [] }; - // Misconfigured judges fail before any request is made. - if (options.judge instanceof RoutedJudgeProvider) options.judge.prepare(rules); + // Built before any node is evaluated, so a misconfigured judge fails before any request is made. + const provider = options.judge && rules.some((r) => r.mode === 'semantic') ? options.judge() : undefined; if (!metadata.hierarchy) { const result = await validateRules(nodes, rules, { - semantic: options.judge !== undefined, - judgeProvider: options.judge, + semantic: provider !== undefined, + judgeProvider: provider, judgeTarget: options.judgeTarget, }); return { violations: result, gates: [] }; } const graph = buildSpaceGraph(nodes, metadata.hierarchy); - return evaluateRulesWithGating(nodes, rules, graph, seeds, options); + return evaluateRulesWithGating(nodes, rules, graph, seeds, { + includeGated: options.includeGated, + provider, + judgeTarget: options.judgeTarget, + }); } /** @@ -363,7 +365,7 @@ export async function validateSpace( ), ...result.hierarchyViolations.map((v): GatingSeed => ({ file: v.file, title: v.nodeTitle, reason: 'hierarchy' })), ]; - const judge = resolveJudge(options.semantic, options.judgeProvider, context.space, context.config, options.judge); + const judge = judgeForRun(options, context.space, context.config); const { violations, gates } = await evaluateSpaceRules(nodes, metadata, seeds, { includeGated: options.includeGated, ruleFilter: options.ruleFilter, @@ -411,7 +413,7 @@ export interface ValidateFileOptions { includeGated?: boolean; /** Evaluate only rules matching this filter (see `parseRuleFilter`). */ ruleFilter?: RuleFilter; - /** Whether to evaluate semantic rules (default: false, or true if configured in space/config). */ + /** Evaluate semantic rules with the configured judge (default: false). */ semantic?: boolean; /** Name of the configured judge provider to use for every semantic rule. Implies `semantic`. */ judge?: string; @@ -540,7 +542,7 @@ export async function validateFile( (v): GatingSeed => ({ file: v.file, title: v.nodeTitle, reason: 'hierarchy' }), ), ]; - const judge = resolveJudge(options.semantic, options.judgeProvider, space, config, options.judge); + const judge = judgeForRun(options, space, config); const { violations, gates } = await evaluateSpaceRules(nodes, metadata, seeds, { includeGated: options.includeGated, ruleFilter: options.ruleFilter, diff --git a/tests/commands/validate-judge-flag.test.ts b/tests/commands/validate-judge-flag.test.ts index e9c3453..dbbdd2c 100644 --- a/tests/commands/validate-judge-flag.test.ts +++ b/tests/commands/validate-judge-flag.test.ts @@ -6,7 +6,7 @@ import { makeTempSpace } from '../helpers/temp-space'; const ROOT = join(import.meta.dir, '..', '..'); const space = makeTempSpace( [{ id: 'step-fit', type: 'step', mode: 'semantic', category: 'coherence', description: 'Step fits its task' }], - { judge: { providers: { f: { type: 'fake' } } } }, + { judging: { modelProviders: { f: { type: 'fake' } } } }, ); const configPath = join(space.dir, 'config.json'); writeFileSync(configPath, JSON.stringify(space.config)); @@ -35,7 +35,7 @@ describe('--judge on the CLI', () => { }); } - it('--no-semantic wins over --judge', () => { - expect(spawn('validate', 'ts', '--judge', 'missing', '--no-semantic').exitCode).toBe(0); + it('runs no judge without --semantic or --judge', () => { + expect(spawn('validate', 'ts').exitCode).toBe(0); }); }); diff --git a/tests/config.test.ts b/tests/config.test.ts index fb0e481..fa12b1a 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -387,19 +387,20 @@ describe('loadConfig with includeSpacesFrom', () => { expect(mainConfig.spaces[0]!.name).toBe('main-space'); }); - it('accepts judge config with the jev provider at root and per space', () => { + it('accepts model providers, a top-level judge and a space judge override', () => { const configPath = join(testDir, 'judge-config.json'); writeFileSync( configPath, JSON.stringify({ - judge: { type: 'jev' }, - semantic: true, - spaces: [{ name: 's', path: '/p', judge: { type: 'anthropic', model: 'claude-haiku-5-5' }, semantic: false }], + modelProviders: { jev: { type: 'jev' }, claude: { type: 'anthropic', model: 'claude-haiku-5-5' } }, + judge: { provider: 'jev' }, + spaces: [{ name: 's', path: '/p', judge: { provider: 'claude' } }], }), ); setConfigPath(configPath); const config = loadConfig(); - expect(config.judge?.type).toBe('jev'); - expect(config.spaces?.[0]?.judge?.model).toBe('claude-haiku-5-5'); + expect(config.judge?.provider).toBe('jev'); + expect(config.modelProviders?.claude?.model).toBe('claude-haiku-5-5'); + expect(config.spaces?.[0]?.judge?.provider).toBe('claude'); }); }); diff --git a/tests/helpers/temp-space.ts b/tests/helpers/temp-space.ts index 263b105..4cde0ae 100644 --- a/tests/helpers/temp-space.ts +++ b/tests/helpers/temp-space.ts @@ -26,7 +26,7 @@ export interface TempSpace { */ export function makeTempSpace( rules: object[], - options: { frontmatter?: Record; judge?: Config['judge'] } = {}, + options: { frontmatter?: Record; judging?: Pick } = {}, ): TempSpace { const dir = mkdtempSync(join(tmpdir(), 'sctx-temp-space-')); const schemaPath = join(dir, 'schema.json'); @@ -50,7 +50,7 @@ export function makeTempSpace( dir, config: { spaces: [{ name: 'ts', path: space, schema: schemaPath }], - ...(options.judge && { judge: options.judge }), + ...options.judging, }, file: (name) => join(space, name), cleanup: () => rmSync(dir, { recursive: true, force: true }), diff --git a/tests/judge/factory.test.ts b/tests/judge/factory.test.ts index bdeaca4..3135c36 100644 --- a/tests/judge/factory.test.ts +++ b/tests/judge/factory.test.ts @@ -1,126 +1,74 @@ import { describe, expect, it } from 'bun:test'; -import { createJudge, createJudgeProvider, JudgeConfigError } from '../../src/judge/factory'; +import { createJudge, createModelProvider, JudgeConfigError } from '../../src/judge/factory'; import { AnthropicJudgeProvider } from '../../src/judge/provider/anthropic'; import { FakeJudgeProvider } from '../../src/judge/provider/fake'; import { JevJudgeProvider } from '../../src/judge/provider/jev'; import { OpenAIJudgeProvider } from '../../src/judge/provider/openai'; -import type { JudgeConfig } from '../../src/judge/types'; -import type { Rule } from '../../src/types'; -import { graded, node, passFailRule } from './helpers'; -describe('createJudgeProvider factory', () => { - it('returns a custom provider when passed directly', () => { - const custom = new FakeJudgeProvider(); - expect(createJudgeProvider(undefined, {}, custom)).toBe(custom); - }); - - it('creates FakeJudgeProvider when provider is "fake"', () => { - expect(createJudgeProvider({ type: 'fake' }, {}) instanceof FakeJudgeProvider).toBe(true); +describe('createModelProvider', () => { + it('creates FakeJudgeProvider for type "fake"', () => { + expect(createModelProvider('f', { type: 'fake' }, {}) instanceof FakeJudgeProvider).toBe(true); }); it.each([ ['anthropic', 'ANTHROPIC_API_KEY', AnthropicJudgeProvider], ['openai', 'OPENAI_API_KEY', OpenAIJudgeProvider], ['jev', 'OPENROUTER_API_KEY', JevJudgeProvider], - ] as const)('creates %s from config provider with its usual env key (%s)', (provider, keyEnv, Class) => { - expect(createJudgeProvider({ type: provider }, { [keyEnv]: 'k' }) instanceof Class).toBe(true); - }); - - it('never takes the provider type from the environment', () => { - expect(() => createJudgeProvider({ apiKey: 'k' }, { SCTX_JUDGE_PROVIDER: 'anthropic' })).toThrow( - 'No semantic judge configured', - ); + ] as const)('creates %s with its usual env key (%s)', (type, keyEnv, Class) => { + expect(createModelProvider('p', { type }, { [keyEnv]: 'k' }) instanceof Class).toBe(true); }); it('reads the key from the variable named by apiKeyEnv', () => { - const config: JudgeConfig = { type: 'jev', apiKeyEnv: 'MY_JEV_KEY' }; - expect(createJudgeProvider(config, { MY_JEV_KEY: 'k' }) instanceof JevJudgeProvider).toBe(true); - expect(() => createJudgeProvider(config, { OPENROUTER_API_KEY: 'k' })).toThrow('MY_JEV_KEY'); - }); - - it('never infers a provider from API keys in the environment', () => { - const env = { - ANTHROPIC_API_KEY: 'a', - OPENAI_API_KEY: 'o', - OPENROUTER_API_KEY: 'r', - OPENAI_BASE_URL: 'http://x/v1', - }; - expect(() => createJudgeProvider(undefined, env)).toThrow(JudgeConfigError); - expect(() => createJudgeProvider({ model: 'm' }, env)).toThrow('No semantic judge configured'); + const def = { type: 'jev', apiKeyEnv: 'MY_JEV_KEY' } as const; + expect(createModelProvider('p', def, { MY_JEV_KEY: 'k' }) instanceof JevJudgeProvider).toBe(true); + expect(() => createModelProvider('p', def, { OPENROUTER_API_KEY: 'k' })).toThrow('MY_JEV_KEY'); }); it("does not borrow another provider's key", () => { - expect(() => createJudgeProvider({ type: 'anthropic' }, { OPENAI_API_KEY: 'o' })).toThrow('ANTHROPIC_API_KEY'); - expect(() => createJudgeProvider({ type: 'jev' }, { ANTHROPIC_API_KEY: 'a' })).toThrow('OPENROUTER_API_KEY'); + expect(() => createModelProvider('p', { type: 'anthropic' }, { OPENAI_API_KEY: 'o' })).toThrow('ANTHROPIC_API_KEY'); + expect(() => createModelProvider('p', { type: 'jev' }, { ANTHROPIC_API_KEY: 'a' })).toThrow('OPENROUTER_API_KEY'); }); it('lets openai run without a key for local endpoints', () => { - const provider = createJudgeProvider({ type: 'openai', endpoint: 'http://localhost:11434/v1' }, {}); + const provider = createModelProvider('local', { type: 'openai', endpoint: 'http://localhost:11434/v1' }, {}); expect(provider instanceof OpenAIJudgeProvider).toBe(true); }); - it('treats provider "none" as an inert placeholder and rejects unknown providers', () => { - expect(() => createJudgeProvider({ type: 'none' }, {})).toThrow('no provider type'); - expect(() => createJudgeProvider({ type: 'gemini' as never }, {})).toThrow('Unknown judge provider'); + it('rejects an unknown type', () => { + expect(() => createModelProvider('p', { type: 'gemini' as never }, {})).toThrow('unknown type "gemini"'); }); }); -describe('named providers and routing', () => { - const config: JudgeConfig = { - providers: { decision: { type: 'jev' }, llm: { type: 'anthropic', model: 'claude-haiku-5-5' } }, - default: 'llm', +describe('createJudge', () => { + const config = { + modelProviders: { jev: { type: 'jev' }, claude: { type: 'anthropic' } } as const, + judge: { provider: 'claude' }, }; const env = { OPENROUTER_API_KEY: 'r', ANTHROPIC_API_KEY: 'a' }; - it('uses the default provider unless the rule names one', () => { - const judge = createJudge(config, env); - expect(judge.resolve().name).toBe('anthropic'); - expect(judge.resolve('decision').name).toBe('jev'); + it('uses the top-level judge provider', () => { + expect(createJudge(config, {}, undefined, env).name).toBe('anthropic'); }); - it('lets a run-level override replace every rule choice', () => { - expect(createJudge(config, env, undefined, 'decision').resolve('llm').name).toBe('jev'); + it("uses the space's judge over the top-level one, and a run's over both", () => { + const space = { judge: { provider: 'jev' } }; + expect(createJudge(config, space, undefined, env).name).toBe('jev'); + expect(createJudge(config, space, 'claude', env).name).toBe('anthropic'); }); - it('needs `default` when several providers are configured and a rule names none', () => { - expect(() => createJudge({ providers: config.providers }, env).resolve()).toThrow('judge.default'); - }); - - it('uses the only configured provider as the default', () => { - expect(createJudge({ providers: { llm: { type: 'anthropic' } } }, env).resolve().name).toBe('anthropic'); - }); - - it('rejects a name that is not configured, listing what is', () => { - expect(() => createJudge(config, env).resolve('nope')).toThrow('Configured: decision, llm'); - }); - - it('creates providers lazily, so an unused one needs no key', () => { - const judge = createJudge(config, { ANTHROPIC_API_KEY: 'a' }); - expect(judge.resolve().name).toBe('anthropic'); - expect(() => judge.resolve('decision')).toThrow('OPENROUTER_API_KEY'); + it('never infers a provider from API keys in the environment', () => { + const keys = { ANTHROPIC_API_KEY: 'a', OPENAI_API_KEY: 'o', OPENROUTER_API_KEY: 'r' }; + expect(() => createJudge({}, {}, undefined, keys)).toThrow(JudgeConfigError); + expect(() => createJudge({ modelProviders: config.modelProviders }, {}, undefined, keys)).toThrow( + 'No judge configured', + ); }); - it('prepare() fails before any request when a rule names an unusable judge', () => { - const rule = { id: 'r', category: 'best-practice', description: 'd', mode: 'semantic', judge: 'decision' } as Rule; - expect(() => createJudge(config, { ANTHROPIC_API_KEY: 'a' }).prepare([rule])).toThrow(JudgeConfigError); - expect(() => createJudge(config, env).prepare([rule])).not.toThrow(); + it('rejects a provider name that is not configured, listing what is', () => { + expect(() => createJudge(config, {}, 'nope', env)).toThrow('Configured: jev, claude'); }); - it('sends each rule to the provider it names, in one call per provider', async () => { - const a = new FakeJudgeProvider({ 'customer-voice': { verdict: 'pass' } }); - const b = new FakeJudgeProvider({ framing: { verdict: 'good' } }); - const judge = createJudge({ type: 'fake' }, {}); - // Swap in recording providers for the two names. - const routed = Object.assign(judge, { - resolve: (name?: string) => (name === 'b' ? b : a), - }); - const results = await routed.evaluate({ - node, - rules: [{ ...passFailRule }, { ...graded, judge: 'b' }], - }); - expect(results['customer-voice']).toEqual({ verdict: 'pass' }); - expect(results.framing).toEqual({ verdict: 'good' }); - expect(a.callHistory.map((c) => c.rules.map((r) => r.id))).toEqual([['customer-voice']]); - expect(b.callHistory.map((c) => c.rules.map((r) => r.id))).toEqual([['framing']]); + it('fails when the chosen provider has no key', () => { + expect(() => createJudge(config, {}, undefined, { OPENROUTER_API_KEY: 'r' })).toThrow('ANTHROPIC_API_KEY'); }); }); diff --git a/tests/judge/provider/jev.test.ts b/tests/judge/provider/jev.test.ts index f2fabb2..7b278f4 100644 --- a/tests/judge/provider/jev.test.ts +++ b/tests/judge/provider/jev.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, it } from 'bun:test'; -import { createJudgeProvider } from '../../../src/judge/factory'; +import { createModelProvider } from '../../../src/judge/factory'; import { JevJudgeProvider } from '../../../src/judge/provider/jev'; import { graded, node, passFailRule } from '../helpers'; @@ -70,8 +70,7 @@ describe('JevJudgeProvider', () => { }); it('is created explicitly via provider "jev" with OPENROUTER_API_KEY, never auto-detected', () => { - expect(createJudgeProvider({ type: 'jev' }, { OPENROUTER_API_KEY: 'k' }).name).toBe('jev'); - expect(() => createJudgeProvider({ type: 'jev' }, {})).toThrow('OPENROUTER_API_KEY'); - expect(() => createJudgeProvider(undefined, { OPENROUTER_API_KEY: 'k' })).toThrow('No semantic judge configured'); + expect(createModelProvider('jev', { type: 'jev' }, { OPENROUTER_API_KEY: 'k' }).name).toBe('jev'); + expect(() => createModelProvider('jev', { type: 'jev' }, {})).toThrow('OPENROUTER_API_KEY'); }); }); diff --git a/tests/judge/verdicts.test.ts b/tests/judge/verdicts.test.ts index de924fa..15b62e0 100644 --- a/tests/judge/verdicts.test.ts +++ b/tests/judge/verdicts.test.ts @@ -37,10 +37,6 @@ describe('resolveSemanticRule', () => { expect(resolved.guidance).toEqual({ vague: 'Name the customer and the pain.' }); }); - it("carries the rule's judge name to the spec", () => { - expect(resolveSemanticRule({ ...base, judge: 'llm' }).spec.judge).toBe('llm'); - }); - it('rejects custom verdicts without a severity map, unknown verdicts, and sets with no passing verdict', () => { expect(() => resolveSemanticRule({ ...base, verdicts: { a: 'A', b: 'B' } })).toThrow( 'must map the failing verdicts', diff --git a/tests/schema/semantic-rule-contract.test.ts b/tests/schema/semantic-rule-contract.test.ts index 3a5cc70..9c349a1 100644 --- a/tests/schema/semantic-rule-contract.test.ts +++ b/tests/schema/semantic-rule-contract.test.ts @@ -45,11 +45,10 @@ describe('rule contract: check and semantic mode', () => { expect(() => load('semantic-global', { ...base, mode: 'semantic', scope: 'global' })).toThrow(); }); - it('accepts a verdict map with guidance, a severity map (including ignore), a judge and a when', () => { + it('accepts a verdict map with guidance, a severity map (including ignore) and a when', () => { const rule = { ...base, mode: 'semantic', - judge: 'llm', when: "current.status = 'ready'", verdicts: { ok: 'Fine', vague: { description: 'Too vague', guidance: 'Be specific.' } }, severity: { vague: 'warning' }, @@ -67,7 +66,9 @@ describe('rule contract: check and semantic mode', () => { it('keeps semantic-only fields off expression rules', () => { const expr = { ...base, check: 'true' }; expect(() => load('expr-map', { ...expr, severity: { fail: 'error' } })).toThrow(); + // Rules do not choose a judge: that is config, not schema. expect(() => load('expr-judge', { ...expr, judge: 'llm' })).toThrow(); + expect(() => load('semantic-judge', { ...base, mode: 'semantic', judge: 'llm' })).toThrow(); expect(() => load('expr-verdicts', { ...expr, verdicts: { a: 'A', b: 'B' } })).toThrow(); expect(load('expr-when', { ...expr, when: 'true' }).rules).toHaveLength(1); }); diff --git a/tests/validate/semantic-rule-shape.test.ts b/tests/validate/semantic-rule-shape.test.ts index 3f60f56..6325a15 100644 --- a/tests/validate/semantic-rule-shape.test.ts +++ b/tests/validate/semantic-rule-shape.test.ts @@ -1,6 +1,5 @@ import { afterAll, describe, expect, it } from 'bun:test'; import { FakeJudgeProvider } from '../../src/judge/provider/fake'; -import type { JudgeConfig } from '../../src/judge/types'; import { createSpaceContext } from '../../src/space-context'; import type { FileValidationResult } from '../../src/types'; import { validateFile, validateSpace } from '../../src/validate'; @@ -134,14 +133,35 @@ describe('rule `when` precondition', () => { }); }); -describe('run-level judge override', () => { - it('fails before any request when the named judge is not configured', async () => { - const { context } = setup([stepRule], { judge: { providers: { llm: { type: 'fake' } } } }); - await expect(validateSpace(context(), { judge: 'nope' })).rejects.toThrow('Judge "nope" is not configured'); +describe('judge configuration', () => { + const judging = { modelProviders: { llm: { type: 'fake' } } as const, judge: { provider: 'llm' } }; + + it('fails before any request when the named provider is not configured', async () => { + const { context } = setup([stepRule], { judging }); + await expect(validateSpace(context(), { judge: 'nope' })).rejects.toThrow( + 'Model provider "nope" is not configured', + ); }); - it('--judge implies semantic evaluation, and semantic: false still wins', async () => { - const { context } = setup([stepRule], { judge: { providers: { llm: { type: 'fake' } } } }); + it('fails before any request when no judge is configured', async () => { + const { context } = setup([stepRule]); + await expect(validateSpace(context(), { semantic: true })).rejects.toThrow('No judge configured'); + }); + + it('needs no judge configuration while semantic evaluation is not requested', async () => { + const { context } = setup([stepRule]); + await expect(validateSpace(context(), {})).resolves.toBeDefined(); + }); + + it('builds no judge when the rule filter leaves no semantic rule', async () => { + const { context } = setup([stepRule]); + await expect( + validateSpace(context(), { semantic: true, ruleFilter: { mode: ['expression'] } }), + ).resolves.toBeDefined(); + }); + + it('a judge name implies semantic evaluation, and semantic: false still wins', async () => { + const { context } = setup([stepRule], { judging }); const on = new FakeJudgeProvider(); await validateSpace(context(), { judge: 'llm', judgeProvider: on }); expect(on.callHistory.length).toBeGreaterThan(0); @@ -151,22 +171,6 @@ describe('run-level judge override', () => { }); }); -describe('judge type "none"', () => { - it('throws, not skips, when semantic evaluation is attempted', async () => { - const judges: JudgeConfig[] = [{ type: 'none' }, { providers: { llm: { type: 'none' } }, default: 'llm' }]; - for (const judge of judges) { - const { context } = setup([stepRule], { judge }); - await expect(validateSpace(context(), { semantic: true })).rejects.toThrow('no provider type'); - await expect(validateSpace(context(), { judge: 'default' })).rejects.toThrow(); - } - }); - - it('is harmless while semantic evaluation is not requested', async () => { - const { context } = setup([stepRule], { judge: { type: 'none' } }); - await expect(validateSpace(context(), {})).resolves.toBeDefined(); - }); -}); - describe('rule expression errors', () => { const broken = { id: 'broken', category: 'validation', description: 'Broken check', check: '$notAFunction(' }; const brokenWhen = { From 18564dfb4754f97de6b7f988d682981f4fb3e38a Mon Sep 17 00:00:00 2001 From: Roger Barnes Date: Wed, 14 Oct 2026 18:27:33 +1100 Subject: [PATCH 15/22] refactor(validate): one judge option on the API; the CLI decides when to judge validateSpace and validateFile take an optional resolved judge instead of semantic, a judge name and an injected provider: a judge means judge. --semantic and --judge are CLI concerns, turned into a judge by judgeForRun. validateRules follows the same shape internally. --- docs/semantic-rules.md | 2 +- src/commands/validate-file.ts | 7 ++- src/commands/validate.ts | 7 +-- src/judge/factory.ts | 13 ++++ src/schema/validate-rules.ts | 20 +++---- src/validate.ts | 50 ++++------------ tests/judge/factory.test.ts | 18 +++++- tests/schema/validate-rules-semantic.test.ts | 17 ++---- tests/validate/semantic-rule-shape.test.ts | 63 ++++++-------------- tests/validate/semantic-validation.test.ts | 35 ++++------- 10 files changed, 93 insertions(+), 139 deletions(-) diff --git a/docs/semantic-rules.md b/docs/semantic-rules.md index 1193e4b..7736088 100644 --- a/docs/semantic-rules.md +++ b/docs/semantic-rules.md @@ -38,7 +38,7 @@ Semantic rules cost money and send node content to the judge's provider. Thus th - `--semantic` runs them with the configured judge. - `--judge ` runs them with that model provider instead. -Both work on `sctx validate` and `sctx validate-file`. Config cannot turn semantic rules on. The API is the same: pass `semantic` or `judge` to `validateSpace` or `validateFile`. +Both work on `sctx validate` and `sctx validate-file`. Config cannot turn semantic rules on. ## Rule fields diff --git a/src/commands/validate-file.ts b/src/commands/validate-file.ts index 8c69c75..b1d2c6a 100644 --- a/src/commands/validate-file.ts +++ b/src/commands/validate-file.ts @@ -1,7 +1,8 @@ import { loadConfig } from '../config'; +import { judgeForRun } from '../judge/factory'; import { parseRuleFilter } from '../schema/rule-filter'; import type { FileValidationIssue, FileValidationResult } from '../types'; -import { validateFile } from '../validate'; +import { resolveFileSpace, validateFile } from '../validate'; export { validateFile } from '../validate'; @@ -15,10 +16,10 @@ export async function validateFileCommand( options: { json?: boolean; includeGated?: boolean; semantic?: boolean; judge?: string; ruleFilter?: string[] } = {}, ): Promise { const config = loadConfig(); + const space = resolveFileSpace(filePath, config)?.space; const result = await validateFile(filePath, config, { includeGated: options.includeGated, - semantic: options.semantic, - judge: options.judge, + judge: space && judgeForRun(config, space, options), ruleFilter: options.ruleFilter && parseRuleFilter(options.ruleFilter), }); diff --git a/src/commands/validate.ts b/src/commands/validate.ts index b02f78e..ce9722b 100644 --- a/src/commands/validate.ts +++ b/src/commands/validate.ts @@ -1,7 +1,7 @@ import { dirname } from 'node:path'; import chokidar from 'chokidar'; import { getConfigSourceFiles } from '../config'; -import type { SemanticJudgeProvider } from '../judge/types'; +import { judgeForRun } from '../judge/factory'; import { resolveSpacePluginSchemas } from '../plugins/util'; import { parseRuleFilter } from '../schema/rule-filter'; import { bundledSchemasDir } from '../schema/schema'; @@ -17,17 +17,14 @@ export async function validate( semantic?: boolean; judge?: string; ruleFilter?: string[]; - judgeProvider?: SemanticJudgeProvider; } = {}, ): Promise { const { schema, schemaRefRegistry } = context; const result = await validateSpace(context, { depth: options.depth, includeGated: options.includeGated, - semantic: options.semantic, - judge: options.judge, + judge: judgeForRun(context.config, context.space, options), ruleFilter: options.ruleFilter && parseRuleFilter(options.ruleFilter), - judgeProvider: options.judgeProvider, }); const ruleCounts: Record = { error: 0, warning: 0, info: 0 }; for (const v of result.ruleViolations) ruleCounts[v.severity]++; diff --git a/src/judge/factory.ts b/src/judge/factory.ts index edab056..442b650 100644 --- a/src/judge/factory.ts +++ b/src/judge/factory.ts @@ -54,6 +54,19 @@ export function createModelProvider( } } +/** + * The judge a CLI run asks for: none without `--semantic` or `--judge `; the named provider with + * `--judge`; else the configured one. + */ +export function judgeForRun( + config: Parameters[0], + space: Parameters[1], + options: { semantic?: boolean; judge?: string }, +): SemanticJudgeProvider | undefined { + if (!options.semantic && options.judge === undefined) return undefined; + return createJudge(config, space, options.judge); +} + /** * The judge for a run: the model provider named by `name` (the `--judge` override), else the space's * `judge.provider`, else the top-level one. Throws {@link JudgeConfigError} when it cannot be built. diff --git a/src/schema/validate-rules.ts b/src/schema/validate-rules.ts index 83b6c74..30afe3f 100644 --- a/src/schema/validate-rules.ts +++ b/src/schema/validate-rules.ts @@ -7,10 +7,8 @@ import { ruleSeverity } from './rule-severity'; export { DEFAULT_SEVERITY_BY_CATEGORY, ruleSeverity } from './rule-severity'; export interface ValidateRulesOptions { - /** Whether to run semantic rules (default: false). */ - semantic?: boolean; - /** The semantic judge provider to use if semantic is true. */ - judgeProvider?: SemanticJudgeProvider; + /** Judge for semantic rules. Without one, semantic rules are not evaluated. */ + judge?: SemanticJudgeProvider; /** Label of the only node to judge (`validate-file`); other nodes' semantic rules are skipped. */ judgeTarget?: string; /** Set of node labels that have already failed upstream structural checks. */ @@ -23,7 +21,7 @@ export interface ValidateRulesOptions { * * @param nodes - All nodes in the space * @param rules - Rules metadata list - * @param options - Validation options including semantic opt-in and judge provider + * @param options - Validation options, including the judge for semantic rules * @returns Rule violations */ export async function validateRules( @@ -110,13 +108,9 @@ export async function validateRules( } } - // 2. Evaluate semantic rules only if explicitly requested - if (options.semantic) { - const judgeProvider = options.judgeProvider; - if (!judgeProvider) { - throw new Error('Semantic evaluation was requested but no judgeProvider was provided to validateRules.'); - } - + // 2. Evaluate semantic rules only when a judge is supplied + const judge = options.judge; + if (judge) { for (const node of nodes) { if (options.judgeTarget !== undefined && node.label !== options.judgeTarget) continue; const applicable: Rule[] = []; @@ -137,7 +131,7 @@ export async function validateRules( // Safe cascading default: gate semantic evaluation if node failed upstream checks if (failedNodeLabels.has(node.label)) continue; - for (const j of await judgeNode(judgeProvider, node, applicable)) { + for (const j of await judgeNode(judge, node, applicable)) { if (!j.failed) continue; violations.push({ file: node.label, diff --git a/src/validate.ts b/src/validate.ts index 93040b2..546c149 100644 --- a/src/validate.ts +++ b/src/validate.ts @@ -1,7 +1,6 @@ import { isAbsolute, relative, resolve } from 'node:path'; import type { ErrorObject } from 'ajv'; import type { Config, SpaceConfig } from './config'; -import { createJudge } from './judge/factory'; import type { SemanticJudgeProvider } from './judge/types'; import { readSpace } from './read/read-space'; import { computeHierarchyDepths, evaluateRulesWithGating, type GatingSeed } from './schema/gating'; @@ -170,27 +169,8 @@ export interface ValidateSpaceOptions { includeGated?: boolean; /** Evaluate only rules matching this filter (see `parseRuleFilter`). */ ruleFilter?: RuleFilter; - /** Evaluate semantic rules with the configured judge (default: false). */ - semantic?: boolean; - /** Name of the configured judge provider to use for every semantic rule, overriding the rules' own. Implies `semantic`. */ - judge?: string; - /** Optional custom judge provider (for testing or injection). */ - judgeProvider?: SemanticJudgeProvider; -} - -/** - * How a run gets its judge: none unless the caller asks (`semantic`, or a `judge` name, which implies it; - * `semantic: false` wins). The judge is built only if a semantic rule is to be judged, so a misconfigured - * judge fails before any request and only when it is needed. An injected provider replaces the configured one. - */ -function judgeForRun( - options: { semantic?: boolean; judge?: string; judgeProvider?: SemanticJudgeProvider }, - space: SpaceConfig, - config: Config, -): (() => SemanticJudgeProvider) | undefined { - const requested = options.semantic ?? options.judge !== undefined; - if (!requested) return undefined; - return () => options.judgeProvider ?? createJudge(config, space, options.judge); + /** Judge for `mode: semantic` rules. Without one, semantic rules are not evaluated. */ + judge?: SemanticJudgeProvider; } /** One-line text for a rule violation: rule, description, and any judge evidence and verdict guidance. */ @@ -209,7 +189,7 @@ type SpaceRuleEvaluation = { * Evaluate a space's rules. With a main hierarchy, evaluation cascades: schema, hierarchy * and gating-rule failures (the `seeds`) suppress rule evaluation for the failing node's subtree. * Shared by `validateSpace` and `validateFile` so both report identical rule results for a node. - * Semantic rules run only when `judge` is supplied; it is called once, and only if a semantic rule survives the filter. + * Semantic rules run only when a `judge` is supplied. */ async function evaluateSpaceRules( nodes: SpaceNode[], @@ -217,19 +197,17 @@ async function evaluateSpaceRules( seeds: GatingSeed[], options: { includeGated?: boolean; - judge?: () => SemanticJudgeProvider; + judge?: SemanticJudgeProvider; judgeTarget?: string; ruleFilter?: RuleFilter; } = {}, ): Promise { const rules = metadata.rules && filterRules(metadata.rules, options.ruleFilter); if (!rules) return { violations: [], gates: [] }; - // Built before any node is evaluated, so a misconfigured judge fails before any request is made. - const provider = options.judge && rules.some((r) => r.mode === 'semantic') ? options.judge() : undefined; + const provider = options.judge; if (!metadata.hierarchy) { const result = await validateRules(nodes, rules, { - semantic: provider !== undefined, - judgeProvider: provider, + judge: provider, judgeTarget: options.judgeTarget, }); return { violations: result, gates: [] }; @@ -365,11 +343,10 @@ export async function validateSpace( ), ...result.hierarchyViolations.map((v): GatingSeed => ({ file: v.file, title: v.nodeTitle, reason: 'hierarchy' })), ]; - const judge = judgeForRun(options, context.space, context.config); const { violations, gates } = await evaluateSpaceRules(nodes, metadata, seeds, { includeGated: options.includeGated, ruleFilter: options.ruleFilter, - judge, + judge: options.judge, }); result.ruleViolations = violations; result.coverageGates = gates; @@ -393,7 +370,7 @@ export async function validateSpace( } /** Find which space a file belongs to by checking directory containment. */ -function resolveFileSpace(filePath: string, config: Config): { space: SpaceConfig; label: string } | null { +export function resolveFileSpace(filePath: string, config: Config): { space: SpaceConfig; label: string } | null { const absFile = isAbsolute(filePath) ? filePath : resolve(process.cwd(), filePath); for (const space of config.spaces) { @@ -413,12 +390,8 @@ export interface ValidateFileOptions { includeGated?: boolean; /** Evaluate only rules matching this filter (see `parseRuleFilter`). */ ruleFilter?: RuleFilter; - /** Evaluate semantic rules with the configured judge (default: false). */ - semantic?: boolean; - /** Name of the configured judge provider to use for every semantic rule. Implies `semantic`. */ - judge?: string; - /** Optional custom judge provider (for testing or injection). */ - judgeProvider?: SemanticJudgeProvider; + /** Judge for `mode: semantic` rules. Without one, semantic rules are not evaluated. */ + judge?: SemanticJudgeProvider; } /** @@ -542,11 +515,10 @@ export async function validateFile( (v): GatingSeed => ({ file: v.file, title: v.nodeTitle, reason: 'hierarchy' }), ), ]; - const judge = judgeForRun(options, space, config); const { violations, gates } = await evaluateSpaceRules(nodes, metadata, seeds, { includeGated: options.includeGated, ruleFilter: options.ruleFilter, - judge, + judge: options.judge, judgeTarget: label, }); for (const v of violations) { diff --git a/tests/judge/factory.test.ts b/tests/judge/factory.test.ts index 3135c36..f651aa6 100644 --- a/tests/judge/factory.test.ts +++ b/tests/judge/factory.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'bun:test'; -import { createJudge, createModelProvider, JudgeConfigError } from '../../src/judge/factory'; +import { createJudge, createModelProvider, JudgeConfigError, judgeForRun } from '../../src/judge/factory'; import { AnthropicJudgeProvider } from '../../src/judge/provider/anthropic'; import { FakeJudgeProvider } from '../../src/judge/provider/fake'; import { JevJudgeProvider } from '../../src/judge/provider/jev'; @@ -72,3 +72,19 @@ describe('createJudge', () => { expect(() => createJudge(config, {}, undefined, { OPENROUTER_API_KEY: 'r' })).toThrow('ANTHROPIC_API_KEY'); }); }); + +describe('judgeForRun (CLI)', () => { + const config = { modelProviders: { f: { type: 'fake' }, g: { type: 'fake' } } as const, judge: { provider: 'f' } }; + + it('gives no judge unless the run asks with --semantic or --judge', () => { + expect(judgeForRun(config, {}, {})).toBeUndefined(); + expect(judgeForRun({}, {}, {})).toBeUndefined(); // no judge config needed when not asked + expect(judgeForRun(config, {}, { semantic: true })).toBeInstanceOf(FakeJudgeProvider); + expect(judgeForRun(config, {}, { judge: 'g' })).toBeInstanceOf(FakeJudgeProvider); + }); + + it('fails when asked and no judge is configured, or the named provider is missing', () => { + expect(() => judgeForRun({}, {}, { semantic: true })).toThrow('No judge configured'); + expect(() => judgeForRun(config, {}, { judge: 'nope' })).toThrow('Model provider "nope" is not configured'); + }); +}); diff --git a/tests/schema/validate-rules-semantic.test.ts b/tests/schema/validate-rules-semantic.test.ts index e5679b4..7b29f56 100644 --- a/tests/schema/validate-rules-semantic.test.ts +++ b/tests/schema/validate-rules-semantic.test.ts @@ -67,15 +67,13 @@ describe('validateRules - semantic evaluation', () => { }, ]; - it('skips semantic rules when semantic option is false / omitted (default validate)', async () => { + it('skips semantic rules without a judge (default validate)', async () => { // Default validate should only run expression rules and remain deterministic & free - const fake = new FakeJudgeProvider(); - const violations = await validateRules(sampleNodes, rules, { judgeProvider: fake }); + const violations = await validateRules(sampleNodes, rules); expect(violations).toHaveLength(0); - expect(fake.callHistory).toHaveLength(0); }); - it('evaluates semantic rules when semantic option is true', async () => { + it('evaluates semantic rules with a judge', async () => { const fake = new FakeJudgeProvider( new Map([ [ @@ -90,8 +88,7 @@ describe('validateRules - semantic evaluation', () => { ); const violations = await validateRules(sampleNodes, rules, { - semantic: true, - judgeProvider: fake, + judge: fake, }); expect(violations).toHaveLength(1); @@ -111,8 +108,7 @@ describe('validateRules - semantic evaluation', () => { const gatedNodeLabels = new Set(['bad-opp.md']); const violations = await validateRules(sampleNodes, rules, { - semantic: true, - judgeProvider: fake, + judge: fake, gatedNodeLabels, }); @@ -143,8 +139,7 @@ describe('validateRules - semantic evaluation', () => { const fake = new FakeJudgeProvider(); const violations = await validateRules(sampleNodes, strictRules, { - semantic: true, - judgeProvider: fake, + judge: fake, }); // bad-opp.md source is "Engineering suggestion", so it failed require-interviews-source diff --git a/tests/validate/semantic-rule-shape.test.ts b/tests/validate/semantic-rule-shape.test.ts index 6325a15..829c85f 100644 --- a/tests/validate/semantic-rule-shape.test.ts +++ b/tests/validate/semantic-rule-shape.test.ts @@ -39,8 +39,7 @@ describe('semantic rules: verdict severity, ignore and guidance', () => { it('reports each verdict at its mapped severity with its guidance', async () => { const { context } = setup([stepRule]); const minor = await validateSpace(context(), { - semantic: true, - judgeProvider: new FakeJudgeProvider({ 'step-fit': { verdict: 'minor' } }), + judge: new FakeJudgeProvider({ 'step-fit': { verdict: 'minor' } }), }); expect(minor.ruleViolations).toEqual([ expect.objectContaining({ @@ -52,8 +51,7 @@ describe('semantic rules: verdict severity, ignore and guidance', () => { ]); const major = await validateSpace(context(), { - semantic: true, - judgeProvider: new FakeJudgeProvider({ 'step-fit': { verdict: 'major' } }), + judge: new FakeJudgeProvider({ 'step-fit': { verdict: 'major' } }), }); expect(major.ruleViolations[0]).toMatchObject({ severity: 'error', verdict: 'major' }); }); @@ -61,7 +59,7 @@ describe('semantic rules: verdict severity, ignore and guidance', () => { it('judges an ignored verdict without reporting it', async () => { const { context } = setup([stepRule]); const fake = new FakeJudgeProvider({ 'step-fit': { verdict: 'unsure' } }); - const result = await validateSpace(context(), { semantic: true, judgeProvider: fake }); + const result = await validateSpace(context(), { judge: fake }); expect(fake.callHistory).toHaveLength(1); expect(result.ruleViolations).toEqual([]); }); @@ -71,12 +69,12 @@ describe('semantic rules: verdict severity, ignore and guidance', () => { const { context } = setup([taskRule, { ...stepRule, id: 'step-fit' }]); const minor = new FakeJudgeProvider({ 'task-fit': { verdict: 'minor' } }); - const ungated = await validateSpace(context(), { semantic: true, judgeProvider: minor }); + const ungated = await validateSpace(context(), { judge: minor }); expect(ungated.coverageGates).toEqual([]); expect(minor.callHistory.some((c) => c.node.label === 'S.md')).toBe(true); const major = new FakeJudgeProvider({ 'task-fit': { verdict: 'major' } }); - const gated = await validateSpace(context(), { semantic: true, judgeProvider: major }); + const gated = await validateSpace(context(), { judge: major }); expect(gated.coverageGates[0]).toMatchObject({ reason: 'rule', ruleId: 'task-fit' }); expect(major.callHistory.some((c) => c.node.label === 'S.md')).toBe(false); }); @@ -84,8 +82,7 @@ describe('semantic rules: verdict severity, ignore and guidance', () => { it('shows guidance in validate-file output', async () => { const { space } = setup([stepRule]); const result = (await validateFile(space.file('S.md'), space.config, { - semantic: true, - judgeProvider: new FakeJudgeProvider({ 'step-fit': { verdict: 'minor' } }), + judge: new FakeJudgeProvider({ 'step-fit': { verdict: 'minor' } }), configDir: space.dir, })) as FileValidationResult; expect(result.warnings['rule:step-fit']?.message).toContain('Guidance: Tighten the wording.'); @@ -94,10 +91,10 @@ describe('semantic rules: verdict severity, ignore and guidance', () => { it('--rule-filter severity matches any severity a verdict map can report', async () => { const { context } = setup([stepRule]); const judge = new FakeJudgeProvider({ 'step-fit': { verdict: 'minor' } }); - await validateSpace(context(), { semantic: true, judgeProvider: judge, ruleFilter: { severity: ['warning'] } }); + await validateSpace(context(), { judge: judge, ruleFilter: { severity: ['warning'] } }); expect(judge.callHistory.length).toBeGreaterThan(0); const none = new FakeJudgeProvider(); - await validateSpace(context(), { semantic: true, judgeProvider: none, ruleFilter: { severity: ['info'] } }); + await validateSpace(context(), { judge: none, ruleFilter: { severity: ['info'] } }); expect(none.callHistory).toHaveLength(0); }); }); @@ -122,52 +119,30 @@ describe('rule `when` precondition', () => { const rule = { ...stepRule, id: 'task-fit', type: 'task', when }; const { context } = setup([rule], { frontmatter: { T: 'status: ready' } }); const judge = new FakeJudgeProvider(); - await validateSpace(context(), { semantic: true, judgeProvider: judge }); + await validateSpace(context(), { judge: judge }); expect(judge.callHistory.map((c) => c.node.label)).toEqual(['T.md']); const none = setup([rule]); const idle = new FakeJudgeProvider(); - const result = await validateSpace(none.context(), { semantic: true, judgeProvider: idle }); + const result = await validateSpace(none.context(), { judge: idle }); expect(idle.callHistory).toHaveLength(0); expect(result.coverageGates).toEqual([]); // not applicable is not gated }); }); -describe('judge configuration', () => { - const judging = { modelProviders: { llm: { type: 'fake' } } as const, judge: { provider: 'llm' } }; - - it('fails before any request when the named provider is not configured', async () => { - const { context } = setup([stepRule], { judging }); - await expect(validateSpace(context(), { judge: 'nope' })).rejects.toThrow( - 'Model provider "nope" is not configured', - ); - }); - - it('fails before any request when no judge is configured', async () => { - const { context } = setup([stepRule]); - await expect(validateSpace(context(), { semantic: true })).rejects.toThrow('No judge configured'); - }); - - it('needs no judge configuration while semantic evaluation is not requested', async () => { +describe('judge option', () => { + it('evaluates no semantic rules without a judge', async () => { const { context } = setup([stepRule]); - await expect(validateSpace(context(), {})).resolves.toBeDefined(); + const result = await validateSpace(context(), {}); + expect(result.ruleViolations).toEqual([]); }); - it('builds no judge when the rule filter leaves no semantic rule', async () => { + it('judges with the judge it is given', async () => { const { context } = setup([stepRule]); - await expect( - validateSpace(context(), { semantic: true, ruleFilter: { mode: ['expression'] } }), - ).resolves.toBeDefined(); - }); - - it('a judge name implies semantic evaluation, and semantic: false still wins', async () => { - const { context } = setup([stepRule], { judging }); - const on = new FakeJudgeProvider(); - await validateSpace(context(), { judge: 'llm', judgeProvider: on }); - expect(on.callHistory.length).toBeGreaterThan(0); - const off = new FakeJudgeProvider(); - await validateSpace(context(), { judge: 'llm', semantic: false, judgeProvider: off }); - expect(off.callHistory).toHaveLength(0); + const judge = new FakeJudgeProvider({ 'step-fit': { verdict: 'minor' } }); + const result = await validateSpace(context(), { judge }); + expect(judge.callHistory.length).toBeGreaterThan(0); + expect(result.ruleViolations[0]).toMatchObject({ verdict: 'minor' }); }); }); diff --git a/tests/validate/semantic-validation.test.ts b/tests/validate/semantic-validation.test.ts index a322585..845a47c 100644 --- a/tests/validate/semantic-validation.test.ts +++ b/tests/validate/semantic-validation.test.ts @@ -26,22 +26,19 @@ describe('Semantic rule evaluation integration (validateSpace & validateFile)', const validContext = createSpaceContext('strict-valid', validConfig, { configDir: import.meta.dir }); describe('validateSpace', () => { - it('skips semantic rules by default (deterministic & free)', async () => { - const fake = new FakeJudgeProvider({ pass: false }); - const result = await validateSpace(validContext, { judgeProvider: fake }); - expect(fake.callHistory).toHaveLength(0); + it('skips semantic rules without a judge (deterministic & free)', async () => { + const result = await validateSpace(validContext); expect(result.ruleViolations.some((v) => v.ruleId === 'opportunity-customer-voice')).toBe(false); }); - it('evaluates semantic rules when opt-in flag semantic: true is passed', async () => { + it('evaluates semantic rules when a judge is passed', async () => { const fakeProvider = new FakeJudgeProvider({ pass: true, evidence: 'Opportunity is framed in clear customer language describing difficulty with sign-up.', }); const result = await validateSpace(validContext, { - semantic: true, - judgeProvider: fakeProvider, + judge: fakeProvider, }); expect(fakeProvider.callHistory.some((c) => c.rules.some((r) => r.id === 'opportunity-customer-voice'))).toBe( @@ -59,8 +56,7 @@ describe('Semantic rule evaluation integration (validateSpace & validateFile)', }); const result = await validateSpace(validContext, { - semantic: true, - judgeProvider: fakeProvider, + judge: fakeProvider, }); const violation = result.ruleViolations.find((v) => v.ruleId === 'opportunity-customer-voice'); @@ -84,8 +80,7 @@ describe('Semantic rule evaluation integration (validateSpace & validateFile)', const fakeProvider = new FakeJudgeProvider(); const result = await validateSpace(invalidContext, { - semantic: true, - judgeProvider: fakeProvider, + judge: fakeProvider, }); // opportunity-no-source.md fails schema validation (missing required 'source' field) @@ -119,8 +114,7 @@ describe('Semantic rule evaluation integration (validateSpace & validateFile)', }); const result = (await validateFile(targetFile, validConfig, { - semantic: true, - judgeProvider: failingProvider, + judge: failingProvider, configDir: import.meta.dir, })) as FileValidationResult; @@ -156,8 +150,7 @@ describe('validateFile judges only the target and its gate-capable ancestor rule const run = (judge: FakeJudgeProvider, file: string, extra = {}) => validateFile(space.file(file), space.config, { - semantic: true, - judgeProvider: judge, + judge: judge, configDir: space.dir, ...extra, }) as Promise; @@ -199,7 +192,7 @@ describe('Semantic rules: verdicts, batching and rule filter', () => { it('judges each node once with all of its semantic rules', async () => { const fake = new FakeJudgeProvider(); - await validateSpace(context, { semantic: true, judgeProvider: fake }); + await validateSpace(context, { judge: fake }); const labels = fake.callHistory.map((c) => c.node.label); expect(new Set(labels).size).toBe(labels.length); expect(fake.callHistory.every((c) => c.rules.length >= 1)).toBe(true); @@ -207,14 +200,14 @@ describe('Semantic rules: verdicts, batching and rule filter', () => { it('reports a verdict that has a severity as a violation carrying the verdict and evidence', async () => { const fake = new FakeJudgeProvider({ verdict: 'fail', confidence: 0.7, evidence: 'solution in disguise' }); - const result = await validateSpace(context, { semantic: true, judgeProvider: fake }); + const result = await validateSpace(context, { judge: fake }); const violation = result.ruleViolations.find((v) => v.ruleId === 'opportunity-customer-voice'); expect(violation).toMatchObject({ verdict: 'fail', evidence: 'solution in disguise' }); }); it('reports a per-rule judge error as a violation without gating the subtree', async () => { const fake = new FakeJudgeProvider({ error: 'rate limited' }); - const result = await validateSpace(context, { semantic: true, judgeProvider: fake }); + const result = await validateSpace(context, { judge: fake }); const violation = result.ruleViolations.find((v) => v.ruleId === 'opportunity-customer-voice'); expect(violation?.description).toContain('Judge error: rate limited'); expect(result.coverageGates.filter((g) => g.ruleId === 'opportunity-customer-voice')).toEqual([]); @@ -222,16 +215,14 @@ describe('Semantic rules: verdicts, batching and rule filter', () => { it('--rule-filter narrows evaluation: semantic-only runs no expression rules, expression-only runs no judge', async () => { const semanticOnly = await validateSpace(context, { - semantic: true, - judgeProvider: new FakeJudgeProvider({ pass: false }), + judge: new FakeJudgeProvider({ pass: false }), ruleFilter: { mode: ['semantic'] }, }); expect(semanticOnly.ruleViolations.every((v) => v.ruleId === 'opportunity-customer-voice')).toBe(true); const fake = new FakeJudgeProvider(); const expressionOnly = await validateSpace(context, { - semantic: true, - judgeProvider: fake, + judge: fake, ruleFilter: { mode: ['expression'] }, }); expect(fake.callHistory).toHaveLength(0); From 828c04d0f01a276d92d61dc0831b9447b8adc40f Mon Sep 17 00:00:00 2001 From: Roger Barnes Date: Thu, 15 Oct 2026 15:16:09 +1100 Subject: [PATCH 16/22] feat(judge): judge shorthand by bundled type; docs pass A judge names either a bundled provider type ({ "type": "jev" }) or an entry in modelProviders ({ "provider": "..." }); a modelProviders entry wins over a bundled type of the same name. No judge in config means no model runs, and a run still asks with --semantic or --judge. Docs: merge config.md's provider and judge sections, adopt a Documentation Principles section in AGENTS.md (one idea lives in one place; document what it does; structured-context in prose; front-load the happy path), and trim rules.md per review. --- AGENTS.md | 10 +++++++++- config.example.json | 5 +---- docs/concepts.md | 2 +- docs/config.md | 14 ++++++++------ docs/rules.md | 12 +++++------- docs/semantic-rules.md | 19 ++++++++----------- src/config.ts | 5 ++++- src/judge/factory.ts | 21 +++++++++++++-------- src/judge/types.ts | 2 ++ tests/judge/factory.test.ts | 19 +++++++++++++++++-- 10 files changed, 68 insertions(+), 41 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index da8eece..ab815a7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -42,6 +42,14 @@ There are several places that need reviewing and updating with any new feature o - **Where a setting lives:** Config says *what* to use, CLI flags override it for one run, and the environment holds secrets and the config location only (`apiKeyEnv` names a secret; `SCTX_CONFIG` locates the config). Do not add env vars that set config values, and never infer a choice (such as a judge provider) from a variable that happens to be set: a key present for another tool must not be spent silently. Config also never switches on costly or outward-facing behaviour (such as semantic judging): each run asks for it, through a flag or an API option. - **Failures are visible:** A rule or expression that cannot be evaluated is reported as a violation, never skipped or treated as a pass. Likewise a misconfiguration fails before any work or spend. +## Documentation Principles + +- **One idea lives in one place:** Decide which doc owns each concept, document it there once, and link instead of repeating. Small, non-obvious tidbits may be repeated; explanations may not. +- **Document what it does, not what it doesn't:** Describe behaviour and purpose. State a limit only when a new user would realistically assume otherwise. Specification and decision history do not leak into docs. +- **Proper name in docs:** `structured-context` is the project name for prose; `sctx` is only the CLI command in command examples. +- **Approachability:** Write short, declarative sentences in active voice, one idea per sentence — about 80% of the way to ASD-STE100. Introduce a concept once, before sections that need it. Frame options by purpose (what each is for, and for whom). +- **Front-load the happy path:** Quick starts give the fewest steps that run. Tuning, alternatives and edge cases come after. + ## Key Files - `src/api.ts` — Public library entry point (`structured-context/api`). Re-exports the supported programmatic surface. Keep CLI-only concerns out of it. Do not extend it without agreement (see **Public API surface** above). @@ -58,7 +66,7 @@ For most development only the main unit tests need re-running regularly. - `bun run test:hook` — unit test plugin hooks (`hook-test/unit/`) - hook development only - `bun run test:hook:e2e` — test plugin hooks in Claude Code (`hook-test/`) - hook development only - `bun run test:smoke` — smoke tests run against locally configured spaces - only use when changes could affect compatibility. -- `bun run test:judge` — live judge tests (`judge-test/`): real provider calls on a fixture space, for each provider whose API key is set (`OPENROUTER_API_KEY`, `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`). Costs money; only use when changing judges, prompts or semantic rule behaviour. +- `bun run test:judge` — live judge tests (`judge-test/`): real provider calls on a fixture space, for each provider whose API key is set. Costs money; use sparingly when changing judges, prompts or semantic rule behaviour. ## Debugging diff --git a/config.example.json b/config.example.json index ff78093..138c28c 100644 --- a/config.example.json +++ b/config.example.json @@ -18,8 +18,5 @@ } ], "schema": "/path/to/custom/schemas/my-general-schema.json", - "modelProviders": { - "jev": { "type": "jev" } - }, - "judge": { "provider": "jev" } + "judge": { "type": "jev" } } diff --git a/docs/concepts.md b/docs/concepts.md index 2584146..73d5021 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -155,7 +155,7 @@ See [docs/rules.md](rules.md) for the rules reference, including JSONata express A **semantic rule** (`mode: semantic`) asks a model to judge a node, for example whether an opportunity is a customer need. It has a `rubric` instead of a `check`. -A **judge** is the model that judges semantic rules: a decision model (Jev) or a chat model (Claude, OpenAI-compatible). For each rule, the judge gives one **verdict** from the rule's `verdicts`, for example `customer-need`, `vague` or `solution`. The rule's `severity` says which verdicts report a violation. Semantic rules cost money, so they run only when a run asks for them (`--semantic` or `--judge`). The judge is a configured **model provider**. See [docs/semantic-rules.md](semantic-rules.md). +A **judge** is the model that judges semantic rules: a decision model (Jev) or a chat model (Claude, OpenAI-compatible). For each rule, the judge gives one **verdict** from the rule's `verdicts`, for example `customer-need`, `vague` or `solution`. The rule's `severity` says which verdicts report a violation. Semantic rules cost money, so they run only when a run asks for them (`--semantic` or `--judge`). The judge is a configured **model provider**: either a bundled provider type or an entry in `modelProviders`. See [docs/semantic-rules.md](semantic-rules.md). ### Cascade gating diff --git a/docs/config.md b/docs/config.md index e0bde76..8e2a9fb 100644 --- a/docs/config.md +++ b/docs/config.md @@ -65,9 +65,9 @@ All plugin names must start with `sctx-` (the prefix is optional in config and n Plugins can also contribute schemas and partials via a `/schemas/` directory (or `"sctx": { "schemas": "..." }` in `package.json`). If a space specifies a plugin-prefixed schema (e.g. `"sctx-wardley-mapping/wardley_map.json"`), schemas are resolved statically without executing plugin code, and the plugin's code hooks are loaded when parsing or rendering. -## Model providers +## Model providers and judge -`modelProviders` names the models that sctx can use, and says how to reach each one. Set it at the top level only. +structured-context judges [semantic rules](semantic-rules.md) with a **model provider**: a named entry in `modelProviders` that says which model to use and how to reach it. Set `modelProviders` at the top level when the bundled defaults do not suit. ```json { @@ -87,14 +87,16 @@ Plugins can also contribute schemas and partials via a `/schemas/` d | `apiKeyEnv` | The environment variable that holds the key. Defaults: `OPENROUTER_API_KEY` (jev), `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`. A local `openai` endpoint can need no key. | | `apiKey` | The key itself. Use `apiKeyEnv` instead, so that keys stay out of config files. | -sctx creates a provider only when it uses it, so a provider that is not used needs no key. The environment holds only keys: no environment variable selects or configures a provider. +The environment holds only keys: no environment variable selects or configures a provider. -## Judge (semantic rules) - -`judge.provider` names the model provider that judges [semantic rules](semantic-rules.md). Set it at the top level. A space can set its own `judge`, which replaces the top-level one for that space, for example to keep a private space on a local model: +The top-level `judge` names the model provider that judges semantic rules: either an entry in `modelProviders` (`provider`) or a bundled provider type (`type`), which takes the type's defaults. A `modelProviders` entry wins over a bundled type of the same name. A space can set its own `judge`, which replaces the top-level one for that space, for example to keep a private space on a local model: ```json { + "modelProviders": { + "jev": { "type": "jev" }, + "local": { "type": "openai", "endpoint": "http://localhost:11434/v1" } + }, "judge": { "provider": "jev" }, "spaces": [ { "name": "private-notes", "path": "/path/to/notes", "judge": { "provider": "local" } } diff --git a/docs/rules.md b/docs/rules.md index bd9c1d9..0fdd11f 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -9,8 +9,6 @@ Rules check content after JSON Schema validation, for example cross-node checks - An **expression rule** (the default) is a JSONata `check` that must return `true`. - A **semantic rule** asks a model to judge a node. See [Semantic rules](semantic-rules.md). -This page applies to both kinds, unless it says otherwise. - For metadata structure and composition behavior, see [docs/schemas.md](schemas.md). ## Rule shape @@ -33,7 +31,7 @@ Rules are a flat array. |---|---|---| | `id` | yes | Unique rule identifier | | `category` | yes | `validation` \| `coherence` \| `workflow` \| `best-practice` | -| `severity` | no | `error` \| `warning` \| `info`; overrides the category default (see [Severity](#severity)). A semantic rule can map verdicts to severities | +| `severity` | no | `error` \| `warning` \| `info`; overrides the category default (see [Severity](#severity)) | | `description` | yes | Human-readable rule intent | | `check` | yes, unless `mode: semantic` | JSONata expression; must evaluate to `true` | | `when` | no | JSONata test. The rule runs only on nodes where it returns `true`. See [Conditional rules](#conditional-rules-when) | @@ -42,7 +40,7 @@ Rules are a flat array. | `type` | no | Restrict rule to nodes of this `resolvedType` | | `scope` | no | Use `"global"` to evaluate once for the whole space | | `override` | no | Only for merge conflicts: allows later duplicate `id` to replace earlier | -| `gatesDescendants` | no | Whether a violation gates downstream checks on the node's hierarchy subtree. Defaults to `true` for rules with `error` severity, `false` for `warning` and `info`. Set explicitly to override either way. | +| `gatesDescendants` | no | Whether a violation gates downstream checks on the node's hierarchy subtree. Defaults to `true` for `error` rules, `false` otherwise. | ## Categories and severity @@ -123,7 +121,7 @@ $count(nodes[resolvedParentTitle=$$.current.title and resolvedType='solution']) ## Conditional rules (`when`) -`when` is a JSONata test with the same context as `check`. sctx evaluates it on each node before the rule. The rule runs only where `when` returns `true`. On other nodes the rule does not run: it reports nothing and does not gate. Use `when` when `type` cannot select the nodes a rule applies to: +`when` is a JSONata test that determines whether a rule runs at all — effectively, under what circumstances the rule applies. It has the same context as `check` and runs on each node before the rule. Use `when` when `type` cannot select the nodes a rule applies to: ```json { "id": "ready-has-owner", "category": "workflow", "type": "task", @@ -133,7 +131,7 @@ $count(nodes[resolvedParentTitle=$$.current.title and resolvedType='solution']) `when` costs almost nothing, so use it to stop an expensive rule from running where it does not apply. For a semantic rule, this saves a model call on each node that `when` excludes. -If sctx cannot evaluate a `when` or a `check` (for example, because of a syntax error), it reports an `error` violation on the node, with `Rule error:` in its description. This violation does not gate descendants. A rule with `scope: "global"` cannot have `when`. +An error in a `when` or `check` is reported as a non-gating `error` violation. A rule with `scope: "global"` cannot have `when`. ## Reusing rules @@ -220,7 +218,7 @@ the next). Excluded nodes are reported under `skippedByDepth`, separate from gat - A semantic rule with a severity map matches each severity that it can report. - Gating comes only from the rules that ran. -For example, `--rule-filter mode=expression` skips semantic rules, and `--rule-filter category=workflow,coherence --rule-filter severity=error` runs only workflow and coherence rules that can report an error. +For example, `--rule-filter category=workflow,coherence --rule-filter severity=error` runs only workflow and coherence rules that can report an error. ## Common patterns diff --git a/docs/semantic-rules.md b/docs/semantic-rules.md index 7736088..8f5c3f7 100644 --- a/docs/semantic-rules.md +++ b/docs/semantic-rules.md @@ -7,16 +7,13 @@ This page assumes you know [Rules](rules.md). Semantic rules use the same fields ## Quick start 1. Set `OPENROUTER_API_KEY` in your environment. -2. Add a model provider and choose it as the judge in `config.json`: +2. Choose Jev as the judge in `config.json`. Jev is a decision model and the recommended judge (see [Choosing a judge](#choosing-a-judge)): ```json - { - "modelProviders": { "jev": { "type": "jev" } }, - "judge": { "provider": "jev" } - } + { "judge": { "type": "jev" } } ``` - Jev is a decision model and the recommended judge (see [Choosing a judge](#choosing-a-judge)). To use Claude instead, add a provider with `"type": "anthropic"` and set `ANTHROPIC_API_KEY`. + To use Claude instead, set `"type": "anthropic"` and `ANTHROPIC_API_KEY`. 3. Add a rule to the schema's `$metadata.rules`: ```json @@ -58,7 +55,7 @@ For each node, the judge gives exactly one verdict for each rule. The verdict mu Each verdict has a purpose for one audience: - The **description** tells the judge when to choose that verdict. -- The **guidance** (optional) tells the user what to do when the judge chooses that verdict. sctx shows it with the violation. sctx does not send it to the judge. +- The **guidance** (optional) tells the user what to do when the judge chooses that verdict. structured-context shows it with the violation and does not send it to the judge. A verdict is a description string, or an object with `description` and `guidance`: @@ -88,9 +85,9 @@ Gating uses the severity of the verdict. A verdict with `error` severity gates t ## How judging works -- **Order.** On each node, expression rules run first. Then sctx sends all of the node's semantic rules to the judge in one call. -- **Gating.** sctx does not judge a gated node. [Coverage output](rules.md#coverage-output) shows which nodes were gated. -- **Judge errors.** If the judge fails (for example, a network error or output that sctx cannot read), sctx reports a violation. A judge error does not gate. +- **Order.** On each node, expression rules run first. Then structured-context sends all of the node's semantic rules to the judge in one call. +- **Gating.** structured-context does not judge a gated node. [Coverage output](rules.md#coverage-output) shows which nodes were gated. +- **Judge errors.** If the judge fails (for example, a network error or output that structured-context cannot read), structured-context reports a violation. A judge error does not gate. - **Input.** The judge gets the node's title, type, parent titles and schema data, including frontmatter. - **Output.** A violation from a semantic rule also includes the verdict, the judge's evidence and any guidance. @@ -106,7 +103,7 @@ Use a decision model when you can. Its results are repeatable and cheap, and the | `anthropic` | Claude. Default model: `claude-haiku-5-5` | `ANTHROPIC_API_KEY` | | `openai` | Any OpenAI-compatible chat endpoint, for example OpenAI, Ollama or vLLM. Set `endpoint` for servers other than OpenAI. Local servers do not need a key | `OPENAI_API_KEY` | -sctx uses only the provider that you configure. It does not choose a provider from the keys in your environment. See [Model providers](config.md#model-providers) and [Judge](config.md#judge-semantic-rules) for all fields. +structured-context uses only the provider that you configure. It does not choose a provider from the keys in your environment. Each provider type above works as a judge directly (`{ "judge": { "type": "jev" } }`); see [Model providers and judge](config.md#model-providers-and-judge) to override models, endpoints or keys. A rule does not choose its judge: that is configuration, not schema. To use a different provider for one run, name it with `--judge`. For example, to get a second opinion from a chat model on one rule and one file: diff --git a/src/config.ts b/src/config.ts index c50cb16..33f9a88 100644 --- a/src/config.ts +++ b/src/config.ts @@ -28,7 +28,10 @@ const MODEL_PROVIDER_SCHEMA = { const JUDGE_CONFIG_SCHEMA = { type: 'object', - properties: { provider: { type: 'string', minLength: 1 } }, + properties: { + provider: { type: 'string', minLength: 1 }, + type: { type: 'string', enum: MODEL_PROVIDER_TYPES }, + }, additionalProperties: false, } as const; diff --git a/src/judge/factory.ts b/src/judge/factory.ts index 442b650..6f63a67 100644 --- a/src/judge/factory.ts +++ b/src/judge/factory.ts @@ -69,26 +69,31 @@ export function judgeForRun( /** * The judge for a run: the model provider named by `name` (the `--judge` override), else the space's - * `judge.provider`, else the top-level one. Throws {@link JudgeConfigError} when it cannot be built. + * `judge`, else the top-level one. A judge names either an entry in `modelProviders` (`provider`) or a + * bundled provider type (`type`), which takes the type's defaults. An entry in `modelProviders` wins over + * a bundled type of the same name. Throws {@link JudgeConfigError} when it cannot be built. */ export function createJudge( - config: { modelProviders?: Record; judge?: { provider?: string } }, - space: { judge?: { provider?: string } } = {}, + config: { modelProviders?: Record; judge?: { provider?: string; type?: string } }, + space: { judge?: { provider?: string; type?: string } } = {}, name?: string, env: Env = process.env, ): SemanticJudgeProvider { - const chosen = name ?? space.judge?.provider ?? config.judge?.provider; + const chosen = name ?? space.judge?.provider ?? space.judge?.type ?? config.judge?.provider ?? config.judge?.type; const known = Object.keys(config.modelProviders ?? {}); if (!chosen) { throw new JudgeConfigError( - 'No judge configured. Set "judge.provider" in config.json to the name of an entry in "modelProviders".', + 'No judge configured. Set "judge" in config.json to a bundled type (e.g. { "type": "jev" }) or to the name of an entry in "modelProviders".', ); } const def = config.modelProviders?.[chosen]; if (!def) { - throw new JudgeConfigError( - `Model provider "${chosen}" is not configured.${known.length ? ` Configured: ${known.join(', ')}.` : ' Add it under "modelProviders" in config.json.'}`, - ); + if (!(MODEL_PROVIDER_TYPES as readonly string[]).includes(chosen)) { + throw new JudgeConfigError( + `Judge "${chosen}" is neither a bundled type (${MODEL_PROVIDER_TYPES.join(', ')}) nor a configured model provider.${known.length ? ` Configured: ${known.join(', ')}.` : ''}`, + ); + } + return createModelProvider(chosen, { type: chosen as ModelProviderConfig['type'] }, env); } return createModelProvider(chosen, def, env); } diff --git a/src/judge/types.ts b/src/judge/types.ts index a190a4b..d503490 100644 --- a/src/judge/types.ts +++ b/src/judge/types.ts @@ -55,4 +55,6 @@ export interface ModelProviderConfig { export interface JudgeConfig { /** Name of an entry in `modelProviders`. */ provider?: string; + /** A bundled provider type, used with its defaults when no `modelProviders` entry of that name exists. */ + type?: string; } diff --git a/tests/judge/factory.test.ts b/tests/judge/factory.test.ts index f651aa6..79a29dc 100644 --- a/tests/judge/factory.test.ts +++ b/tests/judge/factory.test.ts @@ -56,6 +56,20 @@ describe('createJudge', () => { expect(createJudge(config, space, 'claude', env).name).toBe('anthropic'); }); + it('accepts a bundled type as the judge, with the type defaults', () => { + expect(createJudge({}, { judge: { type: 'jev' } }, undefined, env).name).toBe('jev'); + expect(createJudge({ judge: { type: 'anthropic' } }, {}, undefined, env).name).toBe('anthropic'); + expect(createJudge({}, {}, 'fake', env) instanceof FakeJudgeProvider).toBe(true); + }); + + it('prefers a configured model provider over a bundled type of the same name', () => { + const overridden = { + modelProviders: { fake: { type: 'anthropic' } } as const, + judge: { type: 'fake' }, + }; + expect(createJudge(overridden, {}, undefined, env).name).toBe('anthropic'); + }); + it('never infers a provider from API keys in the environment', () => { const keys = { ANTHROPIC_API_KEY: 'a', OPENAI_API_KEY: 'o', OPENROUTER_API_KEY: 'r' }; expect(() => createJudge({}, {}, undefined, keys)).toThrow(JudgeConfigError); @@ -64,7 +78,8 @@ describe('createJudge', () => { ); }); - it('rejects a provider name that is not configured, listing what is', () => { + it('rejects a judge that is neither a bundled type nor a configured provider', () => { + expect(() => createJudge(config, {}, 'nope', env)).toThrow('neither a bundled type'); expect(() => createJudge(config, {}, 'nope', env)).toThrow('Configured: jev, claude'); }); @@ -85,6 +100,6 @@ describe('judgeForRun (CLI)', () => { it('fails when asked and no judge is configured, or the named provider is missing', () => { expect(() => judgeForRun({}, {}, { semantic: true })).toThrow('No judge configured'); - expect(() => judgeForRun(config, {}, { judge: 'nope' })).toThrow('Model provider "nope" is not configured'); + expect(() => judgeForRun(config, {}, { judge: 'nope' })).toThrow('Judge "nope" is neither a bundled type'); }); }); From 6f83b03b0607d0092929ca66655c5b0e8d3351c1 Mon Sep 17 00:00:00 2001 From: Roger Barnes Date: Thu, 15 Oct 2026 16:59:27 +1100 Subject: [PATCH 17/22] refactor(judge): bundled providers, not judge.type; live tests inline judge.provider names a provider: a modelProviders entry first, else a bundled provider (jev, anthropic, openai). A user entry replaces a bundled one of the same name, keeping type defaults for fields it leaves out. The fake provider stays test-only. The judge-test fixture space is gone: the live provider test now judges one inline node directly (SCTX_LIVE_JUDGE_TESTS=1, per-provider key check), and pipeline coverage stays with the existing fake-judge tests. --- config.example.json | 2 +- docs/config.md | 4 +- docs/semantic-rules.md | 6 +- judge-test/fixtures/schema.json | 52 ------------- .../space/Build an AI receipt scanner.md | 5 -- .../fixtures/space/Faster tax filing.md | 4 - .../I cannot find my receipts at tax time.md | 5 -- .../fixtures/space/Receipt inbox by email.md | 5 -- .../space/Use a third-party OCR API.md | 5 -- judge-test/judges.test.ts | 73 ------------------- package.json | 2 +- src/config.ts | 5 +- src/judge/factory.ts | 34 +++++---- src/judge/types.ts | 4 +- tests/judge/factory.test.ts | 30 ++++---- tests/judge/live.test.ts | 40 ++++++++++ 16 files changed, 85 insertions(+), 191 deletions(-) delete mode 100644 judge-test/fixtures/schema.json delete mode 100644 judge-test/fixtures/space/Build an AI receipt scanner.md delete mode 100644 judge-test/fixtures/space/Faster tax filing.md delete mode 100644 judge-test/fixtures/space/I cannot find my receipts at tax time.md delete mode 100644 judge-test/fixtures/space/Receipt inbox by email.md delete mode 100644 judge-test/fixtures/space/Use a third-party OCR API.md delete mode 100644 judge-test/judges.test.ts create mode 100644 tests/judge/live.test.ts diff --git a/config.example.json b/config.example.json index 138c28c..ee902fb 100644 --- a/config.example.json +++ b/config.example.json @@ -18,5 +18,5 @@ } ], "schema": "/path/to/custom/schemas/my-general-schema.json", - "judge": { "type": "jev" } + "judge": { "provider": "jev" } } diff --git a/docs/config.md b/docs/config.md index 8e2a9fb..c03da51 100644 --- a/docs/config.md +++ b/docs/config.md @@ -67,7 +67,7 @@ Plugins can also contribute schemas and partials via a `/schemas/` d ## Model providers and judge -structured-context judges [semantic rules](semantic-rules.md) with a **model provider**: a named entry in `modelProviders` that says which model to use and how to reach it. Set `modelProviders` at the top level when the bundled defaults do not suit. +structured-context judges [semantic rules](semantic-rules.md) with a **model provider**: a named definition that says which model to use and how to reach it. The bundled providers `jev`, `anthropic` and `openai` take their type defaults; an entry in `modelProviders` replaces a bundled provider of the same name, and fields it leaves out still take the defaults. Set `modelProviders` at the top level when the bundled defaults do not suit. ```json { @@ -89,7 +89,7 @@ structured-context judges [semantic rules](semantic-rules.md) with a **model pro The environment holds only keys: no environment variable selects or configures a provider. -The top-level `judge` names the model provider that judges semantic rules: either an entry in `modelProviders` (`provider`) or a bundled provider type (`type`), which takes the type's defaults. A `modelProviders` entry wins over a bundled type of the same name. A space can set its own `judge`, which replaces the top-level one for that space, for example to keep a private space on a local model: +The top-level `judge` names the model provider that judges semantic rules: a bundled provider or an entry in `modelProviders`. A space can set its own `judge`, which replaces the top-level one for that space, for example to keep a private space on a local model: ```json { diff --git a/docs/semantic-rules.md b/docs/semantic-rules.md index 8f5c3f7..dc6e889 100644 --- a/docs/semantic-rules.md +++ b/docs/semantic-rules.md @@ -10,10 +10,10 @@ This page assumes you know [Rules](rules.md). Semantic rules use the same fields 2. Choose Jev as the judge in `config.json`. Jev is a decision model and the recommended judge (see [Choosing a judge](#choosing-a-judge)): ```json - { "judge": { "type": "jev" } } + { "judge": { "provider": "jev" } } ``` - To use Claude instead, set `"type": "anthropic"` and `ANTHROPIC_API_KEY`. + `jev`, `anthropic` and `openai` are bundled providers. To use Claude instead, set `"provider": "anthropic"` and `ANTHROPIC_API_KEY`. 3. Add a rule to the schema's `$metadata.rules`: ```json @@ -103,7 +103,7 @@ Use a decision model when you can. Its results are repeatable and cheap, and the | `anthropic` | Claude. Default model: `claude-haiku-5-5` | `ANTHROPIC_API_KEY` | | `openai` | Any OpenAI-compatible chat endpoint, for example OpenAI, Ollama or vLLM. Set `endpoint` for servers other than OpenAI. Local servers do not need a key | `OPENAI_API_KEY` | -structured-context uses only the provider that you configure. It does not choose a provider from the keys in your environment. Each provider type above works as a judge directly (`{ "judge": { "type": "jev" } }`); see [Model providers and judge](config.md#model-providers-and-judge) to override models, endpoints or keys. +structured-context uses only the provider that you configure. It does not choose a provider from the keys in your environment. Each provider type above is also a bundled provider, usable as a judge by name (`{ "judge": { "provider": "jev" } }`). To override models, endpoints or keys, add an entry of the same name to `modelProviders` (see [Model providers and judge](config.md#model-providers-and-judge)). A rule does not choose its judge: that is configuration, not schema. To use a different provider for one run, name it with `--judge`. For example, to get a second opinion from a chat model on one rule and one file: diff --git a/judge-test/fixtures/schema.json b/judge-test/fixtures/schema.json deleted file mode 100644 index a12843d..0000000 --- a/judge-test/fixtures/schema.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "$schema": "https://raw.githubusercontent.com/mindsocket/structured-context/main/schemas/generated/latest/_structured_context_schema_meta.json", - "$id": "sctx://judge-test", - "$metadata": { - "hierarchy": { "levels": ["outcome", "opportunity", "solution"] }, - "rules": [ - { - "id": "opportunity-framing", - "category": "coherence", - "mode": "semantic", - "type": "opportunity", - "description": "The opportunity is a customer need or pain point, not a solution or a business goal", - "verdicts": { - "customer-need": "A need or pain point, in terms of what the customer experiences", - "solution": { - "description": "A feature, product or solution the business could build", - "guidance": "Restate it as the customer problem this solution would solve." - } - }, - "severity": { "solution": "error" } - }, - { - "id": "solution-addresses-parent", - "category": "best-practice", - "mode": "semantic", - "type": "solution", - "description": "The solution plausibly addresses its parent opportunity" - } - ] - }, - "oneOf": [ - { - "type": "object", - "properties": { "type": { "const": "outcome" }, "title": { "type": "string" } }, - "required": ["type", "title"] - }, - { - "type": "object", - "properties": { - "type": { "const": "opportunity" }, - "title": { "type": "string" }, - "parent": { "type": "string" } - }, - "required": ["type", "title"] - }, - { - "type": "object", - "properties": { "type": { "const": "solution" }, "title": { "type": "string" }, "parent": { "type": "string" } }, - "required": ["type", "title"] - } - ] -} diff --git a/judge-test/fixtures/space/Build an AI receipt scanner.md b/judge-test/fixtures/space/Build an AI receipt scanner.md deleted file mode 100644 index 09c381c..0000000 --- a/judge-test/fixtures/space/Build an AI receipt scanner.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -type: opportunity -parent: "[[Faster tax filing]]" ---- -Add an AI-powered receipt scanner to the mobile app with OCR and automatic categorisation. diff --git a/judge-test/fixtures/space/Faster tax filing.md b/judge-test/fixtures/space/Faster tax filing.md deleted file mode 100644 index 8b92684..0000000 --- a/judge-test/fixtures/space/Faster tax filing.md +++ /dev/null @@ -1,4 +0,0 @@ ---- -type: outcome ---- -Customers file their tax return in under an hour. diff --git a/judge-test/fixtures/space/I cannot find my receipts at tax time.md b/judge-test/fixtures/space/I cannot find my receipts at tax time.md deleted file mode 100644 index 820fe85..0000000 --- a/judge-test/fixtures/space/I cannot find my receipts at tax time.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -type: opportunity -parent: "[[Faster tax filing]]" ---- -In interviews, customers said they spend hours at tax time hunting through email and drawers for receipts, and often give up on deductions they are owed. diff --git a/judge-test/fixtures/space/Receipt inbox by email.md b/judge-test/fixtures/space/Receipt inbox by email.md deleted file mode 100644 index ad6d40c..0000000 --- a/judge-test/fixtures/space/Receipt inbox by email.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -type: solution -parent: "[[I cannot find my receipts at tax time]]" ---- -Forward receipts to a personal email address; we store and tag them by tax category. diff --git a/judge-test/fixtures/space/Use a third-party OCR API.md b/judge-test/fixtures/space/Use a third-party OCR API.md deleted file mode 100644 index c5dcec8..0000000 --- a/judge-test/fixtures/space/Use a third-party OCR API.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -type: solution -parent: "[[Build an AI receipt scanner]]" ---- -Use a third-party OCR API. diff --git a/judge-test/judges.test.ts b/judge-test/judges.test.ts deleted file mode 100644 index cd3e821..0000000 --- a/judge-test/judges.test.ts +++ /dev/null @@ -1,73 +0,0 @@ -/** - * Live judge tests: call real providers on a small fixture space. They cost money, so they run only with - * SCTX_LIVE_JUDGE_TESTS=1 (`bun run test:judge`), and each provider only when its API key is set. - */ -import { describe, expect, it } from 'bun:test'; -import { join } from 'node:path'; - -const ROOT = join(import.meta.dir, '..'); -const CONFIG = join(import.meta.dir, 'fixtures', 'config.json'); -const LIVE = process.env.SCTX_LIVE_JUDGE_TESTS === '1'; - -const PROVIDERS = [ - { name: 'jev', keyEnv: 'OPENROUTER_API_KEY' }, - { name: 'anthropic', keyEnv: 'ANTHROPIC_API_KEY' }, - { name: 'openai', keyEnv: 'OPENAI_API_KEY' }, -]; - -const SOLUTION = 'Build an AI receipt scanner.md'; -const GATED = 'Use a third-party OCR API.md'; - -type Output = { - ruleViolations: Array<{ - file: string; - ruleId: string; - severity: string; - description: string; - verdict?: string; - guidance?: string; - }>; - coverageGates: Array<{ file: string; ruleId?: string; gatedFiles: string[] }>; -}; - -function validate(...args: string[]): { exitCode: number; output: Output } { - const result = Bun.spawnSync(['bun', 'run', 'src/index.ts', 'validate', 'judge-test', '--json', ...args], { - cwd: ROOT, - env: { ...process.env, SCTX_CONFIG: CONFIG }, - stdout: 'pipe', - stderr: 'pipe', - }); - const stdout = new TextDecoder().decode(result.stdout); - if (!stdout.trim().startsWith('{')) throw new Error(new TextDecoder().decode(result.stderr) || stdout); - return { exitCode: result.exitCode, output: JSON.parse(stdout) }; -} - -describe.skipIf(!LIVE)('live judges', () => { - for (const { name, keyEnv } of PROVIDERS) { - it.skipIf(!process.env[keyEnv])( - `${name}: classifies clear-cut opportunities and gates below an error verdict`, - () => { - const { exitCode, output } = validate('--judge', name); - - // No judge errors, and the only violation is the solution in disguise. - expect(output.ruleViolations.filter((v) => v.description.includes('Judge error'))).toEqual([]); - expect(output.ruleViolations).toHaveLength(1); - const violation = output.ruleViolations[0]; - expect(violation).toMatchObject({ - file: SOLUTION, - ruleId: 'opportunity-framing', - severity: 'error', - verdict: 'solution', - }); - expect(violation?.guidance).toContain('customer problem'); - expect(exitCode).toBe(1); - - // The error verdict gates the solution beneath it, so that solution is not judged. - expect(output.coverageGates).toEqual([ - expect.objectContaining({ file: SOLUTION, ruleId: 'opportunity-framing', gatedFiles: [GATED] }), - ]); - }, - 60_000, - ); - } -}); diff --git a/package.json b/package.json index bce31bd..43be01e 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,7 @@ "diagram": "bun run src/index.ts diagram", "test": "bun test tests/", "test:smoke": "bun test smoke/", - "test:judge": "SCTX_LIVE_JUDGE_TESTS=1 bun test judge-test/", + "test:judge": "SCTX_LIVE_JUDGE_TESTS=1 bun test tests/judge/live.test.ts", "test:hook": "bun test hook-test/unit/", "test:hook:e2e": "bun test hook-test/hooks.test.ts", "test:all": "bun test", diff --git a/src/config.ts b/src/config.ts index 33f9a88..c50cb16 100644 --- a/src/config.ts +++ b/src/config.ts @@ -28,10 +28,7 @@ const MODEL_PROVIDER_SCHEMA = { const JUDGE_CONFIG_SCHEMA = { type: 'object', - properties: { - provider: { type: 'string', minLength: 1 }, - type: { type: 'string', enum: MODEL_PROVIDER_TYPES }, - }, + properties: { provider: { type: 'string', minLength: 1 } }, additionalProperties: false, } as const; diff --git a/src/judge/factory.ts b/src/judge/factory.ts index 6f63a67..62ef7ab 100644 --- a/src/judge/factory.ts +++ b/src/judge/factory.ts @@ -13,6 +13,13 @@ const DEFAULT_KEY_ENV = { jev: 'OPENROUTER_API_KEY', } as const; +/** Bundled model providers: usable as a judge by name, without a `modelProviders` entry. */ +const BUNDLED_PROVIDERS: Record = { + jev: { type: 'jev' }, + anthropic: { type: 'anthropic' }, + openai: { type: 'openai' }, +} as const; + /** The judge is misconfigured (no provider, unknown provider, missing key). Reported as a plain CLI error. */ export class JudgeConfigError extends Error {} @@ -69,31 +76,28 @@ export function judgeForRun( /** * The judge for a run: the model provider named by `name` (the `--judge` override), else the space's - * `judge`, else the top-level one. A judge names either an entry in `modelProviders` (`provider`) or a - * bundled provider type (`type`), which takes the type's defaults. An entry in `modelProviders` wins over - * a bundled type of the same name. Throws {@link JudgeConfigError} when it cannot be built. + * `judge.provider`, else the top-level one. The name resolves against `modelProviders` first, then the + * bundled providers. A user entry replaces a bundled provider of the same name; fields it leaves out + * still take the type defaults. Throws {@link JudgeConfigError} when it cannot be built. */ export function createJudge( - config: { modelProviders?: Record; judge?: { provider?: string; type?: string } }, - space: { judge?: { provider?: string; type?: string } } = {}, + config: { modelProviders?: Record; judge?: { provider?: string } }, + space: { judge?: { provider?: string } } = {}, name?: string, env: Env = process.env, ): SemanticJudgeProvider { - const chosen = name ?? space.judge?.provider ?? space.judge?.type ?? config.judge?.provider ?? config.judge?.type; - const known = Object.keys(config.modelProviders ?? {}); + const chosen = name ?? space.judge?.provider ?? config.judge?.provider; if (!chosen) { throw new JudgeConfigError( - 'No judge configured. Set "judge" in config.json to a bundled type (e.g. { "type": "jev" }) or to the name of an entry in "modelProviders".', + 'No judge configured. Set "judge.provider" in config.json to a bundled provider (e.g. "jev") or to the name of an entry in "modelProviders".', ); } - const def = config.modelProviders?.[chosen]; + const def = config.modelProviders?.[chosen] ?? BUNDLED_PROVIDERS[chosen]; if (!def) { - if (!(MODEL_PROVIDER_TYPES as readonly string[]).includes(chosen)) { - throw new JudgeConfigError( - `Judge "${chosen}" is neither a bundled type (${MODEL_PROVIDER_TYPES.join(', ')}) nor a configured model provider.${known.length ? ` Configured: ${known.join(', ')}.` : ''}`, - ); - } - return createModelProvider(chosen, { type: chosen as ModelProviderConfig['type'] }, env); + const known = [...Object.keys(config.modelProviders ?? {}), ...Object.keys(BUNDLED_PROVIDERS)]; + throw new JudgeConfigError( + `Model provider "${chosen}" is not configured.${known.length ? ` Known providers: ${known.join(', ')}.` : ''}`, + ); } return createModelProvider(chosen, def, env); } diff --git a/src/judge/types.ts b/src/judge/types.ts index d503490..f7b8fac 100644 --- a/src/judge/types.ts +++ b/src/judge/types.ts @@ -53,8 +53,6 @@ export interface ModelProviderConfig { /** Which model provider judges semantic rules: config `judge`, at the top level or on a space. */ export interface JudgeConfig { - /** Name of an entry in `modelProviders`. */ + /** Name of an entry in `modelProviders`, or a bundled provider ("jev", "anthropic", "openai"). */ provider?: string; - /** A bundled provider type, used with its defaults when no `modelProviders` entry of that name exists. */ - type?: string; } diff --git a/tests/judge/factory.test.ts b/tests/judge/factory.test.ts index 79a29dc..04fc38d 100644 --- a/tests/judge/factory.test.ts +++ b/tests/judge/factory.test.ts @@ -56,18 +56,22 @@ describe('createJudge', () => { expect(createJudge(config, space, 'claude', env).name).toBe('anthropic'); }); - it('accepts a bundled type as the judge, with the type defaults', () => { - expect(createJudge({}, { judge: { type: 'jev' } }, undefined, env).name).toBe('jev'); - expect(createJudge({ judge: { type: 'anthropic' } }, {}, undefined, env).name).toBe('anthropic'); - expect(createJudge({}, {}, 'fake', env) instanceof FakeJudgeProvider).toBe(true); + it('accepts a bundled provider as the judge, with the type defaults', () => { + expect(createJudge({}, { judge: { provider: 'jev' } }, undefined, env).name).toBe('jev'); + expect(createJudge({ judge: { provider: 'anthropic' } }, {}, undefined, env).name).toBe('anthropic'); }); - it('prefers a configured model provider over a bundled type of the same name', () => { - const overridden = { - modelProviders: { fake: { type: 'anthropic' } } as const, - judge: { type: 'fake' }, + it('does not bundle the test-only fake provider', () => { + expect(() => createJudge({}, { judge: { provider: 'fake' } }, undefined, env)).toThrow('not configured'); + }); + + it('lets a configured model provider replace a bundled provider of the same name, keeping type defaults for fields it leaves out', () => { + const augmented = { + modelProviders: { jev: { type: 'jev', apiKeyEnv: 'MY_JEV_KEY' } } as const, + judge: { provider: 'jev' }, }; - expect(createJudge(overridden, {}, undefined, env).name).toBe('anthropic'); + expect(() => createJudge(augmented, {}, undefined, { MY_JEV_KEY: 'k' })).not.toThrow(); + expect(() => createJudge(augmented, {}, undefined, { OPENROUTER_API_KEY: 'r' })).toThrow('MY_JEV_KEY'); }); it('never infers a provider from API keys in the environment', () => { @@ -78,9 +82,9 @@ describe('createJudge', () => { ); }); - it('rejects a judge that is neither a bundled type nor a configured provider', () => { - expect(() => createJudge(config, {}, 'nope', env)).toThrow('neither a bundled type'); - expect(() => createJudge(config, {}, 'nope', env)).toThrow('Configured: jev, claude'); + it('rejects a judge that is neither a bundled provider nor a configured one', () => { + expect(() => createJudge(config, {}, 'nope', env)).toThrow('not configured'); + expect(() => createJudge(config, {}, 'nope', env)).toThrow('Known providers: jev, claude'); }); it('fails when the chosen provider has no key', () => { @@ -100,6 +104,6 @@ describe('judgeForRun (CLI)', () => { it('fails when asked and no judge is configured, or the named provider is missing', () => { expect(() => judgeForRun({}, {}, { semantic: true })).toThrow('No judge configured'); - expect(() => judgeForRun(config, {}, { judge: 'nope' })).toThrow('Judge "nope" is neither a bundled type'); + expect(() => judgeForRun(config, {}, { judge: 'nope' })).toThrow('Model provider "nope" is not configured'); }); }); diff --git a/tests/judge/live.test.ts b/tests/judge/live.test.ts new file mode 100644 index 0000000..464a57c --- /dev/null +++ b/tests/judge/live.test.ts @@ -0,0 +1,40 @@ +/** + * Live judge tests: call real providers on one inline node. They cost money, so they run only with + * SCTX_LIVE_JUDGE_TESTS=1 (`bun run test:judge`), and each provider only when its API key is set. + */ +import { describe, expect, it } from 'bun:test'; +import { createModelProvider } from '../../src/judge/factory'; +import type { SemanticJudgeInput, SemanticRuleSpec } from '../../src/judge/types'; +import { node, passFailRule } from './helpers'; + +const CASES = [ + { type: 'jev' as const, keyEnv: 'OPENROUTER_API_KEY' }, + { type: 'anthropic' as const, keyEnv: 'ANTHROPIC_API_KEY' }, + { type: 'openai' as const, keyEnv: 'OPENAI_API_KEY' }, +]; + +describe.skipIf(process.env.SCTX_LIVE_JUDGE_TESTS !== '1')('live judges', () => { + for (const { type, keyEnv } of CASES) { + it.skipIf(!process.env[keyEnv])( + `${type}: returns a verdict from the rule's closed set`, + async () => { + const provider = createModelProvider(type, { type }, { [keyEnv]: process.env[keyEnv] }); + const rule: SemanticRuleSpec = { + ...passFailRule, + description: 'The node title names a customer need or pain point', + }; + const input: SemanticJudgeInput = { node, rules: [rule] }; + const results = await provider.evaluate(input); + const result = results[rule.id]; + expect(result).toBeDefined(); + if (!result || 'error' in result) { + expect(result).not.toHaveProperty('error'); + return; + } + expect(result.verdict).toBeOneOf(['pass', 'fail']); + expect(typeof result.evidence).toBe('string'); + }, + 60_000, + ); + } +}); From 7f8f330937a44acbda59cc6c91029f040c1e71fc Mon Sep 17 00:00:00 2001 From: Roger Barnes Date: Thu, 15 Oct 2026 18:59:03 +1100 Subject: [PATCH 18/22] refactor(judge): provider.api with vendor bundles; sctx providers Provider definitions name a wire API (provider.api: openai-completions, anthropic-messages, openrouter-decisions, typesafe-systemone) instead of a model-flavoured type. The two decisions APIs share one implementation. Hydration happens in the factory, not the providers: explicit value > named env var (set-but-empty is an error) > API default. Provider implementations read no environment; only the key has a per-API default variable. The bundled catalogue now carries vendor bundles: openrouter-jev (typesafe/jev-latest), typesafe-jev, anthropic, anthropic-haiku-5-5, anthropic-env, openai, openai-env, openrouter. New sctx providers lists bundled and configured providers with their effective config, keys never shown. --- AGENTS.md | 4 +- config.example.json | 2 +- docs/concepts.md | 2 +- docs/config.md | 41 ++++--- docs/semantic-rules.md | 19 +-- src/commands/providers.ts | 65 ++++++++++ src/config.ts | 8 +- src/index.ts | 13 ++ src/judge/factory.ts | 136 ++++++++++++++------- src/judge/judge-node.ts | 4 + src/judge/provider/anthropic.ts | 20 +-- src/judge/provider/jev.ts | 18 ++- src/judge/provider/openai.ts | 19 +-- src/judge/types.ts | 37 ++++-- tests/commands/validate-judge-flag.test.ts | 2 +- tests/config.test.ts | 5 +- tests/judge/factory.test.ts | 117 ++++++++++++------ tests/judge/judge-node.test.ts | 38 ++++++ tests/judge/live.test.ts | 12 +- tests/judge/provider/jev.test.ts | 25 ++-- 20 files changed, 429 insertions(+), 158 deletions(-) create mode 100644 src/commands/providers.ts create mode 100644 tests/judge/judge-node.test.ts diff --git a/AGENTS.md b/AGENTS.md index ab815a7..6fb80e9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -57,7 +57,7 @@ There are several places that need reviewing and updating with any new feature o - `src/schema/metadata-contract.ts` — Single source of truth for the `$metadata` contract - `schemas/generated/latest/_structured_context_schema_meta.json` — Generated metaschema for the newest dialect (generated on build or with `bun run generate:schema-meta`). `schemas/generated//` folders are frozen copies, created by the `version` script (`generate:schema-meta --release`) only when a release changes the dialect; never edit them. `src/schema/meta-schemas.ts` loads them at runtime. - `src/schema/gating.ts` — Cascade gating: computes hierarchy depth and evaluates rules with subtree gating (schema/hierarchy/blocking-rule failures suppress downstream rule checks). See [docs/rules.md](docs/rules.md#cascade-gating). -- `src/judge/` — Semantic rule judges: provider interface and types (`types.ts`), verdict resolution (`verdicts.ts`), per-node dispatch (`judge-node.ts`), and providers (`provider/anthropic.ts`, `provider/openai.ts`, `provider/jev.ts`, `provider/fake.ts`). See [docs/semantic-rules.md](docs/semantic-rules.md). +- `src/judge/` — Semantic rule judges: provider interface and types (`types.ts`), verdict resolution (`verdicts.ts`), per-node dispatch (`judge-node.ts`), and providers (`provider/anthropic.ts`, `provider/openai.ts`, `provider/jev.ts`, `provider/fake.ts`), provider hydration and the bundled catalogue (`factory.ts`), and `sctx providers` (`src/commands/providers.ts`). See [docs/semantic-rules.md](docs/semantic-rules.md). - `src/schema/rule-filter.ts` — `--rule-filter` parsing and rule filtering by mode, category, severity, type or id. ## Testing @@ -66,7 +66,7 @@ For most development only the main unit tests need re-running regularly. - `bun run test:hook` — unit test plugin hooks (`hook-test/unit/`) - hook development only - `bun run test:hook:e2e` — test plugin hooks in Claude Code (`hook-test/`) - hook development only - `bun run test:smoke` — smoke tests run against locally configured spaces - only use when changes could affect compatibility. -- `bun run test:judge` — live judge tests (`judge-test/`): real provider calls on a fixture space, for each provider whose API key is set. Costs money; use sparingly when changing judges, prompts or semantic rule behaviour. +- `bun run test:judge` — live judge tests (`tests/judge/live.test.ts`): one real provider call per provider whose API key is set. Costs money; use sparingly when changing judges, prompts or semantic rule behaviour. ## Debugging diff --git a/config.example.json b/config.example.json index ee902fb..10b0d94 100644 --- a/config.example.json +++ b/config.example.json @@ -18,5 +18,5 @@ } ], "schema": "/path/to/custom/schemas/my-general-schema.json", - "judge": { "provider": "jev" } + "judge": { "provider": "openrouter-jev" } } diff --git a/docs/concepts.md b/docs/concepts.md index 73d5021..04e584b 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -155,7 +155,7 @@ See [docs/rules.md](rules.md) for the rules reference, including JSONata express A **semantic rule** (`mode: semantic`) asks a model to judge a node, for example whether an opportunity is a customer need. It has a `rubric` instead of a `check`. -A **judge** is the model that judges semantic rules: a decision model (Jev) or a chat model (Claude, OpenAI-compatible). For each rule, the judge gives one **verdict** from the rule's `verdicts`, for example `customer-need`, `vague` or `solution`. The rule's `severity` says which verdicts report a violation. Semantic rules cost money, so they run only when a run asks for them (`--semantic` or `--judge`). The judge is a configured **model provider**: either a bundled provider type or an entry in `modelProviders`. See [docs/semantic-rules.md](semantic-rules.md). +A **judge** is the model that judges semantic rules: a decision model (Jev) or a chat model (Claude, OpenAI-compatible). For each rule, the judge gives one **verdict** from the rule's `verdicts`, for example `customer-need`, `vague` or `solution`. The rule's `severity` says which verdicts report a violation. Semantic rules cost money, so they run only when a run asks for them (`--semantic` or `--judge`). The judge is a configured **model provider**: a bundled provider or an entry in `modelProviders`. See [docs/semantic-rules.md](semantic-rules.md). ### Cascade gating diff --git a/docs/config.md b/docs/config.md index c03da51..a4d6460 100644 --- a/docs/config.md +++ b/docs/config.md @@ -67,37 +67,50 @@ Plugins can also contribute schemas and partials via a `/schemas/` d ## Model providers and judge -structured-context judges [semantic rules](semantic-rules.md) with a **model provider**: a named definition that says which model to use and how to reach it. The bundled providers `jev`, `anthropic` and `openai` take their type defaults; an entry in `modelProviders` replaces a bundled provider of the same name, and fields it leaves out still take the defaults. Set `modelProviders` at the top level when the bundled defaults do not suit. +structured-context judges [semantic rules](semantic-rules.md) with a **model provider**: a named definition that says which wire API to speak, which model to use and where. A provider definition combines an `api`, explicit values or environment-variable names, and the key source. `sctx providers` lists every usable provider with its effective config (keys are never shown). + +**Bundled providers** work without any configuration. Each combines an API, an endpoint, a key source and an opinionated model: + +| Provider | API | Model | Key variable | +|---|---|---|---| +| `openrouter-jev` | `openrouter-decisions` | `typesafe/jev-latest` | `OPENROUTER_API_KEY` | +| `typesafe-jev` | `typesafe-systemone` | `jev-latest` | `TYPESAFE_API_KEY` | +| `anthropic` | `anthropic-messages` | `claude-haiku-5-5` | `ANTHROPIC_API_KEY` | +| `openai` | `openai-completions` | `gpt-4o-mini` | `OPENAI_API_KEY` | +| `openrouter` | `openai-completions` | `openai/gpt-4o-mini` | `OPENROUTER_API_KEY` | + +The `-env` variants (`openai-env`, `anthropic-env`) also read the vendor's usual model and endpoint variables (`OPENAI_MODEL`, `OPENAI_BASE_URL`, `ANTHROPIC_MODEL`), falling back to the bundle's defaults where a variable is unset. Bundled providers without `-env` read no environment beyond the key: they behave the same on every machine. `anthropic-haiku-5-5` pins the model, so it does not drift when the `anthropic` bundle's default changes. + +**Configured providers** live in `modelProviders` at the top level, and replace a bundled provider of the same name. Fields a configured provider leaves out still take the API defaults: ```json { "modelProviders": { - "jev": { "type": "jev" }, - "claude": { "type": "anthropic", "model": "claude-haiku-5-5" }, - "local": { "type": "openai", "endpoint": "http://localhost:11434/v1" } + "mine": { "api": "openrouter-decisions", "apiKeyEnv": "MY_JEV_KEY" }, + "local": { "api": "openai-completions", "endpoint": "http://localhost:11434/v1" } } } ``` | Field | Description | |---|---| -| `type` | **Required.** `jev`, `anthropic` or `openai` (see [Choosing a judge](semantic-rules.md#choosing-a-judge)), or `fake` (for tests). | -| `model` | Defaults: `typesafe/jev-1.13` (jev), `claude-haiku-5-5` (anthropic), `gpt-4o-mini` (openai). | -| `endpoint` | Override the API URL. Jev defaults to `https://openrouter.ai/api/alpha/decisions`. | -| `apiKeyEnv` | The environment variable that holds the key. Defaults: `OPENROUTER_API_KEY` (jev), `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`. A local `openai` endpoint can need no key. | +| `api` | **Required.** The wire API: `openrouter-decisions`, `typesafe-systemone`, `anthropic-messages` or `openai-completions` (see [Choosing a judge](semantic-rules.md#choosing-a-judge)), or `fake` (for tests). | +| `model` | A fixed model. Its default depends on the API. | +| `modelEnv` | Environment variable holding a model override. An explicit `model` wins; a variable set but empty is an error. | +| `endpoint` | A fixed API URL. | +| `endpointEnv` | Environment variable holding an endpoint override. Same precedence as `modelEnv`. | +| `apiKeyEnv` | The environment variable that holds the key. Defaults to the API's usual variable (see the table above). A local `openai-completions` endpoint can need no key. | | `apiKey` | The key itself. Use `apiKeyEnv` instead, so that keys stay out of config files. | -The environment holds only keys: no environment variable selects or configures a provider. +Resolution per field: the explicit value wins, then the named environment variable, then the API default. Provider implementations read no environment; all hydration happens before one runs. + +The environment holds only keys and the variables a `-env` bundle names. No environment variable selects a provider. The top-level `judge` names the model provider that judges semantic rules: a bundled provider or an entry in `modelProviders`. A space can set its own `judge`, which replaces the top-level one for that space, for example to keep a private space on a local model: ```json { - "modelProviders": { - "jev": { "type": "jev" }, - "local": { "type": "openai", "endpoint": "http://localhost:11434/v1" } - }, - "judge": { "provider": "jev" }, + "judge": { "provider": "openrouter-jev" }, "spaces": [ { "name": "private-notes", "path": "/path/to/notes", "judge": { "provider": "local" } } ] diff --git a/docs/semantic-rules.md b/docs/semantic-rules.md index dc6e889..d6e067f 100644 --- a/docs/semantic-rules.md +++ b/docs/semantic-rules.md @@ -10,10 +10,10 @@ This page assumes you know [Rules](rules.md). Semantic rules use the same fields 2. Choose Jev as the judge in `config.json`. Jev is a decision model and the recommended judge (see [Choosing a judge](#choosing-a-judge)): ```json - { "judge": { "provider": "jev" } } + { "judge": { "provider": "openrouter-jev" } } ``` - `jev`, `anthropic` and `openai` are bundled providers. To use Claude instead, set `"provider": "anthropic"` and `ANTHROPIC_API_KEY`. + `openrouter-jev` is a bundled provider. To use Claude instead, set `"provider": "anthropic"` and `ANTHROPIC_API_KEY`. 3. Add a rule to the schema's `$metadata.rules`: ```json @@ -97,13 +97,16 @@ Gating uses the severity of the verdict. A verdict with `error` severity gates t Use a decision model when you can. Its results are repeatable and cheap, and they include a confidence. Use a chat model when the rubric needs open reasoning or a written explanation. -| Provider type | Model | Default key variable | -|---|---|---| -| `jev` | TypeSafe's Jev decision model, through OpenRouter. One request for each node | `OPENROUTER_API_KEY` | -| `anthropic` | Claude. Default model: `claude-haiku-5-5` | `ANTHROPIC_API_KEY` | -| `openai` | Any OpenAI-compatible chat endpoint, for example OpenAI, Ollama or vLLM. Set `endpoint` for servers other than OpenAI. Local servers do not need a key | `OPENAI_API_KEY` | +| Bundled provider | API | Model | Key variable | +|---|---|---|---| +| `openrouter-jev` | `openrouter-decisions` | `typesafe/jev-latest` | `OPENROUTER_API_KEY` | +| `typesafe-jev` | `typesafe-systemone` | `jev-latest` | `TYPESAFE_API_KEY` | +| `anthropic` | `anthropic-messages` | `claude-haiku-5-5` | `ANTHROPIC_API_KEY` | +| `openai` | `openai-completions` | `gpt-4o-mini` | `OPENAI_API_KEY` | + +The `openai-completions` API speaks to any OpenAI-compatible endpoint, for example Ollama or vLLM; local servers do not need a key. `sctx providers` lists every bundled and configured provider with its effective config. -structured-context uses only the provider that you configure. It does not choose a provider from the keys in your environment. Each provider type above is also a bundled provider, usable as a judge by name (`{ "judge": { "provider": "jev" } }`). To override models, endpoints or keys, add an entry of the same name to `modelProviders` (see [Model providers and judge](config.md#model-providers-and-judge)). +structured-context uses only the provider that you configure. It does not choose a provider from the keys in your environment. To override models, endpoints or keys, add an entry of the same name to `modelProviders`, or define a new one (see [Model providers and judge](config.md#model-providers-and-judge)). A rule does not choose its judge: that is configuration, not schema. To use a different provider for one run, name it with `--judge`. For example, to get a second opinion from a chat model on one rule and one file: diff --git a/src/commands/providers.ts b/src/commands/providers.ts new file mode 100644 index 0000000..ae92030 --- /dev/null +++ b/src/commands/providers.ts @@ -0,0 +1,65 @@ +import { loadConfig } from '../config'; +import { BUNDLED_PROVIDERS, hydrateProvider } from '../judge/factory'; +import type { ModelProviderConfig } from '../judge/types'; + +interface ProviderRow { + name: string; + source: 'bundled' | 'config'; + api: string; + model: string | undefined; + endpoint: string | undefined; + keyEnv: string | undefined; + /** Whether the provider would resolve with a key right now. */ + valid: boolean; + error?: string; +} + +function row(name: string, def: ModelProviderConfig, source: 'bundled' | 'config'): ProviderRow { + try { + const h = hydrateProvider(name, def); + const needsKey = h.api !== 'openai-completions' && h.api !== 'fake'; + return { + name, + source, + api: h.api, + model: h.model, + endpoint: h.endpoint, + keyEnv: h.keyEnv, + valid: !needsKey || h.apiKey !== undefined, + error: needsKey && h.apiKey === undefined ? `no API key; set ${h.keyEnv}` : undefined, + }; + } catch (err) { + return { + name, + source, + api: def.api, + model: def.model, + endpoint: def.endpoint, + keyEnv: def.apiKeyEnv, + valid: false, + error: err instanceof Error ? err.message : String(err), + }; + } +} + +/** List bundled and configured model providers with their effective config; keys are never shown. */ +export function listProviders(options: { json?: boolean } = {}): void { + const config = loadConfig(); + const configured = config.modelProviders ?? {}; + const rows: ProviderRow[] = [ + ...Object.entries(configured).map(([name, def]) => row(name, def, 'config')), + ...Object.entries(BUNDLED_PROVIDERS) + .filter(([name]) => !(name in configured)) + .map(([name, def]) => row(name, def, 'bundled')), + ]; + if (options.json) { + console.log(JSON.stringify(rows, null, 2)); + return; + } + for (const r of rows) { + const origin = r.source === 'bundled' ? '' : ' (config)'; + const model = r.model ?? '(api default)'; + const key = r.valid ? `key via ${r.keyEnv}` : (r.error ?? 'no key'); + console.log(`${r.name}${origin} ${r.api} model=${model} ${key}`); + } +} diff --git a/src/config.ts b/src/config.ts index c50cb16..7f90c84 100644 --- a/src/config.ts +++ b/src/config.ts @@ -4,7 +4,7 @@ import { dirname, isAbsolute, join, resolve } from 'node:path'; import Ajv2020 from 'ajv/dist/2020'; import JSON5 from 'json5'; import { ENV_CONFIG_VAR, XDG_CONFIG_DIR } from './constants'; -import { type JudgeConfig, MODEL_PROVIDER_TYPES, type ModelProviderConfig } from './judge/types'; +import { type JudgeConfig, MODEL_PROVIDER_APIS, type ModelProviderConfig } from './judge/types'; import { normalizePluginName, resolvePluginSchemasDir, @@ -16,13 +16,15 @@ import { bundledSchemasDir } from './schema/schema'; const MODEL_PROVIDER_SCHEMA = { type: 'object', properties: { - type: { type: 'string', enum: MODEL_PROVIDER_TYPES }, + api: { type: 'string', enum: MODEL_PROVIDER_APIS }, model: { type: 'string' }, + modelEnv: { type: 'string' }, endpoint: { type: 'string' }, + endpointEnv: { type: 'string' }, apiKey: { type: 'string' }, apiKeyEnv: { type: 'string' }, }, - required: ['type'], + required: ['api'], additionalProperties: false, } as const; diff --git a/src/index.ts b/src/index.ts index 2dd3b06..3c69d75 100755 --- a/src/index.ts +++ b/src/index.ts @@ -191,6 +191,19 @@ spacesCmd .action((options) => listSpaces(options)); program.addCommand(spacesCmd); +const providersCmd = new Command('providers').description( + 'List model providers (bundled and configured) with their effective config', +); +providersCmd + .command('list', { isDefault: true }) + .description('List bundled and configured model providers; keys are never shown') + .option('--json', 'Output as JSON') + .action(async (options) => { + const { listProviders } = await import('./commands/providers'); + listProviders(options); + }); +program.addCommand(providersCmd); + const schemasCmd = new Command('schemas').alias('schema').description('List and inspect schemas'); schemasCmd .command('list', { isDefault: true }) diff --git a/src/judge/factory.ts b/src/judge/factory.ts index 62ef7ab..7f09721 100644 --- a/src/judge/factory.ts +++ b/src/judge/factory.ts @@ -1,63 +1,117 @@ +import type { AnthropicJudgeProviderOptions } from './provider/anthropic'; import { AnthropicJudgeProvider } from './provider/anthropic'; import { FakeJudgeProvider } from './provider/fake'; +import type { JevJudgeProviderOptions } from './provider/jev'; import { JevJudgeProvider } from './provider/jev'; import { OpenAIJudgeProvider } from './provider/openai'; -import { MODEL_PROVIDER_TYPES, type ModelProviderConfig, type SemanticJudgeProvider } from './types'; +import { + type HydratedProvider, + MODEL_PROVIDER_APIS, + type ModelProviderConfig, + type SemanticJudgeProvider, +} from './types'; type Env = Record; -/** Environment variable each provider type reads its API key from unless `apiKeyEnv` says otherwise. */ -const DEFAULT_KEY_ENV = { - anthropic: 'ANTHROPIC_API_KEY', - openai: 'OPENAI_API_KEY', - jev: 'OPENROUTER_API_KEY', +/** + * Bundled providers: usable as a judge by name, without a `modelProviders` entry. A user entry of the + * same name replaces one wholesale; fields it leaves out take the type defaults in the constructors. + * Every value here is explicit: no bundle reads the environment. + */ +export const BUNDLED_PROVIDERS: Record = { + // Jev decision model. The TypeSafe and OpenRouter decision endpoints share a wire shape, so both + // speak a decisions API; the vendors' endpoints stay distinct bundles. + 'typesafe-jev': { api: 'typesafe-systemone', model: 'jev-latest', apiKeyEnv: 'TYPESAFE_API_KEY' }, + 'openrouter-jev': { api: 'openrouter-decisions', model: 'typesafe/jev-latest', apiKeyEnv: 'OPENROUTER_API_KEY' }, + // Chat models. `` is a fixed opinionated bundle; `-env` hydrates model and endpoint + // from the vendor's usual environment variables, falling back to the bundle's defaults. + anthropic: { api: 'anthropic-messages', model: 'claude-haiku-5-5', apiKeyEnv: 'ANTHROPIC_API_KEY' }, + 'anthropic-haiku-5-5': { api: 'anthropic-messages', model: 'claude-haiku-5-5', apiKeyEnv: 'ANTHROPIC_API_KEY' }, + 'anthropic-env': { + api: 'anthropic-messages', + model: 'claude-haiku-5-5', + modelEnv: 'ANTHROPIC_MODEL', + apiKeyEnv: 'ANTHROPIC_API_KEY', + }, + openai: { api: 'openai-completions', model: 'gpt-4o-mini', apiKeyEnv: 'OPENAI_API_KEY' }, + 'openai-env': { + api: 'openai-completions', + model: 'gpt-4o-mini', + modelEnv: 'OPENAI_MODEL', + endpointEnv: 'OPENAI_BASE_URL', + apiKeyEnv: 'OPENAI_API_KEY', + }, + openrouter: { api: 'openai-completions', model: 'openai/gpt-4o-mini', apiKeyEnv: 'OPENROUTER_API_KEY' }, } as const; -/** Bundled model providers: usable as a judge by name, without a `modelProviders` entry. */ -const BUNDLED_PROVIDERS: Record = { - jev: { type: 'jev' }, - anthropic: { type: 'anthropic' }, - openai: { type: 'openai' }, +/** Each API reads its key from this variable unless `apiKeyEnv` names another one. Model and endpoint never read env vars. */ +const DEFAULT_KEY_ENV: Record = { + 'anthropic-messages': 'ANTHROPIC_API_KEY', + 'openai-completions': 'OPENAI_API_KEY', + 'openrouter-decisions': 'OPENROUTER_API_KEY', + 'typesafe-systemone': 'TYPESAFE_API_KEY', } as const; /** The judge is misconfigured (no provider, unknown provider, missing key). Reported as a plain CLI error. */ export class JudgeConfigError extends Error {} /** - * Instantiate one provider from its definition. The type is always explicit and is never inferred from API keys - * in the environment, so a key set for another tool is never spent on judging. Once the type is chosen, its key - * comes from `apiKey`, else the environment variable named by `apiKeyEnv` (default: the type's usual variable). + * Hydrate a provider definition into concrete values, in one place. Per field: the explicit value + * wins, then the named environment variable if it is set (a set-but-empty variable is an error, not + * a silent fall-through), then the definition's own default. The key is special: it must resolve. + */ +export function hydrateProvider(name: string, def: ModelProviderConfig, env: Env = process.env): HydratedProvider { + if (!(MODEL_PROVIDER_APIS as readonly string[]).includes(def.api)) { + throw new JudgeConfigError( + `Model provider "${name}" has unknown api "${def.api}". Use one of: ${MODEL_PROVIDER_APIS.join(', ')}.`, + ); + } + const fromEnv = (value: string | undefined, envName: string | undefined): string | undefined => { + if (value !== undefined) return value; + if (!envName) return undefined; + const envValue = env[envName]; + if (envValue !== undefined && envValue.trim() === '') { + throw new JudgeConfigError( + `Model provider "${name}" has ${envName}=${envName} set but empty; set a value or unset the variable.`, + ); + } + return envValue; + }; + const model = fromEnv(def.model, def.modelEnv); + const endpoint = fromEnv(def.endpoint, def.endpointEnv); + const keyEnv = def.apiKeyEnv ?? DEFAULT_KEY_ENV[def.api]; + const apiKey = def.apiKey ?? (keyEnv ? env[keyEnv] : undefined); + return { api: def.api, model, endpoint, apiKey, keyEnv }; +} + +/** + * Instantiate one provider from a definition. The API is always explicit and is never inferred from + * API keys in the environment, so a key set for another tool is never spent on judging. Constructors + * apply their own API defaults for fields left undefined after hydration. */ export function createModelProvider( name: string, def: ModelProviderConfig, env: Env = process.env, ): SemanticJudgeProvider { - const type = def.type; - if (!(MODEL_PROVIDER_TYPES as readonly string[]).includes(type)) { + const h = hydrateProvider(name, def, env); + // OpenAI-compatible endpoints (Ollama, vLLM, ...) may need no key; the others must resolve one. + // A key never borrows another provider's variable: only the named apiKeyEnv (or literal apiKey) counts. + if (!h.apiKey && h.api !== 'openai-completions' && h.api !== 'fake') { throw new JudgeConfigError( - `Model provider "${name}" has unknown type "${type}". Use one of: ${MODEL_PROVIDER_TYPES.join(', ')}.`, + `Model provider "${name}" (${h.api}) has no API key. Set ${h.keyEnv} in the environment, or "apiKey" / "apiKeyEnv" on the provider.`, ); } - if (type === 'fake') return new FakeJudgeProvider(); - - const keyEnv = def.apiKeyEnv ?? DEFAULT_KEY_ENV[type as keyof typeof DEFAULT_KEY_ENV]; - const apiKey = def.apiKey || env[keyEnv]; - const { model, endpoint } = def; - - // OpenAI-compatible endpoints (Ollama, vLLM, ...) may need no key. - if (!apiKey && type !== 'openai') { - throw new JudgeConfigError( - `Model provider "${name}" (${type}) has no API key. Set ${keyEnv} in the environment, or "apiKey" / "apiKeyEnv" on the provider.`, - ); - } - switch (type) { - case 'anthropic': - return new AnthropicJudgeProvider({ apiKey: apiKey as string, model, endpoint }); - case 'jev': - return new JevJudgeProvider({ apiKey: apiKey as string, model, endpoint }); - default: - return new OpenAIJudgeProvider({ apiKey, model, endpoint }); + switch (h.api) { + case 'fake': + return new FakeJudgeProvider(); + case 'anthropic-messages': + return new AnthropicJudgeProvider(h as AnthropicJudgeProviderOptions); + case 'openrouter-decisions': + case 'typesafe-systemone': + return new JevJudgeProvider(h as JevJudgeProviderOptions); + case 'openai-completions': + return new OpenAIJudgeProvider(h); } } @@ -76,9 +130,8 @@ export function judgeForRun( /** * The judge for a run: the model provider named by `name` (the `--judge` override), else the space's - * `judge.provider`, else the top-level one. The name resolves against `modelProviders` first, then the - * bundled providers. A user entry replaces a bundled provider of the same name; fields it leaves out - * still take the type defaults. Throws {@link JudgeConfigError} when it cannot be built. + * `judge.provider`, else the top-level one. The name resolves against `modelProviders` first, then + * the bundled providers. Throws {@link JudgeConfigError} when it cannot be built. */ export function createJudge( config: { modelProviders?: Record; judge?: { provider?: string } }, @@ -89,12 +142,13 @@ export function createJudge( const chosen = name ?? space.judge?.provider ?? config.judge?.provider; if (!chosen) { throw new JudgeConfigError( - 'No judge configured. Set "judge.provider" in config.json to a bundled provider (e.g. "jev") or to the name of an entry in "modelProviders".', + 'No judge configured. Set "judge.provider" in config.json to a bundled provider (e.g. "openrouter-jev") or to the name of an entry in "modelProviders".', ); } const def = config.modelProviders?.[chosen] ?? BUNDLED_PROVIDERS[chosen]; if (!def) { - const known = [...Object.keys(config.modelProviders ?? {}), ...Object.keys(BUNDLED_PROVIDERS)]; + const known = [...Object.keys(BUNDLED_PROVIDERS), ...Object.keys(config.modelProviders ?? {})]; + throw new JudgeConfigError( `Model provider "${chosen}" is not configured.${known.length ? ` Known providers: ${known.join(', ')}.` : ''}`, ); diff --git a/src/judge/judge-node.ts b/src/judge/judge-node.ts index 4e7ff51..7ab7edb 100644 --- a/src/judge/judge-node.ts +++ b/src/judge/judge-node.ts @@ -48,6 +48,10 @@ export async function judgeNode( return { rule, failed: true, evidence: `Error during evaluation: ${message}`, error: message }; } const severity = resolved[i]!.severities[result.verdict]; + if (severity === undefined && !(result.verdict in resolved[i]!.spec.verdicts)) { + const message = `Judge returned unknown verdict "${result.verdict}" (allowed: ${Object.keys(resolved[i]!.spec.verdicts).join(', ')})`; + return { rule, failed: true, evidence: `Error during evaluation: ${message}`, error: message }; + } const failed = severity !== undefined; return { rule, diff --git a/src/judge/provider/anthropic.ts b/src/judge/provider/anthropic.ts index 1c3778b..c895323 100644 --- a/src/judge/provider/anthropic.ts +++ b/src/judge/provider/anthropic.ts @@ -1,16 +1,16 @@ import { buildJudgeSystemPrompt, buildJudgeUserPrompt, parseJudgeResponse } from '../prompt'; -import type { SemanticJudgeInput, SemanticJudgeResult, SemanticRuleSpec } from '../types'; +import type { HydratedProvider, SemanticJudgeInput, SemanticJudgeResult, SemanticRuleSpec } from '../types'; import { PerRuleJudgeProvider } from './per-rule'; -export interface AnthropicJudgeProviderOptions { - apiKey: string; - model?: string; - endpoint?: string; +export interface AnthropicJudgeProviderOptions extends HydratedProvider { + /** The factory requires a resolved key before constructing a non-OpenAI provider. */ + apiKey?: string; } -export const DEFAULT_ANTHROPIC_JUDGE_MODEL = 'claude-haiku-5-5'; +export const DEFAULT_ANTHROPIC_MODEL = 'claude-haiku-5-5'; +export const ANTHROPIC_MESSAGES_ENDPOINT = 'https://api.anthropic.com/v1/messages'; -/** Built-in judge provider for the Anthropic Messages API. */ +/** Judge provider for the Anthropic Messages API. Reads no environment: it takes hydrated values. */ export class AnthropicJudgeProvider extends PerRuleJudgeProvider { readonly name = 'anthropic'; private apiKey: string; @@ -19,9 +19,9 @@ export class AnthropicJudgeProvider extends PerRuleJudgeProvider { constructor(options: AnthropicJudgeProviderOptions) { super(); - this.apiKey = options.apiKey; - this.model = options.model || process.env.ANTHROPIC_MODEL || DEFAULT_ANTHROPIC_JUDGE_MODEL; - this.endpoint = options.endpoint || 'https://api.anthropic.com/v1/messages'; + this.apiKey = options.apiKey ?? ''; + this.model = options.model || DEFAULT_ANTHROPIC_MODEL; + this.endpoint = options.endpoint || ANTHROPIC_MESSAGES_ENDPOINT; } protected async judgeRule(node: SemanticJudgeInput['node'], rule: SemanticRuleSpec): Promise { diff --git a/src/judge/provider/jev.ts b/src/judge/provider/jev.ts index 1af476c..f2f214c 100644 --- a/src/judge/provider/jev.ts +++ b/src/judge/provider/jev.ts @@ -1,13 +1,11 @@ -import type { SemanticJudgeInput, SemanticJudgeProvider, SemanticJudgeResult } from '../types'; +import type { HydratedProvider, SemanticJudgeInput, SemanticJudgeProvider, SemanticJudgeResult } from '../types'; -export interface JevJudgeProviderOptions { - apiKey: string; - model?: string; - endpoint?: string; +export interface JevJudgeProviderOptions extends HydratedProvider { + /** The factory requires a resolved key before constructing a non-OpenAI provider. */ + apiKey?: string; } -export const DEFAULT_JEV_MODEL = 'typesafe/jev-1.13'; -export const DEFAULT_JEV_ENDPOINT = 'https://openrouter.ai/api/alpha/decisions'; +export const OPENROUTER_DECISIONS_ENDPOINT = 'https://openrouter.ai/api/alpha/decisions'; type JevChoiceAnswer = { type?: string; @@ -30,9 +28,9 @@ export class JevJudgeProvider implements SemanticJudgeProvider { private endpoint: string; constructor(options: JevJudgeProviderOptions) { - this.apiKey = options.apiKey; - this.model = options.model || DEFAULT_JEV_MODEL; - this.endpoint = options.endpoint || DEFAULT_JEV_ENDPOINT; + this.apiKey = options.apiKey ?? ''; + this.model = options.model || 'typesafe/jev-latest'; + this.endpoint = options.endpoint || OPENROUTER_DECISIONS_ENDPOINT; } async evaluate(input: SemanticJudgeInput): Promise> { diff --git a/src/judge/provider/openai.ts b/src/judge/provider/openai.ts index fedd9f0..9795b6b 100644 --- a/src/judge/provider/openai.ts +++ b/src/judge/provider/openai.ts @@ -1,16 +1,18 @@ import { buildJudgeSystemPrompt, buildJudgeUserPrompt, parseJudgeResponse } from '../prompt'; -import type { SemanticJudgeInput, SemanticJudgeResult, SemanticRuleSpec } from '../types'; +import type { HydratedProvider, SemanticJudgeInput, SemanticJudgeResult, SemanticRuleSpec } from '../types'; import { PerRuleJudgeProvider } from './per-rule'; -export interface OpenAIJudgeProviderOptions { +export interface OpenAIJudgeProviderOptions extends HydratedProvider { + /** Optional: local OpenAI-compatible servers need no key. */ apiKey?: string; - model?: string; - endpoint?: string; } +export const DEFAULT_OPENAI_MODEL = 'gpt-4o-mini'; +export const OPENAI_COMPLETIONS_ENDPOINT = 'https://api.openai.com/v1'; + /** - * Built-in judge provider for OpenAI-compatible Chat Completions API. - * Supports OpenAI, Ollama, vLLM, Groq, OpenRouter, and other compatible endpoints. + * Judge provider for the OpenAI-compatible Chat Completions API (OpenAI, Ollama, vLLM, ...). + * Reads no environment: it takes hydrated values. */ export class OpenAIJudgeProvider extends PerRuleJudgeProvider { readonly name = 'openai'; @@ -21,10 +23,9 @@ export class OpenAIJudgeProvider extends PerRuleJudgeProvider { constructor(options: OpenAIJudgeProviderOptions) { super(); this.apiKey = options.apiKey; - this.model = options.model || process.env.OPENAI_MODEL || 'gpt-4o-mini'; + this.model = options.model || DEFAULT_OPENAI_MODEL; - const base = - options.endpoint || process.env.OPENAI_BASE_URL || process.env.OPENAI_ENDPOINT || 'https://api.openai.com/v1'; + const base = options.endpoint || OPENAI_COMPLETIONS_ENDPOINT; this.endpoint = base.endsWith('/chat/completions') ? base : `${base.replace(/\/+$/, '')}/chat/completions`; } diff --git a/src/judge/types.ts b/src/judge/types.ts index f7b8fac..d0dad0a 100644 --- a/src/judge/types.ts +++ b/src/judge/types.ts @@ -36,23 +36,46 @@ export interface SemanticJudgeProvider { evaluate(input: SemanticJudgeInput): Promise>; } -/** Model provider types. `fake` is for tests. */ -export const MODEL_PROVIDER_TYPES = ['anthropic', 'openai', 'jev', 'fake'] as const; -export type ModelProviderType = (typeof MODEL_PROVIDER_TYPES)[number]; +/** Wire APIs a provider can speak. Names the request shape, never the vendor or model. */ +export const MODEL_PROVIDER_APIS = [ + 'openai-completions', + 'anthropic-messages', + 'openrouter-decisions', + 'typesafe-systemone', + 'fake', +] as const; +export type ModelProviderApi = (typeof MODEL_PROVIDER_APIS)[number]; -/** How to reach a model: config `modelProviders.`. */ +/** How to reach a model: a bundled provider or config `modelProviders.`, before hydration. */ export interface ModelProviderConfig { - type: ModelProviderType; + /** The wire API to speak. */ + api: ModelProviderApi; + /** Fixed model override. Beats `modelEnv` and the bundled default. */ model?: string; + /** Environment variable that holds the model override. */ + modelEnv?: string; + /** Fixed endpoint override. Beats `endpointEnv` and the bundled default. */ endpoint?: string; + /** Environment variable that holds the endpoint override. */ + endpointEnv?: string; /** Prefer `apiKeyEnv`: keys do not belong in config files. */ apiKey?: string; - /** Environment variable holding the key. Defaults to the provider type's usual one (e.g. ANTHROPIC_API_KEY). */ + /** Environment variable holding the key. */ apiKeyEnv?: string; } +/** A provider after hydration: every field is a concrete value; the implementation reads no environment. */ +export interface HydratedProvider { + api: ModelProviderApi; + model?: string; + endpoint?: string; + apiKey?: string; + /** The environment variable the key was read from, when one was named or defaulted. */ + keyEnv?: string; +} + /** Which model provider judges semantic rules: config `judge`, at the top level or on a space. */ export interface JudgeConfig { - /** Name of an entry in `modelProviders`, or a bundled provider ("jev", "anthropic", "openai"). */ + /** Name of an entry in `modelProviders`, or a bundled provider. */ provider?: string; } diff --git a/tests/commands/validate-judge-flag.test.ts b/tests/commands/validate-judge-flag.test.ts index dbbdd2c..e88d6f6 100644 --- a/tests/commands/validate-judge-flag.test.ts +++ b/tests/commands/validate-judge-flag.test.ts @@ -6,7 +6,7 @@ import { makeTempSpace } from '../helpers/temp-space'; const ROOT = join(import.meta.dir, '..', '..'); const space = makeTempSpace( [{ id: 'step-fit', type: 'step', mode: 'semantic', category: 'coherence', description: 'Step fits its task' }], - { judging: { modelProviders: { f: { type: 'fake' } } } }, + { judging: { modelProviders: { f: { api: 'fake' } } } }, ); const configPath = join(space.dir, 'config.json'); writeFileSync(configPath, JSON.stringify(space.config)); diff --git a/tests/config.test.ts b/tests/config.test.ts index fa12b1a..96f1993 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -392,7 +392,10 @@ describe('loadConfig with includeSpacesFrom', () => { writeFileSync( configPath, JSON.stringify({ - modelProviders: { jev: { type: 'jev' }, claude: { type: 'anthropic', model: 'claude-haiku-5-5' } }, + modelProviders: { + jev: { api: 'openrouter-decisions' }, + claude: { api: 'anthropic-messages', model: 'claude-haiku-5-5' }, + }, judge: { provider: 'jev' }, spaces: [{ name: 's', path: '/p', judge: { provider: 'claude' } }], }), diff --git a/tests/judge/factory.test.ts b/tests/judge/factory.test.ts index 04fc38d..1af5444 100644 --- a/tests/judge/factory.test.ts +++ b/tests/judge/factory.test.ts @@ -1,77 +1,117 @@ import { describe, expect, it } from 'bun:test'; -import { createJudge, createModelProvider, JudgeConfigError, judgeForRun } from '../../src/judge/factory'; +import { + createJudge, + createModelProvider, + hydrateProvider, + JudgeConfigError, + judgeForRun, +} from '../../src/judge/factory'; import { AnthropicJudgeProvider } from '../../src/judge/provider/anthropic'; import { FakeJudgeProvider } from '../../src/judge/provider/fake'; import { JevJudgeProvider } from '../../src/judge/provider/jev'; import { OpenAIJudgeProvider } from '../../src/judge/provider/openai'; -describe('createModelProvider', () => { - it('creates FakeJudgeProvider for type "fake"', () => { - expect(createModelProvider('f', { type: 'fake' }, {}) instanceof FakeJudgeProvider).toBe(true); +describe('hydrateProvider', () => { + it('prefers the explicit value over the env var and leaves unset fields undefined', () => { + const env = { MY_MODEL: 'from-env', MY_URL: 'https://from-env' }; + const h = hydrateProvider( + 'p', + { api: 'openai-completions', model: 'explicit', modelEnv: 'MY_MODEL', endpointEnv: 'MY_URL' }, + env, + ); + expect(h).toMatchObject({ + api: 'openai-completions', + model: 'explicit', + endpoint: 'https://from-env', + apiKey: undefined, + }); + expect(hydrateProvider('p', { api: 'openai-completions' }, env).model).toBeUndefined(); + }); + + it('errors when a named env var is set but empty, instead of falling through silently', () => { + expect(() => hydrateProvider('p', { api: 'openai-completions', modelEnv: 'MY_MODEL' }, { MY_MODEL: ' ' })).toThrow( + 'MY_MODEL set but empty', + ); }); - it.each([ - ['anthropic', 'ANTHROPIC_API_KEY', AnthropicJudgeProvider], - ['openai', 'OPENAI_API_KEY', OpenAIJudgeProvider], - ['jev', 'OPENROUTER_API_KEY', JevJudgeProvider], - ] as const)('creates %s with its usual env key (%s)', (type, keyEnv, Class) => { - expect(createModelProvider('p', { type }, { [keyEnv]: 'k' }) instanceof Class).toBe(true); + it('reads the key from apiKeyEnv, or takes the literal apiKey', () => { + expect(hydrateProvider('p', { api: 'anthropic-messages', apiKeyEnv: 'MY_KEY' }, { MY_KEY: 'k' }).apiKey).toBe('k'); + expect(hydrateProvider('p', { api: 'anthropic-messages', apiKey: 'lit' }, {}).apiKey).toBe('lit'); + }); + + it('rejects an unknown api', () => { + expect(() => hydrateProvider('p', { api: 'gemini' as never }, {})).toThrow('unknown api "gemini"'); + }); +}); + +describe('createModelProvider', () => { + it('creates FakeJudgeProvider for api "fake"', () => { + expect(createModelProvider('f', { api: 'fake' }, {}) instanceof FakeJudgeProvider).toBe(true); }); - it('reads the key from the variable named by apiKeyEnv', () => { - const def = { type: 'jev', apiKeyEnv: 'MY_JEV_KEY' } as const; - expect(createModelProvider('p', def, { MY_JEV_KEY: 'k' }) instanceof JevJudgeProvider).toBe(true); - expect(() => createModelProvider('p', def, { OPENROUTER_API_KEY: 'k' })).toThrow('MY_JEV_KEY'); + it.each([ + ['anthropic-messages', 'ANTHROPIC_API_KEY', AnthropicJudgeProvider], + ['openai-completions', 'OPENAI_API_KEY', OpenAIJudgeProvider], + ['openrouter-decisions', 'OPENROUTER_API_KEY', JevJudgeProvider], + ['typesafe-systemone', 'TYPESAFE_API_KEY', JevJudgeProvider], + ] as const)('creates %s with its key from the given env (%s)', (api, keyEnv, Class) => { + expect(createModelProvider('p', { api }, { [keyEnv]: 'k' }) instanceof Class).toBe(true); }); it("does not borrow another provider's key", () => { - expect(() => createModelProvider('p', { type: 'anthropic' }, { OPENAI_API_KEY: 'o' })).toThrow('ANTHROPIC_API_KEY'); - expect(() => createModelProvider('p', { type: 'jev' }, { ANTHROPIC_API_KEY: 'a' })).toThrow('OPENROUTER_API_KEY'); + expect(() => createModelProvider('p', { api: 'anthropic-messages' }, { OPENAI_API_KEY: 'o' })).toThrow( + 'ANTHROPIC_API_KEY', + ); + expect(() => createModelProvider('p', { api: 'openrouter-decisions' }, { ANTHROPIC_API_KEY: 'a' })).toThrow( + 'OPENROUTER_API_KEY', + ); }); it('lets openai run without a key for local endpoints', () => { - const provider = createModelProvider('local', { type: 'openai', endpoint: 'http://localhost:11434/v1' }, {}); + const provider = createModelProvider( + 'local', + { api: 'openai-completions', endpoint: 'http://localhost:11434/v1' }, + {}, + ); expect(provider instanceof OpenAIJudgeProvider).toBe(true); }); - - it('rejects an unknown type', () => { - expect(() => createModelProvider('p', { type: 'gemini' as never }, {})).toThrow('unknown type "gemini"'); - }); }); describe('createJudge', () => { const config = { - modelProviders: { jev: { type: 'jev' }, claude: { type: 'anthropic' } } as const, + modelProviders: { mine: { api: 'openrouter-decisions' }, claude: { api: 'anthropic-messages' } } as const, judge: { provider: 'claude' }, }; - const env = { OPENROUTER_API_KEY: 'r', ANTHROPIC_API_KEY: 'a' }; + const env = { OPENROUTER_API_KEY: 'r', ANTHROPIC_API_KEY: 'a', TYPESAFE_API_KEY: 't' }; it('uses the top-level judge provider', () => { expect(createJudge(config, {}, undefined, env).name).toBe('anthropic'); }); it("uses the space's judge over the top-level one, and a run's over both", () => { - const space = { judge: { provider: 'jev' } }; + const space = { judge: { provider: 'mine' } }; expect(createJudge(config, space, undefined, env).name).toBe('jev'); expect(createJudge(config, space, 'claude', env).name).toBe('anthropic'); }); it('accepts a bundled provider as the judge, with the type defaults', () => { - expect(createJudge({}, { judge: { provider: 'jev' } }, undefined, env).name).toBe('jev'); + expect(createJudge({}, { judge: { provider: 'openrouter-jev' } }, undefined, env).name).toBe('jev'); expect(createJudge({ judge: { provider: 'anthropic' } }, {}, undefined, env).name).toBe('anthropic'); }); - it('does not bundle the test-only fake provider', () => { - expect(() => createJudge({}, { judge: { provider: 'fake' } }, undefined, env)).toThrow('not configured'); + it('ships the pinned and env variants of each vendor bundle', () => { + expect(createJudge({}, { judge: { provider: 'anthropic-haiku-5-5' } }, undefined, env).name).toBe('anthropic'); + expect(createJudge({}, { judge: { provider: 'anthropic-env' } }, undefined, env).name).toBe('anthropic'); + expect(createJudge({}, { judge: { provider: 'openai-env' } }, undefined, env).name).toBe('openai'); + expect(createJudge({}, { judge: { provider: 'typesafe-jev' } }, undefined, env).name).toBe('jev'); }); it('lets a configured model provider replace a bundled provider of the same name, keeping type defaults for fields it leaves out', () => { const augmented = { - modelProviders: { jev: { type: 'jev', apiKeyEnv: 'MY_JEV_KEY' } } as const, - judge: { provider: 'jev' }, + modelProviders: { anthropic: { api: 'anthropic-messages', model: 'claude-sonnet-5-5' } } as const, + judge: { provider: 'anthropic' }, }; - expect(() => createJudge(augmented, {}, undefined, { MY_JEV_KEY: 'k' })).not.toThrow(); - expect(() => createJudge(augmented, {}, undefined, { OPENROUTER_API_KEY: 'r' })).toThrow('MY_JEV_KEY'); + expect(createJudge(augmented, {}, undefined, env).name).toBe('anthropic'); }); it('never infers a provider from API keys in the environment', () => { @@ -84,22 +124,29 @@ describe('createJudge', () => { it('rejects a judge that is neither a bundled provider nor a configured one', () => { expect(() => createJudge(config, {}, 'nope', env)).toThrow('not configured'); - expect(() => createJudge(config, {}, 'nope', env)).toThrow('Known providers: jev, claude'); + expect(() => createJudge(config, {}, 'nope', env)).toThrow('Known providers:'); }); it('fails when the chosen provider has no key', () => { expect(() => createJudge(config, {}, undefined, { OPENROUTER_API_KEY: 'r' })).toThrow('ANTHROPIC_API_KEY'); }); + + it('does not bundle the test-only fake provider', () => { + expect(() => createJudge({}, { judge: { provider: 'fake' } }, undefined, env)).toThrow('not configured'); + }); }); describe('judgeForRun (CLI)', () => { - const config = { modelProviders: { f: { type: 'fake' }, g: { type: 'fake' } } as const, judge: { provider: 'f' } }; + const config = { + modelProviders: { f: { api: 'fake' }, g: { api: 'fake' } } as const, + judge: { provider: 'f' }, + }; it('gives no judge unless the run asks with --semantic or --judge', () => { expect(judgeForRun(config, {}, {})).toBeUndefined(); expect(judgeForRun({}, {}, {})).toBeUndefined(); // no judge config needed when not asked - expect(judgeForRun(config, {}, { semantic: true })).toBeInstanceOf(FakeJudgeProvider); - expect(judgeForRun(config, {}, { judge: 'g' })).toBeInstanceOf(FakeJudgeProvider); + expect(judgeForRun(config, {}, { semantic: true }) instanceof FakeJudgeProvider).toBe(true); + expect(judgeForRun(config, {}, { judge: 'g' }) instanceof FakeJudgeProvider).toBe(true); }); it('fails when asked and no judge is configured, or the named provider is missing', () => { diff --git a/tests/judge/judge-node.test.ts b/tests/judge/judge-node.test.ts new file mode 100644 index 0000000..1948012 --- /dev/null +++ b/tests/judge/judge-node.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from 'bun:test'; +import { judgeNode, type NodeJudgement } from '../../src/judge/judge-node'; +import type { SemanticJudgeProvider, SemanticJudgeResult } from '../../src/judge/types'; +import type { Rule } from '../../src/schema/metadata-contract'; +import type { SpaceNode } from '../../src/types'; +import { node, passFailRule } from './helpers'; + +/** Provider returning canned results keyed by rule id, bypassing response parsing. */ +function canned(results: Record): SemanticJudgeProvider { + return { + name: 'canned', + evaluate: (input) => + Promise.resolve(Object.fromEntries(input.rules.map((r) => [r.id, results[r.id] ?? { verdict: 'pass' }]))), + }; +} + +describe('judgeNode', () => { + it('treats an unknown verdict as a judge error, not a pass', async () => { + const rule = { + id: passFailRule.id, + description: passFailRule.description, + category: passFailRule.category, + mode: 'semantic', + severity: { fail: 'error' }, + verdicts: passFailRule.verdicts, + } as unknown as Rule; + const judgements: NodeJudgement[] = await judgeNode( + canned({ 'customer-voice': { verdict: 'unexpected', evidence: 'hmm' } }), + node as SpaceNode, + [rule], + ); + expect(judgements).toHaveLength(1); + expect(judgements[0]).toMatchObject({ + failed: true, + error: expect.stringContaining('unknown verdict "unexpected"'), + }); + }); +}); diff --git a/tests/judge/live.test.ts b/tests/judge/live.test.ts index 464a57c..315a79c 100644 --- a/tests/judge/live.test.ts +++ b/tests/judge/live.test.ts @@ -8,17 +8,17 @@ import type { SemanticJudgeInput, SemanticRuleSpec } from '../../src/judge/types import { node, passFailRule } from './helpers'; const CASES = [ - { type: 'jev' as const, keyEnv: 'OPENROUTER_API_KEY' }, - { type: 'anthropic' as const, keyEnv: 'ANTHROPIC_API_KEY' }, - { type: 'openai' as const, keyEnv: 'OPENAI_API_KEY' }, + { name: 'openrouter-jev', api: 'openrouter-decisions' as const, keyEnv: 'OPENROUTER_API_KEY' }, + { name: 'anthropic', api: 'anthropic-messages' as const, keyEnv: 'ANTHROPIC_API_KEY' }, + { name: 'openai', api: 'openai-completions' as const, keyEnv: 'OPENAI_API_KEY' }, ]; describe.skipIf(process.env.SCTX_LIVE_JUDGE_TESTS !== '1')('live judges', () => { - for (const { type, keyEnv } of CASES) { + for (const { name, api, keyEnv } of CASES) { it.skipIf(!process.env[keyEnv])( - `${type}: returns a verdict from the rule's closed set`, + `${name}: returns a verdict from the rule's closed set`, async () => { - const provider = createModelProvider(type, { type }, { [keyEnv]: process.env[keyEnv] }); + const provider = createModelProvider(name, { api }, { [keyEnv]: process.env[keyEnv] }); const rule: SemanticRuleSpec = { ...passFailRule, description: 'The node title names a customer need or pain point', diff --git a/tests/judge/provider/jev.test.ts b/tests/judge/provider/jev.test.ts index 7b278f4..79937d5 100644 --- a/tests/judge/provider/jev.test.ts +++ b/tests/judge/provider/jev.test.ts @@ -35,13 +35,13 @@ describe('JevJudgeProvider', () => { }, }, }); - const jev = new JevJudgeProvider({ apiKey: 'k' }); + const jev = new JevJudgeProvider({ api: 'openrouter-decisions', apiKey: 'k' }); const results = await jev.evaluate({ node, rules: [passFailRule, graded] }); expect(calls).toHaveLength(1); expect(calls[0]?.url).toBe('https://openrouter.ai/api/alpha/decisions'); const sent = JSON.parse(calls[0]?.init.body as string); - expect(sent.model).toBe('typesafe/jev-1.13'); + expect(sent.model).toBe('typesafe/jev-latest'); expect(sent.state).toMatchObject({ title: 'I cannot find my receipts', parents: ['Faster Tax Filing'] }); expect(sent.questions.framing).toEqual({ type: 'choice', @@ -57,20 +57,27 @@ describe('JevJudgeProvider', () => { it('reports an error for a rule with a missing or unknown choice, leaving others intact', async () => { mockFetch({ answers: { framing: { choice: 'nonsense' }, 'customer-voice': { choice: 'fail' } } }); - const results = await new JevJudgeProvider({ apiKey: 'k' }).evaluate({ node, rules: [passFailRule, graded] }); + const results = await new JevJudgeProvider({ api: 'openrouter-decisions', apiKey: 'k' }).evaluate({ + node, + rules: [passFailRule, graded], + }); expect(results.framing).toHaveProperty('error'); expect(results['customer-voice']).toMatchObject({ verdict: 'fail' }); }); it('throws on an HTTP error', async () => { mockFetch({ error: 'bad' }, 401); - await expect(new JevJudgeProvider({ apiKey: 'k' }).evaluate({ node, rules: [passFailRule] })).rejects.toThrow( - 'Jev API error (401)', - ); + await expect( + new JevJudgeProvider({ api: 'openrouter-decisions', apiKey: 'k' }).evaluate({ node, rules: [passFailRule] }), + ).rejects.toThrow('Jev API error (401)'); }); - it('is created explicitly via provider "jev" with OPENROUTER_API_KEY, never auto-detected', () => { - expect(createModelProvider('jev', { type: 'jev' }, { OPENROUTER_API_KEY: 'k' }).name).toBe('jev'); - expect(() => createModelProvider('jev', { type: 'jev' }, {})).toThrow('OPENROUTER_API_KEY'); + it('is created explicitly via a bundled provider with its key env, never auto-detected', () => { + expect( + createModelProvider('openrouter-jev', { api: 'openrouter-decisions' }, { OPENROUTER_API_KEY: 'k' }).name, + ).toBe('jev'); + expect(() => createModelProvider('openrouter-jev', { api: 'openrouter-decisions' }, {})).toThrow( + 'OPENROUTER_API_KEY', + ); }); }); From a112497848abe8db30ee9dda499d3935c88e618f Mon Sep 17 00:00:00 2001 From: Roger Barnes Date: Thu, 15 Oct 2026 19:52:52 +1100 Subject: [PATCH 19/22] fix(judge): pin the openrouter bundle's endpoint; show endpoints in sctx providers The openrouter bundle spoke openai-completions with no endpoint, so its key would be sent to api.openai.com. It now pins https://openrouter.ai/api/v1. sctx providers shows each provider's effective endpoint, and bundle defaults are pinned by tests. --- src/commands/providers.ts | 3 ++- src/judge/factory.ts | 7 ++++- tests/judge/factory.test.ts | 51 +++++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/src/commands/providers.ts b/src/commands/providers.ts index ae92030..3808504 100644 --- a/src/commands/providers.ts +++ b/src/commands/providers.ts @@ -59,7 +59,8 @@ export function listProviders(options: { json?: boolean } = {}): void { for (const r of rows) { const origin = r.source === 'bundled' ? '' : ' (config)'; const model = r.model ?? '(api default)'; + const endpoint = r.endpoint ?? '(api default)'; const key = r.valid ? `key via ${r.keyEnv}` : (r.error ?? 'no key'); - console.log(`${r.name}${origin} ${r.api} model=${model} ${key}`); + console.log(`${r.name}${origin}\n api=${r.api}\n endpoint=${endpoint}\n model=${model}\n ${key}`); } } diff --git a/src/judge/factory.ts b/src/judge/factory.ts index 7f09721..c28c0f8 100644 --- a/src/judge/factory.ts +++ b/src/judge/factory.ts @@ -41,7 +41,12 @@ export const BUNDLED_PROVIDERS: Record = { endpointEnv: 'OPENAI_BASE_URL', apiKeyEnv: 'OPENAI_API_KEY', }, - openrouter: { api: 'openai-completions', model: 'openai/gpt-4o-mini', apiKeyEnv: 'OPENROUTER_API_KEY' }, + openrouter: { + api: 'openai-completions', + endpoint: 'https://openrouter.ai/api/v1', + model: 'openai/gpt-4o-mini', + apiKeyEnv: 'OPENROUTER_API_KEY', + }, } as const; /** Each API reads its key from this variable unless `apiKeyEnv` names another one. Model and endpoint never read env vars. */ diff --git a/tests/judge/factory.test.ts b/tests/judge/factory.test.ts index 1af5444..0042d58 100644 --- a/tests/judge/factory.test.ts +++ b/tests/judge/factory.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'bun:test'; import { + BUNDLED_PROVIDERS, createJudge, createModelProvider, hydrateProvider, @@ -44,7 +45,57 @@ describe('hydrateProvider', () => { }); }); +describe('bundled providers', () => { + it.each([ + ['openrouter-jev', 'openrouter-decisions', undefined, 'typesafe/jev-latest', 'OPENROUTER_API_KEY'], + ['typesafe-jev', 'typesafe-systemone', undefined, 'jev-latest', 'TYPESAFE_API_KEY'], + ['anthropic', 'anthropic-messages', undefined, 'claude-haiku-5-5', 'ANTHROPIC_API_KEY'], + ['openai', 'openai-completions', undefined, 'gpt-4o-mini', 'OPENAI_API_KEY'], + ['openrouter', 'openai-completions', 'https://openrouter.ai/api/v1', 'openai/gpt-4o-mini', 'OPENROUTER_API_KEY'], + ] as const)('%s carries its vendor defaults', (name, api, endpoint, model, keyEnv) => { + const env = { [keyEnv]: 'k' }; + const judge = createJudge({ judge: { provider: name } }, {}, undefined, env); + expect(judge.name).toBe( + api === 'openrouter-decisions' || api === 'typesafe-systemone' + ? 'jev' + : api.startsWith('anthropic') + ? 'anthropic' + : 'openai', + ); + expect(BUNDLED_PROVIDERS[name]).toMatchObject({ api, model, apiKeyEnv: keyEnv }); + if (endpoint) expect(BUNDLED_PROVIDERS[name]!.endpoint).toBe(endpoint); // only where the bundle pins one + }); +}); + describe('createModelProvider', () => { + it('the decisions API defaults to the OpenRouter decisions endpoint', async () => { + const calls: string[] = []; + const realFetch = globalThis.fetch; + globalThis.fetch = (async (url: string | URL) => { + calls.push(String(url)); + throw new Error('stop'); + }) as unknown as typeof fetch; + try { + const judge = createJudge({ judge: { provider: 'openrouter-jev' } }, {}, undefined, { OPENROUTER_API_KEY: 'k' }); + await judge + .evaluate({ + node: { + label: 'n', + title: 't', + type: 'opportunity', + resolvedType: 'opportunity', + schemaData: {}, + resolvedParents: [], + }, + rules: [], + }) + .catch(() => {}); + expect(calls[0]).toBe('https://openrouter.ai/api/alpha/decisions'); + } finally { + globalThis.fetch = realFetch; + } + }); + it('creates FakeJudgeProvider for api "fake"', () => { expect(createModelProvider('f', { api: 'fake' }, {}) instanceof FakeJudgeProvider).toBe(true); }); From 898e96ac35d5016363290e3731fa3bdc2a477e70 Mon Sep 17 00:00:00 2001 From: Roger Barnes Date: Thu, 15 Oct 2026 22:48:40 +1100 Subject: [PATCH 20/22] feat(judge): OpenAI Decisions API provider; tighter bundled-provider setup - new openai-decisions API and OpenAIDecisionsJudgeProvider (one request per node, choice questions from the rule's verdict set; verified live against api.openai.com/v1/decisions) - new bundles: openai-decisions, openrouter-openai-decisions; openrouter chat bundle renamed to openrouter-openai - model constants live in the provider modules; the factory imports them (recommended models per vendor are this project's judgement; pinned bundles keep their literal) - OpenRouter model slug verified: ~typesafe/jev-latest resolves on the decisions endpoint; typesafe/jev-latest does not - docs: bundled-provider tables replaced by "sctx providers" plus the wire-API table and naming conventions - bun run test:models smoke-checks each bundled model resolves (keyless where the endpoint allows; skip per provider without a key) Co-authored-by: Claude --- AGENTS.md | 5 +- docs/config.md | 33 +++-- docs/semantic-rules.md | 9 +- package.json | 1 + src/judge/factory.ts | 71 +++++++---- src/judge/provider/anthropic.ts | 4 +- src/judge/provider/jev.ts | 5 +- src/judge/provider/openai-decisions.ts | 101 ++++++++++++++++ src/judge/provider/openai.ts | 4 +- src/judge/types.ts | 1 + tests/judge/factory.test.ts | 23 ---- tests/judge/model-validation.test.ts | 98 +++++++++++++++ tests/judge/provider/jev.test.ts | 2 +- tests/judge/provider/openai-decisions.test.ts | 114 ++++++++++++++++++ 14 files changed, 399 insertions(+), 72 deletions(-) create mode 100644 src/judge/provider/openai-decisions.ts create mode 100644 tests/judge/model-validation.test.ts create mode 100644 tests/judge/provider/openai-decisions.test.ts diff --git a/AGENTS.md b/AGENTS.md index 6fb80e9..7a7d763 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,7 +9,9 @@ Before starting new work, review [docs/concepts.md](docs/concepts.md) for canoni - `bun run src/index.ts --help` — list CLI commands - `bun run typecheck` — type-check all code (`tsconfig.json`) - `bun run lint` — biome +- `bun run lint:fix` — biome with fixes - `bun run build` — compile `src/` to `dist/` (`tsconfig.build.json`) +- see `package.json` - Releases go through the `/release` skill. Space names (e.g. `personal`, `politics`) are resolved via a config file - `$SCTX_CONFIG`, `$XDG_CONFIG_HOME/structured-context/config.json`, `--config ` param, or `./config.json` @@ -66,7 +68,8 @@ For most development only the main unit tests need re-running regularly. - `bun run test:hook` — unit test plugin hooks (`hook-test/unit/`) - hook development only - `bun run test:hook:e2e` — test plugin hooks in Claude Code (`hook-test/`) - hook development only - `bun run test:smoke` — smoke tests run against locally configured spaces - only use when changes could affect compatibility. -- `bun run test:judge` — live judge tests (`tests/judge/live.test.ts`): one real provider call per provider whose API key is set. Costs money; use sparingly when changing judges, prompts or semantic rule behaviour. +- `bun run test:judge` — live judge tests (`tests/judge/live.test.ts`): one real provider call per provider whose API key is set. Costs money; use sparingly when changing judges or semantic rule behaviour. +- `bun run test:models` — check that every bundled provider's configured model exists (no API keys) ## Debugging diff --git a/docs/config.md b/docs/config.md index a4d6460..240989f 100644 --- a/docs/config.md +++ b/docs/config.md @@ -67,19 +67,30 @@ Plugins can also contribute schemas and partials via a `/schemas/` d ## Model providers and judge -structured-context judges [semantic rules](semantic-rules.md) with a **model provider**: a named definition that says which wire API to speak, which model to use and where. A provider definition combines an `api`, explicit values or environment-variable names, and the key source. `sctx providers` lists every usable provider with its effective config (keys are never shown). +structured-context judges [semantic rules](semantic-rules.md) with a **model provider**: a named definition that says which wire API to speak, which model to use and where. A provider definition combines an `api`, explicit values or environment-variable names, and the key source. -**Bundled providers** work without any configuration. Each combines an API, an endpoint, a key source and an opinionated model: +**Bundled providers** work without any configuration: a fixed model per vendor chosen by this project, plus `-env` variants that also read the vendor's usual model and endpoint variables (`OPENAI_MODEL`, `OPENAI_BASE_URL`, `ANTHROPIC_MODEL`), falling back to the bundle's defaults where a variable is unset. Bundles without `-env` read no environment beyond the key: they behave the same on every machine. -| Provider | API | Model | Key variable | -|---|---|---|---| -| `openrouter-jev` | `openrouter-decisions` | `typesafe/jev-latest` | `OPENROUTER_API_KEY` | -| `typesafe-jev` | `typesafe-systemone` | `jev-latest` | `TYPESAFE_API_KEY` | -| `anthropic` | `anthropic-messages` | `claude-haiku-5-5` | `ANTHROPIC_API_KEY` | -| `openai` | `openai-completions` | `gpt-4o-mini` | `OPENAI_API_KEY` | -| `openrouter` | `openai-completions` | `openai/gpt-4o-mini` | `OPENROUTER_API_KEY` | +Run `sctx providers` for the authoritative list of bundled and configured providers with their effective API, endpoint, model and key variable. Keys are never shown. `bun run test:models` checks that each bundled model resolves on its endpoint. -The `-env` variants (`openai-env`, `anthropic-env`) also read the vendor's usual model and endpoint variables (`OPENAI_MODEL`, `OPENAI_BASE_URL`, `ANTHROPIC_MODEL`), falling back to the bundle's defaults where a variable is unset. Bundled providers without `-env` read no environment beyond the key: they behave the same on every machine. `anthropic-haiku-5-5` pins the model, so it does not drift when the `anthropic` bundle's default changes. +### Wire APIs and bundled-provider conventions + +A provider's `api` names the **request shape**, potentially supported by multiple vendors. + +| API | Shape | Bundled providers | +|---|---|---| +| `openrouter-decisions` | Decisions API, closed-set `choice` questions, answers with confidence and probabilities | `openrouter-jev`, `openrouter-openai-decisions` | +| `typesafe-systemone` | Same decisions shape (currently) | `typesafe-jev` | +| `openai-decisions` | OpenAI's Decisions API: same concept, its own request/response shape | `openai-decisions` | +| `anthropic-messages` | Chat: one request per rule, structured JSON verdict | `anthropic`, `anthropic-haiku-5-5`, `anthropic-env` | +| `openai-completions` | Chat: OpenAI-compatible; works with Ollama, vLLM and OpenRouter | `openai`, `openai-env`, `openrouter-openai` | + +Bundled provider names follow the pattern: + +- `` — this project's recommended model for that vendor; can change between releases. +- `-` (e.g. `anthropic-haiku-5-5`) — pins a model id, so it does not drift when the `` bundle's recommendation moves. +- `-env` — hydrates model, endpoint etc from the vendor's usual environment variables. +- `openrouter--` (e.g. `openrouter-openai-decisions`) — aggregating vendor's endpoint variant **Configured providers** live in `modelProviders` at the top level, and replace a bundled provider of the same name. Fields a configured provider leaves out still take the API defaults: @@ -94,7 +105,7 @@ The `-env` variants (`openai-env`, `anthropic-env`) also read the vendor's usual | Field | Description | |---|---| -| `api` | **Required.** The wire API: `openrouter-decisions`, `typesafe-systemone`, `anthropic-messages` or `openai-completions` (see [Choosing a judge](semantic-rules.md#choosing-a-judge)), or `fake` (for tests). | +| `api` | **Required.** The wire API (see the table above and [Choosing a judge](semantic-rules.md#choosing-a-judge)), or `fake` (for tests). | | `model` | A fixed model. Its default depends on the API. | | `modelEnv` | Environment variable holding a model override. An explicit `model` wins; a variable set but empty is an error. | | `endpoint` | A fixed API URL. | diff --git a/docs/semantic-rules.md b/docs/semantic-rules.md index d6e067f..ba192cd 100644 --- a/docs/semantic-rules.md +++ b/docs/semantic-rules.md @@ -97,14 +97,7 @@ Gating uses the severity of the verdict. A verdict with `error` severity gates t Use a decision model when you can. Its results are repeatable and cheap, and they include a confidence. Use a chat model when the rubric needs open reasoning or a written explanation. -| Bundled provider | API | Model | Key variable | -|---|---|---|---| -| `openrouter-jev` | `openrouter-decisions` | `typesafe/jev-latest` | `OPENROUTER_API_KEY` | -| `typesafe-jev` | `typesafe-systemone` | `jev-latest` | `TYPESAFE_API_KEY` | -| `anthropic` | `anthropic-messages` | `claude-haiku-5-5` | `ANTHROPIC_API_KEY` | -| `openai` | `openai-completions` | `gpt-4o-mini` | `OPENAI_API_KEY` | - -The `openai-completions` API speaks to any OpenAI-compatible endpoint, for example Ollama or vLLM; local servers do not need a key. `sctx providers` lists every bundled and configured provider with its effective config. +Bundled providers give you a model with no configuration; the decision models among them speak a decisions API. Run `sctx providers` for the full list with effective models, endpoints and key variables. structured-context uses only the provider that you configure. It does not choose a provider from the keys in your environment. To override models, endpoints or keys, add an entry of the same name to `modelProviders`, or define a new one (see [Model providers and judge](config.md#model-providers-and-judge)). diff --git a/package.json b/package.json index 43be01e..9c5d6d7 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "test": "bun test tests/", "test:smoke": "bun test smoke/", "test:judge": "SCTX_LIVE_JUDGE_TESTS=1 bun test tests/judge/live.test.ts", + "test:models": "SCTX_MODEL_VALIDATION=1 bun test tests/judge/model-validation.test.ts", "test:hook": "bun test hook-test/unit/", "test:hook:e2e": "bun test hook-test/hooks.test.ts", "test:all": "bun test", diff --git a/src/judge/factory.ts b/src/judge/factory.ts index c28c0f8..66f5a34 100644 --- a/src/judge/factory.ts +++ b/src/judge/factory.ts @@ -1,9 +1,11 @@ import type { AnthropicJudgeProviderOptions } from './provider/anthropic'; -import { AnthropicJudgeProvider } from './provider/anthropic'; +import { AnthropicJudgeProvider, RECOMMENDED_ANTHROPIC_MODEL } from './provider/anthropic'; import { FakeJudgeProvider } from './provider/fake'; import type { JevJudgeProviderOptions } from './provider/jev'; -import { JevJudgeProvider } from './provider/jev'; -import { OpenAIJudgeProvider } from './provider/openai'; +import { JEV_LATEST, JevJudgeProvider } from './provider/jev'; +import { OpenAIJudgeProvider, RECOMMENDED_OPENAI_MODEL } from './provider/openai'; +import type { OpenAIDecisionsJudgeProviderOptions } from './provider/openai-decisions'; +import { OpenAIDecisionsJudgeProvider, RECOMMENDED_OPENAI_DECISIONS_MODEL } from './provider/openai-decisions'; import { type HydratedProvider, MODEL_PROVIDER_APIS, @@ -13,39 +15,61 @@ import { type Env = Record; +const OPENROUTER_COMPLETIONS_ENDPOINT = 'https://openrouter.ai/api/v1'; + +const RECOMMENDED_OPENROUTER_OPENAI_DECISIONS_MODEL = `openai/${RECOMMENDED_OPENAI_DECISIONS_MODEL}-decisions`; +const RECOMMENDED_OPENROUTER_OPENAI_MODEL = `openai/${RECOMMENDED_OPENAI_MODEL}`; + +const ANTHROPIC_KEY_ENV = 'ANTHROPIC_API_KEY'; +const OPENAI_KEY_ENV = 'OPENAI_API_KEY'; +const OPENROUTER_KEY_ENV = 'OPENROUTER_API_KEY'; + +const ANTHROPIC_BASE: ModelProviderConfig = { + api: 'anthropic-messages', + model: RECOMMENDED_ANTHROPIC_MODEL, + apiKeyEnv: ANTHROPIC_KEY_ENV, +}; +const OPENAI_BASE: ModelProviderConfig = { + api: 'openai-completions', + model: RECOMMENDED_OPENAI_MODEL, + apiKeyEnv: OPENAI_KEY_ENV, +}; + /** * Bundled providers: usable as a judge by name, without a `modelProviders` entry. A user entry of the * same name replaces one wholesale; fields it leaves out take the type defaults in the constructors. * Every value here is explicit: no bundle reads the environment. */ export const BUNDLED_PROVIDERS: Record = { - // Jev decision model. The TypeSafe and OpenRouter decision endpoints share a wire shape, so both - // speak a decisions API; the vendors' endpoints stay distinct bundles. - 'typesafe-jev': { api: 'typesafe-systemone', model: 'jev-latest', apiKeyEnv: 'TYPESAFE_API_KEY' }, - 'openrouter-jev': { api: 'openrouter-decisions', model: 'typesafe/jev-latest', apiKeyEnv: 'OPENROUTER_API_KEY' }, - // Chat models. `` is a fixed opinionated bundle; `-env` hydrates model and endpoint - // from the vendor's usual environment variables, falling back to the bundle's defaults. - anthropic: { api: 'anthropic-messages', model: 'claude-haiku-5-5', apiKeyEnv: 'ANTHROPIC_API_KEY' }, - 'anthropic-haiku-5-5': { api: 'anthropic-messages', model: 'claude-haiku-5-5', apiKeyEnv: 'ANTHROPIC_API_KEY' }, + 'typesafe-jev': { api: 'typesafe-systemone', model: JEV_LATEST, apiKeyEnv: 'TYPESAFE_API_KEY' }, + 'openrouter-jev': { api: 'openrouter-decisions', model: `~typesafe/${JEV_LATEST}`, apiKeyEnv: OPENROUTER_KEY_ENV }, + 'openrouter-openai-decisions': { + api: 'openrouter-decisions', + model: RECOMMENDED_OPENROUTER_OPENAI_DECISIONS_MODEL, + apiKeyEnv: OPENROUTER_KEY_ENV, + }, + 'openai-decisions': { + api: 'openai-decisions', + model: RECOMMENDED_OPENAI_DECISIONS_MODEL, + apiKeyEnv: OPENAI_KEY_ENV, + }, + anthropic: ANTHROPIC_BASE, + 'anthropic-haiku-5-5': { api: 'anthropic-messages', model: 'claude-haiku-5-5', apiKeyEnv: ANTHROPIC_KEY_ENV }, 'anthropic-env': { - api: 'anthropic-messages', - model: 'claude-haiku-5-5', + ...ANTHROPIC_BASE, modelEnv: 'ANTHROPIC_MODEL', - apiKeyEnv: 'ANTHROPIC_API_KEY', }, - openai: { api: 'openai-completions', model: 'gpt-4o-mini', apiKeyEnv: 'OPENAI_API_KEY' }, + openai: OPENAI_BASE, 'openai-env': { - api: 'openai-completions', - model: 'gpt-4o-mini', + ...OPENAI_BASE, modelEnv: 'OPENAI_MODEL', endpointEnv: 'OPENAI_BASE_URL', - apiKeyEnv: 'OPENAI_API_KEY', }, - openrouter: { + 'openrouter-openai': { api: 'openai-completions', - endpoint: 'https://openrouter.ai/api/v1', - model: 'openai/gpt-4o-mini', - apiKeyEnv: 'OPENROUTER_API_KEY', + endpoint: OPENROUTER_COMPLETIONS_ENDPOINT, + model: RECOMMENDED_OPENROUTER_OPENAI_MODEL, + apiKeyEnv: OPENROUTER_KEY_ENV, }, } as const; @@ -53,6 +77,7 @@ export const BUNDLED_PROVIDERS: Record = { const DEFAULT_KEY_ENV: Record = { 'anthropic-messages': 'ANTHROPIC_API_KEY', 'openai-completions': 'OPENAI_API_KEY', + 'openai-decisions': 'OPENAI_API_KEY', 'openrouter-decisions': 'OPENROUTER_API_KEY', 'typesafe-systemone': 'TYPESAFE_API_KEY', } as const; @@ -115,6 +140,8 @@ export function createModelProvider( case 'openrouter-decisions': case 'typesafe-systemone': return new JevJudgeProvider(h as JevJudgeProviderOptions); + case 'openai-decisions': + return new OpenAIDecisionsJudgeProvider(h as OpenAIDecisionsJudgeProviderOptions); case 'openai-completions': return new OpenAIJudgeProvider(h); } diff --git a/src/judge/provider/anthropic.ts b/src/judge/provider/anthropic.ts index c895323..9f3bd6c 100644 --- a/src/judge/provider/anthropic.ts +++ b/src/judge/provider/anthropic.ts @@ -7,7 +7,7 @@ export interface AnthropicJudgeProviderOptions extends HydratedProvider { apiKey?: string; } -export const DEFAULT_ANTHROPIC_MODEL = 'claude-haiku-5-5'; +export const RECOMMENDED_ANTHROPIC_MODEL = 'claude-haiku-5-5'; export const ANTHROPIC_MESSAGES_ENDPOINT = 'https://api.anthropic.com/v1/messages'; /** Judge provider for the Anthropic Messages API. Reads no environment: it takes hydrated values. */ @@ -20,7 +20,7 @@ export class AnthropicJudgeProvider extends PerRuleJudgeProvider { constructor(options: AnthropicJudgeProviderOptions) { super(); this.apiKey = options.apiKey ?? ''; - this.model = options.model || DEFAULT_ANTHROPIC_MODEL; + this.model = options.model || RECOMMENDED_ANTHROPIC_MODEL; this.endpoint = options.endpoint || ANTHROPIC_MESSAGES_ENDPOINT; } diff --git a/src/judge/provider/jev.ts b/src/judge/provider/jev.ts index f2f214c..6ab9b10 100644 --- a/src/judge/provider/jev.ts +++ b/src/judge/provider/jev.ts @@ -1,10 +1,11 @@ import type { HydratedProvider, SemanticJudgeInput, SemanticJudgeProvider, SemanticJudgeResult } from '../types'; export interface JevJudgeProviderOptions extends HydratedProvider { - /** The factory requires a resolved key before constructing a non-OpenAI provider. */ apiKey?: string; } +export const JEV_LATEST = 'jev-latest'; + export const OPENROUTER_DECISIONS_ENDPOINT = 'https://openrouter.ai/api/alpha/decisions'; type JevChoiceAnswer = { @@ -29,7 +30,7 @@ export class JevJudgeProvider implements SemanticJudgeProvider { constructor(options: JevJudgeProviderOptions) { this.apiKey = options.apiKey ?? ''; - this.model = options.model || 'typesafe/jev-latest'; + this.model = options.model || JEV_LATEST; this.endpoint = options.endpoint || OPENROUTER_DECISIONS_ENDPOINT; } diff --git a/src/judge/provider/openai-decisions.ts b/src/judge/provider/openai-decisions.ts new file mode 100644 index 0000000..072c999 --- /dev/null +++ b/src/judge/provider/openai-decisions.ts @@ -0,0 +1,101 @@ +import type { HydratedProvider, SemanticJudgeInput, SemanticJudgeProvider, SemanticJudgeResult } from '../types'; + +export interface OpenAIDecisionsJudgeProviderOptions extends HydratedProvider { + apiKey?: string; +} + +export const RECOMMENDED_OPENAI_DECISIONS_MODEL = 'gpt-6-luna'; +export const OPENAI_DECISIONS_ENDPOINT = 'https://api.openai.com/v1/decisions'; + +type OpenAIDecisionAnswer = { + type?: string; + name?: string; + choice?: string; + confidence?: number; + probabilities?: Array<{ value: string; probability: number }>; +}; + +/** + * Judge provider for OpenAI's Decisions API (https://developers.openai.com/api/docs/guides/decisions). + * + * Each node is sent once as the `input` text, with one `choice` question per semantic rule whose + * choices are the rule's closed verdict set. The API returns the chosen value with per-choice + * probabilities and a confidence, but no prose, so `evidence` is a summary of the distribution. + */ +export class OpenAIDecisionsJudgeProvider implements SemanticJudgeProvider { + readonly name = 'openai-decisions'; + private apiKey: string; + private model: string; + private endpoint: string; + + constructor(options: OpenAIDecisionsJudgeProviderOptions) { + this.apiKey = options.apiKey ?? ''; + this.model = options.model || RECOMMENDED_OPENAI_DECISIONS_MODEL; + this.endpoint = options.endpoint || OPENAI_DECISIONS_ENDPOINT; + } + + async evaluate(input: SemanticJudgeInput): Promise> { + const { node, rules } = input; + + const stateLines = [ + `Title: ${node.title}`, + `Type: ${node.resolvedType}`, + ...(node.resolvedParents?.length ? [`Parents: ${node.resolvedParents.map((p) => p.title).join(', ')}`] : []), + ...Object.entries(node.schemaData ?? {}) + .filter(([field]) => field !== 'title' && field !== 'type') + .map(([field, value]) => `${field}: ${JSON.stringify(value) ?? String(value)}`), + ]; + + const res = await fetch(this.endpoint, { + method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${this.apiKey}` }, + body: JSON.stringify({ + model: this.model, + input: stateLines.join('\n'), + questions: rules.map((rule) => ({ + type: 'choice', + name: rule.id, + instructions: rule.rubric || rule.description, + choices: Object.entries(rule.verdicts).map(([value, description]) => ({ value, description })), + })), + }), + }); + + if (!res.ok) { + const errorText = await res.text().catch(() => ''); + throw new Error(`OpenAI Decisions API error (${res.status}): ${errorText}`); + } + + const data = (await res.json()) as { answers?: OpenAIDecisionAnswer[] }; + const answers = new Map((data.answers ?? []).filter((a) => a.name).map((a) => [a.name as string, a])); + const results: Record = {}; + for (const rule of rules) { + const answer = answers.get(rule.id); + if (!answer || answer.type === 'refusal' || !answer.choice || !(answer.choice in rule.verdicts)) { + results[rule.id] = { error: `OpenAI Decisions returned no valid verdict for rule "${rule.id}"` }; + continue; + } + const probabilities = Object.fromEntries((answer.probabilities ?? []).map((p) => [p.value, p.probability])); + results[rule.id] = { + verdict: answer.choice, + confidence: answer.confidence, + probabilities, + evidence: describeDistribution(answer.choice, answer.confidence, probabilities), + }; + } + return results; + } +} + +function describeDistribution( + choice: string, + confidence: number | undefined, + probabilities: Record, +): string { + const parts = Object.entries(probabilities) + .sort(([, a], [, b]) => b - a) + .map(([verdict, p]) => `${verdict} ${p.toFixed(2)}`); + const conf = confidence !== undefined ? `confidence ${confidence.toFixed(2)}` : undefined; + const detail = [conf, ...parts].filter(Boolean).join('; '); + return `OpenAI Decisions chose "${choice}"${detail ? ` (${detail})` : ''}`; +} diff --git a/src/judge/provider/openai.ts b/src/judge/provider/openai.ts index 9795b6b..a624306 100644 --- a/src/judge/provider/openai.ts +++ b/src/judge/provider/openai.ts @@ -7,7 +7,7 @@ export interface OpenAIJudgeProviderOptions extends HydratedProvider { apiKey?: string; } -export const DEFAULT_OPENAI_MODEL = 'gpt-4o-mini'; +export const RECOMMENDED_OPENAI_MODEL = 'gpt-6-luna'; export const OPENAI_COMPLETIONS_ENDPOINT = 'https://api.openai.com/v1'; /** @@ -23,7 +23,7 @@ export class OpenAIJudgeProvider extends PerRuleJudgeProvider { constructor(options: OpenAIJudgeProviderOptions) { super(); this.apiKey = options.apiKey; - this.model = options.model || DEFAULT_OPENAI_MODEL; + this.model = options.model || RECOMMENDED_OPENAI_MODEL; const base = options.endpoint || OPENAI_COMPLETIONS_ENDPOINT; this.endpoint = base.endsWith('/chat/completions') ? base : `${base.replace(/\/+$/, '')}/chat/completions`; diff --git a/src/judge/types.ts b/src/judge/types.ts index d0dad0a..79417d1 100644 --- a/src/judge/types.ts +++ b/src/judge/types.ts @@ -39,6 +39,7 @@ export interface SemanticJudgeProvider { /** Wire APIs a provider can speak. Names the request shape, never the vendor or model. */ export const MODEL_PROVIDER_APIS = [ 'openai-completions', + 'openai-decisions', 'anthropic-messages', 'openrouter-decisions', 'typesafe-systemone', diff --git a/tests/judge/factory.test.ts b/tests/judge/factory.test.ts index 0042d58..4742146 100644 --- a/tests/judge/factory.test.ts +++ b/tests/judge/factory.test.ts @@ -1,6 +1,5 @@ import { describe, expect, it } from 'bun:test'; import { - BUNDLED_PROVIDERS, createJudge, createModelProvider, hydrateProvider, @@ -45,28 +44,6 @@ describe('hydrateProvider', () => { }); }); -describe('bundled providers', () => { - it.each([ - ['openrouter-jev', 'openrouter-decisions', undefined, 'typesafe/jev-latest', 'OPENROUTER_API_KEY'], - ['typesafe-jev', 'typesafe-systemone', undefined, 'jev-latest', 'TYPESAFE_API_KEY'], - ['anthropic', 'anthropic-messages', undefined, 'claude-haiku-5-5', 'ANTHROPIC_API_KEY'], - ['openai', 'openai-completions', undefined, 'gpt-4o-mini', 'OPENAI_API_KEY'], - ['openrouter', 'openai-completions', 'https://openrouter.ai/api/v1', 'openai/gpt-4o-mini', 'OPENROUTER_API_KEY'], - ] as const)('%s carries its vendor defaults', (name, api, endpoint, model, keyEnv) => { - const env = { [keyEnv]: 'k' }; - const judge = createJudge({ judge: { provider: name } }, {}, undefined, env); - expect(judge.name).toBe( - api === 'openrouter-decisions' || api === 'typesafe-systemone' - ? 'jev' - : api.startsWith('anthropic') - ? 'anthropic' - : 'openai', - ); - expect(BUNDLED_PROVIDERS[name]).toMatchObject({ api, model, apiKeyEnv: keyEnv }); - if (endpoint) expect(BUNDLED_PROVIDERS[name]!.endpoint).toBe(endpoint); // only where the bundle pins one - }); -}); - describe('createModelProvider', () => { it('the decisions API defaults to the OpenRouter decisions endpoint', async () => { const calls: string[] = []; diff --git a/tests/judge/model-validation.test.ts b/tests/judge/model-validation.test.ts new file mode 100644 index 0000000..3f1b0c2 --- /dev/null +++ b/tests/judge/model-validation.test.ts @@ -0,0 +1,98 @@ +/** + * Model-validation smoke tests: check that every bundled provider's configured model actually + * exists on its endpoint. Unlike live.test.ts these make no judgement call — they only prove the + * model id resolves, so the check stays cheap and never sends real content. + * + * Opt in with SCTX_MODEL_VALIDATION=1 (`bun run test:models`); each check runs only when its + * provider's key is set. OpenRouter chat models validate keylessly against the public models + * list; decisions endpoints expose no keyless catalog, so they take a minimal probe. + */ +import { describe, expect, it } from 'bun:test'; +import { BUNDLED_PROVIDERS, hydrateProvider } from '../../src/judge/factory'; +import type { HydratedProvider } from '../../src/judge/types'; + +const env = process.env as Record; + +function resolved(name: string): { model?: string; api: string } { + const h: HydratedProvider = hydrateProvider(name, BUNDLED_PROVIDERS[name]!, env); + return { model: h.model, api: h.api }; +} + +const CHECKABLE = Object.keys(BUNDLED_PROVIDERS).filter((name) => resolved(name).model !== undefined); + +describe.skipIf(process.env.SCTX_MODEL_VALIDATION !== '1')('bundled models exist', () => { + it.each(CHECKABLE)( + '%s: model resolves on its endpoint', + async (name) => { + const { model, api } = resolved(name); + + if (api === 'openai-completions' && name.startsWith('openrouter')) { + // OpenRouter's models list is public and keyless; chat models appear in it. + const res = await fetch('https://openrouter.ai/api/v1/models'); + expect(res.ok).toBe(true); + const data = (await res.json()) as { data?: Array<{ id: string }> }; + expect(new Set((data.data ?? []).map((m) => m.id)).has(model!)).toBe(true); + return; + } + + const keyEnv = BUNDLED_PROVIDERS[name]!.apiKeyEnv!; + const key = env[keyEnv]; + if (!key) { + console.log(`skip ${name}: ${keyEnv} not set`); + return; + } + + if (api === 'openrouter-decisions') { + // The decisions catalog is not the public models list (verified: live slugs are absent from + // it), so probe the endpoint: an unknown model is a 400 naming the model, a known one is not. + const res = await fetch('https://openrouter.ai/api/alpha/decisions', { + method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${key}` }, + body: JSON.stringify({ + model, + state: {}, + questions: { probe: { type: 'choice', instructions: 'Answer a.', criteria: { a: 'always' } } }, + }), + }); + const text = await res.text(); + expect(res.ok || !/does not exist/i.test(text)).toBe(true); + return; + } + + if (api === 'openai-decisions') { + // No keyless catalog; an unknown model is 404 model_not_found, a known one is not. + const res = await fetch('https://api.openai.com/v1/decisions', { + method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${key}` }, + body: JSON.stringify({ + model, + input: 'model existence check', + questions: [{ type: 'predicate', name: 'probe', instructions: 'Answer yes.' }], + }), + }); + expect(res.status).not.toBe(404); + return; + } + + if (api === 'anthropic-messages' || api === 'openai-completions') { + if (api === 'anthropic-messages') { + const res = await fetch('https://api.anthropic.com/v1/models', { + headers: { 'x-api-key': key, 'anthropic-version': '2023-06-01' }, + }); + expect(res.ok).toBe(true); + const data = (await res.json()) as { data?: Array<{ id: string }> }; + expect(new Set((data.data ?? []).map((m) => m.id)).has(model!)).toBe(true); + } else { + const res = await fetch('https://api.openai.com/v1/models', { headers: { authorization: `Bearer ${key}` } }); + expect(res.ok).toBe(true); + const data = (await res.json()) as { data?: Array<{ id: string }> }; + expect(new Set((data.data ?? []).map((m) => m.id)).has(model!)).toBe(true); + } + return; + } + + console.log(`skip ${name}: no validation mechanism for api "${api}"`); + }, + 30_000, + ); +}); diff --git a/tests/judge/provider/jev.test.ts b/tests/judge/provider/jev.test.ts index 79937d5..da3ae03 100644 --- a/tests/judge/provider/jev.test.ts +++ b/tests/judge/provider/jev.test.ts @@ -41,7 +41,7 @@ describe('JevJudgeProvider', () => { expect(calls).toHaveLength(1); expect(calls[0]?.url).toBe('https://openrouter.ai/api/alpha/decisions'); const sent = JSON.parse(calls[0]?.init.body as string); - expect(sent.model).toBe('typesafe/jev-latest'); + expect(sent.model).toBe('jev-latest'); expect(sent.state).toMatchObject({ title: 'I cannot find my receipts', parents: ['Faster Tax Filing'] }); expect(sent.questions.framing).toEqual({ type: 'choice', diff --git a/tests/judge/provider/openai-decisions.test.ts b/tests/judge/provider/openai-decisions.test.ts new file mode 100644 index 0000000..2c4e0be --- /dev/null +++ b/tests/judge/provider/openai-decisions.test.ts @@ -0,0 +1,114 @@ +import { afterEach, describe, expect, it } from 'bun:test'; +import { createModelProvider } from '../../../src/judge/factory'; +import { OPENAI_DECISIONS_ENDPOINT, OpenAIDecisionsJudgeProvider } from '../../../src/judge/provider/openai-decisions'; +import { graded, node, passFailRule } from '../helpers'; + +const realFetch = globalThis.fetch; +afterEach(() => { + globalThis.fetch = realFetch; +}); + +function mockFetch(body: unknown, status = 200) { + const calls: Array<{ url: string; init: RequestInit }> = []; + globalThis.fetch = (async (url: string, init: RequestInit) => { + calls.push({ url, init }); + return new Response(JSON.stringify(body), { status }); + }) as unknown as typeof fetch; + return calls; +} + +describe('OpenAIDecisionsJudgeProvider', () => { + it('sends one request per node with a choice question per rule', async () => { + const calls = mockFetch({ + answers: [ + { + type: 'choice', + name: 'customer-voice', + choice: 'pass', + confidence: 0.9, + probabilities: [ + { value: 'pass', probability: 0.95 }, + { value: 'fail', probability: 0.05 }, + ], + }, + { + type: 'choice', + name: 'framing', + choice: 'solution', + confidence: 0.6, + probabilities: [ + { value: 'good', probability: 0.1 }, + { value: 'weak', probability: 0.3 }, + { value: 'solution', probability: 0.6 }, + ], + }, + ], + }); + const provider = new OpenAIDecisionsJudgeProvider({ api: 'openai-decisions', apiKey: 'k' }); + const results = await provider.evaluate({ node, rules: [passFailRule, graded] }); + + expect(calls).toHaveLength(1); + expect(calls[0]?.url).toBe(OPENAI_DECISIONS_ENDPOINT); + const sent = JSON.parse(calls[0]?.init.body as string); + expect(sent.model).toBe('gpt-6-luna'); + expect(sent.input).toContain('Title: I cannot find my receipts'); + expect(sent.input).toContain('Parents: Faster Tax Filing'); + expect(sent.questions).toEqual([ + { + type: 'choice', + name: 'customer-voice', + instructions: 'Needs to sound like a customer quote, not a solution or business goal', + choices: [ + { value: 'pass', description: 'Customer voice.' }, + { value: 'fail', description: 'Not customer voice.' }, + ], + }, + { + type: 'choice', + name: 'framing', + instructions: 'Framing quality', + choices: [ + { value: 'good', description: 'Clear need' }, + { value: 'weak', description: 'Vague need' }, + { value: 'solution', description: 'A solution in disguise' }, + ], + }, + ]); + expect(((calls[0]?.init.headers ?? {}) as Record).authorization).toBe('Bearer k'); + + expect(results.framing).toMatchObject({ verdict: 'solution', confidence: 0.6, probabilities: { solution: 0.6 } }); + expect((results.framing as { evidence: string }).evidence).toContain('solution 0.60'); + }); + + it('reports an error for a rule with a missing, refused or unknown choice, leaving others intact', async () => { + mockFetch({ + answers: [ + { type: 'choice', name: 'framing', choice: 'nonsense' }, + { type: 'refusal', name: 'customer-voice' }, + ], + }); + const results = await new OpenAIDecisionsJudgeProvider({ api: 'openai-decisions', apiKey: 'k' }).evaluate({ + node, + rules: [passFailRule, graded], + }); + expect(results.framing).toHaveProperty('error'); + expect(results['customer-voice']).toHaveProperty('error'); + }); + + it('throws on an HTTP error', async () => { + mockFetch({ error: 'bad' }, 401); + await expect( + new OpenAIDecisionsJudgeProvider({ api: 'openai-decisions', apiKey: 'k' }).evaluate({ + node, + rules: [passFailRule], + }), + ).rejects.toThrow('OpenAI Decisions API error (401)'); + }); + + it('is created explicitly via a bundled provider with its key env, never auto-detected', () => { + expect(createModelProvider('openai-decisions', { api: 'openai-decisions' }, { OPENAI_API_KEY: 'k' }).name).toBe( + 'openai-decisions', + ); + expect(() => createModelProvider('openai-decisions', { api: 'openai-decisions' }, {})).toThrow('OPENAI_API_KEY'); + }); +}); From 77eced9512e8254cb3485806a61d6e6aff61e108 Mon Sep 17 00:00:00 2001 From: Roger Barnes Date: Thu, 15 Oct 2026 23:34:13 +1100 Subject: [PATCH 21/22] =?UTF-8?q?docs:=20clarify=20rules.md=20structure=20?= =?UTF-8?q?=E2=80=94=20gating=20defaults,=20overrides,=20single-file=20val?= =?UTF-8?q?idation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- docs/rules.md | 149 +++++++++++++++++++---------------------- docs/semantic-rules.md | 6 +- 3 files changed, 72 insertions(+), 85 deletions(-) diff --git a/README.md b/README.md index 8501255..1834bb4 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ Rule violations: High tech juice press.md: Explore multiple candidate solutions (aim for at least three) for the target opportunity [best-practice: solution-quantity] ``` -Rule violations carry a severity (`error`, `warning` or `info`) that defaults from the rule's category. Only errors fail validation — see [docs/rules.md](docs/rules.md#categories-and-severity). +Rule violations carry a severity (`error`, `warning` or `info`) that defaults from the rule's category. Only errors fail validation — see [docs/rules.md](docs/rules.md#rule-shape). ## Installation diff --git a/docs/rules.md b/docs/rules.md index 0fdd11f..d7c1890 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -27,24 +27,7 @@ Rules are a flat array. ] ``` -| Field | Required | Description | -|---|---|---| -| `id` | yes | Unique rule identifier | -| `category` | yes | `validation` \| `coherence` \| `workflow` \| `best-practice` | -| `severity` | no | `error` \| `warning` \| `info`; overrides the category default (see [Severity](#severity)) | -| `description` | yes | Human-readable rule intent | -| `check` | yes, unless `mode: semantic` | JSONata expression; must evaluate to `true` | -| `when` | no | JSONata test. The rule runs only on nodes where it returns `true`. See [Conditional rules](#conditional-rules-when) | -| `mode` | no | `expression` (default) or `semantic` | -| `rubric`, `verdicts` | semantic only | See [Semantic rules](semantic-rules.md#rule-fields) | -| `type` | no | Restrict rule to nodes of this `resolvedType` | -| `scope` | no | Use `"global"` to evaluate once for the whole space | -| `override` | no | Only for merge conflicts: allows later duplicate `id` to replace earlier | -| `gatesDescendants` | no | Whether a violation gates downstream checks on the node's hierarchy subtree. Defaults to `true` for `error` rules, `false` otherwise. | - -## Categories and severity - -Categories describe a rule's intent and supply its default severity. They do not change expression execution semantics. +Every rule has a category, which describes its intent and supplies a default severity. | Category | Typical use | Default severity | |---|---|---| @@ -53,9 +36,7 @@ Categories describe a rule's intent and supply its default severity. They do not | `workflow` | Process/operating-discipline checks | `warning` | | `best-practice` | Advisory quality checks | `info` | -### Severity - -When a rule finds a problem on a node, it reports a **violation**. Otherwise it reports nothing. Each violation has a severity: +Severity can also be set per rule. | Severity | Meaning | Fails validation | |---|---|---| @@ -80,20 +61,32 @@ Prefer choosing the right category and relying on its default. Set `severity` on `sctx schemas show` lists each category with its default severity and marks rules that override it. -### Output - -- **`sctx validate`** — the summary counts rule violations per severity, and details are grouped by severity with each line showing the rule's category and id. -- **`sctx validate --json`** — `errors` includes only `error` severity rule violations (keyed `rule:`, with a `severity` field). A `ruleViolations` array lists every violation with `file`, `ruleId`, `category`, `severity` and `description` (`file` is empty for global rules). -- **`sctx validate-file`** — `error` severity violations appear under `errors`; `warning` and `info` violations appear under `warnings`. Rule entries carry a `severity` field. - -## Which nodes a rule runs on +## Selecting nodes - A rule with `scope: "global"` runs once for the space. - Other rules run once for each node. - `type` limits a rule to nodes with that `resolvedType`. - `when` limits a rule to nodes where a test returns `true`. See [Conditional rules](#conditional-rules-when). -## Expression context +### Conditional rules (`when`) + +`when` is a JSONata test that determines whether a rule runs on a node. It has the same context as `check` and runs before it. Use `when` when `type` cannot select the nodes a rule applies to: + +```json +{ "id": "ready-has-owner", "category": "workflow", "type": "task", + "description": "Ready tasks have an owner", + "when": "current.status = 'ready'", "check": "$exists(current.owner)" } +``` + +`when` costs almost nothing, so use it to stop an expensive rule from running where it does not apply. For a semantic rule, this saves a model call on each node that `when` excludes. + +An error in a `when` or `check` is reported as a non-gating `error` violation. + +## Writing expressions + +An expression rule's `check` (and `when`) is JSONata. This section covers what an expression can see and how to write checks. + +### Expression context Each evaluation receives: @@ -111,7 +104,7 @@ Useful resolved fields on nodes: Prefer `resolvedType` over raw `type` so aliases are handled correctly. -## Predicate scoping (`$$`) +### Predicate scoping (`$$`) Inside `nodes[...]`, bare names refer to each candidate node. Use `$$` to reference outer variables: @@ -119,23 +112,33 @@ Inside `nodes[...]`, bare names refer to each candidate node. Use `$$` to refere $count(nodes[resolvedParentTitle=$$.current.title and resolvedType='solution']) ``` -## Conditional rules (`when`) +### Common patterns -`when` is a JSONata test that determines whether a rule runs at all — effectively, under what circumstances the rule applies. It has the same context as `check` and runs on each node before the rule. Use `when` when `type` cannot select the nodes a rule applies to: +```jsonata +$exists(current.metric) = true +$count(current.sources) >= 1 +$count(nodes[resolvedType='outcome' and status='active']) <= 1 +current.status != 'active' or $exists(parent) = false or parent.status = 'active' +``` -```json -{ "id": "ready-has-owner", "category": "workflow", "type": "task", - "description": "Ready tasks have an owner", - "when": "current.status = 'ready'", "check": "$exists(current.owner)" } +#### Dates and times + +Dates and datetimes reach rules as ISO 8601 strings. Convert with `$toMillis()` to compare them, including date against datetime, and use `$millis()` for the current time: + +```jsonata +$not($exists(current.due)) or $toMillis(current.due) > $millis() +$toMillis(current.stale_after) > $toMillis(current.generated.at) ``` -`when` costs almost nothing, so use it to stop an expensive rule from running where it does not apply. For a semantic rule, this saves a model call on each node that `when` excludes. +A datetime without an offset (`2026-07-01T00:00:00`) is read in the machine's local time zone, so require an offset with `format: "date-time"` when results must not depend on where validation runs. `$toMillis()` also needs seconds: it cannot parse a minute-precision datetime such as `2026-07-01T09:15`, which `format: "date-or-date-time"` accepts. A value `$toMillis()` cannot parse makes the rule fail. -An error in a `when` or `check` is reported as a non-gating `error` violation. A rule with `scope: "global"` cannot have `when`. +The same applies to [filter expressions](concepts.md#filter-expressions): compare dates with `$toMillis()`, not as strings. -## Reusing rules +## Sharing rules -To share rules, put them in a schema's `$metadata.rules` and list that schema's `$id` in `$metadata.imports`. An import brings the whole schema's metadata, including its own imports: +To share rules, put them in a schema's `$metadata.rules` and list that schema's `$id` in `$metadata.imports`. An import brings the whole schema's metadata, including its own imports. Merged rules combine by `id`; a duplicate `id` is an error, and a later rule replaces an earlier one only with `override: true`. + +### Overriding rules ```json5 "$metadata": { @@ -143,13 +146,6 @@ To share rules, put them in a schema's `$metadata.rules` and list that schema's } ``` -## Merge and conflict behavior - -When metadata is merged from imports (imports in order, then the schema's own metadata): -- Rules are merged by `id`. -- A duplicate `id` is an error. -- A later rule may replace an earlier one only with `override: true`. - Example override: ```json5 @@ -173,18 +169,16 @@ in a way that makes its descendants' checks untrustworthy or wasteful to run: suppressed too, since its data is structurally invalid. - **Hierarchy structure violation** on a node (e.g. an invalid parent) — always gates the node's subtree, though the node's own rules still run. -- **A rule violation with `gatesDescendants: true`** (the default for `error`-severity rules; `warning` and `info` rules don't gate unless they set `gatesDescendants: true`) — gates the node's subtree. The - violating node's own other rules still run (they're the same cheap deterministic layer). - -Gating is per-subtree: once a node is gated, its descendants inherit the same gate (the -original blocker), rather than each generating a new one. Non-hierarchy nodes and -`scope: "global"` rules are never gated — cascade gating only applies within the hierarchy -DAG. +- **A rule violation with `gatesDescendants: true`** — gates the node's subtree, though the + violating node's own other rules still run. -This exists so that expensive or semantic downstream checks (see [Semantic rules](semantic-rules.md)) aren't wasted +Gating exists so that expensive or semantic downstream checks (see [Semantic rules](semantic-rules.md)) aren't wasted evaluating nodes whose ancestors are already known-bad, and so noise doesn't drown out the -actionable failure. Set `gatesDescendants: false` on an error-severity rule whose violations shouldn't suppress -checks further down the tree, or `gatesDescendants: true` on a warning/info rule that should. +actionable failure. It is per-subtree: once a node is gated, its descendants inherit the same +gate (the original blocker), rather than each generating a new one. Cascade gating only applies to nodes within the +hierarchy DAG. + +The default is `gatesDescendants: true` for `error`-severity rules and `false` for `warning` and `info` rules. Set it explicitly when that default doesn't fit. ### Coverage output @@ -200,16 +194,17 @@ don't depend on an ancestor being valid. ### Single-file validation -`sctx validate-file ` applies the same gating as `sctx validate`, so a file's rule results match its entry in a full run. Expression rules are evaluated for the whole space and the output is restricted to the file. Semantic rules are judged only for the file and its gating ancestors (see [Semantic rules](semantic-rules.md#single-file-validation)). If the file is gated, its rule checks are skipped and a `gated:` entry (kind `gated`, severity `info`) under `warnings` names the blocker, so a gated file doesn't read as clean. `--include-gated` evaluates the file's rules anyway. `--depth` is not supported. +`sctx validate-file ` applies the same gating as `sctx validate`, so a file's rule results match its entry in a full run. -### Depth-limited evaluation +- Expression rules are evaluated for the whole space; the output is restricted to the file. +- Semantic rules are judged only for the file and its gating ancestors (see [Semantic rules](semantic-rules.md#single-file-validation)). +- If the file is gated, its rule checks are skipped and a `gated:` entry under `warnings` names the blocker, so a gated file doesn't read as clean. `--include-gated` evaluates the file's rules anyway. -`--depth ` restricts validation to hierarchy nodes within `n` breadth-first levels from -the top (roots and orphans are depth 1). This is a manual, coarser control than cascade -gating — intended for staged workflows (e.g. an agent fixes the top level, then requests -the next). Excluded nodes are reported under `skippedByDepth`, separate from gated nodes. +## Controlling validation runs + +These options restrict which rules run, without changing rule definitions. -## Filtering rules +### Filtering rules `--rule-filter =[,]` runs only the rules that match. It works on `sctx validate` and `sctx validate-file`. @@ -220,24 +215,18 @@ the next). Excluded nodes are reported under `skippedByDepth`, separate from gat For example, `--rule-filter category=workflow,coherence --rule-filter severity=error` runs only workflow and coherence rules that can report an error. -## Common patterns - -```jsonata -$exists(current.metric) = true -$count(current.sources) >= 1 -$count(nodes[resolvedType='outcome' and status='active']) <= 1 -current.status != 'active' or $exists(parent) = false or parent.status = 'active' -``` +### Depth-limited evaluation -### Dates and times +`--depth ` restricts validation to hierarchy nodes within `n` breadth-first levels from +the top (roots and orphans are depth 1). This is a manual, coarser control than cascade +gating — intended for staged workflows (e.g. an agent fixes the top level, then requests +the next). Excluded nodes are reported under `skippedByDepth`, separate from gated nodes. -Dates and datetimes reach rules as ISO 8601 strings. Convert with `$toMillis()` to compare them, including date against datetime, and use `$millis()` for the current time: +## Output -```jsonata -$not($exists(current.due)) or $toMillis(current.due) > $millis() -$toMillis(current.stale_after) > $toMillis(current.generated.at) -``` +- **`sctx validate`** — the summary counts rule violations per severity, and details are grouped by severity with each line showing the rule's category and id. +- **`sctx validate --json`** — `errors` includes only `error` severity rule violations (keyed `rule:`, with a `severity` field). A `ruleViolations` array lists every violation with `file`, `ruleId`, `category`, `severity` and `description` (`file` is empty for global rules). +- **`sctx validate-file`** — `error` severity violations appear under `errors`; `warning` and `info` violations appear under `warnings`. Rule entries carry a `severity` field. -A datetime without an offset (`2026-07-01T00:00:00`) is read in the machine's local time zone, so require an offset with `format: "date-time"` when results must not depend on where validation runs. `$toMillis()` also needs seconds: it cannot parse a minute-precision datetime such as `2026-07-01T09:15`, which `format: "date-or-date-time"` accepts. A value `$toMillis()` cannot parse makes the rule fail. +Gated files: see [Single-file validation](#single-file-validation). -The same applies to [filter expressions](concepts.md#filter-expressions): compare dates with `$toMillis()`, not as strings. diff --git a/docs/semantic-rules.md b/docs/semantic-rules.md index ba192cd..9911045 100644 --- a/docs/semantic-rules.md +++ b/docs/semantic-rules.md @@ -28,14 +28,12 @@ This page assumes you know [Rules](rules.md). Semantic rules use the same fields 4. Run `sctx validate --semantic`. -## Turning semantic rules on +## Run `sctx validate` or `sctx validate-file` with semantic rule checking Semantic rules cost money and send node content to the judge's provider. Thus they run only when you ask, on each run: - `--semantic` runs them with the configured judge. -- `--judge ` runs them with that model provider instead. - -Both work on `sctx validate` and `sctx validate-file`. Config cannot turn semantic rules on. +- `--judge ` runs them with a specific model provider instead. ## Rule fields From d997be9adffc41f42a4144446a79c70967f4d36f Mon Sep 17 00:00:00 2001 From: Roger Barnes Date: Thu, 15 Oct 2026 23:37:38 +1100 Subject: [PATCH 22/22] style: format dates.test.ts per biome --- tests/validate/dates.test.ts | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/validate/dates.test.ts b/tests/validate/dates.test.ts index 2e1d5f2..dc11f9c 100644 --- a/tests/validate/dates.test.ts +++ b/tests/validate/dates.test.ts @@ -54,13 +54,12 @@ describe('dates and format validation', () => { }); describe('format: "date-time" schema validation', () => { - it.each([ - '2026-03-31T09:15:00Z', - '2026-03-31T09:15:00.250Z', - '2026-03-31T09:15:00+10:00', - ])('accepts %s', (value) => { - expect(validateNode({ type: 'note', title: 'Test', updated: value })).toBe(true); - }); + it.each(['2026-03-31T09:15:00Z', '2026-03-31T09:15:00.250Z', '2026-03-31T09:15:00+10:00'])( + 'accepts %s', + (value) => { + expect(validateNode({ type: 'note', title: 'Test', updated: value })).toBe(true); + }, + ); it.each([ '2026-03-31',