diff --git a/evaluation/standards-effectiveness/generated/suite-inputs.json b/evaluation/standards-effectiveness/generated/suite-inputs.json index d12e6834..4c5c9332 100644 --- a/evaluation/standards-effectiveness/generated/suite-inputs.json +++ b/evaluation/standards-effectiveness/generated/suite-inputs.json @@ -7347,7 +7347,7 @@ ] }, { - "digest": "sha256:8d8859b36d951bbf7a747da81d597df19b07b6653f1e36acc48b4b131b90c0a8", + "digest": "sha256:142cffd188781617e4641812e2f9bd33b99edf7bbbbd1d54e613bbcbcc8864a2", "path": "tools/standards_engine/README.md", "state": "present", "uses": [ @@ -7405,7 +7405,7 @@ ] }, { - "digest": "sha256:489fea805dbfd6f84bcb28374b467da192d56243100c0a58d568d9acf6180e08", + "digest": "sha256:146231132e8fbff19b5d917c1268bb75a0195cff4478d0e44df6bf04b8ba6aa2", "path": "tools/standards_engine/contracts/examples/a1-examples.json", "state": "present", "uses": [ @@ -7661,7 +7661,7 @@ ] }, { - "digest": "sha256:bc14652d3aa78ea400495d7f89fe9c4662c4de16acf759cac41d8279239a2a78", + "digest": "sha256:1a6d59145f4b8d43c67c23787fc450b60b569bbd65089fca9e6817b17848edae", "path": "tools/standards_engine/standards_engine/interaction_observer.py", "state": "present", "uses": [ @@ -7685,7 +7685,7 @@ ] }, { - "digest": "sha256:899c6bb6605a9b6e07bee7992c48120f3dd10fe2a8bf77fa9d20a35a3575ab22", + "digest": "sha256:49b9ba34111d7fff7dfb575ec3e7e29d8d2be259ab35ba03bd7af26feb5b8c89", "path": "tools/standards_engine/standards_engine/mcp.py", "state": "present", "uses": [ @@ -7875,7 +7875,7 @@ ] }, { - "digest": "sha256:7f51e403f03f5c57be8b1ca5736a79bcf139c87dd9eb4703ce3f7e59f1ba3d01", + "digest": "sha256:30eec6a6c472938048821b5669d17e054aa7cdee637647961b4012a380245e62", "path": "tools/standards_engine/tests/test_analysis.py", "state": "present", "uses": [ @@ -7887,7 +7887,7 @@ ] }, { - "digest": "sha256:2a5e6761f159c63e3c79f8fed0d90e1edcf987ea78a86fc9878a2866c840ef03", + "digest": "sha256:bcb26ee92ec9e34ca2e04280c9122f43a81a515dd195b73d53dc864d55649e06", "path": "tools/standards_engine/tests/test_coverage_publication.py", "state": "present", "uses": [ @@ -8344,7 +8344,7 @@ ] }, { - "digest": "sha256:828be47647e4332067893b79a508e85ed65ff1d524c41daf8f09fa41f6b7d2a2", + "digest": "sha256:e6f3adf5543fad7cbd24b31b68b21d08029e542e11d80d2899d30b2ced682452", "path": "tools/standards_snapshots/README.md", "state": "present", "uses": [ @@ -8441,7 +8441,7 @@ ] }, { - "digest": "sha256:bda178630b8226af0de5c42a7da6290ace2ed1745e3d2a99057deb0bae6ac617", + "digest": "sha256:6853039f09ef90b9d4ba9f54e4df7050b13b0a1941764af95250b44f7237ecae", "path": "tools/standards_snapshots/standards_snapshots/store.py", "state": "present", "uses": [ @@ -14116,7 +14116,7 @@ "path": "evaluation/standards-effectiveness/suite-registry.toml" }, "repository_index": { - "digest": "sha256:25f5d13e9650e10c5bcc198f50ea81aa00f90799ca3f9ba2a5a433a658262229", + "digest": "sha256:96dd89639c5b1a45d6c6692e969cea93338efdd3e5604fb2c2d30e6cfe254fd7", "uses": [ { "check": "python-package-contract", diff --git a/tools/standards_engine/MCP-OBSERVABILITY.md b/tools/standards_engine/MCP-OBSERVABILITY.md index 44acb8e4..d344509b 100644 --- a/tools/standards_engine/MCP-OBSERVABILITY.md +++ b/tools/standards_engine/MCP-OBSERVABILITY.md @@ -1,193 +1,293 @@ -# Operator MCP interaction observations +# MCP interaction recording -This optional local recorder observes the existing MCP boundary. It does not add -agent tools, modify domain outcomes or workflow authority, make model calls, retry -operations, or change the installed agent catalog. The qualification observer remains -**observer v3**, with its existing raw-event acquisition, assessment and exact-source -replay. The production recorder does not import the harness or rewrite verdicts. +Every Standards Engine MCP server process records how its tools are used, so the +interface can be improved from what agents actually send and receive. Recording is +always on, keeps about seven days, and is invisible to the agent. -The sanitized recorder selected by `--observe-database` and -`--observe-deployment` is the sole supported recording mode. The earlier raw -recorder and its `--observation-db` option are retired. Historical raw databases -retain their original schema and remain separate evidence; configure a new -dedicated database for sanitized observations. +- **Always on.** No flag is needed, in either purpose, for any client. +- **Invisible.** Recording adds no tool, catalog entry or instruction text and + changes no result. It never raises into a request. A recording problem is + reported only on the server's stderr. +- **No dropped records.** A call that cannot be written at once waits in memory + and is written by a later call or when the process closes. +- **Seven days.** Older calls are removed automatically. -## Configuration and authority +This is the only MCP recorder. The qualification observer remains **observer v3** +with its own raw-event evidence and exact-source replay; the recorder neither +imports that harness nor rewrites its verdicts. -Recording is off unless the operator explicitly supplies **both** options below. -The commands illustrate a new operator-owned installation, not instructions to -enable an existing one without its operator's approval. Replace paths and the -nonsensitive deployment label with operator-selected values: +## Where the file is + +```text +${XDG_DATA_HOME:-~/.local/share}/standards-engine/observations/interactions-v2.sqlite3 +``` + +The server creates that directory with mode 0700 if it is missing and the file with +mode 0600. Existing directories and permissions are never changed. + +| Launch option | Default | Meaning | +| --- | --- | --- | +| `--observe-database PATH` | the file above | Record somewhere else. The directory must already exist; it is not created. | +| `--observe-deployment LABEL` | `local` | A nonsensitive label stored with each server process. | +| `--observe-max-calls N` | 10000 | 16–100000 calls kept, across all processes. | +| `--observe-max-age-seconds N` | 604800 (7 days) | 60–31536000 seconds since a call began. | +| `--observe-max-bytes N` | 33554432 (32 MiB) | 1048576–1073741824 bytes of database file. | +| `--no-observe` | recording on | Do not record this process. | + +## Turning it off + +Either of these stops recording for a server process. Nothing is created or opened. ```sh -python3 -m tools.standards_engine.standards_engine.mcp \ - --repo-root /operator/standards --purpose authoring \ - --observe-database /operator/private-diagnostics/interactions.sqlite \ - --observe-deployment qualification-fixture +python3 -m tools.standards_engine.standards_engine.mcp --purpose application --no-observe +STANDARDS_ENGINE_OBSERVE=off python3 -m tools.standards_engine.standards_engine.mcp --purpose application ``` -The database directory must already exist, be operator controlled and exclude -concurrent untrusted replacement of the directory or its ancestors. This initial -recorder supports POSIX regular local files and SQLite rollback journals only. -It refuses final-component symlinks, hardlinked files, existing group/world-readable -files, unsupported journal modes and unsupported or unrelated database schemas. -New files use mode 0600. It does not chmod existing files, create directories or -use an Engine domain database. Do not place it in a repository, tracked directory, -network share or an agent-visible artifact directory. The operator must select -appropriate disk access, encrypted storage/backup and disposal arrangements. - -**Same-user filesystem access is not a security boundary.** A process with the -same OS authority can read, modify or remove the recorder database and stderr. -Tool-catalog invisibility does not create confidentiality or tamper evidence. -The recorder therefore excludes payload values regardless of filesystem modes. -The deployment label must not contain account identities, credentials or secrets. -No network exporter is included. Export/sharing requires separate operator review. - -Policy defaults and valid ranges: - -| Option | Default | Range / meaning | +The environment variable wins over every `--observe-*` option. Tests and harnesses +that start a server pass `--no-observe`, set the variable, or name a temporary +`--observe-database`, so they never write to a developer's own file; a test +enforces this. Constructing +`MCPServer` directly in Python records only when it is given a `RecorderConfig`. + +## What is recorded + +One row per server process (`sessions`), one per request (`interactions`) and one +per response (`events`). Notifications are not recorded. + +**Per server process** + +- A random process id, start time, purpose, interface version and the catalog, + schema and implementation digests the process actually loaded. +- The deployment label, and the client name and version from `initialize` + `clientInfo` (for example `codex-mcp-client 0.163.0`). +- Per-tool input and output schema digests, stored once per distinct catalog. +- How many calls this process pruned, and how many rows it lost (see below). + +**Per request** + +- Time, order within the process, method, installed operation name, and whether the + response was written to stdio or only constructed. +- `input_shape`: the argument structure with every value replaced by its type. +- `arguments`: the values listed in the next section, and nothing else. +- `request_bytes` (the whole JSON-RPC line, client metadata included) and + `arguments_bytes` (the `arguments` object alone). +- `expected_catalog` as `matches`, `differs` or `unspecified`; the operation a + `describe_input`/`describe_output` call asked about. +- `thread`: a 16-hex-digit keyed hash of the client's thread or session id, when + the request metadata carries one, and `origin`: `main`, `subagent` or `unknown`. + Codex supplies both. A client that sends neither is recorded with no thread and + `unknown`; reports then treat the server process as the thread. + +**Per response** + +- Time, duration, `response_bytes` (the whole line) and `result_bytes` (one + serialization of `structuredContent`; the line carries that payload twice). +- An outcome word: the result's own `outcome` or `status` (`complete`, + `needs-facts`, `needs-action`, `requires-change`, `ready`, `draft`, `applied`, + `stale`, `rejected`, `recovery-required`, `invalid`, `unavailable`, `unsupported`, + `unauthorized`), or `input-invalid`, `tool-exception`, a protocol error class, + `response`, or `unknown` for an interrupted call. +- Where input validation failed: schema field names and the constraint keyword. +- `result`: `kind`, `status`, `code` and `outcome`; for a route, the selected + target IDs, the IDs of facts still unresolved, and the content page's offset, + total, item count and item IDs; for reads, the target IDs returned. + +### Argument values that are kept + +Values are kept only for names on an explicit list per operation. Any other +argument appears only as a type in `input_shape`. Strings are cut at 200 +characters, and lists and fact maps at 32 entries, with a `[truncated]` marker. A +whole `arguments` object over 8 KB keeps only its top-level scalars and +`truncated`. + +| Operation | Kept as sent | +| --- | --- | +| `read` | `target`, `detail`, `include_coverage`, `include_routing` | +| `read_many` | the same four names for each item | +| `route` | `detail`; `content.limit` and `content.offset`; every fact name and value | +| `related` | `target`, `direction`, `transitive`, `groups` | +| `relationship_groups` | `limit`, `offset` | +| `describe_input`, `describe_output` | `operation`, `selector`, `limit`, `offset` | +| `inspect` | the handle's `kind` and `child_kind` | +| `query`, `query_proposal`, `preview_application` | the nested request's `kind`, the `read`/`related` names above, and route facts; the revision handle's `kind` | +| `routing_facts`, `runtime_info` | nothing beyond the snapshot note below | + +A supplied `snapshot` is recorded as a 12-hex-digit hash of its id, so reuse of +one snapshot is visible without keeping the handle. + +**Route facts are kept exactly as the agent sent them**, including names and +values the routing vocabulary does not contain. An agent that invents a fact such +as `python`, or passes a sentence as a `task` fact, is the most useful signal this +recording has, so up to 200 characters of that text are stored. + +Every other operation is an authoring operation (`propose`, `revise`, +`resolve_workflow`, `resolve_many`, `review`, `apply`, `recover`, `analyze`, +`workflow_status`, `workflow_details`, `resume`, `maintain_evidence` and the +advanced native operations). For these, a value is kept only when all of the +following hold: + +- its name is declared by the installed input schema at that position; +- it is not inside `body`, `content`, `text`, `prompt`, `message`, `rationale`, + `reason`, `evidence`, `explicit_exclusions`, `details`, `metadata` or a + credential-like field; +- its name is one of: `kind`, `child_kind`, `detail`, `action`, `section`, + `result`, `decision`, `owner`, `disposition`, `relation`, `artifact_kind`, + `authority`, `level`, `role`, `origin`, `lifecycle`, `direction`, `id`, `policy`, + `standard`, `consumer`, `target`, `source_policy`, `subject`, `evidence_owner`, + `after_policy`, `rule`, `fact`, `requires`, `specializes`, `successors`, + `predecessors`, `groups`, `limit`, `offset`, `apply`, `transitive`; +- it is a boolean, a small number, or a string shaped like a canonical identifier + (letters, digits and `._:/-`, no spaces) that contains no digest, UUID or other + long hexadecimal run. + +So an edit is recorded as, for example, +`{"kind":"revise-policy-unit","policy":"workflow.planning.written-plan-applicability"}` +and a handle as `{"kind":"analysis-handle"}`. + +### What is never recorded + +- Standards text, bodies, titles, summaries, rationales, intents, headings, + conditions, evidence text, file paths, or any other prose in an authoring call. +- Result content: no standard text, messages, details or summaries. +- Handle ids, digests, request ids and progress tokens. +- Raw thread, session or turn ids, model names, workspace paths, repository URLs, + or anything else in request `_meta`. +- Argument names the installed schema does not declare (they are counted), other + than route fact names. +- Tool names the server does not install, exception messages and tracebacks. +- Credentials or tokens, under any name. + +## The seven-day window + +Each new call first removes calls that began more than seven days ago, then the +oldest calls beyond the count limit. When the file is within one sixteenth of its +byte limit the oldest sixteenth of calls is removed as well, so a full file never +stops new recording. A call and its response are removed together, and a server +process row goes when its last call does (one that carries a loss count stays +until it is seven days old). + +Removal happens only while something is recording: an idle file keeps its last +contents until the next call. Deleted rows are overwritten in the file +(`secure_delete`), but backups, filesystem snapshots and storage remnants are +outside the recorder's reach. The file does not shrink; freed space is reused. + +## No dropped records, and what can still be lost + +Each request is written before it is dispatched and each response after it has +been delivered. If the database is busy or unavailable, the row stays in an +in-memory backlog of at most 1,000 rows and the call proceeds: + +- A busy database is waited for at most 20 ms per attempt, and 250 ms once at + process start. The backlog is written by the next request or response, and at + orderly close with a wait of up to 5 seconds. +- Recording is never switched off. After any other storage error the connection + is reopened on a later call. +- Several processes creating the file at the same instant all succeed: the first + publishes the whole schema in one commit and the others find it complete. + +What cannot be recovered, and how it shows: + +| Situation | Effect | Visible as | | --- | --- | --- | -| `--observe-max-calls` | 10000 | 16–100000 retained interaction starts, across all processes | -| `--observe-max-age-seconds` | 604800 | 60–31536000 seconds since begin | -| `--observe-max-bytes` | 33554432 | 1048576–1073741824 bytes, SQLite main-file page quota | - -All processes sharing a database must use the same privacy/retention policy. -Each connection enforces its configured page quota; mixed policies do not establish -a shared minimum. Rollback journals may temporarily add up to roughly another -main database's size, plus SQLite bookkeeping and bounded report/sort working space. -The directory's filesystem quota remains the final bound on all files. No WAL, -background thread, queue or telemetry write retry is introduced. Main files do not -shrink on deletion: SQLite reuses freed pages. No automatic VACUUM/rebuild occurs. - -## Persisted contract and lifecycle - -Schema v1 has three owners within one recorder database: - -- `sessions`: exact startup runtime metadata (process instance UUID, interface, - purpose, catalog/schema/implementation digests), deployment label and per-tool - input/output schema digests. These bind the actually loaded source and effective - eager/on-demand catalog. They do not prove a client refreshed its catalog. -- `interactions`: immutable begin evidence, recorder UUID, session-local ordinal, - wall-clock begin instant, method/installed operation, sanitized input shape and - its shape-only digest, safe discovery target, expected-catalog match state, - request byte measurement and delivery scope. This row is the begin event. -- `events`: at most one independent end/error/incomplete observation per begin, - with monotonic elapsed milliseconds, wall-clock finish instant, response bytes, - fixed outcome class and sanitized validation locations/constraint keywords. - -Each mutation owns `BEGIN IMMEDIATE`/commit/rollback. Schema admission is checked -under the same lock, including exact owned DDL and application ID. Zero-length new version-0 -SQLite files migrate transactionally once to v1; v1 reopens without rewriting -observations. Future versions, unrelated/partial schemas and corruption are -unavailable, never deleted or replaced. Migration interruption rolls back under -SQLite recovery. Backup a stopped database before any future supported migration; -no downgrade migration or automatic restoration is currently provided. - -Stdio begins are committed before dispatch. End/error evidence is committed only -after the ordinary response write and flush return. `stdio-flushed` does not prove -the client received/processed the response. The direct Python `dispatch` path uses -`constructed` scope, which proves only that a response was produced. Parsing errors -and malformed requests with responses are observed; valid notifications have no -request/response pair and are excluded. Client IDs and clientInfo are not retained. - -A caught tool exception has a fixed `tool-exception` category; domain outcome may -be unknown. Typed validation rejections retain `input-invalid` plus safe field -locations. Other protocol/domain classes use fixed vocabularies, never exception -messages, tracebacks or arbitrary result codes. Transport/interruption errors -append `incomplete/unknown` if storage remains available. A killed process or a -failed terminal write leaves an unmatched begin. Reports label it unknown; startup -does not complete old calls or presume other live recorders died. Telemetry cannot -prove a domain effect did or did not commit and cannot authorize a mutation retry. - -Age and count retention run transactionally on each new observed request. Expiry -is event-driven: an idle/stopped database retains expired records until recording -resumes or the operator disposes of it. Starts and their terminal evidence are -removed together; unreferenced session manifests are removed in the same -transaction. Retention can evict an in-flight call from another process. Its late -terminal is counted as dropped and recording continues; no start is recreated. -Retention can remove evidence before a later success, so history/repair denominators -are limited to retained observations. Per-session prune counters themselves expire -with their session; they are not installation-lifetime loss totals. Secure deletion -is enabled for recorder deletes; it does not erase backups, filesystem snapshots, -SQLite journal history or storage-device remnants. - -## Privacy projection - -No credentials/tokens, scalar input values, raw request IDs, complete prompts, -model transcripts, standards/repository bodies, domain handles or arbitrary tool -outputs are stored or hashed. Every scalar input becomes only its type. - -At each structural location, only property names declared by the installed input -schema can appear. Recursive references/unions supply possible property names; -this is a disclosure projection, not another input validator. Dynamic map keys -and unknown names are removed and counted. Sensitive subtrees (`body`, `content`, -`text`, `prompt`, `message`, `rationale`, `reason`, `evidence`, `details`, `metadata`, -credentials/password/token/secret/authorization/api_key/environment/env) are replaced -by a redacted marker. Other input fields still retain no scalar values. Shapes -have at most 64 visited nodes, depth 5, four sampled array items and 8192 serialized -ASCII bytes. Truncation is explicit. Validation feedback is limited to eight -issues, depth-five schema-owned pointer components, wildcard indices/dynamic keys -and fixed constraint keywords; messages and offending values are omitted. - -An input-shape digest groups structural samples, not raw values or exact semantic -variants. Two scalar variants can deliberately have the same shape. Discovery -operation values are retained only on exact installed-operation membership. -Expected catalogs become only unspecified/matches/differs. Selector values are -not retained. There is no legitimate cross-process session/task ID in the current -MCP server: reports do not infer continuation from prompts or domain handles. - -## Health and recovery - -Operator stderr receives fixed JSON health at recorder startup, first degradation, -retention-dropped terminal observations, and orderly close. It carries state, -fixed reason, starts/terminals recorded, observations dropped and interactions -pruned in that process. Stderr failure itself is guarded. Health is not advertised -in the agent catalog or appended to an Engine result. - -Unavailable, locked (20 ms busy budget), full, unsafe, unsupported or corrupt -storage disables recording for that process and reports degraded health. Existing -begin evidence remains independent; no completion is invented. No further writes -or retries occur. The MCP operation still returns its original outcome and runs -once, including a domain result committed before the telemetry failure. A failed -start means that interaction may have no retained evidence at all. Health counters -cover attempted observations, not an exact census of lost interactions. - -The operator must monitor stderr/process supervision independently of SQLite, -fix the owned storage condition and restart the MCP process to recover recording. -Preserve a corrupt/unrecognized database for diagnosis; select a separately -approved fresh path rather than silently rebuilding it. A report cannot recover -health messages that were unavailable/lost with stderr or process termination. - -## Read-only inspection +| Process killed during a call | The request has no response row. | "without an outcome" in the report | +| Backlog over 1,000 rows | The oldest waiting rows are dropped. | `lost` on the process row and in the report | +| A response whose request row was already pruned or dropped | The response is not stored. | counted in the same `lost` | +| Database unreachable until the process ends | Its rows are never written. | stderr only: nothing can be stored | +| Process killed while rows are waiting | The waiting rows are gone. | not visible | + +In ordinary use the write takes well under a millisecond, so the last two cases +need a database that stays locked or broken for the life of a process. + +## Storage + +The database uses SQLite write-ahead logging. That is what keeps recording out of +the way: a commit holds the write lock for microseconds instead of the several +milliseconds a synced rollback journal needs, and reading a report never blocks a +server. For the person reading the data it means: + +- While any server is running there are three files: the database, `-wal` and + `-shm`. Recent calls may exist only in `-wal`. Read through SQLite (the report + does) or copy all three; do not copy the database file alone. +- The file must be on a local filesystem. Network shares are not supported. +- After an operating-system crash or power loss the last few commits may be + missing. A killed or crashed server process loses nothing that was committed. +- `-wal` is usually a few megabytes on top of the byte limit and is removed when + the last server closes. + +The recorder accepts only a new empty file or its own schema v2. It refuses +symlinks, hardlinked files, files readable by group or others, unrelated SQLite +files, files larger than the byte limit, and other schema versions, and leaves +them untouched. That includes schema v1 files written by earlier releases: they +are historical evidence, are never migrated or deleted, and are no longer written +because the default file name is new. + +**Same-user file access is not a security boundary.** Any process running as the +same user can read or change the recording. The thread hash key is stored in the +same database: it keeps raw client ids out of the file and out of shared reports, +but it does not hide a thread from someone who holds both the file and the id. Do +not put the file in a repository or an agent-visible directory, and review a +report before sharing it: route facts can contain agent-written text. + +## Health messages + +A healthy recorder writes nothing. When recording changes state, one line goes to +the server's stderr: + +```text +standards-engine observer {"backlog":6,"lost":0,"pruned_interactions":0,"reason":"storage-locked","recorded_starts":2,"recorded_terminals":2,"state":"degraded"} +``` + +`degraded` means rows are waiting in memory for the stated reason; it does not mean +recording stopped. A line with `healthy` follows when they have been written, and a +final line is written at close if anything was reported or lost. Reasons are fixed +words: `storage-locked`, `storage-unavailable`, `storage-full`, `storage-corrupt`, +`storage-error`, `unsafe-database-file`, `unsupported-database-schema`, +`invalid-database-schema`, `nonpristine-database`, `database-over-budget`, +`unsupported-journal-mode`, `unsupported-filesystem` and `recorder-error`. Paths, +SQL and payloads never appear. + +## Reading the recording ```sh -python3 -m tools.standards_engine.standards_engine.interaction_observer \ - --database /operator/private-diagnostics/interactions.sqlite --limit 100 +python3 -m tools.standards_engine.standards_engine.interaction_observer +python3 -m tools.standards_engine.standards_engine.interaction_observer --since-days 2 +python3 -m tools.standards_engine.standards_engine.interaction_observer --format json --limit 100 ``` -This operator CLI opens an existing v1 database in SQLite read-only/query-only -mode and uses one read snapshot. It never creates, migrates, prunes or modifies -observations. JSON goes only to the operator's stdout; no output file is created. -Each group/session/histogram section is bounded by `--limit` (1–1000), with explicit -truncation flags. Sequence analysis scans at most the latest 10000 retained starts; -its scan count and truncation are reported. Aggregate queries scan the bounded -retained database (at most the configured 100000 starts). Sessions show the latest -bounded set in chronological order; catalog comparisons cover those shown manifests. - -The report provides operation usage, error distribution by structural input shape, -sanitized field/constraint failures, observed latency and serialized bytes with -measurement counts, sanitized shapes for successful and failed calls, session -identity and catalog additions/removals/schema changes. Totals include all retained -starts/terminals; operation/failure/shape rows identify their delivery scope and -have their own counts. An end is a non-error MCP response, not evidence the domain -workflow completed. Missing measurements are null and have separate denominators. - -Sequence histograms count preceding observed errors/discoveries before a non-error -tool response, by session, delivery scope and operation. Their denominator is the -reported number of non-error responses in the bounded scan. This is a diagnostic -candidate for investigating repair loops, not proof that errors caused discovery, -that a schema was acquired before use, or that separate tasks share agent intent. -No raw trace or redacted request can be reconstructed. Bytes are JSON/stdio bytes, -not model tokens or provider billing. Preserve independent qualification evidence -for observer-v3 replay and exact behavioral assessment; do not export actual logs -with code or automatically rewrite contracts from these summaries. +`--database PATH` reads another file. The command opens the database read-only, +uses one consistent snapshot and never creates, changes or prunes a recording. +SQLite may leave an empty `-wal` and a `-shm` file beside a database it has read. + +The default output is about one screen of text: + +- calls by client, purpose and interface version; +- per operation: calls, share rejected, p50/p90/max duration and p50/max result + size; +- the most frequent rejections with their code, the field that failed and the + value that was sent; +- route fact names and values the server refused and never accepted in the window; +- the most-read targets, counting `read`, `read_many` and route content pages; +- how often a thread read a target it had already read; +- routes sent without a snapshot; +- threads, tool calls and repeated reads for main agents, subagents and clients + that identify neither; +- calls without an outcome, rows lost and calls pruned. + +`--format json` gives every section in full, plus the structural input shapes, +the latest server processes, catalog differences between them and the +error-then-success sequences. `--limit` (1–1000) bounds the rows per section and +truncation is flagged. + +How to read it: + +- "Refused and never accepted" is inferred from the recording, not checked against + a vocabulary: a fact name counts when its route was rejected for a fact and the + server neither accepted that name nor listed it as an unresolved fact in the + window. A value counts when the rejection points at its fact and that value was + never accepted. +- A thread is the hashed client thread, or the server process when the client sent + none. Repeat reads are counted within a thread. +- An unmatched call has an unknown outcome. The recording cannot show whether a + mutation committed and must not be used to decide a retry. +- Bytes are serialized JSON, not model tokens or billing. +- Recorded requests cannot be replayed. Keep qualification evidence for that. diff --git a/tools/standards_engine/PURPOSE-SEPARATION.md b/tools/standards_engine/PURPOSE-SEPARATION.md index b112edf0..02e66413 100644 --- a/tools/standards_engine/PURPOSE-SEPARATION.md +++ b/tools/standards_engine/PURPOSE-SEPARATION.md @@ -124,9 +124,14 @@ two-entry and 32 MiB accounted-retention limit shared by snapshot and draft projection entries. Snapshot keys contain the exact verified captured paths, bytes and source revision, plus the installed compiler function. Python dictionary equality distinguishes hash collisions. Each new operation -opens and validates its store, checks snapshot lifecycle and verifies complete -content identity before lookup. Missing, quarantined, purged, replaced or corrupt -storage therefore remains observable. Equal verified inputs may share pure +opens its store, checks its application, version and exact schema, checks snapshot +lifecycle and verifies complete content identity before lookup. Missing, +quarantined, purged or replaced storage, and content that no longer matches its +identity, therefore remain observable. The store's complete SQLite +integrity/foreign-key audit is not repeated by every operation: a process runs it +on its first open of a store file, as described under +[Snapshot admission](../standards_snapshots/README.md#admission-and-maintenance-work). +Equal verified inputs may share pure compilation even when obtained from distinct store roots; store-specific handles, access checks and result envelopes remain fresh. No store facts enter the cache. @@ -185,8 +190,9 @@ Review, candidate verification, application and recovery retain their existing fresh replay paths. New snapshot capture retains both independent compiler passes. Main advancement affects new captures, never the base of an existing draft. Old explicit revisions retain historical meaning and can have live stale status. -Corruption, quarantine, purge or missing/replaced stores reject before projection -reuse. Server restart reconstructs cold, and close releases both entry kinds. +Stored content that fails validation, quarantine, purge or missing/replaced stores +reject before projection reuse. Server restart reconstructs cold and repeats the +store's complete SQLite audit; close releases both entry kinds. ### Incremental successor construction diff --git a/tools/standards_engine/README.md b/tools/standards_engine/README.md index 3e0b3fd7..8b42403c 100644 --- a/tools/standards_engine/README.md +++ b/tools/standards_engine/README.md @@ -38,10 +38,13 @@ normative text, policy subjects, consumer relationships and other certificates remain intact. It supplies no replacement coverage claim. Restart/reconnect after installing the coordinated source and generated contracts; preserve stores. -Optional sanitized operator measurements use `--observe-database` with -`--observe-deployment`; see [MCP observability](MCP-OBSERVABILITY.md). Recording is -disabled by default and preserves the existing qualification observer v3 and -agent-visible interface. This is the sole supported MCP recorder. +Every MCP server process records how its tools are used in an operator-only local +SQLite file and keeps about seven days. See +[MCP interaction recording](MCP-OBSERVABILITY.md) for exactly what is and is not +recorded, where the file is, and how to turn it off (`--no-observe` or +`STANDARDS_ENGINE_OBSERVE=off`). Recording changes nothing an agent can see and +preserves the existing qualification observer v3. This is the sole supported MCP +recorder. ## Purpose separation (interface 31) @@ -336,6 +339,10 @@ The process-owned compilation cache also retains exact captured-content identity proofs. Identity and compilation for one capture share one LRU entry and the existing byte budget. The Snapshot codec still defines identity, and every read reloads actual bytes, verifies file integrity and checks the stored identity and current lifecycle. +These per-read checks are separate from the store's complete SQLite +integrity/foreign-key audit, which a process runs on its first open of a store file +and not on each store-backed call; see +[Snapshot admission](../standards_snapshots/README.md#admission-and-maintenance-work). Focused authoring operations reuse canonical revision decoding through `ProposalMaterials`; the single record is released on success or exceptional exit. @@ -420,8 +427,15 @@ after installation without changing launch arguments or deleting stores. The earlier raw invocation recorder was retired on 2026-10-22. Its `--observation-db` launch option is no longer accepted. Historical raw databases remain operator-owned evidence and are not converted, reused or deleted. -Select a new dedicated database with `--observe-database` and a nonsensitive -`--observe-deployment` label. Follow the [sanitized recorder's operating contract](MCP-OBSERVABILITY.md) + +The opt-in sanitized recorder that replaced it (schema v1, selected with +`--observe-database` and `--observe-deployment`) is in turn replaced by always-on +recording in a new default file (schema v2). Its v1 databases are also left +untouched. A registration that still names a v1 file with `--observe-database` +is refused that file and records nothing until the option is removed; remove both +options to use the default location. `--observe-database` now only overrides the +location and `--observe-deployment` defaults to `local`. Follow the +[recording contract](MCP-OBSERVABILITY.md) for permissions, retained data, retention and reports. The [original observation plan](../../docs/plans/agent-interaction-observation/plan.md) retains its historical acceptance and qualification records. diff --git a/tools/standards_engine/contracts/examples/a1-examples.json b/tools/standards_engine/contracts/examples/a1-examples.json index a697ac1f..1f256f60 100644 --- a/tools/standards_engine/contracts/examples/a1-examples.json +++ b/tools/standards_engine/contracts/examples/a1-examples.json @@ -1274,7 +1274,7 @@ "value": { "kind": "input-contract-result", "purpose": "application", - "interface_version": 47, + "interface_version": 48, "catalog_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "operation": "runtime_info", "root": "RuntimeInfoCall", @@ -1420,7 +1420,7 @@ "value": { "kind": "output-contract-result", "purpose": "authoring", - "interface_version": 47, + "interface_version": 48, "catalog_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "operation": "review", "dialect": "https://json-schema.org/draft/2020-12/schema", diff --git a/tools/standards_engine/standards_engine/interaction_observer.py b/tools/standards_engine/standards_engine/interaction_observer.py index 197b060a..12e997f4 100644 --- a/tools/standards_engine/standards_engine/interaction_observer.py +++ b/tools/standards_engine/standards_engine/interaction_observer.py @@ -1,40 +1,61 @@ -"""Opt-in operator observations. Never an Engine store or an agent tool. +"""Always-on operator observations. Never an Engine store or an agent tool. The recorder owns sanitization, SQLite transactions, retention and fail-open -health. No caller-supplied scalar is persisted. A missing terminal event means -unknown outcome; restarting a recorder never manufactures a completion. +health. Caller values are persisted only through the explicit allow-lists below. +A row that cannot be written waits in a bounded in-memory backlog. A missing +terminal event means unknown outcome; a recorder never manufactures a completion. """ from __future__ import annotations import argparse -from collections import Counter +from collections import Counter, deque from dataclasses import dataclass +from datetime import datetime, timezone import hashlib +import hmac +from itertools import islice import json +import math import os from pathlib import Path import re +import secrets import sqlite3 import stat import sys import time -from typing import TextIO +from typing import Iterable, TextIO import uuid -SCHEMA_VERSION = 1 +SCHEMA_VERSION = 2 APPLICATION_ID = 0x53454F42 # SEOB, separate from every Engine domain database. +DEFAULT_DATABASE_NAME = 'interactions-v2.sqlite3' +DEFAULT_DEPLOYMENT = 'local' DEFAULT_MAX_CALLS = 10_000 DEFAULT_MAX_AGE_SECONDS = 7 * 24 * 60 * 60 DEFAULT_MAX_BYTES = 32 * 1024 * 1024 -BUSY_TIMEOUT_MS = 20 +BUSY_TIMEOUT_MS = 20 # Lock wait per begin/finish; the backlog delivers later. +OPEN_TIMEOUT_MS = 250 # Lock wait at process start, before any request is served. +CLOSE_TIMEOUT_MS = 5_000 # Lock wait for the last delivery at orderly close. +LOCK_POLL_SECONDS = 0.0005 +MAX_BACKLOG = 1_000 MAX_SHAPE_NODES = 64 MAX_SHAPE_DEPTH = 5 MAX_SHAPE_BYTES = 8 * 1024 MAX_ARRAY_ITEMS = 4 +MAX_VALUE_CHARS = 200 +MAX_VALUE_ITEMS = 32 +MAX_VALUE_DEPTH = 3 +MAX_VALUE_NODES = 512 +MAX_ARGUMENT_BYTES = 8 * 1024 +MAX_RESULT_IDS = 128 MAX_FEEDBACK_ISSUES = 8 MAX_FEEDBACK_POINTER_CHARS = 512 +MAX_THREAD_CHARS = 256 MAX_REPORT_ROWS = 1000 MAX_SEQUENCE_SCAN = 10_000 +TEXT_ROWS = 10 +TRUNCATED = '[truncated]' METHODS = frozenset({'initialize', 'tools/list', 'tools/call', 'ping'}) SENSITIVE_FIELDS = frozenset({ 'body', 'content', 'text', 'prompt', 'message', 'rationale', 'reason', @@ -45,20 +66,63 @@ 'type', 'required', 'additionalProperties', 'enum', 'const', 'oneOf', 'anyOf', 'allOf', 'not', 'minimum', 'maximum', 'minItems', 'maxItems', 'minLength', 'maxLength', 'pattern', 'format', 'uniqueItems', + 'fact', 'evidence_ref', # The Engine's own fixed keywords. }) -OUTCOMES = frozenset({'invalid', 'unsupported', 'unavailable', 'stale', - 'rejected', 'recovery-required', 'needs-action', - 'requires-change', 'complete', 'ready'}) +OUTCOMES = frozenset({'invalid', 'unsupported', 'unavailable', 'unauthorized', 'stale', + 'rejected', 'recovery-required', 'needs-action', 'needs-facts', + 'requires-change', 'complete', 'ready', 'draft', 'applied'}) + +# Values that may be persisted, by operation. A name absent from these lists is +# never stored as a value; it remains a type in input_shape. +_PAGE = ('limit', 'offset') +_READ = ('target', 'detail', 'include_coverage', 'include_routing') +_RELATED = ('target', 'direction', 'transitive', 'groups') +_DESCRIBE = ('operation', 'selector', *_PAGE) +# Navigation: values exactly as sent. Structured members (snapshot, read_many +# items, route content and facts, handles, nested requests) are projected by +# navigation_values() for the operations named here. +NAVIGATION = { + 'read': _READ, 'read_many': (), 'route': ('detail',), 'routing_facts': (), + 'related': _RELATED, 'relationship_groups': _PAGE, 'inspect': (), + 'describe_input': _DESCRIBE, 'describe_output': _DESCRIBE, 'runtime_info': (), + 'query': (), 'query_proposal': (), 'preview_application': (), +} +HANDLES = {'inspect': 'handle', 'query_proposal': 'revision', 'preview_application': 'revision'} +REQUESTS = {'route': ('kind',), 'read': ('kind', *_READ), 'related': ('kind', *_RELATED)} +# Every other operation is authoring: only these names, and only values shaped +# like a canonical identifier, an enum word, a boolean or a small integer. +IDENTIFIER_FIELDS = frozenset({ + 'kind', 'child_kind', 'detail', 'action', 'section', 'result', 'decision', 'owner', + 'disposition', 'relation', 'artifact_kind', 'authority', 'level', 'role', 'origin', + 'lifecycle', 'direction', + 'id', 'policy', 'standard', 'consumer', 'target', 'source_policy', 'subject', + 'evidence_owner', 'after_policy', 'rule', 'fact', 'requires', 'specializes', + 'successors', 'predecessors', 'groups', + 'limit', 'offset', 'apply', 'transitive', +}) +IDENTIFIER = re.compile(r'[A-Za-z0-9][A-Za-z0-9._:/-]{0,199}') +OPAQUE = re.compile(r'[0-9a-fA-F]{16}|[0-9a-f]{8}-[0-9a-f]{4}-') # Digests, handle ids, UUIDs. +RESULT_KIND = re.compile(r'[a-z][a-z0-9-]{0,63}') +RESULT_CODE = re.compile(r'[A-Z][A-Z0-9_]{0,31}(\.[A-Z][A-Z0-9_]{0,47}){1,3}') SCHEMA_DDL = ( - 'CREATE TABLE sessions (id TEXT PRIMARY KEY, started REAL NOT NULL, runtime TEXT NOT NULL, deployment TEXT NOT NULL, catalog TEXT NOT NULL, pruned INTEGER NOT NULL DEFAULT 0)', - 'CREATE TABLE interactions (id TEXT PRIMARY KEY, session TEXT NOT NULL REFERENCES sessions(id), ordinal INTEGER NOT NULL, started REAL NOT NULL, method TEXT NOT NULL, operation TEXT NOT NULL, input_shape TEXT NOT NULL, shape_digest TEXT NOT NULL, discovery_operation TEXT, catalog_state TEXT NOT NULL, request_bytes INTEGER, scope TEXT NOT NULL, UNIQUE(session, ordinal))', - "CREATE TABLE events (interaction TEXT PRIMARY KEY REFERENCES interactions(id) ON DELETE CASCADE, phase TEXT NOT NULL CHECK(phase IN ('end','error','incomplete')), finished REAL NOT NULL, duration_ms REAL NOT NULL, response_bytes INTEGER, outcome TEXT NOT NULL, feedback TEXT NOT NULL)", + 'CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)', + 'CREATE TABLE catalogs (digest TEXT PRIMARY KEY, tools TEXT NOT NULL)', + 'CREATE TABLE sessions (id TEXT PRIMARY KEY, started REAL NOT NULL, runtime TEXT NOT NULL, deployment TEXT NOT NULL, catalog TEXT NOT NULL REFERENCES catalogs(digest), client_name TEXT, client_version TEXT, pruned INTEGER NOT NULL DEFAULT 0, lost INTEGER NOT NULL DEFAULT 0)', + 'CREATE TABLE interactions (id TEXT PRIMARY KEY, session TEXT NOT NULL REFERENCES sessions(id), ordinal INTEGER NOT NULL, started REAL NOT NULL, method TEXT NOT NULL, operation TEXT NOT NULL, input_shape TEXT NOT NULL, shape_digest TEXT NOT NULL, discovery_operation TEXT, catalog_state TEXT NOT NULL, request_bytes INTEGER, scope TEXT NOT NULL, arguments TEXT NOT NULL, arguments_bytes INTEGER, thread TEXT, origin TEXT NOT NULL, UNIQUE(session, ordinal))', + "CREATE TABLE events (interaction TEXT PRIMARY KEY REFERENCES interactions(id) ON DELETE CASCADE, phase TEXT NOT NULL CHECK(phase IN ('end','error','incomplete')), finished REAL NOT NULL, duration_ms REAL NOT NULL, response_bytes INTEGER, outcome TEXT NOT NULL, feedback TEXT NOT NULL, result TEXT NOT NULL, result_bytes INTEGER)", 'CREATE INDEX interaction_age ON interactions(started)', ) +def default_database() -> Path: + """${XDG_DATA_HOME:-~/.local/share}/standards-engine/observations/.""" + base = os.environ.get('XDG_DATA_HOME', '') + base = Path(base) if os.path.isabs(base) else Path.home() / '.local' / 'share' + return base / 'standards-engine' / 'observations' / DEFAULT_DATABASE_NAME + + def _json(value: object) -> str: return json.dumps(value, ensure_ascii=True, sort_keys=True, separators=(',', ':')) @@ -84,12 +148,158 @@ def _kind(value: object) -> str: return 'object' if isinstance(value, dict) else 'array' if isinstance(value, list) else 'unknown' +def _number(value: object) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) and abs(value) < 2 ** 53 + + +def _identifier(value: object) -> bool: + """A canonical identifier or enum word. Prose, digests and handle ids never match.""" + return isinstance(value, str) and bool(IDENTIFIER.fullmatch(value)) and not OPAQUE.search(value) + + +def _identifiers(values: Iterable[object]) -> list: + return [value for value in islice(values, MAX_RESULT_IDS) if _identifier(value)] + + +def _label(value: object) -> str | None: + """A client's self-reported name or version: bounded printable ASCII.""" + return re.sub(r'[^ -~]', '?', value)[:64] if isinstance(value, str) and value else None + + +def _sent(value: object, depth: int = 0) -> object: + """A caller value as sent, bounded. Cut strings and containers carry a marker.""" + if isinstance(value, str): + return value if len(value) <= MAX_VALUE_CHARS else value[:MAX_VALUE_CHARS] + TRUNCATED + if value is None or isinstance(value, bool) or _number(value): + return value + if isinstance(value, list) and depth < MAX_VALUE_DEPTH: + kept = [_sent(item, depth + 1) for item in value[:MAX_VALUE_ITEMS]] + return kept + [TRUNCATED] if len(value) > MAX_VALUE_ITEMS else kept + if isinstance(value, dict) and depth < MAX_VALUE_DEPTH: + kept = {_sent(str(name)): _sent(item, depth + 1) + for name, item in islice(value.items(), MAX_VALUE_ITEMS)} + if len(value) > MAX_VALUE_ITEMS: + kept[TRUNCATED] = len(value) - MAX_VALUE_ITEMS + return kept + return {'type': _kind(value)} + + +def _pick(value: object, names: tuple[str, ...]) -> dict: + """Allow-listed members as sent. A non-object keeps only its type.""" + if not isinstance(value, dict): + return {'type': _kind(value)} + return {name: _sent(value[name]) for name in names if name in value} + + +def _snapshot(handle: object) -> str: + """Reuse is visible through a short stable hash; the handle id is not kept.""" + identity = handle.get('id') if isinstance(handle, dict) else None + if not isinstance(identity, str): + return 'malformed' + return hashlib.sha256(identity.encode('utf-8', 'replace')).hexdigest()[:12] + + +def navigation_values(operation: str, arguments: dict) -> dict: + """The recorded values of one navigation call, from its explicit allow-list.""" + values = _pick(arguments, NAVIGATION[operation]) + if 'snapshot' in arguments: + values['snapshot'] = _snapshot(arguments['snapshot']) + handle = HANDLES.get(operation) + if handle in arguments: + values[handle] = _pick(arguments[handle], ('kind', 'child_kind')) + if operation == 'read_many' and isinstance(arguments.get('items'), list): + values['items'] = [_pick(item, _READ) for item in arguments['items'][:MAX_VALUE_ITEMS]] + if operation == 'route': + if 'content' in arguments: + values['content'] = _pick(arguments['content'], _PAGE) + if 'facts' in arguments: + # Names and values exactly as sent: invented facts are the signal. + values['facts'] = _sent(arguments['facts']) + request = arguments.get('request') + if operation in ('query', 'query_proposal', 'preview_application') and isinstance(request, dict): + kind = request.get('kind') + values['request'] = _pick(request, REQUESTS.get(kind if isinstance(kind, str) else '', ('kind',))) + if 'facts' in request: + values['request']['facts'] = _sent(request['facts']) + return values + + +def _recorded_facts(values: dict) -> dict: + """Route facts inside a recorded arguments object, top-level or in a nested request.""" + request = values.get('request') + facts = values.get('facts', request.get('facts') if isinstance(request, dict) else None) + return facts if isinstance(facts, dict) else {} + + +def _bounded(values: dict) -> dict: + """Whole-object cap: an oversized projection keeps its scalars and says so.""" + if len(_json(values)) <= MAX_ARGUMENT_BYTES: + return values + return {'truncated': True, **{name: value for name, value in values.items() + if not isinstance(value, (dict, list))}} + + +def _thread(meta: object) -> tuple[str | None, str]: + """Client thread id (hashed with the database salt before storage) and origin. + + Codex supplies both in x-codex-turn-metadata. No other _meta member is read, + so workspace paths, repository URLs, models and turn ids never reach storage. + """ + meta = meta if isinstance(meta, dict) else {} + turn = meta.get('x-codex-turn-metadata') + turn = turn if isinstance(turn, dict) else {} + candidates = (turn.get('thread_id'), meta.get('threadId'), turn.get('session_id'), meta.get('sessionId')) + identity = next((value[:MAX_THREAD_CHARS] for value in candidates + if isinstance(value, str) and value), None) + source = turn.get('thread_source') + origin = {'user': 'main', 'subagent': 'subagent'}.get(source, 'unknown') if isinstance(source, str) else 'unknown' + return identity, origin + + +def _returned(read: object) -> object: + """The canonical id a read result is about, wherever that result kind keeps it.""" + if not isinstance(read, dict): + return None + policy = read.get('policy') + if isinstance(policy, dict): + handle = policy.get('handle') + return policy.get('id') or (handle.get('child_id') if isinstance(handle, dict) else None) + return read.get('canonical_id') + + +def result_summary(value: dict) -> dict: + """Engine-produced identifiers and counts from structuredContent; never text.""" + summary = {} + for name, pattern in (('kind', RESULT_KIND), ('code', RESULT_CODE)): + if isinstance(value.get(name), str) and pattern.fullmatch(value[name]): + summary[name] = value[name] + for name in ('status', 'outcome'): + if isinstance(value.get(name), str) and value[name] in OUTCOMES: + summary[name] = value[name] + plan, questions, page = value.get('reading_plan'), value.get('unresolved_questions'), value.get('content') + if isinstance(plan, list): + summary['selected'] = _identifiers(entry.get('target') for entry in plan + if isinstance(entry, dict) and entry.get('state') == 'selected') + if isinstance(questions, list): + summary['unresolved'] = _identifiers(question['fact'].get('id') for question in questions + if isinstance(question, dict) and isinstance(question.get('fact'), dict)) + if isinstance(page, dict): # A route's content page. A read's content is text and is not inspected. + items = page.get('items') if isinstance(page.get('items'), list) else [] + summary['content'] = {**{name: page[name] for name in ('offset', 'total') if _number(page.get(name))}, + 'items': len(items), 'targets': _identifiers(map(_returned, items))} + reads = value['items'] if isinstance(value.get('items'), list) else [value] + targets = _identifiers(map(_returned, reads)) + if targets: + summary['targets'] = targets + return summary + + class InputShape: - """Value-free disclosure projection, not a second input validator. + """Disclosure projections of one operation's input, not a second validator. - Static property names come only from the installed schema. Unknown names and - every scalar value disappear before SQLite sees anything. Sensitive children - are not traversed even if their properties occur elsewhere in the schema. + Static property names come only from the installed schema. Unknown names + disappear before SQLite sees anything. Sensitive children are not traversed + even if their properties occur elsewhere in the schema. """ def __init__(self, schema: dict) -> None: self.schema = schema @@ -122,6 +332,7 @@ def _items(self, nodes: list[dict]) -> list[dict]: if isinstance(node.get('items'), dict)] def project(self, value: object) -> dict: + """Value-free structure: every scalar becomes only its type.""" budget = MAX_SHAPE_NODES def visit(item, nodes, depth): nonlocal budget @@ -156,7 +367,41 @@ def visit(item, nodes, depth): return {'type': _kind(value), 'truncated': True} return shape - def feedback(self, value: object) -> list[dict]: + def identifiers(self, value: object) -> dict: + """Authoring calls: identifiers and enums at schema-declared, allow-listed names. + + The walk follows project(): dynamic keys and sensitive subtrees are never + entered, nor are evidence references under their other name. A string + survives only if it has the shape of an identifier, so prose placed in an + identifier field is dropped like any body or rationale. + """ + budget = MAX_VALUE_NODES + def visit(item, nodes, name, depth): + nonlocal budget + budget -= 1 + if budget <= 0 or depth > 2 * MAX_SHAPE_DEPTH: + return None + if isinstance(item, dict): + properties, kept = self._properties(nodes), {} + for key, child in islice(item.items(), MAX_SHAPE_NODES): + if (key in properties and key.lower() not in SENSITIVE_FIELDS + and key != 'explicit_exclusions'): + found = visit(child, properties[key], key, depth + 1) + if found is not None and found != {} and found != []: + kept[key] = found + return kept + if isinstance(item, list): + children = self._items(nodes) + found = [visit(child, children, name, depth + 1) for child in item[:MAX_VALUE_ITEMS]] + return [entry for entry in found if entry is not None and entry != {} and entry != []] + if name in IDENTIFIER_FIELDS and (isinstance(item, bool) or _number(item) or _identifier(item)): + return item + return None + found = visit(value, [self.schema], '', 0) + return found if isinstance(found, dict) else {} + + def feedback(self, value: object, names: frozenset = frozenset()) -> list[dict]: + """Validation locations. `names` are route fact names this call already recorded.""" if not isinstance(value, dict) or not isinstance(value.get('issues'), list): return [] result = [] @@ -174,7 +419,8 @@ def feedback(self, value: object) -> list[dict]: safe.append(name.replace('~', '~0').replace('/', '~1')) nodes = [] if name.lower() in SENSITIVE_FIELDS else properties[name] else: - safe.append('*') + known = bool(safe) and safe[-1] == 'facts' and name in names + safe.append(name.replace('~', '~0').replace('/', '~1') if known else '*') nodes = items if part.isdecimal() else [] keyword = issue.get('keyword') result.append({'pointer': '/' + '/'.join(safe) if safe else '', @@ -184,8 +430,8 @@ def feedback(self, value: object) -> list[dict]: @dataclass(frozen=True) class RecorderConfig: - database: Path - deployment: str + database: Path | None = None # None selects default_database(). + deployment: str = DEFAULT_DEPLOYMENT max_calls: int = DEFAULT_MAX_CALLS max_age_seconds: int = DEFAULT_MAX_AGE_SECONDS max_bytes: int = DEFAULT_MAX_BYTES @@ -206,12 +452,37 @@ class Observation: identity: str started: float operation: str + facts: frozenset = frozenset() class StorageRejected(Exception): """Fixed operator code; never includes paths, SQL or payloads.""" +def _busy(error: Exception) -> bool: + return (isinstance(error, sqlite3.Error) and getattr(error, 'sqlite_errorcode', 0) & 255 + in (sqlite3.SQLITE_BUSY, sqlite3.SQLITE_LOCKED)) + + +def _deadline(wait_ms: int) -> float: + return time.monotonic() + wait_ms / 1000 + + +def _locked(db: sqlite3.Connection, deadline: float, statement: str = 'BEGIN IMMEDIATE') -> sqlite3.Cursor: + """Run a statement that needs the write lock, polling until the deadline. + + SQLite's own busy handler backs off in steps of up to 25 ms, far coarser than + a lock that other recorders hold for microseconds, so connections use none. + """ + while True: + try: + return db.execute(statement) + except sqlite3.OperationalError as error: + if not _busy(error) or time.monotonic() >= deadline: + raise + time.sleep(LOCK_POLL_SECONDS) + + class InteractionRecorder: def __init__(self, config: RecorderConfig, runtime: dict, catalog: list[dict], *, diagnostics: TextIO | None = None) -> None: @@ -219,57 +490,79 @@ def __init__(self, config: RecorderConfig, runtime: dict, catalog: list[dict], self.runtime = dict(runtime) self._diagnostics = diagnostics if diagnostics is not None else sys.stderr self._connection: sqlite3.Connection | None = None + self._salt = b'' + self._pages = 0 self._active: dict[str, Observation] = {} + self._backlog: deque[tuple[str, tuple]] = deque() + self._unsaved_lost = 0 + self._started = time.time() + self._client: tuple[str | None, str | None] = (None, None) self._sequence = 0 self._closed = False + self._silent = True self.health = {'state': 'starting', 'reason': None, 'recorded_starts': 0, - 'recorded_terminals': 0, 'dropped_observations': 0, + 'recorded_terminals': 0, 'backlog': 0, 'lost': 0, 'pruned_interactions': 0} self._shapes = {tool['name']: InputShape(tool['inputSchema']) for tool in catalog} - self._catalog = {tool['name']: { + self._catalog = _json({tool['name']: { 'input_digest': _digest(tool['inputSchema']), 'output_digest': (tool.get('_meta', {}).get('standards-engine/output-schema-digest') or _digest(tool.get('outputSchema'))), - } for tool in catalog} + } for tool in catalog}) try: - self._open() - self.health['state'] = 'healthy' - self._emit() + self._connection = self._open(_deadline(OPEN_TIMEOUT_MS)) + self._note('healthy') except Exception as error: self._degrade(error) def _emit(self) -> None: + self._silent = False try: print('standards-engine observer ' + _json(self.health), file=self._diagnostics, flush=True) except Exception: # Operator stderr can itself be broken. Never change a domain result. pass + def _note(self, state: str, reason: str | None = None) -> None: + """Operator stderr only when health changes; a healthy recorder is silent.""" + previous = (self.health['state'], self.health['reason']) + self.health.update(state=state, reason=reason, backlog=len(self._backlog)) + if previous != (state, reason) and 'degraded' in (previous[0], state): + self._emit() + def _degrade(self, error: Exception) -> None: + """Keep the backlog. Reconnect on a later call unless the database was only busy.""" reason = 'storage-unavailable' if isinstance(error, StorageRejected): reason = str(error) + elif _busy(error): + reason = 'storage-locked' elif isinstance(error, sqlite3.Error): - reason = {sqlite3.SQLITE_BUSY: 'storage-locked', sqlite3.SQLITE_LOCKED: 'storage-locked', - sqlite3.SQLITE_FULL: 'storage-full', sqlite3.SQLITE_CORRUPT: 'storage-corrupt', + reason = {sqlite3.SQLITE_FULL: 'storage-full', sqlite3.SQLITE_CORRUPT: 'storage-corrupt', sqlite3.SQLITE_NOTADB: 'storage-corrupt'}.get( getattr(error, 'sqlite_errorcode', 0) & 255, 'storage-error') - self.health.update(state='degraded', reason=reason) - self.health['dropped_observations'] += 1 - if self._connection is not None: + if reason != 'storage-locked' and self._connection is not None: try: self._connection.close() except Exception: pass self._connection = None - self._emit() + self._note('degraded', reason) - def _open(self) -> None: - path = self.config.database + def _lose(self, rows: int) -> None: + self.health['lost'] += rows + self._unsaved_lost += rows + + def _open(self, deadline: float) -> sqlite3.Connection: if os.name != 'posix': raise StorageRejected('unsupported-filesystem') - # The operator owns the existing directory; no recursive mkdir or changes - # to existing paths/permissions. O_NOFOLLOW protects the final component. + path = self.config.database + if path is None: + # Only the default location is created. An operator-supplied directory + # must already exist; existing paths and permissions are never changed. + path = default_database() + path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + # O_NOFOLLOW protects the final component. descriptor = os.open(path, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600) try: info = os.fstat(descriptor) @@ -277,78 +570,153 @@ def _open(self) -> None: raise StorageRejected('unsafe-database-file') finally: os.close(descriptor) - db = sqlite3.connect(path, timeout=BUSY_TIMEOUT_MS / 1000, isolation_level=None) - self._connection = db - db.execute('PRAGMA foreign_keys=ON') - db.execute(f'PRAGMA busy_timeout={BUSY_TIMEOUT_MS}') - db.execute('BEGIN IMMEDIATE') + db = sqlite3.connect(path, timeout=0, isolation_level=None) try: - # Reject unsupported capabilities before publishing a migration. - if db.execute('PRAGMA journal_mode').fetchone()[0] != 'delete': + db.execute('PRAGMA foreign_keys=ON') + _locked(db, deadline) + try: + # Reject unsupported capabilities before publishing a migration. + pages = self.config.max_bytes // db.execute('PRAGMA page_size').fetchone()[0] + if db.execute('PRAGMA page_count').fetchone()[0] > pages: + raise StorageRejected('database-over-budget') + db.execute(f'PRAGMA max_page_count={pages}') + version = db.execute('PRAGMA user_version').fetchone()[0] + app = db.execute('PRAGMA application_id').fetchone()[0] + tables = db.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall() + if version == 0 and app == 0 and not tables: + # Table absence cannot prove privacy: dropped SQLite content may + # remain in freelist pages or sqlite_master slack. Only a truly + # new zero-length source is admitted; never wipe/reuse old bytes. + # Simultaneous creators queue on this lock: the first publishes + # the whole schema in one commit and the rest find it complete. + if info.st_size != 0 or path.stat().st_size != 0: + raise StorageRejected('nonpristine-database') + for statement in SCHEMA_DDL: + db.execute(statement) + db.execute("INSERT INTO meta VALUES('salt',?)", (secrets.token_hex(16),)) + db.execute(f'PRAGMA application_id={APPLICATION_ID}') + db.execute(f'PRAGMA user_version={SCHEMA_VERSION}') + elif version != SCHEMA_VERSION or app != APPLICATION_ID: + # Includes schema v1: historical evidence, never migrated or deleted. + raise StorageRejected('unsupported-database-schema') + actual = {row[0]: row[1] for row in db.execute( + "SELECT name,sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%'")} + expected = {statement.split()[2]: statement for statement in SCHEMA_DDL} + if actual != expected: + raise StorageRejected('invalid-database-schema') + salt = bytes.fromhex(db.execute("SELECT value FROM meta WHERE key='salt'").fetchone()[0]) + _locked(db, deadline, 'COMMIT') + except BaseException: + db.execute('ROLLBACK') + raise + # WAL keeps the write lock to microseconds (no fsync per commit) and lets + # a report read while servers write. Idempotent for every later opener. + if _locked(db, deadline, 'PRAGMA journal_mode=WAL').fetchone()[0] != 'wal': raise StorageRejected('unsupported-journal-mode') - pages = self.config.max_bytes // db.execute('PRAGMA page_size').fetchone()[0] - if db.execute('PRAGMA page_count').fetchone()[0] > pages: - raise StorageRejected('database-over-budget') - db.execute(f'PRAGMA max_page_count={pages}') - version = db.execute('PRAGMA user_version').fetchone()[0] - app = db.execute('PRAGMA application_id').fetchone()[0] - tables = db.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall() - if version == 0 and app == 0 and not tables: - # Table absence cannot prove privacy: dropped SQLite content may - # remain in freelist pages or sqlite_master slack. Only a truly - # new zero-length source is admitted; never wipe/reuse old bytes. - if info.st_size != 0 or path.stat().st_size != 0: - raise StorageRejected('nonpristine-database') - for statement in SCHEMA_DDL: - db.execute(statement) - db.execute(f'PRAGMA application_id={APPLICATION_ID}') - db.execute(f'PRAGMA user_version={SCHEMA_VERSION}') - elif version != SCHEMA_VERSION or app != APPLICATION_ID: - raise StorageRejected('unsupported-database-schema') - actual = {row[0]: row[1] for row in db.execute( - "SELECT name,sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%'")} - expected = {statement.split()[2]: statement for statement in SCHEMA_DDL} - if actual != expected: - raise StorageRejected('invalid-database-schema') - db.execute('COMMIT') + db.execute('PRAGMA synchronous=NORMAL') + db.execute('PRAGMA secure_delete=ON') except BaseException: - db.execute('ROLLBACK') + db.close() raise - db.execute('PRAGMA synchronous=FULL') - db.execute('PRAGMA secure_delete=ON') - - def _transaction(self, action): - db = self._connection - if db is None: - self.health['dropped_observations'] += 1 - return False + self._salt, self._pages = salt, pages + return db + + def _pseudonym(self, thread: str | None) -> str | None: + if thread is None: + return None + return hmac.new(self._salt, thread.encode('utf-8', 'replace'), 'sha256').hexdigest()[:16] + + def _write(self, db: sqlite3.Connection) -> tuple[Counter, int, int]: + """One transaction: prune to policy, then store the whole backlog in order.""" + session, now = self.runtime['instance_id'], time.time() + starts = sum(kind == 'start' for kind, _ in self._backlog) + pruned = 0 + if starts: + pruned = db.execute('DELETE FROM interactions WHERE started < ?', + (now - self.config.max_age_seconds,)).rowcount + count = db.execute('SELECT count(*) FROM interactions').fetchone()[0] + excess = max(0, count + starts - self.config.max_calls) + used = db.execute('PRAGMA page_count').fetchone()[0] - db.execute('PRAGMA freelist_count').fetchone()[0] + if used > self._pages - self._pages // 16: + # The byte cap bounds retention like the count cap: a nearly full file + # gives up its oldest sixteenth instead of refusing new observations. + excess = max(excess, count // 16 + starts) + if excess: + pruned += db.execute('DELETE FROM interactions WHERE id IN ' + '(SELECT id FROM interactions ORDER BY started,id LIMIT ?)', (excess,)).rowcount + if pruned: + # A session that only carries a loss count stays until it ages out. + db.execute('DELETE FROM sessions WHERE id != ? AND (lost = 0 OR started < ?) AND NOT EXISTS ' + '(SELECT 1 FROM interactions WHERE session=sessions.id)', + (session, now - self.config.max_age_seconds)) + db.execute('DELETE FROM catalogs WHERE digest NOT IN (SELECT catalog FROM sessions)') + catalog = _digest(self._catalog) + db.execute('INSERT OR IGNORE INTO catalogs VALUES(?,?)', (catalog, self._catalog)) + db.execute('INSERT OR IGNORE INTO sessions(id,started,runtime,deployment,catalog) VALUES(?,?,?,?,?)', + (session, self._started, _json(self.runtime), self.config.deployment, catalog)) + recorded = Counter() + for kind, row in self._backlog: + if kind == 'start': + recorded[kind] += db.execute( + 'INSERT OR IGNORE INTO interactions VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', + (*row[:-2], self._pseudonym(row[-2]), row[-1])).rowcount + else: + # Retention or a backlog overflow may have removed this start. Do + # not recreate it or claim a terminal event was retained. + recorded[kind] += db.execute( + 'INSERT OR IGNORE INTO events SELECT ?,?,?,?,?,?,?,?,? WHERE EXISTS ' + '(SELECT 1 FROM interactions WHERE id=?)', (*row, row[0])).rowcount + orphans = len(self._backlog) - starts - recorded['end'] + db.execute('UPDATE sessions SET client_name=?,client_version=?,pruned=pruned+?,lost=lost+? WHERE id=?', + (*self._client, pruned, self._unsaved_lost + orphans, session)) + return recorded, pruned, orphans + + def _flush(self, wait_ms: int) -> None: + """Deliver the backlog, or keep all of it for a later call. Never raises.""" + if not self._backlog and not self._unsaved_lost: + return + deadline = _deadline(wait_ms) try: - db.execute('BEGIN IMMEDIATE') - action(db) - db.execute('COMMIT') - return True - except Exception as error: + if self._connection is None: + self._connection = self._open(deadline) + db = self._connection + _locked(db, deadline) try: - db.execute('ROLLBACK') - except sqlite3.Error: - pass + recorded, pruned, orphans = self._write(db) + db.execute('COMMIT') + except BaseException: + try: + db.execute('ROLLBACK') + except sqlite3.Error: + pass + raise + except Exception as error: self._degrade(error) - return False + return + self._backlog.clear() + self._unsaved_lost = 0 + self.health['recorded_starts'] += recorded['start'] + self.health['recorded_terminals'] += recorded['end'] + self.health['pruned_interactions'] += pruned + self.health['lost'] += orphans + self._note('healthy') + + def _record(self, kind: str, row: tuple) -> None: + if len(self._backlog) >= MAX_BACKLOG: + # Bounded memory: the oldest waiting row gives way and is counted. + self._backlog.popleft() + self._lose(1) + self._backlog.append((kind, row)) + self._flush(BUSY_TIMEOUT_MS) def begin(self, message: object, *, request_bytes: int | None, scope: str, parse_error: bool = False) -> Observation | None: if self._closed: return None - if self._active: - self._degrade(StorageRejected('overlapping-recorder-call')) - return None # Notifications have no response pair. Invalid envelopes and parse errors do. if (isinstance(message, dict) and 'id' not in message and not parse_error and message.get('jsonrpc') == '2.0' and isinstance(message.get('method'), str)): return None - if self._connection is None: - self.health['dropped_observations'] += 1 - return None try: params = message.get('params', {}) if isinstance(message, dict) else {} params = params if isinstance(params, dict) else {} @@ -356,43 +724,41 @@ def begin(self, message: object, *, request_bytes: int | None, scope: str, method = method if isinstance(method, str) and method in METHODS else 'invalid-json' if parse_error else 'other' name = params.get('name') operation = name if method == 'tools/call' and isinstance(name, str) and name in self._shapes else 'other' - shape = self._shapes[operation].project(params.get('arguments', {})) if operation in self._shapes else {} arguments = params.get('arguments', {}) + shape, values, arguments_bytes = {}, {}, None + if operation in self._shapes: + shape = self._shapes[operation].project(arguments) + if operation not in NAVIGATION: + values = self._shapes[operation].identifiers(arguments) + elif isinstance(arguments, dict): + values = navigation_values(operation, arguments) + values = _bounded(values) + try: + arguments_bytes = len(json.dumps(arguments, ensure_ascii=False, separators=(',', ':')).encode('utf-8')) + except Exception: + pass discovery = arguments.get('operation') if isinstance(arguments, dict) else None discovery = discovery if operation in {'describe_input', 'describe_output'} and isinstance(discovery, str) and discovery in self._shapes else None expected = arguments.get('expected_catalog') if isinstance(arguments, dict) else None catalog_state = ('unspecified' if expected is None else 'matches' if expected == self.runtime['catalog_digest'] else 'differs') - now = time.time() - observation = Observation(str(uuid.uuid4()), time.monotonic(), operation) + if method == 'initialize' and self._client == (None, None): + client = params.get('clientInfo') + client = client if isinstance(client, dict) else {} + self._client = (_label(client.get('name')), _label(client.get('version'))) + thread, origin = _thread(params.get('_meta')) + observation = Observation(str(uuid.uuid4()), time.monotonic(), operation, + frozenset(_recorded_facts(values))) self._sequence += 1 - pruned = 0 - def write(db): - nonlocal pruned - db.execute('INSERT OR IGNORE INTO sessions(id,started,runtime,deployment,catalog) VALUES(?,?,?,?,?)', - (self.runtime['instance_id'], now, _json(self.runtime), self.config.deployment, _json(self._catalog))) - pruned += db.execute('DELETE FROM interactions WHERE started < ?', - (now - self.config.max_age_seconds,)).rowcount - excess = db.execute('SELECT max(0,count(*)-?) FROM interactions', - (self.config.max_calls - 1,)).fetchone()[0] - if excess: - pruned += db.execute('DELETE FROM interactions WHERE id IN ' - '(SELECT id FROM interactions ORDER BY started,id LIMIT ?)', (excess,)).rowcount - db.execute('DELETE FROM sessions WHERE id != ? AND NOT EXISTS ' - '(SELECT 1 FROM interactions WHERE session=sessions.id)', (self.runtime['instance_id'],)) - db.execute('UPDATE sessions SET pruned=pruned+? WHERE id=?', (pruned, self.runtime['instance_id'])) - db.execute('INSERT INTO interactions VALUES(?,?,?,?,?,?,?,?,?,?,?,?)', - (observation.identity, self.runtime['instance_id'], self._sequence, now, - method, operation, _json(shape), _digest(shape), discovery, catalog_state, - request_bytes, scope)) - if not self._transaction(write): - return None - self.health['recorded_starts'] += 1 - self.health['pruned_interactions'] += pruned + row = (observation.identity, self.runtime['instance_id'], self._sequence, time.time(), + method, operation, _json(shape), _digest(shape), discovery, catalog_state, + request_bytes, scope, _json(values), arguments_bytes, thread, origin) self._active[observation.identity] = observation + self._record('start', row) return observation - except Exception as error: - self._degrade(error) + except Exception: + self._lose(1) + self._note('degraded', 'recorder-error') return None def finish(self, observation: Observation | None, response: object = None, @@ -401,7 +767,7 @@ def finish(self, observation: Observation | None, response: object = None, return self._active.pop(observation.identity, None) try: - phase, outcome, feedback = 'end', 'response', [] + phase, outcome, feedback, summary, result_bytes = 'end', 'response', [], {}, None if incomplete: phase, outcome = 'incomplete', 'unknown' elif isinstance(response, dict): @@ -421,30 +787,24 @@ def finish(self, observation: Observation | None, response: object = None, outcome = value['outcome'] elif isinstance(value.get('status'), str) and value['status'] in OUTCOMES: outcome = value['status'] + summary = result_summary(value) + if 'structuredContent' in result: + try: + # One serialization: the text copy the transport already made. + result_bytes = len(result['content'][0]['text'].encode('utf-8')) + except Exception: + pass if observation.operation in self._shapes: - feedback = self._shapes[observation.operation].feedback(value.get('input_feedback')) + feedback = self._shapes[observation.operation].feedback( + value.get('input_feedback'), observation.facts) if 'input_feedback' in value: phase, outcome = 'error', 'input-invalid' - evicted = False - def write(db): - nonlocal evicted - # Retention in another process may have evicted this start. Do not - # recreate it or claim a terminal event was retained. - if not db.execute('SELECT 1 FROM interactions WHERE id=?', (observation.identity,)).fetchone(): - evicted = True - return - db.execute('INSERT INTO events VALUES(?,?,?,?,?,?,?)', - (observation.identity, phase, time.time(), - max(0, (time.monotonic() - observation.started) * 1000), - response_bytes, outcome, _json(feedback))) - if self._transaction(write): - if evicted: - self.health['dropped_observations'] += 1 - self._emit() - else: - self.health['recorded_terminals'] += 1 - except Exception as error: - self._degrade(error) + self._record('end', (observation.identity, phase, time.time(), + max(0, (time.monotonic() - observation.started) * 1000), + response_bytes, outcome, _json(feedback), _json(summary), result_bytes)) + except Exception: + self._lose(1) + self._note('degraded', 'recorder-error') def close(self) -> None: if self._closed: @@ -452,13 +812,22 @@ def close(self) -> None: for observation in tuple(self._active.values()): self.finish(observation, incomplete=True) self._closed = True + self._flush(CLOSE_TIMEOUT_MS) + if self._backlog: + # The process is ending: nothing can deliver these rows any more. + # Save the count if the database can still be reached at all. + self._lose(len(self._backlog)) + self._backlog.clear() + self._flush(BUSY_TIMEOUT_MS) if self._connection is not None: try: self._connection.close() - except Exception as error: - self._degrade(error) + except Exception: + pass self._connection = None - self._emit() + self.health['backlog'] = 0 + if not self._silent or self.health['lost']: + self._emit() def open_report(database: Path) -> sqlite3.Connection: @@ -476,38 +845,171 @@ def open_report(database: Path) -> sqlite3.Connection: raise -def report(database: Path, *, limit: int = 100) -> dict: +def _percentile(values: list, fraction: float): + ordered = sorted(values) + return ordered[max(0, math.ceil(fraction * len(ordered)) - 1)] if ordered else None + + +def _top(counter: Counter, limit: int, *names: str) -> list[dict]: + """Most common keys as rows; ties break on the key so reports are stable.""" + rows = sorted(counter.items(), key=lambda item: (-item[1], _json(item[0])))[:limit] + return [{**dict(zip(names, key if isinstance(key, tuple) else (key,))), 'count': count} + for key, count in rows] + + +def _fact_values(value: object) -> list[str]: + """The vocabulary words inside one routing fact value as sent.""" + if isinstance(value, dict): # Native typed form {type, state, value}. + value = value.get('value') + return [item for item in (value if isinstance(value, list) else [value]) if isinstance(item, str)] + + +def _usage(db: sqlite3.Connection, since: float, limit: int) -> dict: + """What agents asked for and what came back, from the recorded values.""" + sessions = {} + for row in db.execute('SELECT id,runtime,client_name,client_version FROM sessions'): + runtime = json.loads(row['runtime']) + sessions[row['id']] = (' '.join(filter(None, (row['client_name'], row['client_version']))) or 'unknown', + runtime.get('purpose'), runtime.get('interface_version')) + clients, calls, rejected = {}, Counter(), Counter() + durations, sizes = {}, {} + rejections, read_targets, origins, tool_calls = Counter(), Counter(), {}, Counter() + reads, accepted, declared, refused = {}, {}, set(), [] + routes = without_snapshot = 0 + for row in db.execute( + 'SELECT i.session,i.method,i.operation,i.arguments,i.thread,i.origin,e.phase,e.outcome,' + 'e.feedback,e.duration_ms,e.result,e.result_bytes FROM interactions i LEFT JOIN events e ' + 'ON e.interaction=i.id WHERE i.started >= ? ORDER BY i.started,i.id', (since,)): + client = clients.setdefault(sessions.get(row['session'], ('unknown', None, None)), [set(), 0]) + client[0].add(row['session']) + client[1] += 1 + thread = row['thread'] or 'process:' + row['session'] + if row['method'] == 'tools/call': + origins[thread] = row['origin'] + tool_calls[thread] += 1 + operation = row['operation'] if row['method'] == 'tools/call' else row['method'] + calls[operation] += 1 + if row['duration_ms'] is not None: + durations.setdefault(operation, []).append(row['duration_ms']) + if row['result_bytes'] is not None: + sizes.setdefault(operation, []).append(row['result_bytes']) + arguments = json.loads(row['arguments']) + result = json.loads(row['result']) if row['result'] else {} + facts = _recorded_facts(arguments) + if row['operation'] == 'route': + routes += 1 + without_snapshot += 'snapshot' not in arguments + if row['phase'] == 'end': + declared.update(result.get('unresolved', ())) + for name, value in facts.items(): + accepted.setdefault(name, set()).update(_fact_values(value)) + delivered = result.get('targets', []) + result.get('content', {}).get('targets', []) + read_targets.update(delivered) + if delivered: + reads.setdefault(thread, Counter()).update(delivered) + elif row['phase'] == 'error' and row['method'] == 'tools/call': + rejected[operation] += 1 + issue = (json.loads(row['feedback']) or [{}])[0] + # The offending value is whatever the validation pointer names in the + # recorded arguments; without a pointer, everything recorded but the snapshot. + value = arguments + for part in issue.get('pointer', '').split('/')[1:]: + part = part.replace('~1', '/').replace('~0', '~') + inner = (value.get(part) if isinstance(value, dict) + else value[int(part)] if isinstance(value, list) and part.isdecimal() and int(part) < len(value) + else None) + if inner is None: + break + value = inner + if value is arguments: + value = {name: item for name, item in arguments.items() if name != 'snapshot'} + rejections[(operation, result.get('code') or row['outcome'], issue.get('pointer', ''), + issue.get('keyword', ''), _json(value)[:MAX_VALUE_CHARS])] += 1 + if issue.get('keyword') == 'fact': + pointed = issue['pointer'].rpartition('/facts/')[2].replace('~1', '/').replace('~0', '~') + refused.append((facts, pointed if '/facts/' in issue['pointer'] else None)) + # Out-of-vocabulary is inferred, never looked up: a name or value counts when + # the server refused its route and never accepted or asked for it in this window. + names, values = Counter(), Counter() + for facts, pointed in refused: + names.update(name for name in facts if name not in accepted and name not in declared) + for name in ([pointed] if pointed in facts else []): + values.update((name, word) for word in _fact_values(facts[name]) + if word not in accepted.get(name, ())) + repeats = {thread: sum(seen.values()) - len(seen) for thread, seen in reads.items()} + total_reads = sum(sum(seen.values()) for seen in reads.values()) + threads = {} + for thread, origin in origins.items(): + entry = threads.setdefault(origin, {'origin': origin, 'threads': 0, 'calls': 0, 'reads': 0, 'repeats': 0}) + entry['threads'] += 1 + entry['calls'] += tool_calls[thread] + entry['reads'] += sum(reads.get(thread, {}).values()) + entry['repeats'] += repeats.get(thread, 0) + return { + 'clients': [{'client': key[0], 'purpose': key[1], 'interface_version': key[2], + 'sessions': len(value[0]), 'calls': value[1]} + for key, value in sorted(clients.items(), key=lambda item: (-item[1][1], _json(item[0])))][:limit], + 'usage': [{'operation': operation, 'calls': count, 'rejected': rejected[operation], + 'rejection_rate': rejected[operation] / count, + 'duration_p50_ms': _percentile(durations.get(operation, []), 0.5), + 'duration_p90_ms': _percentile(durations.get(operation, []), 0.9), + 'duration_max_ms': max(durations.get(operation, []), default=None), + 'result_p50_bytes': _percentile(sizes.get(operation, []), 0.5), + 'result_max_bytes': max(sizes.get(operation, []), default=None)} + for operation, count in sorted(calls.items(), key=lambda item: (-item[1], item[0]))][:limit], + 'rejections': _top(rejections, limit, 'operation', 'code', 'pointer', 'keyword', 'value'), + 'unaccepted_facts': {'names': _top(names, limit, 'name'), 'values': _top(values, limit, 'fact', 'value')}, + 'read_targets': _top(read_targets, limit, 'target'), + 'repeat_reads': {'reads': total_reads, 'repeats': sum(repeats.values()), 'threads': len(reads), + 'threads_repeating': sum(count > 0 for count in repeats.values()), + 'worst_thread_rate': max((count / sum(reads[thread].values()) + for thread, count in repeats.items() if count), default=0.0)}, + 'routes': {'calls': routes, 'without_snapshot': without_snapshot}, + 'threads': [threads[origin] for origin in sorted(threads)], + } + + +def report(database: Path, *, limit: int = 100, since_days: float | None = None) -> dict: if not 1 <= limit <= MAX_REPORT_ROWS: raise ValueError('report limit must be in 1..1000') + if since_days is not None and not 0 < since_days <= 3650: + raise ValueError('since_days must be in 0..3650') + since = time.time() - since_days * 24 * 60 * 60 if since_days is not None else 0.0 db = open_report(database) db.row_factory = sqlite3.Row try: db.execute('BEGIN') # One consistent read snapshot, including denominators. - total = db.execute('SELECT count(*) FROM interactions').fetchone()[0] - terminals = db.execute('SELECT count(*) FROM events').fetchone()[0] + total, first, last = db.execute('SELECT count(*),min(started),max(started) FROM interactions ' + 'WHERE started >= ?', (since,)).fetchone() + terminals = db.execute('SELECT count(*) FROM events e JOIN interactions i ON i.id=e.interaction ' + 'WHERE i.started >= ?', (since,)).fetchone()[0] + lost, pruned = db.execute('SELECT coalesce(sum(lost),0),coalesce(sum(pruned),0) FROM sessions').fetchone() operations = [dict(row) for row in db.execute( 'SELECT i.scope,i.method,i.operation,count(*) AS starts,count(e.interaction) AS terminals,' 'sum(CASE WHEN e.phase=\'error\' THEN 1 ELSE 0 END) AS errors,' 'sum(CASE WHEN e.interaction IS NULL OR e.phase=\'incomplete\' THEN 1 ELSE 0 END) AS incomplete,' 'sum(i.request_bytes) AS request_bytes,count(i.request_bytes) AS request_measurements,' + 'sum(i.arguments_bytes) AS arguments_bytes,count(i.arguments_bytes) AS arguments_measurements,' 'sum(e.response_bytes) AS response_bytes,count(e.response_bytes) AS response_measurements,' + 'sum(e.result_bytes) AS result_bytes,count(e.result_bytes) AS result_measurements,' 'count(e.duration_ms) AS duration_measurements,' 'avg(e.duration_ms) AS observed_duration_ms FROM interactions i LEFT JOIN events e ' - 'ON e.interaction=i.id GROUP BY i.scope,i.method,i.operation ORDER BY starts DESC,i.scope,i.method,i.operation LIMIT ?', (limit + 1,))] + 'ON e.interaction=i.id WHERE i.started >= ? GROUP BY i.scope,i.method,i.operation ' + 'ORDER BY starts DESC,i.scope,i.method,i.operation LIMIT ?', (since, limit + 1))] operations_truncated = len(operations) > limit operations = operations[:limit] failures = [dict(row) for row in db.execute( 'SELECT i.scope,i.operation,i.shape_digest,e.outcome,e.feedback,count(*) AS count FROM interactions i ' - 'JOIN events e ON e.interaction=i.id WHERE e.phase=\'error\' ' - 'GROUP BY i.scope,i.operation,i.shape_digest,e.outcome,e.feedback ORDER BY count DESC,i.scope,i.operation,i.shape_digest,e.outcome,e.feedback LIMIT ?', (limit + 1,))] + 'JOIN events e ON e.interaction=i.id WHERE e.phase=\'error\' AND i.started >= ? ' + 'GROUP BY i.scope,i.operation,i.shape_digest,e.outcome,e.feedback ORDER BY count DESC,i.scope,i.operation,i.shape_digest,e.outcome,e.feedback LIMIT ?', (since, limit + 1))] failures_truncated = len(failures) > limit failures = failures[:limit] shapes = [dict(row) for row in db.execute( 'SELECT i.scope,i.operation,i.shape_digest,i.input_shape,count(*) AS starts,' 'sum(CASE WHEN e.phase=\'error\' THEN 1 ELSE 0 END) AS errors FROM interactions i ' - 'LEFT JOIN events e ON e.interaction=i.id WHERE i.method=\'tools/call\' ' + 'LEFT JOIN events e ON e.interaction=i.id WHERE i.method=\'tools/call\' AND i.started >= ? ' 'GROUP BY i.scope,i.operation,i.shape_digest,i.input_shape ' - 'ORDER BY starts DESC,i.scope,i.operation,i.shape_digest LIMIT ?', (limit + 1,))] + 'ORDER BY starts DESC,i.scope,i.operation,i.shape_digest LIMIT ?', (since, limit + 1))] shapes_truncated = len(shapes) > limit shapes = shapes[:limit] for shape in shapes: @@ -518,7 +1020,6 @@ def report(database: Path, *, limit: int = 100) -> dict: 'SELECT * FROM (SELECT * FROM sessions ORDER BY started DESC,id DESC LIMIT ?) ORDER BY started,id', (limit,))] for session in sessions: session['runtime'] = json.loads(session['runtime']) - session['catalog'] = json.loads(session['catalog']) # Chronology within a process only: these are observed attempts, not proof # of agent intent, a repair relation, or an acquired definition. pending = Counter() @@ -529,8 +1030,8 @@ def report(database: Path, *, limit: int = 100) -> dict: scanned = 0 for row in db.execute( 'SELECT * FROM (SELECT i.session,i.ordinal,i.scope,i.operation,i.discovery_operation,e.phase ' - 'FROM interactions i LEFT JOIN events e ON e.interaction=i.id ' - 'ORDER BY i.started DESC,i.id DESC LIMIT ?) ORDER BY session,ordinal', (MAX_SEQUENCE_SCAN,)): + 'FROM interactions i LEFT JOIN events e ON e.interaction=i.id WHERE i.started >= ? ' + 'ORDER BY i.started DESC,i.id DESC LIMIT ?) ORDER BY session,ordinal', (since, MAX_SEQUENCE_SCAN)): scanned += 1 scope = row['scope'] key = (row['session'], scope, row['operation']) @@ -551,20 +1052,26 @@ def report(database: Path, *, limit: int = 100) -> dict: 'preceding_errors_histogram': dict(sorted(errors.items())[:limit]), 'preceding_discoveries_histogram': dict(sorted(discovery.items())[:limit]), 'histograms_truncated': len(errors) > limit or len(discovery) > limit}) + # Each distinct catalog is stored once; compare them in order of first use. revisions = {} for session in sessions: - revisions.setdefault(session['runtime']['catalog_digest'], session['catalog']) + revisions.setdefault(session['runtime'].get('catalog_digest'), session['catalog']) + tools = {digest: json.loads(db.execute('SELECT tools FROM catalogs WHERE digest=?', (digest,)).fetchone()[0]) + for digest in set(revisions.values())} comparisons = [] digests = list(revisions) for before, after in zip(digests, digests[1:]): - left, right = revisions[before], revisions[after] + left, right = tools[revisions[before]], tools[revisions[after]] comparisons.append({'before': before, 'after': after, 'added': sorted(right.keys() - left.keys()), 'removed': sorted(left.keys() - right.keys()), 'changed': sorted(name for name in left.keys() & right.keys() if left[name] != right[name])}) - return {'schema_version': SCHEMA_VERSION, 'retained_starts': total, + return {'schema_version': SCHEMA_VERSION, 'since_days': since_days, + 'first_start': first, 'last_start': last, 'retained_starts': total, 'retained_terminals': terminals, 'unmatched_starts': total - terminals, + 'lost_rows': lost, 'pruned_interactions': pruned, + **_usage(db, since, limit), 'operations': operations, 'operations_truncated': operations_truncated, 'failures': failures, 'failures_truncated': failures_truncated, 'shapes': shapes, 'shapes_truncated': shapes_truncated, 'sessions': sessions, @@ -575,24 +1082,87 @@ def report(database: Path, *, limit: int = 100) -> dict: 'limits': {'group_rows': limit, 'sequence_scan_limit': MAX_SEQUENCE_SCAN, 'meaning': 'Retained observations only; unmatched or incomplete outcomes are unknown. ' 'End means response produced/delivered per scope, not domain completion. ' - 'Sequence denominator is end responses per session/scope/operation in the bounded scan. No cross-process ' - 'continuation or intent is inferred. Retention can remove prior attempts. ' + 'Sequence denominator is end responses per session/scope/operation in the bounded scan. ' + 'A thread is a hashed client thread id, or the server process when the client sent none. ' + 'Unaccepted facts are inferred from refused routes, not checked against a vocabulary. ' + 'Retention can remove prior attempts; lost_rows and pruned_interactions cover retained sessions. ' 'Bytes are serialized JSON/stdio bytes, never tokens or billing. ' - 'Redacted requests cannot be replayed; use independent qualification evidence.'}} + 'Recorded requests cannot be replayed; use independent qualification evidence.'}} finally: db.close() +def _stamp(seconds: float | None) -> str: + return datetime.fromtimestamp(seconds, timezone.utc).strftime('%Y-%m-%d %H:%M') if seconds else '-' + + +def _amount(value: float | None, unit: str = '') -> str: + if value is None: + return '-' + if unit == 'B': + return (f'{value} B' if value < 1000 else f'{value / 1000:.1f} kB' if value < 1_000_000 + else f'{value / 1_000_000:.1f} MB') + return f'{value:.1f}' if value < 10 else f'{value:.0f}' + + +def _share(part: int, whole: int) -> str: + return f'{part} of {whole} ({100 * part / whole:.1f}%)' if whole else '0 of 0' + + +def text_report(summary: dict) -> str: + """About a screenful for a person; the JSON form holds every row.""" + window = f'last {summary["since_days"]:g} days' if summary['since_days'] else 'all retained' + lines = [ + f'Standards Engine MCP observations, schema v{summary["schema_version"]} ({window})', + f'{_stamp(summary["first_start"])} to {_stamp(summary["last_start"])} UTC', + f'{summary["retained_starts"]} calls: {summary["unmatched_starts"]} without an outcome ' + f'(process ended mid-call), {summary["lost_rows"]} rows lost, ' + f'{summary["pruned_interactions"]} pruned by retention', + '', f'{"Client":<34}{"purpose":<13}{"interface":>9}{"sessions":>10}{"calls":>8}'] + lines += [f'{row["client"][:33]:<34}{row["purpose"] or "-":<13}{row["interface_version"] or "-":>9}' + f'{row["sessions"]:>10}{row["calls"]:>8}' for row in summary['clients'][:TEXT_ROWS]] + lines += ['', f'{"Operation":<22}{"calls":>7}{"rejected":>10}{"p50 ms":>9}{"p90 ms":>9}{"max ms":>9}' + f'{"p50 result":>12}{"max result":>12}'] + lines += [f'{row["operation"][:21]:<22}{row["calls"]:>7}{100 * row["rejection_rate"]:>9.1f}%' + f'{_amount(row["duration_p50_ms"]):>9}{_amount(row["duration_p90_ms"]):>9}' + f'{_amount(row["duration_max_ms"]):>9}{_amount(row["result_p50_bytes"], "B"):>12}' + f'{_amount(row["result_max_bytes"], "B"):>12}' for row in summary['usage'][:2 * TEXT_ROWS]] + lines += ['', 'Top rejections (count, operation, code, where, value sent)'] + lines += [f'{row["count"]:>5} {row["operation"]} {row["code"]} ' + f'{row["pointer"] or "-"}{" (" + row["keyword"] + ")" if row["keyword"] else ""} {row["value"][:70]}' + for row in summary['rejections'][:TEXT_ROWS]] or [' none'] + facts = summary['unaccepted_facts'] + lines += ['', 'Route facts the server refused and never accepted in this window', + ' names: ' + (', '.join(f'{row["name"][:40]} x{row["count"]}' for row in facts['names'][:TEXT_ROWS]) or 'none'), + ' values: ' + (', '.join(f'{row["fact"][:40]}={row["value"][:40]} x{row["count"]}' + for row in facts['values'][:TEXT_ROWS]) or 'none'), + '', 'Most-read targets (read, read_many and route content)', + ' ' + (', '.join(f'{row["target"]} x{row["count"]}' for row in summary['read_targets'][:TEXT_ROWS]) or 'none')] + repeats, routes = summary['repeat_reads'], summary['routes'] + lines += ['', f'Reads repeating a target within one thread: {_share(repeats["repeats"], repeats["reads"])}; ' + f'{repeats["threads_repeating"]} of {repeats["threads"]} reading threads repeated ' + f'(worst {100 * repeats["worst_thread_rate"]:.1f}%)', + f'Routes sent without a snapshot: {_share(routes["without_snapshot"], routes["calls"])}', + '', f'{"Tool-calling threads":<22}{"threads":>7}{"calls":>8} reads repeated'] + lines += [f'{row["origin"]:<22}{row["threads"]:>7}{row["calls"]:>8} {_share(row["repeats"], row["reads"])}' + for row in summary['threads']] + return '\n'.join(lines) + + def main() -> int: parser = argparse.ArgumentParser(description='Read-only operator MCP observation report; no agent tools.') - parser.add_argument('--database', type=Path, required=True) - parser.add_argument('--limit', type=int, default=100) + parser.add_argument('--database', type=Path, help='Defaults to the always-on recording location.') + parser.add_argument('--format', choices=('text', 'json'), default='text') + parser.add_argument('--since-days', type=float, help='Only calls started within this many days.') + parser.add_argument('--limit', type=int, default=100, help='Rows per section in the JSON form.') arguments = parser.parse_args() try: - print(json.dumps(report(arguments.database, limit=arguments.limit), sort_keys=True, indent=2)) - except (OSError, sqlite3.Error, StorageRejected, ValueError): + summary = report(arguments.database or default_database(), limit=arguments.limit, + since_days=arguments.since_days) + except (OSError, RuntimeError, sqlite3.Error, StorageRejected, ValueError): print('Observation report unavailable; check database access and schema.', file=sys.stderr) return 1 + print(json.dumps(summary, sort_keys=True, indent=2) if arguments.format == 'json' else text_report(summary)) return 0 diff --git a/tools/standards_engine/standards_engine/mcp.py b/tools/standards_engine/standards_engine/mcp.py index 5280fca6..0a74a5f8 100644 --- a/tools/standards_engine/standards_engine/mcp.py +++ b/tools/standards_engine/standards_engine/mcp.py @@ -10,6 +10,7 @@ from contextlib import redirect_stdout from copy import deepcopy import json +import os from pathlib import Path import sys import traceback @@ -22,7 +23,7 @@ from .context_projection import Purpose from .mcp_catalog import OutputSchemaDelivery, tool_catalog from .interaction_observer import ( - DEFAULT_MAX_AGE_SECONDS, DEFAULT_MAX_BYTES, DEFAULT_MAX_CALLS, + DEFAULT_DEPLOYMENT, DEFAULT_MAX_AGE_SECONDS, DEFAULT_MAX_BYTES, DEFAULT_MAX_CALLS, InteractionRecorder, RecorderConfig, serialized_bytes, ) @@ -297,16 +298,19 @@ def main() -> int: help="Advertise complete output schemas (eager, default), or retrieve exact contracts through describe_output (on-demand).", ) parser.add_argument('--observe-database', type=Path, - help='Opt in to operator-only local SQLite observations in a trusted directory.') - parser.add_argument('--observe-deployment', help='Required nonsensitive deployment label when recording.') + help='Record operator-only observations in this SQLite file instead of the default location; its directory must exist.') + parser.add_argument('--observe-deployment', default=DEFAULT_DEPLOYMENT, + help='Nonsensitive deployment label stored with each recorded session.') + parser.add_argument('--no-observe', action='store_true', + help='Do not record this process. STANDARDS_ENGINE_OBSERVE=off does the same.') parser.add_argument('--observe-max-calls', type=int, default=DEFAULT_MAX_CALLS) parser.add_argument('--observe-max-age-seconds', type=int, default=DEFAULT_MAX_AGE_SECONDS) parser.add_argument('--observe-max-bytes', type=int, default=DEFAULT_MAX_BYTES) arguments = parser.parse_args() - if (arguments.observe_database is None) != (arguments.observe_deployment is None): - parser.error('--observe-database and --observe-deployment must be supplied together') + # Recording is on for every server process. Only these two switches turn it + # off; in-process MCPServer construction records only when given a configuration. observation = None - if arguments.observe_database is not None: + if not arguments.no_observe and os.environ.get('STANDARDS_ENGINE_OBSERVE', '').strip().lower() != 'off': try: observation = RecorderConfig(arguments.observe_database, arguments.observe_deployment, arguments.observe_max_calls, arguments.observe_max_age_seconds, diff --git a/tools/standards_engine/tests/agent_efficiency_trace.py b/tools/standards_engine/tests/agent_efficiency_trace.py index 82135c23..7711b255 100644 --- a/tools/standards_engine/tests/agent_efficiency_trace.py +++ b/tools/standards_engine/tests/agent_efficiency_trace.py @@ -49,7 +49,7 @@ def call(self, name: str, arguments: dict, *, rejected: bool = False) -> dict: ] response = subprocess.run( [sys.executable, "-P", "-m", "tools.standards_engine.standards_engine.mcp", - "--repo-root", str(self.root), "--purpose", self.purpose], + "--repo-root", str(self.root), "--purpose", self.purpose, "--no-observe"], input="".join(json.dumps(item) + "\n" for item in messages), text=True, capture_output=True, check=True, env=os.environ.copy(), ) diff --git a/tools/standards_engine/tests/codex_discovery_client.py b/tools/standards_engine/tests/codex_discovery_client.py index 770bcdfa..d036d884 100644 --- a/tools/standards_engine/tests/codex_discovery_client.py +++ b/tools/standards_engine/tests/codex_discovery_client.py @@ -422,7 +422,7 @@ async def run(arguments) -> dict: # Run-owned fixture coordinates take precedence over operator surface choices. command.extend(['-c', 'analytics.enabled=false', '-c', f'mcp_servers.{server}.command={json.dumps(sys.executable)}', - '-c', f'mcp_servers.{server}.args={json.dumps(["-P", "-m", "tools.standards_engine.standards_engine.mcp", "--repo-root", str(fixture), "--purpose", "authoring", "--output-schemas", arguments.output_schemas])}', + '-c', f'mcp_servers.{server}.args={json.dumps(["-P", "-m", "tools.standards_engine.standards_engine.mcp", "--repo-root", str(fixture), "--purpose", "authoring", "--output-schemas", arguments.output_schemas, "--no-observe"])}', '-c', f'mcp_servers.{server}.env.PYTHONPATH={json.dumps(str(ROOT))}']) events = [] report = {'observer_version': OBSERVER_VERSION, 'status': 'unavailable', 'client_version': version, 'requested_model': arguments.model, diff --git a/tools/standards_engine/tests/codex_navigation_client.py b/tools/standards_engine/tests/codex_navigation_client.py index ab4419d8..db153307 100644 --- a/tools/standards_engine/tests/codex_navigation_client.py +++ b/tools/standards_engine/tests/codex_navigation_client.py @@ -26,6 +26,10 @@ async def main(server_name, output_schemas="eager"): "--stdio", "-c", "analytics.enabled=false", + # A scripted check is not agent behaviour: keep it out of the + # configured server's always-on interaction recording. + "-c", + f'mcp_servers.{server_name}.env.STANDARDS_ENGINE_OBSERVE="off"', stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=log, diff --git a/tools/standards_engine/tests/mcp_policy_registration_client.py b/tools/standards_engine/tests/mcp_policy_registration_client.py index 89d18e7d..4002c155 100644 --- a/tools/standards_engine/tests/mcp_policy_registration_client.py +++ b/tools/standards_engine/tests/mcp_policy_registration_client.py @@ -66,7 +66,7 @@ def parameters(purpose="authoring"): return StdioServerParameters( command=engine_python, args=["-P", "-m", "tools.standards_engine.standards_engine.mcp", - "--purpose", purpose, "--repo-root", str(repo)], + "--purpose", purpose, "--repo-root", str(repo), "--no-observe"], env={**os.environ, "PYTHONPATH": str(ROOT)}, ) diff --git a/tools/standards_engine/tests/mcp_workflow_client.py b/tools/standards_engine/tests/mcp_workflow_client.py index c55530a8..d64bf2a0 100644 --- a/tools/standards_engine/tests/mcp_workflow_client.py +++ b/tools/standards_engine/tests/mcp_workflow_client.py @@ -51,7 +51,7 @@ def parameters(interrupted=False, *, purpose="authoring"): ) return StdioServerParameters( command=engine_python, - args=["-P", *launch, "--purpose", purpose, "--repo-root", str(repo)], + args=["-P", *launch, "--purpose", purpose, "--repo-root", str(repo), "--no-observe"], env={**os.environ, "PYTHONPATH": str(ROOT)}, ) diff --git a/tools/standards_engine/tests/test_agent_interaction_transport.py b/tools/standards_engine/tests/test_agent_interaction_transport.py index 03970523..a7a42d36 100644 --- a/tools/standards_engine/tests/test_agent_interaction_transport.py +++ b/tools/standards_engine/tests/test_agent_interaction_transport.py @@ -38,7 +38,7 @@ def call(self, purpose, operation, arguments, *, rejected=False, delivery="eager process = subprocess.run( [sys.executable, '-P', '-m', 'tools.standards_engine.standards_engine.mcp', '--repo-root', str(self.root), '--purpose', purpose, - '--output-schemas', delivery], + '--output-schemas', delivery, '--no-observe'], env={**os.environ, 'PYTHONPATH': str(fixture.ROOT)}, text=True, input=''.join(json.dumps(value) + '\n' for value in messages), capture_output=True, timeout=90, diff --git a/tools/standards_engine/tests/test_analysis.py b/tools/standards_engine/tests/test_analysis.py index 0eeaf032..0ae8a9e6 100644 --- a/tools/standards_engine/tests/test_analysis.py +++ b/tools/standards_engine/tests/test_analysis.py @@ -687,8 +687,39 @@ def apply_ready(readiness_id: str) -> ProposalApplication: } ) ) + # Apply advances this unchanged main checkout to its publication. + self.assertEqual(applied.checkout.status, "current") return engine._authoring.read_application(applied.application.id) + def assert_checkout_presents( + application: ProposalApplication, + present: tuple[str, ...], + absent: tuple[str, ...] = (), + ) -> None: + # Interface 47: HEAD, index and worktree all present the exact + # published candidate, and no other local difference appears. + self.assertEqual( + git_output(repository, ("rev-parse", "HEAD")).strip(), + application.candidate.oid.encode("ascii"), + ) + for path in present: + self.assertEqual( + (repository / path).read_bytes(), + engine._repository.read_file( + application.candidate, RepositoryPath.parse(path) + ), + path, + ) + for path in absent: + self.assertFalse((repository / path).exists(), path) + self.assertEqual( + git_output( + repository, + ("status", "--porcelain=v1", "--untracked-files=all"), + ), + b"", + ) + def assert_commit( application: ProposalApplication, revision: ProposalRevision, @@ -798,7 +829,10 @@ def assert_commit( impact_path: "100644", }, ) - self.assertFalse((repository / standard_path).exists()) + assert_checkout_presents( + create_application, + (standard_path, policy_path, impact_path), + ) facade.close() engine = StandardsEngine.open_repository( @@ -933,7 +967,11 @@ def assert_commit( ) self.assertIn(b"[[tombstone]]", retired_policy) self.assertNotIn(b"[[policy_unit]]", retired_policy) - self.assertFalse((repository / standard_path).exists()) + assert_checkout_presents( + retire_application, + (policy_path,), + absent=(standard_path, impact_path), + ) facade.close() engine = StandardsEngine.open_repository( diff --git a/tools/standards_engine/tests/test_consumer_publication.py b/tools/standards_engine/tests/test_consumer_publication.py index 43021335..4d01fa31 100644 --- a/tools/standards_engine/tests/test_consumer_publication.py +++ b/tools/standards_engine/tests/test_consumer_publication.py @@ -11,13 +11,23 @@ import subprocess import sys import tempfile +import tomllib import unittest +from tools.standards_engine.standards_engine.tools import INTERFACE_CONTRACT from tools.standards_engine.tests.test_agent_workflow import ROOT, decisions, evidence, prepare_repository from tools.standards_engine.tests import test_proposal_consumers as fixtures from tools.standards_engine.tests.test_proposal_consumers import registration +def installed_interface_version(root): + # The server announces the interface declared by the repository it serves. + # Read that declaration instead of repeating a number that each interface + # edition would otherwise leave stale here. + with (root / INTERFACE_CONTRACT).open('rb') as source: + return tomllib.load(source)['interface_schema_version'] + + class ConsumerPublicationTest(unittest.TestCase): def setUp(self): self.temp = tempfile.TemporaryDirectory(prefix='consumer-publication-') @@ -36,14 +46,15 @@ def rpc(self, method, params, purpose='authoring'): ] process=subprocess.run( [sys.executable,'-P','-m','tools.standards_engine.standards_engine.mcp', - '--repo-root',str(self.root),'--purpose',purpose,'--advanced'], + '--repo-root',str(self.root),'--purpose',purpose,'--advanced','--no-observe'], input=''.join(json.dumps(m)+'\n' for m in messages),text=True,capture_output=True, env={**os.environ,'PYTHONPATH':str(ROOT)},timeout=180, ) self.assertEqual(process.returncode,0,process.stderr) self.assertEqual(process.stderr,'') responses={row['id']:row for row in map(json.loads,process.stdout.splitlines())} - self.assertIn('interface 47;',responses[1]['result']['instructions']) + self.assertIn(f'interface {installed_interface_version(self.root)};', + responses[1]['result']['instructions']) return responses[2] def call(self,name,arguments,purpose='authoring',error=False): diff --git a/tools/standards_engine/tests/test_coverage_publication.py b/tools/standards_engine/tests/test_coverage_publication.py index bd2ddeba..799682b4 100644 --- a/tools/standards_engine/tests/test_coverage_publication.py +++ b/tools/standards_engine/tests/test_coverage_publication.py @@ -610,6 +610,7 @@ def test_review_verify_apply_recover_and_read_without_original_database(self): str(repository), "--purpose", "authoring", + "--no-observe", ], input="".join(json.dumps(message) + "\n" for message in messages), text=True, diff --git a/tools/standards_engine/tests/test_discovered_workflow.py b/tools/standards_engine/tests/test_discovered_workflow.py index 78503e1a..da1691f7 100644 --- a/tools/standards_engine/tests/test_discovered_workflow.py +++ b/tools/standards_engine/tests/test_discovered_workflow.py @@ -52,7 +52,7 @@ def call(operation, arguments): {'jsonrpc': '2.0', 'method': 'notifications/initialized'}, selected] process = subprocess.run([sys.executable, '-m', 'tools.standards_engine.standards_engine.mcp', '--repo-root', str(self.root), - '--purpose', 'authoring', '--output-schemas', delivery], + '--purpose', 'authoring', '--output-schemas', delivery, '--no-observe'], input=''.join(json.dumps(message) + '\n' for message in messages), text=True, capture_output=True, check=True) self.assertEqual(process.stderr, '') diff --git a/tools/standards_engine/tests/test_immutable_work_reuse.py b/tools/standards_engine/tests/test_immutable_work_reuse.py index e8caaf76..aeb93f09 100644 --- a/tools/standards_engine/tests/test_immutable_work_reuse.py +++ b/tools/standards_engine/tests/test_immutable_work_reuse.py @@ -389,7 +389,7 @@ def test_stdio_cold_restarts_and_warm_calls_preserve_results(self): results=[] for _ in range(2): process=subprocess.run( - [sys.executable,'-P','-m','tools.standards_engine.standards_engine.mcp','--repo-root',str(self.root),'--purpose','authoring','--output-schemas','on-demand'], + [sys.executable,'-P','-m','tools.standards_engine.standards_engine.mcp','--repo-root',str(self.root),'--purpose','authoring','--output-schemas','on-demand','--no-observe'], input=''.join(json.dumps(item)+'\n' for item in messages),text=True,capture_output=True,check=True, env={**os.environ,'PYTHONPATH':str(ROOT)},cwd=self.temporary.name, ) diff --git a/tools/standards_engine/tests/test_input_discovery.py b/tools/standards_engine/tests/test_input_discovery.py index 312de9a5..1810b75c 100644 --- a/tools/standards_engine/tests/test_input_discovery.py +++ b/tools/standards_engine/tests/test_input_discovery.py @@ -283,7 +283,7 @@ def cold(arguments): messages = [request('initialize', {'protocolVersion': '2025-11-25', 'capabilities': {}, 'clientInfo': {'name': 'discovery-fixture', 'version': '1'}}), {'jsonrpc': '2.0', 'method': 'notifications/initialized'}, request('tools/call', {'name': 'describe_input', 'arguments': arguments}, identifier=2)] - process = subprocess.run([sys.executable, '-P', '-m', 'tools.standards_engine.standards_engine.mcp', '--repo-root', str(root), '--purpose', 'authoring', '--advanced'], + process = subprocess.run([sys.executable, '-P', '-m', 'tools.standards_engine.standards_engine.mcp', '--repo-root', str(root), '--purpose', 'authoring', '--advanced', '--no-observe'], input=''.join(json.dumps(m)+'\n' for m in messages), capture_output=True, text=True, env=environment, timeout=60) self.assertEqual(process.returncode, 0, process.stderr) self.assertEqual(process.stderr, '') diff --git a/tools/standards_engine/tests/test_interaction_observer.py b/tools/standards_engine/tests/test_interaction_observer.py index 01aa31b5..90b77b79 100644 --- a/tools/standards_engine/tests/test_interaction_observer.py +++ b/tools/standards_engine/tests/test_interaction_observer.py @@ -1,4 +1,8 @@ -"""Synthetic operator evidence; never records a real agent/model session.""" +"""Synthetic operator evidence; never records a real agent/model session. + +Every recorder here writes to a temporary directory. Child processes get a +temporary HOME so the always-on default can never reach a developer's database. +""" from __future__ import annotations from contextlib import redirect_stderr @@ -6,17 +10,20 @@ import json import os from pathlib import Path +import re import sqlite3 +import stat import subprocess import sys import tempfile +import threading import time import unittest from unittest.mock import patch from tools.standards_engine.standards_engine.interaction_observer import ( - APPLICATION_ID, InputShape, InteractionRecorder, RecorderConfig, SCHEMA_VERSION, - report, + APPLICATION_ID, DEFAULT_DATABASE_NAME, InputShape, InteractionRecorder, RecorderConfig, + SCHEMA_VERSION, StorageRejected, TRUNCATED, default_database, report, text_report, ) from tools.standards_engine.standards_engine.mcp import MCPServer, serve from tools.standards_engine.standards_engine.tools import AgentToolFacade @@ -24,6 +31,52 @@ MODULE = 'tools.standards_engine.standards_engine.interaction_observer' MCP_MODULE = 'tools.standards_engine.standards_engine.mcp' +SNAPSHOT = {'kind': 'snapshot-handle', 'id': 'snapshot:v1:5e17e1a1-0000-4000-8000-000000000001', 'schema_version': 5} +ANALYSIS = {'kind': 'analysis-handle', 'id': 'analysis:sha256:' + 'ab' * 32, 'schema_version': 7} +# The request metadata Codex attaches to every tool call, with synthetic values. +CODEX_META = { + 'callId': 'exec-SENTINEL_CALL', 'threadId': 'SENTINEL_THREAD', 'sessionId': 'SENTINEL_SESSION', 'progressToken': 9, + 'x-codex-turn-metadata': { + 'session_id': 'SENTINEL_SESSION', 'thread_id': 'SENTINEL_THREAD', 'turn_id': 'SENTINEL_TURN', + 'model': 'SENTINEL_MODEL', 'thread_source': 'subagent', + 'workspaces': {'/SENTINEL/WORKSPACE': {'associated_remote_urls': {'origin': 'git@example.invalid:SENTINEL/REPO.git'}}}}, +} +V1_DDL = ( + 'CREATE TABLE sessions (id TEXT PRIMARY KEY, started REAL NOT NULL, runtime TEXT NOT NULL, deployment TEXT NOT NULL, catalog TEXT NOT NULL, pruned INTEGER NOT NULL DEFAULT 0)', + 'CREATE TABLE interactions (id TEXT PRIMARY KEY, session TEXT NOT NULL REFERENCES sessions(id), ordinal INTEGER NOT NULL, started REAL NOT NULL, method TEXT NOT NULL, operation TEXT NOT NULL, input_shape TEXT NOT NULL, shape_digest TEXT NOT NULL, discovery_operation TEXT, catalog_state TEXT NOT NULL, request_bytes INTEGER, scope TEXT NOT NULL, UNIQUE(session, ordinal))', + "CREATE TABLE events (interaction TEXT PRIMARY KEY REFERENCES interactions(id) ON DELETE CASCADE, phase TEXT NOT NULL CHECK(phase IN ('end','error','incomplete')), finished REAL NOT NULL, duration_ms REAL NOT NULL, response_bytes INTEGER, outcome TEXT NOT NULL, feedback TEXT NOT NULL)", + 'CREATE INDEX interaction_age ON interactions(started)', +) + + +def environment(home: Path, **extra: str) -> dict: + """A child environment whose default observation location is under `home`.""" + env = {name: value for name, value in os.environ.items() + if name not in ('XDG_DATA_HOME', 'STANDARDS_ENGINE_OBSERVE')} + return {**env, 'PYTHONPATH': str(ROOT), 'HOME': str(home), **extra} + + +def rejection(code, pointer=None, keyword='fact', **extra): + value = {'kind': 'rejected-result', 'code': code, 'outcome': 'invalid', 'message': 'SENTINEL_MESSAGE', + 'details': {'SENTINEL_DETAIL': 'SENTINEL_DETAIL'}, 'next_operations': [], **extra} + if pointer is not None: + value['input_feedback'] = {'issues': [{'instance_pointer': pointer, 'keyword': keyword, + 'message': 'SENTINEL_MESSAGE'}], 'truncated': False} + return value + + +def route_result(unresolved=('routing.topics',)): + return {'kind': 'compact-route-result', 'status': 'needs-facts' if unresolved else 'complete', 'snapshot': SNAPSHOT, + 'summary': 'SENTINEL_SUMMARY', + 'reading_plan': [{'target': 'core', 'state': 'selected'}, {'target': 'router', 'state': 'selected'}, + {'target': 'topic.security', 'state': 'conditional'}], + 'unresolved_questions': [{'fact': {'id': name, 'prompt': 'SENTINEL_PROMPT'}} for name in unresolved], + 'content': {'offset': 0, 'total': 2, 'items': [read_result('core')]}} + + +def read_result(target): + return {'kind': 'compact-read-result', 'snapshot': SNAPSHOT, 'content': 'SENTINEL_STANDARD_TEXT', + 'policy': {'handle': {'kind': 'snapshot-child-handle', 'child_kind': 'policy', 'child_id': target}}} class ObserverTests(unittest.TestCase): @@ -33,9 +86,9 @@ def setUp(self): self.database = Path(self.temporary.name) / 'observations.sqlite' self.diagnostics = io.StringIO() - def server(self, **options): + def server(self, *, advanced=False, **options): with redirect_stderr(self.diagnostics): - server = MCPServer(ROOT, purpose='authoring', + server = MCPServer(ROOT, purpose='authoring', advanced=advanced, observation=RecorderConfig(self.database, 'synthetic', **options)) self.addCleanup(server.close) return server @@ -45,15 +98,23 @@ def rows(self, sql): db.row_factory = sqlite3.Row return [dict(row) for row in db.execute(sql)] - def test_opt_in_does_not_change_catalog_or_results(self): + def call(self, server, name, arguments, value, meta=None): + """One tool call whose Engine result is `value`; no store is opened.""" + params = {'name': name, 'arguments': arguments, **({'_meta': meta} if meta else {})} + with patch.object(server, '_request', return_value=server._tool_result(value)): + return server.dispatch(request('tools/call', params)) + + def stored_bytes(self): + return b''.join(path.read_bytes() for path in self.database.parent.iterdir()) + + def test_recording_does_not_change_catalog_instructions_or_results(self): plain = MCPServer(ROOT, purpose='authoring') self.addCleanup(plain.close) observed = self.server() self.assertEqual(plain.tools, observed.tools) - for server in (plain, observed): - initialize(server) # A fixed process identity makes these full public responses comparable. observed._runtime_identity._instance = plain._runtime_identity._instance + self.assertEqual(initialize(plain), initialize(observed)) for message in (request('tools/list'), request('tools/call', {'name': 'runtime_info'}), request('tools/call', {'name': 'describe_input', 'arguments': {'operation': 'apply'}}), request('tools/call', {'name': 'runtime_info', 'arguments': {'secret': 'private'}}), @@ -68,6 +129,7 @@ def test_opt_in_does_not_change_catalog_or_results(self): self.assertEqual(result['sessions'][0]['runtime']['catalog_digest'], plain._runtime_identity.metadata()['catalog_digest']) self.assertNotIn('observer', json.dumps(observed.tools)) + self.assertEqual(self.diagnostics.getvalue(), '', 'a healthy recorder is silent') def test_shape_and_feedback_disclose_no_values_or_dynamic_keys(self): schema = {'type': 'object', 'properties': { @@ -85,18 +147,165 @@ def test_shape_and_feedback_disclose_no_values_or_dynamic_keys(self): self.assertNotIn(secret, encoded) self.assertEqual(shape.project(value)['fields']['body'], 'redacted') self.assertEqual(shape.project(value)['fields']['facts']['fields'], {}) - feedback = shape.feedback({'issues': [ + issues = {'issues': [ {'instance_pointer': '/facts/facts/body', 'keyword': 'type', 'message': 'CREDENTIAL'}, {'instance_pointer': '/items/8/selector', 'keyword': 'required'}, {'instance_pointer': '/body/token', 'keyword': 'CREDENTIAL'}, - ]}) - self.assertEqual(feedback, [{'pointer': '/facts/*/*', 'keyword': 'type'}, - {'pointer': '/items/*/selector', 'keyword': 'required'}, - {'pointer': '/body/*', 'keyword': 'other'}]) + {'instance_pointer': '/facts/routing.topics', 'keyword': 'fact'}, + {'instance_pointer': '/items/routing.topics', 'keyword': 'evidence_ref'}, + ]} + self.assertEqual(shape.feedback(issues), [ + {'pointer': '/facts/*/*', 'keyword': 'type'}, {'pointer': '/items/*/selector', 'keyword': 'required'}, + {'pointer': '/body/*', 'keyword': 'other'}, {'pointer': '/facts/*', 'keyword': 'fact'}, + {'pointer': '/items/*', 'keyword': 'evidence_ref'}]) + # A fact name the same call already recorded may locate its own rejection; + # the same text under any other map stays a wildcard. + named = shape.feedback(issues, frozenset({'routing.topics'})) + self.assertEqual([issue['pointer'] for issue in named][3:], ['/facts/routing.topics', '/items/*']) # Deep/large adversarial data is bounded without traversing unknown maps. projected = shape.project({'items': [{'selector': 'x' * 1_000_000}] * 100_000}) self.assertLess(len(json.dumps(projected)), 8192) + def test_navigation_values_results_sizes_and_task_identity_are_recorded(self): + server = self.server() + initialize(server) + facts = {'routing.activities': ['implementation', 'testing'], 'python': True, 'task': 'p' * 300} + items = [{'target': 'core'}, {'target': 'router', 'detail': 'full', 'include_routing': True, 'extra': 'DROPPED'}] + calls = ( + ('route', {'facts': facts, 'content': {'limit': 1}, 'detail': 'compact'}, route_result()), + ('read', {'snapshot': SNAPSHOT, 'target': 'core', 'detail': 'full', 'handle': {'kind': 'DROPPED'}, + 'request': {'kind': 'route', 'facts': {'DROPPED': 'DROPPED'}}, 'facts': {'DROPPED': True}}, + read_result('core')), + ('read_many', {'snapshot': SNAPSHOT, 'items': items}, + {'kind': 'read-many-result', 'snapshot': SNAPSHOT, 'items': [read_result('core'), read_result('router')]}), + ('related', {'snapshot': SNAPSHOT, 'target': 'core', 'groups': ['semantic'], 'direction': 'both', 'transitive': False}, + rejection('GRAPH.UNKNOWN_GROUP')), + ('query_proposal', {'revision': {'kind': 'proposal-revision-handle', 'id': 'proposal-revision:sha256:' + 'cd' * 32}, + 'request': {'kind': 'route', 'facts': {'nested.fact': {'type': 'string', 'state': 'known', 'value': 'nested-value'}}}}, + rejection('AGGREGATE.UNAVAILABLE')), + ) + for name, arguments, value in calls: + self.call(server, name, arguments, value, CODEX_META) + server.dispatch(request('tools/call', {'name': 'describe_input', 'arguments': { + 'operation': 'route', 'selector': 'RouteCall', 'limit': 2, 'expected_catalog': 'sha256:' + '0' * 64}})) + recorded = {row['operation']: row for row in self.rows( + 'SELECT i.*,e.outcome,e.result,e.result_bytes FROM interactions i JOIN events e ON e.interaction=i.id ' + "WHERE i.method='tools/call'")} + values = {name: json.loads(row['arguments']) for name, row in recorded.items()} + results = {name: json.loads(row['result']) for name, row in recorded.items()} + self.assertEqual(values['route'], {'detail': 'compact', 'content': {'limit': 1}, 'facts': { + 'routing.activities': ['implementation', 'testing'], 'python': True, 'task': 'p' * 200 + TRUNCATED}}) + self.assertEqual(results['route'], { + 'kind': 'compact-route-result', 'status': 'needs-facts', 'selected': ['core', 'router'], + 'unresolved': ['routing.topics'], 'content': {'offset': 0, 'total': 2, 'items': 1, 'targets': ['core']}}) + self.assertEqual(recorded['route']['outcome'], 'needs-facts') + reused = values['read']['snapshot'] + self.assertRegex(reused, '^[0-9a-f]{12}$') + self.assertEqual(values['read'], {'snapshot': reused, 'target': 'core', 'detail': 'full'}) + self.assertEqual(values['read_many'], {'snapshot': reused, 'items': [ + {'target': 'core'}, {'target': 'router', 'detail': 'full', 'include_routing': True}]}) + self.assertEqual(results['read_many'], {'kind': 'read-many-result', 'targets': ['core', 'router']}) + self.assertEqual(values['related'], {'snapshot': reused, 'target': 'core', 'groups': ['semantic'], + 'direction': 'both', 'transitive': False}) + self.assertEqual(results['related'], {'kind': 'rejected-result', 'code': 'GRAPH.UNKNOWN_GROUP', 'outcome': 'invalid'}) + self.assertEqual(values['query_proposal'], {'revision': {'kind': 'proposal-revision-handle'}, 'request': { + 'kind': 'route', 'facts': {'nested.fact': {'type': 'string', 'state': 'known', 'value': 'nested-value'}}}}) + self.assertEqual(values['describe_input'], {'operation': 'route', 'selector': 'RouteCall', 'limit': 2}) + self.assertEqual(recorded['describe_input']['catalog_state'], 'differs') + # Sizes: the arguments object alone and one serialization of structuredContent. + self.assertEqual(recorded['read']['arguments_bytes'], len(json.dumps(calls[1][1], separators=(',', ':')))) + self.assertEqual(recorded['read']['result_bytes'], len(json.dumps(read_result('core')))) + self.assertGreater(recorded['read']['request_bytes'], recorded['read']['arguments_bytes']) + # Task identity: one salted hash per client thread, never the id itself. + threads = {(row['thread'], row['origin']) for name, row in recorded.items() if name != 'describe_input'} + self.assertEqual(len(threads), 1) + thread, origin = threads.pop() + self.assertRegex(thread, '^[0-9a-f]{16}$') + self.assertEqual(origin, 'subagent') + self.assertEqual((recorded['describe_input']['thread'], recorded['describe_input']['origin']), (None, 'unknown')) + self.assertEqual(self.rows('SELECT client_name,client_version FROM sessions'), + [{'client_name': 'test', 'client_version': '1'}]) + stored = self.stored_bytes() + for private in (b'SENTINEL', b'5e17e1a1', b'cdcdcdcd', b'DROPPED'): + self.assertNotIn(private, stored) + # The salt belongs to the database: another process maps the thread to the same hash. + second = self.server() + self.call(second, 'read', {'target': 'core'}, read_result('core'), CODEX_META) + self.assertEqual({row['thread'] for row in self.rows('SELECT thread FROM interactions WHERE thread IS NOT NULL')}, {thread}) + + def test_authoring_prose_never_reaches_disk_while_route_facts_do(self): + server = self.server(advanced=True) + initialize(server) + prose = {name: f'SENTINEL_{name.upper()}' for name in ( + 'body', 'title', 'summary', 'rationale', 'intent', 'heading', 'condition', 'path', 'label', + 'evidence', 'applies', 'verification', 'description', 'provenance', 'value', 'tool')} + evidence = {'id': prose['evidence'], 'digest': 'sha256:' + 'ef' * 32, 'provider_contract': 'review-evidence', + 'provider_contract_version': '1'} + change_set = {'purpose': {'summary': prose['summary'], 'rationale': prose['rationale'], 'evidence': [{'evidence_ref': prose['label']}]}, 'edits': [ + {'kind': 'revise-policy-unit', 'policy': 'workflow.planning.written-plan-applicability', 'title': prose['title'], 'body': prose['body'], + 'semantics': {'kind': 'change', 'accepted_semantic_revision': 1, 'proposed_semantic_revision': 2, 'intent': prose['intent']}}, + {'kind': 'create-standard', 'requires': ['core'], 'specializes': [], 'standard': { + 'id': 'topic.recorded-identifier', 'title': prose['title'], 'body': prose['body'], 'level': 'SHOULD', 'role': 'topic', + 'applies_when': prose['applies'], 'does_not_apply_when': prose['applies'], 'verification': prose['verification']}, + 'policy_units': [{'id': 'topic.recorded-identifier.unit', 'heading_chain': [prose['heading']], 'intent': prose['intent'], + 'semantic_revision': 1, 'aliases': [], 'predecessors': [], 'successors': []}]}, + {'kind': 'put-policy-relationship', 'relationship': { + 'source_policy': 'core.application-guidance', 'consumer': 'prompts/planning.md', 'relation': 'prompt-projection', + 'evidence_owner': 'review:consumer', 'rationale': prose['rationale'], 'applicability': {'operator': 'always'}}}, + {'kind': 'register-consumer', 'consumer': 'consumer.recorded', 'artifact_kind': 'documentation', 'authority': 'projection', + 'path': prose['path']}, + {'kind': 'put-routing-rule', 'rationale': prose['rationale'], 'rule': { + 'id': 'rule.recorded', 'target': 'topic.security', 'condition': prose['condition'], + 'when': {'operator': 'equals', 'fact': 'routing.topics', 'value': prose['value']}}}, + # Prose placed in an identifier field is dropped by its shape. + {'kind': 'audit-policy-unit', 'policy': 'SENTINEL PROSE in an identifier field', 'rationale': prose['rationale']}, + ]} + child = {'kind': 'analysis-child-handle', 'analysis': ANALYSIS, 'child_kind': 'obligation', 'child_id': 'sha256:' + '12' * 32} + calls = ( + ('propose', {'snapshot': SNAPSHOT, 'detail': 'full', 'change_set': change_set, 'evidence': {prose['label']: evidence}}), + ('revise', {'context': ANALYSIS, 'change_set': change_set}), + ('review', {'context': ANALYSIS, 'decisions': [ + {'owner': 'consumer', 'decision': 'accept', 'rationale': prose['rationale'], 'evidence': [evidence]}]}), + ('resolve_workflow', {'context': ANALYSIS, 'submission': { + 'kind': 'coverage-attestation', 'claim': {'requirement': child, 'conclusion': 'complete', 'rationale': prose['rationale'], + 'auditor_provenance': prose['provenance'], 'evidence': [evidence], + 'explicit_exclusions': [evidence]}}}), + ('recover', {'context': ANALYSIS, 'action': 'observe'}), + ('maintain_evidence', {'kind': 'maintain-evidence', 'expected_revision': prose['value'], 'apply': False, 'evidence': [evidence], + 'plan': {'suite_descriptions': [{'suite': prose['label'], 'description': prose['description']}], + 'replacement_evidence_rationale': prose['rationale'], + 'consumer_registrations': [{'path': prose['path'], 'rationale': prose['rationale'], + 'artifact_kind': 'fixture', 'relation': 'fixture-projection'}]}}), + ) + for name, arguments in calls: + self.call(server, name, arguments, rejection('INTERFACE.INVALID_ARGUMENTS', '/change_set/purpose/' + prose['label'], 'minItems'), + CODEX_META) + server.dispatch(request('tools/call', {'name': prose['tool'], 'arguments': {'body': prose['body']}}, identifier=prose['value'])) + # Route facts are kept as sent, in the focused form and inside a preview request. + self.call(server, 'route', {'facts': {'OOV_NAME_python': True, 'task': 'OOV free text about a parser', + 'routing.activities': ['implementation', 'OOV_VALUE_testing']}}, + rejection('ROUTE.INPUT_INVALID', '/facts')) + self.call(server, 'preview_application', {'revision': {'kind': 'proposal-revision-handle', 'id': 'proposal-revision:sha256:' + '34' * 32}, + 'request': {'kind': 'route', 'facts': {'OOV_NESTED': {'type': 'string', 'state': 'known', 'value': 'OOV_NESTED_VALUE'}}}}, + rejection('ROUTE.INPUT_INVALID', '/request/facts')) + for closed in (False, True): + stored = self.stored_bytes() # The main file, and the write-ahead log while it exists. + for private in (b'SENTINEL', b'abababab', b'efefefef', b'12121212', b'34343434', b'5e17e1a1'): + self.assertNotIn(private, stored, (private, closed)) + for kept in (b'OOV_NAME_python', b'OOV free text about a parser', b'OOV_VALUE_testing', b'OOV_NESTED_VALUE', + b'workflow.planning.written-plan-applicability', b'topic.recorded-identifier.unit', + b'prompts/planning.md', b'consumer.recorded', b'revise-policy-unit', b'analysis-handle', + b'INTERFACE.INVALID_ARGUMENTS'): + self.assertIn(kept, stored, (kept, closed)) + server.close() + summary = report(self.database) + self.assertNotIn('SENTINEL', json.dumps(summary) + text_report(summary)) + propose = json.loads(self.rows("SELECT arguments FROM interactions WHERE operation='propose'")[0]['arguments']) + self.assertEqual(propose['change_set']['edits'][0], { + 'kind': 'revise-policy-unit', 'policy': 'workflow.planning.written-plan-applicability', 'semantics': {'kind': 'change'}}) + self.assertEqual(propose['change_set']['edits'][5], {'kind': 'audit-policy-unit'}) + self.assertEqual((propose['snapshot'], propose['detail']), ({'kind': 'snapshot-handle'}, 'full')) + def test_privacy_at_real_dispatch_and_disk_boundary(self): server = self.server() initialize(server) @@ -104,7 +313,8 @@ def test_privacy_at_real_dispatch_and_disk_boundary(self): server.dispatch(request('tools/call', {'name': 'runtime_info', 'arguments': { 'expected_catalog': secrets[0], secrets[1]: secrets[2]}}, identifier=secrets[0])) server.dispatch(request('tools/call', {'name': secrets[1], 'arguments': {'body': secrets[2]}})) - server.dispatch(request('initialize', {'clientInfo': {'name': secrets[0]}})) + # Only the first initialize names the client; other clientInfo members are never read. + server.dispatch(request('initialize', {'clientInfo': {'name': secrets[0], 'title': secrets[1]}})) server.close() for path in self.database.parent.iterdir(): contents = path.read_bytes() @@ -120,13 +330,17 @@ def test_privacy_at_real_dispatch_and_disk_boundary(self): def test_malformed_envelopes_and_parse_errors_do_not_disable_recorder(self): server = self.server() for value in (None, [], {}, {'method': 'ping'}, {'id': 1, 'method': [], 'jsonrpc': '2.0'}, - request('tools/call', {'name': []}), request('ping', [])): + request('tools/call', {'name': []}), request('ping', []), + request('tools/call', {'name': 'route', 'arguments': [], '_meta': []}), + request('tools/call', {'name': 'read_many', 'arguments': {'items': 'x', 'snapshot': []}, + '_meta': {'x-codex-turn-metadata': {'thread_id': [], 'thread_source': {}}}})): server.dispatch(value) output = io.StringIO() serve(server, io.StringIO('{BAD JSON\n'), output) self.assertEqual(json.loads(output.getvalue())['error']['code'], -32700) self.assertEqual(server._observer.health['state'], 'healthy') - self.assertEqual(report(self.database)['retained_starts'], 8) + self.assertEqual(server._observer.health['lost'], 0) + self.assertEqual(report(self.database)['retained_starts'], 10) self.assertIn('parse-error', {row['outcome'] for row in self.rows('SELECT outcome FROM events')}) def test_tool_exception_interruption_and_delivery_failure(self): @@ -146,33 +360,89 @@ def broken(response): events = self.rows('SELECT phase,outcome,response_bytes FROM events') self.assertEqual([r['outcome'] for r in events], ['response', 'tool-exception', 'unknown', 'unknown']) self.assertEqual(sum(r['phase'] == 'incomplete' for r in events), 2) - self.assertNotIn('SENSITIVE_', self.database.read_bytes().decode(errors='ignore')) + self.assertNotIn(b'SENSITIVE_', self.stored_bytes()) - def test_unavailable_locked_and_failed_health_sink_preserve_responses(self): - unavailable = Path(self.temporary.name) / 'missing' / 'db' - with redirect_stderr(io.StringIO()): - server = MCPServer(ROOT, purpose='authoring', observation=RecorderConfig(unavailable, 'synthetic')) - self.addCleanup(server.close) - self.assertEqual(server.dispatch(request('ping'))['result'], {}) - self.assertEqual(server._observer.health['state'], 'degraded') + def test_locked_database_delays_rows_but_never_drops_or_slows_a_call(self): observed = self.server() with sqlite3.connect(self.database, isolation_level=None) as lock: lock.execute('BEGIN IMMEDIATE') before = time.monotonic() - response = observed.dispatch(request('ping')) + responses = [observed.dispatch(request('ping')) for _ in range(3)] elapsed = time.monotonic() - before + self.assertEqual(report(self.database)['retained_starts'], 0) lock.execute('ROLLBACK') - self.assertEqual(response['result'], {}) - self.assertEqual(observed._observer.health['reason'], 'storage-locked') - self.assertLess(elapsed, 1, '20ms SQLite busy policy must not wait indefinitely') + self.assertEqual([response['result'] for response in responses], [{}, {}, {}]) + self.assertLess(elapsed, 1, 'six lock waits of 20 ms must not delay three calls materially') + health = observed._observer.health + self.assertEqual((health['state'], health['reason'], health['backlog']), ('degraded', 'storage-locked', 6)) + # The next call delivers the whole backlog in order. + observed.dispatch(request('ping')) + self.assertEqual((health['state'], health['backlog'], health['lost']), ('healthy', 0, 0)) + self.assertEqual((health['recorded_starts'], health['recorded_terminals']), (4, 4)) + self.assertEqual([row['ordinal'] for row in self.rows('SELECT ordinal FROM interactions ORDER BY started')], [1, 2, 3, 4]) + # Orderly close waits longer for a writer than a call may. + lock = sqlite3.connect(self.database, isolation_level=None, check_same_thread=False) + lock.execute('BEGIN IMMEDIATE') + observed.dispatch(request('ping')) + self.assertEqual(health['backlog'], 2) + release = threading.Timer(0.3, lambda: (lock.execute('ROLLBACK'), lock.close())) + release.start() + observed.close() + release.join() + summary = report(self.database) + self.assertEqual((summary['retained_starts'], summary['retained_terminals'], summary['lost_rows']), (5, 5, 0)) + notes = [json.loads(line.split(' ', 2)[2]) for line in self.diagnostics.getvalue().splitlines()] + # One line per change of health, then the totals at close; nothing while healthy. + self.assertEqual([(note['state'], note['reason']) for note in notes], + [('degraded', 'storage-locked'), ('healthy', None), ('degraded', 'storage-locked'), + ('healthy', None), ('healthy', None)]) + self.assertEqual((notes[-1]['recorded_starts'], notes[-1]['recorded_terminals'], notes[-1]['lost']), (5, 5, 0)) + + def test_backlog_overflow_is_counted_and_the_count_is_persisted(self): + observed = self.server() + with patch(MODULE + '.MAX_BACKLOG', 4): + with sqlite3.connect(self.database, isolation_level=None) as lock: + lock.execute('BEGIN IMMEDIATE') + for _ in range(5): + observed.dispatch(request('ping')) + lock.execute('ROLLBACK') + self.assertEqual((observed._observer.health['backlog'], observed._observer.health['lost']), (4, 6)) + observed.dispatch(request('ping')) + # Calls 1-3 overflowed (6 rows); call 4 lost its start to the sixth call and + # so its terminal as well (2 rows). Calls 5 and 6 are complete. + health = observed._observer.health + self.assertEqual((health['state'], health['lost'], health['recorded_starts'], health['recorded_terminals']), + ('healthy', 8, 2, 2)) + summary = report(self.database) + self.assertEqual((summary['retained_starts'], summary['retained_terminals'], summary['lost_rows']), (2, 2, 8)) + self.assertIn('8 rows lost', text_report(summary)) + + def test_unavailable_storage_reconnects_later_and_reports_what_it_never_wrote(self): + unavailable = Path(self.temporary.name) / 'missing' / 'db' + with redirect_stderr(io.StringIO()): + server = MCPServer(ROOT, purpose='authoring', observation=RecorderConfig(unavailable, 'synthetic')) + self.addCleanup(server.close) + self.assertEqual(server.dispatch(request('ping'))['result'], {}) + health = server._observer.health + self.assertEqual((health['state'], health['reason'], health['backlog']), ('degraded', 'storage-unavailable', 2)) + self.assertFalse(unavailable.parent.exists(), 'an operator-supplied directory is never created') + unavailable.parent.mkdir() + self.assertEqual(server.dispatch(request('ping'))['result'], {}) + self.assertEqual((health['state'], health['recorded_starts'], health['recorded_terminals'], health['lost']), + ('healthy', 2, 2, 0)) + # A process that can never write says so on stderr at close, with its count. class BrokenDiagnostics: def write(self, value): raise OSError('private') def flush(self): raise OSError('private') - recorder = InteractionRecorder(RecorderConfig(unavailable, 'synthetic'), {}, [], diagnostics=BrokenDiagnostics()) - recorder.close() - self.assertEqual(recorder.health['state'], 'degraded') + for diagnostics in (self.diagnostics, BrokenDiagnostics()): + recorder = InteractionRecorder(RecorderConfig(Path(self.temporary.name) / 'absent' / 'db', 'synthetic'), + {'instance_id': 'never', 'catalog_digest': 'c'}, [], diagnostics=diagnostics) + recorder.finish(recorder.begin(request('ping'), request_bytes=1, scope='constructed'), {'result': {}}) + recorder.close() + self.assertEqual((recorder.health['state'], recorder.health['lost']), ('degraded', 2)) + self.assertIn('"lost":2', self.diagnostics.getvalue().splitlines()[-1]) def test_full_storage_after_domain_result_never_repeats_work(self): server = self.server() @@ -195,6 +465,25 @@ def domain(method, params): self.assertEqual(calls, ['ping']) self.assertEqual(server._observer.health['reason'], 'storage-full') self.assertEqual(report(self.database)['unmatched_starts'], 1) + # The terminal waited in the backlog; a later call reconnects and delivers it. + server.dispatch(request('ping')) + self.assertEqual(server._observer.health['state'], 'healthy') + self.assertEqual(report(self.database)['unmatched_starts'], 0) + + def test_byte_cap_evicts_the_oldest_calls_instead_of_refusing_new_ones(self): + recorder = self.server(max_bytes=1024 * 1024)._observer + message = request('tools/call', {'name': 'route', 'arguments': { + 'facts': {f'routing.fact-{index}': ['v' * 150] * 2 for index in range(8)}}}) + for _ in range(900): + recorder.finish(recorder.begin(message, request_bytes=1, scope='constructed'), {'result': {}}, response_bytes=1) + health = recorder.health + self.assertEqual((health['state'], health['lost'], health['recorded_starts']), ('healthy', 0, 900)) + self.assertGreater(health['pruned_interactions'], 0) + retained = self.rows('SELECT min(ordinal) AS oldest,max(ordinal) AS newest,count(*) AS count FROM interactions')[0] + self.assertEqual(retained['newest'], 900) + self.assertEqual(retained['count'], 900 - health['pruned_interactions']) + recorder.close() + self.assertLessEqual(self.database.stat().st_size, 1024 * 1024) def test_migration_reopen_unsupported_corrupt_and_spoofed(self): server = self.server() @@ -203,14 +492,17 @@ def test_migration_reopen_unsupported_corrupt_and_spoofed(self): with sqlite3.connect(self.database) as db: self.assertEqual(db.execute('PRAGMA user_version').fetchone()[0], SCHEMA_VERSION) self.assertEqual(db.execute('PRAGMA application_id').fetchone()[0], APPLICATION_ID) + self.assertEqual(db.execute('PRAGMA journal_mode').fetchone()[0], 'wal') + self.assertRegex(db.execute("SELECT value FROM meta WHERE key='salt'").fetchone()[0], '^[0-9a-f]{32}$') again = self.server() again.dispatch(request('ping')) again.close() self.assertEqual(report(self.database)['retained_starts'], 2) - for name, setup in ( - ('future', 'PRAGMA user_version=999'), - ('unrelated', 'CREATE TABLE business (secret TEXT)'), - ('spoofed', f'PRAGMA user_version=1; PRAGMA application_id={APPLICATION_ID}; CREATE TABLE sessions(id TEXT)'), + for name, setup, reason in ( + ('future', 'PRAGMA user_version=999', 'unsupported-database-schema'), + ('unrelated', 'CREATE TABLE business (secret TEXT)', 'unsupported-database-schema'), + ('spoofed', f'PRAGMA user_version={SCHEMA_VERSION}; PRAGMA application_id={APPLICATION_ID}; CREATE TABLE sessions(id TEXT)', + 'invalid-database-schema'), ): target = Path(self.temporary.name) / name with sqlite3.connect(target) as db: @@ -219,7 +511,7 @@ def test_migration_reopen_unsupported_corrupt_and_spoofed(self): before = target.read_bytes() with redirect_stderr(io.StringIO()): recorder = InteractionRecorder(RecorderConfig(target, 'synthetic'), {}, []) - self.assertEqual(recorder.health['state'], 'degraded') + self.assertEqual((recorder.health['state'], recorder.health['reason']), ('degraded', reason)) self.assertEqual(target.read_bytes(), before) corrupt = Path(self.temporary.name) / 'corrupt' corrupt.write_bytes(b'not sqlite sensitive synthetic') @@ -229,7 +521,25 @@ def test_migration_reopen_unsupported_corrupt_and_spoofed(self): self.assertEqual(recorder.health['state'], 'degraded') self.assertEqual(corrupt.read_bytes(), before) - def test_unsupported_journal_and_page_budget_never_publish_migration(self): + def test_schema_v1_database_is_refused_and_left_untouched(self): + with sqlite3.connect(self.database) as db: + db.executescript(';'.join(V1_DDL) + f';PRAGMA application_id={APPLICATION_ID};PRAGMA user_version=1;' + "INSERT INTO sessions VALUES('historical',1.0,'{}','codex-local','{}',0)") + self.database.chmod(0o600) + before = self.database.read_bytes() + server = self.server() + self.assertEqual(server.dispatch(request('ping'))['result'], {}) + health = server._observer.health + self.assertEqual((health['state'], health['reason'], health['recorded_starts']), + ('degraded', 'unsupported-database-schema', 0)) + server.close() + self.assertEqual(self.database.read_bytes(), before) + self.assertEqual(sorted(path.name for path in self.database.parent.iterdir()), [self.database.name]) + with self.assertRaises(StorageRejected): + report(self.database) + self.assertIn('unsupported-database-schema', self.diagnostics.getvalue()) + + def test_foreign_wal_store_and_page_budget_never_publish_migration(self): for name, pragma in (('wal', 'PRAGMA journal_mode=WAL'),): path = Path(self.temporary.name) / name with sqlite3.connect(path) as db: @@ -238,7 +548,7 @@ def test_unsupported_journal_and_page_budget_never_publish_migration(self): before = path.read_bytes() recorder = InteractionRecorder(RecorderConfig(path, 'synthetic', max_bytes=1024*1024), {}, [], diagnostics=io.StringIO()) - self.assertEqual(recorder.health['state'], 'degraded') + self.assertEqual((recorder.health['state'], recorder.health['reason']), ('degraded', 'nonpristine-database')) self.assertEqual(path.read_bytes(), before) with sqlite3.connect(path) as db: self.assertEqual(db.execute('PRAGMA user_version').fetchone()[0], 0) @@ -307,6 +617,24 @@ def test_private_regular_file_and_no_symlink_or_hardlink(self): self.assertEqual(recorder.health['reason'], 'unsafe-database-file') self.assertEqual(public.stat().st_mode & 0o777, 0o644) + def test_default_location_follows_xdg_and_is_created_private(self): + home = Path(self.temporary.name) / 'home' + with patch.dict(os.environ, {'HOME': str(home), 'XDG_DATA_HOME': ''}): + self.assertEqual(default_database(), home / '.local/share/standards-engine/observations' / DEFAULT_DATABASE_NAME) + with patch.dict(os.environ, {'XDG_DATA_HOME': 'relative/ignored'}): + self.assertEqual(default_database().parents[2], home / '.local/share') + with patch.dict(os.environ, {'XDG_DATA_HOME': str(home / 'xdg')}): + recorder = InteractionRecorder(RecorderConfig(), {'instance_id': 'default', 'catalog_digest': 'c'}, [], + diagnostics=self.diagnostics) + recorder.finish(recorder.begin(request('ping'), request_bytes=1, scope='constructed'), {'result': {}}) + recorder.close() + database = home / 'xdg/standards-engine/observations' / DEFAULT_DATABASE_NAME + self.assertEqual(default_database(), database) + self.assertEqual(recorder.health['recorded_terminals'], 1) + self.assertEqual(stat.S_IMODE(database.stat().st_mode), 0o600) + self.assertEqual(stat.S_IMODE(database.parent.stat().st_mode), 0o700) + self.assertEqual(self.diagnostics.getvalue(), '') + def test_count_age_and_session_retention_and_read_only_reports(self): # Every process-like recorder has its own session; old empty sessions must # disappear along with their interactions rather than accumulating forever. @@ -317,6 +645,7 @@ def test_count_age_and_session_retention_and_read_only_reports(self): summary = report(self.database, limit=1) self.assertEqual(summary['retained_starts'], 16) self.assertEqual(len(self.rows('SELECT id FROM sessions')), 16) + self.assertEqual(len(self.rows('SELECT digest FROM catalogs')), 1, 'one catalog row serves every session') self.assertTrue(summary['session_rows_truncated']) with sqlite3.connect(self.database) as db: db.execute('UPDATE interactions SET started=0') @@ -330,6 +659,8 @@ def test_count_age_and_session_retention_and_read_only_reports(self): for limit in (0, 1001): with self.assertRaises(ValueError): report(self.database, limit=limit) + with self.assertRaises(ValueError): + report(self.database, since_days=0) missing = Path(self.temporary.name) / 'absent.sqlite' with self.assertRaises(FileNotFoundError): report(missing) @@ -343,12 +674,13 @@ def test_retention_can_evict_inflight_evidence_without_disabling_a_live_process( second.dispatch(request('ping')) first._observer.finish(ticket, {'result': {}}, response_bytes=14) self.assertEqual(first._observer.health['state'], 'healthy') - self.assertEqual(first._observer.health['dropped_observations'], 1) + self.assertEqual(first._observer.health['lost'], 1) first.dispatch(request('ping')) self.assertEqual(first._observer.health['recorded_terminals'], 1) summary = report(self.database) self.assertEqual(summary['retained_starts'], 16) self.assertEqual(summary['retained_terminals'], 16) + self.assertEqual(summary['lost_rows'], 1) def test_observed_discovery_and_error_sequences_have_explicit_denominators(self): server = self.server() @@ -403,13 +735,69 @@ def test_report_scope_truncation_and_bounded_scan_are_explicit(self): self.assertTrue(bounded['sequence_scan_truncated']) self.assertIn('input_shape', summary['shapes'][0]) + def test_text_report_answers_the_interface_questions_in_a_screenful(self): + server = self.server() + initialize(server) + thread = {'x-codex-turn-metadata': {'thread_id': 'thread-a', 'thread_source': 'user'}} + self.call(server, 'route', {'facts': {}, 'content': {}}, route_result(('routing.activities', 'routing.topics')), thread) + self.call(server, 'route', {'snapshot': SNAPSHOT, 'facts': {'routing.activities': ['implementation']}}, route_result(), thread) + self.call(server, 'route', {'facts': {'python': True, 'task': 'write a parser test', 'routing.topics': ['security']}}, + rejection('ROUTE.INPUT_INVALID', '/facts'), thread) + for _ in range(2): + self.call(server, 'route', {'facts': {'routing.activities': ['implementation', 'testing']}}, + rejection('ROUTE.INPUT_INVALID', '/facts/routing.activities'), thread) + for target in ('core', 'core', 'router'): + self.call(server, 'read', {'snapshot': SNAPSHOT, 'target': target}, read_result(target), thread) + self.call(server, 'read', {'target': 'python-standards'}, rejection('NAVIGATION.UNKNOWN_POLICY')) + open_call = server._observer.begin(request('ping'), request_bytes=1, scope='constructed') + summary = report(self.database) + text = text_report(summary) + lines = text.splitlines() + self.assertLessEqual(len(lines), 50) + self.assertLessEqual(max(map(len, lines)), 130) + self.assertEqual(lines[2], '11 calls: 1 without an outcome (process ended mid-call), 0 rows lost, 0 pruned by retention') + self.assertRegex(text, r'test 1 +authoring +\d+ +1 +11') + usage = {row['operation']: row for row in summary['usage']} + self.assertEqual((usage['route']['calls'], usage['route']['rejected'], usage['route']['rejection_rate']), (5, 3, 0.6)) + self.assertGreaterEqual(usage['route']['duration_max_ms'], usage['route']['duration_p90_ms']) + self.assertGreaterEqual(usage['route']['duration_p90_ms'], usage['route']['duration_p50_ms']) + self.assertEqual(usage['read']['result_max_bytes'], + max(len(json.dumps(value)) for value in (read_result('router'), rejection('NAVIGATION.UNKNOWN_POLICY')))) + self.assertRegex(text, r'route +5 +60\.0% +[\d.]+ +[\d.]+ +[\d.]+ +[\d.]+ k?B +[\d.]+ k?B') + self.assertIn(' 2 route ROUTE.INPUT_INVALID /facts/routing.activities (fact) ["implementation","testing"]', lines) + self.assertIn(' 1 read NAVIGATION.UNKNOWN_POLICY - {"target":"python-standards"}', lines) + self.assertEqual(summary['unaccepted_facts'], { + 'names': [{'name': 'python', 'count': 1}, {'name': 'task', 'count': 1}], + 'values': [{'fact': 'routing.activities', 'value': 'testing', 'count': 2}]}) + self.assertIn(' names: python x1, task x1', lines) + self.assertIn(' values: routing.activities=testing x2', lines) + # Two route content pages delivered core; three reads returned core, core, router. + self.assertIn(' core x4, router x1', lines) + self.assertEqual(summary['repeat_reads'], {'reads': 5, 'repeats': 3, 'threads': 1, 'threads_repeating': 1, + 'worst_thread_rate': 0.6}) + self.assertIn('Routes sent without a snapshot: 4 of 5 (80.0%)', lines) + self.assertEqual(lines[-3:], ['Tool-calling threads threads calls reads repeated', + 'main 1 8 3 of 5 (60.0%)', + 'unknown 1 1 0 of 0']) + # The window filter drops older calls from every section. + with sqlite3.connect(self.database) as db: + db.execute("UPDATE interactions SET started=started-3*86400 WHERE operation='route'") + recent = report(self.database, since_days=1) + self.assertEqual(recent['retained_starts'], 6) + self.assertNotIn('route', {row['operation'] for row in recent['usage']}) + self.assertIn('(last 1 days)', text_report(recent)) + server._observer.finish(open_call, {'result': {}}) + def test_arbitrary_outputs_are_never_retained(self): server = self.server() initialize(server) secret = 'SYNTHETIC_OUTPUT_CREDENTIAL' - result = {'structuredContent': {'body': secret, 'outcome': secret, + result = {'structuredContent': {'body': secret, 'outcome': secret, 'kind': secret, 'code': secret, 'status': secret, + 'content': {'offset': secret, 'total': secret, 'items': [{'policy': {'id': secret + ' prose'}, 'content': secret}]}, + 'reading_plan': [{'target': secret + ' prose', 'state': 'selected'}], 'input_feedback': {'issues': [{'instance_pointer': '/' + secret, - 'keyword': secret, 'message': secret}]}}, 'isError': True} + 'keyword': secret, 'message': secret}]}}, 'isError': True, + 'content': [{'type': 'text', 'text': secret}]} with patch.object(server, '_request', return_value=result): response = server.dispatch(request('tools/call', {'name': 'runtime_info', 'arguments': {}})) self.assertEqual(response['result'], result) @@ -422,59 +810,135 @@ def test_unencodable_text_measurement_never_changes_stdio_behavior(self): server = self.server() if observed else MCPServer(ROOT, purpose='authoring') self.addCleanup(server.close) output = io.StringIO() - payload = '{"jsonrpc":"2.0","id":"\ud800","method":"ping"}\n' + payload = '{"jsonrpc":"2.0","id":"\ud800","method":"ping","params":{"_meta":{"threadId":"\ud800"}}}\n' serve(server, io.StringIO(payload), output) self.assertEqual(json.loads(output.getvalue())['result'], {}) + self.assertEqual(report(self.database)['retained_terminals'], 1) + def test_no_test_or_harness_launch_can_reach_the_default_database(self): + # Always-on recording must not follow the test suite into a developer's home. + # In-process MCPServer construction records only when given a configuration; + # every file that launches the server module as a process must disable + # recording or name a database explicitly. + launch = re.compile(r'''standards_engine\.mcp["']|standards_engine\.mcp import main''') + guards = ('--no-observe', '--observe-database', 'STANDARDS_ENGINE_OBSERVE') + launchers = [path for folder in ('tools', 'evaluation', '.agents') for path in sorted((ROOT / folder).rglob('*.py')) + if launch.search(path.read_text())] + self.assertGreater(len(launchers), 20) + for path in launchers: + self.assertTrue(any(guard in path.read_text() for guard in guards), path) class ProcessTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.home = Path(self.temporary.name) / 'home' + self.home.mkdir() + self.database = Path(self.temporary.name) / 'observer.sqlite' + + def census(self): + with sqlite3.connect(self.database) as db: + self.assertEqual(db.execute('PRAGMA integrity_check').fetchone()[0], 'ok') + self.assertEqual(db.execute('PRAGMA foreign_key_check').fetchall(), []) + return (db.execute('SELECT count(*) FROM interactions').fetchone()[0], + db.execute('SELECT count(*) FROM events').fetchone()[0], + db.execute('SELECT count(*) FROM (SELECT 1 FROM interactions i JOIN events e ON e.interaction=i.id ' + 'GROUP BY i.session HAVING count(*)=22)').fetchone()[0], + db.execute('SELECT coalesce(sum(lost),0) FROM sessions').fetchone()[0]) + + def test_always_on_by_default_in_both_purposes_invisible_and_two_off_switches(self): + messages = [request('initialize', {'protocolVersion': '2025-11-25', 'capabilities': {}, + 'clientInfo': {'name': 'default-on-fixture', 'version': '7'}}), + {'jsonrpc': '2.0', 'method': 'notifications/initialized'}, request('tools/list', identifier=2), + request('tools/call', {'name': 'runtime_info', 'arguments': {}}, identifier=3)] + payload = ''.join(json.dumps(message) + '\n' for message in messages) + def run(purpose, *flags, **variables): + process = subprocess.run([sys.executable, '-m', MCP_MODULE, '--repo-root', str(ROOT), '--purpose', purpose, *flags], + cwd=ROOT, env=environment(self.home, **variables), input=payload, + capture_output=True, text=True, timeout=60) + self.assertEqual(process.returncode, 0, process.stderr) + self.assertEqual(process.stderr, '', 'recording health is silent unless something is wrong') + return re.sub('[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}', 'INSTANCE', process.stdout) + silent = [run('application', '--no-observe'), run('application', STANDARDS_ENGINE_OBSERVE='off'), + run('application', '--observe-database', str(self.database), STANDARDS_ENGINE_OBSERVE=' OFF ')] + self.assertEqual(list(self.home.iterdir()), [], 'both off switches leave no file or directory') + self.assertFalse(self.database.exists()) + observed = run('application') + self.assertEqual({observed}, set(silent), 'recording changes no protocol byte') + run('authoring') + default = self.home / '.local/share/standards-engine/observations' / DEFAULT_DATABASE_NAME + self.assertEqual(stat.S_IMODE(default.stat().st_mode), 0o600) + self.assertEqual(stat.S_IMODE(default.parent.stat().st_mode), 0o700) + summary = report(default) + self.assertEqual((summary['retained_starts'], summary['retained_terminals']), (6, 6)) + self.assertEqual([(row['client'], row['purpose'], row['sessions'], row['calls']) for row in summary['clients']], + [('default-on-fixture 7', 'application', 1, 3), ('default-on-fixture 7', 'authoring', 1, 3)]) + self.assertEqual({row['deployment'] for row in summary['sessions']}, {'local'}) + run('application', XDG_DATA_HOME=str(self.home / 'xdg'), STANDARDS_ENGINE_OBSERVE='on') + self.assertEqual(report(self.home / 'xdg/standards-engine/observations' / DEFAULT_DATABASE_NAME)['retained_starts'], 3) + run('application', '--observe-database', str(self.database), '--observe-deployment', 'explicit') + self.assertEqual(report(self.database)['sessions'][0]['deployment'], 'explicit') + self.assertEqual(report(default)['retained_starts'], 6) + def test_cold_stdio_restart_bytes_identity_and_operator_cli(self): - with tempfile.TemporaryDirectory() as temporary: - database = Path(temporary) / 'observer.sqlite' - messages = [request('initialize', {'protocolVersion': '2025-11-25', 'capabilities': {}, 'clientInfo': {}}), - {'jsonrpc': '2.0', 'method': 'notifications/initialized'}, request('tools/list'), - request('tools/call', {'name': 'describe_input', 'arguments': {'operation': 'runtime_info'}}), - request('tools/call', {'name': 'runtime_info', 'arguments': {'unexpected': 'SYNTHETIC_SECRET'}}), - request('tools/call', {'name': 'runtime_info', 'arguments': {}})] - payload = ''.join(json.dumps(message) + '\n' for message in messages) - command = [sys.executable, '-m', MCP_MODULE, '--repo-root', str(ROOT), '--purpose', 'authoring', - '--output-schemas', 'on-demand', '--observe-database', str(database), '--observe-deployment', 'cold-test'] - identities = [] - for _ in range(2): - process = subprocess.run(command, cwd=ROOT, input=payload, capture_output=True, text=True, timeout=30) - self.assertEqual(process.returncode, 0, process.stderr) - responses = [json.loads(line) for line in process.stdout.splitlines()] - identity = responses[0]['result']['_meta']['standards-engine/runtime'] - identities.append(identity) - with sqlite3.connect(database) as db: - row = db.execute('SELECT runtime FROM sessions WHERE id=?', (identity['instance_id'],)).fetchone() - self.assertEqual(json.loads(row[0]), identity) - counts = db.execute('SELECT sum(request_bytes) FROM interactions WHERE session=?', (identity['instance_id'],)).fetchone()[0] - notification_bytes = len((json.dumps(messages[1]) + '\n').encode()) - self.assertEqual(counts, len(payload.encode()) - notification_bytes) - output_bytes = db.execute('SELECT sum(e.response_bytes) FROM events e JOIN interactions i ' - 'ON i.id=e.interaction WHERE session=?', (identity['instance_id'],)).fetchone()[0] - self.assertEqual(output_bytes, len(process.stdout.encode())) - self.assertIn('"state":"healthy"', process.stderr) - self.assertNotEqual(identities[0]['instance_id'], identities[1]['instance_id']) - for key in ('implementation_digest', 'schema_digest', 'catalog_digest', 'interface_version'): - self.assertEqual(identities[0][key], identities[1][key]) - summary = report(database) - self.assertEqual(summary['retained_starts'], 10) - self.assertEqual(summary['retained_terminals'], 10) - self.assertEqual({row['scope'] for row in summary['operations']}, {'stdio-flushed'}) - before = database.read_bytes() - result = subprocess.run([sys.executable, '-m', MODULE, '--database', str(database), '--limit', '2'], - cwd=ROOT, capture_output=True, text=True, timeout=30) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(json.loads(result.stdout)['retained_starts'], 10) - self.assertEqual(database.read_bytes(), before) + messages = [request('initialize', {'protocolVersion': '2025-11-25', 'capabilities': {}, 'clientInfo': {}}), + {'jsonrpc': '2.0', 'method': 'notifications/initialized'}, request('tools/list'), + request('tools/call', {'name': 'describe_input', 'arguments': {'operation': 'runtime_info'}}), + request('tools/call', {'name': 'runtime_info', 'arguments': {'unexpected': 'SYNTHETIC_SECRET'}}), + request('tools/call', {'name': 'runtime_info', 'arguments': {}})] + payload = ''.join(json.dumps(message) + '\n' for message in messages) + command = [sys.executable, '-m', MCP_MODULE, '--repo-root', str(ROOT), '--purpose', 'authoring', + '--output-schemas', 'on-demand', '--observe-database', str(self.database), '--observe-deployment', 'cold-test'] + identities = [] + for _ in range(2): + process = subprocess.run(command, cwd=ROOT, env=environment(self.home), input=payload, + capture_output=True, text=True, timeout=30) + self.assertEqual(process.returncode, 0, process.stderr) + self.assertEqual(process.stderr, '') + responses = [json.loads(line) for line in process.stdout.splitlines()] + identity = responses[0]['result']['_meta']['standards-engine/runtime'] + identities.append(identity) + with sqlite3.connect(self.database) as db: + row = db.execute('SELECT runtime FROM sessions WHERE id=?', (identity['instance_id'],)).fetchone() + self.assertEqual(json.loads(row[0]), identity) + counts = db.execute('SELECT sum(request_bytes) FROM interactions WHERE session=?', (identity['instance_id'],)).fetchone()[0] + notification_bytes = len((json.dumps(messages[1]) + '\n').encode()) + self.assertEqual(counts, len(payload.encode()) - notification_bytes) + output_bytes, result_bytes = db.execute( + 'SELECT sum(e.response_bytes),sum(e.result_bytes) FROM events e JOIN interactions i ' + 'ON i.id=e.interaction WHERE session=?', (identity['instance_id'],)).fetchone() + self.assertEqual(output_bytes, len(process.stdout.encode())) + self.assertEqual(result_bytes, sum(len(response['result']['content'][0]['text']) for response in responses[2:])) + self.assertLess(2 * result_bytes, output_bytes) + self.assertNotEqual(identities[0]['instance_id'], identities[1]['instance_id']) + for key in ('implementation_digest', 'schema_digest', 'catalog_digest', 'interface_version'): + self.assertEqual(identities[0][key], identities[1][key]) + summary = report(self.database) + self.assertEqual(summary['retained_starts'], 10) + self.assertEqual(summary['retained_terminals'], 10) + self.assertEqual({row['scope'] for row in summary['operations']}, {'stdio-flushed'}) + before = self.database.read_bytes() + cli = [sys.executable, '-m', MODULE, '--database', str(self.database)] + result = subprocess.run(cli + ['--format', 'json', '--limit', '2'], cwd=ROOT, env=environment(self.home), + capture_output=True, text=True, timeout=30) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(json.loads(result.stdout)['retained_starts'], 10) + self.assertLess(len(result.stdout), 16_000, 'no per-session schema digests in the report') + result = subprocess.run(cli + ['--since-days', '1'], cwd=ROOT, env=environment(self.home), + capture_output=True, text=True, timeout=30) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('10 calls: 0 without an outcome', result.stdout) + self.assertRegex(result.stdout, r'runtime_info +4 +50\.0%') + self.assertNotIn('SYNTHETIC_SECRET', result.stdout) + self.assertEqual(self.database.read_bytes(), before) + # Without --database the report reads the always-on location, here an empty home. + result = subprocess.run(cli[:3], cwd=ROOT, env=environment(self.home), capture_output=True, text=True, timeout=30) + self.assertEqual((result.returncode, result.stdout), (1, '')) + self.assertEqual(list(self.home.iterdir()), []) def test_process_kill_leaves_unmatched_start_and_replacement_does_not_complete_it(self): - with tempfile.TemporaryDirectory() as temporary: - database = Path(temporary) / 'observer.sqlite' - script = ''' + script = ''' import sys, threading from pathlib import Path from tools.standards_engine.standards_engine.mcp import MCPServer, serve @@ -486,62 +950,119 @@ def blocked(method, params): server._request=blocked serve(server,sys.stdin,sys.stdout) ''' - process = subprocess.Popen([sys.executable, '-c', script, str(ROOT), str(database)], cwd=ROOT, - stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) - try: - process.stdin.write(json.dumps(request('ping')) + '\n') - process.stdin.flush() - self.assertIn('healthy', process.stderr.readline()) - self.assertEqual(process.stderr.readline().strip(), 'entered-domain') + process = subprocess.Popen([sys.executable, '-c', script, str(ROOT), str(self.database)], cwd=ROOT, + env=environment(self.home), + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + try: + process.stdin.write(json.dumps(request('ping')) + '\n') + process.stdin.flush() + self.assertEqual(process.stderr.readline().strip(), 'entered-domain') + process.kill() + process.communicate(timeout=10) + finally: + if process.poll() is None: process.kill() process.communicate(timeout=10) - finally: - if process.poll() is None: - process.kill() - process.communicate(timeout=10) - replacement = subprocess.run([sys.executable, '-m', MCP_MODULE, '--purpose', 'authoring', - '--observe-database', str(database), '--observe-deployment', 'replacement'], - cwd=ROOT, input=json.dumps(request('ping')) + '\n', - text=True, capture_output=True, timeout=30) - self.assertEqual(replacement.returncode, 0, replacement.stderr) - summary = report(database) - self.assertEqual(summary['retained_starts'], 2) - self.assertEqual(summary['retained_terminals'], 1) - self.assertEqual(summary['unmatched_starts'], 1) - - def test_concurrent_processes_and_first_open_never_overwrite_or_duplicate(self): - with tempfile.TemporaryDirectory() as temporary: - database = Path(temporary) / 'observer.sqlite' - command = [sys.executable, '-m', MCP_MODULE, '--purpose', 'authoring', - '--observe-database', str(database), '--observe-deployment', 'concurrent'] - workers = [subprocess.Popen(command, cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, text=True) for _ in range(4)] - try: - outputs = [] - for worker in workers: - worker.stdin.write(''.join(json.dumps(request('ping', identifier=i)) + '\n' for i in range(5))) - worker.stdin.flush() - for worker in workers: - output, errors = worker.communicate(timeout=30) - self.assertEqual(worker.returncode, 0, errors) - self.assertEqual(len(output.splitlines()), 5) - outputs.append(errors) - finally: - for worker in workers: - if worker.poll() is None: - worker.kill() - worker.communicate(timeout=10) - # Contention may intentionally degrade a process after the 20ms budget. - # A failed terminal write may leave an unmatched begin; no duplicate or - # fabricated terminal is permitted. Domain replies still total 20. - summary = report(database) - self.assertGreater(summary['retained_starts'], 0) - self.assertLessEqual(summary['retained_starts'], 20) - self.assertLessEqual(summary['retained_terminals'], summary['retained_starts']) - with sqlite3.connect(database) as db: - self.assertEqual(db.execute('PRAGMA integrity_check').fetchone()[0], 'ok') - self.assertEqual(db.execute('PRAGMA foreign_key_check').fetchall(), []) - self.assertTrue(all('healthy' in output or 'degraded' in output for output in outputs)) + replacement = subprocess.run([sys.executable, '-m', MCP_MODULE, '--purpose', 'authoring', + '--observe-database', str(self.database), '--observe-deployment', 'replacement'], + cwd=ROOT, env=environment(self.home), input=json.dumps(request('ping')) + '\n', + text=True, capture_output=True, timeout=30) + self.assertEqual(replacement.returncode, 0, replacement.stderr) + summary = report(self.database) + self.assertEqual(summary['retained_starts'], 2) + self.assertEqual(summary['retained_terminals'], 1) + self.assertEqual(summary['unmatched_starts'], 1) + self.assertIn('2 calls: 1 without an outcome', text_report(summary)) + + def test_eight_recorders_creating_one_database_at_the_same_instant_lose_nothing(self): + script = ''' +import io, sys, time +from pathlib import Path +from tools.standards_engine.standards_engine.interaction_observer import InteractionRecorder, RecorderConfig +while time.time() < float(sys.argv[2]): + pass +recorder = InteractionRecorder(RecorderConfig(Path(sys.argv[1]), 'simultaneous'), {'instance_id': sys.argv[3], 'catalog_digest': 'c'}, + [{'name': 'runtime_info', 'inputSchema': {'type': 'object'}}], diagnostics=io.StringIO()) +calls = [('initialize', {}), ('tools/list', {})] + [('tools/call', {'name': 'runtime_info', 'arguments': {}})] * 20 +for identifier, (method, params) in enumerate(calls): + ticket = recorder.begin({'jsonrpc': '2.0', 'id': identifier, 'method': method, 'params': params}, request_bytes=1, scope='constructed') + recorder.finish(ticket, {'result': {}}, response_bytes=1) +recorder.close() +''' + for attempt in range(3): + self.database = Path(self.temporary.name) / f'simultaneous-{attempt}.sqlite' + instant = time.time() + 1.5 # After every interpreter has imported the Engine. + workers = [subprocess.Popen([sys.executable, '-c', script, str(self.database), str(instant), f'worker-{index}'], + cwd=ROOT, env=environment(self.home), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + for index in range(8)] + for worker in workers: + output, errors = worker.communicate(timeout=60) + self.assertEqual(worker.returncode, 0, errors) + self.assertEqual(self.census(), (176, 176, 8, 0)) + + def servers(self, count=8): + command = [sys.executable, '-m', MCP_MODULE, '--repo-root', str(ROOT), '--purpose', 'application', + '--output-schemas', 'on-demand', '--observe-database', str(self.database)] + return [subprocess.Popen(command, cwd=ROOT, env=environment(self.home), stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) for _ in range(count)] + + def lines(self, *messages): + return ''.join(json.dumps(message) + '\n' for message in messages) + + def collect(self, workers, responses): + errors = [] + try: + for worker in workers: + output, error = worker.communicate(timeout=60) + self.assertEqual(worker.returncode, 0, error) + self.assertEqual(len(output.splitlines()), responses) + errors.append(error) + finally: + for worker in workers: + if worker.poll() is None: + worker.kill() + worker.communicate(timeout=10) + return errors + + def test_eight_server_processes_started_together_record_every_call(self): + opening = self.lines(request('initialize', {'protocolVersion': '2025-11-25', 'capabilities': {}, + 'clientInfo': {'name': 'fan-out', 'version': '1'}}), + {'jsonrpc': '2.0', 'method': 'notifications/initialized'}) + calls = self.lines(request('tools/list', identifier=2), + *(request('tools/call', {'name': 'runtime_info', 'arguments': {}}, identifier=3 + index) for index in range(20))) + # No database exists yet: all eight create it while starting. + workers = self.servers() + for worker in workers: + worker.stdin.write(opening + calls) + worker.stdin.flush() + self.collect(workers, 22) + self.assertEqual(self.census(), (176, 176, 8, 0)) + # Again while another connection holds the write lock for 500 ms. + workers = self.servers() + try: + for worker in workers: + worker.stdin.write(opening) + worker.stdin.flush() + for worker in workers: + self.assertIn('"result"', worker.stdout.readline()) + lock = sqlite3.connect(self.database, isolation_level=None) + lock.execute('BEGIN IMMEDIATE') + for worker in workers: + worker.stdin.write(calls) + worker.stdin.flush() + time.sleep(0.5) + self.assertEqual(self.census()[0], 184, 'nothing is written while the lock is held') + lock.execute('ROLLBACK') + lock.close() + finally: + errors = self.collect(workers, 21) + self.assertEqual(self.census(), (352, 352, 16, 0)) + locked = [error for error in errors if '"reason":"storage-locked"' in error] + self.assertTrue(locked, 'the lock must have forced at least one process to use its backlog') + for error in errors: + if error: + final = json.loads(error.splitlines()[-1].split(' ', 2)[2]) + self.assertEqual((final['state'], final['backlog'], final['lost']), ('healthy', 0, 0)) if __name__ == '__main__': diff --git a/tools/standards_engine/tests/test_mcp.py b/tools/standards_engine/tests/test_mcp.py index 6547ef9b..8c8ff6eb 100644 --- a/tools/standards_engine/tests/test_mcp.py +++ b/tools/standards_engine/tests/test_mcp.py @@ -396,6 +396,7 @@ def test_module_stdio_launch_from_unrelated_working_directory(self): "authoring", "--repo-root", str(ROOT), + "--no-observe", ], input="".join(json.dumps(message) + "\n" for message in messages), text=True, diff --git a/tools/standards_engine/tests/test_navigation_indexes.py b/tools/standards_engine/tests/test_navigation_indexes.py index 7508828e..c999ecf2 100644 --- a/tools/standards_engine/tests/test_navigation_indexes.py +++ b/tools/standards_engine/tests/test_navigation_indexes.py @@ -387,6 +387,7 @@ def test_fresh_mcp_process_exposes_snapshot_bound_navigation(self): "authoring", "--repo-root", str(self.root), + "--no-observe", ], input="".join(json.dumps(message) + "\n" for message in messages), text=True, diff --git a/tools/standards_engine/tests/test_numeric_boundary.py b/tools/standards_engine/tests/test_numeric_boundary.py index 202f7fa3..e8a6bb37 100644 --- a/tools/standards_engine/tests/test_numeric_boundary.py +++ b/tools/standards_engine/tests/test_numeric_boundary.py @@ -111,7 +111,7 @@ def cold_call(self, operation, arguments, rejected=False): ] result = subprocess.run([sys.executable, '-P', '-m', 'tools.standards_engine.standards_engine.mcp', '--repo-root', str(self.root), - '--purpose', 'authoring', '--output-schemas', 'on-demand', '--advanced'], + '--purpose', 'authoring', '--output-schemas', 'on-demand', '--advanced', '--no-observe'], input=''.join(json.dumps(m) + '\n' for m in messages), text=True, capture_output=True, env={**os.environ, 'PYTHONPATH': str(ROOT)}, timeout=90) self.assertEqual(result.returncode, 0, result.stderr) diff --git a/tools/standards_engine/tests/test_output_delivery.py b/tools/standards_engine/tests/test_output_delivery.py index 7a0cafd1..06466ae7 100644 --- a/tools/standards_engine/tests/test_output_delivery.py +++ b/tools/standards_engine/tests/test_output_delivery.py @@ -213,7 +213,7 @@ def run(call): messages=[init,{'jsonrpc':'2.0','method':'notifications/initialized'},request('tools/list',identifier=2), request('tools/call',{'name':'describe_output','arguments':call},identifier=3)] process=subprocess.run([sys.executable,'-m','tools.standards_engine.standards_engine.mcp', - '--repo-root',str(ROOT),'--purpose','authoring','--output-schemas','on-demand'], + '--repo-root',str(ROOT),'--purpose','authoring','--output-schemas','on-demand','--no-observe'], input=''.join(json.dumps(x)+'\n' for x in messages),capture_output=True,text=True,cwd=ROOT,check=True) replies=[json.loads(x) for x in process.stdout.splitlines()] self.assertTrue(all('outputSchema' not in t for t in replies[1]['result']['tools'])) diff --git a/tools/standards_engine/tests/test_process_reuse.py b/tools/standards_engine/tests/test_process_reuse.py index b09716d7..6e8a0507 100644 --- a/tools/standards_engine/tests/test_process_reuse.py +++ b/tools/standards_engine/tests/test_process_reuse.py @@ -220,7 +220,7 @@ def test_real_stdio_restarts_read_the_same_retained_snapshot(self): for _ in range(2): completed = subprocess.run( [sys.executable, "-P", "-m", "tools.standards_engine.standards_engine.mcp", - "--repo-root", str(self.root), "--purpose", "authoring"], + "--repo-root", str(self.root), "--purpose", "authoring", "--no-observe"], input="".join(json.dumps(value) + "\n" for value in messages), text=True, capture_output=True, check=True, timeout=60, cwd=self.temporary.name, env={**os.environ, "PYTHONPATH": str(ROOT)}, diff --git a/tools/standards_engine/tests/test_proposal_reuse.py b/tools/standards_engine/tests/test_proposal_reuse.py index afa9cfa6..5937100a 100644 --- a/tools/standards_engine/tests/test_proposal_reuse.py +++ b/tools/standards_engine/tests/test_proposal_reuse.py @@ -495,7 +495,7 @@ def test_two_fresh_stdio_servers_reproduce_same_historical_draft(self): observed = [] for _ in range(2): process = subprocess.run( - [sys.executable, "-P", "-m", "tools.standards_engine.standards_engine.mcp", "--repo-root", str(self.root), "--purpose", "authoring"], + [sys.executable, "-P", "-m", "tools.standards_engine.standards_engine.mcp", "--repo-root", str(self.root), "--purpose", "authoring", "--no-observe"], input="".join(json.dumps(item) + "\n" for item in messages), text=True, capture_output=True, check=True, timeout=90, env={**os.environ, "PYTHONPATH": str(ROOT)}, cwd=self.temporary.name, diff --git a/tools/standards_engine/tests/test_purpose_transport.py b/tools/standards_engine/tests/test_purpose_transport.py index d96351ba..ba18e8a7 100644 --- a/tools/standards_engine/tests/test_purpose_transport.py +++ b/tools/standards_engine/tests/test_purpose_transport.py @@ -98,7 +98,7 @@ def request(identifier, method, params): ] process = subprocess.run( [sys.executable, "-P", "-m", "tools.standards_engine.standards_engine.mcp", - "--repo-root", str(self.root), "--purpose", "application", "--advanced"], + "--repo-root", str(self.root), "--purpose", "application", "--advanced", "--no-observe"], input="".join(json.dumps(value) + "\n" for value in messages), text=True, capture_output=True, env={**os.environ, "PYTHONPATH": str(ROOT)}, timeout=120, ) diff --git a/tools/standards_engine/tests/test_read_many.py b/tools/standards_engine/tests/test_read_many.py index 2813135c..74414e30 100644 --- a/tools/standards_engine/tests/test_read_many.py +++ b/tools/standards_engine/tests/test_read_many.py @@ -208,7 +208,7 @@ def message(i, method, params): {"jsonrpc": "2.0", "method": "notifications/initialized"}, message(2, "tools/call", {"name": "read_many", "arguments": arguments}), message(3, "tools/call", {"name": "read_many", "arguments": {**arguments, "items": [{"target": "core"}, {"target": "provenance.fixture"}]}})] - command = [sys.executable, "-P", "-m", "tools.standards_engine.standards_engine.mcp", "--repo-root", str(self.root), "--purpose", "application"] + command = [sys.executable, "-P", "-m", "tools.standards_engine.standards_engine.mcp", "--repo-root", str(self.root), "--purpose", "application", "--no-observe"] for _ in range(2): response = subprocess.run(command, input="".join(json.dumps(m) + "\n" for m in messages), text=True, capture_output=True, env=env, timeout=60) self.assertEqual(response.returncode, 0, response.stderr) diff --git a/tools/standards_engine/tests/test_routing_fact_transport.py b/tools/standards_engine/tests/test_routing_fact_transport.py index b46acb04..faaefbf6 100644 --- a/tools/standards_engine/tests/test_routing_fact_transport.py +++ b/tools/standards_engine/tests/test_routing_fact_transport.py @@ -88,7 +88,7 @@ def cold(self, purpose, operation, args, *, error=False): {'jsonrpc':'2.0','method':'notifications/initialized'}, {'jsonrpc':'2.0','id':2,'method':'tools/call','params':{'name':operation,'arguments':args}}] proc=subprocess.run([sys.executable,'-P','-m','tools.standards_engine.standards_engine.mcp', - '--repo-root',str(self.root),'--purpose',purpose,'--output-schemas','on-demand'], + '--repo-root',str(self.root),'--purpose',purpose,'--output-schemas','on-demand','--no-observe'], cwd=ROOT,env={**os.environ,'PYTHONPATH':str(ROOT)},input=''.join(json.dumps(m)+'\n' for m in messages), capture_output=True,text=True,check=True) self.assertEqual(proc.stderr,'') diff --git a/tools/standards_engine/tests/test_runtime_identity.py b/tools/standards_engine/tests/test_runtime_identity.py index ae5bdd72..0a03ecef 100644 --- a/tools/standards_engine/tests/test_runtime_identity.py +++ b/tools/standards_engine/tests/test_runtime_identity.py @@ -100,7 +100,7 @@ def test_actual_mcp_and_cli_observe_without_a_git_repository(self): request('tools/call', {'name': 'runtime_info', 'arguments': {}}, identifier=2)] environment = {**os.environ, 'PYTHONPATH': str(ROOT)} mcp = subprocess.run([sys.executable, '-P', '-m', 'tools.standards_engine.standards_engine.mcp', - '--repo-root', str(root), '--purpose', 'application', '--advanced'], + '--repo-root', str(root), '--purpose', 'application', '--advanced', '--no-observe'], input=''.join(json.dumps(item)+'\n' for item in messages), text=True, capture_output=True, env=environment, timeout=60) self.assertEqual(mcp.returncode, 0, mcp.stderr) diff --git a/tools/standards_engine/tests/test_schema_presentation.py b/tools/standards_engine/tests/test_schema_presentation.py index f3d96f87..4da1b27d 100644 --- a/tools/standards_engine/tests/test_schema_presentation.py +++ b/tools/standards_engine/tests/test_schema_presentation.py @@ -115,7 +115,7 @@ def test_retired_cli_flags_reject_without_creating_paths(self): with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) / 'untouched' commands = ( - ('tools.standards_engine.standards_engine.mcp', ['--repo-root', str(root), '--purpose', 'authoring']), + ('tools.standards_engine.standards_engine.mcp', ['--repo-root', str(root), '--purpose', 'authoring', '--no-observe']), ('tools.standards_engine.tests.catalog_inventory', ['--repo-root', str(root)]), ('tools.standards_engine.tests.codex_navigation_client', []), ('tools.standards_engine.tests.codex_discovery_client', ['--model', 'fixture', '--surface', 'fixture', '--evidence-dir', str(root)]), diff --git a/tools/standards_engine/tests/test_storage_lifecycle_transport.py b/tools/standards_engine/tests/test_storage_lifecycle_transport.py index a9b5eb47..6e84b144 100644 --- a/tools/standards_engine/tests/test_storage_lifecycle_transport.py +++ b/tools/standards_engine/tests/test_storage_lifecycle_transport.py @@ -1,5 +1,8 @@ """Cold MCP reads coexist with a reserved writer only when no purge is due. +One server process audits its store once, on its first call; a newly started +process audits again and still rejects a corrupt store. + The fixture owns its database, evidence and processes. Publication, authorization and interpreter timing are not simulated; no production store is accessed. """ @@ -7,19 +10,27 @@ import json import os +import shutil import sqlite3 import subprocess import sys import time import unittest from contextlib import closing +from pathlib import Path +from unittest import mock +from tools.standards_engine.standards_engine.mcp import MCPServer from tools.standards_engine.tests import test_agent_interaction_transport as transport +from tools.standards_engine.tests.test_mcp import initialize, request from tools.standards_engine.tests.test_route_content import known_assertions from tools.standards_snapshots.standards_snapshots import ( AggregateChild, AggregateRecord, AggregateRoot, ChildHandle, SnapshotError, SnapshotId, SnapshotModule, ) +from tools.standards_snapshots.standards_snapshots import store as snapshot_store + +AUDIT_STATEMENTS = ('PRAGMA integrity_check', 'PRAGMA foreign_key_check') class StorageLifecycleTransportTest(unittest.TestCase): @@ -39,6 +50,85 @@ def call(self, purpose, operation, arguments, *, rejected=False): self, purpose, operation, arguments, rejected=rejected, delivery='on-demand', ) + def cold_open_failure(self): + """Run one cold authoring read whose store open fails; return its private stderr.""" + # Open-time failure retains the existing MCP private-stderr + # boundary (no structured result), not an invented tool outcome. + messages = [ + {'jsonrpc': '2.0', 'id': 1, 'method': 'initialize', 'params': { + 'protocolVersion': '2025-11-25', 'capabilities': {}, + 'clientInfo': {'name': 'storage-open-fixture', 'version': '1'}}}, + {'jsonrpc': '2.0', 'method': 'notifications/initialized'}, + {'jsonrpc': '2.0', 'id': 2, 'method': 'tools/call', 'params': { + 'name': 'read', 'arguments': {'snapshot': self.snapshot, 'target': 'core'}}}, + ] + process = subprocess.run( + [sys.executable, '-P', '-m', 'tools.standards_engine.standards_engine.mcp', + '--repo-root', str(self.root), '--purpose', 'authoring', + '--output-schemas', 'on-demand', '--no-observe'], + env={**os.environ, 'PYTHONPATH': str(transport.fixture.ROOT)}, + input=''.join(json.dumps(message) + '\n' for message in messages), + text=True, capture_output=True, timeout=90, + ) + self.assertEqual(process.returncode, 0, process.stderr) + failed = next(json.loads(line)['result'] for line in process.stdout.splitlines() + if json.loads(line).get('id') == 2) + self.assertTrue(failed['isError']) + self.assertNotIn('structuredContent', failed) + self.assertIn('Engine invocation failed', failed['content'][0]['text']) + return process.stderr + + def test_one_server_process_audits_its_store_once_across_calls(self): + # The fixture built this store inside the test process. Discard that + # record so the server below starts as a newly launched process does. + with snapshot_store._AUDITED_FILES_LOCK: + snapshot_store._AUDITED_FILES.clear() + opens = [] + connect = sqlite3.connect + + def traced_connect(database, *arguments, **options): + connection = connect(database, *arguments, **options) + if Path(database).resolve() == self.path.resolve(): + opens.append([]) + connection.set_trace_callback(opens[-1].append) + return connection + + server = MCPServer(self.root, purpose='application') + self.addCleanup(server.close) + initialize(server) + arguments = {'snapshot': self.snapshot, 'target': 'core'} + with mock.patch.object(snapshot_store.sqlite3, 'connect', traced_connect): + results = [ + server.dispatch(request('tools/call', {'name': 'read', 'arguments': arguments}, + identifier))['result'] + for identifier in (2, 3, 4) + ] + for result in results: + self.assertFalse(result['isError'], result) + self.assertEqual(result['structuredContent']['content'], + transport.fixture.PurposeTransportTest.expected) + # Every call still opens and closes its own store connection. + self.assertEqual(len(opens), 3) + self.assertEqual([[statement for statement in statements if statement in AUDIT_STATEMENTS] + for statements in opens], [list(AUDIT_STATEMENTS), [], []]) + for statements in opens: + self.assertIn('PRAGMA application_id', statements) + self.assertIn('PRAGMA user_version', statements) + self.assertTrue(any('FROM sqlite_schema' in statement for statement in statements)) + + def test_cold_server_process_rejects_a_corrupt_store_on_its_first_call(self): + retained = self.path.with_name('retained-valid-store.sqlite3') + shutil.copyfile(self.path, retained) + try: + with closing(sqlite3.connect(self.path, isolation_level=None)) as corruptor: + corruptor.execute( + "INSERT INTO content_files VALUES ('missing', 'a', X'', 0, 'digest')") + self.assertIn('SNAPSHOT_STORE.FOREIGN_KEY_FAILURE', self.cold_open_failure()) + finally: + os.replace(retained, self.path) + recovered = self.call('authoring', 'read', {'snapshot': self.snapshot, 'target': 'core'}) + self.assertEqual(recovered['content'], transport.fixture.PurposeTransportTest.expected) + def test_cold_agent_observations_match_while_another_process_holds_writer(self): cases = [ ('read', {'snapshot': self.snapshot, 'target': 'core'}), @@ -82,31 +172,7 @@ def test_cold_due_maintenance_keeps_busy_then_expires_all_dependents_atomically( with closing(sqlite3.connect(self.path, isolation_level=None)) as writer: writer.execute('BEGIN IMMEDIATE') try: - # Open-time failure retains the existing MCP private-stderr - # boundary (no structured result), not an invented tool outcome. - messages = [ - {'jsonrpc': '2.0', 'id': 1, 'method': 'initialize', 'params': { - 'protocolVersion': '2025-11-25', 'capabilities': {}, - 'clientInfo': {'name': 'storage-busy-fixture', 'version': '1'}}}, - {'jsonrpc': '2.0', 'method': 'notifications/initialized'}, - {'jsonrpc': '2.0', 'id': 2, 'method': 'tools/call', 'params': { - 'name': 'read', 'arguments': {'snapshot': self.snapshot, 'target': 'core'}}}, - ] - process = subprocess.run( - [sys.executable, '-P', '-m', 'tools.standards_engine.standards_engine.mcp', - '--repo-root', str(self.root), '--purpose', 'authoring', - '--output-schemas', 'on-demand'], - env={**os.environ, 'PYTHONPATH': str(transport.fixture.ROOT)}, - input=''.join(json.dumps(message) + '\n' for message in messages), - text=True, capture_output=True, timeout=90, - ) - self.assertEqual(process.returncode, 0, process.stderr) - self.assertIn('SNAPSHOT_STORE.BUSY', process.stderr) - failed = next(json.loads(line)['result'] for line in process.stdout.splitlines() - if json.loads(line).get('id') == 2) - self.assertTrue(failed['isError']) - self.assertNotIn('structuredContent', failed) - self.assertIn('Engine invocation failed', failed['content'][0]['text']) + self.assertIn('SNAPSHOT_STORE.BUSY', self.cold_open_failure()) self.assertIsNotNone(writer.execute( 'SELECT 1 FROM snapshot_roots WHERE snapshot_id = ?', (str(expired),) ).fetchone()) diff --git a/tools/standards_engine/tests/test_typed_routing_workflow.py b/tools/standards_engine/tests/test_typed_routing_workflow.py index 44a12cc9..6d3e6c19 100644 --- a/tools/standards_engine/tests/test_typed_routing_workflow.py +++ b/tools/standards_engine/tests/test_typed_routing_workflow.py @@ -36,7 +36,7 @@ def call(self, operation, arguments, delivery='on-demand', *, error=False): {'jsonrpc':'2.0','method':'notifications/initialized'}, request('tools/call', {'name':operation,'arguments':arguments}, 2)] run = subprocess.run([sys.executable,'-P','-m','tools.standards_engine.standards_engine.mcp', - '--repo-root',str(self.root),'--purpose','authoring','--output-schemas',delivery], + '--repo-root',str(self.root),'--purpose','authoring','--output-schemas',delivery,'--no-observe'], input=''.join(json.dumps(v)+'\n' for v in messages),text=True,capture_output=True,check=True, cwd=ROOT,env={**os.environ,'PYTHONPATH':str(ROOT)}) self.assertEqual(run.stderr, '') diff --git a/tools/standards_engine/tests/test_workflow_presentation.py b/tools/standards_engine/tests/test_workflow_presentation.py index 3636bc84..6b95da24 100644 --- a/tools/standards_engine/tests/test_workflow_presentation.py +++ b/tools/standards_engine/tests/test_workflow_presentation.py @@ -310,7 +310,7 @@ def call(arguments): root = Path(__file__).resolve().parents[3] run = subprocess.run( [sys.executable, "-P", "-m", "tools.standards_engine.standards_engine.mcp", - "--repo-root", str(self.root), "--purpose", "authoring"], + "--repo-root", str(self.root), "--purpose", "authoring", "--no-observe"], input="".join(json.dumps(item) + "\n" for item in messages), text=True, capture_output=True, env={**os.environ, "PYTHONPATH": str(root)}, check=True, diff --git a/tools/standards_engine/tests/workflow_interaction_trace.py b/tools/standards_engine/tests/workflow_interaction_trace.py index 1de38251..d8df0ddb 100644 --- a/tools/standards_engine/tests/workflow_interaction_trace.py +++ b/tools/standards_engine/tests/workflow_interaction_trace.py @@ -38,7 +38,7 @@ def call(name: str, arguments: dict) -> dict: ] completed = subprocess.run( [sys.executable, "-P", "-m", "tools.standards_engine.standards_engine.mcp", - "--repo-root", str(root), "--purpose", "authoring"], + "--repo-root", str(root), "--purpose", "authoring", "--no-observe"], input="".join(json.dumps(item) + "\n" for item in messages), text=True, capture_output=True, check=True, env=os.environ.copy(), ) diff --git a/tools/standards_snapshots/README.md b/tools/standards_snapshots/README.md index a1809843..0d56ee42 100644 --- a/tools/standards_snapshots/README.md +++ b/tools/standards_snapshots/README.md @@ -53,18 +53,33 @@ No identity domain, store format or persisted representation changes. ## Admission and maintenance work -Each open of an existing current-schema database authenticates the application, -exact schema, and complete SQLite integrity/foreign-key state **before** persistent -configuration. Configuration is followed by the existing exact schema check. The -same full audit is not repeated for a store admitted at the current schema version. -This is a point-in-time audit on every open, not cached proof that the database can -never change. Selected content and aggregate reads retain their ordinary checks. +Each open of an existing current-schema database authenticates the application +and exact schema **before** persistent configuration. Configuration is followed by +the existing exact schema check. Those checks run on every open. -New initialization keeps its final integrity audit and owned-file cleanup on failure. +The complete SQLite integrity/foreign-key audit runs once per process for each +store file. A process's first open of a file audits it before persistent +configuration. Later opens of the same file by that process omit only this audit. +The process remembers a passed audit in memory, by the file's device and inode, and +only after that open succeeded; nothing is persisted. A new process audits again, +and so does a file moved into place under a different inode. To audit a store +again, open it from a new process. `maintain` purges expired roots; it does not run +this audit. + +The audit is a point-in-time observation, not proof that the database stays valid. +Damage that appears after a process's audit is not reported by that process's later +opens. A file deleted and then recreated outside this package can receive the +audited file's inode number, and that process does not audit it; replace a store by +renaming a complete file into place, or restart the process. Selected content and +aggregate reads retain their ordinary checks on every call. + +New initialization keeps its final integrity audit and owned-file cleanup on failure; +that audit is the creating process's audit of the new file. A store admitted as v1 keeps the pre-configuration audit, transaction-guarded -pre/post-migration audits, and final destination audit. The final audit also applies +pre/post-migration audits, and final destination audit, even when the process +audited the same file earlier. The final audit also applies when another opener finishes migration before this opener prepares the schema. -No integrity policy, SQLite profile, supported migration or stored format changes. +No SQLite profile, supported migration or stored format changes. Maintenance remains synchronous at the same module call sites. It first performs a short-lived existence read for quarantined roots whose deadline is at or before the diff --git a/tools/standards_snapshots/standards_snapshots/store.py b/tools/standards_snapshots/standards_snapshots/store.py index 9b42f2ac..8c1ec836 100644 --- a/tools/standards_snapshots/standards_snapshots/store.py +++ b/tools/standards_snapshots/standards_snapshots/store.py @@ -5,6 +5,7 @@ import os import sqlite3 import stat +import threading from collections.abc import Iterator from contextlib import closing, contextmanager from functools import lru_cache @@ -35,6 +36,25 @@ USER_VERSION = 2 BUSY_TIMEOUT_MS = 5_000 +# Device and inode of every store file whose complete SQLite integrity and +# foreign-key audit has passed in this process. A later open of the same file +# keeps every other admission check and omits only that audit pair. The record +# is never persisted: each new process audits again, and a file put in place +# under a different inode has no record. It is a point-in-time observation of +# one file, not proof that another process has left the file unchanged. +_AUDITED_FILES: set[tuple[int, int]] = set() +_AUDITED_FILES_LOCK = threading.Lock() + + +def _audit_recorded(identity: tuple[int, int]) -> bool: + with _AUDITED_FILES_LOCK: + return identity in _AUDITED_FILES + + +def _record_audit(identity: tuple[int, int]) -> None: + with _AUDITED_FILES_LOCK: + _AUDITED_FILES.add(identity) + def _root_reference(value: object, description: str) -> None: if type(value) is not str or not value: @@ -274,6 +294,7 @@ def __init__(self, path: Path) -> None: "SNAPSHOT_STORE.UNSUPPORTED_PATH", "SQLite path must be one regular non-symlink file", ) + identity = (observed.st_dev, observed.st_ino) else: path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL @@ -281,7 +302,7 @@ def __init__(self, path: Path) -> None: try: descriptor = os.open(path, flags, 0o600) observed = os.fstat(descriptor) - created_identity = (observed.st_dev, observed.st_ino) + identity = created_identity = (observed.st_dev, observed.st_ino) os.close(descriptor) except OSError as error: raise unavailable("SNAPSHOT_STORE.UNAVAILABLE", str(error)) from error @@ -292,11 +313,18 @@ def __init__(self, path: Path) -> None: ) self._connection = connection self._connection.enable_load_extension(False) + # SQLite has opened whichever file the path named at that moment. + # An audit is reused, and afterwards recorded, only for a path that + # still names the file observed before the connection was made. + same_file = self._path_identity() == identity if existed: - admitted_version = self._verify_existing_authority() + admitted_version = self._verify_existing_authority( + audited=same_file and _audit_recorded(identity) + ) self._configure() self._prepare_existing_schema() - # Current stores were fully audited before configuration. A v1 + # Current stores were fully audited before configuration, in + # this open or an earlier one in this process. A v1 # admission still needs the final destination audit, including # when another opener completed migration in the meantime. if admitted_version != USER_VERSION: @@ -305,6 +333,8 @@ def __init__(self, path: Path) -> None: self._configure() self._initialize_schema() self._verify_integrity() + if same_file: + _record_audit(identity) except Exception as error: self._cleanup_failed_open(connection, created_identity) if isinstance(error, SnapshotError): @@ -318,6 +348,13 @@ def __init__(self, path: Path) -> None: def close(self) -> None: self._connection.close() + def _path_identity(self) -> tuple[int, int] | None: + try: + observed = self.path.lstat() + except OSError: + return None + return (observed.st_dev, observed.st_ino) + def _cleanup_failed_open( self, connection: sqlite3.Connection | None, @@ -1047,8 +1084,12 @@ def _prepare_existing_schema(self) -> None: self._migrate_version_one() self._verify_schema() - def _verify_existing_authority(self) -> int: - """Authenticate before configuration and return the admitted schema version.""" + def _verify_existing_authority(self, *, audited: bool) -> int: + """Authenticate before configuration and return the admitted schema version. + + ``audited`` reports that this process already completed the full audit + of this exact current-schema file. It never shortens a v1 admission. + """ application_id = self._connection.execute("PRAGMA application_id").fetchone()[0] user_version = self._connection.execute("PRAGMA user_version").fetchone()[0] if application_id != APPLICATION_ID: @@ -1071,7 +1112,8 @@ def _verify_existing_authority(self) -> int: f"expected schema version 1 or {USER_VERSION}, observed {user_version}", ) self._verify_schema() - self._verify_integrity() + if not audited: + self._verify_integrity() return user_version def _migrate_version_one(self) -> None: diff --git a/tools/standards_snapshots/tests/test_storage_lifecycle.py b/tools/standards_snapshots/tests/test_storage_lifecycle.py index 66afbafd..ce7edb5e 100644 --- a/tools/standards_snapshots/tests/test_storage_lifecycle.py +++ b/tools/standards_snapshots/tests/test_storage_lifecycle.py @@ -2,26 +2,67 @@ No-work observations must not request a writer. A positive probe is only a hint: transactions still select the current due set and own all deletion effects. + +One process runs the complete integrity/foreign-key audit of a store file once. +Later opens of that file keep the application, version and exact schema checks. """ from __future__ import annotations +import os +import shutil import sqlite3 +import subprocess +import sys import tempfile import unittest from contextlib import closing from pathlib import Path from typing import Callable +from unittest import mock from tools.standards_snapshots.standards_snapshots import ( AggregateChild, AggregateRecord, AggregateRoot, ChildHandle, FindAggregateRootsRequest, FindSnapshotsRequest, SnapshotError, SnapshotModule, ) +from tools.standards_snapshots.standards_snapshots import store as store_module from tools.standards_snapshots.standards_snapshots.store import SQLiteSnapshotStore from tools.standards_snapshots.tests.test_module import capture from tools.standards_snapshots.tests.test_store import ( _populated_version_one_store, _version_one_rows, ) +ROOT = Path(__file__).resolve().parents[3] + +# A separate interpreter has no audit record. It reports the admission outcome +# of the ordinary module open path for one existing store file. +_COLD_OPEN = """ +import sys +from pathlib import Path +from tools.standards_snapshots.standards_snapshots import SnapshotError, SnapshotModule +try: + SnapshotModule.open(Path(sys.argv[1])).close() +except SnapshotError as error: + print(error.failure.code) +else: + print('admitted') +""" + + +def _begin_as_new_process() -> None: + """Discard this process's audit record, as a newly started process has none.""" + with store_module._AUDITED_FILES_LOCK: + store_module._AUDITED_FILES.clear() + + +def _file_identity(path: Path) -> tuple[int, int]: + observed = path.lstat() + return observed.st_dev, observed.st_ino + + +def _violate_foreign_key(path: Path) -> None: + with closing(sqlite3.connect(path, isolation_level=None)) as corruptor: + corruptor.execute("INSERT INTO content_files VALUES ('missing', 'a', X'', 0, 'digest')") + class _ObservedAdmissionStore(SQLiteSnapshotStore): """Record proof phases while running the unchanged real proof implementations.""" @@ -74,6 +115,25 @@ def setUp(self): self.addCleanup(directory.cleanup) self.path = Path(directory.name) / 'snapshots.sqlite3' self.now = 2_000_000_000 + # Audit expectations must not depend on stores opened by earlier tests, + # whose removed files can lend their inode numbers to these fixtures. + _begin_as_new_process() + + def events(self, path=None): + opened = _ObservedAdmissionStore(path or self.path) + try: + return opened.admission_events + finally: + opened.close() + + def cold_open(self, path=None): + completed = subprocess.run( + [sys.executable, '-P', '-c', _COLD_OPEN, str(path or self.path)], + env={**os.environ, 'PYTHONPATH': str(ROOT)}, + text=True, capture_output=True, timeout=60, + ) + self.assertEqual(completed.returncode, 0, completed.stderr) + return completed.stdout.strip() def module(self): module = SnapshotModule.open(self.path, now=lambda: self.now, quarantine_seconds=10) @@ -311,22 +371,110 @@ def test_probe_sql_error_retains_typed_failure_without_transaction(self): self.assertNotIn('BEGIN IMMEDIATE', statements) self.assertFalse(module._store._connection.in_transaction) - def test_new_and_current_admission_have_one_full_integrity_audit(self): - created = _ObservedAdmissionStore(self.path) - try: - self.assertEqual(created.admission_events, [('configure', 0, False), ('integrity', 2, False)]) - finally: - created.close() + def test_new_and_current_admission_have_one_full_integrity_audit_per_process(self): + # The creating process audits its new file, and does not audit it again. + self.assertEqual(self.events(), [('configure', 0, False), ('integrity', 2, False)]) + for _ in range(2): + self.assertEqual(self.events(), [('configure', 2, False)]) + # A new process audits the existing file once, before configuration. + _begin_as_new_process() + self.assertEqual(self.events(), [('integrity', 2, False), ('configure', 2, False)]) for _ in range(2): - opened = _ObservedAdmissionStore(self.path) - try: - self.assertEqual(opened.admission_events, [('integrity', 2, False), ('configure', 2, False)]) - finally: - opened.close() + self.assertEqual(self.events(), [('configure', 2, False)]) + + def test_two_module_opens_in_one_process_audit_the_file_once(self): + with SnapshotModule.open(self.path, now=lambda: self.now) as initial: + snapshot = initial.create_snapshot(capture()).snapshot + _begin_as_new_process() + opens = [] + connect = sqlite3.connect + + def traced_connect(database, *arguments, **options): + # Observe each store connection from before its first statement. + connection = connect(database, *arguments, **options) + if Path(database).resolve() == self.path.resolve(): + opens.append([]) + connection.set_trace_callback(opens[-1].append) + return connection + + with mock.patch.object(store_module.sqlite3, 'connect', traced_connect): + for _ in range(2): + with SnapshotModule.open(self.path, now=lambda: self.now) as reopened: + self.assertEqual(reopened.load_content(snapshot), capture()) + first, second = opens + audit = ['PRAGMA integrity_check', 'PRAGMA foreign_key_check'] + self.assertEqual([statement for statement in first if statement in audit], audit) + # The later open omits exactly that pair and repeats every other statement. + self.assertEqual([statement for statement in first if statement not in audit], second) + self.assertIn('PRAGMA application_id', second) + self.assertIn('PRAGMA user_version', second) + self.assertTrue(any('FROM sqlite_schema' in statement for statement in second), second) + + def test_replaced_file_is_audited_again(self): + self.assertEqual(self.events(), [('configure', 0, False), ('integrity', 2, False)]) + self.assertEqual(self.events(), [('configure', 2, False)]) + audited = _file_identity(self.path) + # The replacement exists beside the audited file, so it has another inode. + replacement = self.path.with_name('replacement.sqlite3') + shutil.copyfile(self.path, replacement) + self.assertNotEqual(_file_identity(replacement), audited) + os.replace(replacement, self.path) + self.assertEqual(self.events(), [('integrity', 2, False), ('configure', 2, False)]) + self.assertEqual(self.events(), [('configure', 2, False)]) + + def test_corrupt_replacement_of_an_audited_file_is_rejected(self): + SQLiteSnapshotStore(self.path).close() + SQLiteSnapshotStore(self.path).close() + replacement = self.path.with_name('replacement.sqlite3') + shutil.copyfile(self.path, replacement) + _violate_foreign_key(replacement) + os.replace(replacement, self.path) + self.assert_code('SNAPSHOT_STORE.FOREIGN_KEY_FAILURE', lambda: SQLiteSnapshotStore(self.path)) + # A rejected file gains no audit record: the next open audits it again. + self.assert_code('SNAPSHOT_STORE.FOREIGN_KEY_FAILURE', lambda: SQLiteSnapshotStore(self.path)) + self.assertTrue(self.path.is_file()) + + def test_later_opens_keep_application_version_and_schema_checks(self): + for change, code in ( + ('PRAGMA application_id=1', 'SNAPSHOT_STORE.INVALID_APPLICATION'), + ('PRAGMA user_version=3', 'SNAPSHOT_STORE.UNSUPPORTED_VERSION'), + ('PRAGMA user_version=0', 'SNAPSHOT_STORE.INVALID_VERSION'), + ('CREATE TABLE unexpected (value TEXT)', 'SNAPSHOT_STORE.INVALID_SCHEMA'), + ('DROP TRIGGER child_index_no_update', 'SNAPSHOT_STORE.INVALID_SCHEMA'), + ): + with self.subTest(change=change), tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / 'snapshots.sqlite3' + SQLiteSnapshotStore(path).close() + self.assertEqual(self.events(path), [('configure', 2, False)]) + with closing(sqlite3.connect(path, isolation_level=None)) as changer: + changer.execute(change) + # Same file, already audited by this process: only the audit is omitted. + self.assert_code(code, lambda: _ObservedAdmissionStore(path)) + self.assertTrue(path.is_file()) + + def test_new_process_rejects_corruption_in_a_file_audited_elsewhere(self): + for corruption, code in (('foreign-key', 'SNAPSHOT_STORE.FOREIGN_KEY_FAILURE'), + ('check', 'SNAPSHOT_STORE.INTEGRITY_FAILURE')): + with self.subTest(corruption=corruption), tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / 'snapshots.sqlite3' + # This process audits the file; that record stays in this process. + SQLiteSnapshotStore(path).close() + self.assertEqual(self.cold_open(path), 'admitted') + with closing(sqlite3.connect(path, isolation_level=None)) as corruptor: + if corruption == 'foreign-key': + corruptor.execute("INSERT INTO content_files VALUES ('missing', 'a', X'', 0, 'digest')") + else: + corruptor.execute('PRAGMA ignore_check_constraints=ON') + corruptor.execute("INSERT INTO content_sets VALUES ('bad', 0)") + self.assertEqual(self.cold_open(path), code) + self.assertTrue(path.is_file()) def test_version_one_keeps_preconfiguration_guarded_and_final_audits(self): _, record = _populated_version_one_store(self.path) before = _version_one_rows(self.path) + # Building the fixture audited this same file as a current store. That + # record must not shorten any audit of its version-one admission. + self.assertTrue(store_module._audit_recorded(_file_identity(self.path))) opened = _ObservedAdmissionStore(self.path) try: self.assertEqual(opened.admission_events, [ @@ -338,6 +486,18 @@ def test_version_one_keeps_preconfiguration_guarded_and_final_audits(self): opened.close() self.assertEqual(_version_one_rows(self.path), before) + def test_version_one_migration_in_a_new_process_keeps_every_audit(self): + _populated_version_one_store(self.path) + before = _version_one_rows(self.path) + _begin_as_new_process() + self.assertEqual(self.events(), [ + ('integrity', 1, False), ('configure', 1, False), + ('integrity', 1, True), ('integrity', 2, True), ('integrity', 2, False), + ]) + self.assertEqual(_version_one_rows(self.path), before) + # The final destination audit is this process's audit of the migrated file. + self.assertEqual(self.events(), [('configure', 2, False)]) + def test_other_opener_migration_keeps_destination_audit(self): _populated_version_one_store(self.path) before = _version_one_rows(self.path) @@ -404,6 +564,8 @@ def test_current_corruption_is_rejected_before_persistent_configuration(self): else: corruptor.execute('PRAGMA ignore_check_constraints=ON') corruptor.execute("INSERT INTO content_sets VALUES ('bad', 0)") + # The first open by a process is the one that audits. + _begin_as_new_process() self.assert_code(code, lambda: _ObservedAdmissionStore(path)) self.assertTrue(path.is_file()) with closing(sqlite3.connect(path)) as observed: