From 2c7e49ab3479e88e6007a1c2d91486f1d831227c Mon Sep 17 00:00:00 2001 From: Dong1017 <55723162+Dong1017@users.noreply.github.com> Date: Fri, 30 Oct 2026 15:58:18 +0900 Subject: [PATCH 1/2] feat(skilleval): add bounded cloud generation and isolated replay Retain generation-only parser artifacts, bind historical experiment accounting separately from compared revisions, and replay fixed data-only bundles on qualified Linux hosts. Preserve original inconclusive captures and report independent artifact checks separately. Signed-off-by: Codex Agent-assisted-by: Codex design_cloud_trial253 (implementation) Agent-assisted-by: Codex verify_trial_cost_bounds (accounting implementation) Agent-assisted-by: Codex find_skilleval_unblock (replay implementation) --- .github/workflows/skill-eval-cloud-replay.yml | 55 ++++ docs/operations.md | 2 + docs/skill-eval-cloud-replay.md | 54 ++++ docs/skill-eval-generation-only.md | 62 +++++ .../harness/skill_eval_engineering.py | 42 +++ src/inferrsi/harness/skill_evaluation.py | 14 +- tests/test_skill_eval_cloud_replay.py | 107 ++++++++ tests/test_skill_eval_generation.py | 259 ++++++++++++++++++ .../test_skill_eval_historical_accounting.py | 183 +++++++++++++ tools/skill_eval_cloud_export.py | 72 +++++ tools/skill_eval_cloud_replay.py | 143 ++++++++++ tools/skill_eval_compare.py | 91 +++++- tools/skill_eval_historical_accounting.py | 177 ++++++++++++ 13 files changed, 1245 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/skill-eval-cloud-replay.yml create mode 100644 docs/skill-eval-cloud-replay.md create mode 100644 docs/skill-eval-generation-only.md create mode 100644 tests/test_skill_eval_cloud_replay.py create mode 100644 tests/test_skill_eval_generation.py create mode 100644 tests/test_skill_eval_historical_accounting.py create mode 100644 tools/skill_eval_cloud_export.py create mode 100644 tools/skill_eval_cloud_replay.py create mode 100644 tools/skill_eval_historical_accounting.py diff --git a/.github/workflows/skill-eval-cloud-replay.yml b/.github/workflows/skill-eval-cloud-replay.yml new file mode 100644 index 000000000..1f3f6d998 --- /dev/null +++ b/.github/workflows/skill-eval-cloud-replay.yml @@ -0,0 +1,55 @@ +name: SkillEval isolated cloud replay (manual) + +on: + workflow_dispatch: + inputs: + evidence_commit: + description: Exact repository commit containing the fixed-path two-parser bundle + required: true + type: string + bundle_sha256: + description: SHA256 of exact bundle bytes + required: true + type: string + +permissions: {} + +concurrency: + group: skill-eval-cloud-replay + cancel-in-progress: false + +jobs: + replay: + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-22.04 + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Install distribution bubblewrap, without changing security policy + run: | + sudo apt-get update + sudo apt-get install --yes --no-install-recommends bubblewrap + - name: Qualify then replay the pinned data-only bundle + shell: bash + env: + EVIDENCE_COMMIT: ${{ inputs.evidence_commit }} + BUNDLE_SHA256: ${{ inputs.bundle_sha256 }} + run: | + env -i PATH=/usr/bin:/bin LANG=C.UTF-8 PYTHONPATH=src \ + /usr/bin/python3 -m tools.skill_eval_cloud_replay \ + --output "$RUNNER_TEMP/cloud-replay" \ + --evidence-commit "$EVIDENCE_COMMIT" --bundle-sha256 "$BUNDLE_SHA256" \ + --revision "$GITHUB_SHA" --runner-image "${ImageOS:-unknown}/${ImageVersion:-unknown}" + - name: Retain replay receipt, including unavailable isolation + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: cloud-replay-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/cloud-replay/replay.json + if-no-files-found: error + retention-days: 14 diff --git a/docs/operations.md b/docs/operations.md index c61ff6b65..859f7ff97 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -29,7 +29,9 @@ Effect classes used below: | Adapt existing Flare delivery evidence | [Offline qualification adapter](flare-qualification.md), `python -m inferrsi.model.validators.flare` | Read-only | Existing raw packets and a separate receipt; one HTTP delivery metric, no model/server/device execution | | Evaluate selected skill revisions | [Bounded skill-evaluation driver](skill-evaluation.md) | Experiment-executing | Explicitly authorized real Codex calls, independent task verdicts and private evidence; separate from numeric engine-performance scoring | | Compare skill additions or updates | [Configured skill comparison](skill-eval-comparison.md), `python -m tools.skill_eval_compare` | Read-only preparation / experiment-executing / report-generating | Fixed Git snapshots and one shared exam; explicit plan authorization for execution; JSON + Markdown regeneration makes no model calls | +| Export a cloud-generated parser pair | [Generation-only protocol](skill-eval-generation-only.md), `python -m tools.skill_eval_cloud_export` | Local evidence-writing; no execution or publication | Validates frozen inputs and retained sources; writes one bounded inert JSON bundle, never executes parser source | | Qualify an artifact verifier host | [Manual reviewer qualification](skill-eval-reviewer-qualification.md), `python -m tools.skill_eval_qualify` | Trusted-control execution / report-generating | Manual main-only workflow; fixed software controls, no model or external artifact input; unavailable isolation fails rather than skips | +| Replay a cloud-generated parser pair | [Isolated cloud replay](skill-eval-cloud-replay.md), `python -m tools.skill_eval_cloud_replay` | Untrusted-artifact execution / report-generating | Manual pinned data ingress; same-host controls before bubblewrap-only replay; no models or credentials | | Validate one harness-improvement step | [Offline improvement](harness-improvement.md), Python `tools.skill_eval_improve.prepare` / `run_offline` | Snapshot-writing / offline experiment-executing | Supplied declarative INV/DOM edits, shared total limits, rerun and one holdout gate; trusted fixed-output clients only; no real-model or Store execution | | Plan and report skill PR evaluations | [Skill PR integration](skill-eval-pr.md), `python -m tools.skill_eval_pr` | Read-only planning / authorized experiment-executing / PR-publishing | Trusted mapping, exact operator approval, consumed execution identity and credential-separated publication from retained evidence; no automatic paid trials | | Recover goal progress and the next proposal | [Offline goal checkpoints](goal-checkpoints.md), `python -m tools.goal_checkpoint` | Read-only | Explicit scope/criteria, pinned owner decisions and existing comparison receipts; a disposable view, never execution or acceptance authority | diff --git a/docs/skill-eval-cloud-replay.md b/docs/skill-eval-cloud-replay.md new file mode 100644 index 000000000..e56852046 --- /dev/null +++ b/docs/skill-eval-cloud-replay.md @@ -0,0 +1,54 @@ +# Isolated replay of a cloud-generated parser pair + +The manual `skill-eval-cloud-replay.yml` workflow removes any requirement that +an author host the verifier. It performs no model calls and receives no model +credentials. Generation is separate; neither remote replay nor green CI proves +producer isolation, causal comparability, generalization, or campaign acceptance. + +## Data contract and transport + +An operator first reviews and publishes the **data**, not executable controller +code, at `artifacts/skill-eval-cloud/bundle.json` in this repository. Publication +must be authorized and checked for private data. The dispatch takes only the +exact 40-character evidence commit and SHA256 of the complete bundle bytes. +The trusted controller is checked out from the dispatch's exact main revision, +not the evidence commit. Its unauthenticated HTTPS read uses one fixed repository +and path, rejects redirects, and reads at most 1 MiB plus one rejection byte. +There are no archive extraction, arbitrary URLs, user paths, custom graders, +configuration commands, or imports of uploaded Python. + +The UTF-8 JSON object has exactly these keys (duplicate keys are rejected): + +- `schema_version`: `engineering-cloud-replay-v1`. +- `task_id`: 1–128 ASCII letters, numbers, underscores, dots or hyphens; an opaque + provenance identifier, not a path or executable task selector. +- `request_sha256`: SHA256 of the trusted `engineering.REQUEST` UTF-8 bytes. +- `grader_sha256`: SHA256 of the trusted `skill_eval_engineering.py` file bytes. +- `inputs_sha256`, `protocol_sha256`: lowercase 64-character provenance digests. + These are retained, **not independently authenticated by this receipt**. +- `artifacts`: exactly two objects, ordered baseline then candidate, each with + exactly `arm`, `source`, and `sha256`. Each source is nonempty UTF-8, at most + 65,536 bytes, and must match its digest. Both arms are validated before either + is executed. JSON escaping does not change the source-byte hash. + +## Execution and interpretation + +The controller first requalifies the reviewed good and bad controls on this +same runner. If qualification is not PASS, neither submitted artifact runs. +Each parser then executes only through the existing bubblewrap verifier under +its filesystem, network and resource bounds. Missing isolation never falls back +to host execution. The shell clears inherited environment variables and never +passes its repository token to parser execution. No host security policy is +weakened. Model source and outputs are not echoed into workflow logs. + +`replay.json` retains the qualification, controller/evidence identities, bundle +and source hashes, and separate per-arm checks. Its top-level PASS means **both +replays were delivered**, not that either parser passed or that a skill helped; +negative per-arm results are valid evidence. Invalid ingress, infrastructure +failure or lost isolation yields INCONCLUSIVE and a nonzero job. Original +producer/capture failures remain unchanged; reconcile the receipt against the +frozen experiment protocol and original evidence before any comparison claim. + +The workflow is main-only and manual, independent of normal PR/merge CI. It does +not comment, accept, close or merge anything. A reviewer can independently +regenerate the bounded replay from the retained commit and bundle digest. diff --git a/docs/skill-eval-generation-only.md b/docs/skill-eval-generation-only.md new file mode 100644 index 000000000..ab21026eb --- /dev/null +++ b/docs/skill-eval-generation-only.md @@ -0,0 +1,62 @@ +# Engineering parser generation-only mode + +This opt-in #253 transport lets a provider produce parser source without running +model-produced code on the controller. It is **not** an iterative workspace-agent +experiment and does not replace the Linux/bubblewrap verifier. + +In a trusted comparison configuration set both: + +```json +{ + "adapter": "engineering-artifact-v1", + "execution_mode": "provider-generation-only-v1" +} +``` + +The adapter name must match the current engineering adapter constant. The client +must be the existing `anthropic-messages-full-context-v1` client, with reviewed +model, pricing and spend limits. Both arms use the same configuration. Preparation +freezes task, starter, skill snapshots, schema and execution mode before calls. +No default PR mapping or provider configuration is changed by this feature. + +The provider receives the frozen task, starter and its assigned skill only. Its +sole tool returns `artifact`, `summary`, and `artifact_source`; it cannot execute +commands or modify files. A trusted writer accepts exactly one `parser.py` source +string of 1–65,536 UTF-8 bytes and writes those bytes only after checking that the +starter is unchanged. Path alternatives, extra fields and malformed or oversized +payloads fail closed. Source is retained as data; it is not imported or executed +by the writer. Summary claims are not grading evidence. + +Execution always requires the existing request authorization broker, even if a +caller omits the general authorization flag. A stub is usable only in offline +unit tests. This does not authorize paid calls: current provider pricing, +per-request reservations, PR lifetime/day accounting and the precise experiment +identity must be independently authorized and frozen. A new accounting PR must +not be represented as the historical skill revisions being compared. + +When the controller cannot start bubblewrap, the original trial remains +`INCONCLUSIVE` with `artifact_sandbox_unavailable`. A separately qualified Linux +reviewer may replay the retained artifact after frozen-input/source integrity +checks. The report preserves that original result and separately exposes +`artifact_verdict`, the 19 artifact checks and `replayed_artifact_outcome`. +Tampered inputs/source, malformed output and other execution errors are not +eligible for this deferred replay. Existing workspace-agent execution is +unchanged. A passing artifact comparison is not evidence of author-agent +isolation, general skill efficacy or causal improvement from a single pair. + +Offline regression coverage: `tests/test_skill_eval_generation.py`. Real sandbox +qualification, artifact transport and paid trial execution are separate steps; +these unit tests do not establish that a particular host is qualified. + +After the authorized two-call capture, export only retained source and provenance: + +```sh +python -m tools.skill_eval_cloud_export --run /path/to/frozen-run \ + --protocol /path/to/prospective-protocol.json --output /path/to/bundle.json +``` + +The exporter refuses incomplete usage, modified inputs, mismatched receipts or +source, additional pairs and a changed public request. It does not upload the +bundle or establish the timing of protocol approval; the execution admission +controller must bind that protocol hash before calls. Treat both sources as +untrusted until independent qualified replay. diff --git a/src/inferrsi/harness/skill_eval_engineering.py b/src/inferrsi/harness/skill_eval_engineering.py index 32d99a8a1..3fdb79904 100644 --- a/src/inferrsi/harness/skill_eval_engineering.py +++ b/src/inferrsi/harness/skill_eval_engineering.py @@ -18,6 +18,7 @@ from pathlib import Path ADAPTER = "engineering-artifact-v1" +GENERATION_MODE = "provider-generation-only-v1" MAX_SOURCE_BYTES = 65536 STARTER = ( 'def evaluate(status, frames, requested_tokens):\n' @@ -38,6 +39,47 @@ "commits or publication. Return artifact='parser.py' and a concise summary, not a verdict. " "The controller retains the actual file and independently checks its behavior." ) +GENERATION_SCHEMA = { + **SCHEMA, + "required": ["artifact", "summary", "artifact_source"], + "properties": {**SCHEMA["properties"], "artifact_source": {"type": "string"}}, +} +GENERATION_PROMPT = ( + "Read the supplied task.json, parser.py starter and assigned knowledge. Generate the " + "complete repaired parser.py source as artifact_source in submit_answer. " + "Implement evaluate(status, frames, requested_tokens) under the public task contract. " + "You have no shell, execution, file-writing, network or test tool. Do not claim to " + "have run tests. Return artifact='parser.py', a concise summary and artifact_source; " + "do not return a verdict. A trusted controller writes only parser.py from the source " + "string and an independent isolated verifier checks its behavior later." +) + + +def materialize_generated_artifact(workspace: Path, answer: dict) -> dict: + """Convert a bounded provider response to one artifact; never execute its source. + + The provider is a structured-output service, not a process with workspace access. + Fresh workspace ownership and the unchanged starter are required before writing. + """ + if (not isinstance(answer, dict) + or set(answer) != {"artifact", "summary", "artifact_source"} + or answer["artifact"] != "parser.py" + or not isinstance(answer["summary"], str) + or not isinstance(answer["artifact_source"], str)): + raise ValueError("Invalid generated artifact answer") + try: + source = answer["artifact_source"].encode("utf-8") + except UnicodeError: + raise ValueError("Generated artifact must be UTF-8") from None + if not 0 < len(source) <= MAX_SOURCE_BYTES: + raise ValueError("Generated artifact exceeds source bounds") + artifact = workspace / "parser.py" + if read_regular(artifact, MAX_SOURCE_BYTES) != STARTER.encode("utf-8"): + raise ValueError("Generated artifact requires the unchanged frozen starter") + artifact.write_bytes(source) + return {"artifact": "parser.py", "summary": answer["summary"]} + + REQUEST = ( "Repair parser.py: evaluate(status, frames, requested_tokens) returns exactly " "{success: bool, reason: str}. Inputs are an integer HTTP status, decoded frames, " diff --git a/src/inferrsi/harness/skill_evaluation.py b/src/inferrsi/harness/skill_evaluation.py index 4de6960d3..78dcdd788 100644 --- a/src/inferrsi/harness/skill_evaluation.py +++ b/src/inferrsi/harness/skill_evaluation.py @@ -189,7 +189,10 @@ def run_trial( *, timeout_seconds: float | None = None, engineering_artifact: bool = False, + generation_only: bool = False, ) -> dict: + if generation_only and not engineering_artifact: + raise ValueError("Generation-only requires the engineering artifact adapter") trial_dir.mkdir(parents=True) final = trial_dir / "answer.json" before = engineering.input_hashes(workspace) if engineering_artifact else { @@ -233,11 +236,18 @@ def run_trial( evaluation_error = "missing_output" else: try: - answer_bytes = (engineering.read_regular(final, 65536) if engineering_artifact + answer_limit = 1048576 if generation_only else 65536 + answer_bytes = (engineering.read_regular(final, answer_limit) + if engineering_artifact else final.read_bytes()) output = json.loads(answer_bytes.decode("utf-8")) except (ValueError, UnicodeError): evaluation_error = "invalid_output_json" + if evaluation_error is None and generation_only: + try: + output = engineering.materialize_generated_artifact(workspace, output) + except (ValueError, OSError): + evaluation_error = "invalid_generated_artifact" if evaluation_error is None and engineering_artifact: artifact = workspace / "parser.py" if not isinstance(output, dict) or set(output) != {"artifact", "summary"}: @@ -333,5 +343,7 @@ def run_trial( record["artifact_sha256"] = (sha256(retained) if retained.is_file() and not retained.is_symlink() else None) record["artifact_kind"] = engineering.ADAPTER + if generation_only: + record["execution_mode"] = engineering.GENERATION_MODE atomic_write_json(trial_dir / "result.json", record) return record diff --git a/tests/test_skill_eval_cloud_replay.py b/tests/test_skill_eval_cloud_replay.py new file mode 100644 index 000000000..d57999d76 --- /dev/null +++ b/tests/test_skill_eval_cloud_replay.py @@ -0,0 +1,107 @@ +"""Validate data ingress without executing untrusted code or making network calls.""" +import json +from pathlib import Path +from unittest.mock import patch + +import pytest +import yaml + +from tools import skill_eval_cloud_replay as replay + + +def bundle(): + source = replay.engineering.STARTER + return {"schema_version": replay.SCHEMA, "task_id": "public-parser-task", + "request_sha256": replay.digest(replay.engineering.REQUEST.encode()), + "grader_sha256": replay.digest(Path(replay.engineering.__file__).read_bytes()), + "inputs_sha256": "a" * 64, "protocol_sha256": "b" * 64, + "artifacts": [{"arm": arm, "source": source, "sha256": replay.digest(source.encode())} + for arm in ("baseline", "candidate")]} + + +def encoded(value): + raw = json.dumps(value).encode() + return raw, replay.digest(raw) + + +def test_valid_bundle_exact_bytes(): + value = bundle() + assert replay.validate_bundle(*encoded(value)) == value + + +@pytest.mark.parametrize("mutation", ["extra", "wrong_request", "wrong_grader", "duplicate_arm", + "source_hash", "big_source", "surrogate", "path", "missing"]) +def test_invalid_bundle_rejected_before_qualification(mutation): + value = bundle() + if mutation == "extra": + value["command"] = "untrusted" + elif mutation in ("wrong_request", "wrong_grader"): + value[mutation.removeprefix("wrong_") + "_sha256"] = "0" * 64 + elif mutation == "duplicate_arm": + value["artifacts"][1]["arm"] = "baseline" + elif mutation == "source_hash": + value["artifacts"][0]["sha256"] = "0" * 64 + elif mutation == "big_source": + value["artifacts"][0]["source"] = "x" * 65537 + elif mutation == "surrogate": + value["artifacts"][0]["source"] = "\ud800" + elif mutation == "path": + value["task_id"] = "../../oops" + else: + del value["protocol_sha256"] + with patch.object(replay, "qualify") as qualify, pytest.raises(ValueError): + replay.replay(*encoded(value), "a" * 40, "test") + qualify.assert_not_called() + + +def test_duplicate_json_key_size_and_digest_rejected(): + for raw in (b'{"x":1,"x":2}', b"[" * 2000, b"x" * (replay.MAX_BUNDLE_BYTES + 1), b"\xff"): + with pytest.raises(ValueError): + replay.validate_bundle(raw, replay.digest(raw)) + with pytest.raises(ValueError): + replay.validate_bundle(encoded(bundle())[0], "0" * 64) + + +def test_qualification_failure_never_executes_artifacts(): + with patch.object(replay, "qualify", return_value={"verdict": "INCONCLUSIVE"}), \ + patch.object(replay.engineering, "verify") as verify: + result = replay.replay(*encoded(bundle()), "a" * 40, "test") + assert result["verdict"] == "INCONCLUSIVE" + verify.assert_not_called() + + +def test_replay_keeps_negative_controls_as_delivery_not_efficacy(): + with patch.object(replay, "qualify", return_value={"verdict": "PASS"}), \ + patch.object(replay.engineering, "verify", return_value={"passed": False}) as verify: + result = replay.replay(*encoded(bundle()), "a" * 40, "test") + assert result["verdict"] == "PASS" and verify.call_count == 2 + assert all(item["result"]["passed"] is False for item in result["artifacts"]) + assert "source" not in json.dumps(result) + + +def test_lost_isolation_stays_inconclusive(): + with patch.object(replay, "qualify", return_value={"verdict": "PASS"}), \ + patch.object(replay.engineering, "verify", + side_effect=replay.engineering.SandboxUnavailable): + assert replay.replay(*encoded(bundle()), "a" * 40, "test")["verdict"] == "INCONCLUSIVE" + + +def test_no_arbitrary_ingress_url_and_no_redirect(): + for value in ("main", "../evil", "https://evil", "a" * 40 + ";id"): + with pytest.raises(ValueError): + replay.fetch_bundle(value) + with pytest.raises(ValueError): + replay.NoRedirect().redirect_request(None, None, 302, "", {}, "https://elsewhere") + + +def test_workflow_inputs_not_interpolated_into_shell(): + root = Path(__file__).resolve().parents[1] + text = (root / ".github/workflows/skill-eval-cloud-replay.yml").read_text() + workflow = yaml.safe_load(text) + assert set(workflow[True]) == {"workflow_dispatch"} + job = workflow["jobs"]["replay"] + assert job["if"] == "github.ref == 'refs/heads/main'" + assert job["permissions"] == {"contents": "read"} + for step in job["steps"]: + assert "${{ inputs." not in step.get("run", "") + assert "env -i" in text and "secrets." not in text and "sysctl" not in text diff --git a/tests/test_skill_eval_generation.py b/tests/test_skill_eval_generation.py new file mode 100644 index 000000000..688a3d88d --- /dev/null +++ b/tests/test_skill_eval_generation.py @@ -0,0 +1,259 @@ +"""Offline generation-only controller tests; no provider or artifact execution.""" +import hashlib +import json +from pathlib import Path +from unittest import mock + +import pytest + +from inferrsi.harness import skill_eval_engineering as engineering +from inferrsi.harness import skill_evaluation as harness +from inferrsi.infra.artifacts import sha256, write_json +from inferrsi.model import anthropic_agent as agent +from tests.fixtures.skill_eval_compare_support import configuration +from tools import skill_eval_compare as compare + +TASK = {"id": "parser", "split": "development", "request": engineering.REQUEST, + "provenance": "Reconstructed CPU public fixture"} +SOURCE = "def evaluate(status, frames, requested_tokens):\n return {}\n" +ANSWER = {"artifact": "parser.py", "summary": "Generated, not executed", "artifact_source": SOURCE} +SPEC = {"kind": agent.FULL_CONTEXT_KIND, "provider": "yibuapi", "endpoint": agent.ENDPOINT, + "api_version": "2023-06-01", "max_requests": 1, "max_output_tokens": 4096, + "max_request_bytes": 262144, "max_read_bytes": 32768, "max_tool_calls": 12} + + +def prepared(tmp_path): + config, repo = configuration(tmp_path, ids=("parser",)) + data = compare.read_json(config) + data.update(adapter=engineering.ADAPTER, execution_mode=engineering.GENERATION_MODE, + model={"name": "claude-opus-5-5", "effort": "not_applicable"}, client=SPEC) + write_json(config, data) + write_json(tmp_path / "cases.json", [TASK]) + output = tmp_path / "evidence" + digest = compare.prepare(config, repo, output) + return data, output, digest + + +def test_freezes_explicit_mode_schema_starter_and_two_identical_client_conditions(tmp_path): + _, output, digest = prepared(tmp_path) + plan = compare.load_plan(output) + assert plan["execution_mode"] == engineering.GENERATION_MODE + assert plan["prompt"] == engineering.GENERATION_PROMPT + assert compare.read_json(output / "plan/schema.json") == engineering.GENERATION_SCHEMA + assert (output / "plan/starter/parser.py").read_bytes() == engineering.STARTER.encode() + assert plan["files"]["plan/starter/parser.py"] == sha256(output / "plan/starter/parser.py") + assert plan["max_api_requests"] == 2 + assert plan["client"]["settings"] == SPEC # one shared configuration, not arm-specific + report = compare.report(output) + assert report["execution_mode"] == engineering.GENERATION_MODE + assert "not measure iterative" in report["claim"] + with pytest.raises(ValueError, match="pricing/cap"): + compare.execute(output, digest) + assert not list((output / "results").iterdir()) + + +@pytest.mark.parametrize("change", [ + {"execution_mode": "unknown"}, {"adapter": "audit"}, {"client": None}, + {"client": {**SPEC, "kind": agent.KIND, "max_requests": 2}}, +]) +def test_mode_never_silently_falls_back(tmp_path, change): + data, _, _ = prepared(tmp_path) + data.update(change) + if data.get("client") is None: + data.pop("client", None) + with pytest.raises(ValueError): + compare.validate(data, [TASK]) + + +def test_legacy_engineering_still_rejects_provider(tmp_path): + data, _, _ = prepared(tmp_path) + data.pop("execution_mode") + with pytest.raises(ValueError, match="require local execution"): + compare.validate(data, [TASK]) + + +def workspace(tmp_path): + root = tmp_path / "work" + root.mkdir() + write_json(root / "task.json", TASK) + (root / "parser.py").write_bytes(engineering.STARTER.encode()) + (root / "knowledge").mkdir() + (root / "knowledge/SKILL.md").write_text("assigned arm only") + return root + + +def test_context_contains_full_starter_and_assigned_input_only(tmp_path): + root = workspace(tmp_path) + (tmp_path / "other-arm.txt").write_text("not exposed") + payload, manifest = agent.full_context_payload( + root, engineering.GENERATION_SCHEMA, engineering.GENERATION_PROMPT, + "claude-opus-5-5", SPEC) + assert {x["path"] for x in manifest} == {"task.json", "parser.py", "knowledge/SKILL.md"} + assert [x["name"] for x in payload["tools"]] == ["submit_answer"] + assert "not exposed" not in agent.canonical(payload) + assert engineering.STARTER in [x["content"] for x in json.loads( + payload["messages"][0]["content"].split("submission.\n", 1)[1])] + + +@pytest.mark.parametrize("change", [ + {"artifact": "../../outside.py"}, {"artifact_source": ""}, + {"artifact_source": "x" * 65537}, {"artifact_source": "é" * 32769}, + {"artifact_source": "\ud800"}, {"artifact_source": None}, {"summary": []}, + {"files": {"extra.py": "bad"}}, {"passed": True}, +]) +def test_materializer_rejects_bad_or_multiple_outputs_without_writing(tmp_path, change): + root = workspace(tmp_path) + before = {p.relative_to(root): p.read_bytes() for p in root.rglob("*") if p.is_file()} + with pytest.raises(ValueError): + engineering.materialize_generated_artifact(root, {**ANSWER, **change}) + assert before == {p.relative_to(root): p.read_bytes() + for p in root.rglob("*") if p.is_file()} + + +def test_materializer_requires_untouched_starter_and_exact_bytes(tmp_path): + root = workspace(tmp_path) + frozen = engineering.input_hashes(root) + assert engineering.materialize_generated_artifact(root, ANSWER) == { + "artifact": "parser.py", "summary": ANSWER["summary"]} + assert (root / "parser.py").read_bytes() == SOURCE.encode() + assert engineering.input_hashes(root) == frozen + with pytest.raises(ValueError, match="unchanged"): + engineering.materialize_generated_artifact(root, ANSWER) + + +@pytest.mark.parametrize("sandbox_available", [True, False]) +def test_trial_retains_real_source_without_executing_it_on_host(tmp_path, sandbox_available): + root = workspace(tmp_path) + trial = tmp_path / "trial" + + class OfflineClient: + timeout_seconds = 10 + + def command(self, *args): + return ("trusted-stub",) + + def trace_metrics(self, path): + return {"usage": {"input_tokens": 7, "output_tokens": 3}, "commands": [], + "errors": [], "completed_turns": 1} + + def execute(command, cwd, folder, deadline): + assert command == ("trusted-stub",) + folder.mkdir(parents=True) + (folder / "stdout.log").write_text("offline controller fixture") + (folder / "stderr.log").write_text("") + write_json(trial / "answer.json", ANSWER) + return {"status": "completed", "returncode": 0, "elapsed_seconds": 0} + + checks = {"passed": False, "assertions": {"stream_error": False}} + with mock.patch.object(harness, "execute_command", side_effect=execute), \ + mock.patch.object(engineering, "verify", return_value=checks, + side_effect=None if sandbox_available else + engineering.SandboxUnavailable("unavailable")) as verify: + record = harness.run_trial(OfflineClient(), TASK, "baseline", 1, root, trial, + tmp_path / "schema", tmp_path / "verifier", "prompt", + float("inf"), engineering_artifact=True, generation_only=True) + assert verify.call_args.args[1]["artifact_source"] == SOURCE + assert (trial / "parser.py").read_bytes() == SOURCE.encode() + assert record["execution_mode"] == engineering.GENERATION_MODE + assert record["inputs_unchanged"] + assert record["verdict"] == ("FAIL" if sandbox_available else "INCONCLUSIVE") + assert record["evaluation_error"] == (None if sandbox_available else + "artifact_sandbox_unavailable") + + +def test_remote_replay_preserves_generation_failure_and_adds_artifact_verdict(): + plan = {"execution_mode": engineering.GENERATION_MODE, + "exam": {"grader": sha256(Path(engineering.__file__))}} + trial = {"task_id": "parser", "execution_mode": engineering.GENERATION_MODE, + "verdict": "INCONCLUSIVE", "evaluation_error": "artifact_sandbox_unavailable", + "checks": None, "output": json.dumps(ANSWER)} + checks = {"passed": True, "assertions": {"fixture": True}} + with mock.patch.object(engineering, "verify", return_value=checks): + compare.verify_retained_grader(plan, [trial], {"parser": TASK}) + assert trial["verdict"] == "INCONCLUSIVE" + assert trial["evaluation_error"] == "artifact_sandbox_unavailable" + assert trial["artifact_verdict"] == "PASS" + + +def test_generation_execute_requires_request_broker_even_without_cli_flag(tmp_path): + _, output, digest = prepared(tmp_path) + with mock.patch.object(agent.AnthropicAgentClient, "require_live_readiness", return_value={}), \ + mock.patch.object(agent, "gate_environment", side_effect=RuntimeError("broker required")): + with pytest.raises(RuntimeError, match="broker required"): + compare.execute(output, digest) + assert not list((output / "results").iterdir()) + + +def test_cloud_capture_to_remote_report_preserves_error_and_shows_comparison(tmp_path): + _, output, digest = prepared(tmp_path) + + def execute(command, cwd, folder, deadline, **kwargs): + folder.mkdir(parents=True) + (folder / "stdout.log").write_text("offline provider fixture") + (folder / "stderr.log").write_text("") + # This trusted fake writes a structured response only, never parser.py. + source = SOURCE + ("# candidate\n" if "New body" in + (cwd / "knowledge/selected/SKILL.md").read_text() else "# baseline\n") + write_json(folder.parent / "answer.json", {**ANSWER, "artifact_source": source}) + return {"status": "completed", "returncode": 0, "elapsed_seconds": 0} + + trace = {"usage": {"input_tokens": 7, "output_tokens": 3}, "commands": [], + "errors": [], "completed_turns": 1, "client_kind": agent.FULL_CONTEXT_KIND, + "provider_model": "claude-opus-5-5", "api_requests": 1, + "usage_complete": True, "observed_usage": {"input_tokens": 7, "output_tokens": 3}} + with mock.patch.object(agent.AnthropicAgentClient, "require_live_readiness", return_value={}), \ + mock.patch.object(agent.AnthropicAgentClient, "environment", return_value={}), \ + mock.patch.object(agent.AnthropicAgentClient, "trace_metrics", return_value=trace), \ + mock.patch.object(harness, "execute_command", side_effect=execute), \ + mock.patch.object(engineering, "verify", side_effect=engineering.SandboxUnavailable()): + initial = compare.execute(output, digest) + assert initial["status"]["recorded_trials"] == 2 + assert initial["pairs"][0]["outcome"] == "inconclusive" + assert initial["pairs"][0]["replayed_artifact_outcome"] == "inconclusive" + + def verify(task, answer): + passed = "# candidate" in answer["artifact_source"] + return {"passed": passed, "assertions": {"stream_error": passed}} + + with mock.patch.object(engineering, "verify", side_effect=verify): + replay = compare.report(output) + pair = replay["pairs"][0] + assert pair["outcome"] == "inconclusive" + assert pair["comparability"] == "generation_only_provider" + assert pair["replayed_artifact_outcome"] == "artifact_improved" + assert pair["baseline"]["artifact_verdict"] == "FAIL" + assert pair["candidate"]["artifact_verdict"] == "PASS" + assert all(pair[role]["evaluation_error"] == "artifact_sandbox_unavailable" + for role in compare.ROLES) + assert "artifact_improved" in (output / "report.md").read_text() + from tools.skill_eval_cloud_export import export + protocol = tmp_path / "protocol.md" + protocol.write_text("Frozen prospective protocol") + with pytest.raises(ValueError): + export(output, protocol) + frozen = compare.load_plan(output) + write_json(output / "historical-execution-approval.json", { + "protocol_sha256": sha256(protocol), "plan_sha256": digest, + **{role: frozen["arms"][role]["git_revision"] for role in compare.ROLES}}) + bundle = export(output, protocol) + assert bundle["schema_version"] == "engineering-cloud-replay-v1" + assert [a["arm"] for a in bundle["artifacts"]] == ["baseline", "candidate"] + assert all(a["sha256"] == hashlib.sha256(a["source"].encode()).hexdigest() + for a in bundle["artifacts"]) + assert bundle["protocol_sha256"] == hashlib.sha256(protocol.read_bytes()).hexdigest() + from tools.skill_eval_cloud_replay import validate_bundle + raw = json.dumps(bundle).encode() + assert validate_bundle(raw, hashlib.sha256(raw).hexdigest()) == bundle + protocol.write_text("Changed after admission") + with pytest.raises(ValueError, match="approval does not bind"): + export(output, protocol) + protocol.write_text("Frozen prospective protocol") + # Retained errors never authorize replay after an integrity failure. + entry = compare.load_plan(output)["schedule"][0] + trial_dir = compare.trial_directory(output, entry) + (trial_dir / "parser.py").write_text("tampered") + with mock.patch.object(engineering, "verify", side_effect=verify): + tampered = compare.report(output) + assert tampered["pairs"][0]["replayed_artifact_outcome"] == "inconclusive" + with pytest.raises(ValueError, match="invalid retained"): + export(output, protocol) diff --git a/tests/test_skill_eval_historical_accounting.py b/tests/test_skill_eval_historical_accounting.py new file mode 100644 index 000000000..05d1924e7 --- /dev/null +++ b/tests/test_skill_eval_historical_accounting.py @@ -0,0 +1,183 @@ +"""Offline controls; no credentials, provider calls or external ledger writes.""" +import json +from types import SimpleNamespace + +import pytest + +from tools import skill_eval_historical_accounting as historical +from tools.skill_eval_budget import BudgetError, WriteResult + + +@pytest.fixture +def case(tmp_path, monkeypatch): + (tmp_path / "results").mkdir() + approval = {"schema_version": 1, "experiment_id": "offline-history", "repository_id": 123, + "issue": 9, "charge_pr": 7, "charge_head": "a" * 40, + "implementation_sha": "b" * 40, "plan_sha256": "c" * 64, + "protocol_sha256": "c" * 64, + "baseline": "d" * 40, "candidate": "e" * 40, "expires_at": 4102444800, + "max_requests": 2, "currency": "USD", "amount": "20.34", + "approval_evidence": "offline owner approval fixture"} + config = {"repository_id": 123, "provider": {"model": {"name": "fixture"}, + "client": {"max_requests": 1}, "pricing": {}, "spend_limit": {}}} + bound = {"requests": 2, "currency": "USD", "amount": "20.34"} + plan = {"execution_mode": "provider-generation-only-v1", "retry_policy": "none", + "schedule": [{"condition": "baseline"}, {"condition": "candidate"}], + "arms": {"baseline": {"git_revision": "d" * 40}, + "candidate": {"git_revision": "e" * 40}}, + "model": config["provider"]["model"], "client": { + "settings": {"max_requests": 1}, "pricing": {}, "spend_limit": {}}, + "reservation": bound} + current = {"number": 7, "base": {"ref": "main", "repo": {"id": 123}}, + "head": {"sha": "a" * 40}, "state": "closed", "merged": True} + def request(method, path): + assert method == "GET" + if path == "": + return {"id": 123} + if path == "/git/ref/heads/main": + return {"ref": "refs/heads/main", "object": {"type": "commit", "sha": "b" * 40}} + assert path == "/pulls/7" + return current + monkeypatch.setattr(historical.compare, "sha256", lambda p: "c" * 64) + monkeypatch.setattr(historical.compare, "load_plan", lambda p: plan) + monkeypatch.setattr(historical.compare, "provider_client", lambda p: SimpleNamespace( + require_live_readiness=lambda count: bound)) + monkeypatch.setattr(historical.auto, "daily_anchor", lambda c: None) + return SimpleNamespace(approval=approval, config=config, plan=plan, current=current, + api=SimpleNamespace(request=request), output=tmp_path) + + +def test_charge_pr_never_impersonates_historical_arms(case): + assert historical.validate(case.api, case.output, case.approval, case.config, + "protocol") == case.plan + assert case.current["head"]["sha"] != case.approval["candidate"] + case.current["head"]["sha"] = case.approval["candidate"] + with pytest.raises(ValueError, match="Accounting PR"): + historical.validate(case.api, case.output, case.approval, case.config, "protocol") + + +@pytest.mark.parametrize("change", ["mode", "retry", "arms", "schedule", "provider", "bound"]) +def test_plan_scope_is_frozen(case, change): + if change == "mode": + case.plan["execution_mode"] = "local" + elif change == "retry": + case.plan["retry_policy"] = "retry" + elif change == "arms": + case.plan["arms"]["candidate"]["git_revision"] = "f" * 40 + elif change == "schedule": + case.plan["schedule"] *= 2 + elif change == "provider": + case.plan["client"]["settings"] = {"max_requests": 2} + else: + case.plan["reservation"]["amount"] = "20.35" + with pytest.raises(ValueError): + historical.validate(case.api, case.output, case.approval, case.config, "protocol") + + +def test_receipt_hash_and_expiry(case, monkeypatch): + path = case.output / "approval.json" + path.write_text(json.dumps(case.approval)) + assert historical.receipt(path, "c" * 64) == case.approval + with pytest.raises(ValueError, match="changed"): + historical.receipt(path, "f" * 64) + case.approval["expires_at"] = 1 + path.write_text(json.dumps(case.approval)) + with pytest.raises(ValueError, match="expired"): + historical.receipt(path, "c" * 64) + + +@pytest.mark.parametrize("failure", ["missing_key", "daily_conflict", "pr_conflict", "retry"]) +def test_no_launch_on_failed_admission(case, monkeypatch, failure): + calls = [] + monkeypatch.setattr(historical.auto, "configuration", lambda p: case.config) + monkeypatch.setattr(historical, "receipt", lambda *a: case.approval) + monkeypatch.setattr(historical, "validate", lambda *a: case.plan) + monkeypatch.setenv("SKILL_EVAL_YIBUAPI_KEY", "offline-fixture-not-a-credential") + if failure == "missing_key": + monkeypatch.delenv("SKILL_EVAL_YIBUAPI_KEY") + def store(name): + return SimpleNamespace(read_verified=lambda k: SimpleNamespace(ledger=name), + compare_and_swap=lambda *a: ( + calls.append(name) or + (WriteResult.CONFLICT if failure == name + "_conflict" + else WriteResult.COMMITTED))) + monkeypatch.setattr(historical.auto, "ledger_store", lambda *a: (store("pr"), "pr")) + monkeypatch.setattr(historical.auto, "daily_store", lambda *a: (store("daily"), "daily")) + monkeypatch.setattr(historical, "reserve", lambda *a: "successor") + monkeypatch.setattr(historical, "daily_reserve", lambda *a: "successor") + with pytest.raises((ValueError, BudgetError)): + historical.controlled_execute(case.api, case.output, config_path="fixture", + approval_path="fixture", approval_sha256="c" * 64, + protocol_path="protocol", run_id="offline", + attempt=2 if failure == "retry" else 1, + launch=lambda *a, **kw: pytest.fail("must not launch")) + assert calls == ({"daily_conflict": ["daily"], "pr_conflict": ["daily", "pr"]} + .get(failure, [])) + + +def test_real_ledger_claims_preserve_prior_day_and_pre_request_receipt(case, monkeypatch): + from contextlib import contextmanager + + from tools import skill_eval_budget as budget + + key, global_key = budget.PRKey(123, 7), budget.DailyKey(123) + old = budget.DailyEntry(budget.PRKey(123, 8), budget.Binding("f" * 64, "old", 471_010_000), + "2026-10-29", phase="claimed") + class Store: + def __init__(self, ledger, daily=False): + self.ledger, self.daily = ledger, daily + + def read_verified(self, key): + assert self.ledger.key == key + cls = budget.DailySnapshot if self.daily else budget.Snapshot + return cls(str(self.ledger.revision), self.ledger) + + def compare_and_swap(self, before, after): + assert before.ledger == self.ledger + validator = budget.validate_daily_successor if self.daily else budget.validate_successor + validator(self.ledger, after) + self.ledger = after + return WriteResult.COMMITTED + local = Store(budget.Ledger(key, budget.Genesis("offline audited zero", 0))) + daily = Store(budget.DailyLedger(global_key, "offline audit", entries=(old,)), True) + monkeypatch.setattr(historical.auto, "configuration", lambda p: case.config) + monkeypatch.setattr(historical, "receipt", lambda *a: case.approval) + monkeypatch.setattr(historical, "validate", lambda *a: case.plan) + monkeypatch.setattr(historical.auto, "ledger_store", lambda *a: (local, key)) + monkeypatch.setattr(historical.auto, "daily_store", lambda *a: (daily, global_key)) + monkeypatch.setattr(historical, "shanghai_day", lambda: "2026-10-30") + monkeypatch.setenv("SKILL_EVAL_YIBUAPI_KEY", "offline-fixture-not-a-credential") + monkeypatch.setenv("GITHUB_TOKEN", "must-not-enter-child") + @contextmanager + def broker(check, max_requests, *, context): + assert max_requests == 2 + assert (case.output / "historical-execution-approval.json").is_file() + check() + yield {"OFFLINE_GATE": "fixture"} + monkeypatch.setattr(historical, "authorization_broker", broker) + launched = [] + def launch(command, **kwargs): + assert "GITHUB_TOKEN" not in kwargs["env"] + assert kwargs["env"]["OFFLINE_GATE"] == "fixture" + launched.append(command) + args = dict(config_path="fixture", approval_path="fixture", approval_sha256="c" * 64, + protocol_path="fixture", run_id="offline", attempt=1, launch=launch) + historical.controlled_execute(case.api, case.output, **args) + assert len(launched) == 1 + assert daily.ledger.entries[0] == old + assert daily.ledger.charged_on("2026-10-30") == 491_350_000 + assert local.ledger.entries[0].phase == "claimed" + assert local.ledger.entries[0].binding.plan_digest == "c" * 64 + with pytest.raises(BudgetError): + historical.controlled_execute(case.api, case.output, **{**args, "run_id": "retry"}) + assert len(launched) == 1 + + # A changed receipt cannot replenish the same logical experiment. + with pytest.raises(BudgetError): + historical.controlled_execute(case.api, case.output, + **{**args, "approval_sha256": "d" * 64}) + # Nor can changing the experiment identity replenish the same approved plan. + case.approval["experiment_id"] = "renamed-offline-history" + with pytest.raises(BudgetError): + historical.controlled_execute(case.api, case.output, **args) + assert len(launched) == 1 diff --git a/tools/skill_eval_cloud_export.py b/tools/skill_eval_cloud_export.py new file mode 100644 index 000000000..52cb67b72 --- /dev/null +++ b/tools/skill_eval_cloud_export.py @@ -0,0 +1,72 @@ +"""Export two retained generation artifacts as inert, bounded replay input.""" +import argparse +import hashlib +import json +from pathlib import Path + +from inferrsi.harness import skill_eval_engineering as engineering +from inferrsi.infra.artifacts import sha256 +from tools import skill_eval_compare as compare + + +def export(output: Path, protocol: Path) -> dict: + plan = compare.load_plan(output) + if (plan.get("execution_mode") != engineering.GENERATION_MODE + or plan.get("adapter") != engineering.ADAPTER): + raise ValueError("Expected frozen generation-only engineering plan") + schedule = plan["schedule"] + if (len(schedule) != 2 or [e["condition"] for e in schedule] != list(compare.ROLES) + or len({(e["task_id"], e["repeat"], e["split"]) for e in schedule}) != 1): + raise ValueError("Expected exactly one ordered baseline/candidate pair") + tasks = compare.read_json(output / "plan/tasks.json") + if len(tasks) != 1 or tasks[0]["request"] != engineering.REQUEST: + raise ValueError("Replay supports the fixed public engineering task only") + approval = json.loads(engineering.read_regular( + output / "historical-execution-approval.json", 65536)) + if (approval.get("protocol_sha256") != sha256(protocol) + or approval.get("plan_sha256") != sha256(output / "plan/freeze.json") + or any(approval.get(role) != plan["arms"][role]["git_revision"] + for role in compare.ROLES)): + raise ValueError("Historical approval does not bind this protocol and frozen pair") + artifacts = [] + for entry in schedule: + trial = compare.retained_trial(output, entry, engineering.ADAPTER, + engineering.GENERATION_MODE) + if (trial.get("missing") or trial.get("execution_mode") != engineering.GENERATION_MODE + or trial.get("evaluation_error") not in (None, "artifact_sandbox_unavailable") + or trial.get("observations", {}).get("usage_complete") is not True): + raise ValueError("Incomplete or invalid retained generation evidence") + data = engineering.read_regular(compare.trial_directory(output, entry) / "parser.py", + engineering.MAX_SOURCE_BYTES) + if not data: + raise ValueError("Empty artifact") + artifacts.append({"arm": entry["condition"], "source": data.decode("utf-8"), + "sha256": hashlib.sha256(data).hexdigest()}) + # This digest binds the full frozen input inventory, not a claim that a remote + # reviewer independently inspected skill inputs or provider execution. + inputs = json.dumps(plan["files"], sort_keys=True, separators=(",", ":")).encode() + return {"schema_version": "engineering-cloud-replay-v1", "task_id": schedule[0]["task_id"], + "request_sha256": hashlib.sha256(engineering.REQUEST.encode()).hexdigest(), + "grader_sha256": sha256(Path(engineering.__file__)), + "inputs_sha256": hashlib.sha256(inputs).hexdigest(), + "protocol_sha256": sha256(protocol), "artifacts": artifacts} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--run", type=Path, required=True) + parser.add_argument("--protocol", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + bundle = export(args.run, args.protocol) + payload = json.dumps(bundle, ensure_ascii=True, sort_keys=True, indent=2).encode() + b"\n" + if len(payload) > 1024 * 1024: + raise ValueError("Replay bundle exceeds byte bound") + # No replacement of an existing evidence file or accidental publication. + with args.output.open("xb") as stream: + stream.write(payload) + print(json.dumps({"bundle_sha256": hashlib.sha256(payload).hexdigest()})) + + +if __name__ == "__main__": + main() diff --git a/tools/skill_eval_cloud_replay.py b/tools/skill_eval_cloud_replay.py new file mode 100644 index 000000000..22c23c0bd --- /dev/null +++ b/tools/skill_eval_cloud_replay.py @@ -0,0 +1,143 @@ +"""Strict two-parser data ingress and isolated replay; never imports submitted code.""" +import argparse +import hashlib +import json +import re +import urllib.request +from pathlib import Path + +from inferrsi.harness import skill_eval_engineering as engineering +from tools.skill_eval_qualify import qualify + +SCHEMA = "engineering-cloud-replay-v1" +BUNDLE_PATH = "artifacts/skill-eval-cloud/bundle.json" +MAX_BUNDLE_BYTES = 1024 * 1024 + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def require_hash(value, length=64): + if type(value) is not str or not re.fullmatch(r"[0-9a-f]{%d}" % length, value): + raise ValueError("Invalid exact digest/revision") + + +def unique_object(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError("Duplicate JSON key") + result[key] = value + return result + + +def validate_bundle(raw, expected_digest): + require_hash(expected_digest) + if len(raw) > MAX_BUNDLE_BYTES or digest(raw) != expected_digest: + raise ValueError("Bundle size or digest mismatch") + try: + bundle = json.loads(raw.decode("utf-8"), object_pairs_hook=unique_object) + except (UnicodeError, RecursionError, json.JSONDecodeError) as error: + raise ValueError("Invalid bounded UTF-8 JSON") from error + keys = {"schema_version", "task_id", "request_sha256", "grader_sha256", + "inputs_sha256", "protocol_sha256", "artifacts"} + if type(bundle) is not dict or set(bundle) != keys or bundle["schema_version"] != SCHEMA: + raise ValueError("Unsupported bundle schema") + if (type(bundle["task_id"]) is not str + or not re.fullmatch(r"[A-Za-z0-9_.-]{1,128}", bundle["task_id"])): + raise ValueError("Invalid task identifier") + for key in ("request_sha256", "grader_sha256", "inputs_sha256", "protocol_sha256"): + require_hash(bundle[key]) + if (bundle["request_sha256"] != digest(engineering.REQUEST.encode("utf-8")) + or bundle["grader_sha256"] != digest(Path(engineering.__file__).read_bytes())): + raise ValueError("Frozen request/grader differs from trusted controller") + artifacts = bundle["artifacts"] + if type(artifacts) is not list or len(artifacts) != 2: + raise ValueError("Exactly two arms required") + for artifact, arm in zip(artifacts, ("baseline", "candidate")): + if (type(artifact) is not dict or set(artifact) != {"arm", "source", "sha256"} + or artifact["arm"] != arm or type(artifact["source"]) is not str): + raise ValueError("Invalid ordered artifact schema") + require_hash(artifact["sha256"]) + try: + source = artifact["source"].encode("utf-8") + except UnicodeError as error: + raise ValueError("Invalid source UTF-8") from error + if (not 0 < len(source) <= engineering.MAX_SOURCE_BYTES + or digest(source) != artifact["sha256"]): + raise ValueError("Artifact size or digest mismatch") + return bundle + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise ValueError("Unexpected ingress redirect") + + +def fetch_bundle(commit): + require_hash(commit, 40) + url = f"https://raw.githubusercontent.com/InferMatrixLab/InferRSI/{commit}/{BUNDLE_PATH}" + opener = urllib.request.build_opener(NoRedirect()) + with opener.open(url, timeout=20) as response: + return response.read(MAX_BUNDLE_BYTES + 1) + + +def replay(raw, expected_digest, revision, image): + bundle = validate_bundle(raw, expected_digest) + receipt = { + "schema_version": SCHEMA, "bundle_sha256": expected_digest, + "provenance": {key: bundle[key] for key in + ("task_id", "request_sha256", "grader_sha256", + "inputs_sha256", "protocol_sha256")}, + "qualification": qualify(revision, image), "verdict": "INCONCLUSIVE", "artifacts": [], + "scope": "Public parser replay delivery only; captures and comparability unchanged", + } + if receipt["qualification"]["verdict"] != "PASS": + return receipt + task = {"id": bundle["task_id"], "split": "development", "request": engineering.REQUEST, + "provenance": "Fixed public reconstructed parser task; retained cloud artifacts"} + try: + for artifact in bundle["artifacts"]: + checked = engineering.verify(task, { + "artifact": "parser.py", "summary": "Retained artifact", + "artifact_source": artifact["source"], + }) + receipt["artifacts"].append({"arm": artifact["arm"], "sha256": artifact["sha256"], + "result": checked}) + except engineering.SandboxUnavailable: + receipt["error"] = "Artifact sandbox unavailable; no fallback" + return receipt + # PASS describes complete replay delivery, NOT that either parser passed or a skill improved. + receipt["verdict"] = "PASS" + return receipt + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--evidence-commit", required=True) + parser.add_argument("--bundle-sha256", required=True) + parser.add_argument("--revision", required=True) + parser.add_argument("--runner-image", required=True) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + require_hash(args.evidence_commit, 40) + require_hash(args.bundle_sha256) + require_hash(args.revision, 40) + args.output.mkdir(parents=True, exist_ok=False) + try: + receipt = replay(fetch_bundle(args.evidence_commit), args.bundle_sha256, + args.revision, args.runner_image) + except Exception as error: + # Never log exception text that may include input/source; preserve a failure category only. + receipt = {"schema_version": SCHEMA, "verdict": "INCONCLUSIVE", + "error_category": type(error).__name__} + receipt["evidence_commit"] = args.evidence_commit + receipt["controller_revision"] = args.revision + (args.output / "replay.json").write_text(json.dumps(receipt, indent=2) + "\n", encoding="utf-8") + print(receipt["verdict"]) + return 0 if receipt["verdict"] == "PASS" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/skill_eval_compare.py b/tools/skill_eval_compare.py index 07b259f24..7c994406d 100644 --- a/tools/skill_eval_compare.py +++ b/tools/skill_eval_compare.py @@ -23,8 +23,13 @@ ) from inferrsi.infra.artifacts import atomic_write_json, sha256, write_json from inferrsi.knowledge_operations import KnowledgeReader, safe_path +from inferrsi.model.anthropic_agent import ( + FULL_CONTEXT_KIND, + AnthropicAgentClient, + validate_schema, + validate_spec, +) from inferrsi.model.anthropic_agent import SOURCE as AGENT_SOURCE -from inferrsi.model.anthropic_agent import AnthropicAgentClient, validate_schema, validate_spec from inferrsi.model.request_guard import SOURCE as GUARD_SOURCE from tools.skill_eval_pilot import PROMPT, SCHEMA, observed_tokens from tools.skill_eval_snapshot import selected_snapshot @@ -178,7 +183,7 @@ def validate_default_audit(task: dict) -> None: def validate(config: dict, tasks: list[dict], *, default_grader: bool = True) -> None: if set(config) - {"mode", "skill", "skill_root", "cases", "schema", "grader", "snapshots", "model", "limits", "client", "pricing", "spend_limit", - "adapter"}: + "adapter", "execution_mode"}: raise ValueError("Unknown configuration field") adapter = config.get("adapter", "audit") if adapter not in ("audit", diagnostic.ADAPTER, engineering.ADAPTER): @@ -188,7 +193,13 @@ def validate(config: dict, tasks: list[dict], *, default_grader: bool = True) -> raise ValueError("Diagnostic comparisons require paired old/new bundles (update mode)") if "grader" in config or "schema" in config: raise ValueError("Selected adapter owns its frozen schema and verifier") - if adapter == engineering.ADAPTER and "client" in config: + generation_only = config.get("execution_mode") == engineering.GENERATION_MODE + if "execution_mode" in config and (not generation_only or adapter != engineering.ADAPTER): + raise ValueError("Unsupported execution mode for selected adapter") + if generation_only and (not isinstance(config.get("client"), dict) + or config["client"].get("kind") != FULL_CONTEXT_KIND): + raise ValueError("Generation-only requires the bounded full-context provider") + if adapter == engineering.ADAPTER and "client" in config and not generation_only: raise ValueError("Engineering artifacts require local execution, not read/submit provider") if config["mode"] not in ("addition", "update"): raise ValueError("mode must be addition or update") @@ -320,7 +331,9 @@ def prepare(config_path: Path, repo: Path, output: Path) -> str: validate(config, tasks, default_grader=( grader.read_bytes() == (ROOT / "tools/skill_eval_verifier.py").read_bytes())) require_git_version(subprocess.check_output(("git", "--version"), cwd=repo, text=True)) - schema = (engineering.SCHEMA if is_engineering else + generation_only = config.get("execution_mode") == engineering.GENERATION_MODE + schema = (engineering.GENERATION_SCHEMA if generation_only else + engineering.SCHEMA if is_engineering else diagnostic.SCHEMA if is_diagnostic else read_json(config_path.parent / config["schema"]) if "schema" in config else SCHEMA) if "client" in config: @@ -333,7 +346,7 @@ def prepare(config_path: Path, repo: Path, output: Path) -> str: shutil.copyfile(grader, output / "validator/verify.py") if is_engineering: (output / "plan/starter").mkdir() - (output / "plan/starter/parser.py").write_text(engineering.STARTER, encoding="utf-8") + (output / "plan/starter/parser.py").write_bytes(engineering.STARTER.encode("utf-8")) arms = {} with tempfile.TemporaryDirectory(prefix="se-") as temporary: for role, short in zip(ROLES, ("b", "c")): @@ -377,7 +390,8 @@ def prepare(config_path: Path, repo: Path, output: Path) -> str: "exam": {"cases": files["plan/tasks.json"], "schema": files["plan/schema.json"], "grader": files["validator/verify.py"]}, "same_bundle": before is not None and before["revision"] == after["revision"], - "prompt": (engineering.PROMPT if is_engineering else + "prompt": (engineering.GENERATION_PROMPT if generation_only else + engineering.PROMPT if is_engineering else diagnostic.PROMPT if is_diagnostic else PROMPT), "files": files, "controllers": {name: sha256(ROOT / name) for name in CONTROLLERS}, "retry_policy": "none", "cost": None, @@ -407,6 +421,14 @@ def prepare(config_path: Path, repo: Path, output: Path) -> str: plan["isolation"] = ("Fresh workspaces; read/submit tools expose selected files only. " "No shell, network tool, external agent CLI or global config. " "Provider worker and verifier use separate filtered environments.") + if generation_only: + plan["execution_mode"] = engineering.GENERATION_MODE + plan["isolation"] = ( + "Generation only: the provider receives frozen task/starter and its assigned bundle. " + "No shell, file-writing or test tools. " + "A trusted controller materializes only parser.py; " + "artifact code runs only in the independent bubblewrap verifier. " + "This is not a workspace-agent or iterative tool-use experiment.") write_json(output / "plan/freeze.json", plan) return sha256(output / "plan/freeze.json") @@ -544,7 +566,8 @@ def execute(output: Path, authorization: str, *, client=None, raise ValueError("PR execution requires the guarded bounded provider") if "client" in plan: configured = provider_client(plan) - if require_request_authorization: + if (require_request_authorization + or plan.get("execution_mode") == engineering.GENERATION_MODE): configured = replace(configured, require_authorization=True) configured.require_live_readiness(len(plan["schedule"])) if client is not None: @@ -552,6 +575,8 @@ def execute(output: Path, authorization: str, *, client=None, configured.environment() # Missing keys fail before start/trial evidence. client = configured stop_on_unknown_usage = True + if plan.get("execution_mode") == engineering.GENERATION_MODE and "client" not in plan: + raise ValueError("Generation-only requires the frozen bounded provider") if plan.get("adapter") == engineering.ADAPTER and client is None: raise ValueError("Engineering execution requires a verified confined local client; " "the default danger-full-access Codex client is not qualified") @@ -600,7 +625,9 @@ def execute(output: Path, authorization: str, *, client=None, directory, output / "plan/schema.json", output / "validator/verify.py", prompt, deadline, timeout_seconds=plan["limits"]["timeout_seconds"], - engineering_artifact=is_engineering) + engineering_artifact=is_engineering, + generation_only=(plan.get("execution_mode") + == engineering.GENERATION_MODE)) result["evidence_sha256"] = {name: sha256(directory / name) for name in EVIDENCE if (directory / name).is_file()} atomic_write_json(directory / "result.json", result) @@ -625,7 +652,7 @@ def execute(output: Path, authorization: str, *, client=None, return report(output) -def retained_trial(output: Path, entry: dict, expected_adapter=None) -> dict: +def retained_trial(output: Path, entry: dict, expected_adapter=None, execution_mode=None) -> dict: directory = trial_directory(output, entry) path = directory / "result.json" if not path.is_file(): @@ -677,10 +704,20 @@ def retained_trial(output: Path, entry: dict, expected_adapter=None) -> dict: "tool_calls": len(record["observations"]["commands"]) if isinstance(record["observations"].get("commands"), list) else None, } - error = record.get("evaluation_error") + record_error = record.get("evaluation_error") + # Artifact integrity failures take precedence over a replayable host error. + error = None if expected_adapter == engineering.ADAPTER else record_error if (expected_adapter == engineering.ADAPTER and record.get("artifact_kind") != engineering.ADAPTER): error = error or "missing_retained_artifact_identity" + if execution_mode == engineering.GENERATION_MODE: + if record.get("execution_mode") != execution_mode: + error = error or "missing_generation_mode_identity" + elif not record["inputs_unchanged"] or record.get("execution_state") != "completed": + error = "invalid_generation_execution" + else: + selected["execution_mode"] = execution_mode + selected["artifact_verdict"] = None if record.get("execution_state") == "author_reported": selected["provenance"] = record.get("provenance") selected["author_reported_execution"] = True @@ -730,6 +767,7 @@ def retained_trial(output: Path, entry: dict, expected_adapter=None) -> dict: error = error or "retained_artifact_disagrees_with_answer" else: evidence["parser.py"] = artifact.relative_to(output).as_posix() + error = error or record_error if (error or record["execution_state"] not in ("completed", "author_reported") or not record["inputs_unchanged"]): selected["verdict"] = "INCONCLUSIVE" @@ -758,7 +796,8 @@ def condition_summary(trials: list[dict]) -> dict: def read_report(output: Path) -> dict: plan = load_plan(output) - trials = [retained_trial(output, entry, plan.get("adapter")) for entry in plan["schedule"]] + trials = [retained_trial(output, entry, plan.get("adapter"), plan.get("execution_mode")) + for entry in plan["schedule"]] # A receipt is an unkeyed hash: a rewritten tree can carry a consistent one. # Re-grading retained answers with trusted code is what makes a PASS/FAIL # verdict independent of the artifact copy the agent could reach. This runs @@ -788,6 +827,14 @@ def read_report(output: Path) -> dict: pairs.append({"comparability": "unverified_author_reported" if manual else "controller_run", "case_id": entry["task_id"], "repeat": entry["repeat"], "outcome": outcome, **arms}) + if plan.get("execution_mode") == engineering.GENERATION_MODE: + for pair in pairs: + replayed = tuple(pair[role].get("artifact_verdict") for role in ROLES) + pair["comparability"] = "generation_only_provider" + pair["replayed_artifact_outcome"] = { + ("FAIL", "PASS"): "artifact_improved", ("PASS", "FAIL"): "artifact_regressed", + ("PASS", "PASS"): "artifacts_both_pass", ("FAIL", "FAIL"): "artifacts_both_fail", + }.get(replayed, "inconclusive") discovery = [t for t in trials if t["split"] == "discovery"] status = {"execution_state": "not_completed"} status_path = output / "results/status.json" @@ -832,6 +879,14 @@ def read_report(output: Path) -> dict: "usage and costs are self-reported, not independently authenticated. Public tests " "may be overfit. Paired artifact differences do not prove skill causality, original " "engine behavior, hardware efficacy or whole-PR acceptance. Reviewer decides merge.") + if plan.get("execution_mode") == engineering.GENERATION_MODE: + result["execution_mode"] = engineering.GENERATION_MODE + result["claim"] = ( + "Generation-only provider trial: no model shell, file-writing or test tools. " + "The trusted controller materializes parser.py, and independent replay checks " + "the retained artifact. This does not measure iterative workspace-agent performance. " + "Public reconstructed CPU tests may be overfit; artifact differences alone do not " + "establish skill causality, hardware efficacy or whole-PR acceptance.") if "client" in plan: result["client"] = plan["client"] result["max_api_requests"] = plan["max_api_requests"] @@ -871,7 +926,13 @@ def verify_retained_grader(plan: dict, trials: list[dict], tasks: dict) -> None: and trial.get("evaluation_error") in ( None, "reported_agent_failure", "artifact_sandbox_unavailable", "reported_run_exceeded_frozen_timeout")) - if trial["verdict"] not in ("PASS", "FAIL") and not manual_artifact: + generated_artifact = ( + trusted_verifier is engineering.verify + and plan.get("execution_mode") == engineering.GENERATION_MODE + and trial.get("execution_mode") == engineering.GENERATION_MODE + and trial.get("evaluation_error") in (None, "artifact_sandbox_unavailable")) + replay_artifact = manual_artifact or generated_artifact + if trial["verdict"] not in ("PASS", "FAIL") and not replay_artifact: continue try: checks = trusted_verifier(tasks[trial["task_id"]], json.loads(trial["output"])) @@ -881,13 +942,13 @@ def verify_retained_grader(plan: dict, trials: list[dict], tasks: dict) -> None: trial["evaluation_error"] = "artifact_sandbox_unavailable" continue replay_after_blocked_capture = ( - manual_artifact and trial["checks"] is None + replay_artifact and trial["checks"] is None and trial["evaluation_error"] == "artifact_sandbox_unavailable") if not replay_after_blocked_capture and ( checks != trial["checks"] or (trial["verdict"] in ("PASS", "FAIL") and checks["passed"] != (trial["verdict"] == "PASS"))): raise ValueError("Untrusted grading disagrees with the trusted grader") - if manual_artifact: + if replay_artifact: # Report can recheck a portable artifact on a capable reviewer host, # without rewriting capture failure or upgrading pair comparability. trial["artifact_checks"] = checks @@ -968,7 +1029,7 @@ def markdown(result: dict) -> str: outcomes = [pair[role].get("artifact_verdict") or pair[role]["verdict"] or "missing" for role in ROLES] summary.append(f"| {pair['case_id']} | {outcomes[0]} | {outcomes[1]} | " - f"{pair['outcome']} |") + f"{pair.get('replayed_artifact_outcome', pair['outcome'])} |") summary += ["", "## Failed checks and coverage", "", "Public reconstructed response-parser checks only; no original server, GPU, " "retrieval quality or whole-corpus coverage."] diff --git a/tools/skill_eval_historical_accounting.py b/tools/skill_eval_historical_accounting.py new file mode 100644 index 000000000..0ab917907 --- /dev/null +++ b/tools/skill_eval_historical_accounting.py @@ -0,0 +1,177 @@ +"""Explicit owner-approved historical experiment admission, not a PR-label bypass. + +The trusted operator supplies an independently approved receipt hash, never a value +from a PR, model output or dispatch payload. The charge PR identifies accounting +only; the frozen arms identify the compared skill revisions. No genesis creation, +refund, retry, or normal PR-controller policy changes are performed here. +""" + +import json +import os +import re +import subprocess +import sys +import time +from decimal import Decimal +from pathlib import Path + +from inferrsi.model.request_guard import authorization_broker +from tools import skill_eval_auto as auto +from tools import skill_eval_compare as compare +from tools import skill_eval_pr as pr +from tools.skill_eval_budget import ( + Binding, + BudgetError, + WriteResult, + claim_once, + daily_claim_once, + daily_reserve, + reserve, + shanghai_day, +) + + +def receipt(path, expected_sha256): + """Hash must come from the owner's trusted execution control plane.""" + if (not re.fullmatch(r"[0-9a-f]{64}", expected_sha256) + or compare.sha256(Path(path)) != expected_sha256): + raise ValueError("Historical approval receipt changed") + value = json.loads(Path(path).read_text(encoding="utf-8")) + fields = {"schema_version", "experiment_id", "repository_id", "issue", "charge_pr", + "charge_head", "implementation_sha", "plan_sha256", "protocol_sha256", + "baseline", "candidate", + "expires_at", "max_requests", "currency", "amount", "approval_evidence"} + if (not isinstance(value, dict) or set(value) != fields or value["schema_version"] != 1 + or not isinstance(value["experiment_id"], str) + or not re.fullmatch(r"[A-Za-z0-9._-]{1,100}", value["experiment_id"]) + or any(type(value[k]) is not int or value[k] <= 0 + for k in ("repository_id", "issue", "charge_pr", "expires_at")) + or value["expires_at"] <= time.time() + or type(value["max_requests"]) is not int or value["max_requests"] != 2 + or value["currency"] != "USD" or value["amount"] != "20.34" + or not isinstance(value["approval_evidence"], str) + or not value["approval_evidence"].strip() + or any(not isinstance(value[k], str) or not re.fullmatch(r"[0-9a-f]{40}", value[k]) + for k in ("charge_head", "implementation_sha", "baseline", "candidate")) + or not isinstance(value["plan_sha256"], str) + or not re.fullmatch(r"[0-9a-f]{64}", value["plan_sha256"]) + or not isinstance(value["protocol_sha256"], str) + or not re.fullmatch(r"[0-9a-f]{64}", value["protocol_sha256"])): + raise ValueError("Incomplete, expired or unbounded historical approval") + return value + + +def validate(api, output, approval, config, protocol_path): + """Reconcile exact frozen experiment, provider, main and accounting identity.""" + if compare.sha256(Path(protocol_path)) != approval["protocol_sha256"]: + raise ValueError("Prospective protocol changed") + if approval["repository_id"] != config["repository_id"]: + raise ValueError("Historical repository mismatch") + if compare.sha256(output / "plan/freeze.json") != approval["plan_sha256"]: + raise ValueError("Historical plan differs from owner-approved freeze") + plan = compare.load_plan(output) + if (plan.get("execution_mode") != "provider-generation-only-v1" + or plan.get("retry_policy") != "none" or len(plan["schedule"]) != 2 + or {entry["condition"] for entry in plan["schedule"]} != {"baseline", "candidate"} + or plan["arms"]["baseline"]["git_revision"] != approval["baseline"] + or plan["arms"]["candidate"]["git_revision"] != approval["candidate"]): + raise ValueError("Historical experiment scope changed") + provider = config["provider"] + if (plan["model"] != provider["model"] + or plan["client"]["settings"] != provider["client"] + or plan["client"]["pricing"] != provider["pricing"] + or plan["client"]["spend_limit"] != provider["spend_limit"] + or provider["client"]["max_requests"] != 1): + raise ValueError("Historical provider differs from trusted configuration") + bound = compare.provider_client(plan).require_live_readiness(2) + if (bound != plan.get("reservation") or bound["requests"] != 2 + or bound["currency"] != "USD" or Decimal(bound["amount"]) > Decimal("20.34")): + raise ValueError("Historical request reservation changed or exceeds approval") + auto.daily_anchor(config) + if api.request("GET", "")["id"] != approval["repository_id"]: + raise ValueError("Live repository identity changed") + main = api.request("GET", "/git/ref/heads/main") + if (main.get("ref") != "refs/heads/main" or main.get("object", {}).get("type") != "commit" + or main["object"]["sha"] != approval["implementation_sha"]): + raise ValueError("Historical execution requires the approved current main") + current = api.request("GET", f"/pulls/{approval['charge_pr']}") + if (current["number"] != approval["charge_pr"] + or current["base"].get("ref") != "main" + or current["base"].get("repo", {}).get("id") != approval["repository_id"] + or current["head"]["sha"] != approval["charge_head"] + or current.get("draft", False) + or not (current["state"] == "open" or current.get("merged") is True)): + raise ValueError("Accounting PR changed or unavailable; historical arms are separate") + return plan + + +def controlled_execute(api, output, *, config_path, approval_path, approval_sha256, + protocol_path, run_id, attempt, launch=None): + """Trusted orchestration only. No paid operation is reachable without two ledgers.""" + output = Path(output) + if attempt != 1: + raise ValueError("Historical execution retries are forbidden") + config = auto.configuration(config_path) + approval = receipt(approval_path, approval_sha256) + validate(api, output, approval, config, protocol_path) + if any((output / "results").iterdir()): + raise ValueError("Historical execution requires fresh results") + if not os.environ.get("SKILL_EVAL_YIBUAPI_KEY"): + raise ValueError("Provider credential unavailable; no reservation created") + store, key = auto.ledger_store(api, config, approval["charge_pr"]) + global_store, global_key = auto.daily_store(api, config) + # The digest binds every approved byte, including the exact frozen plan. The + # stable logical run prevents a replacement receipt/physical run replenishing + # this experiment; both plan and run identities are unique in each ledger. + binding = Binding(approval_sha256, + f"historical:{approval['experiment_id']}:attempt-1", 20_340_000) + snapshot = store.read_verified(key) + global_snapshot = global_store.read_verified(global_key) + admitted_day = shanghai_day() + successor = daily_reserve(global_snapshot.ledger, key, binding, admitted_day) + if (successor == global_snapshot.ledger or global_store.compare_and_swap( + global_snapshot, successor) is not WriteResult.COMMITTED): + raise BudgetError("Duplicate, conflicted or uncertain global reservation") + successor = reserve(snapshot.ledger, binding) + if (successor == snapshot.ledger + or store.compare_and_swap(snapshot, successor) is not WriteResult.COMMITTED): + raise BudgetError("Duplicate, conflicted or uncertain PR reservation") + daily_claim_once(global_store, key, binding, shanghai_day()) + claim_once(store, key, binding) + + def check(): + fresh = auto.configuration(config_path) + if fresh != config or receipt(approval_path, approval_sha256) != approval: + raise ValueError("Historical authorization changed") + validate(api, output, approval, fresh, protocol_path) + ledger = store.read_verified(key).ledger + daily = global_store.read_verified(global_key).ledger + daily.require_active() + if (not any(e.binding == binding and e.phase == "claimed" for e in ledger.entries) + or not any(e.key == key and e.binding == binding and e.phase == "claimed" + and e.admitted_day == admitted_day for e in daily.entries)): + raise BudgetError("Historical durable claims missing") + if shanghai_day() != admitted_day: + raise BudgetError("Shanghai admission day ended") + + check() + # Pre-request provenance only; never copy private approval text or credentials. + retained = {k: approval[k] for k in ("experiment_id", "repository_id", "issue", + "charge_pr", "charge_head", "baseline", "candidate", "plan_sha256", + "protocol_sha256", "implementation_sha", "max_requests", "currency", "amount")} + retained.update(approval_sha256=approval_sha256, recorded_at=time.time(), + admitted_day=admitted_day, binding_plan_digest=binding.plan_digest, + binding_run_id=binding.run_id, physical_run_id=run_id) + with (output / "historical-execution-approval.json").open("x", encoding="utf-8") as handle: + json.dump(retained, handle, sort_keys=True, indent=2) + with authorization_broker(check, 2, context=f"{approval_sha256}:{binding.run_id}") as gate: + environment = {k: os.environ[k] for k in ("PATH", "LANG", "SYSTEMROOT", "TEMP", "TMP", + "SKILL_EVAL_YIBUAPI_KEY") if k in os.environ} + environment.update(gate) + environment["PYTHONPATH"] = str(pr.ROOT / "src") + command = [sys.executable, "-c", "from pathlib import Path; import sys; " + "from tools.skill_eval_compare import execute; " + "execute(Path(sys.argv[1]),sys.argv[2],require_request_authorization=True)", + str(output), approval["plan_sha256"]] + (launch or subprocess.run)(command, cwd=pr.ROOT, env=environment, check=True) + # No automatic refunds or retries, including partial/uncertain execution. From f77f73539d4a5dc781ab1a956d60c49565d676bc Mon Sep 17 00:00:00 2001 From: Dong1017 <55723162+Dong1017@users.noreply.github.com> Date: Fri, 30 Oct 2026 16:04:01 +0900 Subject: [PATCH 2/2] fix(skilleval): never execute artifacts during provider generation Keep both capture and capture-report phases inert on every host. Preserve explicit deferred verification and independently replay only in the verifier phase. Signed-off-by: Codex Agent-assisted-by: Codex design_cloud_trial253 (implementation) --- docs/skill-eval-generation-only.md | 5 ++-- src/inferrsi/harness/skill_evaluation.py | 19 ++++++++++------ tests/test_skill_eval_generation.py | 22 +++++++++++------- tools/skill_eval_cloud_export.py | 3 ++- tools/skill_eval_compare.py | 29 ++++++++++++++++-------- 5 files changed, 51 insertions(+), 27 deletions(-) diff --git a/docs/skill-eval-generation-only.md b/docs/skill-eval-generation-only.md index ab21026eb..67ec6d09e 100644 --- a/docs/skill-eval-generation-only.md +++ b/docs/skill-eval-generation-only.md @@ -34,8 +34,9 @@ per-request reservations, PR lifetime/day accounting and the precise experiment identity must be independently authorized and frozen. A new accounting PR must not be represented as the historical skill revisions being compared. -When the controller cannot start bubblewrap, the original trial remains -`INCONCLUSIVE` with `artifact_sandbox_unavailable`. A separately qualified Linux +Generation never invokes the artifact verifier, even on a capable host. The +original trial remains `INCONCLUSIVE` with `artifact_verification_deferred`. Its +capture report is also non-executing. A separately qualified Linux reviewer may replay the retained artifact after frozen-input/source integrity checks. The report preserves that original result and separately exposes `artifact_verdict`, the 19 artifact checks and `replayed_artifact_outcome`. diff --git a/src/inferrsi/harness/skill_evaluation.py b/src/inferrsi/harness/skill_evaluation.py index 78dcdd788..27ce1ed3e 100644 --- a/src/inferrsi/harness/skill_evaluation.py +++ b/src/inferrsi/harness/skill_evaluation.py @@ -273,14 +273,19 @@ def run_trial( verification_dir.mkdir() (verification_dir / "stdout.log").write_text("", encoding="utf-8") (verification_dir / "stderr.log").write_text("", encoding="utf-8") - try: - checks = engineering.verify(task, output) - except engineering.SandboxUnavailable: - evaluation_error = "artifact_sandbox_unavailable" + if generation_only: + # The generation process may carry provider credentials. Even + # a capable host must not execute generated source in this phase. + evaluation_error = "artifact_verification_deferred" else: - atomic_write_json(verification_dir / "result.json", checks) - verification = {"status": "completed" if checks["passed"] else "failed", - "returncode": 0 if checks["passed"] else 1} + try: + checks = engineering.verify(task, output) + except engineering.SandboxUnavailable: + evaluation_error = "artifact_sandbox_unavailable" + else: + atomic_write_json(verification_dir / "result.json", checks) + verification = {"status": "completed" if checks["passed"] else "failed", + "returncode": 0 if checks["passed"] else 1} if evaluation_error is None and not engineering_artifact: verifier_environment = getattr(client, "verifier_environment", None) options = {"environment": verifier_environment()} if verifier_environment else {} diff --git a/tests/test_skill_eval_generation.py b/tests/test_skill_eval_generation.py index 688a3d88d..ea4148fa1 100644 --- a/tests/test_skill_eval_generation.py +++ b/tests/test_skill_eval_generation.py @@ -152,27 +152,31 @@ def execute(command, cwd, folder, deadline): record = harness.run_trial(OfflineClient(), TASK, "baseline", 1, root, trial, tmp_path / "schema", tmp_path / "verifier", "prompt", float("inf"), engineering_artifact=True, generation_only=True) - assert verify.call_args.args[1]["artifact_source"] == SOURCE + verify.assert_not_called() assert (trial / "parser.py").read_bytes() == SOURCE.encode() assert record["execution_mode"] == engineering.GENERATION_MODE assert record["inputs_unchanged"] - assert record["verdict"] == ("FAIL" if sandbox_available else "INCONCLUSIVE") - assert record["evaluation_error"] == (None if sandbox_available else - "artifact_sandbox_unavailable") + assert record["verdict"] == "INCONCLUSIVE" + assert record["evaluation_error"] == "artifact_verification_deferred" def test_remote_replay_preserves_generation_failure_and_adds_artifact_verdict(): plan = {"execution_mode": engineering.GENERATION_MODE, "exam": {"grader": sha256(Path(engineering.__file__))}} trial = {"task_id": "parser", "execution_mode": engineering.GENERATION_MODE, - "verdict": "INCONCLUSIVE", "evaluation_error": "artifact_sandbox_unavailable", + "verdict": "INCONCLUSIVE", "evaluation_error": "artifact_verification_deferred", "checks": None, "output": json.dumps(ANSWER)} checks = {"passed": True, "assertions": {"fixture": True}} with mock.patch.object(engineering, "verify", return_value=checks): compare.verify_retained_grader(plan, [trial], {"parser": TASK}) assert trial["verdict"] == "INCONCLUSIVE" - assert trial["evaluation_error"] == "artifact_sandbox_unavailable" + assert trial["evaluation_error"] == "artifact_verification_deferred" assert trial["artifact_verdict"] == "PASS" + with mock.patch.object(engineering, "verify", side_effect=engineering.SandboxUnavailable()): + compare.verify_retained_grader(plan, [trial], {"parser": TASK}) + assert trial["evaluation_error"] == "artifact_verification_deferred" + assert trial["artifact_replay_error"] == "artifact_sandbox_unavailable" + assert trial["artifact_verdict"] is None def test_generation_execute_requires_request_broker_even_without_cli_flag(tmp_path): @@ -205,8 +209,10 @@ def execute(command, cwd, folder, deadline, **kwargs): mock.patch.object(agent.AnthropicAgentClient, "environment", return_value={}), \ mock.patch.object(agent.AnthropicAgentClient, "trace_metrics", return_value=trace), \ mock.patch.object(harness, "execute_command", side_effect=execute), \ - mock.patch.object(engineering, "verify", side_effect=engineering.SandboxUnavailable()): + mock.patch.object(engineering, "verify", + side_effect=AssertionError("must not execute")) as verifier: initial = compare.execute(output, digest) + verifier.assert_not_called() assert initial["status"]["recorded_trials"] == 2 assert initial["pairs"][0]["outcome"] == "inconclusive" assert initial["pairs"][0]["replayed_artifact_outcome"] == "inconclusive" @@ -223,7 +229,7 @@ def verify(task, answer): assert pair["replayed_artifact_outcome"] == "artifact_improved" assert pair["baseline"]["artifact_verdict"] == "FAIL" assert pair["candidate"]["artifact_verdict"] == "PASS" - assert all(pair[role]["evaluation_error"] == "artifact_sandbox_unavailable" + assert all(pair[role]["evaluation_error"] == "artifact_verification_deferred" for role in compare.ROLES) assert "artifact_improved" in (output / "report.md").read_text() from tools.skill_eval_cloud_export import export diff --git a/tools/skill_eval_cloud_export.py b/tools/skill_eval_cloud_export.py index 52cb67b72..8e67e9c34 100644 --- a/tools/skill_eval_cloud_export.py +++ b/tools/skill_eval_cloud_export.py @@ -33,7 +33,8 @@ def export(output: Path, protocol: Path) -> dict: trial = compare.retained_trial(output, entry, engineering.ADAPTER, engineering.GENERATION_MODE) if (trial.get("missing") or trial.get("execution_mode") != engineering.GENERATION_MODE - or trial.get("evaluation_error") not in (None, "artifact_sandbox_unavailable") + or trial.get("evaluation_error") not in (None, "artifact_sandbox_unavailable", + "artifact_verification_deferred") or trial.get("observations", {}).get("usage_complete") is not True): raise ValueError("Incomplete or invalid retained generation evidence") data = engineering.read_regular(compare.trial_directory(output, entry) / "parser.py", diff --git a/tools/skill_eval_compare.py b/tools/skill_eval_compare.py index 7c994406d..7fc1f2ba5 100644 --- a/tools/skill_eval_compare.py +++ b/tools/skill_eval_compare.py @@ -649,7 +649,8 @@ def execute(output: Path, authorization: str, *, client=None, atomic_write_json(output / "results/status.json", status) atomic_write_json(output / "results/status.receipt.json", {"result_sha256": sha256(output / "results/status.json")}) - return report(output) + return report(output, replay_artifacts=( + plan.get("execution_mode") != engineering.GENERATION_MODE)) def retained_trial(output: Path, entry: dict, expected_adapter=None, execution_mode=None) -> dict: @@ -794,7 +795,7 @@ def condition_summary(trials: list[dict]) -> dict: return groups -def read_report(output: Path) -> dict: +def read_report(output: Path, *, replay_artifacts: bool = True) -> dict: plan = load_plan(output) trials = [retained_trial(output, entry, plan.get("adapter"), plan.get("execution_mode")) for entry in plan["schedule"]] @@ -802,8 +803,9 @@ def read_report(output: Path) -> dict: # Re-grading retained answers with trusted code is what makes a PASS/FAIL # verdict independent of the artifact copy the agent could reach. This runs # before pairs are formed so an unverifiable grade cannot become an outcome. - verify_retained_grader(plan, [t for t in trials if not t["missing"]], - {task["id"]: task for task in read_json(output / "plan/tasks.json")}) + if replay_artifacts: + verify_retained_grader(plan, [t for t in trials if not t["missing"]], + {task["id"]: task for task in read_json(output / "plan/tasks.json")}) index = {(t["task_id"], t["repeat"], t["condition"]): t for t in trials} pairs = [] for entry in plan["schedule"]: @@ -930,8 +932,13 @@ def verify_retained_grader(plan: dict, trials: list[dict], tasks: dict) -> None: trusted_verifier is engineering.verify and plan.get("execution_mode") == engineering.GENERATION_MODE and trial.get("execution_mode") == engineering.GENERATION_MODE - and trial.get("evaluation_error") in (None, "artifact_sandbox_unavailable")) + and trial.get("evaluation_error") in (None, "artifact_sandbox_unavailable", + "artifact_verification_deferred")) replay_artifact = manual_artifact or generated_artifact + if replay_artifact: + trial["artifact_verdict"] = None + trial.pop("artifact_checks", None) + trial.pop("artifact_replay_error", None) if trial["verdict"] not in ("PASS", "FAIL") and not replay_artifact: continue try: @@ -939,11 +946,15 @@ def verify_retained_grader(plan: dict, trials: list[dict], tasks: dict) -> None: except engineering.SandboxUnavailable: trial["verdict"] = "INCONCLUSIVE" trial["grader_trust"] = "unverified" - trial["evaluation_error"] = "artifact_sandbox_unavailable" + if generated_artifact: + trial["artifact_replay_error"] = "artifact_sandbox_unavailable" + else: + trial["evaluation_error"] = "artifact_sandbox_unavailable" continue replay_after_blocked_capture = ( replay_artifact and trial["checks"] is None - and trial["evaluation_error"] == "artifact_sandbox_unavailable") + and trial["evaluation_error"] in ("artifact_sandbox_unavailable", + "artifact_verification_deferred")) if not replay_after_blocked_capture and ( checks != trial["checks"] or (trial["verdict"] in ("PASS", "FAIL") and checks["passed"] != (trial["verdict"] == "PASS"))): @@ -956,8 +967,8 @@ def verify_retained_grader(plan: dict, trials: list[dict], tasks: dict) -> None: trial["grader_trust"] = "reverified" -def report(output: Path) -> dict: - result = read_report(output) +def report(output: Path, *, replay_artifacts: bool = True) -> dict: + result = read_report(output, replay_artifacts=replay_artifacts) write_json(output / "report.json", result) (output / "report.md").write_text(markdown(result), encoding="utf-8") return result