From c9b1608d0cf59f53d459fccb99acb3292abdc536 Mon Sep 17 00:00:00 2001 From: Dong1017 <55723162+Dong1017@users.noreply.github.com> Date: Thu, 5 Nov 2026 22:08:14 +0900 Subject: [PATCH 1/2] Support shared projection manifests in trusted planning Extract the backward-compatible raw-Git reader and tests before any projection format migration. Keep existing policy and workflow execution boundaries unchanged. Agent-assisted-by: dot (projection planner) / unknown [effort=unknown] (author) Agent-assisted-by: dot (index implementation) / unknown [effort=unknown] (author) Signed-off-by: Codex --- tests/test_skill_eval_planned_guidance.py | 118 ++++++++++++++++++++++ tools/skill_eval_pr.py | 77 ++++++++++---- 2 files changed, 177 insertions(+), 18 deletions(-) diff --git a/tests/test_skill_eval_planned_guidance.py b/tests/test_skill_eval_planned_guidance.py index 72529290c..1239b4480 100644 --- a/tests/test_skill_eval_planned_guidance.py +++ b/tests/test_skill_eval_planned_guidance.py @@ -1,6 +1,7 @@ """Nonpaid planner guidance: raw Git/data inspection, never candidate/model execution.""" import copy +import hashlib import json import subprocess from unittest.mock import Mock @@ -290,3 +291,120 @@ def test_unrelated_source_ref_presence_cannot_change_full_history_plan(tmp_path) assert plans[0]["projection_check"]["status"] == "unavailable" assert "source_revision" not in plans[0]["projection_check"] assert "automatic command selection" in plans[0]["projection_check"]["reason"] + + + +def manifest_fixture(tmp_path): + repo, readme, canonical, pin, identity, changes = index_fixture(tmp_path) + manifest = readme.with_name("_provenance.json") + data = {"schema_version": 1, "repository": "https://github.com/example/fixture", + "source_revision": pin, + "registry": {"path": pr.LIBRARY_PATH + "/skill-index.json", + "sha256": hashlib.sha256(canonical.read_bytes()).hexdigest()}} + manifest.write_text(json.dumps(data)) + return repo, readme, canonical, pin, identity, changes, manifest, data + + +def commit_projection_fixture(repo, identity): + git(repo, "add", "-A") + git(repo, "commit", "--quiet", "-m", "Synthetic manifest projection") + identity["head"] = git(repo, "rev-parse", "HEAD") + + +def test_manifest_projection_pin_is_authoritative_and_reads_raw_git(tmp_path): + repo, readme, _, pin, identity, changes, manifest, _ = manifest_fixture(tmp_path) + readme.write_text("README no longer carries a source pin.\n") + commit_projection_fixture(repo, identity) + manifest.write_text("Uncommitted hostile metadata must not affect planning") + assert pr.projection_check(repo, identity, changes) == { + "status": "ready", "source_revision": pin, + "declared_in": pr.PROJECTION_PATH + "/_provenance.json"} + + +@pytest.mark.parametrize("mutation", [ + "malformed", "missing_revision", "short_revision", "unknown_commit", "nonancestor", + "wrong_repository", "extra_field", "wrong_version", "boolean_version", "null", "list", + "missing_registry", "registry_null", "registry_extra", "wrong_path", "wrong_digest", + "invalid_digest", "duplicate_field", "duplicate_registry_field", "symlink", "mode", + "oversized", "directory", "non_utf8", "changed_registry", "changed_library", +]) +def test_invalid_manifest_never_falls_back_to_valid_legacy_readme(tmp_path, mutation): + repo, _, canonical, pin, identity, changes, manifest, data = manifest_fixture(tmp_path) + if mutation == "missing_revision": + del data["source_revision"] + elif mutation == "short_revision": + data["source_revision"] = pin[:8] + elif mutation == "unknown_commit": + data["source_revision"] = "f" * 40 + elif mutation == "nonancestor": + tree = git(repo, "rev-parse", "HEAD^{tree}") + data["source_revision"] = git(repo, "commit-tree", tree, "-m", "Synthetic orphan") + elif mutation == "wrong_repository": + data["repository"] = "https://github.com/example/other" + elif mutation == "extra_field": + data["extra"] = "unsupported" + elif mutation in {"wrong_version", "boolean_version"}: + data["schema_version"] = True if mutation == "boolean_version" else 2 + elif mutation == "null": + data = None + elif mutation == "list": + data = [] + elif mutation == "missing_registry": + del data["registry"] + elif mutation == "registry_null": + data["registry"] = None + elif mutation == "registry_extra": + data["registry"]["extra"] = "unsupported" + elif mutation == "wrong_path": + data["registry"]["path"] = pr.LIBRARY_PATH + "/../skill-index.json" + elif mutation == "wrong_digest": + data["registry"]["sha256"] = "0" * 64 + elif mutation == "invalid_digest": + data["registry"]["sha256"] = 123 + elif mutation == "changed_registry": + canonical.write_text("Changed source bytes\n") + elif mutation == "changed_library": + canonical.with_name("resource.md").write_text("New source asset\n") + manifest.write_text(json.dumps(data)) + if mutation == "malformed": + manifest.write_text("{bad json") + elif mutation == "duplicate_field": + manifest.write_text('{"schema_version": 1, ' + manifest.read_text()[1:]) + elif mutation == "duplicate_registry_field": + manifest.write_text(manifest.read_text().replace('"registry": {', + '"registry": {"path": "bad", ')) + elif mutation == "symlink": + manifest.unlink() + manifest.symlink_to(canonical) + elif mutation == "mode": + manifest.chmod(0o755) + elif mutation == "oversized": + manifest.write_text(manifest.read_text() + " " * 65536) + elif mutation == "directory": + manifest.unlink() + manifest.mkdir() + (manifest / "child").write_text("Not a manifest") + elif mutation == "non_utf8": + manifest.write_bytes(json.dumps(data).encode("utf-16")) + commit_projection_fixture(repo, identity) + check = pr.projection_check(repo, identity, changes) + assert check["status"] == "unavailable" and "source_revision" not in check + + +@pytest.mark.parametrize("mode", ["symlink", "executable"]) +def test_manifest_rejects_unsafe_registry_at_both_unchanged_revisions(tmp_path, mode): + repo, _, canonical, _, identity, changes, manifest, data = manifest_fixture(tmp_path) + if mode == "symlink": + canonical.unlink() + canonical.symlink_to("README.md") + canonical.with_name("README.md").write_text("Synthetic registry target\n") + raw = b"README.md" + else: + canonical.chmod(0o755) + raw = canonical.read_bytes() + commit_projection_fixture(repo, identity) + data["source_revision"] = identity["head"] + data["registry"]["sha256"] = hashlib.sha256(raw).hexdigest() + manifest.write_text(json.dumps(data)) + commit_projection_fixture(repo, identity) + assert pr.projection_check(repo, identity, changes)["status"] == "unavailable" diff --git a/tools/skill_eval_pr.py b/tools/skill_eval_pr.py index 5ebe4ef62..7ae84c544 100644 --- a/tools/skill_eval_pr.py +++ b/tools/skill_eval_pr.py @@ -74,46 +74,87 @@ def task_context(tasks): if isinstance(task.get(key), str)} for task in tasks if isinstance(task, dict)] +def projection_blob(repo, revision, path, *, max_bytes=65536): + """Read one exact regular blob, never a candidate checkout or symlink.""" + entry = git(repo, "ls-tree", revision, "--", path).decode().split() + if len(entry) != 4 or entry[:2] != ["100644", "blob"] or entry[3] != path: + raise ValueError("Invalid projection input mode or path") + if int(git(repo, "cat-file", "-s", entry[2])) > max_bytes: + raise ValueError("Oversized projection input") + return git(repo, "cat-file", "blob", entry[2]) + + +def projection_object(pairs): + value = {} + for key, item in pairs: + if key in value: + raise ValueError("Duplicate projection field") + value[key] = item + return value + + def projection_check(repo, identity, changes): """Identify an applicable existing check from raw Git data, without running it.""" if not any(matches_path(c["path"], (LIBRARY_PATH, PROJECTION_PATH)) for c in changes): return None unavailable = {"status": "unavailable", "reason": "No verified ancestor source pin for automatic command selection"} + manifest = PROJECTION_PATH + "/_provenance.json" readme = PROJECTION_PATH + "/README.md" try: - # Do not follow a symlink or read executable/oversized PR metadata. - entry = git(repo, "ls-tree", identity["head"], "--", readme).decode().split() - if len(entry) != 4 or entry[:2] != ["100644", "blob"]: - return unavailable - if int(git(repo, "cat-file", "-s", entry[2])) > 65536: - return unavailable - text = git(repo, "cat-file", "blob", entry[2]).decode("utf-8") - pattern = (r"^Source snapshot: \[(" + SHA + r")\]\(https://github\.com/" - + re.escape(identity["repository"]) + r"/tree/\1/" - + re.escape(LIBRARY_PATH) + r"\)\.$") - pins = re.findall(pattern, text, flags=re.M) - if len(pins) != 1: - return unavailable + # Presence is authoritative: invalid manifests must not fall back to README. + if blob_exists(repo, identity["head"], manifest): + metadata = json.loads(projection_blob(repo, identity["head"], manifest).decode("utf-8"), + object_pairs_hook=projection_object) + if (not isinstance(metadata, dict) + or set(metadata) != {"schema_version", "repository", "source_revision", + "registry"} + or type(metadata["schema_version"]) is not int + or metadata["schema_version"] != 1 + or metadata["repository"] != "https://github.com/" + identity["repository"] + or not isinstance(metadata["source_revision"], str) + or not re.fullmatch(SHA, metadata["source_revision"])): + return unavailable + registry = metadata["registry"] + if (not isinstance(registry, dict) or set(registry) != {"path", "sha256"} + or registry["path"] != LIBRARY_PATH + "/skill-index.json" + or not isinstance(registry["sha256"], str) + or not re.fullmatch(r"[0-9a-f]{64}", registry["sha256"])): + return unavailable + pin, declared_in = metadata["source_revision"], manifest + else: + text = projection_blob(repo, identity["head"], readme).decode("utf-8") + pattern = (r"^Source snapshot: \[(" + SHA + r")\]\(https://github\.com/" + + re.escape(identity["repository"]) + r"/tree/\1/" + + re.escape(LIBRARY_PATH) + r"\)\.$") + pins = re.findall(pattern, text, flags=re.M) + if len(pins) != 1: + return unavailable + pin, declared_in = pins[0], readme # Only the frozen head's full ancestor closure is stable across jobs. # An unrelated locally cached object must not change the plan fingerprint. ancestor = subprocess.run( - [*compare.GIT_READ, "merge-base", "--is-ancestor", pins[0], identity["head"]], + [*compare.GIT_READ, "merge-base", "--is-ancestor", pin, identity["head"]], cwd=repo, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False) if ancestor.returncode != 0: return unavailable - if git(repo, "cat-file", "-t", pins[0]).strip() != b"commit": + if git(repo, "cat-file", "-t", pin).strip() != b"commit": return unavailable library = git(repo, "ls-tree", identity["head"], "--", LIBRARY_PATH).decode().split() if len(library) != 4 or library[:2] != ["040000", "tree"]: return unavailable changed = git(repo, "diff", "--no-ext-diff", "--no-textconv", "--no-renames", - "--name-only", pins[0], identity["head"], "--", LIBRARY_PATH) + "--name-only", pin, identity["head"], "--", LIBRARY_PATH) if changed.strip(): return {"status": "unavailable", "reason": "Declared projection source differs from current canonical library bytes"} - return {"status": "ready", "source_revision": pins[0], "declared_in": readme} - except (subprocess.CalledProcessError, UnicodeError, ValueError): + if declared_in == manifest: + for revision in (pin, identity["head"]): + raw = projection_blob(repo, revision, registry["path"], max_bytes=16 * 1024 * 1024) + if hashlib.sha256(raw).hexdigest() != registry["sha256"]: + return unavailable + return {"status": "ready", "source_revision": pin, "declared_in": declared_in} + except (subprocess.CalledProcessError, UnicodeError, ValueError, RecursionError): return unavailable From 69ce133f1396ede0ddab7b9e2834709b406f17ad Mon Sep 17 00:00:00 2001 From: Dong1017 <55723162+Dong1017@users.noreply.github.com> Date: Mon, 9 Nov 2026 14:47:18 +0800 Subject: [PATCH 2/2] Hash committed registry blobs in manifest fixtures Derive fixture checksums from the stored Git blob instead of working-tree bytes, which checkout conversion such as core.autocrlf can change. Stage executable modes explicitly, since chmod is ignored where core.fileMode is false, and assert the committed mode so each negative case fails for its intended reason. Agent-assisted-by: Claude Code / claude-opus-5-5 [effort=xhigh] (author) Signed-off-by: Dong1017 <55723162+Dong1017@users.noreply.github.com> --- tests/test_skill_eval_planned_guidance.py | 36 ++++++++++++++++++----- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/tests/test_skill_eval_planned_guidance.py b/tests/test_skill_eval_planned_guidance.py index 1239b4480..a58712be3 100644 --- a/tests/test_skill_eval_planned_guidance.py +++ b/tests/test_skill_eval_planned_guidance.py @@ -294,19 +294,32 @@ def test_unrelated_source_ref_presence_cannot_change_full_history_plan(tmp_path) +def committed_blob(repo, revision, path): + # Raw stored bytes: checkout conversion such as core.autocrlf changes only the working tree. + return subprocess.check_output(["git", "cat-file", "blob", f"{revision}:{path}"], cwd=repo) + + +def tree_mode(repo, revision, path): + return git(repo, "ls-tree", revision, "--", path).split()[0] + + def manifest_fixture(tmp_path): repo, readme, canonical, pin, identity, changes = index_fixture(tmp_path) manifest = readme.with_name("_provenance.json") + registry = pr.LIBRARY_PATH + "/skill-index.json" data = {"schema_version": 1, "repository": "https://github.com/example/fixture", "source_revision": pin, - "registry": {"path": pr.LIBRARY_PATH + "/skill-index.json", - "sha256": hashlib.sha256(canonical.read_bytes()).hexdigest()}} + "registry": {"path": registry, + "sha256": hashlib.sha256(committed_blob(repo, pin, registry)).hexdigest()}} manifest.write_text(json.dumps(data)) return repo, readme, canonical, pin, identity, changes, manifest, data -def commit_projection_fixture(repo, identity): +def commit_projection_fixture(repo, identity, executable=()): git(repo, "add", "-A") + # Stage the mode explicitly: chmod alone is ignored where core.fileMode is false. + for path in executable: + git(repo, "update-index", "--chmod=+x", "--", path) git(repo, "commit", "--quiet", "-m", "Synthetic manifest projection") identity["head"] = git(repo, "rev-parse", "HEAD") @@ -386,7 +399,10 @@ def test_invalid_manifest_never_falls_back_to_valid_legacy_readme(tmp_path, muta (manifest / "child").write_text("Not a manifest") elif mutation == "non_utf8": manifest.write_bytes(json.dumps(data).encode("utf-16")) - commit_projection_fixture(repo, identity) + executable = [pr.PROJECTION_PATH + "/_provenance.json"] if mutation == "mode" else [] + commit_projection_fixture(repo, identity, executable) + for path in executable: + assert tree_mode(repo, "HEAD", path) == "100755" check = pr.projection_check(repo, identity, changes) assert check["status"] == "unavailable" and "source_revision" not in check @@ -394,17 +410,21 @@ def test_invalid_manifest_never_falls_back_to_valid_legacy_readme(tmp_path, muta @pytest.mark.parametrize("mode", ["symlink", "executable"]) def test_manifest_rejects_unsafe_registry_at_both_unchanged_revisions(tmp_path, mode): repo, _, canonical, _, identity, changes, manifest, data = manifest_fixture(tmp_path) + registry = pr.LIBRARY_PATH + "/skill-index.json" if mode == "symlink": canonical.unlink() canonical.symlink_to("README.md") canonical.with_name("README.md").write_text("Synthetic registry target\n") - raw = b"README.md" + commit_projection_fixture(repo, identity) + assert tree_mode(repo, "HEAD", registry) == "120000" else: canonical.chmod(0o755) - raw = canonical.read_bytes() - commit_projection_fixture(repo, identity) + commit_projection_fixture(repo, identity, [registry]) + assert tree_mode(repo, "HEAD", registry) == "100755" + # A matching checksum isolates the unsafe mode as the only reason to reject. data["source_revision"] = identity["head"] - data["registry"]["sha256"] = hashlib.sha256(raw).hexdigest() + data["registry"]["sha256"] = hashlib.sha256( + committed_blob(repo, identity["head"], registry)).hexdigest() manifest.write_text(json.dumps(data)) commit_projection_fixture(repo, identity) assert pr.projection_check(repo, identity, changes)["status"] == "unavailable"