diff --git a/tests/test_skill_eval_planned_guidance.py b/tests/test_skill_eval_planned_guidance.py index 72529290..a58712be 100644 --- a/tests/test_skill_eval_planned_guidance.py +++ b/tests/test_skill_eval_planned_guidance.py @@ -1,6 +1,7 @@ """Nonpaid planner guidance: raw Git/data inspection, never candidate/model execution.""" import copy +import hashlib import json import subprocess from unittest.mock import Mock @@ -290,3 +291,140 @@ def test_unrelated_source_ref_presence_cannot_change_full_history_plan(tmp_path) assert plans[0]["projection_check"]["status"] == "unavailable" assert "source_revision" not in plans[0]["projection_check"] assert "automatic command selection" in plans[0]["projection_check"]["reason"] + + + +def committed_blob(repo, revision, path): + # Raw stored bytes: checkout conversion such as core.autocrlf changes only the working tree. + return subprocess.check_output(["git", "cat-file", "blob", f"{revision}:{path}"], cwd=repo) + + +def tree_mode(repo, revision, path): + return git(repo, "ls-tree", revision, "--", path).split()[0] + + +def manifest_fixture(tmp_path): + repo, readme, canonical, pin, identity, changes = index_fixture(tmp_path) + manifest = readme.with_name("_provenance.json") + registry = pr.LIBRARY_PATH + "/skill-index.json" + data = {"schema_version": 1, "repository": "https://github.com/example/fixture", + "source_revision": pin, + "registry": {"path": registry, + "sha256": hashlib.sha256(committed_blob(repo, pin, registry)).hexdigest()}} + manifest.write_text(json.dumps(data)) + return repo, readme, canonical, pin, identity, changes, manifest, data + + +def commit_projection_fixture(repo, identity, executable=()): + git(repo, "add", "-A") + # Stage the mode explicitly: chmod alone is ignored where core.fileMode is false. + for path in executable: + git(repo, "update-index", "--chmod=+x", "--", path) + git(repo, "commit", "--quiet", "-m", "Synthetic manifest projection") + identity["head"] = git(repo, "rev-parse", "HEAD") + + +def test_manifest_projection_pin_is_authoritative_and_reads_raw_git(tmp_path): + repo, readme, _, pin, identity, changes, manifest, _ = manifest_fixture(tmp_path) + readme.write_text("README no longer carries a source pin.\n") + commit_projection_fixture(repo, identity) + manifest.write_text("Uncommitted hostile metadata must not affect planning") + assert pr.projection_check(repo, identity, changes) == { + "status": "ready", "source_revision": pin, + "declared_in": pr.PROJECTION_PATH + "/_provenance.json"} + + +@pytest.mark.parametrize("mutation", [ + "malformed", "missing_revision", "short_revision", "unknown_commit", "nonancestor", + "wrong_repository", "extra_field", "wrong_version", "boolean_version", "null", "list", + "missing_registry", "registry_null", "registry_extra", "wrong_path", "wrong_digest", + "invalid_digest", "duplicate_field", "duplicate_registry_field", "symlink", "mode", + "oversized", "directory", "non_utf8", "changed_registry", "changed_library", +]) +def test_invalid_manifest_never_falls_back_to_valid_legacy_readme(tmp_path, mutation): + repo, _, canonical, pin, identity, changes, manifest, data = manifest_fixture(tmp_path) + if mutation == "missing_revision": + del data["source_revision"] + elif mutation == "short_revision": + data["source_revision"] = pin[:8] + elif mutation == "unknown_commit": + data["source_revision"] = "f" * 40 + elif mutation == "nonancestor": + tree = git(repo, "rev-parse", "HEAD^{tree}") + data["source_revision"] = git(repo, "commit-tree", tree, "-m", "Synthetic orphan") + elif mutation == "wrong_repository": + data["repository"] = "https://github.com/example/other" + elif mutation == "extra_field": + data["extra"] = "unsupported" + elif mutation in {"wrong_version", "boolean_version"}: + data["schema_version"] = True if mutation == "boolean_version" else 2 + elif mutation == "null": + data = None + elif mutation == "list": + data = [] + elif mutation == "missing_registry": + del data["registry"] + elif mutation == "registry_null": + data["registry"] = None + elif mutation == "registry_extra": + data["registry"]["extra"] = "unsupported" + elif mutation == "wrong_path": + data["registry"]["path"] = pr.LIBRARY_PATH + "/../skill-index.json" + elif mutation == "wrong_digest": + data["registry"]["sha256"] = "0" * 64 + elif mutation == "invalid_digest": + data["registry"]["sha256"] = 123 + elif mutation == "changed_registry": + canonical.write_text("Changed source bytes\n") + elif mutation == "changed_library": + canonical.with_name("resource.md").write_text("New source asset\n") + manifest.write_text(json.dumps(data)) + if mutation == "malformed": + manifest.write_text("{bad json") + elif mutation == "duplicate_field": + manifest.write_text('{"schema_version": 1, ' + manifest.read_text()[1:]) + elif mutation == "duplicate_registry_field": + manifest.write_text(manifest.read_text().replace('"registry": {', + '"registry": {"path": "bad", ')) + elif mutation == "symlink": + manifest.unlink() + manifest.symlink_to(canonical) + elif mutation == "mode": + manifest.chmod(0o755) + elif mutation == "oversized": + manifest.write_text(manifest.read_text() + " " * 65536) + elif mutation == "directory": + manifest.unlink() + manifest.mkdir() + (manifest / "child").write_text("Not a manifest") + elif mutation == "non_utf8": + manifest.write_bytes(json.dumps(data).encode("utf-16")) + executable = [pr.PROJECTION_PATH + "/_provenance.json"] if mutation == "mode" else [] + commit_projection_fixture(repo, identity, executable) + for path in executable: + assert tree_mode(repo, "HEAD", path) == "100755" + check = pr.projection_check(repo, identity, changes) + assert check["status"] == "unavailable" and "source_revision" not in check + + +@pytest.mark.parametrize("mode", ["symlink", "executable"]) +def test_manifest_rejects_unsafe_registry_at_both_unchanged_revisions(tmp_path, mode): + repo, _, canonical, _, identity, changes, manifest, data = manifest_fixture(tmp_path) + registry = pr.LIBRARY_PATH + "/skill-index.json" + if mode == "symlink": + canonical.unlink() + canonical.symlink_to("README.md") + canonical.with_name("README.md").write_text("Synthetic registry target\n") + commit_projection_fixture(repo, identity) + assert tree_mode(repo, "HEAD", registry) == "120000" + else: + canonical.chmod(0o755) + commit_projection_fixture(repo, identity, [registry]) + assert tree_mode(repo, "HEAD", registry) == "100755" + # A matching checksum isolates the unsafe mode as the only reason to reject. + data["source_revision"] = identity["head"] + data["registry"]["sha256"] = hashlib.sha256( + committed_blob(repo, identity["head"], registry)).hexdigest() + manifest.write_text(json.dumps(data)) + commit_projection_fixture(repo, identity) + assert pr.projection_check(repo, identity, changes)["status"] == "unavailable" diff --git a/tools/skill_eval_pr.py b/tools/skill_eval_pr.py index 5ebe4ef6..7ae84c54 100644 --- a/tools/skill_eval_pr.py +++ b/tools/skill_eval_pr.py @@ -74,46 +74,87 @@ def task_context(tasks): if isinstance(task.get(key), str)} for task in tasks if isinstance(task, dict)] +def projection_blob(repo, revision, path, *, max_bytes=65536): + """Read one exact regular blob, never a candidate checkout or symlink.""" + entry = git(repo, "ls-tree", revision, "--", path).decode().split() + if len(entry) != 4 or entry[:2] != ["100644", "blob"] or entry[3] != path: + raise ValueError("Invalid projection input mode or path") + if int(git(repo, "cat-file", "-s", entry[2])) > max_bytes: + raise ValueError("Oversized projection input") + return git(repo, "cat-file", "blob", entry[2]) + + +def projection_object(pairs): + value = {} + for key, item in pairs: + if key in value: + raise ValueError("Duplicate projection field") + value[key] = item + return value + + def projection_check(repo, identity, changes): """Identify an applicable existing check from raw Git data, without running it.""" if not any(matches_path(c["path"], (LIBRARY_PATH, PROJECTION_PATH)) for c in changes): return None unavailable = {"status": "unavailable", "reason": "No verified ancestor source pin for automatic command selection"} + manifest = PROJECTION_PATH + "/_provenance.json" readme = PROJECTION_PATH + "/README.md" try: - # Do not follow a symlink or read executable/oversized PR metadata. - entry = git(repo, "ls-tree", identity["head"], "--", readme).decode().split() - if len(entry) != 4 or entry[:2] != ["100644", "blob"]: - return unavailable - if int(git(repo, "cat-file", "-s", entry[2])) > 65536: - return unavailable - text = git(repo, "cat-file", "blob", entry[2]).decode("utf-8") - pattern = (r"^Source snapshot: \[(" + SHA + r")\]\(https://github\.com/" - + re.escape(identity["repository"]) + r"/tree/\1/" - + re.escape(LIBRARY_PATH) + r"\)\.$") - pins = re.findall(pattern, text, flags=re.M) - if len(pins) != 1: - return unavailable + # Presence is authoritative: invalid manifests must not fall back to README. + if blob_exists(repo, identity["head"], manifest): + metadata = json.loads(projection_blob(repo, identity["head"], manifest).decode("utf-8"), + object_pairs_hook=projection_object) + if (not isinstance(metadata, dict) + or set(metadata) != {"schema_version", "repository", "source_revision", + "registry"} + or type(metadata["schema_version"]) is not int + or metadata["schema_version"] != 1 + or metadata["repository"] != "https://github.com/" + identity["repository"] + or not isinstance(metadata["source_revision"], str) + or not re.fullmatch(SHA, metadata["source_revision"])): + return unavailable + registry = metadata["registry"] + if (not isinstance(registry, dict) or set(registry) != {"path", "sha256"} + or registry["path"] != LIBRARY_PATH + "/skill-index.json" + or not isinstance(registry["sha256"], str) + or not re.fullmatch(r"[0-9a-f]{64}", registry["sha256"])): + return unavailable + pin, declared_in = metadata["source_revision"], manifest + else: + text = projection_blob(repo, identity["head"], readme).decode("utf-8") + pattern = (r"^Source snapshot: \[(" + SHA + r")\]\(https://github\.com/" + + re.escape(identity["repository"]) + r"/tree/\1/" + + re.escape(LIBRARY_PATH) + r"\)\.$") + pins = re.findall(pattern, text, flags=re.M) + if len(pins) != 1: + return unavailable + pin, declared_in = pins[0], readme # Only the frozen head's full ancestor closure is stable across jobs. # An unrelated locally cached object must not change the plan fingerprint. ancestor = subprocess.run( - [*compare.GIT_READ, "merge-base", "--is-ancestor", pins[0], identity["head"]], + [*compare.GIT_READ, "merge-base", "--is-ancestor", pin, identity["head"]], cwd=repo, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False) if ancestor.returncode != 0: return unavailable - if git(repo, "cat-file", "-t", pins[0]).strip() != b"commit": + if git(repo, "cat-file", "-t", pin).strip() != b"commit": return unavailable library = git(repo, "ls-tree", identity["head"], "--", LIBRARY_PATH).decode().split() if len(library) != 4 or library[:2] != ["040000", "tree"]: return unavailable changed = git(repo, "diff", "--no-ext-diff", "--no-textconv", "--no-renames", - "--name-only", pins[0], identity["head"], "--", LIBRARY_PATH) + "--name-only", pin, identity["head"], "--", LIBRARY_PATH) if changed.strip(): return {"status": "unavailable", "reason": "Declared projection source differs from current canonical library bytes"} - return {"status": "ready", "source_revision": pins[0], "declared_in": readme} - except (subprocess.CalledProcessError, UnicodeError, ValueError): + if declared_in == manifest: + for revision in (pin, identity["head"]): + raw = projection_blob(repo, revision, registry["path"], max_bytes=16 * 1024 * 1024) + if hashlib.sha256(raw).hexdigest() != registry["sha256"]: + return unavailable + return {"status": "ready", "source_revision": pin, "declared_in": declared_in} + except (subprocess.CalledProcessError, UnicodeError, ValueError, RecursionError): return unavailable