TypedPython: cache generated C for the pypackpack compile slot (#113)

docs/SPEC.md

@@ -784,7 +784,16 @@ All in `PM/commonTest`, so they run wherever the interpreter loads.
   the compile step. `Status: partial`: the per-module cache and its key are implemented in Python
   (`incremental.py`, `pipeline.compile_project`; `test_incremental.py`, `test_pipeline_incremental.py`);
   the Gradle task with declared inputs and outputs does not exist yet, because compiled modules are built
-  through pypackpack's compile slot (#25, #113).
+  through pypackpack's compile slot (#25).
+  For that slot, `pipeline.generate_project` caches source to C instead of source to extension (#113):
+  one `.c` per compiled module plus the directory of the header-only `tp_runtime.h`; its key is the
+  source, the dependency interface hashes and the compiler version (which covers the runtime header),
+  with no platform, C compiler or flag in it, because the C depends on none of them. The slot caches
+  C to extension itself. Tests (`test_generate_project.py`): an unchanged module is reused with a
+  byte-identical `.c`, a fresh cache produces the same bytes, a body change regenerates that module
+  only, changing toolchain flags or platform reuses every entry, the C compiler is never run, and the
+  emitted `.c` built by `cbuild` behaves as `compile_project`'s extension. `compile_project` keeps
+  building in process.
 - **N-11** TypedPython fixed-layout classes (M2b, 2026-11-21; design §4.3.3). A class compiles to fixed
   field access only when its layout is fixed **in CPython too**: it declares `__slots__` listing exactly
   its annotated instance fields, has no base but `object`, no metaclass, no decorator but `@compiled`,

docs/design/typedpython.md

@@ -162,6 +162,12 @@ API 안정성을 확인한 뒤로 미룬다.
 인터페이스 해시가 그대로라 의존 모듈은 다시 컴파일하지 않는다. C 오브젝트는 모듈별로 캐시한다.
 Gradle 태스크는 입력·출력을 정확히 선언해 up-to-date 판정과 빌드 캐시가 함께 작동한다.
 
+pypackpack 의 native compile slot 이 C 를 확장으로 빌드하므로 캐시는 두 층으로 나뉜다(#113).
+`pipeline.generate_project` 는 소스 → C 만 캐시한다: 모듈마다 `.c` 하나, 그리고 `tp_runtime.h` 가 있는
+디렉터리. 키는 소스 + 의존 모듈 인터페이스 해시 + 컴파일러 버전이고, C 는 플랫폼·컴파일러·플래그를 읽지
+않으므로 그것들은 키에 넣지 않는다. 바뀌지 않은 모듈은 바이트가 같은 `.c` 로 재사용되어 slot 의 C → 확장
+캐시도 맞는다. 테스트가 쓰는 `compile_project`(프로세스 안에서 빌드)는 그대로 둔다.
+
 ### 4.3.3 고정 레이아웃 클래스 (SPEC N-11, M2b)
 
 **레이아웃이 CPython 에서도 고정일 때만** 고정 접근으로 컴파일한다, 사용자가 `__slots__` 를 직접 쓴 클래스.

python-multiplatform-ksp/src/main/python/typedpython/pipeline.py

@@ -154,27 +154,24 @@ class ModuleResult:
     interface: str = ""
     kind: str = "compiled"                  # "compiled" | "nothing_compiled" | "not_marked"
     skipped_items: list[SkippedItem] = field(default_factory=list)   # `skipped`, structured
+    c_source: Path | None = None            # generate_project only: the module's one `.c` (#113)
 
 
 @dataclass
 class ProjectResult:
     modules: list[ModuleResult]
+    runtime_include: Path | None = None     # generate_project only: the dir holding tp_runtime.h
 
     @property
     def rebuilt(self) -> list[ModuleResult]:
         return [m for m in self.modules if m.status == "rebuilt"]
 
 
-def compile_project(
-    sources: list[Path],
-    project_root: Path,
-    cache_dir: Path,
-    *,
-    python_include: Path | None = None,
-    ext_suffix: str | None = None,
-    runtime_dir: Path | None = None,
-    cost_rule: bool = True,
-) -> ProjectResult:
+def _project(sources: list[Path], project_root: Path, cache_dir: Path, *, cost_rule: bool,
+             compile_c, compiler_version: str, platform: str, flags: list[str],
+             c_mode: bool) -> list[ModuleResult]:
+    """The cached source -> artefact walk shared by compile_project (artefact: the extension) and
+    generate_project (artefact: the `.c`)."""
     sources = [Path(s).resolve() for s in sources]
     cache_dir = Path(cache_dir)
     verified: dict[Path, list[SkippedItem]] = {}
@@ -193,27 +190,12 @@ def lower(path: Path) -> ir.Module:
     def generate(module: ir.Module, path: Path) -> str:
         return cgen.generate(module, path, _relative(path, root)) if module.functions else ""
 
-    def compile_c(c_source: str, name: str, out_dir: Path) -> Path:
-        if not c_source:
-            marker = Path(out_dir) / (name.rpartition(".")[2] + _NO_EXTENSION)
-            marker.write_text("nothing compiled\n")
-            return marker
-        return cbuild.build(c_source, name, out_dir, runtime_dir=runtime_dir, flags=cbuild.DEFAULT_FLAGS,
-                            python_include=python_include, ext_suffix=ext_suffix)
-
-    version = COMPILER_VERSION
-    if runtime_dir is not None:
-        version += ":" + _tree_hash(Path(runtime_dir))
     if not cost_rule:
-        version += ":nocost"
-    suffix = ext_suffix if ext_suffix is not None else sysconfig.get_config_var("EXT_SUFFIX")
-    include = cbuild._python_include(python_include)
-    flags = [*cbuild.DEFAULT_FLAGS, f"-I{include}"]
-
+        compiler_version += ":nocost"
     cache = incremental.BuildCache(cache_dir)
     report = cache.build_project(
         sources, project_root, lower=lower, generate=generate, compile_c=compile_c,
-        compiler_version=version, platform=f"{sysconfig.get_platform()} {suffix}", flags=flags)
+        compiler_version=compiler_version, platform=platform, flags=flags)
 
     side = cache_dir / "skipped"
     side.mkdir(exist_ok=True)
@@ -230,9 +212,78 @@ def compile_c(c_source: str, name: str, out_dir: Path) -> Path:
             items = [SkippedItem(**i) for i in data["items"]]
         except (OSError, ValueError, KeyError, TypeError):
             skipped, items = {}, []
-        ext = None if m.artifact is None or m.artifact.suffix == _NO_EXTENSION else m.artifact
+        art = None if m.artifact is None or m.artifact.suffix == _NO_EXTENSION else m.artifact
+        ext, c_file = (None, art) if c_mode else (art, None)
         kind = ("not_marked" if not frontend.is_opted_in(m.path)
-                else "compiled" if ext is not None else "nothing_compiled")
+                else "compiled" if art is not None else "nothing_compiled")
         out.append(ModuleResult(m.path, m.name, ext, m.status, list(m.reasons), skipped, m.interface,
-                                kind, items))
-    return ProjectResult(out)
+                                kind, items, c_file))
+    return out
+
+
+def _no_c_marker(name: str, out_dir: Path) -> Path:
+    marker = Path(out_dir) / (name.rpartition(".")[2] + _NO_EXTENSION)
+    marker.write_text("nothing compiled\n")
+    return marker
+
+
+def compile_project(
+    sources: list[Path],
+    project_root: Path,
+    cache_dir: Path,
+    *,
+    python_include: Path | None = None,
+    ext_suffix: str | None = None,
+    runtime_dir: Path | None = None,
+    cost_rule: bool = True,
+) -> ProjectResult:
+    def compile_c(c_source: str, name: str, out_dir: Path) -> Path:
+        if not c_source:
+            return _no_c_marker(name, out_dir)
+        return cbuild.build(c_source, name, out_dir, runtime_dir=runtime_dir, flags=cbuild.DEFAULT_FLAGS,
+                            python_include=python_include, ext_suffix=ext_suffix)
+
+    version = COMPILER_VERSION
+    if runtime_dir is not None:
+        version += ":" + _tree_hash(Path(runtime_dir))
+    suffix = ext_suffix if ext_suffix is not None else sysconfig.get_config_var("EXT_SUFFIX")
+    include = cbuild._python_include(python_include)
+    flags = [*cbuild.DEFAULT_FLAGS, f"-I{include}"]
+    return ProjectResult(_project(
+        sources, project_root, cache_dir, cost_rule=cost_rule, compile_c=compile_c,
+        compiler_version=version, platform=f"{sysconfig.get_platform()} {suffix}", flags=flags,
+        c_mode=False))
+
+
+# The platform component of a generated-C cache key. The C does not depend on the host, the target,
+# the C compiler or its flags (cgen reads none of them; the target's headers and the toolchain are
+# the compile slot's inputs and its own cache key, pypackpack docs/design/compile-slot.md §9), so
+# one constant stands for all of them and a `.c` built on one machine is reused on another.
+GENERATED_C_PLATFORM = "generated-c"
+
+
+def generate_project(
+    sources: list[Path],
+    project_root: Path,
+    cache_dir: Path,
+    *,
+    cost_rule: bool = True,
+) -> ProjectResult:
+    """Source -> C for every module, cached (#113): the input of pypackpack's native compile slot,
+    which builds the extensions and caches C -> extension itself. Each compiled module gets exactly
+    one `.c` (`ModuleResult.c_source`, named after the module's last dotted part); a module with
+    nothing compiled gets none and ships as Python. `runtime_include` is the directory holding the
+    header-only `tp_runtime.h`, for the slot's include dirs. The key is the source, the dependency
+    interfaces and COMPILER_VERSION (which covers the runtime header); no toolchain flag enters it.
+    An unchanged module is reused with a byte-identical `.c`."""
+    def write_c(c_source: str, name: str, out_dir: Path) -> Path:
+        if not c_source:
+            return _no_c_marker(name, out_dir)
+        c_file = Path(out_dir) / (name.rpartition(".")[2] + ".c")
+        c_file.write_text(c_source)
+        return c_file
+
+    return ProjectResult(_project(
+        sources, project_root, cache_dir, cost_rule=cost_rule, compile_c=write_c,
+        compiler_version=COMPILER_VERSION, platform=GENERATED_C_PLATFORM, flags=[], c_mode=True),
+        runtime_include=cbuild.RUNTIME_DIR)

python-multiplatform-ksp/src/test/python/test_generate_project.py

@@ -0,0 +1,111 @@
+"""generate_project: TypedPython caches source -> C, the compile slot builds C -> extension (#113).
+
+Real Pyrefly; the only C build here is the one that proves the emitted `.c` is the same program
+compile_project builds.
+"""
+import textwrap
+from pathlib import Path
+
+import pytest
+
+from conftest import decorate_compiled
+from typedpython import cbuild, frontend, pipeline
+
+B = decorate_compiled(textwrap.dedent("""\
+    def twice(n: int) -> int:
+        return n * 2
+    """))
+A = decorate_compiled(textwrap.dedent("""\
+    import b
+
+
+    def inc(n: int) -> int:
+        return n + 1
+    """))
+PLAIN = "def hello() -> str:\n    return 'hi'\n"
+
+
+@pytest.fixture
+def project(tmp_path):
+    root = tmp_path / "proj"
+    (root / "pkg").mkdir(parents=True)
+    (root / "a.py").write_text(A)
+    (root / "b.py").write_text(B)
+    (root / "pkg" / "__init__.py").write_text("")
+    (root / "pkg" / "plain.py").write_text(PLAIN)
+    return root
+
+
+@pytest.fixture
+def counts(monkeypatch):
+    n = {"lower": 0, "build": 0}
+    real_lower = frontend.lower
+
+    def lower(*a, **k):
+        n["lower"] += 1
+        return real_lower(*a, **k)
+
+    def build(*a, **k):
+        n["build"] += 1
+        raise AssertionError("generate_project must not run the C compiler")
+
+    monkeypatch.setattr(frontend, "lower", lower)
+    monkeypatch.setattr(cbuild, "build", build)
+    return n
+
+
+def gen(project, cache):
+    return pipeline.generate_project(
+        [project / "a.py", project / "b.py", project / "pkg" / "plain.py"], project, cache)
+
+
+def by_stem(result):
+    return {m.path.stem: m for m in result.modules}
+
+
+def test_emits_one_c_per_compiled_module_and_the_runtime_dir(project, tmp_path, counts):
+    r = gen(project, tmp_path / "cache")
+    m = by_stem(r)
+    assert m["a"].c_source.name == "a.c" and m["b"].c_source.name == "b.c"
+    assert "PyInit_a" in m["a"].c_source.read_text()
+    assert m["a"].extension is None and m["a"].kind == "compiled"
+    assert m["plain"].c_source is None and m["plain"].kind == "not_marked"
+    assert (r.runtime_include / "tp_runtime.h").is_file()
+    assert counts == {"lower": 3, "build": 0}
+
+
+def test_unchanged_module_hits_with_byte_identical_c(project, tmp_path, counts):
+    first = by_stem(gen(project, tmp_path / "cache"))
+    c_a = first["a"].c_source.read_bytes()
+    second = gen(project, tmp_path / "cache")
+    assert {m.path.stem: m.status for m in second.modules} == {"a": "reused", "b": "reused", "plain": "reused"}
+    assert by_stem(second)["a"].c_source.read_bytes() == c_a
+    assert counts["lower"] == 3                      # nothing lowered again
+
+
+def test_fresh_cache_gives_byte_identical_c(project, tmp_path):
+    one = by_stem(gen(project, tmp_path / "c1"))["a"].c_source.read_bytes()
+    two = by_stem(gen(project, tmp_path / "c2"))["a"].c_source.read_bytes()
+    assert one == two
+
+
+def test_body_change_regenerates_only_that_module(project, tmp_path):
+    gen(project, tmp_path / "cache")
+    (project / "b.py").write_text(B.replace("n * 2", "n + n"))
+    st = {m.path.stem: m.status for m in gen(project, tmp_path / "cache").modules}
+    assert st == {"a": "reused", "b": "rebuilt", "plain": "reused"}
+
+
+def test_key_ignores_toolchain(project, tmp_path, monkeypatch):
+    gen(project, tmp_path / "cache")
+    monkeypatch.setattr(cbuild, "DEFAULT_FLAGS", (*cbuild.DEFAULT_FLAGS, "-DDIFFERENT"))
+    monkeypatch.setattr(pipeline.sysconfig, "get_platform", lambda: "other-platform")
+    assert all(m.status == "reused" for m in gen(project, tmp_path / "cache").modules)
+
+
+def test_emitted_c_is_what_compile_project_builds(project, tmp_path):
+    c_mod = by_stem(gen(project, tmp_path / "ccache"))["b"]
+    built = by_stem(pipeline.compile_project([project / "b.py"], project, tmp_path / "bcache"))["b"]
+    so = cbuild.build(c_mod.c_source.read_text(), "b", tmp_path / "slot")
+    assert cbuild.load("b", so).twice(21) == 42
+    assert built.extension is not None and built.c_source is None