diff --git a/docs/SPEC.md b/docs/SPEC.md index 4d982f37..2e65b6f4 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -784,7 +784,16 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. the compile step. `Status: partial`: the per-module cache and its key are implemented in Python (`incremental.py`, `pipeline.compile_project`; `test_incremental.py`, `test_pipeline_incremental.py`); the Gradle task with declared inputs and outputs does not exist yet, because compiled modules are built - through pypackpack's compile slot (#25, #113). + through pypackpack's compile slot (#25). + For that slot, `pipeline.generate_project` caches source to C instead of source to extension (#113): + one `.c` per compiled module plus the directory of the header-only `tp_runtime.h`; its key is the + source, the dependency interface hashes and the compiler version (which covers the runtime header), + with no platform, C compiler or flag in it, because the C depends on none of them. The slot caches + C to extension itself. Tests (`test_generate_project.py`): an unchanged module is reused with a + byte-identical `.c`, a fresh cache produces the same bytes, a body change regenerates that module + only, changing toolchain flags or platform reuses every entry, the C compiler is never run, and the + emitted `.c` built by `cbuild` behaves as `compile_project`'s extension. `compile_project` keeps + building in process. - **N-11** TypedPython fixed-layout classes (M2b, 2026-11-21; design §4.3.3). A class compiles to fixed field access only when its layout is fixed **in CPython too**: it declares `__slots__` listing exactly its annotated instance fields, has no base but `object`, no metaclass, no decorator but `@compiled`, diff --git a/docs/design/typedpython.md b/docs/design/typedpython.md index 99120785..eaf3a02d 100644 --- a/docs/design/typedpython.md +++ b/docs/design/typedpython.md @@ -162,6 +162,12 @@ API 안정성을 확인한 뒤로 미룬다. 인터페이스 해시가 그대로라 의존 모듈은 다시 컴파일하지 않는다. C 오브젝트는 모듈별로 캐시한다. Gradle 태스크는 입력·출력을 정확히 선언해 up-to-date 판정과 빌드 캐시가 함께 작동한다. +pypackpack 의 native compile slot 이 C 를 확장으로 빌드하므로 캐시는 두 층으로 나뉜다(#113). +`pipeline.generate_project` 는 소스 → C 만 캐시한다: 모듈마다 `.c` 하나, 그리고 `tp_runtime.h` 가 있는 +디렉터리. 키는 소스 + 의존 모듈 인터페이스 해시 + 컴파일러 버전이고, C 는 플랫폼·컴파일러·플래그를 읽지 +않으므로 그것들은 키에 넣지 않는다. 바뀌지 않은 모듈은 바이트가 같은 `.c` 로 재사용되어 slot 의 C → 확장 +캐시도 맞는다. 테스트가 쓰는 `compile_project`(프로세스 안에서 빌드)는 그대로 둔다. + ### 4.3.3 고정 레이아웃 클래스 (SPEC N-11, M2b) **레이아웃이 CPython 에서도 고정일 때만** 고정 접근으로 컴파일한다, 사용자가 `__slots__` 를 직접 쓴 클래스. diff --git a/python-multiplatform-ksp/src/main/python/typedpython/pipeline.py b/python-multiplatform-ksp/src/main/python/typedpython/pipeline.py index 3f504651..f056da35 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/pipeline.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/pipeline.py @@ -154,27 +154,24 @@ class ModuleResult: interface: str = "" kind: str = "compiled" # "compiled" | "nothing_compiled" | "not_marked" skipped_items: list[SkippedItem] = field(default_factory=list) # `skipped`, structured + c_source: Path | None = None # generate_project only: the module's one `.c` (#113) @dataclass class ProjectResult: modules: list[ModuleResult] + runtime_include: Path | None = None # generate_project only: the dir holding tp_runtime.h @property def rebuilt(self) -> list[ModuleResult]: return [m for m in self.modules if m.status == "rebuilt"] -def compile_project( - sources: list[Path], - project_root: Path, - cache_dir: Path, - *, - python_include: Path | None = None, - ext_suffix: str | None = None, - runtime_dir: Path | None = None, - cost_rule: bool = True, -) -> ProjectResult: +def _project(sources: list[Path], project_root: Path, cache_dir: Path, *, cost_rule: bool, + compile_c, compiler_version: str, platform: str, flags: list[str], + c_mode: bool) -> list[ModuleResult]: + """The cached source -> artefact walk shared by compile_project (artefact: the extension) and + generate_project (artefact: the `.c`).""" sources = [Path(s).resolve() for s in sources] cache_dir = Path(cache_dir) verified: dict[Path, list[SkippedItem]] = {} @@ -193,27 +190,12 @@ def lower(path: Path) -> ir.Module: def generate(module: ir.Module, path: Path) -> str: return cgen.generate(module, path, _relative(path, root)) if module.functions else "" - def compile_c(c_source: str, name: str, out_dir: Path) -> Path: - if not c_source: - marker = Path(out_dir) / (name.rpartition(".")[2] + _NO_EXTENSION) - marker.write_text("nothing compiled\n") - return marker - return cbuild.build(c_source, name, out_dir, runtime_dir=runtime_dir, flags=cbuild.DEFAULT_FLAGS, - python_include=python_include, ext_suffix=ext_suffix) - - version = COMPILER_VERSION - if runtime_dir is not None: - version += ":" + _tree_hash(Path(runtime_dir)) if not cost_rule: - version += ":nocost" - suffix = ext_suffix if ext_suffix is not None else sysconfig.get_config_var("EXT_SUFFIX") - include = cbuild._python_include(python_include) - flags = [*cbuild.DEFAULT_FLAGS, f"-I{include}"] - + compiler_version += ":nocost" cache = incremental.BuildCache(cache_dir) report = cache.build_project( sources, project_root, lower=lower, generate=generate, compile_c=compile_c, - compiler_version=version, platform=f"{sysconfig.get_platform()} {suffix}", flags=flags) + compiler_version=compiler_version, platform=platform, flags=flags) side = cache_dir / "skipped" side.mkdir(exist_ok=True) @@ -230,9 +212,78 @@ def compile_c(c_source: str, name: str, out_dir: Path) -> Path: items = [SkippedItem(**i) for i in data["items"]] except (OSError, ValueError, KeyError, TypeError): skipped, items = {}, [] - ext = None if m.artifact is None or m.artifact.suffix == _NO_EXTENSION else m.artifact + art = None if m.artifact is None or m.artifact.suffix == _NO_EXTENSION else m.artifact + ext, c_file = (None, art) if c_mode else (art, None) kind = ("not_marked" if not frontend.is_opted_in(m.path) - else "compiled" if ext is not None else "nothing_compiled") + else "compiled" if art is not None else "nothing_compiled") out.append(ModuleResult(m.path, m.name, ext, m.status, list(m.reasons), skipped, m.interface, - kind, items)) - return ProjectResult(out) + kind, items, c_file)) + return out + + +def _no_c_marker(name: str, out_dir: Path) -> Path: + marker = Path(out_dir) / (name.rpartition(".")[2] + _NO_EXTENSION) + marker.write_text("nothing compiled\n") + return marker + + +def compile_project( + sources: list[Path], + project_root: Path, + cache_dir: Path, + *, + python_include: Path | None = None, + ext_suffix: str | None = None, + runtime_dir: Path | None = None, + cost_rule: bool = True, +) -> ProjectResult: + def compile_c(c_source: str, name: str, out_dir: Path) -> Path: + if not c_source: + return _no_c_marker(name, out_dir) + return cbuild.build(c_source, name, out_dir, runtime_dir=runtime_dir, flags=cbuild.DEFAULT_FLAGS, + python_include=python_include, ext_suffix=ext_suffix) + + version = COMPILER_VERSION + if runtime_dir is not None: + version += ":" + _tree_hash(Path(runtime_dir)) + suffix = ext_suffix if ext_suffix is not None else sysconfig.get_config_var("EXT_SUFFIX") + include = cbuild._python_include(python_include) + flags = [*cbuild.DEFAULT_FLAGS, f"-I{include}"] + return ProjectResult(_project( + sources, project_root, cache_dir, cost_rule=cost_rule, compile_c=compile_c, + compiler_version=version, platform=f"{sysconfig.get_platform()} {suffix}", flags=flags, + c_mode=False)) + + +# The platform component of a generated-C cache key. The C does not depend on the host, the target, +# the C compiler or its flags (cgen reads none of them; the target's headers and the toolchain are +# the compile slot's inputs and its own cache key, pypackpack docs/design/compile-slot.md §9), so +# one constant stands for all of them and a `.c` built on one machine is reused on another. +GENERATED_C_PLATFORM = "generated-c" + + +def generate_project( + sources: list[Path], + project_root: Path, + cache_dir: Path, + *, + cost_rule: bool = True, +) -> ProjectResult: + """Source -> C for every module, cached (#113): the input of pypackpack's native compile slot, + which builds the extensions and caches C -> extension itself. Each compiled module gets exactly + one `.c` (`ModuleResult.c_source`, named after the module's last dotted part); a module with + nothing compiled gets none and ships as Python. `runtime_include` is the directory holding the + header-only `tp_runtime.h`, for the slot's include dirs. The key is the source, the dependency + interfaces and COMPILER_VERSION (which covers the runtime header); no toolchain flag enters it. + An unchanged module is reused with a byte-identical `.c`.""" + def write_c(c_source: str, name: str, out_dir: Path) -> Path: + if not c_source: + return _no_c_marker(name, out_dir) + c_file = Path(out_dir) / (name.rpartition(".")[2] + ".c") + c_file.write_text(c_source) + return c_file + + return ProjectResult(_project( + sources, project_root, cache_dir, cost_rule=cost_rule, compile_c=write_c, + compiler_version=COMPILER_VERSION, platform=GENERATED_C_PLATFORM, flags=[], c_mode=True), + runtime_include=cbuild.RUNTIME_DIR) diff --git a/python-multiplatform-ksp/src/test/python/test_generate_project.py b/python-multiplatform-ksp/src/test/python/test_generate_project.py new file mode 100644 index 00000000..f5ae50a9 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_generate_project.py @@ -0,0 +1,111 @@ +"""generate_project: TypedPython caches source -> C, the compile slot builds C -> extension (#113). + +Real Pyrefly; the only C build here is the one that proves the emitted `.c` is the same program +compile_project builds. +""" +import textwrap +from pathlib import Path + +import pytest + +from conftest import decorate_compiled +from typedpython import cbuild, frontend, pipeline + +B = decorate_compiled(textwrap.dedent("""\ + def twice(n: int) -> int: + return n * 2 + """)) +A = decorate_compiled(textwrap.dedent("""\ + import b + + + def inc(n: int) -> int: + return n + 1 + """)) +PLAIN = "def hello() -> str:\n return 'hi'\n" + + +@pytest.fixture +def project(tmp_path): + root = tmp_path / "proj" + (root / "pkg").mkdir(parents=True) + (root / "a.py").write_text(A) + (root / "b.py").write_text(B) + (root / "pkg" / "__init__.py").write_text("") + (root / "pkg" / "plain.py").write_text(PLAIN) + return root + + +@pytest.fixture +def counts(monkeypatch): + n = {"lower": 0, "build": 0} + real_lower = frontend.lower + + def lower(*a, **k): + n["lower"] += 1 + return real_lower(*a, **k) + + def build(*a, **k): + n["build"] += 1 + raise AssertionError("generate_project must not run the C compiler") + + monkeypatch.setattr(frontend, "lower", lower) + monkeypatch.setattr(cbuild, "build", build) + return n + + +def gen(project, cache): + return pipeline.generate_project( + [project / "a.py", project / "b.py", project / "pkg" / "plain.py"], project, cache) + + +def by_stem(result): + return {m.path.stem: m for m in result.modules} + + +def test_emits_one_c_per_compiled_module_and_the_runtime_dir(project, tmp_path, counts): + r = gen(project, tmp_path / "cache") + m = by_stem(r) + assert m["a"].c_source.name == "a.c" and m["b"].c_source.name == "b.c" + assert "PyInit_a" in m["a"].c_source.read_text() + assert m["a"].extension is None and m["a"].kind == "compiled" + assert m["plain"].c_source is None and m["plain"].kind == "not_marked" + assert (r.runtime_include / "tp_runtime.h").is_file() + assert counts == {"lower": 3, "build": 0} + + +def test_unchanged_module_hits_with_byte_identical_c(project, tmp_path, counts): + first = by_stem(gen(project, tmp_path / "cache")) + c_a = first["a"].c_source.read_bytes() + second = gen(project, tmp_path / "cache") + assert {m.path.stem: m.status for m in second.modules} == {"a": "reused", "b": "reused", "plain": "reused"} + assert by_stem(second)["a"].c_source.read_bytes() == c_a + assert counts["lower"] == 3 # nothing lowered again + + +def test_fresh_cache_gives_byte_identical_c(project, tmp_path): + one = by_stem(gen(project, tmp_path / "c1"))["a"].c_source.read_bytes() + two = by_stem(gen(project, tmp_path / "c2"))["a"].c_source.read_bytes() + assert one == two + + +def test_body_change_regenerates_only_that_module(project, tmp_path): + gen(project, tmp_path / "cache") + (project / "b.py").write_text(B.replace("n * 2", "n + n")) + st = {m.path.stem: m.status for m in gen(project, tmp_path / "cache").modules} + assert st == {"a": "reused", "b": "rebuilt", "plain": "reused"} + + +def test_key_ignores_toolchain(project, tmp_path, monkeypatch): + gen(project, tmp_path / "cache") + monkeypatch.setattr(cbuild, "DEFAULT_FLAGS", (*cbuild.DEFAULT_FLAGS, "-DDIFFERENT")) + monkeypatch.setattr(pipeline.sysconfig, "get_platform", lambda: "other-platform") + assert all(m.status == "reused" for m in gen(project, tmp_path / "cache").modules) + + +def test_emitted_c_is_what_compile_project_builds(project, tmp_path): + c_mod = by_stem(gen(project, tmp_path / "ccache"))["b"] + built = by_stem(pipeline.compile_project([project / "b.py"], project, tmp_path / "bcache"))["b"] + so = cbuild.build(c_mod.c_source.read_text(), "b", tmp_path / "slot") + assert cbuild.load("b", so).twice(21) == 42 + assert built.extension is not None and built.c_source is None