From eb17d997427b31d1df8afc6c320579bf6ec4fa4a Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:49:26 +0900 Subject: [PATCH] TypedPython: emit-c CLI and manifest for pypackpack's compile slot (#25 step 1) python -m typedpython emit-c runs generate_project and writes a JSON manifest whose nativeModules / notNativeModules entries map one to one onto pypackpack's ExtensionModuleSource and NotNativeModule. The format (version 1) is fixed in docs/design/typedpython.md 4.3.2a for the Gradle task (#25 step 2). --- docs/SPEC.md | 9 ++ docs/design/typedpython.md | 46 +++++++ .../src/main/python/typedpython/__main__.py | 61 +++++++++ .../src/test/python/test_emit_c.py | 121 ++++++++++++++++++ 4 files changed, 237 insertions(+) create mode 100644 python-multiplatform-ksp/src/test/python/test_emit_c.py diff --git a/docs/SPEC.md b/docs/SPEC.md index 13e3a29f..43b78379 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -795,6 +795,15 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. only, changing toolchain flags or platform reuses every entry, the C compiler is never run, and the emitted `.c` built by `cbuild` behaves as `compile_project`'s extension. `compile_project` keeps building in process. + `python -m typedpython emit-c PATHS... --root DIR --cache DIR --manifest FILE` (#25, step 1) runs + `generate_project` and writes a JSON manifest that maps one to one onto pypackpack's + `BundleRequest.nativeModules` (`ExtensionModuleSource`: `moduleName`, `sources`, `includeDirs`, + `defines`) and `notNativeModules` (`NotNativeModule`: `module`, `kind` `NOT_MARKED` or + `NOTHING_COMPILED`, `items`). Every developer module given is in exactly one list except a package + `__init__`, which is in neither; a compiled module's `sources` is its one `.c` and its + `includeDirs` the runtime header directory; paths are absolute. Exit status 0 (manifest written), 1 + (a gate error; no manifest), 2 (the gate could not run). The C compiler is never run. The format is + fixed in `docs/design/typedpython.md` §4.3.2a. Tests: `test_emit_c.py`. - **N-11** TypedPython fixed-layout classes (M2b, 2026-11-21; design §4.3.3). A class compiles to fixed field access only when its layout is fixed **in CPython too**: it declares `__slots__` listing exactly its annotated instance fields, has no base but `object`, no metaclass, no decorator but `@compiled`, diff --git a/docs/design/typedpython.md b/docs/design/typedpython.md index eaf3a02d..1c95c37e 100644 --- a/docs/design/typedpython.md +++ b/docs/design/typedpython.md @@ -168,6 +168,52 @@ pypackpack 의 native compile slot 이 C 를 확장으로 빌드하므로 캐시 않으므로 그것들은 키에 넣지 않는다. 바뀌지 않은 모듈은 바이트가 같은 `.c` 로 재사용되어 slot 의 C → 확장 캐시도 맞는다. 테스트가 쓰는 `compile_project`(프로세스 안에서 빌드)는 그대로 둔다. +### 4.3.2a `emit-c` manifest (SPEC N-10, #25 1단계) + +Gradle 쪽(#25 2단계, python-multiplatform-gradle-plugin)은 TypedPython 을 이 CLI 로 돌리고 manifest 를 +pypackpack `BundleRequest` 로 옮긴다. pypackpack 은 TypedPython 을 실행하지 않는다. + + python -m typedpython emit-c PATHS... --root DIR --cache DIR --manifest FILE + +- `PATHS`: 파일 또는 디렉터리(재귀, `typedpython check` 와 같은 규칙). `--root`: 모듈 이름의 기준 + 디렉터리(ppp 패키지의 `python/` 루트에 해당). `--cache`: `generate_project` 의 소스 → C 캐시. + `--manifest`: 쓸 파일. 상위 디렉터리는 만든다. +- 종료 코드: 0 manifest 를 씀, 1 게이트 오류(manifest 를 쓰지 않음, 원인은 stderr), 2 게이트를 실행할 수 없음. + C 컴파일러는 실행하지 않는다. + +형식(버전 1). 키 이름은 pypackpack `docs/design/compile-slot.md` 의 Kotlin 필드 이름과 같다. + +```json +{ + "format": "typedpython-emit-c", + "version": 1, + "compilerVersion": "", + "projectRoot": "/abs/proj", + "runtimeInclude": "/abs/.../typedpython/runtime", + "nativeModules": [ + {"moduleName": "app.physics.nbody", "file": "app/physics/nbody.py", + "sources": ["/abs/cache/.../nbody.c"], "includeDirs": ["/abs/.../typedpython/runtime"], + "defines": {}, "status": "rebuilt", + "skippedItems": [{"name": "main", "file": "app/physics/nbody.py", "line": 30, "message": "..."}]} + ], + "notNativeModules": [ + {"module": "app.util", "file": "app/util.py", "kind": "NOT_MARKED", "items": []}, + {"module": "app.text", "file": "app/text.py", "kind": "NOTHING_COMPILED", + "items": [{"name": "f", "file": "app/text.py", "line": 2, "message": "..."}]} + ] +} +``` + +- `nativeModules[i]` → `ExtensionModuleSource(moduleName, sources, includeDirs, defines)`. `file`, `status`, + `skippedItems` 는 보고용이다(일부 함수가 인터프리트로 남는 것은 경고가 아니다, compile-slot.md §4). +- `notNativeModules[i]` → `NotNativeModule(module, kind, items)`, `items[j]` → `SkippedItem(name, file, line, message)`. + `line` 은 없으면 `null`. +- 모든 경로는 절대 경로(`/` 구분). `file` 만 `projectRoot` 기준 상대 경로다. +- 패키지 `__init__.py` 는 어느 목록에도 넣지 않는다(compile-slot.md: TypedPython 은 `__init__` 을 넘기지 않는다). + 그 밖의 모듈은 정확히 한 목록에 들어간다. 순서는 입력 파일 순서(디렉터리는 정렬)다. +- `sources` 는 캐시 안의 `.c` 를 가리킨다. 바뀌지 않은 모듈은 같은 경로, 같은 바이트다. +- 버전을 올리는 변경: 키 제거나 의미 변경. 키 추가는 버전을 올리지 않는다. + ### 4.3.3 고정 레이아웃 클래스 (SPEC N-11, M2b) **레이아웃이 CPython 에서도 고정일 때만** 고정 접근으로 컴파일한다, 사용자가 `__slots__` 를 직접 쓴 클래스. diff --git a/python-multiplatform-ksp/src/main/python/typedpython/__main__.py b/python-multiplatform-ksp/src/main/python/typedpython/__main__.py index 4db82478..f368b72c 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/__main__.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/__main__.py @@ -1,4 +1,5 @@ """`python -m typedpython check [--mode checked|compiled] [--format text|json] PATHS...` +`python -m typedpython emit-c PATHS... --root DIR [--cache DIR] --manifest FILE` Exit status: 0 no errors (warnings allowed), 1 errors, 2 the gate itself could not run. """ @@ -36,8 +37,17 @@ def main(argv: list[str]) -> int: help="incremental cache directory (default: build/typedpython-cache)") build.add_argument("--out", type=Path, default=None, help="copy the built extensions here (default: leave them in the cache)") + emit = commands.add_parser( + "emit-c", help="generate C per module (cached) and a manifest for pypackpack's compile slot") + emit.add_argument("paths", nargs="+", type=Path, help="files, or directories to scan recursively") + emit.add_argument("--root", type=Path, required=True, help="directory module names are relative to") + emit.add_argument("--cache", type=Path, default=Path("build/typedpython-c-cache"), + help="source -> C cache directory (default: build/typedpython-c-cache)") + emit.add_argument("--manifest", type=Path, required=True, help="the JSON manifest to write") args = parser.parse_args(argv) + if args.command == "emit-c": + return _emit_c(args.paths, args.root, args.cache, args.manifest) if args.command == "build": return _build(args.paths, args.cache, args.out) if args.command == "demo": @@ -94,6 +104,57 @@ def _build(paths: list[Path], cache: Path, out: Path | None) -> int: return EXIT_CLEAN +MANIFEST_FORMAT, MANIFEST_VERSION = "typedpython-emit-c", 1 + + +def _emit_c(paths: list[Path], root: Path, cache: Path, manifest: Path) -> int: + """SPEC N-10 / #25 step 1; the manifest format is docs/design/typedpython.md section 4.3.2a.""" + from typedpython import pipeline + + sources = python_files(paths) + if not sources: + print("typedpython: no Python files to emit", file=sys.stderr) + return EXIT_FAILURE + try: + result = pipeline.generate_project(sources, root, cache) + except pipeline.CompileError as e: + print(f"typedpython: {e}", file=sys.stderr) + return EXIT_ERRORS + except PyreflyError as e: + print(f"typedpython: {e}", file=sys.stderr) + return EXIT_FAILURE + root = root.resolve() + runtime = Path(result.runtime_include).resolve().as_posix() + + def items(m: "pipeline.ModuleResult") -> list[dict]: + return [{"name": i.name, "file": i.file, "line": i.line, "message": i.message} + for i in m.skipped_items] + + native, other = [], [] + for m in result.modules: + if m.path.name == "__init__.py": + continue # never listed (pypackpack compile-slot.md) + rel = m.path.resolve().relative_to(root).as_posix() + if m.kind == "compiled": + assert m.c_source is not None + native.append({"moduleName": m.name, "file": rel, + "sources": [Path(m.c_source).resolve().as_posix()], + "includeDirs": [runtime], "defines": {}, "status": m.status, + "skippedItems": items(m)}) + else: + other.append({"module": m.name, "file": rel, + "kind": "NOT_MARKED" if m.kind == "not_marked" else "NOTHING_COMPILED", + "items": items(m)}) + doc = {"format": MANIFEST_FORMAT, "version": MANIFEST_VERSION, + "compilerVersion": pipeline.COMPILER_VERSION, "projectRoot": root.as_posix(), + "runtimeInclude": runtime, "nativeModules": native, "notNativeModules": other} + manifest.parent.mkdir(parents=True, exist_ok=True) + tmp = manifest.with_name(manifest.name + ".tmp") + tmp.write_text(json.dumps(doc, indent=2) + "\n") + tmp.replace(manifest) + return EXIT_CLEAN + + def _demo(module: Path, module_argv: list[str], out: Path) -> int: import contextlib import importlib.util diff --git a/python-multiplatform-ksp/src/test/python/test_emit_c.py b/python-multiplatform-ksp/src/test/python/test_emit_c.py new file mode 100644 index 00000000..73713789 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_emit_c.py @@ -0,0 +1,121 @@ +"""`python -m typedpython emit-c` (#25 step 1, SPEC N-10): generate_project plus a JSON manifest that +maps onto pypackpack's BundleRequest.nativeModules / notNativeModules. Format: docs/design/typedpython.md +section 4.3.2a. + +Real Pyrefly; the C compiler is never run here. +""" +import json +import textwrap +from pathlib import Path + +import pytest + +from conftest import decorate_compiled +from typedpython import cbuild +from typedpython.__main__ import main + +A = decorate_compiled(textwrap.dedent("""\ + def inc(n: int) -> int: + return n + 1 + """)) +PLAIN = "def hello() -> str:\n return 'hi'\n" +NOTHING = "@compiled\ndef f(s: str) -> str:\n return s[1:]\n" +BROKEN = "@compiled\ndef f(n: int) -> int:\n return n + 'x'\n" + + +@pytest.fixture +def project(tmp_path): + root = tmp_path / "proj" + (root / "app").mkdir(parents=True) + (root / "app" / "__init__.py").write_text("") + (root / "app" / "fast.py").write_text(A) + (root / "app" / "plain.py").write_text(PLAIN) + (root / "app" / "text.py").write_text(NOTHING) + return root + + +@pytest.fixture(autouse=True) +def no_c_compiler(monkeypatch): + def build(*a, **k): + raise AssertionError("emit-c must not run the C compiler") + monkeypatch.setattr(cbuild, "build", build) + + +def emit(project, tmp_path, *paths): + manifest = tmp_path / "out" / "manifest.json" + rc = main(["emit-c", *(str(p) for p in (paths or [project])), "--root", str(project), + "--cache", str(tmp_path / "cache"), "--manifest", str(manifest)]) + return rc, manifest + + +def test_manifest_lists_every_module_once_in_the_slot_shape(project, tmp_path): + rc, manifest = emit(project, tmp_path) + assert rc == 0 + m = json.loads(manifest.read_text()) + assert m["format"] == "typedpython-emit-c" and m["version"] == 1 + assert m["projectRoot"] == project.resolve().as_posix() + runtime = Path(m["runtimeInclude"]) + assert runtime.is_absolute() and (runtime / "tp_runtime.h").is_file() + assert m["compilerVersion"] + + native = {n["moduleName"]: n for n in m["nativeModules"]} + assert list(native) == ["app.fast"] + fast = native["app.fast"] + assert set(fast) >= {"moduleName", "sources", "includeDirs", "defines", "file", "status", "skippedItems"} + assert fast["file"] == "app/fast.py" and fast["defines"] == {} + assert fast["includeDirs"] == [m["runtimeInclude"]] + assert len(fast["sources"]) == 1 + c_file = Path(fast["sources"][0]) + assert c_file.is_absolute() and c_file.name == "fast.c" and c_file.is_file() + assert "PyInit_fast" in c_file.read_text() + + other = {n["module"]: n for n in m["notNativeModules"]} + assert list(other) == ["app.plain", "app.text"] + assert other["app.plain"] == {"module": "app.plain", "file": "app/plain.py", "kind": "NOT_MARKED", "items": []} + text = other["app.text"] + assert text["kind"] == "NOTHING_COMPILED" and text["items"] + item = text["items"][0] + assert set(item) == {"name", "file", "line", "message"} + assert item["name"] == "f" and item["file"] == "app/text.py" and isinstance(item["line"], int) + + every = [n["moduleName"] for n in m["nativeModules"]] + [n["module"] for n in m["notNativeModules"]] + assert "app" not in every and "app.__init__" not in every # a package __init__ is in neither + + +def test_a_second_run_reuses_the_same_c_files(project, tmp_path): + _, manifest = emit(project, tmp_path) + first = json.loads(manifest.read_text()) + c_file = Path(first["nativeModules"][0]["sources"][0]) + before = c_file.read_bytes() + assert first["nativeModules"][0]["status"] == "rebuilt" + rc, manifest = emit(project, tmp_path) + second = json.loads(manifest.read_text()) + assert rc == 0 + assert second["nativeModules"][0]["status"] == "reused" + assert second["nativeModules"][0]["sources"] == first["nativeModules"][0]["sources"] + assert c_file.read_bytes() == before + strip = lambda d: {**d, "nativeModules": [{**n, "status": None} for n in d["nativeModules"]]} + assert strip(second) == strip(first) + + +def test_files_given_one_by_one_keep_their_order(project, tmp_path): + rc, manifest = emit(project, tmp_path, project / "app" / "text.py", project / "app" / "plain.py") + assert rc == 0 + m = json.loads(manifest.read_text()) + assert m["nativeModules"] == [] + assert [n["module"] for n in m["notNativeModules"]] == ["app.text", "app.plain"] + + +def test_a_gate_error_exits_1_and_writes_no_manifest(project, tmp_path, capsys): + (project / "app" / "broken.py").write_text(BROKEN) + rc, manifest = emit(project, tmp_path) + assert rc == 1 + assert not manifest.exists() + assert "typedpython:" in capsys.readouterr().err + + +def test_no_python_files_exits_2(tmp_path): + empty = tmp_path / "empty" + empty.mkdir() + rc, manifest = emit(empty, tmp_path) + assert rc == 2 and not manifest.exists()