TypedPython: emit-c CLI and manifest for pypackpack's compile slot (#25 step 1)

docs/SPEC.md

@@ -795,6 +795,15 @@ All in `PM/commonTest`, so they run wherever the interpreter loads.
   only, changing toolchain flags or platform reuses every entry, the C compiler is never run, and the
   emitted `.c` built by `cbuild` behaves as `compile_project`'s extension. `compile_project` keeps
   building in process.
+  `python -m typedpython emit-c PATHS... --root DIR --cache DIR --manifest FILE` (#25, step 1) runs
+  `generate_project` and writes a JSON manifest that maps one to one onto pypackpack's
+  `BundleRequest.nativeModules` (`ExtensionModuleSource`: `moduleName`, `sources`, `includeDirs`,
+  `defines`) and `notNativeModules` (`NotNativeModule`: `module`, `kind` `NOT_MARKED` or
+  `NOTHING_COMPILED`, `items`). Every developer module given is in exactly one list except a package
+  `__init__`, which is in neither; a compiled module's `sources` is its one `.c` and its
+  `includeDirs` the runtime header directory; paths are absolute. Exit status 0 (manifest written), 1
+  (a gate error; no manifest), 2 (the gate could not run). The C compiler is never run. The format is
+  fixed in `docs/design/typedpython.md` §4.3.2a. Tests: `test_emit_c.py`.
 - **N-11** TypedPython fixed-layout classes (M2b, 2026-11-21; design §4.3.3). A class compiles to fixed
   field access only when its layout is fixed **in CPython too**: it declares `__slots__` listing exactly
   its annotated instance fields, has no base but `object`, no metaclass, no decorator but `@compiled`,

docs/design/typedpython.md

@@ -168,6 +168,52 @@ pypackpack 의 native compile slot 이 C 를 확장으로 빌드하므로 캐시
 않으므로 그것들은 키에 넣지 않는다. 바뀌지 않은 모듈은 바이트가 같은 `.c` 로 재사용되어 slot 의 C → 확장
 캐시도 맞는다. 테스트가 쓰는 `compile_project`(프로세스 안에서 빌드)는 그대로 둔다.
 
+### 4.3.2a `emit-c` manifest (SPEC N-10, #25 1단계)
+
+Gradle 쪽(#25 2단계, python-multiplatform-gradle-plugin)은 TypedPython 을 이 CLI 로 돌리고 manifest 를
+pypackpack `BundleRequest` 로 옮긴다. pypackpack 은 TypedPython 을 실행하지 않는다.
+
+    python -m typedpython emit-c PATHS... --root DIR --cache DIR --manifest FILE
+
+- `PATHS`: 파일 또는 디렉터리(재귀, `typedpython check` 와 같은 규칙). `--root`: 모듈 이름의 기준
+  디렉터리(ppp 패키지의 `python/` 루트에 해당). `--cache`: `generate_project` 의 소스 → C 캐시.
+  `--manifest`: 쓸 파일. 상위 디렉터리는 만든다.
+- 종료 코드: 0 manifest 를 씀, 1 게이트 오류(manifest 를 쓰지 않음, 원인은 stderr), 2 게이트를 실행할 수 없음.
+  C 컴파일러는 실행하지 않는다.
+
+형식(버전 1). 키 이름은 pypackpack `docs/design/compile-slot.md` 의 Kotlin 필드 이름과 같다.
+
+```json
+{
+  "format": "typedpython-emit-c",
+  "version": 1,
+  "compilerVersion": "<pipeline.COMPILER_VERSION>",
+  "projectRoot": "/abs/proj",
+  "runtimeInclude": "/abs/.../typedpython/runtime",
+  "nativeModules": [
+    {"moduleName": "app.physics.nbody", "file": "app/physics/nbody.py",
+     "sources": ["/abs/cache/.../nbody.c"], "includeDirs": ["/abs/.../typedpython/runtime"],
+     "defines": {}, "status": "rebuilt",
+     "skippedItems": [{"name": "main", "file": "app/physics/nbody.py", "line": 30, "message": "..."}]}
+  ],
+  "notNativeModules": [
+    {"module": "app.util", "file": "app/util.py", "kind": "NOT_MARKED", "items": []},
+    {"module": "app.text", "file": "app/text.py", "kind": "NOTHING_COMPILED",
+     "items": [{"name": "f", "file": "app/text.py", "line": 2, "message": "..."}]}
+  ]
+}
+```
+
+- `nativeModules[i]` → `ExtensionModuleSource(moduleName, sources, includeDirs, defines)`. `file`, `status`,
+  `skippedItems` 는 보고용이다(일부 함수가 인터프리트로 남는 것은 경고가 아니다, compile-slot.md §4).
+- `notNativeModules[i]` → `NotNativeModule(module, kind, items)`, `items[j]` → `SkippedItem(name, file, line, message)`.
+  `line` 은 없으면 `null`.
+- 모든 경로는 절대 경로(`/` 구분). `file` 만 `projectRoot` 기준 상대 경로다.
+- 패키지 `__init__.py` 는 어느 목록에도 넣지 않는다(compile-slot.md: TypedPython 은 `__init__` 을 넘기지 않는다).
+  그 밖의 모듈은 정확히 한 목록에 들어간다. 순서는 입력 파일 순서(디렉터리는 정렬)다.
+- `sources` 는 캐시 안의 `.c` 를 가리킨다. 바뀌지 않은 모듈은 같은 경로, 같은 바이트다.
+- 버전을 올리는 변경: 키 제거나 의미 변경. 키 추가는 버전을 올리지 않는다.
+
 ### 4.3.3 고정 레이아웃 클래스 (SPEC N-11, M2b)
 
 **레이아웃이 CPython 에서도 고정일 때만** 고정 접근으로 컴파일한다, 사용자가 `__slots__` 를 직접 쓴 클래스.

python-multiplatform-ksp/src/main/python/typedpython/__main__.py

@@ -1,4 +1,5 @@
 """`python -m typedpython check [--mode checked|compiled] [--format text|json] PATHS...`
+`python -m typedpython emit-c PATHS... --root DIR [--cache DIR] --manifest FILE`
 
 Exit status: 0 no errors (warnings allowed), 1 errors, 2 the gate itself could not run.
 """
@@ -36,8 +37,17 @@ def main(argv: list[str]) -> int:
                        help="incremental cache directory (default: build/typedpython-cache)")
     build.add_argument("--out", type=Path, default=None,
                        help="copy the built extensions here (default: leave them in the cache)")
+    emit = commands.add_parser(
+        "emit-c", help="generate C per module (cached) and a manifest for pypackpack's compile slot")
+    emit.add_argument("paths", nargs="+", type=Path, help="files, or directories to scan recursively")
+    emit.add_argument("--root", type=Path, required=True, help="directory module names are relative to")
+    emit.add_argument("--cache", type=Path, default=Path("build/typedpython-c-cache"),
+                      help="source -> C cache directory (default: build/typedpython-c-cache)")
+    emit.add_argument("--manifest", type=Path, required=True, help="the JSON manifest to write")
     args = parser.parse_args(argv)
 
+    if args.command == "emit-c":
+        return _emit_c(args.paths, args.root, args.cache, args.manifest)
     if args.command == "build":
         return _build(args.paths, args.cache, args.out)
     if args.command == "demo":
@@ -94,6 +104,57 @@ def _build(paths: list[Path], cache: Path, out: Path | None) -> int:
     return EXIT_CLEAN
 
 
+MANIFEST_FORMAT, MANIFEST_VERSION = "typedpython-emit-c", 1
+
+
+def _emit_c(paths: list[Path], root: Path, cache: Path, manifest: Path) -> int:
+    """SPEC N-10 / #25 step 1; the manifest format is docs/design/typedpython.md section 4.3.2a."""
+    from typedpython import pipeline
+
+    sources = python_files(paths)
+    if not sources:
+        print("typedpython: no Python files to emit", file=sys.stderr)
+        return EXIT_FAILURE
+    try:
+        result = pipeline.generate_project(sources, root, cache)
+    except pipeline.CompileError as e:
+        print(f"typedpython: {e}", file=sys.stderr)
+        return EXIT_ERRORS
+    except PyreflyError as e:
+        print(f"typedpython: {e}", file=sys.stderr)
+        return EXIT_FAILURE
+    root = root.resolve()
+    runtime = Path(result.runtime_include).resolve().as_posix()
+
+    def items(m: "pipeline.ModuleResult") -> list[dict]:
+        return [{"name": i.name, "file": i.file, "line": i.line, "message": i.message}
+                for i in m.skipped_items]
+
+    native, other = [], []
+    for m in result.modules:
+        if m.path.name == "__init__.py":
+            continue                        # never listed (pypackpack compile-slot.md)
+        rel = m.path.resolve().relative_to(root).as_posix()
+        if m.kind == "compiled":
+            assert m.c_source is not None
+            native.append({"moduleName": m.name, "file": rel,
+                           "sources": [Path(m.c_source).resolve().as_posix()],
+                           "includeDirs": [runtime], "defines": {}, "status": m.status,
+                           "skippedItems": items(m)})
+        else:
+            other.append({"module": m.name, "file": rel,
+                          "kind": "NOT_MARKED" if m.kind == "not_marked" else "NOTHING_COMPILED",
+                          "items": items(m)})
+    doc = {"format": MANIFEST_FORMAT, "version": MANIFEST_VERSION,
+           "compilerVersion": pipeline.COMPILER_VERSION, "projectRoot": root.as_posix(),
+           "runtimeInclude": runtime, "nativeModules": native, "notNativeModules": other}
+    manifest.parent.mkdir(parents=True, exist_ok=True)
+    tmp = manifest.with_name(manifest.name + ".tmp")
+    tmp.write_text(json.dumps(doc, indent=2) + "\n")
+    tmp.replace(manifest)
+    return EXIT_CLEAN
+
+
 def _demo(module: Path, module_argv: list[str], out: Path) -> int:
     import contextlib
     import importlib.util

python-multiplatform-ksp/src/test/python/test_emit_c.py

@@ -0,0 +1,121 @@
+"""`python -m typedpython emit-c` (#25 step 1, SPEC N-10): generate_project plus a JSON manifest that
+maps onto pypackpack's BundleRequest.nativeModules / notNativeModules. Format: docs/design/typedpython.md
+section 4.3.2a.
+
+Real Pyrefly; the C compiler is never run here.
+"""
+import json
+import textwrap
+from pathlib import Path
+
+import pytest
+
+from conftest import decorate_compiled
+from typedpython import cbuild
+from typedpython.__main__ import main
+
+A = decorate_compiled(textwrap.dedent("""\
+    def inc(n: int) -> int:
+        return n + 1
+    """))
+PLAIN = "def hello() -> str:\n    return 'hi'\n"
+NOTHING = "@compiled\ndef f(s: str) -> str:\n    return s[1:]\n"
+BROKEN = "@compiled\ndef f(n: int) -> int:\n    return n + 'x'\n"
+
+
+@pytest.fixture
+def project(tmp_path):
+    root = tmp_path / "proj"
+    (root / "app").mkdir(parents=True)
+    (root / "app" / "__init__.py").write_text("")
+    (root / "app" / "fast.py").write_text(A)
+    (root / "app" / "plain.py").write_text(PLAIN)
+    (root / "app" / "text.py").write_text(NOTHING)
+    return root
+
+
+@pytest.fixture(autouse=True)
+def no_c_compiler(monkeypatch):
+    def build(*a, **k):
+        raise AssertionError("emit-c must not run the C compiler")
+    monkeypatch.setattr(cbuild, "build", build)
+
+
+def emit(project, tmp_path, *paths):
+    manifest = tmp_path / "out" / "manifest.json"
+    rc = main(["emit-c", *(str(p) for p in (paths or [project])), "--root", str(project),
+               "--cache", str(tmp_path / "cache"), "--manifest", str(manifest)])
+    return rc, manifest
+
+
+def test_manifest_lists_every_module_once_in_the_slot_shape(project, tmp_path):
+    rc, manifest = emit(project, tmp_path)
+    assert rc == 0
+    m = json.loads(manifest.read_text())
+    assert m["format"] == "typedpython-emit-c" and m["version"] == 1
+    assert m["projectRoot"] == project.resolve().as_posix()
+    runtime = Path(m["runtimeInclude"])
+    assert runtime.is_absolute() and (runtime / "tp_runtime.h").is_file()
+    assert m["compilerVersion"]
+
+    native = {n["moduleName"]: n for n in m["nativeModules"]}
+    assert list(native) == ["app.fast"]
+    fast = native["app.fast"]
+    assert set(fast) >= {"moduleName", "sources", "includeDirs", "defines", "file", "status", "skippedItems"}
+    assert fast["file"] == "app/fast.py" and fast["defines"] == {}
+    assert fast["includeDirs"] == [m["runtimeInclude"]]
+    assert len(fast["sources"]) == 1
+    c_file = Path(fast["sources"][0])
+    assert c_file.is_absolute() and c_file.name == "fast.c" and c_file.is_file()
+    assert "PyInit_fast" in c_file.read_text()
+
+    other = {n["module"]: n for n in m["notNativeModules"]}
+    assert list(other) == ["app.plain", "app.text"]
+    assert other["app.plain"] == {"module": "app.plain", "file": "app/plain.py", "kind": "NOT_MARKED", "items": []}
+    text = other["app.text"]
+    assert text["kind"] == "NOTHING_COMPILED" and text["items"]
+    item = text["items"][0]
+    assert set(item) == {"name", "file", "line", "message"}
+    assert item["name"] == "f" and item["file"] == "app/text.py" and isinstance(item["line"], int)
+
+    every = [n["moduleName"] for n in m["nativeModules"]] + [n["module"] for n in m["notNativeModules"]]
+    assert "app" not in every and "app.__init__" not in every          # a package __init__ is in neither
+
+
+def test_a_second_run_reuses_the_same_c_files(project, tmp_path):
+    _, manifest = emit(project, tmp_path)
+    first = json.loads(manifest.read_text())
+    c_file = Path(first["nativeModules"][0]["sources"][0])
+    before = c_file.read_bytes()
+    assert first["nativeModules"][0]["status"] == "rebuilt"
+    rc, manifest = emit(project, tmp_path)
+    second = json.loads(manifest.read_text())
+    assert rc == 0
+    assert second["nativeModules"][0]["status"] == "reused"
+    assert second["nativeModules"][0]["sources"] == first["nativeModules"][0]["sources"]
+    assert c_file.read_bytes() == before
+    strip = lambda d: {**d, "nativeModules": [{**n, "status": None} for n in d["nativeModules"]]}
+    assert strip(second) == strip(first)
+
+
+def test_files_given_one_by_one_keep_their_order(project, tmp_path):
+    rc, manifest = emit(project, tmp_path, project / "app" / "text.py", project / "app" / "plain.py")
+    assert rc == 0
+    m = json.loads(manifest.read_text())
+    assert m["nativeModules"] == []
+    assert [n["module"] for n in m["notNativeModules"]] == ["app.text", "app.plain"]
+
+
+def test_a_gate_error_exits_1_and_writes_no_manifest(project, tmp_path, capsys):
+    (project / "app" / "broken.py").write_text(BROKEN)
+    rc, manifest = emit(project, tmp_path)
+    assert rc == 1
+    assert not manifest.exists()
+    assert "typedpython:" in capsys.readouterr().err
+
+
+def test_no_python_files_exits_2(tmp_path):
+    empty = tmp_path / "empty"
+    empty.mkdir()
+    rc, manifest = emit(empty, tmp_path)
+    assert rc == 2 and not manifest.exists()