From 8f27c263e384046174d36b8de582f2f80fe380ee Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 7 Nov 2026 10:14:37 +0900 Subject: [PATCH] Feat: add TypedPythonEmitCTask and manifest v1 mapper in Gradle plugin (#25 step 2, SPEC N-10) --- docs/SPEC.md | 17 +- .../gradle/TypedPythonEmitCTask.kt | 359 ++++++++++++++++++ .../gradle/TypedPythonEmitCTaskTest.kt | 321 ++++++++++++++++ 3 files changed, 694 insertions(+), 3 deletions(-) create mode 100644 python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/TypedPythonEmitCTask.kt create mode 100644 python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/TypedPythonEmitCTaskTest.kt diff --git a/docs/SPEC.md b/docs/SPEC.md index 1882bc19..76f1380d 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -792,9 +792,11 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. The Gradle task declares exact inputs and outputs so up-to-date checks and the build cache apply. Tests: changing one function body recompiles that module only; an unchanged rebuild spends ~0 s in the compile step. `Status: partial`: the per-module cache and its key are implemented in Python - (`incremental.py`, `pipeline.compile_project`; `test_incremental.py`, `test_pipeline_incremental.py`); - the Gradle task with declared inputs and outputs does not exist yet, because compiled modules are built - through pypackpack's compile slot (#25). + (`incremental.py`, `pipeline.compile_project`; `test_incremental.py`, `test_pipeline_incremental.py`), + and the Gradle `emit-c` task with declared inputs and outputs is implemented in + `python-multiplatform-gradle-plugin` (#25, step 2; `TypedPythonEmitCTask.kt`, + `TypedPythonEmitCTaskTest.kt`); wiring the resulting `nativeModules` / `notNativeModules` into + pypackpack's compile slot in the bundler (#25, step 3) remains. For that slot, `pipeline.generate_project` caches source to C instead of source to extension (#113): one `.c` per compiled module plus the directory of the header-only `tp_runtime.h`; its key is the source, the dependency interface hashes and the compiler version (which covers the runtime header), @@ -813,6 +815,15 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. `includeDirs` the runtime header directory; paths are absolute. Exit status 0 (manifest written), 1 (a gate error; no manifest), 2 (the gate could not run). The C compiler is never run. The format is fixed in `docs/design/typedpython.md` §4.3.2a. Tests: `test_emit_c.py`. + `TypedPythonEmitCTask` in `python-multiplatform-gradle-plugin` (#25, step 2) runs + `python -m typedpython emit-c` with declared Gradle inputs (`@InputFiles` `sources` with + `PathSensitivity.RELATIVE` and `@SkipWhenEmpty`, `@Input` `pythonExecutable` and `@Optional @InputFiles` + `pythonPath` with `PathSensitivity.RELATIVE`) and outputs (`@OutputDirectory` `cacheDirectory` and + `@OutputFile` `manifestFile`), fails with `GradleException` naming the stderr diagnostics on exit 1 or 2, + and parses the written v1 manifest into `TypedPythonEmitCManifest` (`nativeModules` mapped to + `ExtensionModuleSource(moduleName, sources, includeDirs, defines)` and `notNativeModules` mapped to + `NotNativeModule(module, kind, items)` with `SkippedItem(name, file, line, message)`). Tests: + `TypedPythonEmitCTaskTest.kt`. - **N-11** TypedPython fixed-layout classes (M2b, 2026-11-21; design §4.3.3). A class compiles to fixed field access only when its layout is fixed **in CPython too**: it declares `__slots__` listing exactly its annotated instance fields, has no base but `object`, no metaclass, no decorator but `@compiled`, diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/TypedPythonEmitCTask.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/TypedPythonEmitCTask.kt new file mode 100644 index 00000000..21f23326 --- /dev/null +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/TypedPythonEmitCTask.kt @@ -0,0 +1,359 @@ +package python.multiplatform.gradle + +import groovy.json.JsonSlurper +import org.gradle.api.DefaultTask +import org.gradle.api.GradleException +import org.gradle.api.file.ConfigurableFileCollection +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.provider.Property +import org.gradle.api.tasks.IgnoreEmptyDirectories +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.InputFiles +import org.gradle.api.tasks.Internal +import org.gradle.api.tasks.Optional +import org.gradle.api.tasks.OutputDirectory +import org.gradle.api.tasks.OutputFile +import org.gradle.api.tasks.PathSensitive +import org.gradle.api.tasks.PathSensitivity +import org.gradle.api.tasks.SkipWhenEmpty +import org.gradle.api.tasks.TaskAction +import java.io.File + +/** `docs/design/typedpython.md` section 4.3.2a (`emit-c` manifest format). */ +const val TYPEDPYTHON_EMIT_C_FORMAT = "typedpython-emit-c" + +/** `docs/design/typedpython.md` section 4.3.2a (`emit-c` manifest version). */ +const val TYPEDPYTHON_EMIT_C_VERSION = 1 + +/** + * One extension module's generated C and header/define metadata for pypackpack's compile slot + * (`pypackpack` `ExtensionCompiler.kt`, `BundleRequest.nativeModules`). + */ +data class ExtensionModuleSource( + val moduleName: String, + val sources: List, + val includeDirs: List = emptyList(), + val defines: Map = emptyMap(), +) + +/** + * One function or class that stayed interpreted, and why (`pypackpack` `docs/design/compile-slot.md` section 2). + * + * @param file path relative to `projectRoot`, `/`-separated. + * @param line 1-based line number, or `null` when unavailable. + */ +data class SkippedItem( + val name: String, + val file: String, + val line: Int?, + val message: String, +) + +/** + * One developer module for which TypedPython produced no C, and why (`pypackpack` `BundleRequest.notNativeModules`). + */ +data class NotNativeModule( + val module: String, + val kind: Kind, + val items: List = emptyList(), +) { + enum class Kind { + NOT_MARKED, + NOTHING_COMPILED, + } +} + +/** + * One entry of `nativeModules` in the `emit-c` manifest (`docs/design/typedpython.md` section 4.3.2a), + * carrying both the compile-slot fields ([toExtensionModuleSource]) and the reporting fields + * ([file], [status], [skippedItems]). + */ +data class TypedPythonNativeModuleEntry( + val moduleName: String, + val file: String, + val sources: List, + val includeDirs: List = emptyList(), + val defines: Map = emptyMap(), + val status: String, + val skippedItems: List = emptyList(), +) { + fun toExtensionModuleSource(): ExtensionModuleSource = + ExtensionModuleSource( + moduleName = moduleName, + sources = sources, + includeDirs = includeDirs, + defines = defines, + ) +} + +/** + * Parsed `typedpython-emit-c` v1 manifest (`docs/design/typedpython.md` section 4.3.2a, SPEC N-10). + */ +data class TypedPythonEmitCManifest( + val format: String, + val version: Int, + val compilerVersion: String, + val projectRoot: File, + val runtimeInclude: File, + val nativeModules: List, + val notNativeModules: List, +) { + /** Maps [nativeModules] onto pypackpack's `BundleRequest.nativeModules`. */ + fun toExtensionModuleSources(): List = + nativeModules.map { it.toExtensionModuleSource() } + + /** Maps [notNativeModules] onto pypackpack's `BundleRequest.notNativeModules`. */ + fun toNotNativeModules(): List = notNativeModules +} + +/** + * Parses a `typedpython-emit-c` v1 JSON manifest (`docs/design/typedpython.md` section 4.3.2a) + * into [TypedPythonEmitCManifest]. + * + * Unknown top-level or entry keys are ignored as specified in section 4.3.2a ("adding keys does not + * bump the version"). Unknown `format`, `version` or `notNativeModules[i].kind` fails with + * [IllegalArgumentException]. + */ +fun parseTypedPythonEmitCManifest(json: String): TypedPythonEmitCManifest { + val root = try { + JsonSlurper().parseText(json) + } catch (e: Exception) { + throw IllegalArgumentException("typedpython emit-c manifest is not valid JSON: ${e.message}", e) + } as? Map<*, *> ?: throw IllegalArgumentException("typedpython emit-c manifest is not a JSON object.") + + val format = root["format"] as? String + require(format == TYPEDPYTHON_EMIT_C_FORMAT) { + "typedpython emit-c manifest has format '$format'; expected '$TYPEDPYTHON_EMIT_C_FORMAT'." + } + val version = (root["version"] as? Number)?.toInt() + require(version == TYPEDPYTHON_EMIT_C_VERSION) { + "typedpython emit-c manifest has version '${root["version"]}'; expected version $TYPEDPYTHON_EMIT_C_VERSION." + } + + fun requireString(map: Map<*, *>, key: String, owner: String): String = + (map[key] as? String)?.takeIf { it.isNotEmpty() } + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner.$key' is missing or empty.") + + fun parseSkippedItems(rawList: Any?, owner: String): List { + val list = rawList as? List<*> + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner' must be a list.") + return list.mapIndexed { idx, elem -> + val item = elem as? Map<*, *> + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner[$idx]' is not an object.") + val lineVal = item["line"] + val line = when (lineVal) { + null -> null + is Number -> lineVal.toInt() + else -> throw IllegalArgumentException( + "typedpython emit-c manifest: '$owner[$idx].line' must be an integer or null, got '$lineVal'.", + ) + } + SkippedItem( + name = requireString(item, "name", "$owner[$idx]"), + file = requireString(item, "file", "$owner[$idx]"), + line = line, + message = (item["message"] as? String) + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner[$idx].message' is missing."), + ) + } + } + + val rawNative = root["nativeModules"] as? List<*> + ?: throw IllegalArgumentException("typedpython emit-c manifest: 'nativeModules' is missing or not a list.") + val nativeModules = rawNative.mapIndexed { idx, elem -> + val owner = "nativeModules[$idx]" + val map = elem as? Map<*, *> + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner' is not an object.") + val sourcesRaw = map["sources"] as? List<*> + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner.sources' is missing or not a list.") + val sources = sourcesRaw.map { path -> + val str = (path as? String)?.takeIf { it.isNotEmpty() } + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner.sources' contains a non-string entry.") + File(str) + } + val includeDirsRaw = map["includeDirs"] as? List<*> + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner.includeDirs' is missing or not a list.") + val includeDirs = includeDirsRaw.map { path -> + val str = (path as? String)?.takeIf { it.isNotEmpty() } + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner.includeDirs' contains a non-string entry.") + File(str) + } + val definesRaw = map["defines"] as? Map<*, *> + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner.defines' is missing or not an object.") + val defines = definesRaw.entries.associate { (k, v) -> + val key = (k as? String)?.takeIf { it.isNotEmpty() } + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner.defines' has a blank key.") + val value = when (v) { + null -> null + is String -> v + else -> v.toString() + } + key to value + } + TypedPythonNativeModuleEntry( + moduleName = requireString(map, "moduleName", owner), + file = requireString(map, "file", owner), + sources = sources, + includeDirs = includeDirs, + defines = defines, + status = requireString(map, "status", owner), + skippedItems = parseSkippedItems(map["skippedItems"] ?: emptyList(), "$owner.skippedItems"), + ) + } + + val rawNotNative = root["notNativeModules"] as? List<*> + ?: throw IllegalArgumentException("typedpython emit-c manifest: 'notNativeModules' is missing or not a list.") + val notNativeModules = rawNotNative.mapIndexed { idx, elem -> + val owner = "notNativeModules[$idx]" + val map = elem as? Map<*, *> + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner' is not an object.") + val kindRaw = requireString(map, "kind", owner) + val kind = try { + NotNativeModule.Kind.valueOf(kindRaw) + } catch (e: IllegalArgumentException) { + throw IllegalArgumentException( + "typedpython emit-c manifest: '$owner.kind' has unknown kind '$kindRaw'; " + + "expected one of ${NotNativeModule.Kind.values().joinToString()}.", + e, + ) + } + NotNativeModule( + module = requireString(map, "module", owner), + kind = kind, + items = parseSkippedItems(map["items"] ?: emptyList(), "$owner.items"), + ) + } + + return TypedPythonEmitCManifest( + format = format, + version = version, + compilerVersion = requireString(root, "compilerVersion", "root"), + projectRoot = File(requireString(root, "projectRoot", "root")), + runtimeInclude = File(requireString(root, "runtimeInclude", "root")), + nativeModules = nativeModules, + notNativeModules = notNativeModules, + ) +} + +/** + * Builds the `python -m typedpython emit-c` command line (`docs/design/typedpython.md` section 4.3.2a). + * Input files are sorted by their path relative to [rootDir] so the manifest module order is deterministic. + */ +fun typedPythonEmitCCommand( + pythonExecutable: String, + files: List, + rootDir: File, + cacheDir: File, + manifestFile: File, +): List { + val rootAbs = rootDir.absoluteFile + val sorted = files.map { it.absoluteFile }.sortedBy { file -> + file.relativeToOrSelf(rootAbs).invariantSeparatorsPath + } + return buildList { + add(pythonExecutable) + add("-m") + add("typedpython") + add("emit-c") + sorted.forEach { add(it.absolutePath) } + add("--root") + add(rootAbs.absolutePath) + add("--cache") + add(cacheDir.absoluteFile.absolutePath) + add("--manifest") + add(manifestFile.absoluteFile.absolutePath) + } +} + +/** + * SPEC N-10 (`python-multiplatform#25` step 2): runs `python -m typedpython emit-c` over [sources] + * and writes the per-module C cache ([cacheDirectory]) and the v1 JSON manifest ([manifestFile]) + * for pypackpack's native compile slot. + */ +abstract class TypedPythonEmitCTask : DefaultTask() { + + /** The developer `.py` files to compile to C. */ + @get:InputFiles + @get:SkipWhenEmpty + @get:IgnoreEmptyDirectories + @get:PathSensitive(PathSensitivity.RELATIVE) + abstract val sources: ConfigurableFileCollection + + /** + * The directory module names are derived relative to (`--root`, the ppp package's `python/` + * or `src/main` root). Fingerprinted through [sources]'s `PathSensitivity.RELATIVE`. + */ + @get:Internal + abstract val rootDirectory: DirectoryProperty + + /** The Python interpreter executable that runs `-m typedpython emit-c` (defaults to `"python3"`). */ + @get:Input + abstract val pythonExecutable: Property + + /** Optional `PYTHONPATH` entries for locating the `typedpython` package when not installed in the interpreter. */ + @get:InputFiles + @get:Optional + @get:PathSensitive(PathSensitivity.RELATIVE) + abstract val pythonPath: ConfigurableFileCollection + + /** `generate_project`'s incremental source-to-C cache directory (`--cache`). */ + @get:OutputDirectory + abstract val cacheDirectory: DirectoryProperty + + /** The JSON manifest written by `emit-c` (`--manifest`). */ + @get:OutputFile + abstract val manifestFile: RegularFileProperty + + init { + pythonExecutable.convention("python3") + } + + /** Parses the manifest written at [manifestFile] into [TypedPythonEmitCManifest]. */ + fun readManifest(): TypedPythonEmitCManifest = + parseTypedPythonEmitCManifest(manifestFile.get().asFile.readText()) + + @TaskAction + fun emitC() { + val pyFiles = sources.files.filter { it.isFile && it.extension == "py" } + val root = rootDirectory.get().asFile + val cache = cacheDirectory.get().asFile + val manifest = manifestFile.get().asFile + val executable = pythonExecutable.getOrElse("python3") + + val command = typedPythonEmitCCommand( + pythonExecutable = executable, + files = pyFiles, + rootDir = root, + cacheDir = cache, + manifestFile = manifest, + ) + logger.info("typedPythonEmitC: ${command.joinToString(" ")}") + + val work = temporaryDir.apply { mkdirs() } + val tmp = File(work, "tmp").apply { mkdirs() } + val stdoutFile = File(work, "emit-c.stdout") + val stderrFile = File(work, "emit-c.stderr") + + val pb = ProcessBuilder(command) + .directory(project.projectDir) + .redirectOutput(stdoutFile) + .redirectError(stderrFile) + pb.environment()["TMPDIR"] = tmp.absolutePath + val extraPythonPath = pythonPath.files.map { it.absolutePath }.filter { it.isNotEmpty() } + if (extraPythonPath.isNotEmpty()) { + val existing = pb.environment()["PYTHONPATH"].orEmpty() + val joined = extraPythonPath.joinToString(File.pathSeparator) + pb.environment()["PYTHONPATH"] = if (existing.isEmpty()) joined else "$joined${File.pathSeparator}$existing" + } + + val exitCode = pb.start().waitFor() + if (exitCode != 0) { + val details = listOf(stderrFile.readText().trim(), stdoutFile.readText().trim()) + .filter { it.isNotEmpty() } + .joinToString("\n") + .ifEmpty { "(no output)" } + throw GradleException("typedpython emit-c failed (exit $exitCode):\n$details") + } + } +} diff --git a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/TypedPythonEmitCTaskTest.kt b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/TypedPythonEmitCTaskTest.kt new file mode 100644 index 00000000..23893acd --- /dev/null +++ b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/TypedPythonEmitCTaskTest.kt @@ -0,0 +1,321 @@ +package python.multiplatform.gradle + +import org.gradle.api.GradleException +import org.gradle.api.tasks.IgnoreEmptyDirectories +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.InputFiles +import org.gradle.api.tasks.Internal +import org.gradle.api.tasks.Optional +import org.gradle.api.tasks.OutputDirectory +import org.gradle.api.tasks.OutputFile +import org.gradle.api.tasks.PathSensitive +import org.gradle.api.tasks.PathSensitivity +import org.gradle.api.tasks.SkipWhenEmpty +import org.gradle.testfixtures.ProjectBuilder +import java.io.File +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertNull +import kotlin.test.assertTrue + +/** + * SPEC N-10 (python-multiplatform#25 step 2): Gradle task running `python -m typedpython emit-c` + * with declared inputs/outputs and mapping the v1 manifest (`docs/design/typedpython.md` section 4.3.2a) + * onto pypackpack's `BundleRequest.nativeModules` (`ExtensionModuleSource`) and + * `BundleRequest.notNativeModules` (`NotNativeModule` / `SkippedItem`). + */ +class TypedPythonEmitCTaskTest { + + private val sampleManifestJson = + """ + { + "format": "typedpython-emit-c", + "version": 1, + "compilerVersion": "abc123def456", + "projectRoot": "/abs/proj", + "runtimeInclude": "/abs/site-packages/typedpython/runtime", + "nativeModules": [ + { + "moduleName": "app.physics.nbody", + "file": "app/physics/nbody.py", + "sources": ["/abs/cache/nbody.c"], + "includeDirs": ["/abs/site-packages/typedpython/runtime"], + "defines": {"TP_FLAG": "1", "TP_BARE": null}, + "status": "rebuilt", + "skippedItems": [ + {"name": "main", "file": "app/physics/nbody.py", "line": 30, "message": "not pure"} + ] + } + ], + "notNativeModules": [ + { + "module": "app.util", + "file": "app/util.py", + "kind": "NOT_MARKED", + "items": [] + }, + { + "module": "app.text", + "file": "app/text.py", + "kind": "NOTHING_COMPILED", + "items": [ + {"name": "f", "file": "app/text.py", "line": 2, "message": "str slice is not supported"}, + {"name": "g", "file": "app/text.py", "line": null, "message": "no line"} + ] + } + ] + } + """.trimIndent() + + @Test + fun parsesManifestV1AndMapsToBundleRequestShapes() { + val manifest = parseTypedPythonEmitCManifest(sampleManifestJson) + assertEquals(TYPEDPYTHON_EMIT_C_FORMAT, manifest.format) + assertEquals(TYPEDPYTHON_EMIT_C_VERSION, manifest.version) + assertEquals("abc123def456", manifest.compilerVersion) + assertEquals(File("/abs/proj"), manifest.projectRoot) + assertEquals(File("/abs/site-packages/typedpython/runtime"), manifest.runtimeInclude) + + assertEquals(1, manifest.nativeModules.size) + val nativeEntry = manifest.nativeModules.single() + assertEquals("app.physics.nbody", nativeEntry.moduleName) + assertEquals("app/physics/nbody.py", nativeEntry.file) + assertEquals(listOf(File("/abs/cache/nbody.c")), nativeEntry.sources) + assertEquals(listOf(File("/abs/site-packages/typedpython/runtime")), nativeEntry.includeDirs) + assertEquals(mapOf("TP_FLAG" to "1", "TP_BARE" to null), nativeEntry.defines) + assertEquals("rebuilt", nativeEntry.status) + assertEquals( + listOf(SkippedItem("main", "app/physics/nbody.py", 30, "not pure")), + nativeEntry.skippedItems, + ) + + val mappedNative = manifest.toExtensionModuleSources() + assertEquals( + listOf( + ExtensionModuleSource( + moduleName = "app.physics.nbody", + sources = listOf(File("/abs/cache/nbody.c")), + includeDirs = listOf(File("/abs/site-packages/typedpython/runtime")), + defines = mapOf("TP_FLAG" to "1", "TP_BARE" to null), + ), + ), + mappedNative, + ) + + val mappedNotNative = manifest.toNotNativeModules() + assertEquals(2, mappedNotNative.size) + assertEquals( + NotNativeModule( + module = "app.util", + kind = NotNativeModule.Kind.NOT_MARKED, + items = emptyList(), + ), + mappedNotNative[0], + ) + assertEquals( + NotNativeModule( + module = "app.text", + kind = NotNativeModule.Kind.NOTHING_COMPILED, + items = listOf( + SkippedItem("f", "app/text.py", 2, "str slice is not supported"), + SkippedItem("g", "app/text.py", null, "no line"), + ), + ), + mappedNotNative[1], + ) + assertNull(mappedNotNative[1].items[1].line) + } + + @Test + fun AdditiveTopLevelOrEntryKeysDoNotBreakVersion1Parsing() { + val withExtraKeys = sampleManifestJson.replace( + "\"version\": 1,", + "\"version\": 1, \"futureTopKey\": 42,", + ) + val manifest = parseTypedPythonEmitCManifest(withExtraKeys) + assertEquals(1, manifest.nativeModules.size) + assertEquals(2, manifest.notNativeModules.size) + } + + @Test + fun rejectsUnknownFormatVersionOrKind() { + val wrongFormat = sampleManifestJson.replace("\"typedpython-emit-c\"", "\"other-format\"") + val formatErr = assertFailsWith { parseTypedPythonEmitCManifest(wrongFormat) } + assertTrue("typedpython-emit-c" in formatErr.message.orEmpty()) + + val wrongVersion = sampleManifestJson.replace("\"version\": 1", "\"version\": 2") + val versionErr = assertFailsWith { parseTypedPythonEmitCManifest(wrongVersion) } + assertTrue("version" in versionErr.message.orEmpty()) + + val wrongKind = sampleManifestJson.replace("\"NOT_MARKED\"", "\"UNKNOWN_KIND\"") + val kindErr = assertFailsWith { parseTypedPythonEmitCManifest(wrongKind) } + assertTrue("UNKNOWN_KIND" in kindErr.message.orEmpty()) + } + + @Test + fun buildsDeterministicCommandLineWithSortedInputFiles() { + val root = File("/proj/src/main") + val cache = File("/proj/build/typedpython/c-cache") + val manifest = File("/proj/build/typedpython/emit-c-manifest.json") + val b = File("/proj/src/main/app/b.py") + val a = File("/proj/src/main/app/a.py") + + val cmd = typedPythonEmitCCommand( + pythonExecutable = "/venv/bin/python3", + files = listOf(b, a), + rootDir = root, + cacheDir = cache, + manifestFile = manifest, + ) + assertEquals( + listOf( + "/venv/bin/python3", + "-m", + "typedpython", + "emit-c", + a.absolutePath, + b.absolutePath, + "--root", + root.absolutePath, + "--cache", + cache.absolutePath, + "--manifest", + manifest.absolutePath, + ), + cmd, + ) + } + + @Test + fun taskDeclaresGradleInputsAndOutputsForIncrementalAndBuildCacheSupport() { + val getters = TypedPythonEmitCTask::class.java.methods.associateBy { it.name } + + val getSources = getters.getValue("getSources") + assertTrue(getSources.isAnnotationPresent(InputFiles::class.java)) + assertTrue(getSources.isAnnotationPresent(SkipWhenEmpty::class.java)) + assertTrue(getSources.isAnnotationPresent(IgnoreEmptyDirectories::class.java)) + assertEquals( + PathSensitivity.RELATIVE, + getSources.getAnnotation(PathSensitive::class.java)?.value, + ) + + assertTrue(getters.getValue("getRootDirectory").isAnnotationPresent(Internal::class.java)) + assertTrue(getters.getValue("getPythonExecutable").isAnnotationPresent(Input::class.java)) + + val getPythonPath = getters.getValue("getPythonPath") + assertTrue(getPythonPath.isAnnotationPresent(InputFiles::class.java)) + assertTrue(getPythonPath.isAnnotationPresent(Optional::class.java)) + assertEquals( + PathSensitivity.RELATIVE, + getPythonPath.getAnnotation(PathSensitive::class.java)?.value, + ) + + assertTrue(getters.getValue("getCacheDirectory").isAnnotationPresent(OutputDirectory::class.java)) + assertTrue(getters.getValue("getManifestFile").isAnnotationPresent(OutputFile::class.java)) + } + + @Test + fun taskActionExecutesStubEmitCAndParsesResultingManifest() { + val project = ProjectBuilder.builder().build() + val pkgRoot = project.file("pkg/src/main").apply { mkdirs() } + val pyFile = File(pkgRoot, "app/fast.py").apply { + parentFile.mkdirs() + writeText("@compiled\ndef inc(n: int) -> int:\n return n + 1\n") + } + val fakeStubRoot = project.file("fake-pythonpath").apply { mkdirs() } + val tpPkg = File(fakeStubRoot, "typedpython").apply { mkdirs() } + val runtimeDir = File(tpPkg, "runtime").apply { mkdirs() } + File(runtimeDir, "tp_runtime.h").writeText("/* header */\n") + File(tpPkg, "__init__.py").writeText("") + File(tpPkg, "__main__.py").writeText( + """ + import argparse, json, sys + from pathlib import Path + p = argparse.ArgumentParser() + p.add_argument("cmd") + p.add_argument("paths", nargs="+") + p.add_argument("--root", required=True) + p.add_argument("--cache", required=True) + p.add_argument("--manifest", required=True) + a = p.parse_args() + cache = Path(a.cache).resolve() + cache.mkdir(parents=True, exist_ok=True) + c_file = cache / "fast.c" + c_file.write_text("/* generated C */\n") + runtime = (Path(__file__).resolve().parent / "runtime").as_posix() + doc = { + "format": "typedpython-emit-c", + "version": 1, + "compilerVersion": "stub-1", + "projectRoot": Path(a.root).resolve().as_posix(), + "runtimeInclude": runtime, + "nativeModules": [{ + "moduleName": "app.fast", + "file": "app/fast.py", + "sources": [c_file.as_posix()], + "includeDirs": [runtime], + "defines": {}, + "status": "rebuilt", + "skippedItems": [], + }], + "notNativeModules": [], + } + mf = Path(a.manifest) + mf.parent.mkdir(parents=True, exist_ok=True) + mf.write_text(json.dumps(doc)) + """.trimIndent() + "\n", + ) + + val task = project.tasks.register("emitTypedPythonC", TypedPythonEmitCTask::class.java) { + sources.from(pyFile) + rootDirectory.set(pkgRoot) + pythonExecutable.set("python3") + pythonPath.from(fakeStubRoot) + cacheDirectory.set(project.layout.buildDirectory.dir("typedpython/c-cache")) + manifestFile.set(project.layout.buildDirectory.file("typedpython/emit-c-manifest.json")) + }.get() + + task.emitC() + + val parsed = task.readManifest() + assertEquals(1, parsed.nativeModules.size) + val mapped = parsed.toExtensionModuleSources().single() + assertEquals("app.fast", mapped.moduleName) + assertTrue(mapped.sources.single().isFile) + assertEquals("fast.c", mapped.sources.single().name) + } + + @Test + fun taskActionFailsWithDiagnosticsOnExit1Or2() { + val project = ProjectBuilder.builder().build() + val pkgRoot = project.file("pkg/src/main").apply { mkdirs() } + val pyFile = File(pkgRoot, "app/broken.py").apply { + parentFile.mkdirs() + writeText("def f(x): return x\n") + } + val fakeStubRoot = project.file("fake-failing-tp").apply { mkdirs() } + val tpPkg = File(fakeStubRoot, "typedpython").apply { mkdirs() } + File(tpPkg, "__init__.py").writeText("") + File(tpPkg, "__main__.py").writeText( + """ + import sys + print("typedpython: broken.py does not pass the gate in compiled mode", file=sys.stderr) + sys.exit(1) + """.trimIndent() + "\n", + ) + + val task = project.tasks.register("emitBroken", TypedPythonEmitCTask::class.java) { + sources.from(pyFile) + rootDirectory.set(pkgRoot) + pythonExecutable.set("python3") + pythonPath.from(fakeStubRoot) + cacheDirectory.set(project.layout.buildDirectory.dir("typedpython/c-cache")) + manifestFile.set(project.layout.buildDirectory.file("typedpython/emit-c-manifest.json")) + }.get() + + val err = assertFailsWith { task.emitC() } + assertTrue("exit 1" in err.message.orEmpty(), err.message) + assertTrue("broken.py does not pass the gate" in err.message.orEmpty(), err.message) + } +}