Feat: add macosArm64 and linuxX64 Kotlin/Native desktop targets (#191)

.github/workflows/desktop.yml

@@ -51,6 +51,31 @@ jobs:
         # Using --console=plain so logs are readable in GitHub Actions
         run: ./gradlew :python-multiplatform:desktopTest --console=plain
 
+  desktop-native-linux:
+    name: Desktop Kotlin/Native Tests (Linux x86_64)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Java
+        uses: actions/setup-java@v4
+        with:
+          java-version: '21'
+          distribution: 'zulu'
+          cache: 'gradle'
+
+      - name: Cache Konan
+        uses: actions/cache@v4
+        with:
+          path: ~/.konan
+          key: ${{ runner.os }}-konan-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
+          restore-keys: |
+            ${{ runner.os }}-konan-
+
+      - name: Run Linux x86_64 Kotlin/Native Tests
+        run: ./gradlew :python-multiplatform:linuxX64Test --rerun --console=plain
+
   # The dev agent's protocol on the other desktop OSes (#238). Its socket options differ per platform
   # (SO_EXCLUSIVEADDRUSE on Windows, SO_REUSEADDR on POSIX), and the port-takeover and restart tests are
   # what check them. Stdlib only: no JDK, no Gradle.

docs/SPEC.md

@@ -26,12 +26,13 @@ fail, then implement.
 | Platform | Kotlin target(s) | Downcall mechanism | Upcall entry | Test path |
 |---|---|---|---|---|
 | Desktop JVM (macOS; Linux/Windows wired, unverified) | `jvm("desktop")` | Panama FFM, `invokeExact` | FFM upcall stub | `desktopTest → jvmTest → commonTest` |
+| Desktop native (macOS Arm64, Linux x86_64) | `macosArm64`, `linuxX64` | cinterop | `@CName` symbol | `{macosArm64Test, linuxX64Test} → desktopNativeTest → nativeTest → commonTest` |
 | Android (ART) | `androidTarget` | JNI via `RegisterNatives` | JNI upcall | `androidInstrumentedTest → jvmTest → commonTest` (device) |
 | iOS | `iosArm64`, `iosSimulatorArm64`, `iosX64` | cinterop | `@CName` symbol | `iosSimulatorArm64Test → nativeTest → commonTest` |
 | Android native | `androidNativeArm64`, `androidNativeX64` | cinterop | `@CName` symbol | `androidNativeArm64Test → artTest → nativeTest → commonTest` (device, via adb) |
 | Web (experimental) | `wasmJs` (Node, browser) | JS / Emscripten CPython | `@WasmExport` | `wasmJsTest` (Node suite; browser subset) |
 
-- **P-1** The library compiles for every target above. `Status: implemented` for desktop, Android,
+- **P-1** The library compiles for every target above. `Status: implemented` for desktop, `macosArm64`, `linuxX64`, Android,
   iosSimulatorArm64, androidNativeArm64 and wasmJs (each has a test source set). `partial` for
   `iosArm64`, `iosX64`, `androidNativeX64` (compile-wired, no tests run there).
 - **P-2** Linux and Windows desktop: the build downloads and links the right CPython archive.

python-multiplatform/build.gradle.kts

@@ -89,6 +89,7 @@ buildkonfig {
 }
 
 val libVersion = pythonVersion.split('.').subList(0, 2).joinToString(".")
+val abiSuffix = if (pythonFreeThreaded) "t" else ""
 
 /**
  * The `ftp/python/<dir>/` directory holding a release, which drops any pre-release suffix:
@@ -1245,7 +1246,8 @@ kotlin {
      */
     listOf(
         iosArm64(), iosSimulatorArm64(), iosX64(),
-        androidNativeArm64(), androidNativeX64()
+        androidNativeArm64(), androidNativeX64(),
+        macosArm64(), linuxX64()
     ).forEach { nativeTarget ->
         nativeTarget.apply {
             val targetABI = when(konanTarget) {
@@ -1254,16 +1256,20 @@ kotlin {
                 IOS_ARM64 -> "ios-arm64"
                 IOS_X64 -> "ios-arm64_x86_64-simulator"
                 IOS_SIMULATOR_ARM64 -> "ios-arm64_x86_64-simulator"
+                MACOS_ARM64 -> "macos-aarch64"
+                LINUX_X64 -> "linux-x86_64"
                 else -> throw RuntimeException("Unsupported ABI: $konanTarget")
             }
             val targetExtractDir = when(konanTarget.family) {
                 Family.ANDROID -> "$extractedDir/android-${if (targetABI == "arm64-v8a") "aarch64" else "x86_64"}/prefix"
                 Family.IOS -> "$extractedDir/ios/Python.xcframework"
+                Family.OSX, Family.LINUX -> "$extractedDir/$targetABI$desktopFlavourSuffix/python"
                 else -> throw RuntimeException("Unsupported target family: ${konanTarget.family}")
             }
             val targetIncludePath = when(konanTarget.family) {
                 Family.ANDROID -> "$targetExtractDir/include/python$libVersion"
                 Family.IOS -> "$targetExtractDir/$targetABI/Python.framework/Headers"
+                Family.OSX, Family.LINUX -> "$targetExtractDir/include/python$libVersion$abiSuffix"
                 else -> throw RuntimeException("Unsupported target family: ${konanTarget.family}")
             }
 
@@ -1331,6 +1337,28 @@ kotlin {
                         "-framework", "Python", "-F$targetExtractDir/$targetABI",
                         "-rpath", "$targetExtractDir/$targetABI"
                     ))
+                } else if (konanTarget.family == Family.OSX) {
+                    getTest(NativeBuildType.DEBUG).linkerOpts.addAll(listOf(
+                        "-L$targetExtractDir/lib",
+                        "-lpython$libVersion$abiSuffix",
+                        "-Wl,-rpath,$targetExtractDir/lib"
+                    ))
+                } else if (konanTarget.family == Family.LINUX) {
+                    // python-build-standalone ships libpythonX.Y.so as a symlink to libpythonX.Y.so.1.0,
+                    // which Gradle's tarTree extracts as a 0-byte file; stageDesktopLibraries resolves
+                    // it into a real libpythonX.Y.so under build/desktop-libs/lib/linux-x86_64.
+                    val stagedLinuxLibDir = layout.buildDirectory.dir("desktop-libs/lib/linux-x86_64").get().asFile.absolutePath
+                    getTest(NativeBuildType.DEBUG).apply {
+                        linkerOpts.addAll(listOf(
+                            "-L$stagedLinuxLibDir",
+                            "-L$targetExtractDir/lib",
+                            "-lpython$libVersion$abiSuffix",
+                            "-Wl,-rpath,$targetExtractDir/lib"
+                        ))
+                        linkTaskProvider.configure {
+                            dependsOn("stageDesktopLibraries")
+                        }
+                    }
                 }
             }
         }
@@ -1426,6 +1454,15 @@ kotlin {
         androidNativeX64Main.dependsOn(artMain)
         androidNativeArm64Main.dependsOn(artMain)
 
+        val desktopNativeMain by creating {
+            kotlin.srcDir("src/desktopNativeMain/kotlin")
+        }
+        val macosArm64Main by getting
+        val linuxX64Main by getting
+        desktopNativeMain.dependsOn(nativeMain)
+        macosArm64Main.dependsOn(desktopNativeMain)
+        linuxX64Main.dependsOn(desktopNativeMain)
+
         // `src/nativeTest` existed as a directory long before this line, and nothing pointed at
         // it -- so its `GCLeakTest.native.kt` was dead source, and the same file had been copied
         // byte-for-byte into all five target test source sets to compensate. This is the mirror of
@@ -1439,6 +1476,13 @@ kotlin {
             "androidNativeX64Test", "androidNativeArm64Test",
         ).forEach { getByName(it).dependsOn(nativeTest) }
 
+        val desktopNativeTest by creating {
+            kotlin.srcDir("src/desktopNativeTest/kotlin")
+        }
+        desktopNativeTest.dependsOn(nativeTest)
+        listOf("macosArm64Test", "linuxX64Test")
+            .forEach { getByName(it).dependsOn(desktopNativeTest) }
+
         // ...and `artTest` completes the mirror for the other half: things true of androidNative
         // but not of iOS. Without it such a test has nowhere to live except duplicated into both
         // androidNative target source sets, which is the arrangement the comment above describes
@@ -1452,6 +1496,18 @@ kotlin {
     }
 }
 
+tasks.named<org.jetbrains.kotlin.gradle.targets.native.tasks.KotlinNativeHostTest>("macosArm64Test") {
+    dependsOn("downloadPython_macos_aarch64")
+    val pythonHome = file("$extractedDir/macos-aarch64$desktopFlavourSuffix/python").absolutePath
+    environment("PYTHONHOME", pythonHome)
+}
+
+tasks.named<org.jetbrains.kotlin.gradle.targets.native.tasks.KotlinNativeHostTest>("linuxX64Test") {
+    dependsOn("downloadPython_linux_x86_64")
+    val pythonHome = file("$extractedDir/linux-x86_64$desktopFlavourSuffix/python").absolutePath
+    environment("PYTHONHOME", pythonHome)
+}
+
 /**
  * The iOS `Python.framework` ships only the interpreter binary and headers, it carries no
  * standard library. `Py_Initialize()` therefore aborts the process with

python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/env/PackagedPythonHome.desktopNative.kt

@@ -0,0 +1,100 @@
+package python.multiplatform.env
+
+import kotlinx.cinterop.ExperimentalForeignApi
+import kotlinx.cinterop.alloc
+import kotlinx.cinterop.memScoped
+import kotlinx.cinterop.ptr
+import platform.posix.S_IFDIR
+import platform.posix.S_IFMT
+import platform.posix.stat
+
+/**
+ * The directory containing the running executable, or null when the OS cannot report it.
+ */
+internal expect fun executableDirectory(): String?
+
+/**
+ * Where a desktop Kotlin/Native application (`macosArm64`, `linuxX64`) finds its packaged CPython
+ * prefix (SPEC L-9 / P-1, #191).
+ *
+ * A desktop native binary has no JVM, no system properties and no launcher to set `PYTHONHOME`:
+ * like a GraalVM native image (#246), its prefix ships beside the executable under
+ * [DIRECTORY_NAME] (or, inside a macOS `.app` bundle, under `../Resources/`[DIRECTORY_NAME]).
+ */
+internal object DesktopNativePythonHome {
+
+    /** Matches `PackagedPythonHome.DIRECTORY_NAME` (`jvmMain`) and `IosPythonHome.DIRECTORY_NAME` (`iosMain`). */
+    const val DIRECTORY_NAME = "python-multiplatform-home"
+
+    /** A resolved prefix and a human-readable account of where it came from, for error messages. */
+    data class Resolved(val path: String, val source: String)
+
+    /** [resolve] against this process's environment and executable directory. */
+    fun resolve(): Resolved? = resolve(
+        environmentHome = readEnvVar("PYTHONHOME"),
+        executableDir = executableDirectory(),
+        isDirectory = ::isDirectory,
+    )
+
+    /**
+     * The packaged prefix, or null when there is none to apply.
+     *
+     * 1. A non-blank `PYTHONHOME` in the environment wins, and the answer is null: CPython reads it
+     *    itself and [PythonHomeCheck] already checked it. This keeps the Gradle host test tasks
+     *    (`macosArm64Test`, `linuxX64Test`) and an explicit caller environment working unchanged.
+     * 2. `<executableDir>/`[DIRECTORY_NAME], when that directory exists.
+     * 3. `<executableDir>/../Resources/`[DIRECTORY_NAME], when that directory exists (a macOS
+     *    `.app` bundle places the binary in `Contents/MacOS/` and resources in `Contents/Resources/`).
+     */
+    fun resolve(
+        environmentHome: String?,
+        executableDir: String?,
+        isDirectory: (String) -> Boolean,
+    ): Resolved? {
+        if (!environmentHome.isNullOrBlank()) return null
+        val dir = executableDir?.takeIf { it.isNotBlank() }?.trimEnd('/') ?: return null
+        val beside = "$dir/$DIRECTORY_NAME"
+        if (isDirectory(beside)) {
+            return Resolved(beside, "$DIRECTORY_NAME beside the executable ($dir)")
+        }
+        val parent = dir.substringBeforeLast('/', missingDelimiterValue = "")
+        if (parent.isNotEmpty()) {
+            val resources = "$parent/Resources/$DIRECTORY_NAME"
+            if (isDirectory(resources)) {
+                return Resolved(resources, "$DIRECTORY_NAME in the app bundle's Resources directory ($parent/Resources)")
+            }
+        }
+        return null
+    }
+
+    /** `stat(2)`'s answer: a directory exists at [path] (a regular file there is not one). */
+    @OptIn(ExperimentalForeignApi::class)
+    internal fun isDirectory(path: String): Boolean = memScoped {
+        val info = alloc<stat>()
+        if (stat(path, info.ptr) != 0) return false
+        (info.st_mode.toInt() and S_IFMT) == S_IFDIR
+    }
+}
+
+/**
+ * Hands a packaged prefix to CPython with `Py_SetPythonHome` before `Py_Initialize()`.
+ *
+ * `Py_DecodeLocale` is documented by CPython as safe to call before initialisation and produces
+ * storage that lives for the duration of the process, as `Py_SetPythonHome` requires.
+ */
+@OptIn(ExperimentalForeignApi::class)
+@Suppress("DEPRECATION")
+internal actual fun applyPackagedPythonHome() {
+    val home = DesktopNativePythonHome.resolve() ?: return
+    try {
+        PythonHomeCheck.verifyOrThrow(home.path)
+    } catch (e: IllegalStateException) {
+        throw IllegalStateException(
+            "The Python home this application was packaged with (${home.source}) is unusable: ${e.message}",
+            e,
+        )
+    }
+    val wide = python.native.ffi.bindings.Py_DecodeLocale(home.path, null)
+        ?: throw IllegalStateException("Py_DecodeLocale could not decode the packaged Python home '${home.path}'.")
+    python.native.ffi.bindings.Py_SetPythonHome(wide)
+}

python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/env/PythonPayload.desktopNative.kt

@@ -0,0 +1,24 @@
+package python.multiplatform.env
+
+/**
+ * Desktop Kotlin/Native's answer to "where did the payload land": `python/` beside the executable
+ * or, in a macOS `.app` bundle, under `../Resources/python`.
+ */
+internal actual fun discoverStagedPayloadRoots(): List<String> =
+    discoverStagedPayloadRoots(executableDirectory(), ::pathIsAccessible)
+
+internal fun discoverStagedPayloadRoots(
+    executableDir: String?,
+    isAccessible: (String) -> Boolean,
+): List<String> {
+    val dir = executableDir?.takeIf { it.isNotBlank() }?.trimEnd('/') ?: return emptyList()
+    val roots = mutableListOf<String>()
+    val beside = "$dir/${PythonPayload.PAYLOAD_ROOT}"
+    if (isAccessible(beside)) roots += beside
+    val parent = dir.substringBeforeLast('/', missingDelimiterValue = "")
+    if (parent.isNotEmpty()) {
+        val resources = "$parent/Resources/${PythonPayload.PAYLOAD_ROOT}"
+        if (resources != beside && isAccessible(resources)) roots += resources
+    }
+    return roots
+}

python-multiplatform/src/desktopNativeTest/kotlin/python/multiplatform/env/DesktopNativePythonHomeTest.kt

@@ -0,0 +1,93 @@
+package python.multiplatform.env
+
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertFalse
+import kotlin.test.assertNotNull
+import kotlin.test.assertNull
+import kotlin.test.assertTrue
+
+/**
+ * Where a desktop Kotlin/Native application (`macosArm64`, `linuxX64`) resolves its packaged
+ * CPython prefix and payload from.
+ */
+class DesktopNativePythonHomeTest {
+
+    private val besideBinary = "/apps/demo/python-multiplatform-home"
+    private val inAppBundle = "/apps/Demo.app/Contents/Resources/python-multiplatform-home"
+
+    @Test
+    fun theDirectoryNameMatchesThePluginAndDesktopAndIos() {
+        assertEquals("python-multiplatform-home", DesktopNativePythonHome.DIRECTORY_NAME)
+    }
+
+    @Test
+    fun anEnvironmentPythonHomeWinsAndNothingIsApplied() {
+        assertNull(
+            DesktopNativePythonHome.resolve(
+                environmentHome = "/from/env",
+                executableDir = "/apps/demo",
+                isDirectory = { true },
+            )
+        )
+    }
+
+    @Test
+    fun thePrefixBesideTheExecutableIsUsedWhenPresent() {
+        for (environment in listOf(null, "", "   ")) {
+            val resolved = DesktopNativePythonHome.resolve(
+                environmentHome = environment,
+                executableDir = "/apps/demo",
+                isDirectory = { it == besideBinary },
+            )
+            assertEquals(besideBinary, resolved?.path, "PYTHONHOME='$environment'")
+            assertTrue("beside the executable" in resolved!!.source, resolved.source)
+        }
+    }
+
+    @Test
+    fun anAppBundleResourceDirectoryIsCheckedWhenNothingSitsBesideTheBinary() {
+        val resolved = DesktopNativePythonHome.resolve(
+            environmentHome = null,
+            executableDir = "/apps/Demo.app/Contents/MacOS",
+            isDirectory = { it == inAppBundle },
+        )
+        assertEquals(inAppBundle, resolved?.path)
+        assertTrue("Resources" in resolved!!.source, resolved.source)
+    }
+
+    @Test
+    fun anUnpackagedExecutableBehavesAsBefore() {
+        assertNull(DesktopNativePythonHome.resolve(null, "/apps/demo") { false })
+        assertNull(DesktopNativePythonHome.resolve(null, null) { true })
+        assertNull(DesktopNativePythonHome.resolve(null, "") { true })
+    }
+
+    @Test
+    fun theHostTestTaskPassesPythonHomeAndExecutableDirectoryResolves() {
+        val home = assertNotNull(readEnvVar("PYTHONHOME"), "the host test task must pass PYTHONHOME")
+        assertNull(DesktopNativePythonHome.resolve())
+        val exeDir = assertNotNull(executableDirectory(), "executableDirectory() returned null")
+        assertTrue(DesktopNativePythonHome.isDirectory(exeDir), exeDir)
+        assertTrue(DesktopNativePythonHome.isDirectory(home), home)
+        assertFalse(DesktopNativePythonHome.isDirectory("$home/does-not-exist"))
+    }
+
+    @Test
+    fun payloadDiscoveryFindsBesideExecutableAndBundleResources() {
+        assertEquals(
+            listOf("/apps/demo/python"),
+            discoverStagedPayloadRoots("/apps/demo") { it == "/apps/demo/python" },
+        )
+        assertEquals(
+            listOf("/apps/Demo.app/Contents/Resources/python"),
+            discoverStagedPayloadRoots("/apps/Demo.app/Contents/MacOS") {
+                it == "/apps/Demo.app/Contents/Resources/python"
+            },
+        )
+        assertEquals(
+            emptyList(),
+            discoverStagedPayloadRoots("/apps/demo") { false },
+        )
+    }
+}