diff --git a/.github/scripts/ft-mobile/build-android.sh b/.github/scripts/ft-mobile/build-android.sh new file mode 100755 index 000000000..cc5ced80d --- /dev/null +++ b/.github/scripts/ft-mobile/build-android.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Build free-threaded CPython (--disable-gil) for aarch64-linux-android with +# CPython's own tooling (Platforms/Android). Usage: build-android.sh [VERSION] +# Needs ANDROID_HOME (SDK with cmdline-tools/latest) and a JDK 17, 21 or 25. +set -euo pipefail +here="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=common.sh +source "$here/common.sh" + +PY_VERSION="${1:-3.15.0}" +HOST=aarch64-linux-android +: "${ANDROID_HOME:?ANDROID_HOME must point at the Android SDK}" + +src="$(fetch_source "$PY_VERSION")" +cd "$src" + +# `Platforms/Android package` lists the files to ship with `git ls-files`, so +# the unpacked tarball needs an index that tracks Platforms/Android. +if [ ! -d .git ]; then + git init -q . + git add Platforms/Android +fi + +# Install the NDK version the tooling pins (android-env.sh would do it on first +# use; doing it here makes a missing cmdline-tools fail early and visibly). +ndk_version="$(sed -n 's/^ndk_version=//p' Platforms/Android/android-env.sh)" +[ -n "$ndk_version" ] || die "no ndk_version in Platforms/Android/android-env.sh" +log "NDK required by CPython $PY_VERSION: $ndk_version" +if [ ! -e "$ANDROID_HOME/ndk/$ndk_version/package.xml" ]; then + yes | "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" "ndk;$ndk_version" >/dev/null || true +fi +[ -e "$ANDROID_HOME/ndk/$ndk_version/package.xml" ] || die "NDK $ndk_version not installed" + +# --disable-gil goes to configure after `--`, for the build Python and the host. +python3 Platforms/Android configure-build -- --disable-gil +python3 Platforms/Android make-build +python3 Platforms/Android pythoninfo-build +python3 Platforms/Android configure-host "$HOST" -- --disable-gil +python3 Platforms/Android make-host "$HOST" +python3 Platforms/Android package "$HOST" + +# ---- verification ------------------------------------------------------- +minor="$(py_minor "$PY_VERSION")" +archive="$(ls cross-build/$HOST/dist/python-*-$HOST.tar.gz)" +log "package: $archive" + +# The build Python must itself be free-threaded. +cross-build/build/python -c 'import sys, sysconfig; assert sysconfig.get_config_var("Py_GIL_DISABLED") == 1; print("build python free-threaded:", sys.version)' + +check="$FT_WORK/android-check" +rm -rf "$check"; mkdir -p "$check" +tar -xzf "$archive" -C "$check" + +require_gil_disabled "$check/prefix/include/python${minor}t/pyconfig.h" +lib="$(ls "$check"/prefix/lib/libpython${minor}t*.so* 2>/dev/null | head -n 1 || true)" +[ -n "$lib" ] || die "no libpython${minor}t*.so in the package" +log "ok: $lib" +file "$lib" | tee /dev/stderr | grep -q 'ARM aarch64' || die "$lib is not an aarch64 ELF" +[ -d "$check/prefix/lib/python${minor}t" ] || die "no lib/python${minor}t in the package" + +cp "$archive" "$FT_OUT/" +echo "Android archive:" >> "${GITHUB_STEP_SUMMARY:-/dev/null}" +record_sha256 "$FT_OUT/$(basename "$archive")" diff --git a/.github/scripts/ft-mobile/build-ios.sh b/.github/scripts/ft-mobile/build-ios.sh new file mode 100755 index 000000000..db48d0049 --- /dev/null +++ b/.github/scripts/ft-mobile/build-ios.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Build free-threaded CPython (--disable-gil) as an iOS XCframework with +# CPython's own tooling (Platforms/Apple). Usage: build-ios.sh [VERSION] +# Needs macOS with a full Xcode and python3 >= 3.10. +set -euo pipefail +here="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=common.sh +source "$here/common.sh" + +PY_VERSION="${1:-3.15.0}" +# The XCframework needs every slice Platforms/Apple declares: one device host +# and two simulator hosts (arm64, x86_64). +HOSTS=(arm64-apple-ios arm64-apple-ios-simulator x86_64-apple-ios-simulator) + +src="$(fetch_source "$PY_VERSION")" +cd "$src" +xcodebuild -version + +# Platforms/Apple names the standard library directory "python" +# when it packages and in the Xcode build helper, but a free-threaded build +# installs "pythont" (LIBDEST has $(ABI_THREAD)). Without these two +# edits `package` fails copying the xcprivacy files and the helper script would +# not find the standard library. Each edit is asserted so a changed upstream +# script fails loudly instead of silently skipping the patch. +python3 -I - <<'PY' +import pathlib +def patch(path, old, new): + p = pathlib.Path(path) + s = p.read_text() + assert s.count(old) == 1, f"{path}: expected exactly one {old!r}" + p.write_text(s.replace(old, new)) +patch("Platforms/Apple/__main__.py", + 'version_tag = ".".join(version.split(".")[:2])', + 'version_tag = ".".join(version.split(".")[:2]) + "t"') +patch("Platforms/Apple/testbed/Python.xcframework/build/utils.sh", + 'grep -E "^python3\\.\\d+$"', + 'grep -E "^python3\\.[0-9]+t?$"') +PY + +# --disable-gil goes to configure after `--`, for the build Python and each host. +python3 Platforms/Apple configure-build -- --disable-gil +python3 Platforms/Apple make-build +python3 Platforms/Apple pythoninfo-build +for host in "${HOSTS[@]}"; do + python3 Platforms/Apple configure-host iOS "$host" -- --disable-gil + python3 Platforms/Apple make-host iOS "$host" +done +python3 Platforms/Apple package iOS + +# ---- verification ------------------------------------------------------- +minor="$(py_minor "$PY_VERSION")" +archive="$(ls cross-build/dist/python-*-iOS-XCframework.tar.gz)" +log "package: $archive" + +cross-build/build/python -c 'import sys, sysconfig; assert sysconfig.get_config_var("Py_GIL_DISABLED") == 1; print("build python free-threaded:", sys.version)' + +check="$FT_WORK/ios-check" +rm -rf "$check"; mkdir -p "$check" +tar -xzf "$archive" -C "$check" +xcf="$check/Python.xcframework" +[ -d "$xcf" ] || die "no Python.xcframework in the package" + +[ -d "$xcf/lib/python${minor}t" ] || die "no lib/python${minor}t in the XCframework" +slices=0 +for slice in "$xcf"/ios-*; do + slices=$((slices + 1)) + # Headers/pyconfig.h is a wrapper that includes pyconfig-.h. + found=0 + for h in "$slice"/Python.framework/Headers/pyconfig-*.h; do + require_gil_disabled "$h"; found=$((found + 1)) + done + [ "$found" -gt 0 ] || die "no pyconfig-.h in $slice" + [ -e "$slice/lib/libpython${minor}t.dylib" ] || die "no libpython${minor}t.dylib in $slice" + lipo -info "$slice/Python.framework/Python" +done +[ "$slices" -eq 2 ] || die "expected 2 slices (device, simulator), found $slices" + +cp "$archive" "$FT_OUT/" +echo "iOS archive:" >> "${GITHUB_STEP_SUMMARY:-/dev/null}" +record_sha256 "$FT_OUT/$(basename "$archive")" diff --git a/.github/scripts/ft-mobile/common.sh b/.github/scripts/ft-mobile/common.sh new file mode 100755 index 000000000..74ddd4035 --- /dev/null +++ b/.github/scripts/ft-mobile/common.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Helpers shared by build-android.sh and build-ios.sh. Source this file, do not run it. + +# FT_WORK: scratch directory (source tree and cross-build output). +# FT_OUT : directory that receives the archives the workflow uploads. +FT_WORK="${FT_WORK:-$PWD/.tmp/ft-mobile}" +FT_OUT="${FT_OUT:-$PWD/.tmp/ft-mobile-out}" + +log() { echo "[ft-mobile] $*" >&2; } +die() { echo "[ft-mobile] ERROR: $*" >&2; exit 1; } + +# "3.15.0" -> "3.15" (also "3.15.0rc1" -> "3.15"). +py_minor() { echo "$1" | sed -E 's/^([0-9]+\.[0-9]+).*/\1/'; } + +# Download and unpack the CPython source tarball; print the source directory. +# CPython's mobile tooling needs the directory to be a git work tree only for +# `Platforms/Android package` (it runs `git ls-files` there), so the caller +# decides whether to `git init`. +fetch_source() { + local version="$1" + local dirver="${version%%[a-z]*}" # the ftp directory has no rc/alpha suffix + local tgz="$FT_WORK/Python-$version.tgz" + mkdir -p "$FT_WORK" "$FT_OUT" + if [ ! -f "$tgz" ]; then + log "downloading CPython $version" + curl -fL --retry 5 --retry-all-errors -o "$tgz" \ + "https://www.python.org/ftp/python/$dirver/Python-$version.tgz" + fi + if [ ! -d "$FT_WORK/Python-$version" ]; then + tar -xzf "$tgz" -C "$FT_WORK" + fi + echo "$FT_WORK/Python-$version" +} + +# Fail unless FILE contains a line matching `#define Py_GIL_DISABLED 1`. +require_gil_disabled() { + local file="$1" + grep -Eq '^#define Py_GIL_DISABLED 1$' "$file" \ + || die "$file does not define Py_GIL_DISABLED 1" + log "ok: Py_GIL_DISABLED 1 in $file" +} + +# Print "sha256 name" lines for the given files, to stdout and the job summary. +record_sha256() { + local f line + for f in "$@"; do + line="$(cd "$(dirname "$f")" && shasum -a 256 "$(basename "$f")")" + echo "$line" + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + echo "$line" >> "$GITHUB_STEP_SUMMARY" + fi + done +} diff --git a/.github/workflows/build-ft-mobile.yml b/.github/workflows/build-ft-mobile.yml new file mode 100644 index 000000000..d33bd5067 --- /dev/null +++ b/.github/workflows/build-ft-mobile.yml @@ -0,0 +1,80 @@ +name: Build free-threaded CPython for mobile + +# Builds CPython with --disable-gil ("3.15t") for Android arm64 and iOS with +# CPython's own tooling (Platforms/Android, Platforms/Apple). Manual only: the +# builds take a long time and the archives are inputs to later work (#163). +# See docs/platforms/free-threaded-mobile.md. + +on: + # Runs on develop when the build itself changes (workflow_dispatch needs the file on the + # default branch, main, which develop reaches only through a release). + push: + branches: [develop] + paths: + - ".github/workflows/build-ft-mobile.yml" + - ".github/scripts/ft-mobile/**" + workflow_dispatch: + inputs: + python_version: + description: CPython version to build (a python.org source release) + default: "3.15.0" + required: true + type: string + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + android: + name: Android aarch64 (free-threaded) + runs-on: ubuntu-24.04 + timeout-minutes: 120 + env: + FT_WORK: ${{ github.workspace }}/.tmp/ft-mobile + FT_OUT: ${{ github.workspace }}/.tmp/ft-mobile-out + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "21" + + - name: Install build Python prerequisites + run: > + sudo apt-get update && sudo apt-get install -y + build-essential pkg-config libssl-dev zlib1g-dev libffi-dev + libbz2-dev liblzma-dev libsqlite3-dev libzstd-dev + + - name: Build + run: .github/scripts/ft-mobile/build-android.sh "${{ inputs.python_version || '3.15.0' }}" + + - uses: actions/upload-artifact@v4 + with: + name: python-${{ inputs.python_version || '3.15.0' }}t-android-aarch64 + path: ${{ env.FT_OUT }}/*.tar.gz + if-no-files-found: error + + ios: + name: iOS XCframework (free-threaded) + runs-on: macos-15 + timeout-minutes: 180 + env: + FT_WORK: ${{ github.workspace }}/.tmp/ft-mobile + FT_OUT: ${{ github.workspace }}/.tmp/ft-mobile-out + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Build + run: .github/scripts/ft-mobile/build-ios.sh "${{ inputs.python_version || '3.15.0' }}" + + - uses: actions/upload-artifact@v4 + with: + name: python-${{ inputs.python_version || '3.15.0' }}t-ios-xcframework + path: ${{ env.FT_OUT }}/*.tar.gz + if-no-files-found: error diff --git a/docs/platforms/free-threaded-mobile.md b/docs/platforms/free-threaded-mobile.md new file mode 100644 index 000000000..8146231ae --- /dev/null +++ b/docs/platforms/free-threaded-mobile.md @@ -0,0 +1,120 @@ +# Free-threaded CPython for Android and iOS + +Status: build recipe written from the CPython 3.15.0 source; **not yet run in CI, +not yet run on a device or emulator.** Tracking issue: #163. + +python-multiplatform gets its parallelism from free-threaded CPython (decision 8 in +`AGENTS.md`). Desktop uses the python.org 3.14t builds. No prebuilt 3.15t exists for +Android or iOS, so the workflow `.github/workflows/build-ft-mobile.yml` builds it with +CPython's own mobile tooling. + +## What is built + +| Platform | Host | Tooling | Output | +|---|---|---|---| +| Android | `aarch64-linux-android` | `Platforms/Android` | `python-3.15.0-aarch64-linux-android.tar.gz` | +| iOS | `arm64-apple-ios` (device), `arm64-apple-ios-simulator`, `x86_64-apple-ios-simulator` | `Platforms/Apple` | `python-3.15.0-iOS-XCframework.tar.gz` | + +The iOS tooling only builds an XCframework from all of its slices, so the simulator +hosts are built as well as the device. There is no cheaper arm64-only path in the +script. + +## How `--disable-gil` is passed + +Both scripts accept extra `configure` arguments after `--` +(`Platforms/Android/__main__.py`: `"args", nargs="*", help="Extra arguments to pass to +`configure`"`; `Platforms/Apple/__main__.py`: `args", nargs="*", help="Extra arguments +to pass to `configure`"`). The workflow uses the discrete steps so that the flag reaches +both the build Python and every host: + +```sh +python3 Platforms/Android configure-build -- --disable-gil +python3 Platforms/Android make-build +python3 Platforms/Android configure-host aarch64-linux-android -- --disable-gil +python3 Platforms/Android make-host aarch64-linux-android +python3 Platforms/Android package aarch64-linux-android + +python3 Platforms/Apple configure-build -- --disable-gil +python3 Platforms/Apple make-build +python3 Platforms/Apple configure-host iOS -- --disable-gil # per host +python3 Platforms/Apple make-host iOS +python3 Platforms/Apple package iOS +``` + +`configure.ac` turns the flag into `#define Py_GIL_DISABLED 1` and appends `t` to +`ABIFLAGS`, so the library, include directory and standard library directory become +`libpython3.15t`, `include/python3.15t` and `lib/python3.15t`. `--disable-gil` needs +mimalloc, which is on by default. + +The build Python is also built with `--disable-gil`. The cross-build only requires the +same major.minor version (`--with-build-python`), so a GIL build Python would probably +work too, but matching them removes a variable. + +## Scripts + +`.github/scripts/ft-mobile/`: + +- `common.sh`: source download, `Py_GIL_DISABLED` check, sha256 recording. +- `build-android.sh [VERSION]`: unpacks `Python-.tgz`, installs the NDK the + tooling pins (`ndk_version=` in `Platforms/Android/android-env.sh`, 27.3.13750724 in + 3.15.0), runs the steps above, then verifies the package. +- `build-ios.sh [VERSION]`: same for iOS, with the two edits below. + +Local use is possible on a Linux or macOS machine with the prerequisites, but the heavy +build is meant for CI (`AGENTS.md` rule 10). Both scripts honour `FT_WORK` and `FT_OUT` +(default under `.tmp/`). + +### Android notes + +- `Platforms/Android package` lists files with `git ls-files`, which does not work in an + unpacked tarball. `build-android.sh` runs `git init` and `git add Platforms/Android` + in the source tree first. +- The package contains `prefix/include/python3.15t/` and `prefix/lib/libpython3.15t.so` + plus `lib/python3.15t/`, OpenSSL and SQLite libraries, and the `android.py` helper. +- Dependencies (OpenSSL, SQLite, libffi, xz, bzip2, zstd) come from the + `beeware/cpython-android-source-deps` releases, as in upstream. + +### iOS notes + +- Upstream `Platforms/Apple/__main__.py` computes `version_tag` as `3.15`, but a + free-threaded framework installs `lib/python3.15t`. Unpatched, `package` fails when it + copies the `.xcprivacy` files into `lib-/python3.15/lib-dynload`. `build-ios.sh` + appends `t` to `version_tag`; this also names the symlink `lib/libpython3.15t.dylib`. +- `testbed/Python.xcframework/build/utils.sh` finds the standard library with + `grep -E "^python3\.\d+$"`, which does not match `python3.15t`. `build-ios.sh` changes + it to `^python3\.[0-9]+t?$`. Each edit is asserted, so an upstream change fails the + build instead of skipping the patch. +- The framework and its binary are still named `Python` / `Python.framework/Python`; + only the `lib` side carries the `t`. The `Py_GIL_DISABLED` define is in + `Headers/pyconfig-.h`; `Headers/pyconfig.h` is a wrapper that includes it. +- Dependencies come from `beeware/cpython-apple-source-deps` releases. The runner needs + a full Xcode (`macos-15`); the Xcode version is printed in the log. + +## Verification built into the workflow + +Each script fails the job unless: + +- the build Python reports `Py_GIL_DISABLED == 1`; +- Android: `include/python3.15t/pyconfig.h` has `#define Py_GIL_DISABLED 1`, + `lib/libpython3.15t*.so` exists and is an aarch64 ELF, `lib/python3.15t` exists; +- iOS: every slice has `Headers/pyconfig-.h` with `#define Py_GIL_DISABLED 1`, + `lib/libpython3.15t.dylib`, and a `Python.framework/Python` binary (`lipo -info` + is logged); the XCframework has `lib/python3.15t` and exactly two slices. + +The sha256 of each archive is written to the job summary and the archives are uploaded +as workflow artifacts (`python-t-android-aarch64`, +`python-t-ios-xcframework`). + +## Not verified + +- The workflow has not run yet. Whether 3.15.0 configures and compiles with + `--disable-gil` for these targets is unknown; free-threaded support on Android and iOS + is not a documented CPython target. +- Nothing has been loaded on a device, emulator or simulator. No import of `_ssl`, + `_ctypes` or extension modules, no GIL-off test run, no thread-safety check. +- Whether the prebuilt dependency tarballs and the Xcode on `macos-15` suit 3.15.0 is + unconfirmed. +- Integration with the Gradle plugin's `PYTHONHOME` staging, with the Android + `libpython` loading code, and with the iOS framework embedding (which expects + `Python.framework` and a non-`t` library name in places) is not done. +- The x86_64 simulator slice is built only because the packaging step requires it.