From fb5b531c50bff13c2f672e95604cba253d72abf8 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sun, 4 Oct 2026 00:10:52 +0900 Subject: [PATCH 01/29] Bench: Add Cython (pure and typed) and numba columns to the TypedPython benchmarks --- .gitignore | 7 + .../typedpython/cython/binary_trees.pyx | 46 ++ benchmarks/typedpython/cython/build.py | 46 ++ benchmarks/typedpython/cython/fannkuch.pyx | 61 +++ benchmarks/typedpython/cython/nbody.pyx | 80 ++++ .../typedpython/cython/spectral_norm.pyx | 61 +++ benchmarks/typedpython/cython/wordfreq.pyx | 54 +++ benchmarks/typedpython/numba/binary_trees.py | 72 +++ benchmarks/typedpython/numba/fannkuch.py | 62 +++ benchmarks/typedpython/numba/nbody.py | 82 ++++ benchmarks/typedpython/numba/spectral_norm.py | 61 +++ benchmarks/typedpython/numba/wordfreq.py | 69 +++ benchmarks/typedpython/run.py | 436 +++++++++++++++--- 13 files changed, 1083 insertions(+), 54 deletions(-) create mode 100644 benchmarks/typedpython/cython/binary_trees.pyx create mode 100644 benchmarks/typedpython/cython/build.py create mode 100644 benchmarks/typedpython/cython/fannkuch.pyx create mode 100644 benchmarks/typedpython/cython/nbody.pyx create mode 100644 benchmarks/typedpython/cython/spectral_norm.pyx create mode 100644 benchmarks/typedpython/cython/wordfreq.pyx create mode 100644 benchmarks/typedpython/numba/binary_trees.py create mode 100644 benchmarks/typedpython/numba/fannkuch.py create mode 100644 benchmarks/typedpython/numba/nbody.py create mode 100644 benchmarks/typedpython/numba/spectral_norm.py create mode 100644 benchmarks/typedpython/numba/wordfreq.py diff --git a/.gitignore b/.gitignore index 670457d97..c9ffafa6b 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,10 @@ google-services.json # scratch space for agents and scripts; delete when done .tmp/ + +# TypedPython benchmark harness: uv venv, Cython/numba build output and numba caches +benchmarks/typedpython/.venv/ +benchmarks/typedpython/build/ +benchmarks/typedpython/target/ +benchmarks/typedpython/java-out/ +benchmarks/typedpython/tp-build/ diff --git a/benchmarks/typedpython/cython/binary_trees.pyx b/benchmarks/typedpython/cython/binary_trees.pyx new file mode 100644 index 000000000..1f25e4e66 --- /dev/null +++ b/benchmarks/typedpython/cython/binary_trees.pyx @@ -0,0 +1,46 @@ +# cython: language_level=3, boundscheck=False, wraparound=False, cdivision=True, initializedcheck=False +# Same algorithm as py/binary_trees.py: one heap object per node, reference counted, freed by +# refcounting (not an arena). Semantics that differ from CPython: Node is a cdef class (no __dict__, +# attributes not visible from Python), C long counters, recursion uses the C stack (no +# RecursionError), no None checks on typed attributes. +import sys + + +cdef class Node: + cdef Node left + cdef Node right + + def __init__(self, Node left, Node right): + self.left = left + self.right = right + + +cdef Node make(int depth): + if depth > 0: + return Node(make(depth - 1), make(depth - 1)) + return Node(None, None) + + +cdef long check(Node node) noexcept: + cdef Node left = node.left + cdef Node right = node.right + if left is None or right is None: + return 1 + return 1 + check(left) + check(right) + + +def main(): + cdef int n = int(sys.argv[1]) + cdef int min_depth = 4 + cdef int max_depth = max(min_depth + 2, n) + cdef int depth, i + cdef long iterations, chk, total = 0 + cdef long stretch = check(make(max_depth + 1)) + cdef Node long_lived = make(max_depth) + for depth in range(min_depth, max_depth + 1, 2): + iterations = 1 << (max_depth - depth + min_depth) + chk = 0 + for i in range(iterations): + chk += check(make(depth)) + total += chk + print(f"{stretch} {total} {check(long_lived)}") diff --git a/benchmarks/typedpython/cython/build.py b/benchmarks/typedpython/cython/build.py new file mode 100644 index 000000000..1ed61b65a --- /dev/null +++ b/benchmarks/typedpython/cython/build.py @@ -0,0 +1,46 @@ +"""Build the Cython columns. Run with the harness venv's python (see ../README in run.py's docstring). + + .venv/bin/python cython/build.py + +Writes /pure/.so (the unmodified py/*.py compiled in pure Python mode, annotations +as written) and /typed/_typed.so (cython/*.pyx). Not part of any timed run. +Flags: -O3 -ffp-contract=off (no fused multiply-add, so float results match CPython bit for bit). +""" +import shutil +import sys +from pathlib import Path + +from Cython.Build import cythonize +from setuptools import Extension, setup + +HERE = Path(__file__).resolve().parent +PY = HERE.parent / "py" +NAMES = {"nbody": "nbody", "spectral_norm": "spectral_norm", "fannkuch": "fannkuch", + "binary_trees": "binary_trees", "wordfreq": "wordfreq"} +CFLAGS = ["-O3", "-ffp-contract=off"] + + +def build(out: Path, kind: str) -> None: + work = out / f"src-{kind}" + shutil.rmtree(work, ignore_errors=True) + work.mkdir(parents=True) + exts = [] + for name in NAMES: + if kind == "pure": + src = work / f"{name}.py" + shutil.copy(PY / f"{name}.py", src) # unmodified copy + mod = name + else: + mod = f"{name}_typed" + src = work / f"{mod}.pyx" + shutil.copy(HERE / f"{name}.pyx", src) + exts.append(Extension(mod, [str(src)], extra_compile_args=CFLAGS)) + ext_modules = cythonize(exts, language_level=3, quiet=True, build_dir=str(work / "c")) + setup(name=f"tp-bench-{kind}", ext_modules=ext_modules, script_name="setup.py", + script_args=["build_ext", "--build-lib", str(out / kind), "--build-temp", str(work / "obj"), "-q"]) + + +if __name__ == "__main__": + target = Path(sys.argv[1]).resolve() + for k in ("pure", "typed"): + build(target, k) diff --git a/benchmarks/typedpython/cython/fannkuch.pyx b/benchmarks/typedpython/cython/fannkuch.pyx new file mode 100644 index 000000000..9927c47fc --- /dev/null +++ b/benchmarks/typedpython/cython/fannkuch.pyx @@ -0,0 +1,61 @@ +# cython: language_level=3, boundscheck=False, wraparound=False, cdivision=True, initializedcheck=False +# Same algorithm as py/fannkuch.py. Semantics that differ from CPython: C int arrays and C long +# counters (wrap instead of growing), no bounds checks, fixed capacity of 16 elements (n > 16 raises). +import sys + + +cdef (long, long) fannkuch(int n) noexcept: + cdef int perm1[16] + cdef int count[16] + cdef int perm[16] + cdef long max_flips = 0, checksum = 0, perm_count = 0, flips + cdef int r = n, i, k, lo, hi, t, p0 + for i in range(n): + perm1[i] = i + count[i] = 0 + while True: + while r != 1: + count[r - 1] = r + r -= 1 + for i in range(n): + perm[i] = perm1[i] + flips = 0 + k = perm[0] + while k != 0: + lo = 0 + hi = k + while lo < hi: + t = perm[lo] + perm[lo] = perm[hi] + perm[hi] = t + lo += 1 + hi -= 1 + flips += 1 + k = perm[0] + if flips > max_flips: + max_flips = flips + if perm_count % 2 == 0: + checksum += flips + else: + checksum -= flips + while True: + if r == n: + return checksum, max_flips + p0 = perm1[0] + for i in range(r): + perm1[i] = perm1[i + 1] + perm1[r] = p0 + count[r] -= 1 + if count[r] > 0: + break + r += 1 + perm_count += 1 + + +def main(): + cdef int n = int(sys.argv[1]) + if n > 16: + raise ValueError("fannkuch.pyx supports n <= 16") + cdef long checksum, max_flips + checksum, max_flips = fannkuch(n) + print(f"{checksum} {max_flips}") diff --git a/benchmarks/typedpython/cython/nbody.pyx b/benchmarks/typedpython/cython/nbody.pyx new file mode 100644 index 000000000..a12fd632a --- /dev/null +++ b/benchmarks/typedpython/cython/nbody.pyx @@ -0,0 +1,80 @@ +# cython: language_level=3, boundscheck=False, wraparound=False, cdivision=True, initializedcheck=False +# Same algorithm as py/nbody.py. Semantics that differ from CPython: float division does not raise +# ZeroDivisionError (cdivision), no bounds checks (arrays are C arrays), loop counters are C long. +from libc.math cimport sqrt +import sys + +cdef double PI = 3.141592653589793 +cdef double SOLAR_MASS = 4 * PI * PI +cdef double DAYS_PER_YEAR = 365.24 + + +cdef void offset_momentum(double* vx, double* vy, double* vz, double* mass) noexcept nogil: + cdef double px = 0.0, py = 0.0, pz = 0.0 + cdef int i + for i in range(5): + px += vx[i] * mass[i] + py += vy[i] * mass[i] + pz += vz[i] * mass[i] + vx[0] = -px / SOLAR_MASS + vy[0] = -py / SOLAR_MASS + vz[0] = -pz / SOLAR_MASS + + +cdef void advance(double dt, long steps, double* x, double* y, double* z, + double* vx, double* vy, double* vz, double* mass) noexcept nogil: + cdef long s + cdef int i, j + cdef double dx, dy, dz, d2, mag + for s in range(steps): + for i in range(5): + for j in range(i + 1, 5): + dx = x[i] - x[j] + dy = y[i] - y[j] + dz = z[i] - z[j] + d2 = dx * dx + dy * dy + dz * dz + mag = dt / (d2 * sqrt(d2)) + vx[i] -= dx * mass[j] * mag + vy[i] -= dy * mass[j] * mag + vz[i] -= dz * mass[j] * mag + vx[j] += dx * mass[i] * mag + vy[j] += dy * mass[i] * mag + vz[j] += dz * mass[i] * mag + for i in range(5): + x[i] += dt * vx[i] + y[i] += dt * vy[i] + z[i] += dt * vz[i] + + +cdef double energy(double* x, double* y, double* z, double* vx, double* vy, double* vz, + double* mass) noexcept nogil: + cdef double e = 0.0, dx, dy, dz + cdef int i, j + for i in range(5): + e += 0.5 * mass[i] * (vx[i] * vx[i] + vy[i] * vy[i] + vz[i] * vz[i]) + for j in range(i + 1, 5): + dx = x[i] - x[j] + dy = y[i] - y[j] + dz = z[i] - z[j] + e -= mass[i] * mass[j] / sqrt(dx * dx + dy * dy + dz * dz) + return e + + +def main(): + cdef long n = int(sys.argv[1]) + cdef double[5] x = [0.0, 4.84143144246472090e+00, 8.34336671824457987e+00, 1.28943695621391310e+01, 1.53796971148509165e+01] + cdef double[5] y = [0.0, -1.16032004402742839e+00, 4.12479856412430479e+00, -1.51111514016986312e+01, -2.59193146099879641e+01] + cdef double[5] z = [0.0, -1.03622044471123109e-01, -4.03523417114321381e-01, -2.23307578892655734e-01, 1.79258772950371181e-01] + cdef double[5] vx = [0.0, 1.66007664274403694e-03 * DAYS_PER_YEAR, -2.76742510726862411e-03 * DAYS_PER_YEAR, + 2.96460137564761618e-03 * DAYS_PER_YEAR, 2.68067772490389322e-03 * DAYS_PER_YEAR] + cdef double[5] vy = [0.0, 7.69901118419740425e-03 * DAYS_PER_YEAR, 4.99852801234917238e-03 * DAYS_PER_YEAR, + 2.37847173959480950e-03 * DAYS_PER_YEAR, 1.62824170038242295e-03 * DAYS_PER_YEAR] + cdef double[5] vz = [0.0, -6.90460016972063023e-05 * DAYS_PER_YEAR, 2.30417297573763929e-05 * DAYS_PER_YEAR, + -2.96589568540237556e-05 * DAYS_PER_YEAR, -9.51592254519715870e-05 * DAYS_PER_YEAR] + cdef double[5] mass = [SOLAR_MASS, 9.54791938424326609e-04 * SOLAR_MASS, 2.85885980666130812e-04 * SOLAR_MASS, + 4.36624404335156298e-05 * SOLAR_MASS, 5.15138902046611451e-05 * SOLAR_MASS] + offset_momentum(vx, vy, vz, mass) + cdef double before = energy(x, y, z, vx, vy, vz, mass) + advance(0.01, n, x, y, z, vx, vy, vz, mass) + cdef double after = energy(x, y, z, vx, vy, vz, mass) + print(f"{before:.9f} {after:.9f}") diff --git a/benchmarks/typedpython/cython/spectral_norm.pyx b/benchmarks/typedpython/cython/spectral_norm.pyx new file mode 100644 index 000000000..0b73cd8bc --- /dev/null +++ b/benchmarks/typedpython/cython/spectral_norm.pyx @@ -0,0 +1,61 @@ +# cython: language_level=3, boundscheck=False, wraparound=False, cdivision=True, initializedcheck=False +# Same algorithm as py/spectral_norm.py. Semantics that differ from CPython: loop and index +# arithmetic is C long (wraps instead of growing; not reachable at these sizes), cdivision (no +# ZeroDivisionError), no bounds checks (malloc'd C arrays). +from libc.math cimport sqrt +from libc.stdlib cimport malloc, free +import sys + + +cdef inline double eval_a(long i, long j) noexcept nogil: + return 1.0 / ((i + j) * (i + j + 1) // 2 + i + 1) + + +cdef void mul_av(long n, double* v, double* out) noexcept nogil: + cdef long i, j + cdef double s + for i in range(n): + s = 0.0 + for j in range(n): + s += eval_a(i, j) * v[j] + out[i] = s + + +cdef void mul_atv(long n, double* v, double* out) noexcept nogil: + cdef long i, j + cdef double s + for i in range(n): + s = 0.0 + for j in range(n): + s += eval_a(j, i) * v[j] + out[i] = s + + +cdef void mul_atav(long n, double* v, double* out, double* tmp) noexcept nogil: + mul_av(n, v, tmp) + mul_atv(n, tmp, out) + + +def main(): + cdef long n = int(sys.argv[1]) + cdef long i + cdef double* u = malloc(n * sizeof(double)) + cdef double* v = malloc(n * sizeof(double)) + cdef double* tmp = malloc(n * sizeof(double)) + cdef double vbv = 0.0, vv = 0.0 + try: + for i in range(n): + u[i] = 1.0 + v[i] = 0.0 + tmp[i] = 0.0 + for i in range(10): + mul_atav(n, u, v, tmp) + mul_atav(n, v, u, tmp) + for i in range(n): + vbv += u[i] * v[i] + vv += v[i] * v[i] + finally: + free(u) + free(v) + free(tmp) + print(f"{sqrt(vbv / vv):.9f}") diff --git a/benchmarks/typedpython/cython/wordfreq.pyx b/benchmarks/typedpython/cython/wordfreq.pyx new file mode 100644 index 000000000..2b1734a91 --- /dev/null +++ b/benchmarks/typedpython/cython/wordfreq.pyx @@ -0,0 +1,54 @@ +# cython: language_level=3, boundscheck=False, wraparound=False, cdivision=True, initializedcheck=False +# Same algorithm as py/wordfreq.py: the counts stay a Python dict[str, int] (the algorithm's data +# structure), only the arithmetic is typed and words are built in a C buffer. Semantics that differ +# from CPython: C long arithmetic (wraps; the LCG product stays below 2**47 and the weighted sum +# below 2**63 at benchmark size), cdivision. +import sys + +cdef long VOCAB = 50000 +cdef long MOD = 2147483647 + + +cdef inline long next_state(long s) noexcept nogil: + return (s * 48271) % MOD + + +cdef str word_of(long word_id): + cdef char buf[32] + cdef int pos = 32 + cdef long x = word_id + 17576 + while x > 0: + pos -= 1 + buf[pos] = (97 + x % 26) + x //= 26 + return (buf[pos:32]).decode("ascii") + + +def main(): + cdef long n = int(sys.argv[1]) + cdef dict counts = {} + cdef long s = 12345, a, b, i, c, weighted = 0, best_count + cdef str word, w, best_word + cdef list taken = [] + cdef list parts = [] + for i in range(n): + s = next_state(s) + a = s % VOCAB + s = next_state(s) + b = s % VOCAB + word = word_of(a * b // VOCAB) + counts[word] = counts.get(word, 0) + 1 + for w, c in counts.items(): + weighted += c * c + for i in range(5): + best_word = "" + best_count = -1 + for w, c in counts.items(): + if w in taken: + continue + if c > best_count or (c == best_count and w < best_word): + best_word = w + best_count = c + taken.append(best_word) + parts.append(f"{best_word}:{best_count}") + print(f"{len(counts)} {weighted} {','.join(parts)}") diff --git a/benchmarks/typedpython/numba/binary_trees.py b/benchmarks/typedpython/numba/binary_trees.py new file mode 100644 index 000000000..35a4b7fcc --- /dev/null +++ b/benchmarks/typedpython/numba/binary_trees.py @@ -0,0 +1,72 @@ +"""numba version of py/binary_trees.py, using numba.experimental.jitclass. + +Node is a jitclass with a deferred (self-referential) Optional type, the closest numba has to a +heap object per node; allocation and release go through numba's NRT reference counting, like the +CPython program. (An index-based tree in preallocated arrays would be faster but changes the +allocation pattern into an arena, so it is not used.) + +Semantics that differ from CPython: counters are int64, recursion uses the native stack (no +RecursionError). The module must be registered in sys.modules before it runs (run.py does). +""" +from numba import deferred_type, int64, njit, optional +from numba.core.types import UniTuple +from numba.experimental import jitclass + +node_type = deferred_type() + + +@jitclass([("left", optional(node_type)), ("right", optional(node_type))]) +class Node: + def __init__(self, left, right): + self.left = left + self.right = right + + +NodeT = Node.class_type.instance_type +node_type.define(NodeT) + +# cache=True is not usable here: numba raises AttributeError: 'Optional' object has no attribute +# 'type' while hashing the recursive jitclass type for the cache index. So this benchmark has no +# cached run; every process compiles (at import, see below). +OPTS = dict(cache=False) + + +# Explicit signatures: numba cannot infer the return type of the self-recursive make() through +# the jitclass's deferred Optional type (AttributeError: 'Optional' object has no attribute +# 'type'). The consequence is that these two functions compile when this module is imported, not +# on the first call; run.py counts the import time as compile time. +@njit(NodeT(int64), **OPTS) +def make(depth): + if depth > 0: + return Node(make(depth - 1), make(depth - 1)) + return Node(None, None) + + +@njit(int64(optional(NodeT)), **OPTS) +def check(node): + left = node.left + right = node.right + if left is None or right is None: + return 1 + return 1 + check(left) + check(right) + + +@njit(UniTuple(int64, 3)(int64), **OPTS) +def kernel(n): + min_depth = 4 + max_depth = max(min_depth + 2, n) + stretch = check(make(max_depth + 1)) + long_lived = make(max_depth) + total = 0 + for depth in range(min_depth, max_depth + 1, 2): + iterations = 1 << (max_depth - depth + min_depth) + chk = 0 + for _ in range(iterations): + chk += check(make(depth)) + total += chk + return stretch, total, check(long_lived) + + +def run(n: int) -> str: + stretch, total, long_lived = kernel(n) + return f"{stretch} {total} {long_lived}" diff --git a/benchmarks/typedpython/numba/fannkuch.py b/benchmarks/typedpython/numba/fannkuch.py new file mode 100644 index 000000000..5f21f092e --- /dev/null +++ b/benchmarks/typedpython/numba/fannkuch.py @@ -0,0 +1,62 @@ +"""numba @njit version of py/fannkuch.py. `run(n) -> str` is the whole benchmark. + +Semantics that differ from CPython: int64 numpy arrays and int64 counters (wrap instead of +growing), no bounds checks. +""" +import numpy as np +from numba import njit + +OPTS = dict(cache=True, error_model="numpy") + + +@njit(**OPTS) +def fannkuch(n): + perm1 = np.arange(n) + count = np.zeros(n, dtype=np.int64) + perm = np.zeros(n, dtype=np.int64) + max_flips = 0 + checksum = 0 + perm_count = 0 + r = n + while True: + while r != 1: + count[r - 1] = r + r -= 1 + for i in range(n): + perm[i] = perm1[i] + flips = 0 + k = perm[0] + while k != 0: + lo = 0 + hi = k + while lo < hi: + t = perm[lo] + perm[lo] = perm[hi] + perm[hi] = t + lo += 1 + hi -= 1 + flips += 1 + k = perm[0] + if flips > max_flips: + max_flips = flips + if perm_count % 2 == 0: + checksum += flips + else: + checksum -= flips + while True: + if r == n: + return checksum, max_flips + p0 = perm1[0] + for i in range(r): + perm1[i] = perm1[i + 1] + perm1[r] = p0 + count[r] -= 1 + if count[r] > 0: + break + r += 1 + perm_count += 1 + + +def run(n: int) -> str: + checksum, max_flips = fannkuch(n) + return f"{checksum} {max_flips}" diff --git a/benchmarks/typedpython/numba/nbody.py b/benchmarks/typedpython/numba/nbody.py new file mode 100644 index 000000000..74d8b8bee --- /dev/null +++ b/benchmarks/typedpython/numba/nbody.py @@ -0,0 +1,82 @@ +"""numba @njit version of py/nbody.py. Loaded by run.py; `run(n) -> str` is the whole benchmark. + +Semantics that differ from CPython: float64 numpy arrays instead of lists; error_model="numpy" +(float division by zero gives inf instead of raising ZeroDivisionError); no bounds checks. +""" +import math + +import numpy as np +from numba import njit + +PI = 3.141592653589793 +SOLAR_MASS = 4 * PI * PI +DAYS_PER_YEAR = 365.24 +OPTS = dict(cache=True, error_model="numpy") + + +@njit(**OPTS) +def offset_momentum(vx, vy, vz, mass): + px = 0.0 + py = 0.0 + pz = 0.0 + for i in range(5): + px += vx[i] * mass[i] + py += vy[i] * mass[i] + pz += vz[i] * mass[i] + vx[0] = -px / SOLAR_MASS + vy[0] = -py / SOLAR_MASS + vz[0] = -pz / SOLAR_MASS + + +@njit(**OPTS) +def advance(dt, steps, x, y, z, vx, vy, vz, mass): + for _ in range(steps): + for i in range(5): + for j in range(i + 1, 5): + dx = x[i] - x[j] + dy = y[i] - y[j] + dz = z[i] - z[j] + d2 = dx * dx + dy * dy + dz * dz + mag = dt / (d2 * math.sqrt(d2)) + vx[i] -= dx * mass[j] * mag + vy[i] -= dy * mass[j] * mag + vz[i] -= dz * mass[j] * mag + vx[j] += dx * mass[i] * mag + vy[j] += dy * mass[i] * mag + vz[j] += dz * mass[i] * mag + for i in range(5): + x[i] += dt * vx[i] + y[i] += dt * vy[i] + z[i] += dt * vz[i] + + +@njit(**OPTS) +def energy(x, y, z, vx, vy, vz, mass): + e = 0.0 + for i in range(5): + e += 0.5 * mass[i] * (vx[i] * vx[i] + vy[i] * vy[i] + vz[i] * vz[i]) + for j in range(i + 1, 5): + dx = x[i] - x[j] + dy = y[i] - y[j] + dz = z[i] - z[j] + e -= mass[i] * mass[j] / math.sqrt(dx * dx + dy * dy + dz * dz) + return e + + +def run(n: int) -> str: + x = np.array([0.0, 4.84143144246472090e+00, 8.34336671824457987e+00, 1.28943695621391310e+01, 1.53796971148509165e+01]) + y = np.array([0.0, -1.16032004402742839e+00, 4.12479856412430479e+00, -1.51111514016986312e+01, -2.59193146099879641e+01]) + z = np.array([0.0, -1.03622044471123109e-01, -4.03523417114321381e-01, -2.23307578892655734e-01, 1.79258772950371181e-01]) + vx = np.array([0.0, 1.66007664274403694e-03 * DAYS_PER_YEAR, -2.76742510726862411e-03 * DAYS_PER_YEAR, + 2.96460137564761618e-03 * DAYS_PER_YEAR, 2.68067772490389322e-03 * DAYS_PER_YEAR]) + vy = np.array([0.0, 7.69901118419740425e-03 * DAYS_PER_YEAR, 4.99852801234917238e-03 * DAYS_PER_YEAR, + 2.37847173959480950e-03 * DAYS_PER_YEAR, 1.62824170038242295e-03 * DAYS_PER_YEAR]) + vz = np.array([0.0, -6.90460016972063023e-05 * DAYS_PER_YEAR, 2.30417297573763929e-05 * DAYS_PER_YEAR, + -2.96589568540237556e-05 * DAYS_PER_YEAR, -9.51592254519715870e-05 * DAYS_PER_YEAR]) + mass = np.array([SOLAR_MASS, 9.54791938424326609e-04 * SOLAR_MASS, 2.85885980666130812e-04 * SOLAR_MASS, + 4.36624404335156298e-05 * SOLAR_MASS, 5.15138902046611451e-05 * SOLAR_MASS]) + offset_momentum(vx, vy, vz, mass) + before = energy(x, y, z, vx, vy, vz, mass) + advance(0.01, n, x, y, z, vx, vy, vz, mass) + after = energy(x, y, z, vx, vy, vz, mass) + return f"{before:.9f} {after:.9f}" diff --git a/benchmarks/typedpython/numba/spectral_norm.py b/benchmarks/typedpython/numba/spectral_norm.py new file mode 100644 index 000000000..bfa99bae5 --- /dev/null +++ b/benchmarks/typedpython/numba/spectral_norm.py @@ -0,0 +1,61 @@ +"""numba @njit version of py/spectral_norm.py. `run(n) -> str` is the whole benchmark. + +Semantics that differ from CPython: float64 numpy arrays, int64 index arithmetic (wraps), +error_model="numpy" (no ZeroDivisionError), no bounds checks. +""" +import math + +import numpy as np +from numba import njit + +OPTS = dict(cache=True, error_model="numpy") + + +@njit(**OPTS) +def eval_a(i, j): + return 1.0 / ((i + j) * (i + j + 1) // 2 + i + 1) + + +@njit(**OPTS) +def mul_av(n, v, out): + for i in range(n): + s = 0.0 + for j in range(n): + s += eval_a(i, j) * v[j] + out[i] = s + + +@njit(**OPTS) +def mul_atv(n, v, out): + for i in range(n): + s = 0.0 + for j in range(n): + s += eval_a(j, i) * v[j] + out[i] = s + + +@njit(**OPTS) +def mul_atav(n, v, out, tmp): + mul_av(n, v, tmp) + mul_atv(n, tmp, out) + + +@njit(**OPTS) +def kernel(n): + u = np.ones(n) + v = np.zeros(n) + tmp = np.zeros(n) + for _ in range(10): + mul_atav(n, u, v, tmp) + mul_atav(n, v, u, tmp) + vbv = 0.0 + vv = 0.0 + for i in range(n): + vbv += u[i] * v[i] + vv += v[i] * v[i] + return vbv, vv + + +def run(n: int) -> str: + vbv, vv = kernel(n) + return f"{math.sqrt(vbv / vv):.9f}" diff --git a/benchmarks/typedpython/numba/wordfreq.py b/benchmarks/typedpython/numba/wordfreq.py new file mode 100644 index 000000000..5bc8599fb --- /dev/null +++ b/benchmarks/typedpython/numba/wordfreq.py @@ -0,0 +1,69 @@ +"""numba @njit version of py/wordfreq.py: numba.typed.Dict[unicode_type, int64]. + +Semantics that differ from CPython: int64 arithmetic (wraps), words are numba unicode strings, +the counts dict is a numba typed.Dict (hash order differs, but the result does not depend on +iteration order because ties are broken by string comparison). +""" +import numpy as np +from numba import njit, types +from numba.typed import Dict, List + +VOCAB = 50000 +MOD = 2147483647 +OPTS = dict(cache=True, error_model="numpy") + + +@njit(**OPTS) +def next_state(s): + return (s * 48271) % MOD + + +@njit(**OPTS) +def word_of(word_id): + x = word_id + 17576 + w = "" + while x > 0: + w = chr(97 + x % 26) + w + x //= 26 + return w + + +@njit(**OPTS) +def kernel(n): + counts = Dict.empty(key_type=types.unicode_type, value_type=types.int64) + s = 12345 + for _ in range(n): + s = next_state(s) + a = s % VOCAB + s = next_state(s) + b = s % VOCAB + word = word_of(a * b // VOCAB) + if word in counts: + counts[word] += 1 + else: + counts[word] = 1 + weighted = 0 + for w in counts: + c = counts[w] + weighted += c * c + taken = List.empty_list(types.unicode_type) + best_counts = np.zeros(5, dtype=np.int64) + for k in range(5): + best_word = "" + best_count = -1 + for w in counts: + c = counts[w] + if w in taken: + continue + if c > best_count or (c == best_count and w < best_word): + best_word = w + best_count = c + taken.append(best_word) + best_counts[k] = best_count + return len(counts), weighted, taken, best_counts + + +def run(n: int) -> str: + size, weighted, taken, best_counts = kernel(n) + parts = [f"{w}:{c}" for w, c in zip(taken, best_counts)] + return f"{size} {weighted} {','.join(parts)}" diff --git a/benchmarks/typedpython/run.py b/benchmarks/typedpython/run.py index c7fced567..b6ce8a280 100644 --- a/benchmarks/typedpython/run.py +++ b/benchmarks/typedpython/run.py @@ -13,6 +13,27 @@ the `# typedpython: compiled` marker, compiled through `typedpython.pipeline`, and its `main()` is run from the built extension. Without the option the column reports n/a. +Cython and numba columns (issue: compare TypedPython fairly against them). They need a harness venv +that holds Cython, numba and a C compiler (Xcode command line tools), created ONLY with uv: + + uv venv --python 3.14 benchmarks/typedpython/.venv + uv pip install --python benchmarks/typedpython/.venv/bin/python cython setuptools numba + +`--tools-python` (default benchmarks/typedpython/.venv/bin/python) names that interpreter; without it +the three columns report n/a. Columns, each in the form most favourable to it: + "Cython (pure)" py/*.py compiled unmodified by cythonize (annotations as written), same CPython; + "Cython (typed)" cython/*.pyx: cdef types, C arrays, boundscheck/wraparound off, cdef class Node; + "numba" numba/*.py with @njit(cache=True) (binary-trees: jitclass, no cache). The time + reported is the STEADY-STATE call at benchmark size, inside the process, after a + warm-up call that triggered the JIT. The JIT cost is the separate record + "numba compile" (first call: cold cache vs cached; numba import excluded). +Cython is built by cython/build.py before any timing (build/cython/, git-ignored). numba runs in a +fresh process with its own NUMBA_CACHE_DIR (build/numba-cache/, wiped per cold run). +"CPython (tools python)" is the same program under the tools interpreter, and is the baseline for the +Cython/numba ratios in `interleaved` (so those pairs use one interpreter); it reads "same as CPython" +when --python and --tools-python are the same version. Per-column tool version, flags and a semantics +note are written to the JSON ("tools") and printed after the table. + binary-trees declares `__slots__` (TypedPython compiles fixed-layout classes only, SPEC N-11). The speedup is reported against that CPython version; `CPython (no __slots__)` runs the same program with the `__slots__` line removed, so what adding the slots changes in CPython is recorded too. @@ -21,6 +42,7 @@ import datetime import json import os +import shutil import socket import statistics import subprocess @@ -45,7 +67,31 @@ "wordfreq": ("wordfreq.py", "wordfreq.mjs", "Wordfreq", "wordfreq", "100000", "3000000"), } NOSLOTS = "CPython (no __slots__)" -COLUMNS = ["CPython", NOSLOTS, "Node", "Java", "Rust", "TypedPython @compiled"] +CPY_TOOLS = "CPython (tools python)" +CY_PURE = "Cython (pure)" +CY_TYPED = "Cython (typed)" +NUMBA = "numba" +TOOL_COLS = [CY_PURE, CY_TYPED, NUMBA] +COLUMNS = ["CPython", NOSLOTS, CPY_TOOLS, "Node", "Java", "Rust", "TypedPython @compiled", *TOOL_COLS] +TOOLS_PY_DEFAULT = ROOT / ".venv" / "bin" / "python" +CYTHON_OUT = ROOT / "build" / "cython" +NUMBA_CACHE = ROOT / "build" / "numba-cache" +# tiny sizes for numba's warm-up (first) call: the types are the same at any size, so the first +# call is almost all compile time +NUMBA_WARM = {"nbody": "10", "spectral-norm": "4", "fannkuch-redux": "4", "binary-trees": "1", + "wordfreq": "10"} +CYTHON_FLAGS = "cython/build.py: cythonize(language_level=3), cc -O3 -ffp-contract=off (no fused multiply-add)" +SEMANTICS = { + CY_PURE: "py/*.py unmodified; Cython honours the existing annotations (float is a C double, int stays " + "a Python int, list stays a list), so semantics are CPython's apart from float arithmetic on typed locals", + CY_TYPED: "C semantics where declared: C long/int arithmetic wraps instead of growing, cdivision (no " + "ZeroDivisionError), boundscheck/wraparound off on C arrays, cdef class Node (no __dict__), " + "C-stack recursion (no RecursionError); wordfreq keeps a Python dict", + NUMBA: "fixed-width int64/float64 (ints wrap, no Python big ints, no OverflowError), error_model=numpy " + "(no ZeroDivisionError), no bounds checks, numpy arrays for lists, typed.Dict/List for dict/list " + "(wordfreq), jitclass for the tree (binary-trees); reported time is steady state, JIT is separate", +} + SLOTS_LINE = ' __slots__ = ("left", "right")\n' NOSLOTS_BENCHMARKS = {"binary-trees"} NOT_BUILT = "n/a (not built)" @@ -118,8 +164,64 @@ def noslots_source(name: str) -> Path: return out +TOOLS_PROBE = """ +import json, sys +out = {"python": sys.version.split()[0]} +for mod in ("Cython", "numba", "llvmlite", "numpy", "setuptools"): + try: + out[mod] = __import__(mod).__version__ + except Exception as e: + out[mod] = f"missing ({type(e).__name__}: {e})" +print(json.dumps(out)) +""" + + +def _cython_up_to_date() -> bool: + sources = [*(ROOT / "py").glob("*.py"), *(ROOT / "cython").glob("*"), ] + newest = max(p.stat().st_mtime for p in sources) + for name, (py, *_r) in BENCHMARKS.items(): + stem = Path(py).stem + for kind, mod in (("pure", stem), ("typed", stem + "_typed")): + found = list((CYTHON_OUT / kind).glob(f"{mod}.*.so")) + if not found or found[0].stat().st_mtime < newest: + return False + return True + + +def build_tools(tools_python: str | None, python: str) -> dict[str, object]: + """Probe the tools interpreter and cythonize + C-compile the Cython columns (never in a timed run). + Returns {"python", "versions", "na": {column: reason}, "same_python"}.""" + tools: dict[str, object] = {"python": None, "versions": {}, "na": {}, "same_python": True} + na: dict[str, str] = tools["na"] # type: ignore[assignment] + if not tools_python or not Path(tools_python).exists(): + why = ("n/a (no harness venv; create it with: uv venv --python 3.14 benchmarks/typedpython/.venv && " + "uv pip install --python benchmarks/typedpython/.venv/bin/python cython setuptools numba)") + return {**tools, "na": {c: why for c in TOOL_COLS}} + probe = subprocess.run([tools_python, "-c", TOOLS_PROBE], capture_output=True, text=True) + if probe.returncode != 0: + return {**tools, "na": {c: f"n/a (tools python failed: {probe.stderr.strip()[-200:]})" for c in TOOL_COLS}} + versions = json.loads(probe.stdout.strip().splitlines()[-1]) + tools.update(python=tools_python, versions=versions) + base = subprocess.run([python, "-c", "import sys; print(sys.version.split()[0])"], + capture_output=True, text=True).stdout.strip() + tools["same_python"] = base == versions["python"] + tools["baseline_python"] = base + if "missing" in versions["Cython"] or "missing" in versions["setuptools"]: + na[CY_PURE] = na[CY_TYPED] = f"n/a (Cython {versions['Cython']}; setuptools {versions['setuptools']})" + elif not _cython_up_to_date(): + build = subprocess.run([tools_python, str(ROOT / "cython" / "build.py"), str(CYTHON_OUT)], + capture_output=True, text=True) + if build.returncode != 0: + reason = f"n/a (cython build failed: {' | '.join(build.stderr.strip().splitlines()[-3:])})" + na[CY_PURE] = na[CY_TYPED] = reason + if "missing" in versions["numba"] or "missing" in versions["numpy"]: + na[NUMBA] = f"n/a (numba {versions['numba']}; numpy {versions['numpy']})" + return tools + + def commands(name: str, size: str, python: str, java_source: bool, - compiled: dict[str, tuple[str, list[str]] | str] | None = None) -> dict[str, list[str]]: + compiled: dict[str, tuple[str, list[str]] | str] | None = None, + tools: dict[str, object] | None = None) -> dict[str, list[str]]: py, js, cls, rs, _, _ = BENCHMARKS[name] java = ["java", str(ROOT / "java" / f"{cls}.java"), size] if java_source \ else ["java", "-cp", str(JAVA_OUT), cls, size] @@ -134,37 +236,188 @@ def commands(name: str, size: str, python: str, java_source: bool, entry = (compiled or {}).get(name) if isinstance(entry, tuple): cmds["TypedPython @compiled"] = [python, "-c", RUN_SNIPPET, Path(py).stem, entry[0], size] + if tools and tools.get("python"): + tp = str(tools["python"]) + na: dict[str, str] = tools["na"] # type: ignore[assignment] + stem = Path(py).stem + cmds[CPY_TOOLS] = Cmd([tp, str(ROOT / "py" / py), size]) + for col, kind, mod in ((CY_PURE, "pure", stem), (CY_TYPED, "typed", stem + "_typed")): + so = sorted((CYTHON_OUT / kind).glob(f"{mod}.*.so")) + if col not in na and so: + cmd = Cmd([tp, "-c", RUN_SNIPPET, mod, str(so[0]), size]) + cmd.soft = True + cmds[col] = cmd + if NUMBA not in na: + cmd = Cmd([tp, "-c", NUMBA_SNIPPET, str(ROOT / "numba" / py), stem + "_numba", + NUMBA_WARM[name], size]) + cmd.numba = True + cmd.soft = True + cmd.cache_dir = NUMBA_CACHE / name + cmd.env = {"NUMBA_CACHE_DIR": str(cmd.cache_dir)} + cmds[NUMBA] = cmd return cmds -def run_once(cmd: list[str]) -> tuple[float, str]: +class RunFailed(Exception): + """A Cython/numba column failed to run; reported as n/a with the concrete error.""" + + +class Cmd(list): + """argv plus, for the numba column, an environment, a cache directory and the last timing.""" + env: dict[str, str] | None = None + numba = False + cache_dir: Path | None = None + cold = True + last: dict[str, float] | None = None + soft = False + + +NUMBA_SNIPPET = """ +import importlib.util, json, sys, time +import numba, numpy # imported first: import time is not compile time +path, name, warm, size = sys.argv[1:5] +t0 = time.perf_counter() +spec = importlib.util.spec_from_file_location(name, path) +m = importlib.util.module_from_spec(spec) +sys.modules[name] = m # jitclass resolves its module through sys.modules +spec.loader.exec_module(m) # binary-trees (explicit signatures) compiles here +m.run(int(warm)) # first call: JIT compile (or cache load) at a tiny size +t1 = time.perf_counter() +out = m.run(int(size)) # steady state: already compiled +t2 = time.perf_counter() +print(out) +print("#timing " + json.dumps({"first_call_s": t1 - t0, "steady_s": t2 - t1})) +""" + + +def run_once(cmd: list[str], soft: bool = False) -> tuple[float, str]: + """Wall time and stdout. A numba Cmd returns its steady-state seconds instead (see NUMBA_SNIPPET).""" + cache_dir = getattr(cmd, "cache_dir", None) + if cache_dir is not None and getattr(cmd, "cold", True): + shutil.rmtree(cache_dir, ignore_errors=True) + env = dict(os.environ, **cmd.env) if getattr(cmd, "env", None) else None # type: ignore[attr-defined] start = time.perf_counter() - result = subprocess.run(cmd, capture_output=True, text=True) + result = subprocess.run(cmd, capture_output=True, text=True, env=env) elapsed = time.perf_counter() - start if result.returncode != 0: + if soft or getattr(cmd, "soft", False): + last = (result.stderr.strip().splitlines() or ["no stderr"])[-1] + raise RunFailed(f"exit {result.returncode}: {last}") sys.stderr.write(f"failed ({result.returncode}): {' '.join(cmd)}\n{result.stderr}") sys.exit(1) - return elapsed, result.stdout.strip() + out = result.stdout.strip() + if getattr(cmd, "numba", False): + lines = out.splitlines() + timing = json.loads(lines[-1][len("#timing "):]) + cmd.last = dict(timing, wall_s=elapsed) # type: ignore[attr-defined] + out = "\n".join(lines[:-1]).strip() + elapsed = timing["steady_s"] + return elapsed, out + + +def tools_for(args: argparse.Namespace) -> dict[str, object]: + return build_tools(args.tools_python, args.python) + + +def tools_report(tools: dict[str, object]) -> dict[str, object]: + """Per-column tool version, flags and semantics note, for the printout and the JSON.""" + v: dict[str, str] = tools.get("versions") or {} # type: ignore[assignment] + na: dict[str, str] = tools["na"] # type: ignore[assignment] + py = f"CPython {v.get('python', '?')}" + report: dict[str, object] = { + "tools_python": {"interpreter": py, "baseline_python": tools.get("baseline_python"), + "same_interpreter_as_CPython_column": tools.get("same_python")}, + CY_PURE: {"tool": f"Cython {v.get('Cython')} on {py}", "flags": CYTHON_FLAGS, + "semantics": SEMANTICS[CY_PURE]}, + CY_TYPED: {"tool": f"Cython {v.get('Cython')} on {py}", + "flags": CYTHON_FLAGS + "; # cython: boundscheck=False, wraparound=False, cdivision=True, " + "initializedcheck=False in each .pyx", "semantics": SEMANTICS[CY_TYPED]}, + NUMBA: {"tool": f"numba {v.get('numba')}, llvmlite {v.get('llvmlite')}, numpy {v.get('numpy')} on {py}", + "flags": "@njit(cache=True, error_model='numpy'); binary-trees: jitclass + explicit " + "signatures, cache=False; NUMBA_CACHE_DIR per benchmark", + "semantics": SEMANTICS[NUMBA]}, + } + for col in TOOL_COLS: + if col in na: + report[col]["unavailable"] = na[col] # type: ignore[index] + return report + + +def print_tools_report(report: dict[str, object]) -> None: + print("\ntools:") + for key, val in report.items(): + if key == "tools_python": + print(f" tools python: {val['interpreter']}; baseline CPython column: {val['baseline_python']}; " # type: ignore[index] + f"same interpreter: {val['same_interpreter_as_CPython_column']}") # type: ignore[index] + else: + print(f" {key}: {val['tool']}\n flags: {val['flags']}\n semantics: {val['semantics']}") # type: ignore[index] + + +def run_column(col: str, cmd: list[str]) -> tuple[float, str] | str: + """run_once, but a Cython/numba failure becomes the n/a string instead of ending the run.""" + try: + return run_once(cmd) + except RunFailed as e: + return f"n/a ({e})" def verify(args: argparse.Namespace) -> int: build() compiled = build_compiled(Path(args.typedpython), args.typedpython_python) if args.typedpython else None + tools = tools_for(args) bad = 0 for name, spec in BENCHMARKS.items(): - outputs = {col: run_once(cmd)[1] - for col, cmd in commands(name, spec[4], args.python, True, compiled).items()} + outputs: dict[str, str] = {} + notes: dict[str, str] = dict(tools["na"]) # type: ignore[arg-type] + for col, cmd in commands(name, spec[4], args.python, True, compiled, tools).items(): + res = run_column(col, cmd) + if isinstance(res, str): + notes[col] = res + else: + outputs[col] = res[1] distinct = set(outputs.values()) status = "OK " if len(distinct) == 1 else "FAIL" print(f"{status} {name} (size {spec[4]}): {next(iter(outputs.values())) if len(distinct) == 1 else ''}") + print(f" columns: {', '.join(outputs)}") + for col, why in notes.items(): + print(f" {col}: {why}") if len(distinct) != 1: bad += 1 for col, out in outputs.items(): print(f" {col}: {out}") print("all outputs agree" if bad == 0 else f"{bad} benchmark(s) disagree") + print_tools_report(tools_report(tools)) return 1 if bad else 0 +def measure_numba(cmd: Cmd, repeat: int) -> tuple[dict[str, object], dict[str, object]]: + """Cold-cache runs (each wipes the cache dir, so each compiles) give the steady-state time and the + cold compile time; one more run with the cache left in place gives the cached first-call time.""" + steady, outs, cold_first, walls = [], set(), [], [] + for _ in range(repeat): + t, out = run_once(cmd) + steady.append(t) + outs.add(out) + cold_first.append(cmd.last["first_call_s"]) # type: ignore[index] + walls.append(cmd.last["wall_s"]) # type: ignore[index] + cache_files = len([p for p in cmd.cache_dir.rglob("*") if p.is_file()]) if cmd.cache_dir else 0 + cmd.cold = False + try: + t_c, out_c = run_once(cmd) + outs.add(out_c) + cached_first = cmd.last["first_call_s"] # type: ignore[index] + cached_steady = t_c + finally: + cmd.cold = True + column = {"median_s": statistics.median(steady), "runs_s": steady, "output": sorted(outs), + "process_wall_s_runs": walls, "note": "steady-state call at benchmark size, JIT excluded"} + compile_rec = {"cold_first_call_s_median": statistics.median(cold_first), "cold_first_call_s_runs": cold_first, + "cached_first_call_s": cached_first, "cached_steady_s": cached_steady, + "cache_files_written": cache_files, "cache_used": cache_files > 0, + "note": "first call = module import + first run() at a tiny size; numba import excluded"} + return column, compile_rec + + def measure(args: argparse.Namespace) -> int: load = os.getloadavg() print(f"load average: {load[0]:.2f} {load[1]:.2f} {load[2]:.2f}") @@ -174,6 +427,8 @@ def measure(args: argparse.Namespace) -> int: return 2 build() compiled = build_compiled(Path(args.typedpython), args.typedpython_python) if args.typedpython else None + tools = tools_for(args) + report = tools_report(tools) results: dict[str, dict[str, object]] = {} for name, spec in BENCHMARKS.items(): size = args.size.get(name, spec[5]) @@ -181,15 +436,29 @@ def measure(args: argparse.Namespace) -> int: entry = (compiled or {}).get(name, NOT_BUILT) if isinstance(entry, tuple): row["compiled_functions"] = entry[1] - for col, cmd in commands(name, size, args.python, False, compiled).items(): - runs = [run_once(cmd) for _ in range(args.repeat)] - outs = {out for _, out in runs} - row[col] = {"median_s": statistics.median(t for t, _ in runs), - "runs_s": [t for t, _ in runs], "output": sorted(outs)} + for col, cmd in commands(name, size, args.python, False, compiled, tools).items(): + if col == CPY_TOOLS and tools.get("same_python"): + continue + try: + if col == NUMBA: + row[col], row["numba compile"] = measure_numba(cmd, args.repeat) # type: ignore[arg-type] + outs = set(row[col]["output"]) # type: ignore[index] + else: + runs = [run_once(cmd) for _ in range(args.repeat)] + outs = {out for _, out in runs} + row[col] = {"median_s": statistics.median(t for t, _ in runs), + "runs_s": [t for t, _ in runs], "output": sorted(outs)} + except RunFailed as e: + row[col] = f"n/a ({e})" + continue if len(outs) != 1: print(f"warning: {name}/{col} produced differing outputs across runs") if not isinstance(entry, tuple): row["TypedPython @compiled"] = entry + for col, why in tools["na"].items(): # type: ignore[union-attr] + row[col] = why + if tools.get("same_python"): + row[CPY_TOOLS] = "same as CPython" measured = [c for c in COLUMNS if isinstance(row.get(c), dict)] outs_all = {tuple(row[c]["output"]) for c in measured} # type: ignore[index] row["outputs_agree"] = len(outs_all) == 1 @@ -206,6 +475,13 @@ def measure(args: argparse.Namespace) -> int: flag = "" if row["outputs_agree"] else " (OUTPUTS DIFFER)" print(f"| {name}{flag} | {row['size']} | " + " | ".join(cells) + " |") print(f"\nmedian of {args.repeat} runs, wall time; Java excludes javac, includes JVM startup.") + print("numba column = steady-state call inside the process (JIT excluded); its compile cost:") + for name, row in results.items(): + rec = row.get("numba compile") + if isinstance(rec, dict): + print(f" {name}: cold first call {rec['cold_first_call_s_median']:.3f} s, cached first call " + f"{rec['cached_first_call_s']:.3f} s, cache files written {rec['cache_files_written']}") + print_tools_report(report) if args.write: out_dir = ROOT / "results" @@ -214,7 +490,7 @@ def measure(args: argparse.Namespace) -> int: payload = {"date": datetime.datetime.now().isoformat(timespec="seconds"), "host": socket.gethostname(), "load_average": load, "repeat": args.repeat, "python": subprocess.run([args.python, "--version"], capture_output=True, text=True).stdout.strip(), - "results": results} + "tools": report, "results": results} path.write_text(json.dumps(payload, indent=2) + "\n") print(f"wrote {path}") return 0 @@ -224,46 +500,48 @@ def interleaved(args: argparse.Namespace) -> int: """CPython vs TypedPython @compiled, alternated A/B in the same time window (leader decision, 2026-10-03): when the machine is never quiet, load hits both sides of each pair alike, so the per-pair ratio is stable even though absolute times are not. Reports the median ratio, never - absolute times as the result, with the load average recorded at every pair.""" - if not args.typedpython: - print("interleaved needs --typedpython") + absolute times as the result, with the load average recorded at every pair. + + The Cython and numba columns are paired the same way against the CPython run under the tools + interpreter (`CPython (tools python)`), each pair alternating which side runs first. For numba the + ratio uses its in-process steady-state time (JIT excluded, process start included for CPython, a + small bias in numba's favour at these sizes); its first-call (compile) time is recorded per pair.""" + tools = tools_for(args) + if not args.typedpython and not tools.get("python"): + print("interleaved needs --typedpython or a tools python") return 2 - compiled = build_compiled(Path(args.typedpython), args.typedpython_python) + compiled = build_compiled(Path(args.typedpython), args.typedpython_python) if args.typedpython else {} results: dict[str, dict[str, object]] = {} for name, spec in BENCHMARKS.items(): entry = compiled.get(name) - if not isinstance(entry, tuple): - results[name] = {"skipped": entry} - continue size = args.size.get(name, spec[5]) - cmds = commands(name, size, args.python, False, compiled) - a_cmd, b_cmd = cmds["CPython"], cmds["TypedPython @compiled"] - c_cmd = cmds.get(NOSLOTS) - pairs = [] - for i in range(args.pairs): - order = (a_cmd, b_cmd) if i % 2 == 0 else (b_cmd, a_cmd) # alternate which runs first - load = os.getloadavg() - (t1, o1), (t2, o2) = run_once(order[0]), run_once(order[1]) - t_a, o_a, t_b, o_b = (t1, o1, t2, o2) if order[0] is a_cmd else (t2, o2, t1, o1) - if o_a != o_b: - print(f"{name}: OUTPUTS DIFFER in pair {i}: {o_a!r} vs {o_b!r}") - return 1 - pair = {"cpython_s": t_a, "compiled_s": t_b, "ratio": t_a / t_b, "load": load} - if c_cmd is not None: - t_c, o_c = run_once(c_cmd) - if o_c != o_a: - print(f"{name}: OUTPUTS DIFFER in pair {i} (no __slots__): {o_c!r} vs {o_a!r}") - return 1 - pair.update(cpython_noslots_s=t_c, ratio_vs_noslots=t_c / t_b) - pairs.append(pair) - ratios = [p["ratio"] for p in pairs] - results[name] = {"size": size, "compiled_functions": entry[1], "pairs": pairs, - "median_ratio": statistics.median(ratios), "min_ratio": min(ratios), - "max_ratio": max(ratios)} - if c_cmd is not None: - results[name]["median_ratio_vs_noslots"] = statistics.median( - p["ratio_vs_noslots"] for p in pairs) + cmds = commands(name, size, args.python, False, compiled, tools) + r: dict[str, object] = {"size": size} + if args.typedpython: + if isinstance(entry, tuple): + r.update(measure_pair(name, cmds["CPython"], cmds["TypedPython @compiled"], cmds.get(NOSLOTS), + args.pairs, "compiled")) + r["compiled_functions"] = entry[1] + else: + r["skipped"] = entry + extras: dict[str, object] = {} + base = cmds.get(CPY_TOOLS) + for col in TOOL_COLS: + if col in tools["na"]: # type: ignore[operator] + extras[col] = tools["na"][col] # type: ignore[index] + elif col not in cmds or base is None: + extras[col] = "n/a (not built)" + else: + try: + extras[col] = measure_pair(name, base, cmds[col], None, args.pairs, col) + except RunFailed as e: + extras[col] = f"n/a ({e})" + r["tools_pairs"] = extras + results[name] = r print(f"measured {name}", file=sys.stderr) + if any(isinstance(v, dict) and v.get("outputs_differ") for v in [r, *extras.values()]): + print(f"{name}: OUTPUTS DIFFER, see the JSON/table") + return 1 print() print("| benchmark | size | speedup vs CPython (median of pairs) | range | load during pairs (1-min) |") @@ -271,22 +549,69 @@ def interleaved(args: argparse.Namespace) -> int: for name, r in results.items(): if "skipped" in r: print(f"| {name} | - | {r['skipped']} | | |") - continue - loads = [p["load"][0] for p in r["pairs"]] # type: ignore[index] - print(f"| {name} | {r['size']} | {r['median_ratio']:.1f}x | {r['min_ratio']:.1f}–{r['max_ratio']:.1f}x " - f"| {min(loads):.1f}–{max(loads):.1f} |") - print(f"\n{args.pairs} A/B pairs per benchmark, order alternated; ratio = CPython time / compiled time.") + elif "median_ratio" in r: + loads = [p["load"][0] for p in r["pairs"]] # type: ignore[index] + print(f"| {name} | {r['size']} | {r['median_ratio']:.1f}x | {r['min_ratio']:.1f}\u2013{r['max_ratio']:.1f}x " + f"| {min(loads):.1f}\u2013{max(loads):.1f} |") + if args.typedpython: + print(f"\n{args.pairs} A/B pairs per benchmark, order alternated; ratio = CPython time / compiled time.") + print("\n| benchmark | column | speedup vs CPython (tools python), median of pairs | range | extra |") + print("|---|---|---|---|---|") + for name, r in results.items(): + for col, p in r["tools_pairs"].items(): # type: ignore[union-attr] + if isinstance(p, dict): + extra = "" + if col == NUMBA: + extra = (f"first call (compile) median {statistics.median(q['first_call_s'] for q in p['pairs']):.3f} s; " + f"process wall median {statistics.median(q['wall_s'] for q in p['pairs']):.3f} s") + print(f"| {name} | {col} | {p['median_ratio']:.1f}x | {p['min_ratio']:.1f}\u2013{p['max_ratio']:.1f}x | {extra} |") + else: + print(f"| {name} | {col} | {p} | | |") + print(f"\n{args.pairs} pairs per column, order alternated; ratio = CPython (tools python) time / column time.") + report = tools_report(tools) + print_tools_report(report) if args.write: out_dir = ROOT / "results" out_dir.mkdir(exist_ok=True) path = out_dir / f"{datetime.date.today().isoformat()}-{socket.gethostname()}-interleaved.json" path.write_text(json.dumps({"date": datetime.datetime.now().isoformat(timespec="seconds"), "host": socket.gethostname(), "method": "interleaved A/B", - "pairs": args.pairs, "results": results}, indent=2) + "\n") + "pairs": args.pairs, "tools": report, "results": results}, indent=2) + "\n") print(f"wrote {path}") return 0 +def measure_pair(name: str, a_cmd: list[str], b_cmd: list[str], c_cmd: list[str] | None, n_pairs: int, + label: str) -> dict[str, object]: + """n_pairs A/B pairs (A = CPython, B = the column); which runs first alternates every pair.""" + pairs = [] + for i in range(n_pairs): + order = (a_cmd, b_cmd) if i % 2 == 0 else (b_cmd, a_cmd) # alternate which runs first + load = os.getloadavg() + (t1, o1), (t2, o2) = run_once(order[0]), run_once(order[1]) + t_a, o_a, t_b, o_b = (t1, o1, t2, o2) if order[0] is a_cmd else (t2, o2, t1, o1) + if o_a != o_b: + print(f"{name}/{label}: OUTPUTS DIFFER in pair {i}: {o_a!r} vs {o_b!r}") + return {"outputs_differ": True} + pair: dict[str, object] = {"cpython_s": t_a, "compiled_s": t_b, "ratio": t_a / t_b, "load": load} + last = getattr(b_cmd, "last", None) + if getattr(b_cmd, "numba", False) and last: + pair.update(first_call_s=last["first_call_s"], wall_s=last["wall_s"]) + if c_cmd is not None: + t_c, o_c = run_once(c_cmd) + if o_c != o_a: + print(f"{name}: OUTPUTS DIFFER in pair {i} (no __slots__): {o_c!r} vs {o_a!r}") + return {"outputs_differ": True} + pair.update(cpython_noslots_s=t_c, ratio_vs_noslots=t_c / t_b) + pairs.append(pair) + ratios = [p["ratio"] for p in pairs] + out: dict[str, object] = {"pairs": pairs, "median_ratio": statistics.median(ratios), + "min_ratio": min(ratios), "max_ratio": max(ratios)} + if c_cmd is not None: + out["median_ratio_vs_noslots"] = statistics.median(p["ratio_vs_noslots"] for p in pairs) + return out + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = parser.add_subparsers(dest="cmd", required=True) @@ -294,7 +619,7 @@ def main() -> int: v.add_argument("--python", default=sys.executable) m = sub.add_parser("measure") m.add_argument("--python", default=sys.executable) - ab = sub.add_parser("interleaved", help="CPython vs @compiled alternated A/B; reports median ratios") + ab = sub.add_parser("interleaved", help="CPython vs @compiled/Cython/numba alternated A/B; reports median ratios") ab.add_argument("--python", default=sys.executable) ab.add_argument("--pairs", type=int, default=11) ab.add_argument("--write", action="store_true") @@ -304,6 +629,9 @@ def main() -> int: help="directory holding the typedpython package; enables the @compiled column") p.add_argument("--typedpython-python", default=sys.executable, help="interpreter for compiling (needs pyrefly 1.3.2)") + p.add_argument("--tools-python", default=str(TOOLS_PY_DEFAULT), + help="interpreter of the uv venv holding Cython and numba (default: benchmarks/" + "typedpython/.venv/bin/python); enables the Cython and numba columns") m.add_argument("--repeat", type=int, default=5) m.add_argument("--force", action="store_true") m.add_argument("--write", action="store_true") From b159f27bdfa860add9dba75e177dfc4fa469617e Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 30 Oct 2026 04:45:16 +0900 Subject: [PATCH 02/29] TypedPython: decode cl.exe output in the console code page cl.exe writes in the console code page (CP949 on Korean Windows); decoding it as UTF-8 raised UnicodeDecodeError in subprocess's reader thread and would lose a compile error message. --- .../src/main/python/typedpython/cbuild.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/cbuild.py b/python-multiplatform-ksp/src/main/python/typedpython/cbuild.py index d12ecea39..2f30a9490 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/cbuild.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/cbuild.py @@ -164,7 +164,7 @@ def _find_vcvars() -> Path: f"with C++' workload ({MSVC_COMPONENT}), or run from a 'x64 Native Tools' prompt.") run = subprocess.run( [str(vswhere), "-latest", "-products", "*", "-requires", MSVC_COMPONENT, - "-property", "installationPath"], capture_output=True, text=True) + "-property", "installationPath"], capture_output=True, encoding="oem", errors="replace") root = run.stdout.strip().splitlines()[0].strip() if run.returncode == 0 and run.stdout.strip() else "" if not root: raise CBuildError( @@ -229,7 +229,8 @@ def build(c_source: str, module_name: str, out_dir: Path, runtime_dir: Path | No env = msvc_env() cmd = msvc_command(_msvc_cl(env), c_file, so, include, runtime, flags, gil_disabled=free_threaded(ext_suffix)) - run = subprocess.run(cmd, capture_output=True, text=True, env=env) + # cl.exe writes in the console code page (CP949 on Korean Windows), not UTF-8 + run = subprocess.run(cmd, capture_output=True, encoding="oem", errors="replace", env=env) if run.returncode != 0 or not so.is_file(): # cl.exe reports its errors on stdout raise CBuildError(f"C compiler failed ({run.returncode}): {subprocess.list2cmdline(cmd)}\n" From 2a75e573edcf6671501a2629ab2555297862ff95 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 30 Oct 2026 04:51:56 +0900 Subject: [PATCH 03/29] TypedPython: publish the package to PyPI by trusted publishing A typedpython-v* tag builds the wheel, checks that it alone compiles the release example, and publishes it. INTENT 1.4 records the maintainer's distribution decision. --- .github/workflows/typedpython-pypi.yml | 78 +++++++++++++++++++++++++ docs/INTENT.md | 5 ++ python-multiplatform-ksp/pyproject.toml | 6 ++ 3 files changed, 89 insertions(+) create mode 100644 .github/workflows/typedpython-pypi.yml diff --git a/.github/workflows/typedpython-pypi.yml b/.github/workflows/typedpython-pypi.yml new file mode 100644 index 000000000..b0f98705d --- /dev/null +++ b/.github/workflows/typedpython-pypi.yml @@ -0,0 +1,78 @@ +name: TypedPython to PyPI + +# Publishes the `typedpython` package (python-multiplatform-ksp/pyproject.toml) to PyPI. The package +# is versioned on its own, so it has its own tag: `typedpython-v`, pushed by the coordinator +# on a commit of main. PyPI trusted publishing: no token is stored; the `pypi` environment and the +# PyPI project's trusted publisher (this repository, this workflow file, environment `pypi`) must match. +# Only a wheel is published: it is pure Python (`py3-none-any`), and an sdist would carry the whole +# Kotlin module next to it. + +on: + push: + tags: + - 'typedpython-v*' + workflow_dispatch: + inputs: + tag: + description: 'Existing tag to build and publish (e.g. typedpython-v0.1.0)' + required: true + +permissions: + contents: read + +jobs: + build: + name: Build and check the wheel + runs-on: macos-latest + env: + TAG: ${{ github.event.inputs.tag || github.ref_name }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.tag || github.ref }} + + - name: Setup uv + uses: astral-sh/setup-uv@v6 + + - name: The tag names the package version + run: | + set -euo pipefail + version="$(sed -nE 's/^version = "(.*)"$/\1/p' python-multiplatform-ksp/pyproject.toml | head -1)" + test "$TAG" = "typedpython-v$version" || { echo "error: tag $TAG does not match typedpython $version" >&2; exit 1; } + + - name: Build + run: uv build --wheel --out-dir dist python-multiplatform-ksp + + - name: The wheel alone compiles the release example + run: | + set -euo pipefail + uv venv -q -p 3.15t "$RUNNER_TEMP/tpenv" + uv pip install -q --python "$RUNNER_TEMP/tpenv/bin/python" dist/typedpython-*.whl + "$RUNNER_TEMP/tpenv/bin/typedpython" demo --out "$RUNNER_TEMP/tpbuild" docs/release/desktop/typedpython/kernels.py 400000 | tee "$RUNNER_TEMP/tp.log" + grep -q '^compiled functions: .*dot' "$RUNNER_TEMP/tp.log" + grep -q '^identical output' "$RUNNER_TEMP/tp.log" + + - uses: actions/upload-artifact@v4 + with: + name: dist + path: dist/ + + publish: + name: Publish to PyPI + needs: build + runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write + steps: + - uses: actions/download-artifact@v4 + with: + name: dist + path: dist/ + + - name: Setup uv + uses: astral-sh/setup-uv@v6 + + - name: Publish + run: uv publish --trusted-publishing always dist/* diff --git a/docs/INTENT.md b/docs/INTENT.md index cb707781c..5c45079d1 100644 --- a/docs/INTENT.md +++ b/docs/INTENT.md @@ -40,6 +40,11 @@ CPython interpreter inside a Kotlin Multiplatform application so that: is **checked** by default; **compiling** it to native code is opt-in. The compiled code still runs on the embedded CPython, as an extension module (`docs/design/typedpython.md`). + TypedPython reaches plain Python users as a **distribution**, not as a new interpreter (maintainer, + 2026-10-30: "배포판 방식을 생각했다는거야. 근데 일단 그렇게 하더라도 typedpython 패키지 형태라면 pypi + 배포를 해야겠지"). The thisisthepy CPython build ships with TypedPython in it, and the same + compiler is an ordinary `typedpython` package on PyPI (`uv add typedpython`). + 5. **High-performance kernels are written in Python too** (TypedPython tier 2). The maintainer first asked to explore it ("GPU 관련 최적화까지도 별도 언어 없이 지원할 방안을 모색해보자") and on 2026-10-04 made it a commitment ("GPU 커널 등 탐색을 약속으로 올려"), so that TypedPython can stand diff --git a/python-multiplatform-ksp/pyproject.toml b/python-multiplatform-ksp/pyproject.toml index d5fcbf41c..487554414 100644 --- a/python-multiplatform-ksp/pyproject.toml +++ b/python-multiplatform-ksp/pyproject.toml @@ -4,11 +4,17 @@ [project] name = "typedpython" version = "0.1.0" +description = "TypedPython: statically checked Python, with marked functions compiled to CPython extension modules" +license = "Apache-2.0" +authors = [{ name = "thisisthepy" }] requires-python = ">=3.13" # Pinned exactly: the gate reads Pyrefly's per-expression type report (--report-pysa), whose # format is versioned by Pyrefly, not by a public API. dependencies = ["pyrefly==1.3.2"] +[project.urls] +Homepage = "https://github.com/thisisthepy/python-multiplatform" + [project.scripts] typedpython = "typedpython.__main__:cli" From a59096f40bf5cdd7f89d6889b353001518a1db03 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 30 Oct 2026 04:54:13 +0900 Subject: [PATCH 04/29] TypedPython: @compiled is the only mark; drop the '# typedpython: compiled' module pragma The maintainer decided (2026-10-30) to keep only the builtin @compiled. The pragma is now an ordinary comment; tests, the Kotlin-bridge fixture, benchmarks and the release example decorate the functions the pragma used to cover. --- .../2026-10-17-buffer-atomic-kernels.py | 4 +- benchmarks/typedpython/run.py | 16 +++- docs/SPEC.md | 2 +- docs/design/typedpython.md | 6 +- docs/release/desktop/typedpython/kernels.py | 3 +- .../typedpython/tp_kotlin_bridge.py | 3 +- .../src/main/python/typedpython/__main__.py | 4 +- .../src/main/python/typedpython/frontend.py | 26 ++----- .../src/test/python/conftest.py | 20 +++++ .../src/test/python/test_bitwise.py | 27 ++++++- .../src/test/python/test_buffer_arrays.py | 24 +++++- .../src/test/python/test_buffer_memoryview.py | 11 ++- .../src/test/python/test_buffer_numpy.py | 76 +++++++++++++++++- .../src/test/python/test_class_methods.py | 20 +++-- .../src/test/python/test_cost_rule.py | 19 +++-- .../src/test/python/test_eval_breaker.py | 19 ++++- .../src/test/python/test_fast_entry.py | 13 +++- .../src/test/python/test_free_threaded.py | 20 ++++- .../src/test/python/test_frontend.py | 29 +++++-- .../src/test/python/test_frontend_classes.py | 9 +-- .../src/test/python/test_interval_proofs.py | 17 +++- .../src/test/python/test_module_report.py | 29 +++---- .../src/test/python/test_native_int_impure.py | 35 +++++++-- .../src/test/python/test_osr_exit.py | 4 +- .../src/test/python/test_parallel_loops.py | 78 +++++++++++++++++-- .../src/test/python/test_pipeline.py | 16 ++-- .../test/python/test_pipeline_incremental.py | 19 +++-- .../src/test/python/test_recursion.py | 8 +- .../src/test/python/test_regions.py | 13 +++- .../src/test/python/test_resume.py | 4 +- .../src/test/python/test_str_dict.py | 38 ++++++++- .../src/test/python/test_tracing.py | 5 +- 32 files changed, 498 insertions(+), 119 deletions(-) diff --git a/benchmarks/typedpython/results/2026-10-17-buffer-atomic-kernels.py b/benchmarks/typedpython/results/2026-10-17-buffer-atomic-kernels.py index a0cd59c56..53a917186 100644 --- a/benchmarks/typedpython/results/2026-10-17-buffer-atomic-kernels.py +++ b/benchmarks/typedpython/results/2026-10-17-buffer-atomic-kernels.py @@ -1,7 +1,7 @@ -# typedpython: compiled import array +@compiled def dot_arr(a: array.array[float], b: array.array[float]) -> float: s: float = 0.0 for i in range(len(a)): @@ -9,11 +9,13 @@ def dot_arr(a: array.array[float], b: array.array[float]) -> float: return s +@compiled def scale_arr(a: array.array[float], k: float) -> None: for i in range(len(a)): a[i] = a[i] * k +@compiled def dot_list(a: list[float], b: list[float]) -> float: s: float = 0.0 for i in range(len(a)): diff --git a/benchmarks/typedpython/run.py b/benchmarks/typedpython/run.py index 104ad572d..c6c3aaa47 100644 --- a/benchmarks/typedpython/run.py +++ b/benchmarks/typedpython/run.py @@ -10,7 +10,7 @@ The "TypedPython @compiled" column runs only with `--typedpython ` (the directory that holds the `typedpython` package, python-multiplatform-ksp/src/main/python) and `--typedpython-python` (an interpreter with the gate's dependencies, e.g. that module's .venv): each benchmark is copied with -the `# typedpython: compiled` marker, compiled through `typedpython.pipeline`, and its `main()` is +`@compiled` above every top-level def and class, compiled through `typedpython.pipeline`, and its `main()` is run from the built extension. Without the option the column reports n/a. binary-trees declares `__slots__` (TypedPython compiles fixed-layout classes only, SPEC N-11). The @@ -18,6 +18,7 @@ with the `__slots__` line removed, so what adding the slots changes in CPython is recorded too. """ import argparse +import ast import datetime import json import os @@ -87,6 +88,17 @@ def build() -> None: """ +def with_compiled(source: str) -> str: + """`source` with `@compiled` above every top-level def and class (above their other decorators).""" + tree = ast.parse(source) + lines = source.split("\n") + firsts = [min([n.lineno] + [d.lineno for d in n.decorator_list]) for n in tree.body + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef))] + for first in reversed(firsts): + lines.insert(first - 1, "@compiled") + return "\n".join(lines) + + def build_compiled(tp_dir: Path, tp_python: str) -> dict[str, tuple[str, list[str]] | str]: """Compile each benchmark with TypedPython; name -> (extension, compiled functions) or a reason.""" built: dict[str, tuple[str, list[str]] | str] = {} @@ -95,7 +107,7 @@ def build_compiled(tp_dir: Path, tp_python: str) -> dict[str, tuple[str, list[st work = TP_BUILD / name work.mkdir(parents=True, exist_ok=True) source = work / py - source.write_text("# typedpython: compiled\n" + (ROOT / "py" / py).read_text()) + source.write_text(with_compiled((ROOT / "py" / py).read_text())) result = subprocess.run([tp_python, "-c", COMPILE_SNIPPET, str(source), str(work / "out")], capture_output=True, text=True, env=env) if result.returncode != 0: diff --git a/docs/SPEC.md b/docs/SPEC.md index 1ba12bbc1..61c454c2f 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -724,7 +724,7 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. with `.diagnostics`) before the module runs. Results are cached in memory by file content hash, failures included. `Status: implemented`, `python-multiplatform-ksp/src/test/python/test_importgate.py`. - **N-8** TypedPython compilation to C (INTENT §1.4; design §4.3; #41). `@compiled` (a builtin name, no - import, #42) and a first-line `# typedpython: compiled` mark functions; the compiler lowers them + import, #42) is the only mark for functions (the first-line module pragma was removed, #353); the compiler lowers them through the typed IR (`typedpython/ir.py`) to C against the CPython C API and builds an extension with the platform C compiler. Compiled functions return what CPython returns (IR contract: checked i64 with deopt to the interpreted function, Python floor division/modulo, exact int/float comparison, diff --git a/docs/design/typedpython.md b/docs/design/typedpython.md index 1ae91c1c2..991207854 100644 --- a/docs/design/typedpython.md +++ b/docs/design/typedpython.md @@ -62,7 +62,7 @@ TypedPython 의 목표는 **새 언어를 만들지 않고, 파이썬 문법 그 |---|---|---|---| | 0 | 서드파티 패키지 (의존성) | 없음, 검사하지 않는다 | 바이트코드 (pypackpack 이 받은 그대로) | | 1 `checked` | **사용자 프로젝트의 파이썬 코드 전부, 기본값** | 빌드 타임 타입 오류, `Any` 유출 0 (§4.2) | 바이트코드 | -| 1 `compiled` | `@compiled` (내장 이름, import 없음) 또는 모듈 선언 `# typedpython: compiled` | `checked` + §3.1 금지 목록 | C 확장 모듈 (NEON · AVX 벡터화, §5.3) | +| 1 `compiled` | `@compiled` (내장 이름, import 없음) | `checked` + §3.1 금지 목록 | C 확장 모듈 (NEON · AVX 벡터화, §5.3) | | 2 `kernel` | `@kernel` | 객체·힙 할당 없음, 고정 dtype | SPIR-V / MSL (직접) / WGSL / PTX, 빌드 타임 (§5) | - **사용자 코드는 표시하지 않아도 검사된다.** 타입 오류가 있으면 Kotlin 컴파일 에러처럼 빌드가 @@ -685,9 +685,7 @@ upstream 대비 수치 오차와 커널 시간을 기준선으로 박아 둔다. 인터프리터 시작 시 `builtins.compiled` 를 설치하고(#42), 일반 CPython 에서는 `typedpython` wheel 의 `typedpython_builtins.pth` 가 `builtins` 에 없을 때만 항등 함수를 설치한다(런타임이 넣은 것을 덮어쓰지 않는다). 타입 검사기에는 `__builtins__.pyi` 로 알린다, Pyrefly 1.3.2 는 검색 경로의 그 파일을 추가 - 내장으로 읽는다(실측). 모듈 단위 표기는 `# typedpython: compiled` 주석을 유지한다: `# pyright: strict` 와 같은 - 도구 pragma 형태이고, 실행 시 이름 조회가 필요 없어 일반 CPython 에서 부작용이 없으며, 데코레이터 없이 - 모듈 전체를 지정할 수 있다. + 내장으로 읽는다(실측). 모듈 단위 pragma(`# typedpython: compiled`)는 2026-10-30 메인테이너 결정으로 제거했다(#353). `@compiled` 가 컴파일 대상을 표시하는 유일한 수단이다. 2. ~~고정폭 정수의 오버플로 의미~~, **결정됨:** `int` 는 i64 로 컴파일하고 오버플로 시 인터프리터 재실행으로 큰 정수로 승격(의미 보존). 3. ~~pypackpack Cython 슬롯 인터페이스 소유~~, **결정됨:** pypackpack 이 정한다. TypedPython 은 생성한 C 소스, 빌드 플래그, C 를 낸 모듈 목록을 넘긴다. 확장이 원본 소스를 품으므로 별도 `.py` 폴백은 없다(Cython 폐기 후 갱신). 4. ~~GPU 착지를 torchnative 커스텀 op 로 할 것인가~~, **결정됨 (2026-10-04, 메인테이너):** 커널은 diff --git a/docs/release/desktop/typedpython/kernels.py b/docs/release/desktop/typedpython/kernels.py index 81cc9f952..3fb98914e 100644 --- a/docs/release/desktop/typedpython/kernels.py +++ b/docs/release/desktop/typedpython/kernels.py @@ -1,7 +1,7 @@ -# typedpython: compiled import sys +@compiled def dot(n: int) -> float: s = 0.0 for i in range(n): @@ -10,6 +10,7 @@ def dot(n: int) -> float: return s +@compiled def collatz_steps(limit: int) -> int: total = 0 for start in range(1, limit): diff --git a/python-multiplatform-ksp/fixtures/app/src/desktopTest/typedpython/tp_kotlin_bridge.py b/python-multiplatform-ksp/fixtures/app/src/desktopTest/typedpython/tp_kotlin_bridge.py index 7677eca9c..311eb2f5b 100644 --- a/python-multiplatform-ksp/fixtures/app/src/desktopTest/typedpython/tp_kotlin_bridge.py +++ b/python-multiplatform-ksp/fixtures/app/src/desktopTest/typedpython/tp_kotlin_bridge.py @@ -1,4 +1,3 @@ -# typedpython: compiled """Issue #45: compiled TypedPython code calling Kotlin through the binder. Every name imported below is a binder proxy (`fixture/app/TypedPythonBridge.kt`, published by @@ -21,6 +20,7 @@ from fixture.app import TpSample, tpAccumulate, tpCheck, tpTouch +@compiled def drive(n: int, scale: float, token: object) -> float: """Per step: a Kotlin constructor call that returns a typed proxy, a method call on that proxy, the proxy handed back to a Kotlin function, and a PyObject handed to Kotlin -- inside native @@ -35,6 +35,7 @@ def drive(n: int, scale: float, token: object) -> float: return total +@compiled def checked(n: int, limit: float) -> float: """`tpCheck` throws in Kotlin once `acc` passes `limit`; the caller sees what Python sees.""" acc: float = 0.0 diff --git a/python-multiplatform-ksp/src/main/python/typedpython/__main__.py b/python-multiplatform-ksp/src/main/python/typedpython/__main__.py index bd220e713..a9cda63e2 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/__main__.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/__main__.py @@ -84,7 +84,7 @@ def _build(paths: list[Path], cache: Path, out: Path | None) -> int: print(f"{m.status:7} {m.name}: {what}{why}") rel = m.path.relative_to(root).as_posix() if m.kind == "not_marked": - print(f" {rel}: not marked for compilation (no `# typedpython: compiled`, no @compiled)") + print(f" {rel}: not marked for compilation (no @compiled)") for item in m.skipped_items: where = f"{item.file}:{item.line}" if item.line is not None else item.file print(f" {where}: {item.name}: {item.message}") @@ -112,6 +112,8 @@ def _demo(module: Path, module_argv: list[str], out: Path) -> int: return EXIT_FAILURE def interpreted() -> object: + import builtins # the module's `@compiled` is a builtin name; typedpython_builtins.pth installs it + builtins.compiled = getattr(builtins, "compiled", lambda f, /: f) spec = importlib.util.spec_from_file_location(module.stem + "_interpreted", module) assert spec is not None and spec.loader is not None mod = importlib.util.module_from_spec(spec) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py index 0da894f10..6e0b75aa6 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py @@ -3,8 +3,7 @@ `lower(path)` reads one module and returns an `ir.Module`: - Which functions: module-level `def`s decorated with the builtin `@compiled` (a bare name, no - import, #42), or every module-level `def` when the first non-empty line is - `# typedpython: compiled`. + import, #42). That decorator is the only mark; there is no module-level pragma (#353). - A function is lowered only when **all** of it is expressible in the IR; otherwise it is left out and `Module.skipped[name]` says why, with the line. There is no partial lowering. - Types: parameters and returns from their annotations (`int`/`float`/`bool`, `list[float]` and @@ -80,8 +79,6 @@ from typedpython import gate, ir, pyrefly, rebinding, resume, taint from typedpython.ir import Type -MARKER = "# typedpython: compiled" - I64_MIN, I64_MAX = -2**63, 2**63 - 1 TOP = (I64_MIN, I64_MAX) EXACT_INT_FLOAT = 2**53 # |n| <= this: int/int true division is exact-rounded in C (ir.BinOp) @@ -249,7 +246,7 @@ def lower(path: Path) -> ir.Module: def _parallel_marked(tree: ast.Module, source: str) -> set[str]: """Module-level functions the user marked for SPEC N-15: the comment line `# typedpython: parallel` - directly above the `def` (above its decorators if it has any). A comment, like the module marker, + directly above the `def` (above its decorators if it has any). A comment, so the interpreted module is unchanged and needs no runtime name.""" lines = source.splitlines() out: set[str] = set() @@ -434,17 +431,10 @@ def _binding_targets(stmt: ast.stmt) -> list[ast.expr]: SLOT_ACCESS_DUNDERS = ("__getattribute__", "__getattr__", "__setattr__", "__delattr__", "__new__") -def _whole_module(source: str) -> bool: - return next((line.strip() for line in source.splitlines() if line.strip()), "") == MARKER - - def is_opted_in(path: Path) -> bool: - """Whether the front end considers the module for compilation at all: the `# typedpython: compiled` - marker as its first non-empty line, or a module-level `def` / `class` decorated `@compiled` - (the same tests `_targets` and `_classes` apply).""" + """Whether the front end considers the module for compilation at all: a module-level `def` / `class` + decorated `@compiled` (the same test `_targets` and `_classes` apply).""" source = Path(path).read_text() - if _whole_module(source): - return True return any( isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) and any(isinstance(d, ast.Name) and d.id == "compiled" for d in node.decorator_list) @@ -454,14 +444,13 @@ def is_opted_in(path: Path) -> bool: def _classes( tree: ast.Module, source: str, info: _ModuleInfo, ) -> tuple[dict[str, ir.ClassDecl], dict[str, str]]: - whole = _whole_module(source) decls: dict[str, ir.ClassDecl] = {} skipped: dict[str, str] = {} for node in tree.body: if not isinstance(node, ast.ClassDef): continue marked = any(isinstance(d, ast.Name) and d.id == "compiled" for d in node.decorator_list) - if not (whole or marked): + if not marked: continue result = _class_decl(node, tree, info) if isinstance(result, str): @@ -608,13 +597,12 @@ def _class_of(kind: str, classes: dict[str, ir.ClassDecl]) -> tuple[str, bool] | def _targets( tree: ast.Module, source: str, info: _ModuleInfo, ) -> list[tuple[ast.FunctionDef | ast.AsyncFunctionDef, str | None]]: - whole_module = _whole_module(source) found: list[tuple[ast.FunctionDef | ast.AsyncFunctionDef, str | None]] = [] for node in tree.body: if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): continue marked = [d for d in node.decorator_list if isinstance(d, ast.Name) and d.id == "compiled"] - if not (whole_module or marked): + if not marked: continue reason = None if marked and info.bound("compiled"): @@ -2499,4 +2487,4 @@ def _breaks(body: tuple[ir.Stmt, ...]) -> bool: return False -__all__ = ["lower", "MARKER"] +__all__ = ["lower"] diff --git a/python-multiplatform-ksp/src/test/python/conftest.py b/python-multiplatform-ksp/src/test/python/conftest.py index 452a4e613..669816bf4 100644 --- a/python-multiplatform-ksp/src/test/python/conftest.py +++ b/python-multiplatform-ksp/src/test/python/conftest.py @@ -1,12 +1,32 @@ +import ast +import builtins import sys from pathlib import Path import textwrap import pytest +# The `typedpython_builtins.pth` of an installed wheel; the test interpreter has no such install. +builtins.compiled = getattr(builtins, "compiled", lambda f, /: f) + FIXTURES = Path(__file__).parent / "fixtures" +def decorate_compiled(source: str) -> str: + """`source` with `@compiled` put above every module-level def / class (above its other decorators), + the set a whole-module mark used to cover. A source that does not parse is returned unchanged.""" + try: + tree = ast.parse(source) + except SyntaxError: + return source + lines = source.split("\n") + firsts = [min([n.lineno] + [d.lineno for d in n.decorator_list]) for n in tree.body + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef))] + for first in reversed(firsts): + lines.insert(first - 1, "@compiled") + return "\n".join(lines) + + @pytest.fixture def write(tmp_path): """Write a dedented source file under tmp_path and return its path.""" diff --git a/python-multiplatform-ksp/src/test/python/test_bitwise.py b/python-multiplatform-ksp/src/test/python/test_bitwise.py index 0bd844034..c0d48b3d4 100644 --- a/python-multiplatform-ksp/src/test/python/test_bitwise.py +++ b/python-multiplatform-ksp/src/test/python/test_bitwise.py @@ -20,33 +20,39 @@ from test_native_int_impure import build_source SOURCE = textwrap.dedent(''' - # typedpython: compiled + @compiled def p_and(a: int, b: int) -> int: return a & b + @compiled def p_or(a: int, b: int) -> int: return a | b + @compiled def p_xor(a: int, b: int) -> int: return a ^ b + @compiled def p_inv(a: int) -> int: return ~a + @compiled def p_shl(a: int, b: int) -> int: return a << b + @compiled def p_shr(a: int, b: int) -> int: return a >> b + @compiled def p_aug(a: int, b: int) -> int: x: int = a x ^= b @@ -55,22 +61,26 @@ def p_aug(a: int, b: int) -> int: return x + @compiled def p_augshl(a: int, b: int) -> int: x: int = a x <<= b return x + @compiled def p_augshr(a: int, b: int) -> int: x: int = a x >>= b return x + @compiled def p_mix(a: int, b: int) -> int: return (((a & 255) << 3) | (b >> 2)) ^ ~a + @compiled def p_mask_loop(n: int) -> int: h: int = 0 for i in range(n): @@ -78,6 +88,7 @@ def p_mask_loop(n: int) -> int: return h + @compiled def p_array(xs: list[int]) -> None: for i in range(len(xs)): xs[i] &= 0xFF @@ -85,10 +96,12 @@ def p_array(xs: list[int]) -> None: xs[i] ^= i + @compiled def p_iv(a: int, b: int) -> int: return (a & 0xFFFF) * (b & 0xFFFF) + ~(a & 7) + @compiled def i_and(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a & b @@ -96,6 +109,7 @@ def i_and(log: list[object], a: int, b: int) -> object: return r + @compiled def i_or(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a | b @@ -103,6 +117,7 @@ def i_or(log: list[object], a: int, b: int) -> object: return r + @compiled def i_xor(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a ^ b @@ -110,6 +125,7 @@ def i_xor(log: list[object], a: int, b: int) -> object: return r + @compiled def i_inv(log: list[object], a: int) -> object: log.append("start") r: int = ~a @@ -117,6 +133,7 @@ def i_inv(log: list[object], a: int) -> object: return r + @compiled def i_shl(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a << b @@ -124,6 +141,7 @@ def i_shl(log: list[object], a: int, b: int) -> object: return r + @compiled def i_shr(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a >> b @@ -131,6 +149,7 @@ def i_shr(log: list[object], a: int, b: int) -> object: return r + @compiled def i_aug(log: list[object], a: int, b: int) -> object: log.append("start") x: int = a @@ -143,6 +162,7 @@ def i_aug(log: list[object], a: int, b: int) -> object: return x + @compiled def i_big(log: list[object], a: int, b: int, c: int) -> object: log.append("start") r: int = (a * b) & c @@ -156,6 +176,7 @@ def i_big(log: list[object], a: int, b: int, c: int) -> object: return r + @compiled def o_ops(log: list[object], x: object, y: object) -> object: log.append("start") r: object = x | y @@ -164,21 +185,25 @@ def o_ops(log: list[object], x: object, y: object) -> object: return r + @compiled def o_inplace(log: list[object], x: object, y: object) -> object: log.append("start") x |= y return x + @compiled def f_and(log: list[object], a: float, b: int) -> object: log.append("start") return a & b + @compiled def b_and(a: bool, b: bool) -> bool: return a & b + @compiled def b_inv(a: bool) -> int: return ~a ''') diff --git a/python-multiplatform-ksp/src/test/python/test_buffer_arrays.py b/python-multiplatform-ksp/src/test/python/test_buffer_arrays.py index 71d95f0c0..724f6759e 100644 --- a/python-multiplatform-ksp/src/test/python/test_buffer_arrays.py +++ b/python-multiplatform-ksp/src/test/python/test_buffer_arrays.py @@ -29,10 +29,10 @@ from typedpython import cbuild, cgen, frontend, ir, verify SOURCE = textwrap.dedent(''' - # typedpython: compiled import array + @compiled def sum_ad(a: array.array[float]) -> float: s = 0.0 for i in range(len(a)): @@ -40,6 +40,7 @@ def sum_ad(a: array.array[float]) -> float: return s + @compiled def sum_aq(a: array.array[int]) -> int: s = 0 for i in range(len(a)): @@ -47,70 +48,85 @@ def sum_aq(a: array.array[int]) -> int: return s + @compiled def get_ad(a: array.array[float], i: int) -> float: return a[i] + @compiled def get_aq(a: array.array[int], i: int) -> int: return a[i] + @compiled def set_ad(a: array.array[float], i: int, v: float) -> None: a[i] = v + @compiled def set_aq(a: array.array[int], i: int, v: int) -> None: a[i] = v + @compiled def length_ad(a: array.array[float]) -> int: return len(a) + @compiled def scale_ad(a: array.array[float], k: float) -> None: for i in range(len(a)): a[i] = a[i] * k + @compiled def iota_aq(a: array.array[int]) -> None: for i in range(len(a)): a[i] = i + @compiled def copy_ad(dst: array.array[float], src: array.array[float]) -> None: for i in range(len(dst)): dst[i] = src[i] + @compiled def cascade_ad(a: array.array[float], b: array.array[float]) -> None: for i in range(len(a)): a[i] = b[i] + 1.0 + @compiled def fill_n(a: array.array[float], n: int) -> None: for i in range(n): a[i] = 1.0 + @compiled def mixed(l: list[float], a: array.array[float]) -> None: for i in range(len(a)): a[i] = l[i] + @compiled def four(a: array.array[float], b: array.array[float], c: array.array[float], d: array.array[float]) -> None: for i in range(len(a)): a[i] = b[i] + c[i] + d[i] + @compiled def div_ad(a: array.array[float]) -> float: return a[0] / a[1] + @compiled def open_ad(a: array.array[float], o: object) -> float: return a[0] + float(o.v) + @compiled def rebind_view(a: array.array[float]) -> None: y = a[:] y[0] = 5.0 @@ -505,30 +521,34 @@ def many(n, buffer): # --- vectorised element-wise store loops (#226) --------------------------------------------------- VEC_SOURCE = textwrap.dedent(''' - # typedpython: compiled import array + @compiled def vscale(a: array.array[float], k: float) -> None: for i in range(len(a)): a[i] = a[i] * k + @compiled def vmix(a: array.array[float], k: float) -> None: for i in range(len(a)): a[i] = -(a[i] - k) * 2.5 + k + @compiled def vtwo(a: array.array[float], b: array.array[float]) -> None: for i in range(len(a)): a[i] = a[i] + b[i] + @compiled def vfill(a: array.array[float], k: float) -> None: for i in range(len(a)): a[i] = k + @compiled def vdiv(a: array.array[float], k: float) -> None: for i in range(len(a)): a[i] = a[i] / k diff --git a/python-multiplatform-ksp/src/test/python/test_buffer_memoryview.py b/python-multiplatform-ksp/src/test/python/test_buffer_memoryview.py index 1766b3300..8a616abc8 100644 --- a/python-multiplatform-ksp/src/test/python/test_buffer_memoryview.py +++ b/python-multiplatform-ksp/src/test/python/test_buffer_memoryview.py @@ -23,10 +23,10 @@ same_value) SOURCE = textwrap.dedent(''' - # typedpython: compiled import array + @compiled def sum_mf(m: memoryview[float]) -> float: s = 0.0 for i in range(len(m)): @@ -34,6 +34,7 @@ def sum_mf(m: memoryview[float]) -> float: return s + @compiled def sum_mi(m: memoryview[int]) -> int: s = 0 for i in range(len(m)): @@ -41,31 +42,38 @@ def sum_mi(m: memoryview[int]) -> int: return s + @compiled def get_mf(m: memoryview[float], i: int) -> float: return m[i] + @compiled def get_mi(m: memoryview[int], i: int) -> int: return m[i] + @compiled def set_mf(m: memoryview[float], i: int, v: float) -> None: m[i] = v + @compiled def set_mi(m: memoryview[int], i: int, v: int) -> None: m[i] = v + @compiled def length_mf(m: memoryview[float]) -> int: return len(m) + @compiled def scale_mf(m: memoryview[float], k: float) -> None: for i in range(len(m)): m[i] = m[i] * k + @compiled def spin_mf(m: memoryview[float], n: int) -> float: s = 0.0 L = len(m) @@ -74,6 +82,7 @@ def spin_mf(m: memoryview[float], n: int) -> float: return s + @compiled def spin_af(a: array.array[float], n: int) -> float: s = 0.0 L = len(a) diff --git a/python-multiplatform-ksp/src/test/python/test_buffer_numpy.py b/python-multiplatform-ksp/src/test/python/test_buffer_numpy.py index 582e4b094..f14a32c00 100644 --- a/python-multiplatform-ksp/src/test/python/test_buffer_numpy.py +++ b/python-multiplatform-ksp/src/test/python/test_buffer_numpy.py @@ -21,13 +21,13 @@ from test_buffer_arrays import build_source, load_so # noqa: E402 SOURCE = textwrap.dedent(''' - # typedpython: compiled import array import math import numpy as np import numpy.typing as npt + @compiled def sum_ad(a: array.array[float]) -> float: s = 0.0 for i in range(len(a)): @@ -35,18 +35,22 @@ def sum_ad(a: array.array[float]) -> float: return s + @compiled def nd_first(x: npt.NDArray[np.float64]) -> object: return x[0] + @compiled def nd_first_f(x: npt.NDArray[np.float64]) -> float: return x[0] + @compiled def nd_first_i(x: npt.NDArray[np.int64]) -> int: return x[0] + @compiled def nd_sum(x: npt.NDArray[np.float64]) -> float: s = 0.0 for i in range(len(x)): @@ -54,6 +58,7 @@ def nd_sum(x: npt.NDArray[np.float64]) -> float: return s + @compiled def nd_sum_from_first(x: npt.NDArray[np.float64]) -> float: s = x[0] for i in range(1, len(x)): @@ -61,6 +66,7 @@ def nd_sum_from_first(x: npt.NDArray[np.float64]) -> float: return s + @compiled def nd_isum(x: npt.NDArray[np.int64]) -> int: s = 0 for i in range(len(x)): @@ -68,64 +74,79 @@ def nd_isum(x: npt.NDArray[np.int64]) -> int: return s + @compiled def nd_gt(x: npt.NDArray[np.float64]) -> object: return x[0] > x[1] + @compiled def nd_flag(x: npt.NDArray[np.float64]) -> object: b = x[0] > 0.0 return b + @compiled def nd_plain(x: npt.NDArray[np.float64]) -> object: return float(x[0]) + @compiled def nd_mixed(x: npt.NDArray[np.float64], k: float) -> object: return k + 1.0 + @compiled def nd_sqrt(x: npt.NDArray[np.float64]) -> object: return math.sqrt(x[0] * x[0]) + @compiled def nd_neg(x: npt.NDArray[np.float64]) -> object: return -x[1] + @compiled def nd_len(x: npt.NDArray[np.float64]) -> int: return len(x) + @compiled def nd_set(x: npt.NDArray[np.float64], i: int, v: float) -> None: x[i] = v + @compiled def nd_copy(dst: npt.NDArray[np.float64], src: npt.NDArray[np.float64]) -> None: for i in range(len(dst)): dst[i] = src[i] + @compiled def nd_iset(x: npt.NDArray[np.int64], i: int, v: int) -> None: x[i] = v + @compiled def nd_get(x: npt.NDArray[np.float64], i: int) -> object: return x[i] + @compiled def nd_overflow(x: npt.NDArray[np.float64]) -> object: return x[0] * x[1] + @compiled def nd_invalid(x: npt.NDArray[np.float64]) -> object: return x[0] - x[1] + @compiled def nd_iover(x: npt.NDArray[np.int64]) -> object: return x[0] + x[1] + @compiled def nd_energy(x: npt.NDArray[np.float64], y: npt.NDArray[np.float64], z: npt.NDArray[np.float64], vx: npt.NDArray[np.float64], vy: npt.NDArray[np.float64], vz: npt.NDArray[np.float64], m: npt.NDArray[np.float64]) -> float: @@ -135,6 +156,7 @@ def nd_energy(x: npt.NDArray[np.float64], y: npt.NDArray[np.float64], z: npt.NDA return e + @compiled def nd_copy7(x: npt.NDArray[np.float64], y: npt.NDArray[np.float64], z: npt.NDArray[np.float64], vx: npt.NDArray[np.float64], vy: npt.NDArray[np.float64], vz: npt.NDArray[np.float64], m: npt.NDArray[np.float64]) -> None: @@ -145,20 +167,24 @@ def nd_copy7(x: npt.NDArray[np.float64], y: npt.NDArray[np.float64], z: npt.NDAr m[i] = x[i] + @compiled def nd_div(x: npt.NDArray[np.float64]) -> object: return x[0] / x[1] + @compiled def nd_idiv(x: npt.NDArray[np.int64]) -> object: return x[0] // x[1] + @compiled def nd_param(x: npt.NDArray[np.float64], k: float, use: bool) -> object: if use: k = x[0] return k + @compiled def nd_branch(x: npt.NDArray[np.float64], use: bool) -> object: if use: v = x[0] @@ -167,36 +193,44 @@ def nd_branch(x: npt.NDArray[np.float64], use: bool) -> object: return v + @compiled def nd_tuple(x: npt.NDArray[np.int64]) -> tuple[int, int, bool]: return (x[0], 3, x[1] > 0) + @compiled def nd_abs(x: npt.NDArray[np.float64]) -> object: return abs(x[1]) + @compiled def nd_to_list(x: npt.NDArray[np.float64], dst: list[float]) -> None: dst[0] = x[0] + @compiled def nd_mixed_cmp(x: npt.NDArray[np.int64], f: float) -> object: if x[0] < f: return 1 return 0 + @compiled def whole_sum(x: npt.NDArray[np.float64]) -> object: return np.sum(x) + @compiled def whole_attr(x: npt.NDArray[np.float64]) -> object: return x.size + @compiled def whole_pass(x: npt.NDArray[np.float64]) -> object: return np.add(x, 1.0).sum() + @compiled def nd_int_impure(x: npt.NDArray[np.int64]) -> object: s = 0 for i in range(len(x)): @@ -205,83 +239,103 @@ def nd_int_impure(x: npt.NDArray[np.int64]) -> object: return s + @compiled def nd_scale(x: npt.NDArray[np.float64], k: float) -> None: for i in range(len(x)): x[i] = x[i] * k + @compiled def p_div(x: npt.NDArray[np.float64]) -> object: return x[0] / x[1] + @compiled def p_fdiv(x: npt.NDArray[np.float64]) -> object: return x[0] // x[1] + @compiled def p_mod(x: npt.NDArray[np.float64]) -> object: return x[0] % x[1] + @compiled def p_add(x: npt.NDArray[np.float64]) -> object: return x[0] + x[1] + @compiled def p_sub(x: npt.NDArray[np.float64]) -> object: return x[0] - x[1] + @compiled def p_mul(x: npt.NDArray[np.float64]) -> object: return x[0] * x[1] + @compiled def p_pdiv(x: npt.NDArray[np.float64], k: float) -> object: return k / x[0] + @compiled def p_idiv(x: npt.NDArray[np.int64]) -> object: return x[0] / x[1] + @compiled def p_ifdiv(x: npt.NDArray[np.int64]) -> object: return x[0] // x[1] + @compiled def p_imod(x: npt.NDArray[np.int64]) -> object: return x[0] % x[1] + @compiled def p_imixed(x: npt.NDArray[np.int64]) -> object: return x[0] // 3 + x[1] % 4 + @compiled def i_div(x: npt.NDArray[np.float64]) -> None: x[2] = x[0] / x[1] + @compiled def i_fdiv(x: npt.NDArray[np.float64]) -> None: x[2] = x[0] // x[1] + @compiled def i_mod(x: npt.NDArray[np.float64]) -> None: x[2] = x[0] % x[1] + @compiled def i_add(x: npt.NDArray[np.float64]) -> None: x[2] = x[0] + x[1] + @compiled def i_sub(x: npt.NDArray[np.float64]) -> None: x[2] = x[0] - x[1] + @compiled def i_mul(x: npt.NDArray[np.float64]) -> None: x[2] = x[0] * x[1] + @compiled def i_plain(x: npt.NDArray[np.float64], k: float) -> None: x[2] = k / x[1] x[3] = k * 2.0 + @compiled def i_multiline(x: npt.NDArray[np.float64]) -> None: x[2] = (x[0] * x[1]) @@ -290,12 +344,14 @@ def i_multiline(x: npt.NDArray[np.float64]) -> None: + x[0] * x[1]) + @compiled def i_aug(x: npt.NDArray[np.float64]) -> None: x[0] *= x[1] x[1] /= x[2] x[2] += x[3] + @compiled def i_loop(x: npt.NDArray[np.float64], y: npt.NDArray[np.float64]) -> float: s = 0.0 for i in range(len(x)): @@ -304,21 +360,25 @@ def i_loop(x: npt.NDArray[np.float64], y: npt.NDArray[np.float64]) -> float: return s + @compiled def i_mixed(x: npt.NDArray[np.float64], n: int) -> None: x[0] = n / x[1] x[2] = x[3] * n + @compiled def i_ret(x: npt.NDArray[np.float64]) -> object: x[2] = x[0] * x[1] return x[0] / x[1] + @compiled def i_ifdiv(x: npt.NDArray[np.int64]) -> object: x[2] = x[0] // x[1] return x[2] + @compiled def nd_advance(dt: float, steps: int, x: npt.NDArray[np.float64], y: npt.NDArray[np.float64], z: npt.NDArray[np.float64], vx: npt.NDArray[np.float64], vy: npt.NDArray[np.float64], vz: npt.NDArray[np.float64], @@ -343,6 +403,7 @@ def nd_advance(dt: float, steps: int, x: npt.NDArray[np.float64], y: npt.NDArray z[i] += dt * vz[i] + @compiled def nd_energy_full(x: npt.NDArray[np.float64], y: npt.NDArray[np.float64], z: npt.NDArray[np.float64], vx: npt.NDArray[np.float64], vy: npt.NDArray[np.float64], vz: npt.NDArray[np.float64], @@ -357,10 +418,12 @@ def nd_energy_full(x: npt.NDArray[np.float64], y: npt.NDArray[np.float64], e -= mass[i] * mass[j] / math.sqrt(dx * dx + dy * dy + dz * dz) return e # #261: np.bool_ arrays (1-byte items, format "?") + @compiled def nd_b_get(x: npt.NDArray[np.bool_], i: int) -> object: return x[i] + @compiled def nd_b_any(x: npt.NDArray[np.bool_]) -> bool: for i in range(len(x)): if x[i]: @@ -368,6 +431,7 @@ def nd_b_any(x: npt.NDArray[np.bool_]) -> bool: return False + @compiled def nd_b_count(x: npt.NDArray[np.bool_]) -> int: c = 0 for i in range(len(x)): @@ -376,45 +440,55 @@ def nd_b_count(x: npt.NDArray[np.bool_]) -> int: return c + @compiled def nd_b_set(x: npt.NDArray[np.bool_], i: int, v: bool) -> None: x[i] = v + @compiled def nd_b_flip(x: npt.NDArray[np.bool_]) -> None: for i in range(len(x)): x[i] = not x[i] + @compiled def nd_b_copy(a: npt.NDArray[np.bool_], b: npt.NDArray[np.bool_]) -> None: for i in range(len(a)): b[i] = a[i] + @compiled def nd_b_eq(x: npt.NDArray[np.bool_], y: npt.NDArray[np.bool_], i: int) -> object: return x[i] == y[i] + @compiled def nd_b_gt(x: npt.NDArray[np.float64], m: npt.NDArray[np.bool_]) -> None: for i in range(len(x)): m[i] = x[i] > 2.0 + @compiled def nd_b_len(x: npt.NDArray[np.bool_]) -> int: return len(x) + @compiled def nd_b_store_int(x: npt.NDArray[np.bool_], i: int, v: int) -> None: x[i] = v + @compiled def nd_b_store_float(x: npt.NDArray[np.bool_], i: int, v: float) -> None: x[i] = v + @compiled def nd_b_arith(x: npt.NDArray[np.bool_], i: int) -> object: return x[i] + 1 + @compiled def nd_b_whole(x: npt.NDArray[np.bool_]) -> object: return np.sum(x) ''') diff --git a/python-multiplatform-ksp/src/test/python/test_class_methods.py b/python-multiplatform-ksp/src/test/python/test_class_methods.py index d6693dcad..cd08794e3 100644 --- a/python-multiplatform-ksp/src/test/python/test_class_methods.py +++ b/python-multiplatform-ksp/src/test/python/test_class_methods.py @@ -40,6 +40,7 @@ import pytest +from conftest import decorate_compiled from typedpython import cbuild, cgen, cost, frontend, pipeline, verify HERE = Path(__file__).resolve().parent @@ -51,10 +52,10 @@ BINARY_TREES = REPO / "benchmarks" / "typedpython" / "py" / "binary_trees.py" SOURCE = textwrap.dedent('''\ - # typedpython: compiled """typedpython class-method test module.""" + @compiled class Node: __slots__ = ("left", "right") left: "Node | None" @@ -135,21 +136,25 @@ def no_annotation(self, x) -> int: return 1 + @compiled class Plain: def check(self) -> int: return 5 + @compiled def make(depth: int) -> Node: if depth > 0: return Node(make(depth - 1), make(depth - 1)) return Node(None, None) + @compiled def check_of(n: Node) -> int: return n.check() + @compiled def leaf_of(n: Node) -> bool: return n.leaf() ''') @@ -231,6 +236,7 @@ def test_the_methods_of_a_class_that_is_not_fixed_layout_are_not_considered(tmp_ def test_a_method_named_like_a_module_function_does_not_collide(tmp_path): m = lower(tmp_path, SOURCE + textwrap.dedent('''\ + @compiled def height(n: Node) -> int: return n.height() ''')) @@ -240,7 +246,7 @@ def height(n: Node) -> int: def test_a_method_with_a_foreign_self_annotation_stays_interpreted(tmp_path): src = textwrap.dedent('''\ - # typedpython: compiled + @compiled class A: __slots__ = ("x",) x: int @@ -258,7 +264,7 @@ def get(self: int) -> int: def test_a_method_defined_twice_in_the_class_body_stays_interpreted(tmp_path): src = textwrap.dedent('''\ - # typedpython: compiled + @compiled class A: __slots__ = ("x",) x: int @@ -277,7 +283,7 @@ def get(self) -> int: assert "more than once" in m.skipped["A.get"] -def test_an_at_compiled_class_compiles_its_methods_without_the_module_marker(tmp_path): +def test_an_at_compiled_class_compiles_its_methods(tmp_path): src = textwrap.dedent('''\ @compiled class A: @@ -887,7 +893,7 @@ def run_main(path_or_module, n): def test_binary_trees_with_check_as_a_method_compiles_and_matches(tmp_path): - text = "# typedpython: compiled\n" + method_form(BINARY_TREES.read_text()) + text = decorate_compiled(method_form(BINARY_TREES.read_text())) path = tmp_path / "bt_method.py" path.write_text(text) result = pipeline.compile_module(path, tmp_path / "out") @@ -921,8 +927,8 @@ def test_binary_trees_with_check_as_a_method_compiles_and_matches(tmp_path): @pytest.mark.parametrize("name", sorted(EXPECTED)) def test_the_benchmarks_compile_the_same_functions_as_before(tmp_path, name): path = tmp_path / f"{name}.py" - path.write_text("# typedpython: compiled\n" + (REPO / "benchmarks" / "typedpython" / "py" - / f"{name}.py").read_text()) + path.write_text(decorate_compiled((REPO / "benchmarks" / "typedpython" / "py" + / f"{name}.py").read_text())) proved, diags = verify.verify(frontend.lower(path)) proved, diags = cost.apply(proved, diags) assert {f.name for f in proved.functions} == EXPECTED[name] diff --git a/python-multiplatform-ksp/src/test/python/test_cost_rule.py b/python-multiplatform-ksp/src/test/python/test_cost_rule.py index db6c4999a..c540bb260 100644 --- a/python-multiplatform-ksp/src/test/python/test_cost_rule.py +++ b/python-multiplatform-ksp/src/test/python/test_cost_rule.py @@ -12,20 +12,19 @@ import pytest +from conftest import decorate_compiled from typedpython import cost, pipeline -HEADER = "# typedpython: compiled\n" - -OBJECT_ONLY = HEADER + textwrap.dedent(''' +OBJECT_ONLY = decorate_compiled(textwrap.dedent(''' def copy_words(src: list[str], dst: list[str]) -> str: w: str = "" for x in src: w = x + w dst.append(w) return w -''') +''')) -INT_LOOP = HEADER + textwrap.dedent(''' +INT_LOOP = decorate_compiled(textwrap.dedent(''' def word_of(word_id: int) -> str: x: int = word_id + 17576 w: str = "" @@ -33,7 +32,7 @@ def word_of(word_id: int) -> str: w = chr(97 + x % 26) + w x //= 26 return w -''') +''')) def build(tmp_path, source, **kw): @@ -67,7 +66,7 @@ def test_the_opt_out_compiles_the_same_function(tmp_path): assert names(r) == {"copy_words"} and r.extension is not None -MIXED = HEADER + textwrap.dedent(''' +MIXED = decorate_compiled(textwrap.dedent(''' def mixed(n: int, xs: list[str]) -> float: total: float = 0.0 for i in range(n): @@ -106,7 +105,7 @@ def native_loop_inside_object_loop(rows: list[list[str]]) -> float: for k in range(3): total += 0.5 * 2.0 return total -''') +''')) def test_one_native_loop_with_object_operations_compiles(tmp_path): @@ -148,10 +147,10 @@ def test_the_rule_is_what_keeps_it_interpreted(tmp_path, monkeypatch): test_an_object_only_loop_stays_interpreted_with_the_reason(tmp_path) -STRAIGHT_LINE = HEADER + textwrap.dedent(''' +STRAIGHT_LINE = decorate_compiled(textwrap.dedent(''' def label(n: int, name: str) -> str: return f"{name}:{n + 1}" -''') +''')) def test_a_straight_line_impure_function_still_compiles(tmp_path): diff --git a/python-multiplatform-ksp/src/test/python/test_eval_breaker.py b/python-multiplatform-ksp/src/test/python/test_eval_breaker.py index 51f1bca43..0d6bb643b 100644 --- a/python-multiplatform-ksp/src/test/python/test_eval_breaker.py +++ b/python-multiplatform-ksp/src/test/python/test_eval_breaker.py @@ -23,9 +23,9 @@ posix_signals = pytest.mark.skipif(sys.platform == "win32", reason="needs POSIX signals (SIGALRM/SIGUSR1/setitimer) or fork; Windows has none") SOURCE = """\ -# typedpython: compiled +@compiled def spin(n: int) -> float: acc: float = 0.0 for k in range(n): @@ -33,12 +33,14 @@ def spin(n: int) -> float: return acc +@compiled def fib(n: int) -> int: if n < 2: return n return fib(n - 1) + fib(n - 2) +@compiled def total(xs: list[float], rounds: int) -> float: s: float = 0.0 for r in range(rounds): @@ -165,9 +167,9 @@ def handler(signum, frame): # ---- chunked polling for range loops (#206, SPEC N-8) --------------------------------------------- CHUNK_SOURCE = """\ -# typedpython: compiled +@compiled def brk(n: int, k: int) -> int: s: int = 0 last: int = -1 @@ -179,6 +181,7 @@ def brk(n: int, k: int) -> int: return s * 3 + last +@compiled def cont(n: int, m: int) -> int: s: int = 0 for i in range(n): @@ -188,6 +191,7 @@ def cont(n: int, m: int) -> int: return s +@compiled def ret(n: int, k: int) -> int: for i in range(n): if i == k: @@ -195,6 +199,7 @@ def ret(n: int, k: int) -> int: return -1 +@compiled def down(n: int) -> int: s: int = 0 for i in range(n, 0, -3): @@ -202,6 +207,7 @@ def down(n: int) -> int: return s +@compiled def down1(n: int) -> int: s: int = 0 for i in range(n, -1, -1): @@ -209,6 +215,7 @@ def down1(n: int) -> int: return s +@compiled def up3(a: int, b: int) -> int: s: int = 0 for i in range(a, b, 3): @@ -216,6 +223,7 @@ def up3(a: int, b: int) -> int: return s +@compiled def nested(n: int, m: int) -> int: s: int = 0 for i in range(n): @@ -226,6 +234,7 @@ def nested(n: int, m: int) -> int: return s +@compiled def brk_in_while(n: int) -> int: s: int = 0 for i in range(n): @@ -238,6 +247,7 @@ def brk_in_while(n: int) -> int: return s +@compiled def huge(a: int, b: int, cap: int) -> int: c: int = 0 for i in range(a, b): @@ -247,6 +257,7 @@ def huge(a: int, b: int, cap: int) -> int: return c +@compiled def huge_last(a: int, b: int, cap: int) -> int: c: int = 0 last: int = 0 @@ -346,7 +357,7 @@ def test_range_loop_polls_once_per_chunk(chunk_module): def test_while_loop_keeps_its_per_iteration_countdown(tmp_path): path = tmp_path / "w.py" - path.write_text("# typedpython: compiled\n\n\ndef w(n: int) -> int:\n i: int = 0\n while i < n:\n" + path.write_text("\n\n@compiled\ndef w(n: int) -> int:\n i: int = 0\n while i < n:\n" " i += 1\n return i\n") body = _function_c(_generated_c(path), "w") lines = body.splitlines() @@ -379,9 +390,9 @@ def test_keyboard_interrupt_reaches_a_long_range_loop_in_chunks(extension): MID_SOURCE = """\ -# typedpython: compiled +@compiled def mid(n: int) -> float: x: float = 1.0 for k in range(n): diff --git a/python-multiplatform-ksp/src/test/python/test_fast_entry.py b/python-multiplatform-ksp/src/test/python/test_fast_entry.py index f1e43a1ac..d448e3b41 100644 --- a/python-multiplatform-ksp/src/test/python/test_fast_entry.py +++ b/python-multiplatform-ksp/src/test/python/test_fast_entry.py @@ -25,45 +25,53 @@ posix_signals = pytest.mark.skipif(sys.platform == "win32", reason="needs POSIX signals (SIGALRM/SIGUSR1/setitimer) or fork; Windows has none") SOURCE = """\ -# typedpython: compiled +@compiled def helper(x: int) -> float: d: dict[str, int] = {} d['a'] = x return d['a'] * 2.5 +@compiled def leaf(i: int) -> float: return 1.0 / (i + 1) +@compiled def mid(i: int) -> float: return leaf(i) + leaf(i + 1) +@compiled def top(i: int) -> float: return mid(i) * 2.0 +@compiled def objy(i: int) -> float: return float(helper(i)) + leaf(i) +@compiled def viaobj(i: int) -> float: return objy(i) + 1.0 +@compiled def rec(n: int) -> int: if n == 0: return 0 return 1 + rec(n - 1) +@compiled def callrec(n: int) -> int: return rec(n) + 1 +@compiled def loopy(n: int) -> float: s: float = 0.0 for i in range(n): @@ -71,14 +79,17 @@ def loopy(n: int) -> float: return s +@compiled def outer(n: int) -> float: return loopy(n) + 1.0 +@compiled def cube(x: int) -> float: return float(x * x * x) + leaf(x) +@compiled def fill(x: int, out: list[float]) -> None: out[0] = cube(x) """ diff --git a/python-multiplatform-ksp/src/test/python/test_free_threaded.py b/python-multiplatform-ksp/src/test/python/test_free_threaded.py index 378395e78..e954c03ba 100644 --- a/python-multiplatform-ksp/src/test/python/test_free_threaded.py +++ b/python-multiplatform-ksp/src/test/python/test_free_threaded.py @@ -33,9 +33,9 @@ FREE_THREADED = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) SOURCE = '''\ -# typedpython: compiled +@compiled class Cell: __slots__ = ("a", "b") a: object @@ -46,6 +46,7 @@ def __init__(self, a: object, b: object) -> None: self.b = b +@compiled class Twin: __slots__ = ("a", "b") a: object @@ -56,18 +57,22 @@ def __init__(self, a: object, b: object) -> None: self.b = b +@compiled def geta(c: Cell) -> object: return c.a +@compiled def seta(c: Cell, v: object) -> None: c.a = v +@compiled def mk(a: object, b: object) -> Cell: return Cell(a, b) +@compiled def fresh(a: object, n: int) -> object: c = Cell(a, None) keep: object = c # read as a value: c is a real object, not virtual (N-12a) @@ -77,11 +82,13 @@ def fresh(a: object, n: int) -> object: return s +@compiled def vpick(a: object, b: object) -> object: c = Cell(a, b) return c.b +@compiled def total(xs: list[float], rounds: int) -> float: s: float = 0.0 for r in range(rounds): @@ -90,24 +97,29 @@ def total(xs: list[float], rounds: int) -> float: return s +@compiled def fill(xs: list[float], v: float) -> None: for i in range(len(xs)): xs[i] = v +@compiled def edge(xs: list[float]) -> float: return xs[0] - xs[len(xs) - 1] +@compiled def fill_int(xs: list[int], v: int) -> None: for i in range(len(xs)): xs[i] = v +@compiled def edge_int(xs: list[int]) -> int: return xs[0] - xs[len(xs) - 1] +@compiled def rotate(xs: list[float]) -> None: if len(xs) > 1: first: float = xs[0] @@ -116,6 +128,7 @@ def rotate(xs: list[float]) -> None: xs[len(xs) - 1] = first +@compiled def zero2(xs: list[int], ys: list[int]) -> int: for i in range(len(xs)): xs[i] = 0 @@ -124,10 +137,12 @@ def zero2(xs: list[int], ys: list[int]) -> int: return 0 +@compiled def pair_edge(xs: list[int], ys: list[int]) -> int: return xs[0] - ys[len(ys) - 1] +@compiled def three(a: list[int], b: list[int], c: list[int]) -> int: return len(a) + len(b) + len(c) @@ -135,6 +150,7 @@ def three(a: list[int], b: list[int], c: list[int]) -> int: K: int = 3 +@compiled def usek(x: int) -> int: return x + K ''' @@ -244,7 +260,7 @@ def test_three_list_parameters_stay_interpreted(built): def test_front_end_gives_a_clear_reason(tmp_path): p = tmp_path / "m.py" - p.write_text("# typedpython: compiled\n\n\ndef f(a: list[int], b: list[float], c: list[int]) -> int:\n" + p.write_text("\n\n@compiled\ndef f(a: list[int], b: list[float], c: list[int]) -> int:\n" " return len(a)\n") m = frontend.lower(p) assert not m.functions diff --git a/python-multiplatform-ksp/src/test/python/test_frontend.py b/python-multiplatform-ksp/src/test/python/test_frontend.py index 8b2a55fd9..fe8f4d94c 100644 --- a/python-multiplatform-ksp/src/test/python/test_frontend.py +++ b/python-multiplatform-ksp/src/test/python/test_frontend.py @@ -12,6 +12,7 @@ import pytest +from conftest import decorate_compiled from typedpython import frontend from typedpython.ir import ( And, ArrayParam, Assign, BinOp, BinOpKind, Box, Break, Call, CallObject, Compare, CompareKind, @@ -54,7 +55,7 @@ def nodes(root, cls): def test_scalar_loop_lowers_to_for_range(tmp_path): m = lower(tmp_path, """ - # typedpython: compiled + @compiled def total(n: int, x: float) -> float: acc: float = 0.0 for k in range(n): @@ -633,16 +634,18 @@ def f(x: float) -> object: assert Global(OBJ, "K") in nodes(f, Global) -def test_module_marker_compiles_every_module_level_def(tmp_path): +def test_compiled_marks_every_decorated_module_level_def(tmp_path): m = lower(tmp_path, """ - # typedpython: compiled + @compiled def a(x: float) -> float: return x + @compiled def b(x: float) -> float: return -x + @compiled @staticmethod def c(x: float) -> float: return x @@ -651,7 +654,19 @@ def c(x: float) -> float: assert set(m.skipped) == {"c"} and "decorator" in m.skipped["c"] -def test_without_marker_only_compiled_functions_are_lowered(tmp_path): +def test_the_former_module_pragma_comment_compiles_nothing(tmp_path): + m = lower(tmp_path, """ + # typedpython: compiled + def a(x: float) -> float: + return x + + def b(x: float) -> float: + return -x + """) + assert m.functions == () and m.skipped == {} + + +def test_without_a_decorator_only_compiled_functions_are_lowered(tmp_path): m = lower(tmp_path, """ x: int = 1 # typedpython: compiled @@ -828,7 +843,7 @@ def f(n: int) -> int: def test_nbody_kernels_lower(tmp_path): path = tmp_path / "nbody.py" shutil.copy(BENCH / "nbody.py", path) - path.write_text("# typedpython: compiled\n" + path.read_text()) + path.write_text(decorate_compiled(path.read_text())) m = frontend.lower(path) for name in ("advance", "energy", "offset_momentum"): function(m, name) @@ -847,7 +862,7 @@ def test_nbody_kernels_lower(tmp_path): @pytest.mark.skipif(not BENCH.exists(), reason="benchmark worktree not present") def test_spectral_norm_kernels_lower(tmp_path): path = tmp_path / "spectral_norm.py" - path.write_text("# typedpython: compiled\n" + (BENCH / "spectral_norm.py").read_text()) + path.write_text(decorate_compiled((BENCH / "spectral_norm.py").read_text())) m = frontend.lower(path) eval_a = function(m, "eval_a") assert eval_a.pure and eval_a.may_deopt @@ -1048,7 +1063,7 @@ def f(a: int, b: int) -> {annotation}: @pytest.mark.skipif(not BENCH.exists(), reason="benchmark worktree not present") def test_fannkuch_lowers(tmp_path): path = tmp_path / "fannkuch.py" - path.write_text("# typedpython: compiled\n" + (BENCH / "fannkuch.py").read_text()) + path.write_text(decorate_compiled((BENCH / "fannkuch.py").read_text())) m = frontend.lower(path) assert "fannkuch" in [f.name for f in m.functions], m.skipped.get("fannkuch") f = function(m, "fannkuch") diff --git a/python-multiplatform-ksp/src/test/python/test_frontend_classes.py b/python-multiplatform-ksp/src/test/python/test_frontend_classes.py index 33c0473cc..a2daaeed9 100644 --- a/python-multiplatform-ksp/src/test/python/test_frontend_classes.py +++ b/python-multiplatform-ksp/src/test/python/test_frontend_classes.py @@ -18,6 +18,7 @@ import pytest +from conftest import decorate_compiled from typedpython import frontend from typedpython.ir import ( And, Assign, Box, Call, CallObject, CheckExact, ClassDecl, Const, ExprStmt, FieldGet, FieldSet, @@ -28,8 +29,6 @@ BINARY_TREES = Path(__file__).resolve().parents[4] / "benchmarks/typedpython/py/binary_trees.py" SLOTS_LINE = ' __slots__ = ("left", "right")\n' -MARK = "# typedpython: compiled\n" - POINT = """ class P: __slots__ = ("x", "y") @@ -44,7 +43,7 @@ def __init__(self, x: float, y: float) -> None: def lower(tmp_path, source, name="mod.py"): path = tmp_path / name - path.write_text(MARK + textwrap.dedent(source).lstrip("\n")) + path.write_text(decorate_compiled(textwrap.dedent(source).lstrip("\n"))) return frontend.lower(path) @@ -573,7 +572,7 @@ def test_binary_trees_with_slots_lowers_make_and_check(tmp_path): source = BINARY_TREES.read_text() assert "class Node:\n" + SLOTS_LINE in source # the benchmark itself declares the slots path = tmp_path / "binary_trees.py" - path.write_text(MARK + source) + path.write_text(decorate_compiled(source)) m = frontend.lower(path) assert cls(m, "Node") == ClassDecl("Node", ("left", "right"), True) @@ -598,7 +597,7 @@ def test_binary_trees_without_slots_stays_object_code(tmp_path): path = tmp_path / "binary_trees.py" source = BINARY_TREES.read_text() assert SLOTS_LINE in source - path.write_text(MARK + source.replace(SLOTS_LINE, "", 1)) # the run.py no-slots baseline + path.write_text(decorate_compiled(source.replace(SLOTS_LINE, "", 1))) # the run.py no-slots baseline m = frontend.lower(path) assert m.classes == () assert "`__slots__`" in m.skipped["Node"] diff --git a/python-multiplatform-ksp/src/test/python/test_interval_proofs.py b/python-multiplatform-ksp/src/test/python/test_interval_proofs.py index 045a593e3..64260ddf7 100644 --- a/python-multiplatform-ksp/src/test/python/test_interval_proofs.py +++ b/python-multiplatform-ksp/src/test/python/test_interval_proofs.py @@ -330,39 +330,44 @@ def test_a_mod_result_can_feed_a_proof(): # --- 2. end to end --------------------------------------------------------------------------------- SOURCE = '''\ -# typedpython: compiled +@compiled def dec_if_positive(n: int) -> int: if n > 0: return n - 1 return -1 +@compiled def dec_after_guard(n: int) -> int: if n <= 0: return 0 return n - 1 +@compiled def inc_below_max(n: int) -> int: if n < 9223372036854775807: return n + 1 return 0 +@compiled def inc_below_100(n: int) -> int: if n < 100: return n + 1 return 0 +@compiled def inc_if_positive(n: int) -> int: if n > 0: return n + 1 return 0 +@compiled def reverse_gap(a: int, b: int) -> int: lo: int = a hi: int = b @@ -374,6 +379,7 @@ def reverse_gap(a: int, b: int) -> int: return hi - lo + steps +@compiled def killed_guard(n: int, a: list[int]) -> int: if n < 100: n = a[0] @@ -381,6 +387,7 @@ def killed_guard(n: int, a: list[int]) -> int: return 0 +@compiled def killed_in_loop(c: int, a: list[int]) -> int: n: int = a[0] t: int = 0 @@ -391,6 +398,7 @@ def killed_in_loop(c: int, a: list[int]) -> int: return t +@compiled def parity(n: int) -> int: k: int = 0 s: int = 0 @@ -401,11 +409,13 @@ def parity(n: int) -> int: return s +@compiled def const_local() -> int: x: int = 41 return x + 1 +@compiled def range_shifts(a: int, b: int) -> int: if a < 0: return 0 @@ -415,26 +425,31 @@ def range_shifts(a: int, b: int) -> int: return t +@compiled def guarded_shift(n: int) -> int: if n >= 0: return n // 16 + n % 16 return 0 +@compiled def negative_shift(n: int) -> int: return n // 4 + n % 8 +@compiled def odd_divisor(n: int) -> int: if n >= 0: return n // 3 + n % 6 return 0 +@compiled def mod_then_add(n: int) -> int: return n % 10 + 1 +@compiled def pair_sums(a: list[int]) -> int: best: int = 0 for i in range(len(a)): diff --git a/python-multiplatform-ksp/src/test/python/test_module_report.py b/python-multiplatform-ksp/src/test/python/test_module_report.py index 8625cf18d..86ead4b97 100644 --- a/python-multiplatform-ksp/src/test/python/test_module_report.py +++ b/python-multiplatform-ksp/src/test/python/test_module_report.py @@ -8,16 +8,17 @@ import textwrap from pathlib import Path +from conftest import decorate_compiled from typedpython import cbuild, cgen, frontend, pipeline, verify -SRC = "# typedpython: compiled\n" + textwrap.dedent("""\ +SRC = decorate_compiled(textwrap.dedent("""\ def twice(n: int) -> int: return n * 2 def boom(s: str) -> str: raise ValueError("boom " + s) - """) + """)) def make_project(root: Path) -> Path: @@ -95,14 +96,14 @@ def test_a_moved_file_with_the_same_content_does_not_reuse_the_extension(tmp_pat # --- #128 ---------------------------------------------------------------------------------------- -MIXED = "# typedpython: compiled\n" + textwrap.dedent("""\ +MIXED = decorate_compiled(textwrap.dedent("""\ def ok(n: int) -> int: return n + 1 def greet(name: str) -> str: return name[1:] - """) + """)) def build_one(tmp_path, rel, source, cache="cache"): @@ -119,20 +120,20 @@ def test_kind_compiled(tmp_path): assert m.kind == "compiled" and m.extension is not None [item] = m.skipped_items assert item.name == "greet" and item.file == "pkg/mixed.py" - assert item.line == 7 and not item.message.startswith("line ") - assert m.skipped["greet"].startswith("line 7: ") # ModuleResult.skipped unchanged + assert item.line == 8 and not item.message.startswith("line ") + assert m.skipped["greet"].startswith("line 8: ") # ModuleResult.skipped unchanged def test_kind_nothing_compiled_with_structured_items(tmp_path): _, _, m = build_one(tmp_path, "pkg/plain.py", - "# typedpython: compiled\n\ndef f(s: str) -> str:\n return s[1:]\n") + "@compiled\ndef f(s: str) -> str:\n return s[1:]\n") assert m.kind == "nothing_compiled" and m.extension is None [item] = m.skipped_items - assert (item.name, item.file, item.line) == ("f", "pkg/plain.py", 4) + assert (item.name, item.file, item.line) == ("f", "pkg/plain.py", 3) assert item.message and not re.match(r"line \d+", item.message) -def test_kind_not_marked_when_there_is_neither_marker_nor_decorator(tmp_path): +def test_kind_not_marked_when_there_is_no_decorator(tmp_path): _, _, m = build_one(tmp_path, "pkg/unmarked.py", "def f(x: int) -> int:\n return x\n") assert m.kind == "not_marked" and m.extension is None and m.skipped_items == [] @@ -145,15 +146,15 @@ def test_a_compiled_decorator_alone_opts_in(tmp_path): def test_class_items_carry_name_and_line(tmp_path): - src = ("# typedpython: compiled\n\n\nclass Node:\n __slots__ = ('a',)\n" + src = ("@compiled\nclass Node:\n __slots__ = ('a',)\n" " def __init__(self, a: int) -> None:\n self.a = a\n") _, _, m = build_one(tmp_path, "pkg/cls.py", src) cls = [i for i in m.skipped_items if i.name == "Node"] - assert cls and cls[0].line == 4 and cls[0].file == "pkg/cls.py" + assert cls and cls[0].line == 2 and cls[0].file == "pkg/cls.py" def test_verifier_skips_carry_a_line_and_no_prefix(tmp_path): - src = "# typedpython: compiled\n\n\ndef f(x: int) -> int:\n return g(x)\n\n\ndef g(x: int) -> int:\n return x\n" + src = "@compiled\ndef f(x: int) -> int:\n return g(x)\n\n\n@compiled\ndef g(x: int) -> int:\n return x\n" _, _, m = build_one(tmp_path, "pkg/v.py", src) for i in m.skipped_items: assert isinstance(i.line, int) and not i.message.startswith("line ") @@ -170,7 +171,7 @@ def test_structure_round_trips_through_the_cache_for_reused_modules(tmp_path): def test_not_marked_and_nothing_compiled_survive_reuse(tmp_path): root, path, _ = build_one(tmp_path, "pkg/plain.py", - "# typedpython: compiled\n\ndef f(s: str) -> str:\n return s[1:]\n") + "@compiled\ndef f(s: str) -> str:\n return s[1:]\n") again = pipeline.compile_project([path], root, tmp_path / "cache").modules[0] assert again.status == "reused" and again.kind == "nothing_compiled" and again.skipped_items root2, path2, _ = build_one(tmp_path, "pkg/un.py", "def f(x: int) -> int:\n return x\n") @@ -191,5 +192,5 @@ def test_cli_build_prints_file_line_name_message_and_not_marked(tmp_path): [sys.executable, "-m", "typedpython", "build", str(root), "--cache", str(tmp_path / "c")], capture_output=True, text=True, env=dict(os.environ, PYTHONPATH=str(src))) assert out.returncode == 0, out.stdout + out.stderr - assert re.search(r"^\s*pkg/mixed\.py:7: greet: \S", out.stdout, re.M), out.stdout + assert re.search(r"^\s*pkg/mixed\.py:8: greet: \S", out.stdout, re.M), out.stdout assert re.search(r"pkg/un\.py.*not marked for compilation", out.stdout), out.stdout diff --git a/python-multiplatform-ksp/src/test/python/test_native_int_impure.py b/python-multiplatform-ksp/src/test/python/test_native_int_impure.py index f2ceeb875..2da246f90 100644 --- a/python-multiplatform-ksp/src/test/python/test_native_int_impure.py +++ b/python-multiplatform-ksp/src/test/python/test_native_int_impure.py @@ -28,13 +28,13 @@ import pytest +from conftest import decorate_compiled from typedpython import cbuild, cgen, cost, frontend, ir, verify REPO = Path(__file__).resolve().parents[4] WORDFREQ = REPO / "benchmarks" / "typedpython" / "py" / "wordfreq.py" SOURCE = textwrap.dedent(''' - # typedpython: compiled import math as GMOD @@ -42,14 +42,17 @@ MOD: int = 2147483647 + @compiled def _one() -> int: return 1 + @compiled def _two() -> int: return 2 + @compiled class GTYPE: pass @@ -57,6 +60,7 @@ class GTYPE: GFN = _one + @compiled def add2(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a + b @@ -66,6 +70,7 @@ def add2(log: list[object], a: int, b: int) -> object: return r + @compiled def sub2(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a - b @@ -75,6 +80,7 @@ def sub2(log: list[object], a: int, b: int) -> object: return r + @compiled def mul2(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a * b @@ -84,6 +90,7 @@ def mul2(log: list[object], a: int, b: int) -> object: return r + @compiled def neg2(log: list[object], a: int) -> object: log.append("start") r: int = -a @@ -93,6 +100,7 @@ def neg2(log: list[object], a: int) -> object: return r + @compiled def floordiv2(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a + 1 @@ -104,6 +112,7 @@ def floordiv2(log: list[object], a: int, b: int) -> object: return r + @compiled def mod2(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a - 1 @@ -115,6 +124,7 @@ def mod2(log: list[object], a: int, b: int) -> object: return r + @compiled def lsh(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a << b @@ -122,6 +132,7 @@ def lsh(log: list[object], a: int, b: int) -> object: return r + @compiled def rsh(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a + 1 @@ -131,6 +142,7 @@ def rsh(log: list[object], a: int, b: int) -> object: return r + @compiled def roundtrip(log: list[object], a: int) -> object: log.append(a) x: int = a @@ -145,6 +157,7 @@ def roundtrip(log: list[object], a: int) -> object: return x + @compiled def compare(log: list[object], a: int, b: int) -> None: log.append("start") x: int = a + 1 @@ -167,6 +180,7 @@ def compare(log: list[object], a: int, b: int) -> None: log.append("y-zero") + @compiled def fact(log: list[object], n: int) -> object: log.append(n) f: int = 1 @@ -176,6 +190,7 @@ def fact(log: list[object], n: int) -> object: return f + @compiled def grow(log: list[object], a: int, k: int) -> object: x: int = a i: int = 0 @@ -186,16 +201,19 @@ def grow(log: list[object], a: int, k: int) -> object: return x + @compiled def consts(log: list[object]) -> None: log.append(7000000000) log.append(5) log.append(-7000000000) + @compiled def next_state(s: int) -> int: return (s * 48271) % MOD + @compiled def drive(log: list[object], seed: int, n: int) -> object: s: int = seed + 1 for _ in range(n): @@ -204,37 +222,44 @@ def drive(log: list[object], seed: int, n: int) -> object: return s + @compiled def read_limit(log: list[object]) -> object: log.append(1) return LIMIT + @compiled def read_twice(log: list[object], cb: object) -> None: log.append(LIMIT) cb() log.append(LIMIT) + @compiled def use_builtin(log: list[object], s: str) -> object: log.append(1) return len(s) + @compiled def read_fn(log: list[object]) -> object: log.append(1) return GFN + @compiled def read_type(log: list[object]) -> object: log.append(1) return GTYPE + @compiled def read_mod(log: list[object]) -> object: log.append(1) return GMOD + @compiled def call_g(x: object) -> object: return GFN() ''') @@ -314,7 +339,7 @@ def test_int_locals_of_impure_functions_are_native(built): def test_an_int_fed_from_an_object_stays_an_object(tmp_path): src = textwrap.dedent(''' - # typedpython: compiled + @compiled def f(log: list[object], g: object) -> object: log.append(1) n: int = g() @@ -656,7 +681,7 @@ def test_a_cache_that_is_never_invalidated_is_caught(tmp_path): def test_an_int_loop_is_native_work_for_the_cost_rule(tmp_path): src = textwrap.dedent(''' - # typedpython: compiled + @compiled def digits(log: list[object], n: int) -> None: x: int = n + 1 while x > 0: @@ -672,7 +697,7 @@ def digits(log: list[object], n: int) -> None: def test_an_object_only_loop_is_still_rejected(tmp_path): src = textwrap.dedent(''' - # typedpython: compiled + @compiled def words(xs: list[object], ys: list[object]) -> None: for x in xs: ys.append(x) @@ -688,7 +713,7 @@ def words(xs: list[object], ys: list[object]) -> None: @pytest.fixture(scope="module") def wordfreq(tmp_path_factory): d = tmp_path_factory.mktemp("wordfreq") - src = "# typedpython: compiled\n" + WORDFREQ.read_text() + src = decorate_compiled(WORDFREQ.read_text()) path = d / "wordfreq.py" path.write_text(src) proved, diags = verify.verify(frontend.lower(path)) diff --git a/python-multiplatform-ksp/src/test/python/test_osr_exit.py b/python-multiplatform-ksp/src/test/python/test_osr_exit.py index fbfbdd3cf..227d50ecd 100644 --- a/python-multiplatform-ksp/src/test/python/test_osr_exit.py +++ b/python-multiplatform-ksp/src/test/python/test_osr_exit.py @@ -17,11 +17,11 @@ from typedpython import cbuild, cgen, frontend, verify SOURCE = textwrap.dedent(''' - # typedpython: compiled SCALE = 2.0 + @compiled def scale_all(x: list[float]) -> float: s = 0.0 for i in range(len(x)): @@ -30,6 +30,7 @@ def scale_all(x: list[float]) -> float: return s + @compiled def nested(x: list[float], n: int) -> int: hits = 0 for r in range(n): @@ -43,6 +44,7 @@ def nested(x: list[float], n: int) -> int: return hits + @compiled def countdown(x: list[int]) -> int: k = 0 total = 0 diff --git a/python-multiplatform-ksp/src/test/python/test_parallel_loops.py b/python-multiplatform-ksp/src/test/python/test_parallel_loops.py index 55fc184c6..e86605b69 100644 --- a/python-multiplatform-ksp/src/test/python/test_parallel_loops.py +++ b/python-multiplatform-ksp/src/test/python/test_parallel_loops.py @@ -33,26 +33,29 @@ PAR_RULE = "verify/par-loop" SOURCE = '''\ -# typedpython: compiled # typedpython: parallel +@compiled def scale(xs: list[float], a: float, b: float) -> None: for i in range(len(xs)): xs[i] = xs[i] * a + b +@compiled def scale_unmarked(xs: list[float], a: float, b: float) -> None: for i in range(len(xs)): xs[i] = xs[i] * a + b # typedpython: parallel +@compiled def ramp(xs: list[float], a: float) -> None: for i in range(len(xs)): xs[i] = -(float(i) * a) + xs[i] # typedpython: parallel +@compiled def two_stores(xs: list[float], ys: list[float], a: float) -> None: for i in range(len(xs)): xs[i] = xs[i] + a @@ -60,12 +63,14 @@ def two_stores(xs: list[float], ys: list[float], a: float) -> None: # typedpython: parallel +@compiled def neighbour(xs: list[float]) -> None: for i in range(len(xs) - 1): xs[i] = xs[i + 1] # typedpython: parallel +@compiled def total(xs: list[float]) -> float: s: float = 0.0 for i in range(len(xs)): @@ -73,23 +78,27 @@ def total(xs: list[float]) -> float: return s +@compiled def helper(x: float) -> float: return x * 2.0 # typedpython: parallel +@compiled def with_call(xs: list[float]) -> None: for i in range(len(xs)): xs[i] = helper(xs[i]) # typedpython: parallel +@compiled def divides(xs: list[float], d: float) -> None: for i in range(len(xs)): xs[i] = xs[i] / d # typedpython: parallel +@compiled def local_fill(a: float) -> float: t = [0.0] * 300000 for i in range(len(t)): @@ -98,6 +107,7 @@ def local_fill(a: float) -> float: # typedpython: parallel +@compiled def local_dyn(n: int, a: float) -> float: t = [1.0] * n for i in range(len(t)): @@ -438,9 +448,9 @@ def sh(): # ===================================================================================================== RSOURCE = '''\ -# typedpython: compiled # typedpython: parallel +@compiled def r_sum(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -449,6 +459,7 @@ def r_sum(xs: list[int]) -> int: # typedpython: parallel +@compiled def r_sum_init(xs: list[int], s0: int) -> int: s: int = s0 for i in range(len(xs)): @@ -457,6 +468,7 @@ def r_sum_init(xs: list[int], s0: int) -> int: # typedpython: parallel +@compiled def r_prod(xs: list[int], c: int) -> int: s: int = 1 for i in range(len(xs)): @@ -465,6 +477,7 @@ def r_prod(xs: list[int], c: int) -> int: # typedpython: parallel +@compiled def r_sum_mul(xs: list[int], c: int) -> int: s: int = 0 for i in range(len(xs)): @@ -473,6 +486,7 @@ def r_sum_mul(xs: list[int], c: int) -> int: # typedpython: parallel +@compiled def r_two(xs: list[int]) -> int: a: int = 0 b: int = 7 @@ -483,6 +497,7 @@ def r_two(xs: list[int]) -> int: # typedpython: parallel +@compiled def r_range_sum(xs: list[int]) -> int: s: int = 5 for i in range(len(xs)): @@ -491,6 +506,7 @@ def r_range_sum(xs: list[int]) -> int: # typedpython: parallel +@compiled def r_local(n: int) -> int: t = [0] * n for i in range(len(t)): @@ -501,6 +517,7 @@ def r_local(n: int) -> int: return s +@compiled def r_unmarked(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -509,6 +526,7 @@ def r_unmarked(xs: list[int]) -> int: # typedpython: parallel +@compiled def n_acc_read(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -517,6 +535,7 @@ def n_acc_read(xs: list[int]) -> int: # typedpython: parallel +@compiled def n_two_ops(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -526,6 +545,7 @@ def n_two_ops(xs: list[int]) -> int: # typedpython: parallel +@compiled def n_float(xs: list[float]) -> float: s: float = 0.0 for i in range(len(xs)): @@ -534,6 +554,7 @@ def n_float(xs: list[float]) -> float: # typedpython: parallel +@compiled def n_copy(xs: list[int]) -> int: s: int = 0 w: int = 0 @@ -544,6 +565,7 @@ def n_copy(xs: list[int]) -> int: # typedpython: parallel +@compiled def n_mixed(xs: list[int]) -> int: t = [0.0] * len(xs) s: int = 0 @@ -554,6 +576,7 @@ def n_mixed(xs: list[int]) -> int: # typedpython: parallel +@compiled def n_div(xs: list[int], d: int) -> int: s: int = 0 for i in range(len(xs)): @@ -562,6 +585,7 @@ def n_div(xs: list[int], d: int) -> int: # typedpython: parallel +@compiled def n_shift_step(xs: list[int]) -> int: s: int = 0 for i in range(0, len(xs), 2): @@ -859,9 +883,9 @@ def test_no_reduction_region_without_a_marker(tmp_path): # ===================================================================================================== MSOURCE = '''\ -# typedpython: compiled # typedpython: parallel +@compiled def mn_i(xs: list[int]) -> int: m = xs[0] for i in range(len(xs)): @@ -871,6 +895,7 @@ def mn_i(xs: list[int]) -> int: # typedpython: parallel +@compiled def mx_i(xs: list[int]) -> int: m = xs[0] for i in range(len(xs)): @@ -880,6 +905,7 @@ def mx_i(xs: list[int]) -> int: # typedpython: parallel +@compiled def mn_f(xs: list[float]) -> float: m = xs[0] for i in range(len(xs)): @@ -889,6 +915,7 @@ def mn_f(xs: list[float]) -> float: # typedpython: parallel +@compiled def mx_f(xs: list[float]) -> float: m = xs[0] for i in range(len(xs)): @@ -898,6 +925,7 @@ def mx_f(xs: list[float]) -> float: # typedpython: parallel +@compiled def mn_f_flipped(xs: list[float], init: float) -> float: m = init for i in range(len(xs)): @@ -907,6 +935,7 @@ def mn_f_flipped(xs: list[float], init: float) -> float: # typedpython: parallel +@compiled def mx_f_flipped(xs: list[float], init: float) -> float: m = init for i in range(len(xs)): @@ -916,6 +945,7 @@ def mx_f_flipped(xs: list[float], init: float) -> float: # typedpython: parallel +@compiled def mn_i_expr(xs: list[int], c: int) -> int: m = c for i in range(len(xs)): @@ -925,6 +955,7 @@ def mn_i_expr(xs: list[int], c: int) -> int: # typedpython: parallel +@compiled def mn_f_expr(xs: list[float], a: float) -> float: m = a for i in range(len(xs)): @@ -934,6 +965,7 @@ def mn_f_expr(xs: list[float], a: float) -> float: # typedpython: parallel +@compiled def mn_and_sum(xs: list[int]) -> int: m = xs[0] s = 0 @@ -945,6 +977,7 @@ def mn_and_sum(xs: list[int]) -> int: # typedpython: parallel +@compiled def two_ext(xs: list[float]) -> float: lo = xs[0] hi = xs[0] @@ -957,6 +990,7 @@ def two_ext(xs: list[float]) -> float: # typedpython: parallel +@compiled def mn_i_init(xs: list[int], init: int) -> int: m = init for i in range(len(xs)): @@ -966,6 +1000,7 @@ def mn_i_init(xs: list[int], init: int) -> int: # typedpython: parallel +@compiled def mn_i_big_e(xs: list[int], c: int) -> int: m = 0 for i in range(len(xs)): @@ -974,6 +1009,7 @@ def mn_i_big_e(xs: list[int], c: int) -> int: return m +@compiled def mn_unmarked(xs: list[int]) -> int: m = xs[0] for i in range(len(xs)): @@ -983,6 +1019,7 @@ def mn_unmarked(xs: list[int]) -> int: # typedpython: parallel +@compiled def n_le(xs: list[float]) -> float: m = xs[0] for i in range(len(xs)): @@ -992,6 +1029,7 @@ def n_le(xs: list[float]) -> float: # typedpython: parallel +@compiled def n_other_value(xs: list[int]) -> int: m = xs[0] for i in range(len(xs)): @@ -1001,6 +1039,7 @@ def n_other_value(xs: list[int]) -> int: # typedpython: parallel +@compiled def n_else(xs: list[int]) -> int: m = xs[0] s = 0 @@ -1013,6 +1052,7 @@ def n_else(xs: list[int]) -> int: # typedpython: parallel +@compiled def n_two_in_then(xs: list[int]) -> int: m = xs[0] s = 0 @@ -1024,6 +1064,7 @@ def n_two_in_then(xs: list[int]) -> int: # typedpython: parallel +@compiled def n_acc_in_value(xs: list[int]) -> int: m = xs[0] for i in range(len(xs)): @@ -1034,6 +1075,7 @@ def n_acc_in_value(xs: list[int]) -> int: # typedpython: parallel +@compiled def n_cond_other(xs: list[int], t: int) -> int: m = xs[0] for i in range(len(xs)): @@ -1043,6 +1085,7 @@ def n_cond_other(xs: list[int], t: int) -> int: # typedpython: parallel +@compiled def n_mixed_types(xs: list[int], init: float) -> float: m = init for i in range(len(xs)): @@ -1052,6 +1095,7 @@ def n_mixed_types(xs: list[int], init: float) -> float: # typedpython: parallel +@compiled def n_acc_reads_acc(xs: list[int]) -> int: m = xs[0] for i in range(len(xs)): @@ -1063,6 +1107,7 @@ def n_acc_reads_acc(xs: list[int]) -> int: # typedpython: parallel +@compiled def n_div(xs: list[float], d: float) -> float: m = xs[0] for i in range(len(xs)): @@ -1490,9 +1535,9 @@ def spin(): # ===================================================================================================== BSOURCE = '''\ -# typedpython: compiled # typedpython: parallel +@compiled def b_and(xs: list[int]) -> int: s: int = -1 for i in range(len(xs)): @@ -1501,6 +1546,7 @@ def b_and(xs: list[int]) -> int: # typedpython: parallel +@compiled def b_or(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1509,6 +1555,7 @@ def b_or(xs: list[int]) -> int: # typedpython: parallel +@compiled def b_xor(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1517,6 +1564,7 @@ def b_xor(xs: list[int]) -> int: # typedpython: parallel +@compiled def b_and_swapped(xs: list[int], s0: int) -> int: s: int = s0 for i in range(len(xs)): @@ -1525,6 +1573,7 @@ def b_and_swapped(xs: list[int], s0: int) -> int: # typedpython: parallel +@compiled def b_or_swapped(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1533,6 +1582,7 @@ def b_or_swapped(xs: list[int]) -> int: # typedpython: parallel +@compiled def b_xor_expr(xs: list[int], c: int) -> int: s: int = 0 for i in range(len(xs)): @@ -1541,6 +1591,7 @@ def b_xor_expr(xs: list[int], c: int) -> int: # typedpython: parallel +@compiled def b_or_not(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1549,6 +1600,7 @@ def b_or_not(xs: list[int]) -> int: # typedpython: parallel +@compiled def b_mix(xs: list[int]) -> int: a: int = 0 x: int = 5 @@ -1564,6 +1616,7 @@ def b_mix(xs: list[int]) -> int: # typedpython: parallel +@compiled def b_local(n: int) -> int: t = [0] * n for i in range(len(t)): @@ -1574,6 +1627,7 @@ def b_local(n: int) -> int: return s +@compiled def b_unmarked(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1582,6 +1636,7 @@ def b_unmarked(xs: list[int]) -> int: # typedpython: parallel +@compiled def nb_acc_read(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1590,6 +1645,7 @@ def nb_acc_read(xs: list[int]) -> int: # typedpython: parallel +@compiled def nb_two_ops(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1599,6 +1655,7 @@ def nb_two_ops(xs: list[int]) -> int: # typedpython: parallel +@compiled def nb_and_then_add(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1608,6 +1665,7 @@ def nb_and_then_add(xs: list[int]) -> int: # typedpython: parallel +@compiled def nb_shift(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1616,6 +1674,7 @@ def nb_shift(xs: list[int]) -> int: # typedpython: parallel +@compiled def nb_rshift(xs: list[int], k: int) -> int: s: int = 0 for i in range(len(xs)): @@ -1624,6 +1683,7 @@ def nb_rshift(xs: list[int], k: int) -> int: # typedpython: parallel +@compiled def nb_copy(xs: list[int]) -> int: s: int = 0 w: int = 0 @@ -1634,6 +1694,7 @@ def nb_copy(xs: list[int]) -> int: # typedpython: parallel +@compiled def nb_div(xs: list[int], d: int) -> int: s: int = 0 for i in range(len(xs)): @@ -1794,15 +1855,16 @@ def test_bitwise_equals_interpreter_for_every_thread_count(bbuilt, threads): # ===================================================================================================== TSOURCE = '''\ -# typedpython: compiled # typedpython: parallel +@compiled def t_scale(xs: list[float], a: float, b: float) -> None: for i in range(len(xs)): xs[i] = xs[i] * a + b # typedpython: parallel +@compiled def t_sum(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1811,6 +1873,7 @@ def t_sum(xs: list[int]) -> int: # typedpython: parallel +@compiled def t_prod(xs: list[int], c: int) -> int: s: int = 1 for i in range(len(xs)): @@ -1819,6 +1882,7 @@ def t_prod(xs: list[int], c: int) -> int: # typedpython: parallel +@compiled def t_min(xs: list[int]) -> int: m = xs[0] for i in range(len(xs)): @@ -1828,6 +1892,7 @@ def t_min(xs: list[int]) -> int: # typedpython: parallel +@compiled def t_max(xs: list[float]) -> float: m = xs[0] for i in range(len(xs)): @@ -1837,6 +1902,7 @@ def t_max(xs: list[float]) -> float: # typedpython: parallel +@compiled def t_and(xs: list[int]) -> int: s: int = -1 for i in range(len(xs)): @@ -1845,6 +1911,7 @@ def t_and(xs: list[int]) -> int: # typedpython: parallel +@compiled def t_or(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1853,6 +1920,7 @@ def t_or(xs: list[int]) -> int: # typedpython: parallel +@compiled def t_xor(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): diff --git a/python-multiplatform-ksp/src/test/python/test_pipeline.py b/python-multiplatform-ksp/src/test/python/test_pipeline.py index 6efbeb14d..ed326eb4c 100644 --- a/python-multiplatform-ksp/src/test/python/test_pipeline.py +++ b/python-multiplatform-ksp/src/test/python/test_pipeline.py @@ -20,9 +20,9 @@ def interpreted(path: Path): SCALAR = """\ -# typedpython: compiled +@compiled def total(n: int) -> float: acc: float = 0.0 for k in range(n): @@ -30,6 +30,7 @@ def total(n: int) -> float: return acc +@compiled def fact(n: int) -> int: r: int = 1 for k in range(2, n + 1): @@ -53,6 +54,7 @@ def test_scalar_module_compiles_and_matches_cpython(write, tmp_path): def test_unsupported_functions_stay_interpreted_with_a_reason(write, tmp_path): path = write("mixed.py", SCALAR + """ +@compiled def greet(name: str) -> str: return name[1:] """) @@ -68,16 +70,16 @@ def test_a_module_with_nothing_to_compile_produces_no_extension(write, tmp_path) def test_gate_errors_block_compilation(write, tmp_path): - path = write("bad.py", "# typedpython: compiled\n\ndef f(x):\n return x\n") + path = write("bad.py", "@compiled\ndef f(x):\n return x\n") with pytest.raises(pipeline.CompileError, match="implicit-any-parameter"): pipeline.compile_module(path, tmp_path / "out") DEMO = """\ -# typedpython: compiled import sys +@compiled def total(n: int) -> float: acc: float = 0.0 for k in range(n): @@ -85,6 +87,7 @@ def total(n: int) -> float: return acc +@compiled def main() -> None: print(f"{total(int(sys.argv[1])):.3f}") """ @@ -103,15 +106,16 @@ def test_demo_reports_identical_output(write, tmp_path): ARRAY_CALLBACK = """\ -# typedpython: compiled from typing import Callable +@compiled def first_after(xs: list[float], cb: Callable[[], None]) -> float: cb() return xs[0] +@compiled def fill_then(xs: list[float], cb: Callable[[], None]) -> None: xs[1] = 5.0 cb() @@ -138,9 +142,9 @@ def cb2(): GC_REACHABLE = """\ -# typedpython: compiled +@compiled class Cell: a: object b: object @@ -151,10 +155,12 @@ def __init__(self, a: object, b: object) -> None: self.b = b +@compiled def seta(c: Cell, v: object) -> None: c.a = v +@compiled def fresh(a: object, n: int) -> object: c = Cell(None, None) seta(c, a) diff --git a/python-multiplatform-ksp/src/test/python/test_pipeline_incremental.py b/python-multiplatform-ksp/src/test/python/test_pipeline_incremental.py index dd7cfad86..3735df9bc 100644 --- a/python-multiplatform-ksp/src/test/python/test_pipeline_incremental.py +++ b/python-multiplatform-ksp/src/test/python/test_pipeline_incremental.py @@ -9,25 +9,24 @@ import pytest +from conftest import decorate_compiled from typedpython import cbuild, frontend, pipeline -HEADER = "# typedpython: compiled\n" - -B = HEADER + textwrap.dedent("""\ +B = decorate_compiled(textwrap.dedent("""\ def twice(n: int) -> int: return n * 2 - """) -A = HEADER + textwrap.dedent("""\ + """)) +A = decorate_compiled(textwrap.dedent("""\ import b def inc(n: int) -> int: return n + 1 - """) -C = HEADER + textwrap.dedent("""\ + """)) +C = decorate_compiled(textwrap.dedent("""\ def half(x: float) -> float: return x * 0.5 - """) + """)) def interpreted(path: Path): @@ -133,7 +132,7 @@ def test_rebuilt_and_reused_extensions_compute_the_interpreted_results(project, def test_module_with_nothing_compiled_has_no_extension_and_is_reused(tmp_path): root = tmp_path / "proj" root.mkdir() - (root / "plain.py").write_text(HEADER + "def f(s: str) -> str:\n return s[1:]\n") + (root / "plain.py").write_text("@compiled\ndef f(s: str) -> str:\n return s[1:]\n") cache = tmp_path / "cache" first = pipeline.compile_project([root / "plain.py"], root, cache) m = first.modules[0] @@ -151,7 +150,7 @@ def test_interface_describes_the_verified_module_not_the_lowered_one(tmp_path, m root = tmp_path / "proj" root.mkdir() - (root / "dep.py").write_text(HEADER + "def ok(n: int) -> int:\n return n + 1\n") + (root / "dep.py").write_text("@compiled\ndef ok(n: int) -> int:\n return n + 1\n") real_lower = frontend.lower seen = {} diff --git a/python-multiplatform-ksp/src/test/python/test_recursion.py b/python-multiplatform-ksp/src/test/python/test_recursion.py index 289e1de66..49901b0e6 100644 --- a/python-multiplatform-ksp/src/test/python/test_recursion.py +++ b/python-multiplatform-ksp/src/test/python/test_recursion.py @@ -20,15 +20,16 @@ PACKAGE = Path(__file__).resolve().parents[2] / "main" / "python" SOURCE = """\ -# typedpython: compiled +@compiled def fib(n: int) -> int: if n < 2: return n return fib(n - 1) + fib(n - 2) +@compiled def ack(m: int, n: int) -> int: if m == 0: return n + 1 @@ -37,30 +38,35 @@ def ack(m: int, n: int) -> int: return ack(m - 1, ack(m, n - 1)) +@compiled def is_even(n: int) -> bool: if n == 0: return True return is_odd(n - 1) +@compiled def is_odd(n: int) -> bool: if n == 0: return False return is_even(n - 1) +@compiled def down(n: int) -> int: if n == 0: return 0 return 1 + down(n - 1) +@compiled def pow2(n: int) -> int: if n == 0: return 1 return 2 * pow2(n - 1) +@compiled def fact(n: int) -> int: if n < 2: return 1 diff --git a/python-multiplatform-ksp/src/test/python/test_regions.py b/python-multiplatform-ksp/src/test/python/test_regions.py index 0246fd207..ac7aad6c8 100644 --- a/python-multiplatform-ksp/src/test/python/test_regions.py +++ b/python-multiplatform-ksp/src/test/python/test_regions.py @@ -37,12 +37,12 @@ FAIL_AFTER = "__typedpython_region_fail_after__" # (n) -> the nth region object allocation fails; -1 off SOURCE = textwrap.dedent('''\ - # typedpython: compiled """typedpython region test module.""" KEPT = None + @compiled class Node: __slots__ = ("left", "right") left: "Node | None" @@ -53,12 +53,14 @@ def __init__(self, left: "Node | None", right: "Node | None") -> None: self.right = right + @compiled def make(depth: int) -> Node: if depth > 0: return Node(make(depth - 1), make(depth - 1)) return Node(None, None) + @compiled def check(node: Node) -> int: left: Node | None = node.left right: Node | None = node.right @@ -67,10 +69,12 @@ def check(node: Node) -> int: return 1 + check(left) + check(right) + @compiled def tree_sum(d: int) -> int: return check(make(d)) + @compiled def repeat(d: int, n: int) -> int: t: int = 0 for _ in range(n): @@ -78,35 +82,42 @@ def repeat(d: int, n: int) -> int: return t + @compiled def escapes(d: int) -> Node: return make(d) + @compiled def keeps(d: int) -> int: global KEPT KEPT = make(d) return check(KEPT) + @compiled def prints(d: int) -> int: n = make(d) print("unknown code while the tree is alive") return check(n) + @compiled def ident(d: int) -> int: return id(make(d)) * 0 + @compiled def same(d: int) -> bool: return make(d) is make(d) + @compiled def mixed(d: int) -> int: t = make(d) return check(make(d)) + check(t) + @compiled def noisy(d: int) -> object: print("unknown code") return check(make(d)) diff --git a/python-multiplatform-ksp/src/test/python/test_resume.py b/python-multiplatform-ksp/src/test/python/test_resume.py index 14115cc79..984c7b4b3 100644 --- a/python-multiplatform-ksp/src/test/python/test_resume.py +++ b/python-multiplatform-ksp/src/test/python/test_resume.py @@ -519,7 +519,7 @@ def lower(tmp_path, source): def test_frontend_gives_a_snapshot_holder_one_resume_per_loop(tmp_path): module = lower(tmp_path, """ - # typedpython: compiled + @compiled def f(xs: list[float], n: int) -> float: t = 0.0 for i in range(n): @@ -572,7 +572,7 @@ def f(xs: list[float], n: int) -> float: def test_frontend_function_without_a_snapshot_has_no_resumes(tmp_path): module = lower(tmp_path, """ - # typedpython: compiled + @compiled def g(n: int) -> int: t = 0 for i in range(n): diff --git a/python-multiplatform-ksp/src/test/python/test_str_dict.py b/python-multiplatform-ksp/src/test/python/test_str_dict.py index 2362033c4..e6bc52c7b 100644 --- a/python-multiplatform-ksp/src/test/python/test_str_dict.py +++ b/python-multiplatform-ksp/src/test/python/test_str_dict.py @@ -26,70 +26,84 @@ import pytest +from conftest import decorate_compiled from typedpython import cbuild, cgen, frontend, verify REPO = Path(__file__).resolve().parents[4] WORDFREQ = REPO / "benchmarks" / "typedpython" / "py" / "wordfreq.py" SOURCE = textwrap.dedent(''' - # typedpython: compiled from typing import SupportsIndex + @compiled def get(d: dict[object, object], k: object) -> object: return d[k] + @compiled def get_list(xs: list[object], i: int) -> object: return xs[i] + @compiled def get_index(xs: list[object], i: SupportsIndex) -> object: return xs[i] + @compiled def get_str(s: str, i: int) -> str: return s[i] + @compiled def get_nested(d: dict[object, dict[object, object]], a: object, b: object) -> object: return d[a][b] + @compiled def put(d: dict[object, object], k: object, v: object) -> None: d[k] = v + @compiled def put_list(xs: list[object], i: int, v: object) -> None: xs[i] = v + @compiled def has(c: list[object], x: object) -> bool: return x in c + @compiled def has_not(c: list[object], x: object) -> bool: return x not in c + @compiled def has_dict(c: dict[object, object], x: object) -> bool: return x in c + @compiled def has_str(c: str, x: str) -> bool: return x in c + @compiled def has_branch(c: list[object], x: object) -> str: if x in c: return "yes" return "no" + @compiled def has_chain(c: list[object], x: object, y: object) -> bool: return x in c in y + @compiled def pairs(xs: list[tuple[object, ...]]) -> list[object]: out: list[object] = [] for a, b in xs: @@ -98,6 +112,7 @@ def pairs(xs: list[tuple[object, ...]]) -> list[object]: return out + @compiled def triples(xs: list[tuple[object, ...]]) -> list[object]: out: list[object] = [] for a, b, c in xs: @@ -106,6 +121,7 @@ def triples(xs: list[tuple[object, ...]]) -> list[object]: return out + @compiled def each(xs: list[object]) -> list[object]: out: list[object] = [] for x in xs: @@ -115,6 +131,7 @@ def each(xs: list[object]) -> list[object]: return out + @compiled def first_after(xs: list[object], stop: object) -> object: found: object = None for x in xs: @@ -124,12 +141,14 @@ def first_after(xs: list[object], stop: object) -> object: return found + @compiled def return_inside(xs: list[object]) -> object: for x in xs: return x return None + @compiled def nested(xs: list[list[object]]) -> list[object]: out: list[object] = [] for row in xs: @@ -140,6 +159,7 @@ def nested(xs: list[list[object]]) -> list[object]: return out + @compiled def items_of(d: dict[str, object]) -> list[object]: out: list[object] = [] for k, v in d.items(): @@ -148,6 +168,7 @@ def items_of(d: dict[str, object]) -> list[object]: return out + @compiled def mutate_while_iterating(d: dict[str, object]) -> object: n: int = 0 for k in d: @@ -156,30 +177,37 @@ def mutate_while_iterating(d: dict[str, object]) -> object: return n + @compiled def fmt(x: object, y: float, n: int) -> str: return f"{x}|{x!r}|{x!s}|{x!a}|{y:.3f}|{n:05d}|{x:>5}|{n}" + @compiled def fmt_one(x: object) -> str: return f"{x}" + @compiled def fmt_text(x: object) -> str: return f"a{x}b{x!r}c" + @compiled def fmt_empty() -> str: return f"" + @compiled def fmt_debug(x: object) -> str: return f"{x=}" + @compiled def cat(a: str, b: str) -> str: return a + b + @compiled def cat_in_place(a: str, b: str) -> str: s: str = "" s += a @@ -187,6 +215,7 @@ def cat_in_place(a: str, b: str) -> str: return s + @compiled def grow(xs: list[object], ys: list[object]) -> list[object]: out: list[object] = [] out += xs @@ -194,6 +223,7 @@ def grow(xs: list[object], ys: list[object]) -> list[object]: return out + @compiled def empties() -> object: d: dict[object, object] = {} xs: list[object] = [] @@ -201,6 +231,7 @@ def empties() -> object: return xs + @compiled class Acc: def __init__(self, log: list[object]) -> None: self.log = log @@ -230,6 +261,7 @@ def __imod__(self, other: int) -> Acc: return self + @compiled def aug_ops(a: Acc, b: int) -> Acc: a += b a -= b @@ -240,12 +272,14 @@ def aug_ops(a: Acc, b: int) -> Acc: return a + @compiled def aug_list(xs: list[str], n: int) -> list[str]: xs *= n xs += xs return xs + @compiled def lookup_default(d: dict[str, int], k: str) -> object: counts: dict[str, int] = {} counts[k] = counts.get(k, 0) + 1 @@ -905,7 +939,7 @@ def run_main(main, n): @pytest.fixture(scope="module") def wordfreq(tmp_path_factory): d = tmp_path_factory.mktemp("wordfreq") - src = "# typedpython: compiled\n" + WORDFREQ.read_text() + src = decorate_compiled(WORDFREQ.read_text()) proved, diags, m = build_source(d, src, name="wordfreq") spec = importlib.util.spec_from_file_location("wordfreq_plain", WORDFREQ) plain = importlib.util.module_from_spec(spec) diff --git a/python-multiplatform-ksp/src/test/python/test_tracing.py b/python-multiplatform-ksp/src/test/python/test_tracing.py index 2c6939249..39f3c78b3 100644 --- a/python-multiplatform-ksp/src/test/python/test_tracing.py +++ b/python-multiplatform-ksp/src/test/python/test_tracing.py @@ -19,14 +19,15 @@ from typedpython import pipeline SOURCE = """\ -# typedpython: compiled from typing import Callable +@compiled def sq(x: float) -> float: return x * x +@compiled def loop(n: int) -> float: acc: float = 0.0 for k in range(n): @@ -34,11 +35,13 @@ def loop(n: int) -> float: return acc +@compiled def leaf(cb: Callable[[], None], n: float) -> float: cb() return n + 1.0 +@compiled def outer(cb: Callable[[], None], n: float) -> float: cb() return leaf(cb, n) + leaf(cb, n) From f702f48ff8a3f4f95a7cb086ce37c25b16836d58 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 30 Oct 2026 04:58:51 +0900 Subject: [PATCH 05/29] TypedPython: no PyPI publishing; the wheel ships with the GitHub Release and the distribution Maintainer, 2026-10-30: the GitHub Release and the thisisthepy distribution are enough. Removes typedpython-pypi.yml; INTENT 1.4 says so. --- .github/workflows/typedpython-pypi.yml | 78 -------------------------- docs/INTENT.md | 6 +- 2 files changed, 3 insertions(+), 81 deletions(-) delete mode 100644 .github/workflows/typedpython-pypi.yml diff --git a/.github/workflows/typedpython-pypi.yml b/.github/workflows/typedpython-pypi.yml deleted file mode 100644 index b0f98705d..000000000 --- a/.github/workflows/typedpython-pypi.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: TypedPython to PyPI - -# Publishes the `typedpython` package (python-multiplatform-ksp/pyproject.toml) to PyPI. The package -# is versioned on its own, so it has its own tag: `typedpython-v`, pushed by the coordinator -# on a commit of main. PyPI trusted publishing: no token is stored; the `pypi` environment and the -# PyPI project's trusted publisher (this repository, this workflow file, environment `pypi`) must match. -# Only a wheel is published: it is pure Python (`py3-none-any`), and an sdist would carry the whole -# Kotlin module next to it. - -on: - push: - tags: - - 'typedpython-v*' - workflow_dispatch: - inputs: - tag: - description: 'Existing tag to build and publish (e.g. typedpython-v0.1.0)' - required: true - -permissions: - contents: read - -jobs: - build: - name: Build and check the wheel - runs-on: macos-latest - env: - TAG: ${{ github.event.inputs.tag || github.ref_name }} - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - ref: ${{ github.event.inputs.tag || github.ref }} - - - name: Setup uv - uses: astral-sh/setup-uv@v6 - - - name: The tag names the package version - run: | - set -euo pipefail - version="$(sed -nE 's/^version = "(.*)"$/\1/p' python-multiplatform-ksp/pyproject.toml | head -1)" - test "$TAG" = "typedpython-v$version" || { echo "error: tag $TAG does not match typedpython $version" >&2; exit 1; } - - - name: Build - run: uv build --wheel --out-dir dist python-multiplatform-ksp - - - name: The wheel alone compiles the release example - run: | - set -euo pipefail - uv venv -q -p 3.15t "$RUNNER_TEMP/tpenv" - uv pip install -q --python "$RUNNER_TEMP/tpenv/bin/python" dist/typedpython-*.whl - "$RUNNER_TEMP/tpenv/bin/typedpython" demo --out "$RUNNER_TEMP/tpbuild" docs/release/desktop/typedpython/kernels.py 400000 | tee "$RUNNER_TEMP/tp.log" - grep -q '^compiled functions: .*dot' "$RUNNER_TEMP/tp.log" - grep -q '^identical output' "$RUNNER_TEMP/tp.log" - - - uses: actions/upload-artifact@v4 - with: - name: dist - path: dist/ - - publish: - name: Publish to PyPI - needs: build - runs-on: ubuntu-latest - environment: pypi - permissions: - id-token: write - steps: - - uses: actions/download-artifact@v4 - with: - name: dist - path: dist/ - - - name: Setup uv - uses: astral-sh/setup-uv@v6 - - - name: Publish - run: uv publish --trusted-publishing always dist/* diff --git a/docs/INTENT.md b/docs/INTENT.md index 5c45079d1..b471a8573 100644 --- a/docs/INTENT.md +++ b/docs/INTENT.md @@ -41,9 +41,9 @@ CPython interpreter inside a Kotlin Multiplatform application so that: the embedded CPython, as an extension module (`docs/design/typedpython.md`). TypedPython reaches plain Python users as a **distribution**, not as a new interpreter (maintainer, - 2026-10-30: "배포판 방식을 생각했다는거야. 근데 일단 그렇게 하더라도 typedpython 패키지 형태라면 pypi - 배포를 해야겠지"). The thisisthepy CPython build ships with TypedPython in it, and the same - compiler is an ordinary `typedpython` package on PyPI (`uv add typedpython`). + 2026-10-30: "배포판 방식을 생각했다는거야"). The thisisthepy CPython build ships with TypedPython in + it. The `typedpython` wheel is attached to the GitHub Release and is not published to PyPI + ("그냥 github release로만 해도 충분한거라는거지 ... 그냥 우리 배포판으로 배포하면 되는거"). 5. **High-performance kernels are written in Python too** (TypedPython tier 2). The maintainer first asked to explore it ("GPU 관련 최적화까지도 별도 언어 없이 지원할 방안을 모색해보자") and on From cca2ef5753f544a53c2c19eae5715e3a6b3a6616 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 30 Oct 2026 05:01:16 +0900 Subject: [PATCH 06/29] TypedPython tests: child processes install the compiled builtin themselves Plain CPython without typedpython_builtins.pth has no builtins.compiled; the child scripts now prepend conftest's COMPILED_SHIM. Also fixes test_regions' split that left a dangling @compiled. --- python-multiplatform-ksp/src/test/python/conftest.py | 4 ++++ .../src/test/python/test_class_methods.py | 8 ++++---- .../src/test/python/test_eval_breaker.py | 3 ++- .../src/test/python/test_fast_entry.py | 3 ++- .../src/test/python/test_free_threaded.py | 3 ++- .../src/test/python/test_parallel_loops.py | 3 ++- .../src/test/python/test_recursion.py | 3 ++- python-multiplatform-ksp/src/test/python/test_regions.py | 4 +++- python-multiplatform-ksp/src/test/python/test_tracing.py | 3 ++- 9 files changed, 23 insertions(+), 11 deletions(-) diff --git a/python-multiplatform-ksp/src/test/python/conftest.py b/python-multiplatform-ksp/src/test/python/conftest.py index 669816bf4..85b038234 100644 --- a/python-multiplatform-ksp/src/test/python/conftest.py +++ b/python-multiplatform-ksp/src/test/python/conftest.py @@ -9,6 +9,10 @@ # The `typedpython_builtins.pth` of an installed wheel; the test interpreter has no such install. builtins.compiled = getattr(builtins, "compiled", lambda f, /: f) +# Prepended to every child-process script that loads a module whose source uses `@compiled`: a plain +# CPython has the builtin only when `typedpython_builtins.pth` is installed, so the child installs it. +COMPILED_SHIM = "import builtins\nif not hasattr(builtins, 'compiled'):\n builtins.compiled = lambda f: f\n" + FIXTURES = Path(__file__).parent / "fixtures" diff --git a/python-multiplatform-ksp/src/test/python/test_class_methods.py b/python-multiplatform-ksp/src/test/python/test_class_methods.py index cd08794e3..b35b5db6e 100644 --- a/python-multiplatform-ksp/src/test/python/test_class_methods.py +++ b/python-multiplatform-ksp/src/test/python/test_class_methods.py @@ -40,7 +40,7 @@ import pytest -from conftest import decorate_compiled +from conftest import COMPILED_SHIM, decorate_compiled from typedpython import cbuild, cgen, cost, frontend, pipeline, verify HERE = Path(__file__).resolve().parent @@ -553,7 +553,7 @@ def test_class_info_reports_the_methods(built): # --- class state changed after import: child processes -------------------------------------------- -CHILD = ''' +CHILD = COMPILED_SHIM + ''' import json, sys from importlib.util import module_from_spec, spec_from_file_location from pathlib import Path @@ -736,7 +736,7 @@ def test_reference_counts_are_flat_over_many_calls(built): # --- free-threaded concurrency -------------------------------------------------------------------- -THREADS = ''' +THREADS = COMPILED_SHIM + ''' import sys, threading from pathlib import Path sys.path.insert(0, {main!r}) @@ -800,7 +800,7 @@ def test_concurrent_calls_on_shared_and_private_instances(built): # --- mutation: the exact-type guard of `self` ------------------------------------------------------- -GUARD_PROBE = ''' +GUARD_PROBE = COMPILED_SHIM + ''' import sys from pathlib import Path sys.path.insert(0, {main!r}) diff --git a/python-multiplatform-ksp/src/test/python/test_eval_breaker.py b/python-multiplatform-ksp/src/test/python/test_eval_breaker.py index 0d6bb643b..dece8d3b8 100644 --- a/python-multiplatform-ksp/src/test/python/test_eval_breaker.py +++ b/python-multiplatform-ksp/src/test/python/test_eval_breaker.py @@ -18,6 +18,7 @@ import pytest +from conftest import COMPILED_SHIM from typedpython import pipeline posix_signals = pytest.mark.skipif(sys.platform == "win32", reason="needs POSIX signals (SIGALRM/SIGUSR1/setitimer) or fork; Windows has none") @@ -49,7 +50,7 @@ def total(xs: list[float], rounds: int) -> float: return s """ -LOADER = """\ +LOADER = COMPILED_SHIM + """\ import importlib.machinery, importlib.util, json, signal, sys, threading, time path = sys.argv[1] loader = importlib.machinery.ExtensionFileLoader("loops", path) diff --git a/python-multiplatform-ksp/src/test/python/test_fast_entry.py b/python-multiplatform-ksp/src/test/python/test_fast_entry.py index d448e3b41..633a14487 100644 --- a/python-multiplatform-ksp/src/test/python/test_fast_entry.py +++ b/python-multiplatform-ksp/src/test/python/test_fast_entry.py @@ -20,6 +20,7 @@ import pytest +from conftest import COMPILED_SHIM from typedpython import cbuild, cgen, frontend, verify posix_signals = pytest.mark.skipif(sys.platform == "win32", reason="needs POSIX signals (SIGALRM/SIGUSR1/setitimer) or fork; Windows has none") @@ -170,7 +171,7 @@ def test_the_fast_impl_is_never_addressed(traced): assert "&tp_fimpl_" not in c and "(tp_fimpl_" not in c -LOAD = """\ +LOAD = COMPILED_SHIM + """\ import importlib.machinery, importlib.util, json, sys def load(name, path, ext): if ext: diff --git a/python-multiplatform-ksp/src/test/python/test_free_threaded.py b/python-multiplatform-ksp/src/test/python/test_free_threaded.py index e954c03ba..e857d3950 100644 --- a/python-multiplatform-ksp/src/test/python/test_free_threaded.py +++ b/python-multiplatform-ksp/src/test/python/test_free_threaded.py @@ -27,6 +27,7 @@ import pytest +from conftest import COMPILED_SHIM from typedpython import cbuild, cgen, frontend, pipeline, verify STRESS = max(1, int(os.environ.get("TP_FT_STRESS", "1"))) @@ -155,7 +156,7 @@ def usek(x: int) -> int: return x + K ''' -LOADER = """\ +LOADER = COMPILED_SHIM + """\ import faulthandler, importlib.machinery, importlib.util, json, sys, threading, time faulthandler.enable() path = sys.argv[1] diff --git a/python-multiplatform-ksp/src/test/python/test_parallel_loops.py b/python-multiplatform-ksp/src/test/python/test_parallel_loops.py index e86605b69..ecf4b8c7a 100644 --- a/python-multiplatform-ksp/src/test/python/test_parallel_loops.py +++ b/python-multiplatform-ksp/src/test/python/test_parallel_loops.py @@ -25,6 +25,7 @@ import pytest +from conftest import COMPILED_SHIM from typedpython import cbuild, cgen, frontend, ir, verify posix_signals = pytest.mark.skipif(sys.platform == "win32", reason="needs POSIX signals (SIGALRM/SIGUSR1/setitimer) or fork; Windows has none") @@ -335,7 +336,7 @@ def test_list_untouched_when_the_call_raises_before_the_loop(mod): # --- child processes: thread counts, creation failure, concurrency ----------------------------------- -CHILD = """\ +CHILD = COMPILED_SHIM + """\ import importlib.machinery, importlib.util, json, struct, sys, threading path = sys.argv[1] name = path.replace(chr(92), "/").rsplit("/", 1)[-1].split(".")[0] diff --git a/python-multiplatform-ksp/src/test/python/test_recursion.py b/python-multiplatform-ksp/src/test/python/test_recursion.py index 49901b0e6..84c36f1e6 100644 --- a/python-multiplatform-ksp/src/test/python/test_recursion.py +++ b/python-multiplatform-ksp/src/test/python/test_recursion.py @@ -15,6 +15,7 @@ import pytest +from conftest import COMPILED_SHIM from typedpython import pipeline PACKAGE = Path(__file__).resolve().parents[2] / "main" / "python" @@ -239,7 +240,7 @@ def inner(j): # --- sanitizers: the RecursionError path --------------------------------------------------------- -ASAN_SCRIPT = textwrap.dedent('''\ +ASAN_SCRIPT = COMPILED_SHIM + textwrap.dedent('''\ import sys sys.path.insert(0, {pkg!r}) from typedpython import cbuild diff --git a/python-multiplatform-ksp/src/test/python/test_regions.py b/python-multiplatform-ksp/src/test/python/test_regions.py index ac7aad6c8..7d7080ee9 100644 --- a/python-multiplatform-ksp/src/test/python/test_regions.py +++ b/python-multiplatform-ksp/src/test/python/test_regions.py @@ -25,6 +25,7 @@ import pytest from test_cgen import runtime_dir, need_runtime +from conftest import COMPILED_SHIM from typedpython import cbuild, cgen, cost, frontend, verify posix_signals = pytest.mark.skipif(sys.platform == "win32", reason="needs POSIX signals (SIGALRM/SIGUSR1/setitimer) or fork; Windows has none") @@ -392,10 +393,11 @@ def test_a_forged_module_level_variant_set_is_reported(built): def test_a_finalizer_or_a_field_store_keeps_the_class_out_of_a_region(tmp_path): - base = SOURCE.split("def escapes")[0] + base = SOURCE.split("@compiled\ndef escapes")[0] with_del = base.replace(" self.right = right\n", " self.right = right\n\n def __del__(self) -> None:\n pass\n", 1) with_store = base + textwrap.dedent('''\ + @compiled def poke(node: Node) -> int: node.left = None return 0 diff --git a/python-multiplatform-ksp/src/test/python/test_tracing.py b/python-multiplatform-ksp/src/test/python/test_tracing.py index 39f3c78b3..823c1dd1d 100644 --- a/python-multiplatform-ksp/src/test/python/test_tracing.py +++ b/python-multiplatform-ksp/src/test/python/test_tracing.py @@ -16,6 +16,7 @@ import pytest +from conftest import COMPILED_SHIM from typedpython import pipeline SOURCE = """\ @@ -47,7 +48,7 @@ def outer(cb: Callable[[], None], n: float) -> float: return leaf(cb, n) + leaf(cb, n) """ -LOADER = """\ +LOADER = COMPILED_SHIM + """\ import importlib.machinery, importlib.util, json, sys, threading path = sys.argv[1] loader = importlib.machinery.ExtensionFileLoader("tmod", path) From 0e54f8eeaee9c940491518b15551a3cc25bc5afa Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 31 Oct 2026 23:49:18 +0900 Subject: [PATCH 07/29] Feat: replace # typedpython: parallel comment with @compiled(parallel=True) (#358) --- docs/SPEC.md | 4 +- docs/design/typedpython-parallel-loops.md | 15 +- .../src/main/python/typedpython/__main__.py | 2 +- .../builtin_stubs/__builtins__.pyi | 6 + .../src/main/python/typedpython/frontend.py | 85 ++++--- .../src/main/python/typedpython/ir.py | 4 +- .../src/main/python/typedpython_builtins.pth | 2 +- .../src/test/python/conftest.py | 4 +- .../src/test/python/test_builtin_compiled.py | 13 +- .../src/test/python/test_frontend.py | 23 ++ .../src/test/python/test_parallel_loops.py | 216 ++++++------------ .../python/multiplatform/ffi/Python3.kt | 4 +- .../multiplatform/ffi/BuiltinCompiledTest.kt | 4 +- 13 files changed, 192 insertions(+), 190 deletions(-) diff --git a/docs/SPEC.md b/docs/SPEC.md index 61c454c2f..a41dd5b2e 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -80,7 +80,7 @@ fail, then implement. extracts it. `Status: partial`, exercised by the shared suite on the simulator; no iOS-specific lifecycle test beyond `PM/iosSimulatorArm64Test/.../AsyncioAvailabilityProbeTest.kt`. An installed app carries its own stdlib: L-11. -- **L-8** Initialisation sets `builtins.compiled` to an identity decorator (an existing one is kept), so `@compiled` needs no import on any platform (N-7, issue #42). `Status: implemented`, `PM/commonTest/.../ffi/BuiltinCompiledTest.kt`. +- **L-8** Initialisation sets `builtins.compiled` to an identity decorator accepting both `@compiled` (`compiled(f)`) and `@compiled(**opts)` (`compiled(**opts)(f)`), leaving an existing `builtins.compiled` alone, so `@compiled` needs no import on any platform (N-7, issue #42, #358). `Status: implemented`, `PM/commonTest/.../ffi/BuiltinCompiledTest.kt`. - **L-9** Desktop: a packaged application starts with no `PYTHONHOME` in its environment (issue #60, ROADMAP §15.4). The Gradle plugin copies the prefix `stagePythonHome` stages into Compose Desktop's application resources (every `prepare*AppResources` task, so `createDistributable` and the @@ -985,7 +985,7 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. - **N-15** TypedPython parallel execution of order-independent loops on free-threaded CPython (#151; `docs/design/typedpython-parallel-loops.md`). A compiled `for ... in range(...)` loop may run its iterations on several threads only if its result cannot depend on iteration order, in a function the - user marked with a `# typedpython: parallel` comment line directly above the `def` (above its decorators if any); module-level functions only, and then the + user marked `@compiled(parallel=True)` (#358; the earlier `# typedpython: parallel` comment line was removed); module-level functions only, and then the result, the exception, the partial writes and the deopt behaviour are bit-for-bit those of the sequential loop (INTENT §1.4; N-12: every execution is indistinguishable from some valid CPython execution). diff --git a/docs/design/typedpython-parallel-loops.md b/docs/design/typedpython-parallel-loops.md index e86c3e06b..2834d69a6 100644 --- a/docs/design/typedpython-parallel-loops.md +++ b/docs/design/typedpython-parallel-loops.md @@ -30,8 +30,7 @@ checkable on the existing IR (`typedpython/ir.py`), and says where that rule giv (§7, decision 3). Thread creation cost is measured together with the threshold. 6. A cost rule decides, at run time, whether a region is worth its fixed cost; below the threshold the sequential code runs (§8). The threshold is measured, not guessed here. -7. v1 parallelises only functions the user marks (decision 5). The spelling of the marking is a design - detail settled at implementation. Widening to unmarked functions later is easy; narrowing is hard. +7. v1 parallelises only functions the user marks `@compiled(parallel=True)` (decision 5, #358). Widening to unmarked functions later is easy; narrowing is hard. ## 2. What exists today (the facts this design stands on) @@ -78,7 +77,7 @@ condition that failed (like `verify/array-open`), so users can see why. - The function holds a copy-in list array, a local array, or only scalars (a pure scalar loop, e.g. a reduction over `range`). A function whose loop touches a buffer array (`array.array`) is not parallelised (decision 1). -- The user marked the function (decision 5). +- The user marked the function `@compiled(parallel=True)` (decision 5, #358). - Not under `entry_globals` mutation: entry globals are read once at entry and are locals in the body, so they are read-only and fine. @@ -503,9 +502,11 @@ Recorded 2026-10-25 from the maintainer. The body and SPEC N-15 follow them. measured together with the threshold (`TP_PAR_MIN_WORK`). 4. **Float `+` and `*` reductions are excluded in v1.** Not permanently: a fastmath opt-in (#153) awaits the owner's decision, and N-15 follows that decision. Until then they stay sequential. -5. **Only functions the user marks are parallelised in v1.** How the marking is spelled (decorator - argument, comment, other) is a design detail to settle at implementation. Widening later (implicit - parallelisation) is easy; narrowing later is hard, so v1 starts narrow. +5. **Only functions the user marks are parallelised in v1.** Marked with `@compiled(parallel=True)` + (maintainer decision 2026-10-30, #358: `@compiled` is the only compilation mark, and options are + keyword arguments on `@compiled`, matching `#332` `@compiled(fastmath=True)`; the earlier + `# typedpython: parallel` comment marker was removed). Widening later (implicit parallelisation) is + easy; narrowing later is hard, so v1 starts narrow. 6. **`min`/`max` reductions are in v1 only if exactly equal to Python's** (§6). If an implementation cannot match it, they move to a later step and v1 keeps the integer and bitwise reductions only. 7. **Thresholds are measured, not chosen here.** `TP_PAR_MIN_WORK` and the thread cap (default 8) are @@ -516,4 +517,4 @@ Recorded 2026-10-25 from the maintainer. The body and SPEC N-15 follow them. - (Settled: free-threaded 3.15t is the only build, #158, so there is no GIL-build variant.) - Which devices the threshold is measured on (the Mac mini only, or also an Android device). The measurement itself is decided (item 7); the device set is not. -- The spelling of the marking (item 5), settled at implementation. +- (Settled 2026-10-30, #358: the marking is `@compiled(parallel=True)`.) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/__main__.py b/python-multiplatform-ksp/src/main/python/typedpython/__main__.py index a9cda63e2..4db824787 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/__main__.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/__main__.py @@ -113,7 +113,7 @@ def _demo(module: Path, module_argv: list[str], out: Path) -> int: def interpreted() -> object: import builtins # the module's `@compiled` is a builtin name; typedpython_builtins.pth installs it - builtins.compiled = getattr(builtins, "compiled", lambda f, /: f) + builtins.compiled = getattr(builtins, "compiled", lambda f=None, /, **_k: (lambda g: g) if f is None else f) spec = importlib.util.spec_from_file_location(module.stem + "_interpreted", module) assert spec is not None and spec.loader is not None mod = importlib.util.module_from_spec(spec) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/builtin_stubs/__builtins__.pyi b/python-multiplatform-ksp/src/main/python/typedpython/builtin_stubs/__builtins__.pyi index f773368b9..122e4cdf5 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/builtin_stubs/__builtins__.pyi +++ b/python-multiplatform-ksp/src/main/python/typedpython/builtin_stubs/__builtins__.pyi @@ -1,3 +1,9 @@ # TypedPython's builtin names (docs/design/typedpython.md §2.3). A `__builtins__.pyi` on the type # checker's search path declares names the interpreter provides without an import. +from collections.abc import Callable +from typing import overload + +@overload def compiled[T](f: T, /) -> T: ... +@overload +def compiled[T](*, parallel: bool = ..., fastmath: bool = ...) -> Callable[[T], T]: ... diff --git a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py index 6e0b75aa6..a65ea1b9f 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py @@ -190,7 +190,8 @@ def lower(path: Path) -> ir.Module: info.classes = class_decls skipped.update(class_skips) signatures: dict[str, _Signature] = {} - for node, reason in _targets(tree, source, info): + marked: set[str] = set() + for node, reason, parallel in _targets(tree, source, info): if reason is not None: skipped[node.name] = reason continue @@ -199,6 +200,9 @@ def lower(path: Path) -> ir.Module: signatures[node.name] = _signature(node, info.classes, info) except Skip as e: skipped[node.name] = e.reason + else: + if parallel: + marked.add(node.name) for owner, node, reason in _method_targets(tree, info): q = f"{owner}.{node.name}" if reason is not None: @@ -236,28 +240,10 @@ def lower(path: Path) -> ir.Module: else: skipped[name] = result functions.sort(key=lambda f: f.source_line) - marked = _parallel_marked(tree, source) functions = [dataclasses.replace(f, parallel=True) if f.name in marked else f for f in functions] return ir.Module(path.stem, tuple(functions), skipped, tuple(class_decls.values())) -PARALLEL_MARKER = "# typedpython: parallel" - - -def _parallel_marked(tree: ast.Module, source: str) -> set[str]: - """Module-level functions the user marked for SPEC N-15: the comment line `# typedpython: parallel` - directly above the `def` (above its decorators if it has any). A comment, - so the interpreted module is unchanged and needs no runtime name.""" - lines = source.splitlines() - out: set[str] = set() - for node in tree.body: - if isinstance(node, ast.FunctionDef): - first = min([node.lineno] + [d.lineno for d in node.decorator_list]) - if first >= 2 and lines[first - 2].strip() == PARALLEL_MARKER: - out.add(node.name) - return out - - def _module_info(tree: ast.Module) -> _ModuleInfo: bindings: dict[str, list[tuple[int, str]]] = {} @@ -431,13 +417,45 @@ def _binding_targets(stmt: ast.stmt) -> list[ast.expr]: SLOT_ACCESS_DUNDERS = ("__getattribute__", "__getattr__", "__setattr__", "__delattr__", "__new__") +def _is_compiled_decorator(d: ast.expr) -> bool: + return (isinstance(d, ast.Name) and d.id == "compiled") or ( + isinstance(d, ast.Call) and isinstance(d.func, ast.Name) and d.func.id == "compiled" + ) + + +def _compiled_decorator_options(d: ast.expr) -> tuple[bool, str | None]: + """Parse `@compiled` or `@compiled(parallel=...)` -> (parallel, error_reason_or_None).""" + if isinstance(d, ast.Name): + return False, None + assert isinstance(d, ast.Call) + if d.args: + return False, (f"line {d.lineno}: `@compiled` takes keyword arguments only " + f"(got positional `{_src(d.args[0])}`)") + parallel = False + seen: set[str] = set() + for kw in d.keywords: + if kw.arg is None: + return False, f"line {d.lineno}: `**{_src(kw.value)}` in `@compiled(...)` is not supported" + if kw.arg != "parallel": + return False, (f"line {d.lineno}: `@compiled` keyword argument `{kw.arg}` is not " + "supported (only `parallel=True` or `parallel=False`)") + if kw.arg in seen: + return False, f"line {d.lineno}: `@compiled` keyword argument `{kw.arg}` is repeated" + seen.add(kw.arg) + if not (isinstance(kw.value, ast.Constant) and isinstance(kw.value.value, bool)): + return False, (f"line {d.lineno}: `@compiled(parallel=...)` requires a literal `True` " + f"or `False`, got `{_src(kw.value)}`") + parallel = kw.value.value + return parallel, None + + def is_opted_in(path: Path) -> bool: """Whether the front end considers the module for compilation at all: a module-level `def` / `class` decorated `@compiled` (the same test `_targets` and `_classes` apply).""" source = Path(path).read_text() return any( isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) - and any(isinstance(d, ast.Name) and d.id == "compiled" for d in node.decorator_list) + and any(_is_compiled_decorator(d) for d in node.decorator_list) for node in ast.parse(source, filename=str(path)).body) @@ -596,15 +614,16 @@ def _class_of(kind: str, classes: dict[str, ir.ClassDecl]) -> tuple[str, bool] | def _targets( tree: ast.Module, source: str, info: _ModuleInfo, -) -> list[tuple[ast.FunctionDef | ast.AsyncFunctionDef, str | None]]: - found: list[tuple[ast.FunctionDef | ast.AsyncFunctionDef, str | None]] = [] +) -> list[tuple[ast.FunctionDef | ast.AsyncFunctionDef, str | None, bool]]: + found: list[tuple[ast.FunctionDef | ast.AsyncFunctionDef, str | None, bool]] = [] for node in tree.body: if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): continue - marked = [d for d in node.decorator_list if isinstance(d, ast.Name) and d.id == "compiled"] + marked = [d for d in node.decorator_list if _is_compiled_decorator(d)] if not marked: continue reason = None + parallel = False if marked and info.bound("compiled"): line = info.bindings["compiled"][0][0] reason = (f"line {node.lineno}: `compiled` is bound in this module (line {line}), " @@ -612,12 +631,20 @@ def _targets( elif other := [d for d in node.decorator_list if d not in marked]: reason = (f"line {other[0].lineno}: decorator `@{_src(other[0])}` is not supported " "(only `@compiled`)") - elif isinstance(node, ast.AsyncFunctionDef): - reason = f"line {node.lineno}: async function is not supported" - elif len(info.bindings.get(node.name, [])) > 1: - lines = ", ".join(str(line) for line, _ in info.bindings[node.name]) - reason = f"line {node.lineno}: `{node.name}` is bound more than once (lines {lines})" - found.append((node, reason)) + else: + for d in marked: + par, r = _compiled_decorator_options(d) + if r is not None: + reason = r + break + parallel = parallel or par + if reason is None: + if isinstance(node, ast.AsyncFunctionDef): + reason = f"line {node.lineno}: async function is not supported" + elif len(info.bindings.get(node.name, [])) > 1: + lines = ", ".join(str(line) for line, _ in info.bindings[node.name]) + reason = f"line {node.lineno}: `{node.name}` is bound more than once (lines {lines})" + found.append((node, reason, parallel)) return found diff --git a/python-multiplatform-ksp/src/main/python/typedpython/ir.py b/python-multiplatform-ksp/src/main/python/typedpython/ir.py index 368c1c3ba..1176d2182 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/ir.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/ir.py @@ -817,8 +817,8 @@ class Function: # rejects a function whose declared entries differ (`verify/virtual-escape`). The front end # leaves it empty. virtual: tuple[str, ...] = () - # SPEC N-15: the user marked this function with the comment line `# typedpython: parallel` - # directly above its `def`. Only a marked function can hold a `ForRange.par`. + # SPEC N-15: the user marked this function `@compiled(parallel=True)` (#358). + # Only a marked function can hold a `ForRange.par`. parallel: bool = False # SPEC N-12b (#268): the producers whose result this pure function keeps inside one statement # (`check(make(d))`), so each such call builds its objects in a region. Set only by `verify`, diff --git a/python-multiplatform-ksp/src/main/python/typedpython_builtins.pth b/python-multiplatform-ksp/src/main/python/typedpython_builtins.pth index 98776970a..1b8b0a0ad 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython_builtins.pth +++ b/python-multiplatform-ksp/src/main/python/typedpython_builtins.pth @@ -1 +1 @@ -import builtins; builtins.compiled = getattr(builtins, "compiled", lambda f, /: f) +import builtins; builtins.compiled = getattr(builtins, "compiled", lambda f=None, /, **_k: (lambda g: g) if f is None else f) diff --git a/python-multiplatform-ksp/src/test/python/conftest.py b/python-multiplatform-ksp/src/test/python/conftest.py index 85b038234..975fdc08b 100644 --- a/python-multiplatform-ksp/src/test/python/conftest.py +++ b/python-multiplatform-ksp/src/test/python/conftest.py @@ -7,11 +7,11 @@ import pytest # The `typedpython_builtins.pth` of an installed wheel; the test interpreter has no such install. -builtins.compiled = getattr(builtins, "compiled", lambda f, /: f) +builtins.compiled = getattr(builtins, "compiled", lambda f=None, /, **_k: (lambda g: g) if f is None else f) # Prepended to every child-process script that loads a module whose source uses `@compiled`: a plain # CPython has the builtin only when `typedpython_builtins.pth` is installed, so the child installs it. -COMPILED_SHIM = "import builtins\nif not hasattr(builtins, 'compiled'):\n builtins.compiled = lambda f: f\n" +COMPILED_SHIM = "import builtins\nif not hasattr(builtins, 'compiled'):\n builtins.compiled = lambda f=None, /, **_k: (lambda g: g) if f is None else f\n" FIXTURES = Path(__file__).parent / "fixtures" diff --git a/python-multiplatform-ksp/src/test/python/test_builtin_compiled.py b/python-multiplatform-ksp/src/test/python/test_builtin_compiled.py index 4f184812b..1357c7ed7 100644 --- a/python-multiplatform-ksp/src/test/python/test_builtin_compiled.py +++ b/python-multiplatform-ksp/src/test/python/test_builtin_compiled.py @@ -21,7 +21,16 @@ def test_gate_accepts_compiled_without_an_import(write): def f(x: int) -> int: return x + 1 - y: int = f(1) + @compiled(parallel=True) + def g(x: int) -> int: + return x + 2 + + @compiled(fastmath=True) + def h(x: float) -> float: + return x * 2.0 + + y: int = f(1) + g(2) + z: float = h(1.5) """) assert errors(gate.check([path])) == [] @@ -82,7 +91,7 @@ def test_decorator_works_with_no_import(installed): def test_compiled_is_the_identity(installed): python, _ = installed - r = py(python, "-c", "def f(): pass\nprint(compiled(f) is f)") + r = py(python, "-c", "def f(): pass\nprint(compiled(f) is f and compiled(parallel=True)(f) is f and compiled()(f) is f)") assert r.stdout.strip() == "True", r.stderr diff --git a/python-multiplatform-ksp/src/test/python/test_frontend.py b/python-multiplatform-ksp/src/test/python/test_frontend.py index fe8f4d94c..ddcf3fcc9 100644 --- a/python-multiplatform-ksp/src/test/python/test_frontend.py +++ b/python-multiplatform-ksp/src/test/python/test_frontend.py @@ -654,6 +654,29 @@ def c(x: float) -> float: assert set(m.skipped) == {"c"} and "decorator" in m.skipped["c"] +def test_compiled_parallel_option_and_former_comment_marker(tmp_path): + m = lower(tmp_path, """ + @compiled(parallel=True) + def p(x: float) -> float: + return x + + @compiled(parallel=False) + def s(x: float) -> float: + return -x + + # typedpython: parallel + @compiled + def c(x: float) -> float: + return x * 2.0 + + @compiled(unknown=True) + def bad(x: float) -> float: + return x + """) + assert [(f.name, f.parallel) for f in m.functions] == [("p", True), ("s", False), ("c", False)] + assert set(m.skipped) == {"bad"} and "parallel" in m.skipped["bad"] + + def test_the_former_module_pragma_comment_compiles_nothing(tmp_path): m = lower(tmp_path, """ # typedpython: compiled diff --git a/python-multiplatform-ksp/src/test/python/test_parallel_loops.py b/python-multiplatform-ksp/src/test/python/test_parallel_loops.py index ecf4b8c7a..8f215ecd2 100644 --- a/python-multiplatform-ksp/src/test/python/test_parallel_loops.py +++ b/python-multiplatform-ksp/src/test/python/test_parallel_loops.py @@ -1,14 +1,14 @@ -"""TypedPython parallel order-independent loops, step 1 (SPEC N-15, #151, #277). +"""TypedPython parallel order-independent loops, step 1 (SPEC N-15, #151, #277, #358). -Step 1 scope: in a function marked by the comment line `# typedpython: parallel` directly above its -`def`, a `for i in range(len(a))` loop whose body is only stores `a[i] = ` (the same +Step 1 scope: in a function decorated with `@compiled(parallel=True)`, a `for i in range(len(a))` +loop whose body is only stores `a[i] = ` (the same index i, each proven in bounds), over list copy-in or local arrays, with values built from constants, loop-invariant scalars, `float(i)`, reads `b[i]` at the same i (proven) and F64 + - * and negation, runs on several C threads above TP_PAR_MIN_WORK. The verifier recomputes the set (`ForRange.par`) and rejects a declared one that differs (`verify/par-loop`). Everything else compiles as before. Red record: written before the implementation. Before it, `ir.ForRange` had no `par`, `ir.Function` -had no `parallel`, the marker comment was ignored and the generated module had no +had no `parallel`, the parallel opt-in was ignored and the generated module had no `__typedpython_parallel__`; every test below that touches those failed (see the report in #277). """ from __future__ import annotations @@ -35,8 +35,7 @@ SOURCE = '''\ -# typedpython: parallel -@compiled +@compiled(parallel=True) def scale(xs: list[float], a: float, b: float) -> None: for i in range(len(xs)): xs[i] = xs[i] * a + b @@ -48,30 +47,26 @@ def scale_unmarked(xs: list[float], a: float, b: float) -> None: xs[i] = xs[i] * a + b -# typedpython: parallel -@compiled +@compiled(parallel=True) def ramp(xs: list[float], a: float) -> None: for i in range(len(xs)): xs[i] = -(float(i) * a) + xs[i] -# typedpython: parallel -@compiled +@compiled(parallel=True) def two_stores(xs: list[float], ys: list[float], a: float) -> None: for i in range(len(xs)): xs[i] = xs[i] + a xs[i] = xs[i] * xs[i] -# typedpython: parallel -@compiled +@compiled(parallel=True) def neighbour(xs: list[float]) -> None: for i in range(len(xs) - 1): xs[i] = xs[i + 1] -# typedpython: parallel -@compiled +@compiled(parallel=True) def total(xs: list[float]) -> float: s: float = 0.0 for i in range(len(xs)): @@ -84,22 +79,19 @@ def helper(x: float) -> float: return x * 2.0 -# typedpython: parallel -@compiled +@compiled(parallel=True) def with_call(xs: list[float]) -> None: for i in range(len(xs)): xs[i] = helper(xs[i]) -# typedpython: parallel -@compiled +@compiled(parallel=True) def divides(xs: list[float], d: float) -> None: for i in range(len(xs)): xs[i] = xs[i] / d -# typedpython: parallel -@compiled +@compiled(parallel=True) def local_fill(a: float) -> float: t = [0.0] * 300000 for i in range(len(t)): @@ -107,8 +99,7 @@ def local_fill(a: float) -> float: return t[7] + t[299999] -# typedpython: parallel -@compiled +@compiled(parallel=True) def local_dyn(n: int, a: float) -> float: t = [1.0] * n for i in range(len(t)): @@ -250,7 +241,7 @@ def test_c_has_a_region_only_for_eligible_loops(built): def test_no_par_code_without_a_marker(tmp_path): - src = SOURCE.replace("# typedpython: parallel\n", "") + src = SOURCE.replace("@compiled(parallel=True)", "@compiled") _, c_text, _, _ = build(tmp_path, src, name="nomark") assert "tp_par_run(" not in c_text and "__typedpython_parallel__" not in c_text @@ -450,8 +441,7 @@ def sh(): RSOURCE = '''\ -# typedpython: parallel -@compiled +@compiled(parallel=True) def r_sum(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -459,8 +449,7 @@ def r_sum(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def r_sum_init(xs: list[int], s0: int) -> int: s: int = s0 for i in range(len(xs)): @@ -468,8 +457,7 @@ def r_sum_init(xs: list[int], s0: int) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def r_prod(xs: list[int], c: int) -> int: s: int = 1 for i in range(len(xs)): @@ -477,8 +465,7 @@ def r_prod(xs: list[int], c: int) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def r_sum_mul(xs: list[int], c: int) -> int: s: int = 0 for i in range(len(xs)): @@ -486,8 +473,7 @@ def r_sum_mul(xs: list[int], c: int) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def r_two(xs: list[int]) -> int: a: int = 0 b: int = 7 @@ -497,8 +483,7 @@ def r_two(xs: list[int]) -> int: return a * 3 + b -# typedpython: parallel -@compiled +@compiled(parallel=True) def r_range_sum(xs: list[int]) -> int: s: int = 5 for i in range(len(xs)): @@ -506,8 +491,7 @@ def r_range_sum(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def r_local(n: int) -> int: t = [0] * n for i in range(len(t)): @@ -526,8 +510,7 @@ def r_unmarked(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def n_acc_read(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -535,8 +518,7 @@ def n_acc_read(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def n_two_ops(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -545,8 +527,7 @@ def n_two_ops(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def n_float(xs: list[float]) -> float: s: float = 0.0 for i in range(len(xs)): @@ -554,8 +535,7 @@ def n_float(xs: list[float]) -> float: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def n_copy(xs: list[int]) -> int: s: int = 0 w: int = 0 @@ -565,8 +545,7 @@ def n_copy(xs: list[int]) -> int: return s + w -# typedpython: parallel -@compiled +@compiled(parallel=True) def n_mixed(xs: list[int]) -> int: t = [0.0] * len(xs) s: int = 0 @@ -576,8 +555,7 @@ def n_mixed(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def n_div(xs: list[int], d: int) -> int: s: int = 0 for i in range(len(xs)): @@ -585,8 +563,7 @@ def n_div(xs: list[int], d: int) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def n_shift_step(xs: list[int]) -> int: s: int = 0 for i in range(0, len(xs), 2): @@ -864,7 +841,7 @@ def test_reduction_results_identical_for_every_thread_count(rbuilt): def test_no_reduction_region_without_a_marker(tmp_path): - src = RSOURCE.replace("# typedpython: parallel\n", "") + src = RSOURCE.replace("@compiled(parallel=True)", "@compiled") _, c_text, _, _ = build(tmp_path, src, name="rnomark") assert "tp_par_run(" not in c_text and "__typedpython_parallel__" not in c_text @@ -885,8 +862,7 @@ def test_no_reduction_region_without_a_marker(tmp_path): MSOURCE = '''\ -# typedpython: parallel -@compiled +@compiled(parallel=True) def mn_i(xs: list[int]) -> int: m = xs[0] for i in range(len(xs)): @@ -895,8 +871,7 @@ def mn_i(xs: list[int]) -> int: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def mx_i(xs: list[int]) -> int: m = xs[0] for i in range(len(xs)): @@ -905,8 +880,7 @@ def mx_i(xs: list[int]) -> int: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def mn_f(xs: list[float]) -> float: m = xs[0] for i in range(len(xs)): @@ -915,8 +889,7 @@ def mn_f(xs: list[float]) -> float: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def mx_f(xs: list[float]) -> float: m = xs[0] for i in range(len(xs)): @@ -925,8 +898,7 @@ def mx_f(xs: list[float]) -> float: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def mn_f_flipped(xs: list[float], init: float) -> float: m = init for i in range(len(xs)): @@ -935,8 +907,7 @@ def mn_f_flipped(xs: list[float], init: float) -> float: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def mx_f_flipped(xs: list[float], init: float) -> float: m = init for i in range(len(xs)): @@ -945,8 +916,7 @@ def mx_f_flipped(xs: list[float], init: float) -> float: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def mn_i_expr(xs: list[int], c: int) -> int: m = c for i in range(len(xs)): @@ -955,8 +925,7 @@ def mn_i_expr(xs: list[int], c: int) -> int: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def mn_f_expr(xs: list[float], a: float) -> float: m = a for i in range(len(xs)): @@ -965,8 +934,7 @@ def mn_f_expr(xs: list[float], a: float) -> float: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def mn_and_sum(xs: list[int]) -> int: m = xs[0] s = 0 @@ -977,8 +945,7 @@ def mn_and_sum(xs: list[int]) -> int: return m * 1000003 + s -# typedpython: parallel -@compiled +@compiled(parallel=True) def two_ext(xs: list[float]) -> float: lo = xs[0] hi = xs[0] @@ -990,8 +957,7 @@ def two_ext(xs: list[float]) -> float: return hi - lo -# typedpython: parallel -@compiled +@compiled(parallel=True) def mn_i_init(xs: list[int], init: int) -> int: m = init for i in range(len(xs)): @@ -1000,8 +966,7 @@ def mn_i_init(xs: list[int], init: int) -> int: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def mn_i_big_e(xs: list[int], c: int) -> int: m = 0 for i in range(len(xs)): @@ -1019,8 +984,7 @@ def mn_unmarked(xs: list[int]) -> int: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def n_le(xs: list[float]) -> float: m = xs[0] for i in range(len(xs)): @@ -1029,8 +993,7 @@ def n_le(xs: list[float]) -> float: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def n_other_value(xs: list[int]) -> int: m = xs[0] for i in range(len(xs)): @@ -1039,8 +1002,7 @@ def n_other_value(xs: list[int]) -> int: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def n_else(xs: list[int]) -> int: m = xs[0] s = 0 @@ -1052,8 +1014,7 @@ def n_else(xs: list[int]) -> int: return m + s -# typedpython: parallel -@compiled +@compiled(parallel=True) def n_two_in_then(xs: list[int]) -> int: m = xs[0] s = 0 @@ -1064,8 +1025,7 @@ def n_two_in_then(xs: list[int]) -> int: return m + s -# typedpython: parallel -@compiled +@compiled(parallel=True) def n_acc_in_value(xs: list[int]) -> int: m = xs[0] for i in range(len(xs)): @@ -1075,8 +1035,7 @@ def n_acc_in_value(xs: list[int]) -> int: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def n_cond_other(xs: list[int], t: int) -> int: m = xs[0] for i in range(len(xs)): @@ -1085,8 +1044,7 @@ def n_cond_other(xs: list[int], t: int) -> int: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def n_mixed_types(xs: list[int], init: float) -> float: m = init for i in range(len(xs)): @@ -1095,8 +1053,7 @@ def n_mixed_types(xs: list[int], init: float) -> float: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def n_acc_reads_acc(xs: list[int]) -> int: m = xs[0] for i in range(len(xs)): @@ -1107,8 +1064,7 @@ def n_acc_reads_acc(xs: list[int]) -> int: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def n_div(xs: list[float], d: float) -> float: m = xs[0] for i in range(len(xs)): @@ -1537,8 +1493,7 @@ def spin(): BSOURCE = '''\ -# typedpython: parallel -@compiled +@compiled(parallel=True) def b_and(xs: list[int]) -> int: s: int = -1 for i in range(len(xs)): @@ -1546,8 +1501,7 @@ def b_and(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def b_or(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1555,8 +1509,7 @@ def b_or(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def b_xor(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1564,8 +1517,7 @@ def b_xor(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def b_and_swapped(xs: list[int], s0: int) -> int: s: int = s0 for i in range(len(xs)): @@ -1573,8 +1525,7 @@ def b_and_swapped(xs: list[int], s0: int) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def b_or_swapped(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1582,8 +1533,7 @@ def b_or_swapped(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def b_xor_expr(xs: list[int], c: int) -> int: s: int = 0 for i in range(len(xs)): @@ -1591,8 +1541,7 @@ def b_xor_expr(xs: list[int], c: int) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def b_or_not(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1600,8 +1549,7 @@ def b_or_not(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def b_mix(xs: list[int]) -> int: a: int = 0 x: int = 5 @@ -1616,8 +1564,7 @@ def b_mix(xs: list[int]) -> int: return a + x * 3 + o * 5 + m * 7 -# typedpython: parallel -@compiled +@compiled(parallel=True) def b_local(n: int) -> int: t = [0] * n for i in range(len(t)): @@ -1636,8 +1583,7 @@ def b_unmarked(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def nb_acc_read(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1645,8 +1591,7 @@ def nb_acc_read(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def nb_two_ops(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1655,8 +1600,7 @@ def nb_two_ops(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def nb_and_then_add(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1665,8 +1609,7 @@ def nb_and_then_add(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def nb_shift(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1674,8 +1617,7 @@ def nb_shift(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def nb_rshift(xs: list[int], k: int) -> int: s: int = 0 for i in range(len(xs)): @@ -1683,8 +1625,7 @@ def nb_rshift(xs: list[int], k: int) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def nb_copy(xs: list[int]) -> int: s: int = 0 w: int = 0 @@ -1694,8 +1635,7 @@ def nb_copy(xs: list[int]) -> int: return s + w -# typedpython: parallel -@compiled +@compiled(parallel=True) def nb_div(xs: list[int], d: int) -> int: s: int = 0 for i in range(len(xs)): @@ -1857,15 +1797,13 @@ def test_bitwise_equals_interpreter_for_every_thread_count(bbuilt, threads): TSOURCE = '''\ -# typedpython: parallel -@compiled +@compiled(parallel=True) def t_scale(xs: list[float], a: float, b: float) -> None: for i in range(len(xs)): xs[i] = xs[i] * a + b -# typedpython: parallel -@compiled +@compiled(parallel=True) def t_sum(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1873,8 +1811,7 @@ def t_sum(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def t_prod(xs: list[int], c: int) -> int: s: int = 1 for i in range(len(xs)): @@ -1882,8 +1819,7 @@ def t_prod(xs: list[int], c: int) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def t_min(xs: list[int]) -> int: m = xs[0] for i in range(len(xs)): @@ -1892,8 +1828,7 @@ def t_min(xs: list[int]) -> int: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def t_max(xs: list[float]) -> float: m = xs[0] for i in range(len(xs)): @@ -1902,8 +1837,7 @@ def t_max(xs: list[float]) -> float: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def t_and(xs: list[int]) -> int: s: int = -1 for i in range(len(xs)): @@ -1911,8 +1845,7 @@ def t_and(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def t_or(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1920,8 +1853,7 @@ def t_or(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def t_xor(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): diff --git a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/Python3.kt b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/Python3.kt index 14d153326..5060b9469 100644 --- a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/Python3.kt +++ b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/Python3.kt @@ -124,7 +124,9 @@ object Python3 { def _pm_install_compiled(): import builtins if not hasattr(builtins, "compiled"): - def compiled(f): + def compiled(f=None, /, **_opts): + if f is None: + return lambda g: g return f builtins.compiled = compiled _pm_install_compiled() diff --git a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/BuiltinCompiledTest.kt b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/BuiltinCompiledTest.kt index 5d3ef407f..e7765283f 100644 --- a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/BuiltinCompiledTest.kt +++ b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/BuiltinCompiledTest.kt @@ -11,7 +11,9 @@ class BuiltinCompiledTest { @Test fun compiledResolvesWithoutImportAndIsIdentity() = PythonTestFixture.withInterpreter { assertEquals("True", evalStr("compiled(len) is len")) - Python3.exec("def _bc_f(): return 1\n_bc_r = compiled(_bc_f) is _bc_f") + assertEquals("True", evalStr("compiled(parallel=True)(len) is len")) + assertEquals("True", evalStr("compiled()(len) is len")) + Python3.exec("def _bc_f(): return 1\n_bc_r = compiled(_bc_f) is _bc_f and compiled(parallel=True)(_bc_f) is _bc_f") assertEquals("True", evalStr("_bc_r")) // It must live in builtins, not leak into __main__. assertEquals("False", evalStr("'compiled' in globals()")) From 88243859f91849d45471b3d8e53c66ffe15cc7dd Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 31 Oct 2026 23:59:03 +0900 Subject: [PATCH 08/29] Feat: add per-function @compiled(fastmath=True) opt-in (#332) --- docs/SPEC.md | 9 ++++ .../src/main/python/typedpython/cgen.py | 33 ++++++++++-- .../src/main/python/typedpython/frontend.py | 53 ++++++++++++------- .../src/main/python/typedpython/ir.py | 3 ++ .../src/test/python/test_frontend.py | 37 +++++++++++++ .../src/test/python/test_pipeline.py | 53 +++++++++++++++++++ 6 files changed, 165 insertions(+), 23 deletions(-) diff --git a/docs/SPEC.md b/docs/SPEC.md index a41dd5b2e..1e0697a64 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -739,6 +739,15 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. `continue` and `return` meaning exactly what they mean without chunking (trip counts just below, at and above a multiple of the interval, negative steps, empty ranges and ranges spanning the i64 limits all behave as CPython's `range`). A function holding an array copy-in or entry-globals snapshot polls too (#152, option (i)): at the top of a loop iteration, once per `TP_POLL_INTERVAL`, it writes its copied lists back (CPython's own stores would already be visible there), ends their critical sections, polls, takes the critical sections again, copies the lists in again and re-checks the entry guards (list lengths, exact element types, entry globals). Unchanged: it continues compiled, which is an execution CPython can produce, since every list operation of another thread now falls before or after this point. Changed: it leaves for the interpreter at that point (an OSR-style exit) and does not deopt or raise: a continuation function generated from the function's own source, holding the rest of the function from that loop iteration (the innermost loop from its current index; each enclosing loop's remaining body, then its remaining iterations; then the statements after), receives every live local (a native value as its Python object, a local array as a new list) and each enclosing `range`'s already evaluated bounds, and the compiled call returns what it returns or raises what it raises. The continuation keeps the original file, line numbers and function name, so a traceback reads as the interpreted function's. + **Floating-point exactness and `@compiled(fastmath=True)` (#153, #332, owner decision 2026-10-25):** + by default (`fastmath=False`), floating-point compilation is bit-identical to CPython (`#pragma STDC + FP_CONTRACT OFF` in the generated C, `-ffp-contract=off -fno-fast-math` on GCC/Clang, `/fp:precise` + on MSVC). A module-level function marked `@compiled(fastmath=True)` opts into floating-point + reassociation and FMA contraction for that function only (never on by default, never applied to an + unmarked function or another function in the same module, and never inlined into a non-fastmath + caller without fastmath): **this is the one path in TypedPython that is explicitly NOT + CPython-identical** (a sum or fused multiply-add in a `@compiled(fastmath=True)` function may round + differently from the interpreter so float reductions and FMA loops can vectorise). `Status: implemented` for desktop host builds (`test_pipeline.py`, `test_cgen*.py`, `test_recursion.py`, `test_eval_breaker.py`, `test_fast_entry.py`, `test_free_threaded.py`; free-threaded 3.15t, #159). Not yet: shipping compiled modules in an app through diff --git a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py index fb776ce01..940837ed9 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py @@ -494,10 +494,19 @@ def _runs_no_python(f: ir.Function) -> bool: def _bounded_functions(functions: dict[str, ir.Function]) -> dict[str, int]: """name -> k for every bounded function (module docstring, "Fast entry"): in no call cycle, every callee bounded, `_runs_no_python`. k(f) = 1 + max(k(callee)), 1 without callees: the most - compiled frames a call of `f` can have in flight at once.""" + compiled frames a call of `f` can have in flight at once. + A `fastmath` function is never given an `inline` fast twin (`tp_fimpl_*`) when some other + function in the module calls it without `fastmath`, so a non-fastmath caller can never inline + a fastmath body (#332).""" callees: dict[str, set[str]] = {} for name, f in functions.items(): callees[name] = {e.function for e in _all_exprs(f) if isinstance(e, ir.Call)} + called_from_exact = { + c + for caller_name, cs in callees.items() + if not functions[caller_name].fastmath + for c in cs + } k: dict[str, int | None] = {} # None: not bounded (also while on the DFS stack) def visit(name: str) -> int | None: @@ -505,7 +514,7 @@ def visit(name: str) -> int | None: return k[name] k[name] = None # on the stack: reaching it again is a cycle f = functions[name] - if not _runs_no_python(f): + if not _runs_no_python(f) or (f.fastmath and name in called_from_exact): return None deepest = 0 for c in callees[name]: @@ -743,6 +752,21 @@ def generate(module: ir.Module, source_path: Path, display_path: str | None = No w("#if !defined(_MSC_VER) || defined(__clang__) /* cl.exe warns on an unknown pragma */") w("#pragma STDC FP_CONTRACT OFF") w("#endif") + if any(f.fastmath for f in module.functions): + w("/* SPEC N-8 (#153, #332): per-function `@compiled(fastmath=True)` opt-in (NOT CPython-identical). */") + w("#if defined(__clang__)") + w("#define TP_FASTMATH_BEGIN _Pragma(\"clang fp reassociate(on) contract(fast)\")") + w("#define TP_FASTMATH_END _Pragma(\"clang fp reassociate(off) contract(off)\")") + w("#elif defined(__GNUC__)") + w("#define TP_FASTMATH_BEGIN _Pragma(\"GCC push_options\") _Pragma(\"GCC optimize(\\\"fast-math\\\")\")") + w("#define TP_FASTMATH_END _Pragma(\"GCC pop_options\")") + w("#elif defined(_MSC_VER)") + w("#define TP_FASTMATH_BEGIN __pragma(float_control(precise, off, push)) __pragma(fp_contract(on))") + w("#define TP_FASTMATH_END __pragma(fp_contract(off)) __pragma(float_control(pop))") + w("#else") + w("#define TP_FASTMATH_BEGIN") + w("#define TP_FASTMATH_END") + w("#endif") w("") w(f"static const char tp_source[] =\n {c_source_init(source)};") w(f"static const char tp_source_path[] = {c_string(display_path)};") @@ -2892,7 +2916,8 @@ def generate(self) -> str: """The impl (preceded by its fast twin when bounded) and the wrapper.""" parts = [] if self.rmode: - return self.impl() + text = self.impl() + return f"TP_FASTMATH_BEGIN\n{text}TP_FASTMATH_END\n" if self.f.fastmath else text if self.f.name in self.mg.bounded: twin = _FnGen(self.mg, self.f, fast=True) twin_text = twin.impl() @@ -2901,6 +2926,8 @@ def generate(self) -> str: impl_text = self.impl() parts += self.par_defs parts.append(impl_text) + if self.f.fastmath: + parts = ["TP_FASTMATH_BEGIN", *parts, "TP_FASTMATH_END"] parts.append(self.wrapper()) return "\n".join(parts) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py index a65ea1b9f..83fa2083d 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py @@ -191,7 +191,8 @@ def lower(path: Path) -> ir.Module: skipped.update(class_skips) signatures: dict[str, _Signature] = {} marked: set[str] = set() - for node, reason, parallel in _targets(tree, source, info): + fastmath_marked: set[str] = set() + for node, reason, parallel, fastmath in _targets(tree, source, info): if reason is not None: skipped[node.name] = reason continue @@ -203,6 +204,8 @@ def lower(path: Path) -> ir.Module: else: if parallel: marked.add(node.name) + if fastmath: + fastmath_marked.add(node.name) for owner, node, reason in _method_targets(tree, info): q = f"{owner}.{node.name}" if reason is not None: @@ -240,7 +243,11 @@ def lower(path: Path) -> ir.Module: else: skipped[name] = result functions.sort(key=lambda f: f.source_line) - functions = [dataclasses.replace(f, parallel=True) if f.name in marked else f for f in functions] + functions = [ + dataclasses.replace(f, parallel=f.name in marked, fastmath=f.name in fastmath_marked) + if (f.name in marked or f.name in fastmath_marked) else f + for f in functions + ] return ir.Module(path.stem, tuple(functions), skipped, tuple(class_decls.values())) @@ -423,30 +430,34 @@ def _is_compiled_decorator(d: ast.expr) -> bool: ) -def _compiled_decorator_options(d: ast.expr) -> tuple[bool, str | None]: - """Parse `@compiled` or `@compiled(parallel=...)` -> (parallel, error_reason_or_None).""" +def _compiled_decorator_options(d: ast.expr) -> tuple[bool, bool, str | None]: + """Parse `@compiled` or `@compiled(parallel=..., fastmath=...)` -> (parallel, fastmath, error_reason_or_None).""" if isinstance(d, ast.Name): - return False, None + return False, False, None assert isinstance(d, ast.Call) if d.args: - return False, (f"line {d.lineno}: `@compiled` takes keyword arguments only " - f"(got positional `{_src(d.args[0])}`)") + return False, False, (f"line {d.lineno}: `@compiled` takes keyword arguments only " + f"(got positional `{_src(d.args[0])}`)") parallel = False + fastmath = False seen: set[str] = set() for kw in d.keywords: if kw.arg is None: - return False, f"line {d.lineno}: `**{_src(kw.value)}` in `@compiled(...)` is not supported" - if kw.arg != "parallel": - return False, (f"line {d.lineno}: `@compiled` keyword argument `{kw.arg}` is not " - "supported (only `parallel=True` or `parallel=False`)") + return False, False, f"line {d.lineno}: `**{_src(kw.value)}` in `@compiled(...)` is not supported" + if kw.arg not in ("parallel", "fastmath"): + return False, False, (f"line {d.lineno}: `@compiled` keyword argument `{kw.arg}` is not " + "supported (only `parallel` and `fastmath` with literal `True` or `False`)") if kw.arg in seen: - return False, f"line {d.lineno}: `@compiled` keyword argument `{kw.arg}` is repeated" + return False, False, f"line {d.lineno}: `@compiled` keyword argument `{kw.arg}` is repeated" seen.add(kw.arg) if not (isinstance(kw.value, ast.Constant) and isinstance(kw.value.value, bool)): - return False, (f"line {d.lineno}: `@compiled(parallel=...)` requires a literal `True` " - f"or `False`, got `{_src(kw.value)}`") - parallel = kw.value.value - return parallel, None + return False, False, (f"line {d.lineno}: `@compiled({kw.arg}=...)` requires a literal `True` " + f"or `False`, got `{_src(kw.value)}`") + if kw.arg == "parallel": + parallel = kw.value.value + else: + fastmath = kw.value.value + return parallel, fastmath, None def is_opted_in(path: Path) -> bool: @@ -614,8 +625,8 @@ def _class_of(kind: str, classes: dict[str, ir.ClassDecl]) -> tuple[str, bool] | def _targets( tree: ast.Module, source: str, info: _ModuleInfo, -) -> list[tuple[ast.FunctionDef | ast.AsyncFunctionDef, str | None, bool]]: - found: list[tuple[ast.FunctionDef | ast.AsyncFunctionDef, str | None, bool]] = [] +) -> list[tuple[ast.FunctionDef | ast.AsyncFunctionDef, str | None, bool, bool]]: + found: list[tuple[ast.FunctionDef | ast.AsyncFunctionDef, str | None, bool, bool]] = [] for node in tree.body: if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): continue @@ -624,6 +635,7 @@ def _targets( continue reason = None parallel = False + fastmath = False if marked and info.bound("compiled"): line = info.bindings["compiled"][0][0] reason = (f"line {node.lineno}: `compiled` is bound in this module (line {line}), " @@ -633,18 +645,19 @@ def _targets( "(only `@compiled`)") else: for d in marked: - par, r = _compiled_decorator_options(d) + par, fm, r = _compiled_decorator_options(d) if r is not None: reason = r break parallel = parallel or par + fastmath = fastmath or fm if reason is None: if isinstance(node, ast.AsyncFunctionDef): reason = f"line {node.lineno}: async function is not supported" elif len(info.bindings.get(node.name, [])) > 1: lines = ", ".join(str(line) for line, _ in info.bindings[node.name]) reason = f"line {node.lineno}: `{node.name}` is bound more than once (lines {lines})" - found.append((node, reason, parallel)) + found.append((node, reason, parallel, fastmath)) return found diff --git a/python-multiplatform-ksp/src/main/python/typedpython/ir.py b/python-multiplatform-ksp/src/main/python/typedpython/ir.py index 1176d2182..4b28c068f 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/ir.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/ir.py @@ -820,6 +820,9 @@ class Function: # SPEC N-15: the user marked this function `@compiled(parallel=True)` (#358). # Only a marked function can hold a `ForRange.par`. parallel: bool = False + # SPEC N-8 (#153, #332): the user marked this function `@compiled(fastmath=True)` (reassociation + # and FMA contraction enabled for this function only; NOT CPython-identical). + fastmath: bool = False # SPEC N-12b (#268): the producers whose result this pure function keeps inside one statement # (`check(make(d))`), so each such call builds its objects in a region. Set only by `verify`, # which recomputes it and rejects a function whose declared entries differ diff --git a/python-multiplatform-ksp/src/test/python/test_frontend.py b/python-multiplatform-ksp/src/test/python/test_frontend.py index ddcf3fcc9..cd631efa6 100644 --- a/python-multiplatform-ksp/src/test/python/test_frontend.py +++ b/python-multiplatform-ksp/src/test/python/test_frontend.py @@ -677,6 +677,43 @@ def bad(x: float) -> float: assert set(m.skipped) == {"bad"} and "parallel" in m.skipped["bad"] +def test_compiled_fastmath_option_and_validation(tmp_path): + m = lower(tmp_path, """ + @compiled(fastmath=True) + def fm(x: float) -> float: + return x + + @compiled(fastmath=False) + def exact(x: float) -> float: + return -x + + @compiled(parallel=True, fastmath=True) + def both(x: float) -> float: + return x * 2.0 + + @compiled + def plain(x: float) -> float: + return x + 1.0 + + @compiled(fastmath=1) + def bad_val(x: float) -> float: + return x + + @compiled(other=True) + def bad_kw(x: float) -> float: + return x + """) + assert [(f.name, f.parallel, f.fastmath) for f in m.functions] == [ + ("fm", False, True), + ("exact", False, False), + ("both", True, True), + ("plain", False, False), + ] + assert set(m.skipped) == {"bad_val", "bad_kw"} + assert "fastmath" in m.skipped["bad_val"] + assert "fastmath" in m.skipped["bad_kw"] and "parallel" in m.skipped["bad_kw"] + + def test_the_former_module_pragma_comment_compiles_nothing(tmp_path): m = lower(tmp_path, """ # typedpython: compiled diff --git a/python-multiplatform-ksp/src/test/python/test_pipeline.py b/python-multiplatform-ksp/src/test/python/test_pipeline.py index ed326eb4c..18355aec4 100644 --- a/python-multiplatform-ksp/src/test/python/test_pipeline.py +++ b/python-multiplatform-ksp/src/test/python/test_pipeline.py @@ -185,3 +185,56 @@ def test_a_fresh_local_does_not_exempt_the_callee_class_guard(tmp_path): c = mod.Cell(None, None) mod.seta(c, 7) assert c.a == 7 + + +def test_default_fp_flags_stay_exact_and_fastmath_is_per_function_only(write, tmp_path): + """#153 / #332: by default, `-ffp-contract=off` and `-fno-fast-math` on GCC/Clang and + `/fp:precise` on MSVC are pinned so floating-point arithmetic is bit-identical to CPython. + `@compiled(fastmath=True)` opts in for that function only (not CPython-identical), leaving + sibling `@compiled` functions in the same module exact.""" + from typedpython import cbuild, cgen, frontend, verify + + assert "-ffp-contract=off" in cbuild.DEFAULT_FLAGS + assert "-fno-fast-math" in cbuild.DEFAULT_FLAGS + assert "/fp:precise" in cbuild.MSVC_DEFAULT_FLAGS + assert not any("fast" in f for f in cbuild.MSVC_DEFAULT_FLAGS) + + path = write( + "fp_mod.py", + """ + @compiled + def fma_exact(a: float, b: float, c: float) -> float: + return a * b + c + + @compiled(fastmath=True) + def fma_fast(a: float, b: float, c: float) -> float: + return a * b + c + + @compiled + def calls_fast_and_adds(a: float, b: float, c: float) -> float: + # caller is exact: its own `a * b + c` must not contract even though it calls `fma_fast` + _ = fma_fast(1.0, 2.0, 3.0) + return a * b + c + """, + ) + proved, _ = verify.verify(frontend.lower(path)) + c_text = cgen.generate(proved, path) + assert "#pragma STDC FP_CONTRACT OFF" in c_text + assert "\nTP_FASTMATH_BEGIN\n" in c_text and "\nTP_FASTMATH_END\n" in c_text + # Only `fma_fast` is wrapped in standalone TP_FASTMATH_BEGIN / TP_FASTMATH_END lines: + assert c_text.count("\nTP_FASTMATH_BEGIN\n") == 1 + assert c_text.count("\nTP_FASTMATH_END\n") == 1 + + res = pipeline.compile_module(path, tmp_path / "out", cost_rule=False) + mod = pipeline.load(res) + ref = interpreted(path) + # 1 + 2**-27 squared: exact double product rounds to 1 + 2**-26, while FMA(a, a, -1.0) keeps 2**-54 + a = 1.0 + 2.0**-27 + c = -1.0 + want_exact = ref.fma_exact(a, a, c) + assert mod.fma_exact(a, a, c) == want_exact + assert mod.calls_fast_and_adds(a, a, c) == want_exact + # fma_fast runs and returns a float (on targets with FMA it equals 2**-26 + 2**-54; either is valid under fastmath) + got_fast = mod.fma_fast(a, a, c) + assert isinstance(got_fast, float) + From 2b1bac3a3d623f96acbf3a2ae5c30c30937e527a Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sun, 1 Nov 2026 00:32:50 +0900 Subject: [PATCH 09/29] TypedPython: TSan test ignores races against new_threadstate reusing a freed block Linux CI (run 41212647289) reported tp_traced's read of an exited thread's PyThreadState racing with libpython's new_threadstate memcpy into the same reused mimalloc block; free and reuse are ordered inside the uninstrumented allocator, as in #347. --- .../src/test/python/test_free_threaded.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python-multiplatform-ksp/src/test/python/test_free_threaded.py b/python-multiplatform-ksp/src/test/python/test_free_threaded.py index e857d3950..a1e925f04 100644 --- a/python-multiplatform-ksp/src/test/python/test_free_threaded.py +++ b/python-multiplatform-ksp/src/test/python/test_free_threaded.py @@ -840,12 +840,14 @@ def _access_is_ours(block: str) -> bool: A report where one access is mimalloc initialising a fresh block (`_mi_malloc*` memset) is not ours either: the extension's earlier read hit memory a thread later freed (an exited thread's `PyThreadState`, read by `tp_traced`), and the free and the reuse both happen inside the - uninstrumented allocator, so TSan cannot see that they are ordered (seen on Linux CI).""" + uninstrumented allocator, so TSan cannot see that they are ordered (seen on Linux CI). The same + holds when the reused block is filled by libpython's `new_threadstate` (a memcpy of the template + thread state into the block an exited thread's state was freed from, seen on Linux CI).""" firsts = [] for part in block.split("\n\n")[:2]: frames = [ln for ln in part.splitlines() if ln.strip().startswith("#")] firsts.append(next((f for f in frames if "libtsan" not in f and "sanitizer_common" not in f), "")) - if any(" _mi_malloc" in f or " _mi_page_malloc" in f for f in firsts): + if any(" _mi_malloc" in f or " _mi_page_malloc" in f or " new_threadstate " in f for f in firsts): return False return any("ftmod" in f for f in firsts) From 371e9679ed7f1c6709164e69cd871ec4ec203d2a Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sun, 1 Nov 2026 10:53:02 +0900 Subject: [PATCH 10/29] Test: Make mid() loop heavier in test_eval_breaker to avoid CI timing flake (#280) --- .../src/test/python/test_eval_breaker.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python-multiplatform-ksp/src/test/python/test_eval_breaker.py b/python-multiplatform-ksp/src/test/python/test_eval_breaker.py index dece8d3b8..faea4a5ae 100644 --- a/python-multiplatform-ksp/src/test/python/test_eval_breaker.py +++ b/python-multiplatform-ksp/src/test/python/test_eval_breaker.py @@ -397,7 +397,10 @@ def test_keyboard_interrupt_reaches_a_long_range_loop_in_chunks(extension): def mid(n: int) -> float: x: float = 1.0 for k in range(n): - x = x * 0.9999999 + 1.0 / (k + 1.0) + x = (x + 1.0) / (x + 2.0) + 1.0 / (k + 1.0) + x = (x + 1.0) / (x + 2.0) + 1.0 / (k + 2.0) + x = (x + 1.0) / (x + 2.0) + 1.0 / (k + 3.0) + x = (x + 1.0) / (x + 2.0) + 1.0 / (k + 4.0) return x """ @@ -419,7 +422,7 @@ def handler(signum, frame): t0 = time.perf_counter() m.mid(n) full = time.perf_counter() - t0 - signal.setitimer(signal.ITIMER_REAL, full / 10) + signal.setitimer(signal.ITIMER_REAL, full / 20) t0 = time.perf_counter() try: m.mid(n) @@ -430,4 +433,5 @@ def handler(signum, frame): print(json.dumps({"out": out, "full": full, "elapsed": time.perf_counter() - t0})) """) assert r["out"] == "interrupted", r + assert r["full"] > 0.1, r assert r["elapsed"] < r["full"] * 0.8, r # before #280 the signal waited for the whole loop From c14d26498ffeae54a9bf387bfbc6b6a524c97a4d Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sun, 1 Nov 2026 11:08:49 +0900 Subject: [PATCH 11/29] Feat: Switch notebook-e2e native-image CI to Maven Central and stage sample native-image prefix (#191, #251) --- .github/workflows/desktop.yml | 36 +--- docs/SPEC.md | 2 +- .../fork-desktop-paths-macos.md | 196 +++++++----------- .../fixtures/notebook-e2e/README.md | 5 +- sample/build.gradle.kts | 12 +- 5 files changed, 96 insertions(+), 155 deletions(-) diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index 0b43ff3e5..3956921f1 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -151,26 +151,15 @@ jobs: python-multiplatform-compose/fixtures/notebook-e2e/build/reports/tests/desktopTest python-multiplatform-compose/fixtures/notebook-e2e/build/test-results/desktopTest - # The notebook E2E in a GraalVM native image on macOS (#249, #251). It takes `ui`, `ui-text` and - # `foundation` from compose-multiplatform-core-extended, which a runner reads from the fork's GitHub - # Packages registry; until the repository variable COMPOSE_EXTENDED_VERSION names a published - # development version (`1.11.1-ext.N-dev.`) the job is skipped. + # The notebook E2E in a GraalVM native image on macOS (#249, #251). It takes the fork's Compose + # (`org.thisisthepy.compose.*:1.11.1-ext.2`, `composeExtended.*` in `gradle.properties`) from + # Maven Central, builds the 20-scenario executable with Liberica NIK 25, and runs it with + # JAVA_HOME and PYTHONHOME removed. notebook-e2e-native-image: name: Notebook E2E in a native image (macOS) - if: ${{ vars.COMPOSE_EXTENDED_VERSION != '' }} runs-on: macos-latest - permissions: - contents: read - packages: read env: WHEEL_DIR: ${{ github.workspace }}/.tmp/pythonx-compose-dist - COMPOSE_EXTENDED_VERSION: ${{ vars.COMPOSE_EXTENDED_VERSION }} - COMPOSE_EXTENDED_GROUP_PREFIX: ${{ vars.COMPOSE_EXTENDED_GROUP_PREFIX || 'org.thisisthepy.compose' }} - COMPOSE_EXTENDED_REPOSITORY: ${{ vars.COMPOSE_EXTENDED_REPOSITORY || 'https://maven.pkg.github.com/thisisthepy/compose-multiplatform-core-extended' }} - # GitHub Packages asks for a token even for a public package. A read:packages PAT in the secret - # GH_PACKAGES_TOKEN is used when it exists; otherwise this repository's own token. - GITHUB_TOKEN: ${{ secrets.GH_PACKAGES_TOKEN || secrets.GITHUB_TOKEN }} - TOKEN_SOURCE: ${{ secrets.GH_PACKAGES_TOKEN != '' && 'GH_PACKAGES_TOKEN' || 'GITHUB_TOKEN' }} steps: - name: Checkout uses: actions/checkout@v4 @@ -183,19 +172,6 @@ jobs: path: .tmp/pythonx-compose fetch-depth: 1 - # Before any build: can this token read the fork's packages? One known POM, so a refusal is - # reported as what it is rather than as a dependency resolution error minutes later. - - name: Check access to the fork's packages - run: | - group_path="${COMPOSE_EXTENDED_GROUP_PREFIX//.//}/ui" - pom="$COMPOSE_EXTENDED_REPOSITORY/$group_path/ui-desktop/$COMPOSE_EXTENDED_VERSION/ui-desktop-$COMPOSE_EXTENDED_VERSION.pom" - status=$(curl -s -o /dev/null -w '%{http_code}' -u "${GITHUB_ACTOR}:${GITHUB_TOKEN}" "$pom") - echo "reading $pom with $TOKEN_SOURCE: HTTP $status" - if [ "$status" != "200" ]; then - echo "::error::The fork's packages could not be read with $TOKEN_SOURCE (HTTP $status at $pom). Add a personal access token (classic) with read:packages as the repository secret GH_PACKAGES_TOKEN, or check COMPOSE_EXTENDED_VERSION / COMPOSE_EXTENDED_REPOSITORY." - exit 1 - fi - - name: Setup uv uses: astral-sh/setup-uv@v6 @@ -225,8 +201,4 @@ jobs: ./gradlew :python-multiplatform-compose:fixtures:notebook-e2e:notebookE2eNativeAgent :python-multiplatform-compose:fixtures:notebook-e2e:notebookE2eNativeRun -PpythonxComposeWheel="$WHEEL_DIR" - -PcomposeExtended.version="$COMPOSE_EXTENDED_VERSION" - -PcomposeExtended.groupPrefix="$COMPOSE_EXTENDED_GROUP_PREFIX" - -PcomposeExtended.mavenLocal=false - -PcomposeExtended.repository="$COMPOSE_EXTENDED_REPOSITORY" --console=plain diff --git a/docs/SPEC.md b/docs/SPEC.md index 1e0697a64..4d982f37c 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -628,7 +628,7 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. main-thread work to skiko's Swing queue, which starts AWT, and AWT in an image needs the JDK's `libawt`, which links `libjvm`; the fork sends it to `Dispatchers.Main` when `compose.main.dispatcher=coroutines`, and the fixture supplies a single-thread one with no AWT. - `desktopTest` keeps upstream Compose. Not on CI yet: the fork's artefacts are in mavenLocal only. + CI runs it on every push to `develop` (`desktop.yml`, job `notebook-e2e-native-image`, #251). - **B-11** (#192) The artifact walker skips every declaration whose package has an `internal` segment (`*.internal` and `*.internal.*`, as in `androidx.compose.ui.internal`), on jars and on klibs. That is the Kotlin `.api` dump convention: a library that puts a declaration in such a package has declared it diff --git a/docs/investigations/fork-desktop-paths-macos.md b/docs/investigations/fork-desktop-paths-macos.md index bbdc507c1..df39c1995 100644 --- a/docs/investigations/fork-desktop-paths-macos.md +++ b/docs/investigations/fork-desktop-paths-macos.md @@ -2,128 +2,90 @@ What python-multiplatform needs before a pythonx-compose app runs on the two desktop paths `compose-multiplatform-core-extended` and `compose-multiplatform-extended` add: a GraalVM native -image and a Kotlin/Native executable, macOS arm64 first. Read on 2026-10-17 from the fork's -`origin/extended` branches (core `61906995435`, plugin fork `fca1473a`), `~/.m2/repository`, and -this repository's `develop` (`af4debac`). Nothing here was built; every fact cites where it was read. - -## 1. What the fork has on macOS +image and a Kotlin/Native executable, macOS arm64 first. Originally read on 2026-10-17 and refreshed +on 2026-11-01 after the fork published to Maven Central (#257, #287), the notebook E2E native-image +milestone landed (#246, #249, #251), and the `#191` decisions were recorded (2026-10-25). + +## 1. What the fork publishes on Maven Central + +The fork publishes its Compose libraries, window modules and Gradle plugins to Maven Central under +`org.thisisthepy.compose`: + +- **Libraries (`1.11.1-ext.2`)**: `org.thisisthepy.compose.runtime:*`, `ui:*`, `foundation:*`, + `material:*`, `material3:*`, `animation:*`, `components:*`, `org.thisisthepy.compose.androidx.lifecycle:*`, + including `macosarm64` and `linuxx64` klibs alongside `desktop` (JVM), `android`, `ios*` and + `wasmjs`. (`material-icons-core` stays on JetBrains `1.7.3`.) +- **Window modules (`1.11.1-ext.2`)**: `org.thisisthepy.compose.window:common`, `scene`, + `graalvm-macos`, `graalvm-linux`, `native-macos`, `native-linux`. +- **Gradle plugins (`1.11.1-ext.3`)**: `org.thisisthepy.compose` and + `org.thisisthepy.kotlin.native.desktop` (`compose-gradle-plugin:1.11.1-ext.3` on Maven Central). +- **Path selection in the fork's plugin**: `compose.desktop.application { output.set(ApplicationOutput.NativeImage) }` + (with `windowing.set(ApplicationWindowing.Awt)` or `AwtFree`, and `nativeImage { ... }` registering + `packageNativeImage`, `runNativeImageAgent`, `checkNativeImageNoAwt` and `packageApplication`), or + `compose.desktop.nativeApplication { ... }` for Kotlin/Native (`createDistributableNative*`, + `runNative*`, `packageKotlinNative`). Per decision 5 on #191, consumers select the path through the + fork plugin's setting directly; `python-multiplatform` adds no duplicate path-selection DSL. | | GraalVM native image | Kotlin/Native | |---|---|---| -| Fork's claim | `extended/README.md` (core, line 12): "macOS works on both: the Kotlin/Native window (AppKit and Metal) and the GraalVM native-image window." | same line | -| Window layer | `extended/window/graalvm/graalvm-macos` (JVM lib, AppKit through `@CFunction`, `upcalls.txt` table, no reflection) | `extended/window/native/macos` (`macosArm64`: AppKit, Metal, `MetalSurface.kt`) | -| Compose host | **none for this path yet**: `extended/window/scene` (JVM and `macosArm64`) hosts a Compose scene through `runComposeWindow(platform, config, surfaceFor, content)` (`scene/src/.../ComposeWindow.kt:285`), but `graalvm-macos` provides no `WindowSurface` to draw into | `scene` with `native/macos`'s `MetalSurface` | -| Measured with Compose content | `extended/native-image/hello` (plugin fork): Material 3 in an **AWT** window (`windowing = Awt`), 82 MB, "draws the same content through Skia offscreen into a PNG byte for byte the one the JVM run draws" (`extended/native-image/README.md`, 2026-10-02) | no Gradle demo in the fork; compose-rust's Amper renderer (`darkpyonix/compose-rust/renderer/macos`) consumes `native-macos` | -| AWT-free demo | `extended/native-image/hello-awtfree-link`: opens, pumps and presents an AppKit window; **draws no Compose** | none | -| Build | the plugin fork's `org.jetbrains.compose` `1.11.1-extended-dev`: `compose.desktop.application { nativeImage { graalvmHome; skikoStaticDirectory } }`, `packageNativeImage`, `runNativeImageAgent`; skiko as a static archive (`extended/skiko/build-skiko-static-jvm.sh`) | `extended/gradle-plugin-kn`: `createDistributableNative...`, `runNative...`, `packageKotlinNative` | - -**Published to mavenLocal** (JetBrains coordinates kept, as #191 says): +| Window layer | `org.thisisthepy.compose.window:graalvm-macos`, `graalvm-linux` (JVM lib, `@CFunction` + `upcalls.txt` table, no reflection) | `org.thisisthepy.compose.window:native-macos` (`macosArm64`: AppKit, Metal, `MetalSurface.kt`), `native-linux` (`linuxX64`) | +| Compose host | `org.thisisthepy.compose.window:scene` (`runComposeWindow`) | `scene` with `native-macos` / `native-linux` | +| Build | `org.thisisthepy.compose` `1.11.1-ext.3`: `compose.desktop.application { output.set(ApplicationOutput.NativeImage); nativeImage { ... } }`, `packageNativeImage`, `runNativeImageAgent` | `org.thisisthepy.kotlin.native.desktop` `1.11.1-ext.3`: `compose.desktop.nativeApplication { ... }`, `createDistributableNative*`, `runNative*` | -- `org.jetbrains.compose:ui*`, `foundation*`, `ui-text*` at `1.11.1001` (2026-10-17), including the - `macosarm64` klibs of `ui` and `foundation`. No `material3` macOS klib. -- `org.jetbrains.compose:compose-gradle-plugin:1.11.1-extended-dev`. -- `org.thisisthepy.compose.window:*:0.1.0`: `common`, `graalvm-macos`, `graalvm-linux`, `native-macos`. Not - `scene`, and not `native-linux`. -- `org.jetbrains.skiko:skiko-awt:0.144.6.1`. -- No publish script producing `1.11.1001` was found in either fork repository; the "Extended window - publish" workflow and compose-rust's `scripts/publish-window.sh` publish the window modules. - -## 2. What this repository has +## 2. What this repository has today +- **Switched to the fork's Maven Central artefacts (#257, #287).** `gradle/libs.versions.toml` uses + `org.thisisthepy.compose` `1.11.1-ext.3` (plugins) and `1.11.1-ext.2` (libraries). The artifact + walker (`ArtifactScanner.kt`) resolves `org.thisisthepy.compose.*` coordinates alongside + `org.jetbrains.compose.*`. - **Targets.** `python-multiplatform` declares `jvm("desktop")`, Android, iOS, androidNative and wasm; - no `macosArm64`, `linuxX64` or `mingwX64` (`python-multiplatform/build.gradle.kts`; the plugin's - `hostDesktopPlatform` knows macOS only as a JVM host). The sample has the same set. -- **A native image already runs Python** (`docs/platforms/graal-native-image-verification.md`, - 2026-08-17, `4687070a`): `:sample:nativeCompile` (an `Exec` of Liberica NIK 25.0.4's `native-image`, - `--no-fallback --enable-native-access=ALL-UNNAMED`) builds a headless `NativeImageMainKt`; 28 - downcalls and 5 upcalls registered, `UPCALL_OK`/`PROXY_OK`, no `libjvm`, `libpython` `dlopen`ed. - It is **headless on purpose**: the sample's `application {}` needs AWT. Not automated, not re-run. + `macosArm64` and `linuxX64` are the next interpreter targets to add (Phase A, decision 2; + `mingwX64` in Phase C). +- **A native image runs Python and stages `PYTHONHOME` beside the executable (#246, SPEC L-9).** + `pythonBindings.nativeImageDirectory` and `stageNativeImagePythonHome` copy the pruned + `python-build-standalone` prefix (`lib/python/**` and `lib/libpython.*`) into + `/python-multiplatform-home`. At run time, inside a GraalVM native image + (`org.graalvm.nativeimage.imagecode == "runtime"`), `PackagedPythonHome.resolve()` looks up + `/python-multiplatform-home` via `ProcessHandle.current().info().command()` + and hands it to CPython with `Py_SetPythonHome`. `:sample:nativeCompile` and `:sample:runNativeUpcallDemo` + stage and verify this layout with `JAVA_HOME` and `PYTHONHOME` unset (`docs/platforms/graal-native-image-verification.md`). - **The upcall table on a native image.** Desktop upcalls are FFM (`Panama.kt`), registered by the generated `META-INF/native-image/org.thisisthepy/python-multiplatform/reachability-metadata.json` - (`generateDesktopReachabilityMetadata`) plus `native-image/reachability-metadata.base.json`. That - metadata ships in the jar, so any native-image build that has the jar on its classpath picks it up. - (The verification note says no reflection registration is needed, while its own log shows 1,822 - types registered and the base file registers `UpcallTarget`; the note should be corrected.) -- **PYTHONHOME.** `stagePythonHome` sets `PYTHONHOME` for `JavaExec`/`Test` tasks and copies the prefix - into Compose's `prepare*AppResources`; the native demo is given `PYTHONHOME` by its run task. A - native image has no `java.home` and no packaged-resources directory, so nothing stages a prefix - **beside the executable** or finds it from there at start. -- **The notebook E2E** (`python-multiplatform-compose/fixtures/notebook-e2e`) needs no window: it - draws `PythonAppView` into an offscreen `ImageComposeScene` (`NotebookHost.kt:132`) and compares - pixels. It is a JVM `desktopTest` suite with `compose.desktop.currentOs` (Skia natives) and - `java.awt.headless=true`; no `java.awt`/`javax.swing` import. - -## 3. What each path needs from this side - -### GraalVM native image - -1. **A desktop app built by the plugin fork's `packageNativeImage`** rather than the sample's hand-made - `Exec`: the fork already solves Compose's own native-image configuration and static skiko. The - generated upcall table and FFM registrations come along in the jar's `META-INF/native-image`, so - **the upcall table carries over unchanged** (build-time table, no reflection: AGENTS.md §12.3, §13). - To check: that the fork's plugin passes `--enable-native-access=ALL-UNNAMED` (or lets an app add it) - and does not set `--no-fallback`-incompatible options that hide a missing FFM registration. -2. **PYTHONHOME beside the executable.** A task that stages the prefix and the installed wheels next to - `build/compose/native-image/main/`, and a runtime lookup (`ProcessHandle`'s executable path, a - sibling `python/` directory) when `PYTHONHOME` is unset, like `IosPythonHome` does for an app bundle. - **Staging does not carry over as is**: today's paths are a Gradle task's environment and the JVM - packaging's resources directory. -3. **The walker over the fork's jars.** Same public API, so the walk must produce the same bindings; - guarded by the stub diff of pythonx-compose #54. -4. **Selecting the path.** A Gradle plugin option (upstream Compose by default, the fork's native image - as a choice), #191's first bullet. - -### Kotlin/Native - -1. **An interpreter target**: `macosArm64` in `python-multiplatform` (cinterop against the macOS - python-build-standalone prefix the desktop JVM already downloads, a static or shipped - `libpython`), with the native `HandleTable`/trampoline path iOS and androidNative already use. - This is new work, not a switch. -2. **The klib walker** (B-2) over the fork's `macosarm64` klibs: since #234/#237 it carries handles and - interface members, but not constructors, properties or composables, and Compose's surface is mostly - composables and extensions; the K/N path needs the walker to reach about where the jar walker is. -3. **PYTHONHOME** beside the executable, as for the native image. -4. `material3` for `macosArm64`: no fork klib is published (the fork says only `ui` and `foundation` - differ from JetBrains; JetBrains' own `material3` macOS klib would have to resolve beside them). - -## 4. First milestone: the notebook E2E in a macOS GraalVM native image - -The E2E draws offscreen, so the milestone needs **no window** and none of the AWT-free pieces: - -1. A `notebook-e2e` native-image entry point (a `main` running the existing scenarios and printing a - pass/fail line per scenario, the way the fork's `hello` self-checks through `HELLO_SELF_CHECK`), - because a JUnit `desktopTest` does not run inside an image. -2. Built with the plugin fork's `packageNativeImage` (`windowing = Awt`, which is what the fork measured; - the scenes are offscreen either way), `--enable-native-access=ALL-UNNAMED`, the fork's static skiko. -3. PYTHONHOME and the pythonx-compose wheel staged beside the executable (§3, GraalVM 2). -4. Pass: the same 20 scenarios as the JVM run (pixel comparisons included), the binary run alone from - an empty directory with `JAVA_HOME` unset, and `otool -L` showing no `libjvm`. Then a CI job on a - macOS runner, as `notebook-e2e` has on the JVM (#213). - -Milestone 2 is the same app in the AWT-free window, once the fork has the pieces in §5. Kotlin/Native -follows the interpreter target (§3, K/N 1). - -## 5. Requests for the fork (through the darkpyonix lead) - -1. **Publish `org.thisisthepy.compose.window:scene`** (its `module.yaml` already enables publishing; - it is missing from mavenLocal and from the "Extended window publish" workflow's output). -2. **A `WindowSurface` for `graalvm-macos`** (Metal or GL into a Skia surface, as `native/macos`'s - `MetalSurface`), so `runComposeWindow` can host Compose content on the GraalVM path; and a demo that - draws Compose through it (`hello-awtfree-link` draws nothing today). -3. **A documented, reproducible publish of `1.11.1001`** (no script producing that version was found) - and one version line for the Compose libraries, the window modules and the Gradle plugin, so a - consumer pins one set. -4. **The native-image flags an app can add**: confirm that `packageNativeImage` lets an app pass - `--enable-native-access=ALL-UNNAMED` and merges a library's `META-INF/native-image` metadata (FFM - `foreign` registrations included). -5. **A hook to stage extra files beside the image** (a prefix directory), or confirmation that a - consumer task after `packageNativeImage` is the supported way. -6. For Kotlin/Native: whether `material3` for `macosArm64` is expected to come from JetBrains, and a - Gradle (not Amper-only) example of `scene` plus `native-macos` hosting Compose content. - -## 6. Open questions - -- Whether CPython's own native-image behaviour holds with the fork's Compose in the same image (one - process, Skia and libpython both `dlopen`ed or linked); not tried. -- Where the native-image E2E runs in CI: GitHub's macOS runners have no GraalVM by default, and the - fork's static skiko must be built or cached. + (`generateDesktopReachabilityMetadata`) plus `native-image/reachability-metadata.base.json` (which + registers `python.native.ffi.UpcallTarget`, corrected in #244). That metadata ships in the jar, so + any native-image build that has the jar on its classpath picks it up. +- **The notebook E2E runs in a macOS GraalVM native image (#249, #251, SPEC B-10).** + `python-multiplatform-compose/fixtures/notebook-e2e` builds `notebook-e2e` with `notebookE2eNativeCompile` + and runs all 20 `UI.ipynb` scenarios via `notebookE2eNativeRun` with `JAVA_HOME` and `PYTHONHOME` + removed, `compose.main.dispatcher=coroutines` (`NotebookE2eMainDispatcherFactory`, no AWT), and an + `otool -L` check forbidding `libjvm`. `.github/workflows/desktop.yml` (`notebook-e2e-native-image`) + runs this job on macOS against Maven Central. + +## 3. Remaining work by phase (per #191 decisions, 2026-10-25) + +### Phase A (in progress) + +1. **#251 (`desktop.yml`)**: `notebook-e2e-native-image` switched to Maven Central with the + `COMPOSE_EXTENDED_VERSION` guard and GitHub Packages check removed. +2. **`:sample` as a native-image output**: `:sample` sets `pythonBindings.nativeImageDirectory` so + `stageNativeImagePythonHome` stages `python-multiplatform-home/` beside `upcall-native-demo`, and + `runNativeUpcallDemo` runs the executable with `JAVA_HOME` and `PYTHONHOME` removed. +3. **Kotlin/Native desktop interpreter targets (`macosArm64`, `linuxX64`)**: add `macosArm64()` and + `linuxX64()` to `:python-multiplatform`, cinterop against the `python-build-standalone` headers + already downloaded by `downloadPython_macos_aarch64` and `downloadPython_linux_x86_64`, link and + load `libpython` dynamically beside the executable (decision 3), and wire `PackagedPythonHome` / + `PythonPayload` for desktop K/N executables. + +### Phase B + +1. **AWT-free GraalVM notebook app**: run the notebook app in an AWT-free window (`windowing = AwtFree`). +2. **Kotlin/Native macOS app**: re-check `KlibScanner` (B-2) coverage for `@Composable` functions, + constructors and properties over the fork's `macosarm64` klibs, add `macosArm64` to + `:python-multiplatform-compose`, and run a K/N macOS app. + +### Phase C + +1. **Windows (`mingwX64`)** once the fork publishes Windows artefacts. +2. **Stub diff guard (`pythonx-compose #54`)** and the final acceptance criterion: the same + `pythonx-compose` app running unchanged on upstream Compose, extended GraalVM and extended K/N + desktop builds with the notebook E2E passing in CI. diff --git a/python-multiplatform-compose/fixtures/notebook-e2e/README.md b/python-multiplatform-compose/fixtures/notebook-e2e/README.md index af55ae435..ef0399bf7 100644 --- a/python-multiplatform-compose/fixtures/notebook-e2e/README.md +++ b/python-multiplatform-compose/fixtures/notebook-e2e/README.md @@ -68,11 +68,12 @@ JAVA_HOME= ./gradlew :python-multiplatform-compose:fixtures:notebook-e2e `site-packages/` (the wheel) and `app/`. The image finds them there with no environment. - `notebookE2eNativeRun` removes `JAVA_HOME` and `PYTHONHOME`, requires 20 passed, and fails if `otool -L` shows `libjvm`. -- The image takes `ui`, `ui-text` and `foundation` from compose-multiplatform-core-extended's mavenLocal - build (`composeExtended.*` in the root `gradle.properties`): upstream Compose sends its main-thread +- The image takes `ui`, `ui-text` and `foundation` from compose-multiplatform-core-extended on Maven + Central (`composeExtended.*` in the root `gradle.properties`): upstream Compose sends its main-thread work to skiko's Swing queue, which needs AWT, and AWT needs `libjvm`. The fork sends it to `Dispatchers.Main` under `compose.main.dispatcher=coroutines`, which `NotebookE2eMainDispatcherFactory` supplies (registered by `src/desktopTest/native-image-main/` on the image's classpath only). + CI runs `notebookE2eNativeAgent` and `notebookE2eNativeRun` in `.github/workflows/desktop.yml` (`notebook-e2e-native-image`, #251). ## Scenarios diff --git a/sample/build.gradle.kts b/sample/build.gradle.kts index 2e8a35dcd..2c9f5e32b 100644 --- a/sample/build.gradle.kts +++ b/sample/build.gradle.kts @@ -411,10 +411,14 @@ val nativeImageJavaLauncher = javaToolchains.launcherFor { val nativeImageOutputDir = layout.buildDirectory.dir("native/nativeCompile") +pythonBindings { + nativeImageDirectory.set(nativeImageOutputDir) +} + val nativeCompile by tasks.registering(Exec::class) { group = "native" description = "Builds upcall-native-demo with GraalVM native-image from NativeImageMain.kt" - dependsOn(desktopMainCompilation.compileTaskProvider, pythonMultiplatformDesktopJar) + dependsOn(desktopMainCompilation.compileTaskProvider, pythonMultiplatformDesktopJar, "stageNativeImagePythonHome") val classpathProvider = files( desktopMainCompilation.output.allOutputs, @@ -425,7 +429,7 @@ val nativeCompile by tasks.registering(Exec::class) { val launcher = nativeImageJavaLauncher inputs.files(classpathProvider) - outputs.dir(outDir) + outputs.file(outDir.map { it.file("upcall-native-demo") }) doFirst { outDir.get().asFile.mkdirs() @@ -445,8 +449,10 @@ tasks.register("runNativeUpcallDemo") { group = "verification" description = "Runs the native-image binary built by nativeCompile and checks the upcall path" dependsOn(nativeCompile) + workingDir(nativeImageOutputDir) executable = nativeImageOutputDir.get().asFile.resolve("upcall-native-demo").absolutePath - environment("PYTHONHOME", pythonHomeForHost.get().asFile.absolutePath) + environment.remove("JAVA_HOME") + environment.remove("PYTHONHOME") // A zero exit status is not the same as a working upcall path. `PyRun_SimpleString` reports // failure for an exception raised inside the script, but a Python interpreter that never got From afec35c4f76127e652dbf36dab6e8e65459c8b04 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Thu, 5 Nov 2026 00:26:05 +0900 Subject: [PATCH 12/29] Fix: Point Gradle toolchain resolution at GRAALVM_HOME in native-image CI (#251) --- .github/workflows/desktop.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index 3956921f1..cad58b5fd 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -196,9 +196,13 @@ jobs: # Records Compose and skiko's configuration against the fork version this run uses, then builds # the image and runs it with JAVA_HOME and PYTHONHOME removed: 20 of 20 and no libjvm. + # `setup-graalvm` installs Liberica NIK under `GRAALVM_HOME` in the runner toolcache outside + # `/Users/runner/hostedtoolcache/Java_*`, so pass `org.gradle.java.installations.paths` + # explicitly instead of picking up the runner's pre-installed Temurin 25. - name: Build and run the native image run: > ./gradlew :python-multiplatform-compose:fixtures:notebook-e2e:notebookE2eNativeAgent :python-multiplatform-compose:fixtures:notebook-e2e:notebookE2eNativeRun -PpythonxComposeWheel="$WHEEL_DIR" + -Porg.gradle.java.installations.paths="$GRAALVM_HOME" --console=plain From 5b7ac9d96b8607a2283c516ee7414d19b84f2400 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Thu, 5 Nov 2026 07:42:24 +0900 Subject: [PATCH 13/29] TypedPython: cache generated C for the pypackpack compile slot (#113) --- docs/SPEC.md | 11 +- docs/design/typedpython.md | 6 + .../src/main/python/typedpython/pipeline.py | 113 +++++++++++++----- .../src/test/python/test_generate_project.py | 111 +++++++++++++++++ 4 files changed, 209 insertions(+), 32 deletions(-) create mode 100644 python-multiplatform-ksp/src/test/python/test_generate_project.py diff --git a/docs/SPEC.md b/docs/SPEC.md index 4d982f37c..2e65b6f45 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -784,7 +784,16 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. the compile step. `Status: partial`: the per-module cache and its key are implemented in Python (`incremental.py`, `pipeline.compile_project`; `test_incremental.py`, `test_pipeline_incremental.py`); the Gradle task with declared inputs and outputs does not exist yet, because compiled modules are built - through pypackpack's compile slot (#25, #113). + through pypackpack's compile slot (#25). + For that slot, `pipeline.generate_project` caches source to C instead of source to extension (#113): + one `.c` per compiled module plus the directory of the header-only `tp_runtime.h`; its key is the + source, the dependency interface hashes and the compiler version (which covers the runtime header), + with no platform, C compiler or flag in it, because the C depends on none of them. The slot caches + C to extension itself. Tests (`test_generate_project.py`): an unchanged module is reused with a + byte-identical `.c`, a fresh cache produces the same bytes, a body change regenerates that module + only, changing toolchain flags or platform reuses every entry, the C compiler is never run, and the + emitted `.c` built by `cbuild` behaves as `compile_project`'s extension. `compile_project` keeps + building in process. - **N-11** TypedPython fixed-layout classes (M2b, 2026-11-21; design §4.3.3). A class compiles to fixed field access only when its layout is fixed **in CPython too**: it declares `__slots__` listing exactly its annotated instance fields, has no base but `object`, no metaclass, no decorator but `@compiled`, diff --git a/docs/design/typedpython.md b/docs/design/typedpython.md index 991207854..eaf3a02d2 100644 --- a/docs/design/typedpython.md +++ b/docs/design/typedpython.md @@ -162,6 +162,12 @@ API 안정성을 확인한 뒤로 미룬다. 인터페이스 해시가 그대로라 의존 모듈은 다시 컴파일하지 않는다. C 오브젝트는 모듈별로 캐시한다. Gradle 태스크는 입력·출력을 정확히 선언해 up-to-date 판정과 빌드 캐시가 함께 작동한다. +pypackpack 의 native compile slot 이 C 를 확장으로 빌드하므로 캐시는 두 층으로 나뉜다(#113). +`pipeline.generate_project` 는 소스 → C 만 캐시한다: 모듈마다 `.c` 하나, 그리고 `tp_runtime.h` 가 있는 +디렉터리. 키는 소스 + 의존 모듈 인터페이스 해시 + 컴파일러 버전이고, C 는 플랫폼·컴파일러·플래그를 읽지 +않으므로 그것들은 키에 넣지 않는다. 바뀌지 않은 모듈은 바이트가 같은 `.c` 로 재사용되어 slot 의 C → 확장 +캐시도 맞는다. 테스트가 쓰는 `compile_project`(프로세스 안에서 빌드)는 그대로 둔다. + ### 4.3.3 고정 레이아웃 클래스 (SPEC N-11, M2b) **레이아웃이 CPython 에서도 고정일 때만** 고정 접근으로 컴파일한다, 사용자가 `__slots__` 를 직접 쓴 클래스. diff --git a/python-multiplatform-ksp/src/main/python/typedpython/pipeline.py b/python-multiplatform-ksp/src/main/python/typedpython/pipeline.py index 3f504651c..f056da353 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/pipeline.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/pipeline.py @@ -154,27 +154,24 @@ class ModuleResult: interface: str = "" kind: str = "compiled" # "compiled" | "nothing_compiled" | "not_marked" skipped_items: list[SkippedItem] = field(default_factory=list) # `skipped`, structured + c_source: Path | None = None # generate_project only: the module's one `.c` (#113) @dataclass class ProjectResult: modules: list[ModuleResult] + runtime_include: Path | None = None # generate_project only: the dir holding tp_runtime.h @property def rebuilt(self) -> list[ModuleResult]: return [m for m in self.modules if m.status == "rebuilt"] -def compile_project( - sources: list[Path], - project_root: Path, - cache_dir: Path, - *, - python_include: Path | None = None, - ext_suffix: str | None = None, - runtime_dir: Path | None = None, - cost_rule: bool = True, -) -> ProjectResult: +def _project(sources: list[Path], project_root: Path, cache_dir: Path, *, cost_rule: bool, + compile_c, compiler_version: str, platform: str, flags: list[str], + c_mode: bool) -> list[ModuleResult]: + """The cached source -> artefact walk shared by compile_project (artefact: the extension) and + generate_project (artefact: the `.c`).""" sources = [Path(s).resolve() for s in sources] cache_dir = Path(cache_dir) verified: dict[Path, list[SkippedItem]] = {} @@ -193,27 +190,12 @@ def lower(path: Path) -> ir.Module: def generate(module: ir.Module, path: Path) -> str: return cgen.generate(module, path, _relative(path, root)) if module.functions else "" - def compile_c(c_source: str, name: str, out_dir: Path) -> Path: - if not c_source: - marker = Path(out_dir) / (name.rpartition(".")[2] + _NO_EXTENSION) - marker.write_text("nothing compiled\n") - return marker - return cbuild.build(c_source, name, out_dir, runtime_dir=runtime_dir, flags=cbuild.DEFAULT_FLAGS, - python_include=python_include, ext_suffix=ext_suffix) - - version = COMPILER_VERSION - if runtime_dir is not None: - version += ":" + _tree_hash(Path(runtime_dir)) if not cost_rule: - version += ":nocost" - suffix = ext_suffix if ext_suffix is not None else sysconfig.get_config_var("EXT_SUFFIX") - include = cbuild._python_include(python_include) - flags = [*cbuild.DEFAULT_FLAGS, f"-I{include}"] - + compiler_version += ":nocost" cache = incremental.BuildCache(cache_dir) report = cache.build_project( sources, project_root, lower=lower, generate=generate, compile_c=compile_c, - compiler_version=version, platform=f"{sysconfig.get_platform()} {suffix}", flags=flags) + compiler_version=compiler_version, platform=platform, flags=flags) side = cache_dir / "skipped" side.mkdir(exist_ok=True) @@ -230,9 +212,78 @@ def compile_c(c_source: str, name: str, out_dir: Path) -> Path: items = [SkippedItem(**i) for i in data["items"]] except (OSError, ValueError, KeyError, TypeError): skipped, items = {}, [] - ext = None if m.artifact is None or m.artifact.suffix == _NO_EXTENSION else m.artifact + art = None if m.artifact is None or m.artifact.suffix == _NO_EXTENSION else m.artifact + ext, c_file = (None, art) if c_mode else (art, None) kind = ("not_marked" if not frontend.is_opted_in(m.path) - else "compiled" if ext is not None else "nothing_compiled") + else "compiled" if art is not None else "nothing_compiled") out.append(ModuleResult(m.path, m.name, ext, m.status, list(m.reasons), skipped, m.interface, - kind, items)) - return ProjectResult(out) + kind, items, c_file)) + return out + + +def _no_c_marker(name: str, out_dir: Path) -> Path: + marker = Path(out_dir) / (name.rpartition(".")[2] + _NO_EXTENSION) + marker.write_text("nothing compiled\n") + return marker + + +def compile_project( + sources: list[Path], + project_root: Path, + cache_dir: Path, + *, + python_include: Path | None = None, + ext_suffix: str | None = None, + runtime_dir: Path | None = None, + cost_rule: bool = True, +) -> ProjectResult: + def compile_c(c_source: str, name: str, out_dir: Path) -> Path: + if not c_source: + return _no_c_marker(name, out_dir) + return cbuild.build(c_source, name, out_dir, runtime_dir=runtime_dir, flags=cbuild.DEFAULT_FLAGS, + python_include=python_include, ext_suffix=ext_suffix) + + version = COMPILER_VERSION + if runtime_dir is not None: + version += ":" + _tree_hash(Path(runtime_dir)) + suffix = ext_suffix if ext_suffix is not None else sysconfig.get_config_var("EXT_SUFFIX") + include = cbuild._python_include(python_include) + flags = [*cbuild.DEFAULT_FLAGS, f"-I{include}"] + return ProjectResult(_project( + sources, project_root, cache_dir, cost_rule=cost_rule, compile_c=compile_c, + compiler_version=version, platform=f"{sysconfig.get_platform()} {suffix}", flags=flags, + c_mode=False)) + + +# The platform component of a generated-C cache key. The C does not depend on the host, the target, +# the C compiler or its flags (cgen reads none of them; the target's headers and the toolchain are +# the compile slot's inputs and its own cache key, pypackpack docs/design/compile-slot.md §9), so +# one constant stands for all of them and a `.c` built on one machine is reused on another. +GENERATED_C_PLATFORM = "generated-c" + + +def generate_project( + sources: list[Path], + project_root: Path, + cache_dir: Path, + *, + cost_rule: bool = True, +) -> ProjectResult: + """Source -> C for every module, cached (#113): the input of pypackpack's native compile slot, + which builds the extensions and caches C -> extension itself. Each compiled module gets exactly + one `.c` (`ModuleResult.c_source`, named after the module's last dotted part); a module with + nothing compiled gets none and ships as Python. `runtime_include` is the directory holding the + header-only `tp_runtime.h`, for the slot's include dirs. The key is the source, the dependency + interfaces and COMPILER_VERSION (which covers the runtime header); no toolchain flag enters it. + An unchanged module is reused with a byte-identical `.c`.""" + def write_c(c_source: str, name: str, out_dir: Path) -> Path: + if not c_source: + return _no_c_marker(name, out_dir) + c_file = Path(out_dir) / (name.rpartition(".")[2] + ".c") + c_file.write_text(c_source) + return c_file + + return ProjectResult(_project( + sources, project_root, cache_dir, cost_rule=cost_rule, compile_c=write_c, + compiler_version=COMPILER_VERSION, platform=GENERATED_C_PLATFORM, flags=[], c_mode=True), + runtime_include=cbuild.RUNTIME_DIR) diff --git a/python-multiplatform-ksp/src/test/python/test_generate_project.py b/python-multiplatform-ksp/src/test/python/test_generate_project.py new file mode 100644 index 000000000..f5ae50a94 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_generate_project.py @@ -0,0 +1,111 @@ +"""generate_project: TypedPython caches source -> C, the compile slot builds C -> extension (#113). + +Real Pyrefly; the only C build here is the one that proves the emitted `.c` is the same program +compile_project builds. +""" +import textwrap +from pathlib import Path + +import pytest + +from conftest import decorate_compiled +from typedpython import cbuild, frontend, pipeline + +B = decorate_compiled(textwrap.dedent("""\ + def twice(n: int) -> int: + return n * 2 + """)) +A = decorate_compiled(textwrap.dedent("""\ + import b + + + def inc(n: int) -> int: + return n + 1 + """)) +PLAIN = "def hello() -> str:\n return 'hi'\n" + + +@pytest.fixture +def project(tmp_path): + root = tmp_path / "proj" + (root / "pkg").mkdir(parents=True) + (root / "a.py").write_text(A) + (root / "b.py").write_text(B) + (root / "pkg" / "__init__.py").write_text("") + (root / "pkg" / "plain.py").write_text(PLAIN) + return root + + +@pytest.fixture +def counts(monkeypatch): + n = {"lower": 0, "build": 0} + real_lower = frontend.lower + + def lower(*a, **k): + n["lower"] += 1 + return real_lower(*a, **k) + + def build(*a, **k): + n["build"] += 1 + raise AssertionError("generate_project must not run the C compiler") + + monkeypatch.setattr(frontend, "lower", lower) + monkeypatch.setattr(cbuild, "build", build) + return n + + +def gen(project, cache): + return pipeline.generate_project( + [project / "a.py", project / "b.py", project / "pkg" / "plain.py"], project, cache) + + +def by_stem(result): + return {m.path.stem: m for m in result.modules} + + +def test_emits_one_c_per_compiled_module_and_the_runtime_dir(project, tmp_path, counts): + r = gen(project, tmp_path / "cache") + m = by_stem(r) + assert m["a"].c_source.name == "a.c" and m["b"].c_source.name == "b.c" + assert "PyInit_a" in m["a"].c_source.read_text() + assert m["a"].extension is None and m["a"].kind == "compiled" + assert m["plain"].c_source is None and m["plain"].kind == "not_marked" + assert (r.runtime_include / "tp_runtime.h").is_file() + assert counts == {"lower": 3, "build": 0} + + +def test_unchanged_module_hits_with_byte_identical_c(project, tmp_path, counts): + first = by_stem(gen(project, tmp_path / "cache")) + c_a = first["a"].c_source.read_bytes() + second = gen(project, tmp_path / "cache") + assert {m.path.stem: m.status for m in second.modules} == {"a": "reused", "b": "reused", "plain": "reused"} + assert by_stem(second)["a"].c_source.read_bytes() == c_a + assert counts["lower"] == 3 # nothing lowered again + + +def test_fresh_cache_gives_byte_identical_c(project, tmp_path): + one = by_stem(gen(project, tmp_path / "c1"))["a"].c_source.read_bytes() + two = by_stem(gen(project, tmp_path / "c2"))["a"].c_source.read_bytes() + assert one == two + + +def test_body_change_regenerates_only_that_module(project, tmp_path): + gen(project, tmp_path / "cache") + (project / "b.py").write_text(B.replace("n * 2", "n + n")) + st = {m.path.stem: m.status for m in gen(project, tmp_path / "cache").modules} + assert st == {"a": "reused", "b": "rebuilt", "plain": "reused"} + + +def test_key_ignores_toolchain(project, tmp_path, monkeypatch): + gen(project, tmp_path / "cache") + monkeypatch.setattr(cbuild, "DEFAULT_FLAGS", (*cbuild.DEFAULT_FLAGS, "-DDIFFERENT")) + monkeypatch.setattr(pipeline.sysconfig, "get_platform", lambda: "other-platform") + assert all(m.status == "reused" for m in gen(project, tmp_path / "cache").modules) + + +def test_emitted_c_is_what_compile_project_builds(project, tmp_path): + c_mod = by_stem(gen(project, tmp_path / "ccache"))["b"] + built = by_stem(pipeline.compile_project([project / "b.py"], project, tmp_path / "bcache"))["b"] + so = cbuild.build(c_mod.c_source.read_text(), "b", tmp_path / "slot") + assert cbuild.load("b", so).twice(21) == 42 + assert built.extension is not None and built.c_source is None From 831174c0a4b816d4b73445978f19812bb0c59613 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Thu, 5 Nov 2026 07:59:00 +0900 Subject: [PATCH 14/29] Fix: Disable Gradle toolchain auto-detect in native-image CI so Liberica NIK 25 wins (#251) --- .github/workflows/desktop.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index cad58b5fd..1799d3699 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -196,13 +196,15 @@ jobs: # Records Compose and skiko's configuration against the fork version this run uses, then builds # the image and runs it with JAVA_HOME and PYTHONHOME removed: 20 of 20 and no libjvm. - # `setup-graalvm` installs Liberica NIK under `GRAALVM_HOME` in the runner toolcache outside - # `/Users/runner/hostedtoolcache/Java_*`, so pass `org.gradle.java.installations.paths` - # explicitly instead of picking up the runner's pre-installed Temurin 25. + # `setup-graalvm` installs Liberica NIK under `GRAALVM_HOME` in the runner toolcache, whereas + # macOS runner images also pre-install Temurin 25 in `/Library/Java/JavaVirtualMachines`. + # Disable toolchain auto-detection and pass `org.gradle.java.installations.paths="$GRAALVM_HOME"` + # so Gradle's JavaLanguageVersion(25) launcher selects Liberica NIK 25 instead of Temurin 25. - name: Build and run the native image run: > ./gradlew :python-multiplatform-compose:fixtures:notebook-e2e:notebookE2eNativeAgent :python-multiplatform-compose:fixtures:notebook-e2e:notebookE2eNativeRun -PpythonxComposeWheel="$WHEEL_DIR" + -Porg.gradle.java.installations.auto-detect=false -Porg.gradle.java.installations.paths="$GRAALVM_HOME" --console=plain From 693252cf0764dbf5ab07983059f5e55248a9b84e Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Thu, 5 Nov 2026 08:10:49 +0900 Subject: [PATCH 15/29] Fix: Require BellSoft vendor for JDK 25 native-image toolchain (#251) --- .github/workflows/desktop.yml | 8 +++----- .../fixtures/notebook-e2e/build.gradle.kts | 5 ++++- sample/build.gradle.kts | 1 + 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index 1799d3699..6a80f31e0 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -196,15 +196,13 @@ jobs: # Records Compose and skiko's configuration against the fork version this run uses, then builds # the image and runs it with JAVA_HOME and PYTHONHOME removed: 20 of 20 and no libjvm. - # `setup-graalvm` installs Liberica NIK under `GRAALVM_HOME` in the runner toolcache, whereas - # macOS runner images also pre-install Temurin 25 in `/Library/Java/JavaVirtualMachines`. - # Disable toolchain auto-detection and pass `org.gradle.java.installations.paths="$GRAALVM_HOME"` - # so Gradle's JavaLanguageVersion(25) launcher selects Liberica NIK 25 instead of Temurin 25. + # `setup-graalvm` installs Liberica NIK under `GRAALVM_HOME` in the runner toolcache outside + # `/Users/runner/hostedtoolcache/Java_*`, so pass `org.gradle.java.installations.paths` + # explicitly so Gradle's BellSoft JDK 25 toolchain resolution finds it. - name: Build and run the native image run: > ./gradlew :python-multiplatform-compose:fixtures:notebook-e2e:notebookE2eNativeAgent :python-multiplatform-compose:fixtures:notebook-e2e:notebookE2eNativeRun -PpythonxComposeWheel="$WHEEL_DIR" - -Porg.gradle.java.installations.auto-detect=false -Porg.gradle.java.installations.paths="$GRAALVM_HOME" --console=plain diff --git a/python-multiplatform-compose/fixtures/notebook-e2e/build.gradle.kts b/python-multiplatform-compose/fixtures/notebook-e2e/build.gradle.kts index 8c4915a86..0c269b73a 100644 --- a/python-multiplatform-compose/fixtures/notebook-e2e/build.gradle.kts +++ b/python-multiplatform-compose/fixtures/notebook-e2e/build.gradle.kts @@ -280,7 +280,10 @@ val e2eMainClass = "fixture.notebook.NotebookE2eMainKt" val e2eNativeConfig = layout.projectDirectory.dir("src/desktopTest/native-image") val e2eNativeDir = layout.buildDirectory.dir("native/notebook-e2e") val e2eAppDir = layout.projectDirectory.dir("src/desktopTest/python") -val e2eJdk25 = javaToolchains.launcherFor { languageVersion.set(JavaLanguageVersion.of(25)) } +val e2eJdk25 = javaToolchains.launcherFor { + languageVersion.set(JavaLanguageVersion.of(25)) + vendor.set(JvmVendorSpec.BELLSOFT) +} fun JavaExec.notebookE2eOnTheJvm() { dependsOn(e2eTestCompilation.compileTaskProvider, installPythonxComposeWheel) diff --git a/sample/build.gradle.kts b/sample/build.gradle.kts index 2c9f5e32b..326e10cb7 100644 --- a/sample/build.gradle.kts +++ b/sample/build.gradle.kts @@ -407,6 +407,7 @@ tasks.matching { it.name == "run" }.configureEach { val nativeImageJavaLauncher = javaToolchains.launcherFor { languageVersion.set(JavaLanguageVersion.of(25)) + vendor.set(JvmVendorSpec.BELLSOFT) } val nativeImageOutputDir = layout.buildDirectory.dir("native/nativeCompile") From 986c736a5003c78024a5e8013b9c817b612b740b Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Thu, 5 Nov 2026 11:38:31 +0900 Subject: [PATCH 16/29] TypedPython: narrow GCC fastmath to reassociation and FMA contract (#368) --- docs/SPEC.md | 6 ++ .../src/main/python/typedpython/cgen.py | 6 +- .../src/test/python/test_pipeline.py | 83 +++++++++++++++++++ 3 files changed, 94 insertions(+), 1 deletion(-) diff --git a/docs/SPEC.md b/docs/SPEC.md index 2e65b6f45..14c5ece6f 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -748,6 +748,12 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. caller without fastmath): **this is the one path in TypedPython that is explicitly NOT CPython-identical** (a sum or fused multiply-add in a `@compiled(fastmath=True)` function may round differently from the interpreter so float reductions and FMA loops can vectorise). + The three compilers allow the same scope (#368): reassociation and FMA contraction only. Clang uses + `#pragma clang fp reassociate(on) contract(fast)`, MSVC `float_control(precise, off)` with + `fp_contract(on)`, GCC `optimize("associative-math,no-signed-zeros,no-trapping-math,fp-contract=fast")` + (the two extra options are what associative-math needs to take effect). None of them assumes finite + values, so a NaN check (`x != x`, `isnan`) in a fastmath function still works; GCC's `fast-math` is + not used (`test_pipeline.py`: the macro test, and the NaN-check test run under a real GCC on Linux CI). `Status: implemented` for desktop host builds (`test_pipeline.py`, `test_cgen*.py`, `test_recursion.py`, `test_eval_breaker.py`, `test_fast_entry.py`, `test_free_threaded.py`; free-threaded 3.15t, #159). Not yet: shipping compiled modules in an app through diff --git a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py index 940837ed9..295484182 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py @@ -758,7 +758,11 @@ def generate(module: ir.Module, source_path: Path, display_path: str | None = No w("#define TP_FASTMATH_BEGIN _Pragma(\"clang fp reassociate(on) contract(fast)\")") w("#define TP_FASTMATH_END _Pragma(\"clang fp reassociate(off) contract(off)\")") w("#elif defined(__GNUC__)") - w("#define TP_FASTMATH_BEGIN _Pragma(\"GCC push_options\") _Pragma(\"GCC optimize(\\\"fast-math\\\")\")") + # The same scope as clang and MSVC (#368): reassociation and FMA contraction only. + # associative-math takes effect only with no-signed-zeros and no-trapping-math; never + # fast-math, whose finite-math-only folds `x != x` and isnan away. + w("#define TP_FASTMATH_BEGIN _Pragma(\"GCC push_options\") _Pragma(\"GCC optimize(" + "\\\"associative-math,no-signed-zeros,no-trapping-math,fp-contract=fast\\\")\")") w("#define TP_FASTMATH_END _Pragma(\"GCC pop_options\")") w("#elif defined(_MSC_VER)") w("#define TP_FASTMATH_BEGIN __pragma(float_control(precise, off, push)) __pragma(fp_contract(on))") diff --git a/python-multiplatform-ksp/src/test/python/test_pipeline.py b/python-multiplatform-ksp/src/test/python/test_pipeline.py index 18355aec4..dcfb7d6da 100644 --- a/python-multiplatform-ksp/src/test/python/test_pipeline.py +++ b/python-multiplatform-ksp/src/test/python/test_pipeline.py @@ -238,3 +238,86 @@ def calls_fast_and_adds(a: float, b: float, c: float) -> float: got_fast = mod.fma_fast(a, a, c) assert isinstance(got_fast, float) + + +_NAN_SRC = """ +@compiled(fastmath=True) +def nan_count(xs: list[float]) -> int: + n = 0 + for i in range(len(xs)): + x = xs[i] + if x != x: + n += 1 + return n + +@compiled(fastmath=True) +def is_nan(x: float) -> bool: + return x != x +""" + + +def _real_gcc(): + """A GCC driver that is really GCC (macOS `gcc` is clang), or None.""" + import shutil + import subprocess + for name in ("gcc", "gcc-15", "gcc-14", "gcc-13"): + exe = shutil.which(name) + if exe: + out = subprocess.run([exe, "--version"], capture_output=True, text=True).stdout + if "clang" not in out.lower(): + return exe + return None + + +def test_gcc_fastmath_scope_is_reassociation_and_contract_only(write): + """#368: GCC's fastmath macro allows the same scope as clang and MSVC, reassociation and FMA + contraction (plus the no-signed-zeros / no-trapping-math that associative-math needs); never + `fast-math`, whose finite-math-only folds `x != x` to false.""" + from typedpython import cgen, frontend, verify + + path = write("fm_scope.py", _NAN_SRC) + proved, _ = verify.verify(frontend.lower(path)) + c_text = cgen.generate(proved, path) + gcc_line = next(l for l in c_text.splitlines() if "define TP_FASTMATH_BEGIN" in l and "GCC" in l) + assert "fast-math" not in gcc_line + for opt in ("associative-math", "no-signed-zeros", "no-trapping-math", "fp-contract=fast"): + assert opt in gcc_line, (opt, gcc_line) + assert "finite-math" not in gcc_line + + +def test_fastmath_keeps_nan_checks(write, tmp_path): + """#368: `x != x` inside a fastmath function still detects NaN (host compiler).""" + path = write("fm_nan.py", _NAN_SRC) + res = pipeline.compile_module(path, tmp_path / "out", cost_rule=False) + assert {f.name for f in res.module.functions} == {"nan_count", "is_nan"}, res.skipped + mod = pipeline.load(res) + nan = float("nan") + assert mod.is_nan(nan) is True and mod.is_nan(1.0) is False + assert mod.nan_count([1.0, nan, 2.0, nan]) == 2 + + +def test_fastmath_keeps_nan_checks_under_gcc(write, tmp_path, monkeypatch): + """#368: the same under a real GCC (Linux CI); skipped where no GCC is installed.""" + import sys + from typedpython import cbuild + gcc = _real_gcc() + if gcc is None or sys.platform == "win32": + pytest.skip("no real GCC on this host") + monkeypatch.setattr(cbuild, "compiler", lambda: [gcc]) + path = write("fm_nan_gcc.py", _NAN_SRC) + # GCC only warns on an optimize option it does not know; make that an error so a dropped + # option cannot pass silently. + from typedpython import cgen, frontend, verify + proved, _ = verify.verify(frontend.lower(path)) + c_file = tmp_path / "fm_nan_gcc.c" + c_file.write_text(cgen.generate(proved, path)) + chk = subprocess.run([gcc, "-fsyntax-only", "-Werror=pragmas", "-Werror=attributes", + f"-I{cbuild._python_include(None)}", f"-I{cbuild.RUNTIME_DIR}", str(c_file)], + capture_output=True, text=True) + assert chk.returncode == 0, chk.stderr + res = pipeline.compile_module(path, tmp_path / "out", cost_rule=False) + assert {f.name for f in res.module.functions} == {"nan_count", "is_nan"}, res.skipped + mod = pipeline.load(res) + nan = float("nan") + assert mod.is_nan(nan) is True + assert mod.nan_count([1.0, nan, 2.0, nan]) == 2 From decc90e735808f2e3ae49e33db57f814ba008e49 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Thu, 5 Nov 2026 11:54:20 +0900 Subject: [PATCH 17/29] Feat: add macosArm64 and linuxX64 Kotlin/Native desktop targets (#191) --- .github/workflows/desktop.yml | 25 +++++ docs/SPEC.md | 3 +- python-multiplatform/build.gradle.kts | 58 +++++++++- .../env/PackagedPythonHome.desktopNative.kt | 100 ++++++++++++++++++ .../env/PythonPayload.desktopNative.kt | 24 +++++ .../env/DesktopNativePythonHomeTest.kt | 93 ++++++++++++++++ .../python/multiplatform/Platform.linuxX64.kt | 39 +++++++ .../python/multiplatform/ThreadId.linuxX64.kt | 9 ++ .../env/ExecutableDir.linuxX64.kt | 21 ++++ .../multiplatform/Platform.macosArm64.kt | 49 +++++++++ .../multiplatform/ThreadId.macosArm64.kt | 8 ++ .../env/ExecutableDir.macosArm64.kt | 27 +++++ 12 files changed, 454 insertions(+), 2 deletions(-) create mode 100644 python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/env/PackagedPythonHome.desktopNative.kt create mode 100644 python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/env/PythonPayload.desktopNative.kt create mode 100644 python-multiplatform/src/desktopNativeTest/kotlin/python/multiplatform/env/DesktopNativePythonHomeTest.kt create mode 100644 python-multiplatform/src/linuxX64Main/kotlin/python/multiplatform/Platform.linuxX64.kt create mode 100644 python-multiplatform/src/linuxX64Main/kotlin/python/multiplatform/ThreadId.linuxX64.kt create mode 100644 python-multiplatform/src/linuxX64Main/kotlin/python/multiplatform/env/ExecutableDir.linuxX64.kt create mode 100644 python-multiplatform/src/macosArm64Main/kotlin/python/multiplatform/Platform.macosArm64.kt create mode 100644 python-multiplatform/src/macosArm64Main/kotlin/python/multiplatform/ThreadId.macosArm64.kt create mode 100644 python-multiplatform/src/macosArm64Main/kotlin/python/multiplatform/env/ExecutableDir.macosArm64.kt diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index 6a80f31e0..7d6003ba4 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -51,6 +51,31 @@ jobs: # Using --console=plain so logs are readable in GitHub Actions run: ./gradlew :python-multiplatform:desktopTest --console=plain + desktop-native-linux: + name: Desktop Kotlin/Native Tests (Linux x86_64) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Java + uses: actions/setup-java@v4 + with: + java-version: '21' + distribution: 'zulu' + cache: 'gradle' + + - name: Cache Konan + uses: actions/cache@v4 + with: + path: ~/.konan + key: ${{ runner.os }}-konan-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-konan- + + - name: Run Linux x86_64 Kotlin/Native Tests + run: ./gradlew :python-multiplatform:linuxX64Test --rerun --console=plain + # The dev agent's protocol on the other desktop OSes (#238). Its socket options differ per platform # (SO_EXCLUSIVEADDRUSE on Windows, SO_REUSEADDR on POSIX), and the port-takeover and restart tests are # what check them. Stdlib only: no JDK, no Gradle. diff --git a/docs/SPEC.md b/docs/SPEC.md index 2e65b6f45..13e3a29f4 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -26,12 +26,13 @@ fail, then implement. | Platform | Kotlin target(s) | Downcall mechanism | Upcall entry | Test path | |---|---|---|---|---| | Desktop JVM (macOS; Linux/Windows wired, unverified) | `jvm("desktop")` | Panama FFM, `invokeExact` | FFM upcall stub | `desktopTest → jvmTest → commonTest` | +| Desktop native (macOS Arm64, Linux x86_64) | `macosArm64`, `linuxX64` | cinterop | `@CName` symbol | `{macosArm64Test, linuxX64Test} → desktopNativeTest → nativeTest → commonTest` | | Android (ART) | `androidTarget` | JNI via `RegisterNatives` | JNI upcall | `androidInstrumentedTest → jvmTest → commonTest` (device) | | iOS | `iosArm64`, `iosSimulatorArm64`, `iosX64` | cinterop | `@CName` symbol | `iosSimulatorArm64Test → nativeTest → commonTest` | | Android native | `androidNativeArm64`, `androidNativeX64` | cinterop | `@CName` symbol | `androidNativeArm64Test → artTest → nativeTest → commonTest` (device, via adb) | | Web (experimental) | `wasmJs` (Node, browser) | JS / Emscripten CPython | `@WasmExport` | `wasmJsTest` (Node suite; browser subset) | -- **P-1** The library compiles for every target above. `Status: implemented` for desktop, Android, +- **P-1** The library compiles for every target above. `Status: implemented` for desktop, `macosArm64`, `linuxX64`, Android, iosSimulatorArm64, androidNativeArm64 and wasmJs (each has a test source set). `partial` for `iosArm64`, `iosX64`, `androidNativeX64` (compile-wired, no tests run there). - **P-2** Linux and Windows desktop: the build downloads and links the right CPython archive. diff --git a/python-multiplatform/build.gradle.kts b/python-multiplatform/build.gradle.kts index 46e46c00b..bb3ac4824 100644 --- a/python-multiplatform/build.gradle.kts +++ b/python-multiplatform/build.gradle.kts @@ -89,6 +89,7 @@ buildkonfig { } val libVersion = pythonVersion.split('.').subList(0, 2).joinToString(".") +val abiSuffix = if (pythonFreeThreaded) "t" else "" /** * The `ftp/python//` directory holding a release, which drops any pre-release suffix: @@ -1245,7 +1246,8 @@ kotlin { */ listOf( iosArm64(), iosSimulatorArm64(), iosX64(), - androidNativeArm64(), androidNativeX64() + androidNativeArm64(), androidNativeX64(), + macosArm64(), linuxX64() ).forEach { nativeTarget -> nativeTarget.apply { val targetABI = when(konanTarget) { @@ -1254,16 +1256,20 @@ kotlin { IOS_ARM64 -> "ios-arm64" IOS_X64 -> "ios-arm64_x86_64-simulator" IOS_SIMULATOR_ARM64 -> "ios-arm64_x86_64-simulator" + MACOS_ARM64 -> "macos-aarch64" + LINUX_X64 -> "linux-x86_64" else -> throw RuntimeException("Unsupported ABI: $konanTarget") } val targetExtractDir = when(konanTarget.family) { Family.ANDROID -> "$extractedDir/android-${if (targetABI == "arm64-v8a") "aarch64" else "x86_64"}/prefix" Family.IOS -> "$extractedDir/ios/Python.xcframework" + Family.OSX, Family.LINUX -> "$extractedDir/$targetABI$desktopFlavourSuffix/python" else -> throw RuntimeException("Unsupported target family: ${konanTarget.family}") } val targetIncludePath = when(konanTarget.family) { Family.ANDROID -> "$targetExtractDir/include/python$libVersion" Family.IOS -> "$targetExtractDir/$targetABI/Python.framework/Headers" + Family.OSX, Family.LINUX -> "$targetExtractDir/include/python$libVersion$abiSuffix" else -> throw RuntimeException("Unsupported target family: ${konanTarget.family}") } @@ -1331,6 +1337,28 @@ kotlin { "-framework", "Python", "-F$targetExtractDir/$targetABI", "-rpath", "$targetExtractDir/$targetABI" )) + } else if (konanTarget.family == Family.OSX) { + getTest(NativeBuildType.DEBUG).linkerOpts.addAll(listOf( + "-L$targetExtractDir/lib", + "-lpython$libVersion$abiSuffix", + "-Wl,-rpath,$targetExtractDir/lib" + )) + } else if (konanTarget.family == Family.LINUX) { + // python-build-standalone ships libpythonX.Y.so as a symlink to libpythonX.Y.so.1.0, + // which Gradle's tarTree extracts as a 0-byte file; stageDesktopLibraries resolves + // it into a real libpythonX.Y.so under build/desktop-libs/lib/linux-x86_64. + val stagedLinuxLibDir = layout.buildDirectory.dir("desktop-libs/lib/linux-x86_64").get().asFile.absolutePath + getTest(NativeBuildType.DEBUG).apply { + linkerOpts.addAll(listOf( + "-L$stagedLinuxLibDir", + "-L$targetExtractDir/lib", + "-lpython$libVersion$abiSuffix", + "-Wl,-rpath,$targetExtractDir/lib" + )) + linkTaskProvider.configure { + dependsOn("stageDesktopLibraries") + } + } } } } @@ -1426,6 +1454,15 @@ kotlin { androidNativeX64Main.dependsOn(artMain) androidNativeArm64Main.dependsOn(artMain) + val desktopNativeMain by creating { + kotlin.srcDir("src/desktopNativeMain/kotlin") + } + val macosArm64Main by getting + val linuxX64Main by getting + desktopNativeMain.dependsOn(nativeMain) + macosArm64Main.dependsOn(desktopNativeMain) + linuxX64Main.dependsOn(desktopNativeMain) + // `src/nativeTest` existed as a directory long before this line, and nothing pointed at // it -- so its `GCLeakTest.native.kt` was dead source, and the same file had been copied // byte-for-byte into all five target test source sets to compensate. This is the mirror of @@ -1439,6 +1476,13 @@ kotlin { "androidNativeX64Test", "androidNativeArm64Test", ).forEach { getByName(it).dependsOn(nativeTest) } + val desktopNativeTest by creating { + kotlin.srcDir("src/desktopNativeTest/kotlin") + } + desktopNativeTest.dependsOn(nativeTest) + listOf("macosArm64Test", "linuxX64Test") + .forEach { getByName(it).dependsOn(desktopNativeTest) } + // ...and `artTest` completes the mirror for the other half: things true of androidNative // but not of iOS. Without it such a test has nowhere to live except duplicated into both // androidNative target source sets, which is the arrangement the comment above describes @@ -1452,6 +1496,18 @@ kotlin { } } +tasks.named("macosArm64Test") { + dependsOn("downloadPython_macos_aarch64") + val pythonHome = file("$extractedDir/macos-aarch64$desktopFlavourSuffix/python").absolutePath + environment("PYTHONHOME", pythonHome) +} + +tasks.named("linuxX64Test") { + dependsOn("downloadPython_linux_x86_64") + val pythonHome = file("$extractedDir/linux-x86_64$desktopFlavourSuffix/python").absolutePath + environment("PYTHONHOME", pythonHome) +} + /** * The iOS `Python.framework` ships only the interpreter binary and headers, it carries no * standard library. `Py_Initialize()` therefore aborts the process with diff --git a/python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/env/PackagedPythonHome.desktopNative.kt b/python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/env/PackagedPythonHome.desktopNative.kt new file mode 100644 index 000000000..a338dc59e --- /dev/null +++ b/python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/env/PackagedPythonHome.desktopNative.kt @@ -0,0 +1,100 @@ +package python.multiplatform.env + +import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.alloc +import kotlinx.cinterop.memScoped +import kotlinx.cinterop.ptr +import platform.posix.S_IFDIR +import platform.posix.S_IFMT +import platform.posix.stat + +/** + * The directory containing the running executable, or null when the OS cannot report it. + */ +internal expect fun executableDirectory(): String? + +/** + * Where a desktop Kotlin/Native application (`macosArm64`, `linuxX64`) finds its packaged CPython + * prefix (SPEC L-9 / P-1, #191). + * + * A desktop native binary has no JVM, no system properties and no launcher to set `PYTHONHOME`: + * like a GraalVM native image (#246), its prefix ships beside the executable under + * [DIRECTORY_NAME] (or, inside a macOS `.app` bundle, under `../Resources/`[DIRECTORY_NAME]). + */ +internal object DesktopNativePythonHome { + + /** Matches `PackagedPythonHome.DIRECTORY_NAME` (`jvmMain`) and `IosPythonHome.DIRECTORY_NAME` (`iosMain`). */ + const val DIRECTORY_NAME = "python-multiplatform-home" + + /** A resolved prefix and a human-readable account of where it came from, for error messages. */ + data class Resolved(val path: String, val source: String) + + /** [resolve] against this process's environment and executable directory. */ + fun resolve(): Resolved? = resolve( + environmentHome = readEnvVar("PYTHONHOME"), + executableDir = executableDirectory(), + isDirectory = ::isDirectory, + ) + + /** + * The packaged prefix, or null when there is none to apply. + * + * 1. A non-blank `PYTHONHOME` in the environment wins, and the answer is null: CPython reads it + * itself and [PythonHomeCheck] already checked it. This keeps the Gradle host test tasks + * (`macosArm64Test`, `linuxX64Test`) and an explicit caller environment working unchanged. + * 2. `/`[DIRECTORY_NAME], when that directory exists. + * 3. `/../Resources/`[DIRECTORY_NAME], when that directory exists (a macOS + * `.app` bundle places the binary in `Contents/MacOS/` and resources in `Contents/Resources/`). + */ + fun resolve( + environmentHome: String?, + executableDir: String?, + isDirectory: (String) -> Boolean, + ): Resolved? { + if (!environmentHome.isNullOrBlank()) return null + val dir = executableDir?.takeIf { it.isNotBlank() }?.trimEnd('/') ?: return null + val beside = "$dir/$DIRECTORY_NAME" + if (isDirectory(beside)) { + return Resolved(beside, "$DIRECTORY_NAME beside the executable ($dir)") + } + val parent = dir.substringBeforeLast('/', missingDelimiterValue = "") + if (parent.isNotEmpty()) { + val resources = "$parent/Resources/$DIRECTORY_NAME" + if (isDirectory(resources)) { + return Resolved(resources, "$DIRECTORY_NAME in the app bundle's Resources directory ($parent/Resources)") + } + } + return null + } + + /** `stat(2)`'s answer: a directory exists at [path] (a regular file there is not one). */ + @OptIn(ExperimentalForeignApi::class) + internal fun isDirectory(path: String): Boolean = memScoped { + val info = alloc() + if (stat(path, info.ptr) != 0) return false + (info.st_mode.toInt() and S_IFMT) == S_IFDIR + } +} + +/** + * Hands a packaged prefix to CPython with `Py_SetPythonHome` before `Py_Initialize()`. + * + * `Py_DecodeLocale` is documented by CPython as safe to call before initialisation and produces + * storage that lives for the duration of the process, as `Py_SetPythonHome` requires. + */ +@OptIn(ExperimentalForeignApi::class) +@Suppress("DEPRECATION") +internal actual fun applyPackagedPythonHome() { + val home = DesktopNativePythonHome.resolve() ?: return + try { + PythonHomeCheck.verifyOrThrow(home.path) + } catch (e: IllegalStateException) { + throw IllegalStateException( + "The Python home this application was packaged with (${home.source}) is unusable: ${e.message}", + e, + ) + } + val wide = python.native.ffi.bindings.Py_DecodeLocale(home.path, null) + ?: throw IllegalStateException("Py_DecodeLocale could not decode the packaged Python home '${home.path}'.") + python.native.ffi.bindings.Py_SetPythonHome(wide) +} diff --git a/python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/env/PythonPayload.desktopNative.kt b/python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/env/PythonPayload.desktopNative.kt new file mode 100644 index 000000000..cb3c3d1e0 --- /dev/null +++ b/python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/env/PythonPayload.desktopNative.kt @@ -0,0 +1,24 @@ +package python.multiplatform.env + +/** + * Desktop Kotlin/Native's answer to "where did the payload land": `python/` beside the executable + * or, in a macOS `.app` bundle, under `../Resources/python`. + */ +internal actual fun discoverStagedPayloadRoots(): List = + discoverStagedPayloadRoots(executableDirectory(), ::pathIsAccessible) + +internal fun discoverStagedPayloadRoots( + executableDir: String?, + isAccessible: (String) -> Boolean, +): List { + val dir = executableDir?.takeIf { it.isNotBlank() }?.trimEnd('/') ?: return emptyList() + val roots = mutableListOf() + val beside = "$dir/${PythonPayload.PAYLOAD_ROOT}" + if (isAccessible(beside)) roots += beside + val parent = dir.substringBeforeLast('/', missingDelimiterValue = "") + if (parent.isNotEmpty()) { + val resources = "$parent/Resources/${PythonPayload.PAYLOAD_ROOT}" + if (resources != beside && isAccessible(resources)) roots += resources + } + return roots +} diff --git a/python-multiplatform/src/desktopNativeTest/kotlin/python/multiplatform/env/DesktopNativePythonHomeTest.kt b/python-multiplatform/src/desktopNativeTest/kotlin/python/multiplatform/env/DesktopNativePythonHomeTest.kt new file mode 100644 index 000000000..28fb45434 --- /dev/null +++ b/python-multiplatform/src/desktopNativeTest/kotlin/python/multiplatform/env/DesktopNativePythonHomeTest.kt @@ -0,0 +1,93 @@ +package python.multiplatform.env + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue + +/** + * Where a desktop Kotlin/Native application (`macosArm64`, `linuxX64`) resolves its packaged + * CPython prefix and payload from. + */ +class DesktopNativePythonHomeTest { + + private val besideBinary = "/apps/demo/python-multiplatform-home" + private val inAppBundle = "/apps/Demo.app/Contents/Resources/python-multiplatform-home" + + @Test + fun theDirectoryNameMatchesThePluginAndDesktopAndIos() { + assertEquals("python-multiplatform-home", DesktopNativePythonHome.DIRECTORY_NAME) + } + + @Test + fun anEnvironmentPythonHomeWinsAndNothingIsApplied() { + assertNull( + DesktopNativePythonHome.resolve( + environmentHome = "/from/env", + executableDir = "/apps/demo", + isDirectory = { true }, + ) + ) + } + + @Test + fun thePrefixBesideTheExecutableIsUsedWhenPresent() { + for (environment in listOf(null, "", " ")) { + val resolved = DesktopNativePythonHome.resolve( + environmentHome = environment, + executableDir = "/apps/demo", + isDirectory = { it == besideBinary }, + ) + assertEquals(besideBinary, resolved?.path, "PYTHONHOME='$environment'") + assertTrue("beside the executable" in resolved!!.source, resolved.source) + } + } + + @Test + fun anAppBundleResourceDirectoryIsCheckedWhenNothingSitsBesideTheBinary() { + val resolved = DesktopNativePythonHome.resolve( + environmentHome = null, + executableDir = "/apps/Demo.app/Contents/MacOS", + isDirectory = { it == inAppBundle }, + ) + assertEquals(inAppBundle, resolved?.path) + assertTrue("Resources" in resolved!!.source, resolved.source) + } + + @Test + fun anUnpackagedExecutableBehavesAsBefore() { + assertNull(DesktopNativePythonHome.resolve(null, "/apps/demo") { false }) + assertNull(DesktopNativePythonHome.resolve(null, null) { true }) + assertNull(DesktopNativePythonHome.resolve(null, "") { true }) + } + + @Test + fun theHostTestTaskPassesPythonHomeAndExecutableDirectoryResolves() { + val home = assertNotNull(readEnvVar("PYTHONHOME"), "the host test task must pass PYTHONHOME") + assertNull(DesktopNativePythonHome.resolve()) + val exeDir = assertNotNull(executableDirectory(), "executableDirectory() returned null") + assertTrue(DesktopNativePythonHome.isDirectory(exeDir), exeDir) + assertTrue(DesktopNativePythonHome.isDirectory(home), home) + assertFalse(DesktopNativePythonHome.isDirectory("$home/does-not-exist")) + } + + @Test + fun payloadDiscoveryFindsBesideExecutableAndBundleResources() { + assertEquals( + listOf("/apps/demo/python"), + discoverStagedPayloadRoots("/apps/demo") { it == "/apps/demo/python" }, + ) + assertEquals( + listOf("/apps/Demo.app/Contents/Resources/python"), + discoverStagedPayloadRoots("/apps/Demo.app/Contents/MacOS") { + it == "/apps/Demo.app/Contents/Resources/python" + }, + ) + assertEquals( + emptyList(), + discoverStagedPayloadRoots("/apps/demo") { false }, + ) + } +} diff --git a/python-multiplatform/src/linuxX64Main/kotlin/python/multiplatform/Platform.linuxX64.kt b/python-multiplatform/src/linuxX64Main/kotlin/python/multiplatform/Platform.linuxX64.kt new file mode 100644 index 000000000..dfa1251af --- /dev/null +++ b/python-multiplatform/src/linuxX64Main/kotlin/python/multiplatform/Platform.linuxX64.kt @@ -0,0 +1,39 @@ +package python.multiplatform + +import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.alloc +import kotlinx.cinterop.memScoped +import kotlinx.cinterop.ptr +import kotlinx.cinterop.toKString +import platform.posix.uname +import platform.posix.utsname + +/** + * The Linux Kotlin/Native (`linuxX64`) view of the running host. + */ +object LinuxNativePlatform : Platform { + override val os = OSType.Linux + override val arch: String = machineName() + override val version: String = kernelRelease() + override val versionCode: Int? = null + override val platformType = PlatformType.Native + override val platformVersion: String? = null + + override fun toString(): String = name +} + +@OptIn(ExperimentalForeignApi::class) +private fun machineName(): String = memScoped { + val info = alloc() + if (uname(info.ptr) != 0) return "unknown" + info.machine.toKString().ifEmpty { "unknown" } +} + +@OptIn(ExperimentalForeignApi::class) +private fun kernelRelease(): String = memScoped { + val info = alloc() + if (uname(info.ptr) != 0) return "unknown" + info.release.toKString().ifEmpty { "unknown" } +} + +actual val currentPlatform: Platform = LinuxNativePlatform diff --git a/python-multiplatform/src/linuxX64Main/kotlin/python/multiplatform/ThreadId.linuxX64.kt b/python-multiplatform/src/linuxX64Main/kotlin/python/multiplatform/ThreadId.linuxX64.kt new file mode 100644 index 000000000..94cbe50da --- /dev/null +++ b/python-multiplatform/src/linuxX64Main/kotlin/python/multiplatform/ThreadId.linuxX64.kt @@ -0,0 +1,9 @@ +package python.multiplatform + +import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.convert + +/** glibc's `pthread_t` is a plain unsigned integral typedef (`unsigned long`), not a pointer. */ +@OptIn(ExperimentalForeignApi::class) +internal actual fun currentThreadId(): Long = + platform.posix.pthread_self().convert() diff --git a/python-multiplatform/src/linuxX64Main/kotlin/python/multiplatform/env/ExecutableDir.linuxX64.kt b/python-multiplatform/src/linuxX64Main/kotlin/python/multiplatform/env/ExecutableDir.linuxX64.kt new file mode 100644 index 000000000..72e10a9ea --- /dev/null +++ b/python-multiplatform/src/linuxX64Main/kotlin/python/multiplatform/env/ExecutableDir.linuxX64.kt @@ -0,0 +1,21 @@ +package python.multiplatform.env + +import kotlinx.cinterop.ByteVar +import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.allocArray +import kotlinx.cinterop.convert +import kotlinx.cinterop.memScoped +import kotlinx.cinterop.set +import kotlinx.cinterop.toKString +import platform.posix.PATH_MAX +import platform.posix.readlink + +@OptIn(ExperimentalForeignApi::class) +internal actual fun executableDirectory(): String? = memScoped { + val buf = allocArray(PATH_MAX) + val len = readlink("/proc/self/exe", buf, (PATH_MAX - 1).convert()) + if (len <= 0) return null + buf[len.toInt()] = 0 + val path = buf.toKString() + path.substringBeforeLast('/', missingDelimiterValue = "").takeIf { it.isNotEmpty() } +} diff --git a/python-multiplatform/src/macosArm64Main/kotlin/python/multiplatform/Platform.macosArm64.kt b/python-multiplatform/src/macosArm64Main/kotlin/python/multiplatform/Platform.macosArm64.kt new file mode 100644 index 000000000..6be1e9ab1 --- /dev/null +++ b/python-multiplatform/src/macosArm64Main/kotlin/python/multiplatform/Platform.macosArm64.kt @@ -0,0 +1,49 @@ +package python.multiplatform + +import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.alloc +import kotlinx.cinterop.memScoped +import kotlinx.cinterop.ptr +import kotlinx.cinterop.toKString +import kotlinx.cinterop.useContents +import platform.Foundation.NSProcessInfo +import platform.posix.uname +import platform.posix.utsname + +/** + * The macOS Kotlin/Native (`macosArm64`) view of the running host. + */ +object MacosNativePlatform : Platform { + override val os = OSType.MacOS + override val arch: String = getDeviceArch() + override val version: String = getOSVersionString() + override val versionCode: Int = getOSVersionCode() + override val versionText = "SDK $versionCode" + override val platformType = PlatformType.Native + override val platformVersion: String? = null + + override fun toString(): String = name +} + +@OptIn(ExperimentalForeignApi::class) +private fun getDeviceArch(): String = memScoped { + val systemInfo = alloc() + if (uname(systemInfo.ptr) != 0) { + throw RuntimeException("Failed to get device arch info") + } + systemInfo.machine.toKString() +} + +@OptIn(ExperimentalForeignApi::class) +private fun getOSVersionString(): String = + NSProcessInfo.processInfo.operatingSystemVersion().useContents { + "$majorVersion.$minorVersion.$patchVersion" + } + +@OptIn(ExperimentalForeignApi::class) +private fun getOSVersionCode(): Int = + NSProcessInfo.processInfo.operatingSystemVersion().useContents { + majorVersion.toInt() * 10_000 + minorVersion.toInt() * 100 + patchVersion.toInt() + } + +actual val currentPlatform: Platform = MacosNativePlatform diff --git a/python-multiplatform/src/macosArm64Main/kotlin/python/multiplatform/ThreadId.macosArm64.kt b/python-multiplatform/src/macosArm64Main/kotlin/python/multiplatform/ThreadId.macosArm64.kt new file mode 100644 index 000000000..d19062749 --- /dev/null +++ b/python-multiplatform/src/macosArm64Main/kotlin/python/multiplatform/ThreadId.macosArm64.kt @@ -0,0 +1,8 @@ +package python.multiplatform + +import kotlinx.cinterop.ExperimentalForeignApi + +/** Darwin's `pthread_self()` returns a pointer; its bit pattern is the thread identity. */ +@OptIn(ExperimentalForeignApi::class) +internal actual fun currentThreadId(): Long = + platform.posix.pthread_self()?.rawValue?.toLong() ?: 0L diff --git a/python-multiplatform/src/macosArm64Main/kotlin/python/multiplatform/env/ExecutableDir.macosArm64.kt b/python-multiplatform/src/macosArm64Main/kotlin/python/multiplatform/env/ExecutableDir.macosArm64.kt new file mode 100644 index 000000000..38fb7176b --- /dev/null +++ b/python-multiplatform/src/macosArm64Main/kotlin/python/multiplatform/env/ExecutableDir.macosArm64.kt @@ -0,0 +1,27 @@ +package python.multiplatform.env + +import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.free +import kotlinx.cinterop.nativeHeap +import kotlinx.cinterop.toKString +import platform.Foundation.NSBundle +import platform.Foundation.NSProcessInfo +import platform.posix.realpath + +@OptIn(ExperimentalForeignApi::class) +internal actual fun executableDirectory(): String? { + val raw = NSBundle.mainBundle.executablePath + ?: (NSProcessInfo.processInfo.arguments.firstOrNull() as? String) + ?: return null + val resolved = realpath(raw, null) + val path = if (resolved != null) { + try { + resolved.toKString() + } finally { + nativeHeap.free(resolved) + } + } else { + raw + } + return path.substringBeforeLast('/', missingDelimiterValue = "").takeIf { it.isNotEmpty() } +} From 5d93fe927e21ce3648a5ecc3954542d0e83f4890 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:21:07 +0900 Subject: [PATCH 18/29] TypedPython test: ignore a probe that succeeds only after the call returned The macOS run of develop c50b53d failed in test_other_thread_release_and_append_get_bufferror_during_a_memoryview_call with 113 BufferErrors followed by one success: the compiled call returned between the worker's done check and its next probe. A success now counts only when the call keeps running for 0.05 s after it. --- .../src/test/python/test_buffer_memoryview.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python-multiplatform-ksp/src/test/python/test_buffer_memoryview.py b/python-multiplatform-ksp/src/test/python/test_buffer_memoryview.py index 8a616abc8..e622536b9 100644 --- a/python-multiplatform-ksp/src/test/python/test_buffer_memoryview.py +++ b/python-multiplatform-ksp/src/test/python/test_buffer_memoryview.py @@ -498,7 +498,12 @@ def worker(): except Exception as e: # noqa: BLE001 seen[name][1].append(repr(e)) else: - seen[name][1].append("succeeded") + # The call can return between the `done` check above and this probe; + # the buffer is then legitimately free. A probe that succeeds while the + # buffer is really unheld does so ~0.05 s into a ~0.6 s call, so `done` + # stays unset for far longer than this grace period. + if not done.wait(0.05): + seen[name][1].append("succeeded") return time.sleep(0.001) From 8ccfda4ba2be9c66024fbb8bf17014ae1635aca6 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:39:36 +0900 Subject: [PATCH 19/29] TypedPython: fill a fresh New's slots by direct release stores (#125) tp_new_fixed no longer goes through PyMember_SetOne for each slot of the object it has just allocated. The store is the one SetOne makes for a Py_T_OBJECT_EX slot (release store of a new reference) without the per-object critical section and the release of the old value, which is NULL after tp_alloc. Nothing between tp_alloc and the last store reaches a safepoint, so no stop-the-world can hand the object to another thread before it is filled. The design note also records why a freelist and later GC tracking were not done. --- docs/design/typedpython.md | 28 +++++++++++++++++++ .../main/python/typedpython/runtime/API.md | 3 +- .../python/typedpython/runtime/tp_runtime.h | 22 +++++++++++++-- .../src/test/python/test_cgen_classes.py | 14 ++++++++++ 4 files changed, 64 insertions(+), 3 deletions(-) diff --git a/docs/design/typedpython.md b/docs/design/typedpython.md index eaf3a02d2..31e8a0f64 100644 --- a/docs/design/typedpython.md +++ b/docs/design/typedpython.md @@ -221,6 +221,34 @@ GC 추적), refcount 증감, 해제(`subtype_dealloc`)를 그대로 치르기 더 가려면 객체를 아예 만들지 않거나(가상 객체), 증감을 빼거나(refcount 생략), 한꺼번에 할당·해제해야 한다(영역 할당). 셋 다 `id()`, `__del__` 시점, `gc` 관측과 부딪칠 수 있다. +**실제 객체(N-11)의 생성 비용 (#125, 2026-11-05).** 프로파일(`benchmarks/typedpython/results/2026-10-25-binary-trees-alloc-profile.md`) +가운데 관측을 바꾸지 않고 줄일 수 있는 것은 다음과 같이 처리했다. +- 호출마다 스레드 상태를 한 번만 읽는다(#273). +- `tp_new_fixed` 는 새 객체의 슬롯을 `PyMember_SetOne` 대신 release store 로 직접 채운다. `SetOne` 이 + `Py_T_OBJECT_EX` 슬롯에 하는 일(객체 critical section 안에서 `Py_XNewRef(v)` 의 release store, 그 뒤 옛 값 + `Py_XDECREF`)에서 critical section 과 옛 값 해제만 뺀 것이다. 옛 값은 `tp_alloc` 이 0 으로 채웠으므로 NULL 이다. + critical section 을 빼도 되는 근거: `tp_alloc` 부터 마지막 슬롯 저장까지 스레드 상태를 놓는 호출도, eval + breaker 검사도 없어서 safepoint 가 없다. 그래서 stop-the-world 가 필요한 `gc.get_objects()` / + `gc.get_referrers()` 는 채우기가 끝나기 전에 이 객체를 다른 스레드에 줄 수 없다. 오히려 기존 critical + section 은 경합 시 스레드 상태를 놓을 수 있어서, 반쯤 채운 객체가 보이는 실행이 가능했다(그것도 인터프리터의 + `__init__` 실행과 같은 상태라 유효했다). 슬롯의 종류·쓰기 가능·오프셋 범위는 `tp_class_capture` 가 init 에서 + 확인한다. 참조 수: 슬롯은 `Py_NewRef` 로 자기 참조를 갖는다(이것을 빼는 변이는 `test_cgen_classes.py` 에서 + segfault 로 잡힌다). 측정(참고용, load 24~27): binary-trees `check(make(18))` 의 CPU 시간 중앙값이 develop + 대비 1.08배 빠르다. 같은 부하에서 벽시계 시간은 한 회차 안에서도 2~22초로 흔들려 쓸 수 없었다. + +하지 않은 것과 이유: +- **타입별 freelist.** 해제된 객체의 메모리를 붙잡으려면 클래스의 `tp_dealloc` 를 바꿔야 한다. 이것은 + 2026-10-25 결정(#125 댓글)에서 미뤘다: 클래스가 뒤에 `__del__`, `__weakref__`, 서브클래스를 얻으면 바뀐 dealloc 이 + 관측된다. free-threaded 빌드에서는 해제된 GC 객체의 메모리를 다른 스레드가 lock-free 로 읽을 수 있어서 + (`_Py_TryIncref` 계열) CPython 은 mimalloc 의 지연 해제(QSBR)에 기댄다. 확장이 따로 붙잡는 freelist 는 이 보장 + 밖에 있으므로, 이 결정을 다시 열 때 그 근거도 함께 써야 한다. +- **GC 추적을 늦추기/생략.** 아직 사이클을 만들 수 없는 노드(자식이 `None`)를 추적하지 않는 방식은 + `gc.is_tracked`, `gc.get_objects()` 로 관측된다. 그리고 인터프리터가 나중에 슬롯에 자기 자신을 넣으면 + (`PyMember_SetOne` 은 추적을 다시 켜지 않는다) 사이클이 수거되지 않는다. 컬렉션 횟수를 줄이는 방식(할당 계수 + 묶기, 임계값 조정)은 컬렉션 시점을 바꾼다. 둘 다 SPEC N-12 의 "GC 시점" 판단에 속하므로 오너 결정 사항이다. +- binary-trees 의 `main` 은 정책(#150)으로 컴파일되지 않고 순수하지도 않아서(`print`, `sys.argv`), 실제 객체를 + 아예 없애는 N-12b 영역이 이 벤치마크에는 적용되지 않는다. + **동일성의 정의.** 컴파일된 코드의 모든 실행은 **어떤 유효한 CPython 실행과 구별할 수 없어야 한다.** 구별은 관측으로만 할 수 있으므로, 설계는 "관측 지점" 목록과 "관측 지점에 닿기 전에 실제 객체로 되돌린다"는 규칙으로 이루어진다. CPython 자체가 정하지 않는 것(스레드 전환 시점)은 CPython 이 고를 수 있는 실행 중 diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md index 9c30cdad8..4d0e4f25f 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md @@ -389,7 +389,8 @@ Slot access and the slow paths: int tp_field_set(PyObject *obj, PyMemberDef *m, PyObject *v); /* PyMember_SetOne: 0 / -1 */ int tp_setattr(PyObject *obj, PyObject *name, PyObject *v); /* STORE_ATTR (PyObject_SetAttr): 0 / -1 */ PyObject *tp_new_fixed(PyTypeObject *cls, PyMemberDef *const *slots, PyObject *const *values, Py_ssize_t n); - /* cls->tp_alloc(cls, 0) then set each slot (new references); NULL + MemoryError */ + /* cls->tp_alloc(cls, 0) then a release store of a new reference into each slot + (the fresh object is unreachable until it returns, #125); NULL + MemoryError */ int tp_is_exact(PyObject *obj, PyTypeObject *cls); /* Py_IS_TYPE */ FieldGet's slow path is `tp_getattr`, New's is `tp_global` + `tp_call` (the binding loaded before the arguments), IsExact's compares with `tp_global`; in a pure function every slow path is a deopt. diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h index 8b9f54d8e..845f42ec9 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h @@ -2756,7 +2756,22 @@ static inline int tp_setattr(PyObject *obj, PyObject *name, PyObject *v) /* The observable result of a trivial __init__: allocate with cls's tp_alloc and set slots[i] to * values[i] (new references; the caller keeps its own). NULL with the error set on failure; the - * half-built object is released (its unset slots are NULL, which dealloc handles). */ + * half-built object is released (its unset slots are NULL, which dealloc handles). + * + * Issue #125: each slot is filled by a release store of a new reference, which is what + * PyMember_SetOne does for a Py_T_OBJECT_EX slot (FT_ATOMIC_STORE_PTR_RELEASE of Py_XNewRef(v) + * under the object's critical section, then Py_XDECREF of the old value), minus the critical section + * and the old value. Why that is the same result: + * - every slot was checked at capture (tp_class_capture): Py_T_OBJECT_EX, writable, offset inside + * tp_basicsize; so SetOne's READONLY and type dispatch are decided already; + * - tp_alloc zeroed the object, so the old value is NULL and there is nothing to release; + * - nothing between tp_alloc and the last store can reach a safepoint (no call that detaches the + * thread state, no eval-breaker check), so no stop-the-world (gc.get_objects, gc.get_referrers) + * can hand the object to another thread before it is filled. The critical section, which could + * detach when contended, protected nothing here. A reader that did find it (none can) would see + * NULL or the stored value, as with SetOne's release store. + * A NULL value (the generator never passes one) keeps SetOne's result: AttributeError for deleting + * an unset Py_T_OBJECT_EX slot. */ static inline PyObject *tp_new_fixed(PyTypeObject *cls, PyMemberDef *const *slots, PyObject *const *values, Py_ssize_t n) { @@ -2767,10 +2782,13 @@ static inline PyObject *tp_new_fixed(PyTypeObject *cls, PyMemberDef *const *slot return NULL; } for (i = 0; i < n; i++) { - if (PyMember_SetOne((char *)obj, slots[i], values[i]) < 0) { + if (values[i] == NULL) { + PyErr_SetString(PyExc_AttributeError, slots[i]->name); Py_DECREF(obj); return NULL; } + __atomic_store_n((PyObject **)((char *)obj + slots[i]->offset), Py_NewRef(values[i]), + __ATOMIC_RELEASE); } return obj; } diff --git a/python-multiplatform-ksp/src/test/python/test_cgen_classes.py b/python-multiplatform-ksp/src/test/python/test_cgen_classes.py index dfbeb0058..4858212dc 100644 --- a/python-multiplatform-ksp/src/test/python/test_cgen_classes.py +++ b/python-multiplatform-ksp/src/test/python/test_cgen_classes.py @@ -428,6 +428,20 @@ def test_gen_uses_only_the_runtime_helpers_for_slots(): assert "PyMember_" not in text and "tp_alloc" not in text and "->tp_dictoffset" not in text +def test_new_fixed_fills_its_fresh_object_with_direct_release_stores(): + """#125: a New allocates its object and fills every slot before control can reach a safepoint, so + no other thread holds the object yet. tp_new_fixed stores each new reference straight into the + slot (release store, as PyMember_SetOne's free-threaded store) instead of taking the member + protocol's per-object critical section for every slot.""" + from typedpython import cbuild + h = (cbuild.RUNTIME_DIR / "tp_runtime.h").read_text() + start = h.index("static inline PyObject *tp_new_fixed(") + body = h[start:h.index("\n}\n", start)] + assert "PyMember_SetOne" not in body and "CRITICAL_SECTION" not in body, body + assert "__atomic_store_n(" in body and "__ATOMIC_RELEASE" in body, body + assert "Py_NewRef(" in body, body # the slot owns its own reference + + # --- refcounts --------------------------------------------------------------------------------------------- def test_reference_counts_are_unchanged_over_many_calls(m): From 6f7b669ea28e902ccd0156fa9db3c46650de3663 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:44:19 +0900 Subject: [PATCH 20/29] TypedPython: MSVC fastmath enables FMA contraction only (#368) Windows CI failed test_fastmath_keeps_nan_checks: float_control(precise, off) is /fp:fast, which folds x != x. cl.exe has no reassociation-only switch, so on MSVC a fastmath function now contracts but does not reassociate. SPEC N-8 says so; a macro test pins it. --- docs/SPEC.md | 13 ++++++++----- .../src/main/python/typedpython/cgen.py | 8 +++++--- .../src/test/python/test_pipeline.py | 14 ++++++++++++++ 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/docs/SPEC.md b/docs/SPEC.md index 14c5ece6f..e8ebb2032 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -748,12 +748,15 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. caller without fastmath): **this is the one path in TypedPython that is explicitly NOT CPython-identical** (a sum or fused multiply-add in a `@compiled(fastmath=True)` function may round differently from the interpreter so float reductions and FMA loops can vectorise). - The three compilers allow the same scope (#368): reassociation and FMA contraction only. Clang uses - `#pragma clang fp reassociate(on) contract(fast)`, MSVC `float_control(precise, off)` with - `fp_contract(on)`, GCC `optimize("associative-math,no-signed-zeros,no-trapping-math,fp-contract=fast")` - (the two extra options are what associative-math needs to take effect). None of them assumes finite + No compiler goes beyond reassociation and FMA contraction (#368). Clang uses + `#pragma clang fp reassociate(on) contract(fast)`, GCC + `optimize("associative-math,no-signed-zeros,no-trapping-math,fp-contract=fast")` (the two extra + options are what associative-math needs to take effect), MSVC `fp_contract(on)` only: cl.exe has no + reassociation-only switch, and `float_control(precise, off)` (`/fp:fast`) also folds `x != x` + away, so on MSVC a fastmath function contracts but does not reassociate. None of them assumes finite values, so a NaN check (`x != x`, `isnan`) in a fastmath function still works; GCC's `fast-math` is - not used (`test_pipeline.py`: the macro test, and the NaN-check test run under a real GCC on Linux CI). + not used (`test_pipeline.py`: the macro test, the NaN-check test on the host compiler of every CI OS + including MSVC on Windows, and the same under a real GCC on Linux CI). `Status: implemented` for desktop host builds (`test_pipeline.py`, `test_cgen*.py`, `test_recursion.py`, `test_eval_breaker.py`, `test_fast_entry.py`, `test_free_threaded.py`; free-threaded 3.15t, #159). Not yet: shipping compiled modules in an app through diff --git a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py index 295484182..fe1635e79 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py @@ -758,15 +758,17 @@ def generate(module: ir.Module, source_path: Path, display_path: str | None = No w("#define TP_FASTMATH_BEGIN _Pragma(\"clang fp reassociate(on) contract(fast)\")") w("#define TP_FASTMATH_END _Pragma(\"clang fp reassociate(off) contract(off)\")") w("#elif defined(__GNUC__)") - # The same scope as clang and MSVC (#368): reassociation and FMA contraction only. + # The same scope as clang (#368): reassociation and FMA contraction only. # associative-math takes effect only with no-signed-zeros and no-trapping-math; never # fast-math, whose finite-math-only folds `x != x` and isnan away. w("#define TP_FASTMATH_BEGIN _Pragma(\"GCC push_options\") _Pragma(\"GCC optimize(" "\\\"associative-math,no-signed-zeros,no-trapping-math,fp-contract=fast\\\")\")") w("#define TP_FASTMATH_END _Pragma(\"GCC pop_options\")") w("#elif defined(_MSC_VER)") - w("#define TP_FASTMATH_BEGIN __pragma(float_control(precise, off, push)) __pragma(fp_contract(on))") - w("#define TP_FASTMATH_END __pragma(fp_contract(off)) __pragma(float_control(pop))") + # cl.exe has no reassociation-only switch: float_control(precise, off) is /fp:fast, which also + # folds `x != x` away (Windows CI, #368). So MSVC gets FMA contraction only. + w("#define TP_FASTMATH_BEGIN __pragma(fp_contract(on))") + w("#define TP_FASTMATH_END __pragma(fp_contract(off))") w("#else") w("#define TP_FASTMATH_BEGIN") w("#define TP_FASTMATH_END") diff --git a/python-multiplatform-ksp/src/test/python/test_pipeline.py b/python-multiplatform-ksp/src/test/python/test_pipeline.py index dcfb7d6da..a50a4bfe5 100644 --- a/python-multiplatform-ksp/src/test/python/test_pipeline.py +++ b/python-multiplatform-ksp/src/test/python/test_pipeline.py @@ -321,3 +321,17 @@ def test_fastmath_keeps_nan_checks_under_gcc(write, tmp_path, monkeypatch): nan = float("nan") assert mod.is_nan(nan) is True assert mod.nan_count([1.0, nan, 2.0, nan]) == 2 + + +def test_msvc_fastmath_scope_is_contract_only(write): + """#368: cl.exe has no reassociation-only switch; `float_control(precise, off)` is /fp:fast and + folds `x != x` (seen on Windows CI). MSVC's fastmath macro enables FMA contraction only.""" + from typedpython import cgen, frontend, verify + + path = write("fm_msvc.py", _NAN_SRC) + proved, _ = verify.verify(frontend.lower(path)) + lines = cgen.generate(proved, path).splitlines() + i = lines.index("#elif defined(_MSC_VER)", next(k for k, l in enumerate(lines) if "TP_FASTMATH_BEGIN" in l)) + begin = next(l for l in lines[i:] if "define TP_FASTMATH_BEGIN" in l) + assert "fp_contract(on)" in begin, begin + assert "float_control" not in begin and "precise" not in begin, begin From eb17d997427b31d1df8afc6c320579bf6ec4fa4a Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:49:26 +0900 Subject: [PATCH 21/29] TypedPython: emit-c CLI and manifest for pypackpack's compile slot (#25 step 1) python -m typedpython emit-c runs generate_project and writes a JSON manifest whose nativeModules / notNativeModules entries map one to one onto pypackpack's ExtensionModuleSource and NotNativeModule. The format (version 1) is fixed in docs/design/typedpython.md 4.3.2a for the Gradle task (#25 step 2). --- docs/SPEC.md | 9 ++ docs/design/typedpython.md | 46 +++++++ .../src/main/python/typedpython/__main__.py | 61 +++++++++ .../src/test/python/test_emit_c.py | 121 ++++++++++++++++++ 4 files changed, 237 insertions(+) create mode 100644 python-multiplatform-ksp/src/test/python/test_emit_c.py diff --git a/docs/SPEC.md b/docs/SPEC.md index 13e3a29f4..43b783797 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -795,6 +795,15 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. only, changing toolchain flags or platform reuses every entry, the C compiler is never run, and the emitted `.c` built by `cbuild` behaves as `compile_project`'s extension. `compile_project` keeps building in process. + `python -m typedpython emit-c PATHS... --root DIR --cache DIR --manifest FILE` (#25, step 1) runs + `generate_project` and writes a JSON manifest that maps one to one onto pypackpack's + `BundleRequest.nativeModules` (`ExtensionModuleSource`: `moduleName`, `sources`, `includeDirs`, + `defines`) and `notNativeModules` (`NotNativeModule`: `module`, `kind` `NOT_MARKED` or + `NOTHING_COMPILED`, `items`). Every developer module given is in exactly one list except a package + `__init__`, which is in neither; a compiled module's `sources` is its one `.c` and its + `includeDirs` the runtime header directory; paths are absolute. Exit status 0 (manifest written), 1 + (a gate error; no manifest), 2 (the gate could not run). The C compiler is never run. The format is + fixed in `docs/design/typedpython.md` §4.3.2a. Tests: `test_emit_c.py`. - **N-11** TypedPython fixed-layout classes (M2b, 2026-11-21; design §4.3.3). A class compiles to fixed field access only when its layout is fixed **in CPython too**: it declares `__slots__` listing exactly its annotated instance fields, has no base but `object`, no metaclass, no decorator but `@compiled`, diff --git a/docs/design/typedpython.md b/docs/design/typedpython.md index eaf3a02d2..1c95c37ec 100644 --- a/docs/design/typedpython.md +++ b/docs/design/typedpython.md @@ -168,6 +168,52 @@ pypackpack 의 native compile slot 이 C 를 확장으로 빌드하므로 캐시 않으므로 그것들은 키에 넣지 않는다. 바뀌지 않은 모듈은 바이트가 같은 `.c` 로 재사용되어 slot 의 C → 확장 캐시도 맞는다. 테스트가 쓰는 `compile_project`(프로세스 안에서 빌드)는 그대로 둔다. +### 4.3.2a `emit-c` manifest (SPEC N-10, #25 1단계) + +Gradle 쪽(#25 2단계, python-multiplatform-gradle-plugin)은 TypedPython 을 이 CLI 로 돌리고 manifest 를 +pypackpack `BundleRequest` 로 옮긴다. pypackpack 은 TypedPython 을 실행하지 않는다. + + python -m typedpython emit-c PATHS... --root DIR --cache DIR --manifest FILE + +- `PATHS`: 파일 또는 디렉터리(재귀, `typedpython check` 와 같은 규칙). `--root`: 모듈 이름의 기준 + 디렉터리(ppp 패키지의 `python/` 루트에 해당). `--cache`: `generate_project` 의 소스 → C 캐시. + `--manifest`: 쓸 파일. 상위 디렉터리는 만든다. +- 종료 코드: 0 manifest 를 씀, 1 게이트 오류(manifest 를 쓰지 않음, 원인은 stderr), 2 게이트를 실행할 수 없음. + C 컴파일러는 실행하지 않는다. + +형식(버전 1). 키 이름은 pypackpack `docs/design/compile-slot.md` 의 Kotlin 필드 이름과 같다. + +```json +{ + "format": "typedpython-emit-c", + "version": 1, + "compilerVersion": "", + "projectRoot": "/abs/proj", + "runtimeInclude": "/abs/.../typedpython/runtime", + "nativeModules": [ + {"moduleName": "app.physics.nbody", "file": "app/physics/nbody.py", + "sources": ["/abs/cache/.../nbody.c"], "includeDirs": ["/abs/.../typedpython/runtime"], + "defines": {}, "status": "rebuilt", + "skippedItems": [{"name": "main", "file": "app/physics/nbody.py", "line": 30, "message": "..."}]} + ], + "notNativeModules": [ + {"module": "app.util", "file": "app/util.py", "kind": "NOT_MARKED", "items": []}, + {"module": "app.text", "file": "app/text.py", "kind": "NOTHING_COMPILED", + "items": [{"name": "f", "file": "app/text.py", "line": 2, "message": "..."}]} + ] +} +``` + +- `nativeModules[i]` → `ExtensionModuleSource(moduleName, sources, includeDirs, defines)`. `file`, `status`, + `skippedItems` 는 보고용이다(일부 함수가 인터프리트로 남는 것은 경고가 아니다, compile-slot.md §4). +- `notNativeModules[i]` → `NotNativeModule(module, kind, items)`, `items[j]` → `SkippedItem(name, file, line, message)`. + `line` 은 없으면 `null`. +- 모든 경로는 절대 경로(`/` 구분). `file` 만 `projectRoot` 기준 상대 경로다. +- 패키지 `__init__.py` 는 어느 목록에도 넣지 않는다(compile-slot.md: TypedPython 은 `__init__` 을 넘기지 않는다). + 그 밖의 모듈은 정확히 한 목록에 들어간다. 순서는 입력 파일 순서(디렉터리는 정렬)다. +- `sources` 는 캐시 안의 `.c` 를 가리킨다. 바뀌지 않은 모듈은 같은 경로, 같은 바이트다. +- 버전을 올리는 변경: 키 제거나 의미 변경. 키 추가는 버전을 올리지 않는다. + ### 4.3.3 고정 레이아웃 클래스 (SPEC N-11, M2b) **레이아웃이 CPython 에서도 고정일 때만** 고정 접근으로 컴파일한다, 사용자가 `__slots__` 를 직접 쓴 클래스. diff --git a/python-multiplatform-ksp/src/main/python/typedpython/__main__.py b/python-multiplatform-ksp/src/main/python/typedpython/__main__.py index 4db824787..f368b72c0 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/__main__.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/__main__.py @@ -1,4 +1,5 @@ """`python -m typedpython check [--mode checked|compiled] [--format text|json] PATHS...` +`python -m typedpython emit-c PATHS... --root DIR [--cache DIR] --manifest FILE` Exit status: 0 no errors (warnings allowed), 1 errors, 2 the gate itself could not run. """ @@ -36,8 +37,17 @@ def main(argv: list[str]) -> int: help="incremental cache directory (default: build/typedpython-cache)") build.add_argument("--out", type=Path, default=None, help="copy the built extensions here (default: leave them in the cache)") + emit = commands.add_parser( + "emit-c", help="generate C per module (cached) and a manifest for pypackpack's compile slot") + emit.add_argument("paths", nargs="+", type=Path, help="files, or directories to scan recursively") + emit.add_argument("--root", type=Path, required=True, help="directory module names are relative to") + emit.add_argument("--cache", type=Path, default=Path("build/typedpython-c-cache"), + help="source -> C cache directory (default: build/typedpython-c-cache)") + emit.add_argument("--manifest", type=Path, required=True, help="the JSON manifest to write") args = parser.parse_args(argv) + if args.command == "emit-c": + return _emit_c(args.paths, args.root, args.cache, args.manifest) if args.command == "build": return _build(args.paths, args.cache, args.out) if args.command == "demo": @@ -94,6 +104,57 @@ def _build(paths: list[Path], cache: Path, out: Path | None) -> int: return EXIT_CLEAN +MANIFEST_FORMAT, MANIFEST_VERSION = "typedpython-emit-c", 1 + + +def _emit_c(paths: list[Path], root: Path, cache: Path, manifest: Path) -> int: + """SPEC N-10 / #25 step 1; the manifest format is docs/design/typedpython.md section 4.3.2a.""" + from typedpython import pipeline + + sources = python_files(paths) + if not sources: + print("typedpython: no Python files to emit", file=sys.stderr) + return EXIT_FAILURE + try: + result = pipeline.generate_project(sources, root, cache) + except pipeline.CompileError as e: + print(f"typedpython: {e}", file=sys.stderr) + return EXIT_ERRORS + except PyreflyError as e: + print(f"typedpython: {e}", file=sys.stderr) + return EXIT_FAILURE + root = root.resolve() + runtime = Path(result.runtime_include).resolve().as_posix() + + def items(m: "pipeline.ModuleResult") -> list[dict]: + return [{"name": i.name, "file": i.file, "line": i.line, "message": i.message} + for i in m.skipped_items] + + native, other = [], [] + for m in result.modules: + if m.path.name == "__init__.py": + continue # never listed (pypackpack compile-slot.md) + rel = m.path.resolve().relative_to(root).as_posix() + if m.kind == "compiled": + assert m.c_source is not None + native.append({"moduleName": m.name, "file": rel, + "sources": [Path(m.c_source).resolve().as_posix()], + "includeDirs": [runtime], "defines": {}, "status": m.status, + "skippedItems": items(m)}) + else: + other.append({"module": m.name, "file": rel, + "kind": "NOT_MARKED" if m.kind == "not_marked" else "NOTHING_COMPILED", + "items": items(m)}) + doc = {"format": MANIFEST_FORMAT, "version": MANIFEST_VERSION, + "compilerVersion": pipeline.COMPILER_VERSION, "projectRoot": root.as_posix(), + "runtimeInclude": runtime, "nativeModules": native, "notNativeModules": other} + manifest.parent.mkdir(parents=True, exist_ok=True) + tmp = manifest.with_name(manifest.name + ".tmp") + tmp.write_text(json.dumps(doc, indent=2) + "\n") + tmp.replace(manifest) + return EXIT_CLEAN + + def _demo(module: Path, module_argv: list[str], out: Path) -> int: import contextlib import importlib.util diff --git a/python-multiplatform-ksp/src/test/python/test_emit_c.py b/python-multiplatform-ksp/src/test/python/test_emit_c.py new file mode 100644 index 000000000..737137899 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_emit_c.py @@ -0,0 +1,121 @@ +"""`python -m typedpython emit-c` (#25 step 1, SPEC N-10): generate_project plus a JSON manifest that +maps onto pypackpack's BundleRequest.nativeModules / notNativeModules. Format: docs/design/typedpython.md +section 4.3.2a. + +Real Pyrefly; the C compiler is never run here. +""" +import json +import textwrap +from pathlib import Path + +import pytest + +from conftest import decorate_compiled +from typedpython import cbuild +from typedpython.__main__ import main + +A = decorate_compiled(textwrap.dedent("""\ + def inc(n: int) -> int: + return n + 1 + """)) +PLAIN = "def hello() -> str:\n return 'hi'\n" +NOTHING = "@compiled\ndef f(s: str) -> str:\n return s[1:]\n" +BROKEN = "@compiled\ndef f(n: int) -> int:\n return n + 'x'\n" + + +@pytest.fixture +def project(tmp_path): + root = tmp_path / "proj" + (root / "app").mkdir(parents=True) + (root / "app" / "__init__.py").write_text("") + (root / "app" / "fast.py").write_text(A) + (root / "app" / "plain.py").write_text(PLAIN) + (root / "app" / "text.py").write_text(NOTHING) + return root + + +@pytest.fixture(autouse=True) +def no_c_compiler(monkeypatch): + def build(*a, **k): + raise AssertionError("emit-c must not run the C compiler") + monkeypatch.setattr(cbuild, "build", build) + + +def emit(project, tmp_path, *paths): + manifest = tmp_path / "out" / "manifest.json" + rc = main(["emit-c", *(str(p) for p in (paths or [project])), "--root", str(project), + "--cache", str(tmp_path / "cache"), "--manifest", str(manifest)]) + return rc, manifest + + +def test_manifest_lists_every_module_once_in_the_slot_shape(project, tmp_path): + rc, manifest = emit(project, tmp_path) + assert rc == 0 + m = json.loads(manifest.read_text()) + assert m["format"] == "typedpython-emit-c" and m["version"] == 1 + assert m["projectRoot"] == project.resolve().as_posix() + runtime = Path(m["runtimeInclude"]) + assert runtime.is_absolute() and (runtime / "tp_runtime.h").is_file() + assert m["compilerVersion"] + + native = {n["moduleName"]: n for n in m["nativeModules"]} + assert list(native) == ["app.fast"] + fast = native["app.fast"] + assert set(fast) >= {"moduleName", "sources", "includeDirs", "defines", "file", "status", "skippedItems"} + assert fast["file"] == "app/fast.py" and fast["defines"] == {} + assert fast["includeDirs"] == [m["runtimeInclude"]] + assert len(fast["sources"]) == 1 + c_file = Path(fast["sources"][0]) + assert c_file.is_absolute() and c_file.name == "fast.c" and c_file.is_file() + assert "PyInit_fast" in c_file.read_text() + + other = {n["module"]: n for n in m["notNativeModules"]} + assert list(other) == ["app.plain", "app.text"] + assert other["app.plain"] == {"module": "app.plain", "file": "app/plain.py", "kind": "NOT_MARKED", "items": []} + text = other["app.text"] + assert text["kind"] == "NOTHING_COMPILED" and text["items"] + item = text["items"][0] + assert set(item) == {"name", "file", "line", "message"} + assert item["name"] == "f" and item["file"] == "app/text.py" and isinstance(item["line"], int) + + every = [n["moduleName"] for n in m["nativeModules"]] + [n["module"] for n in m["notNativeModules"]] + assert "app" not in every and "app.__init__" not in every # a package __init__ is in neither + + +def test_a_second_run_reuses_the_same_c_files(project, tmp_path): + _, manifest = emit(project, tmp_path) + first = json.loads(manifest.read_text()) + c_file = Path(first["nativeModules"][0]["sources"][0]) + before = c_file.read_bytes() + assert first["nativeModules"][0]["status"] == "rebuilt" + rc, manifest = emit(project, tmp_path) + second = json.loads(manifest.read_text()) + assert rc == 0 + assert second["nativeModules"][0]["status"] == "reused" + assert second["nativeModules"][0]["sources"] == first["nativeModules"][0]["sources"] + assert c_file.read_bytes() == before + strip = lambda d: {**d, "nativeModules": [{**n, "status": None} for n in d["nativeModules"]]} + assert strip(second) == strip(first) + + +def test_files_given_one_by_one_keep_their_order(project, tmp_path): + rc, manifest = emit(project, tmp_path, project / "app" / "text.py", project / "app" / "plain.py") + assert rc == 0 + m = json.loads(manifest.read_text()) + assert m["nativeModules"] == [] + assert [n["module"] for n in m["notNativeModules"]] == ["app.text", "app.plain"] + + +def test_a_gate_error_exits_1_and_writes_no_manifest(project, tmp_path, capsys): + (project / "app" / "broken.py").write_text(BROKEN) + rc, manifest = emit(project, tmp_path) + assert rc == 1 + assert not manifest.exists() + assert "typedpython:" in capsys.readouterr().err + + +def test_no_python_files_exits_2(tmp_path): + empty = tmp_path / "empty" + empty.mkdir() + rc, manifest = emit(empty, tmp_path) + assert rc == 2 and not manifest.exists() From 8f27c263e384046174d36b8de582f2f80fe380ee Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 7 Nov 2026 10:14:37 +0900 Subject: [PATCH 22/29] Feat: add TypedPythonEmitCTask and manifest v1 mapper in Gradle plugin (#25 step 2, SPEC N-10) --- docs/SPEC.md | 17 +- .../gradle/TypedPythonEmitCTask.kt | 359 ++++++++++++++++++ .../gradle/TypedPythonEmitCTaskTest.kt | 321 ++++++++++++++++ 3 files changed, 694 insertions(+), 3 deletions(-) create mode 100644 python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/TypedPythonEmitCTask.kt create mode 100644 python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/TypedPythonEmitCTaskTest.kt diff --git a/docs/SPEC.md b/docs/SPEC.md index 1882bc197..76f1380db 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -792,9 +792,11 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. The Gradle task declares exact inputs and outputs so up-to-date checks and the build cache apply. Tests: changing one function body recompiles that module only; an unchanged rebuild spends ~0 s in the compile step. `Status: partial`: the per-module cache and its key are implemented in Python - (`incremental.py`, `pipeline.compile_project`; `test_incremental.py`, `test_pipeline_incremental.py`); - the Gradle task with declared inputs and outputs does not exist yet, because compiled modules are built - through pypackpack's compile slot (#25). + (`incremental.py`, `pipeline.compile_project`; `test_incremental.py`, `test_pipeline_incremental.py`), + and the Gradle `emit-c` task with declared inputs and outputs is implemented in + `python-multiplatform-gradle-plugin` (#25, step 2; `TypedPythonEmitCTask.kt`, + `TypedPythonEmitCTaskTest.kt`); wiring the resulting `nativeModules` / `notNativeModules` into + pypackpack's compile slot in the bundler (#25, step 3) remains. For that slot, `pipeline.generate_project` caches source to C instead of source to extension (#113): one `.c` per compiled module plus the directory of the header-only `tp_runtime.h`; its key is the source, the dependency interface hashes and the compiler version (which covers the runtime header), @@ -813,6 +815,15 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. `includeDirs` the runtime header directory; paths are absolute. Exit status 0 (manifest written), 1 (a gate error; no manifest), 2 (the gate could not run). The C compiler is never run. The format is fixed in `docs/design/typedpython.md` §4.3.2a. Tests: `test_emit_c.py`. + `TypedPythonEmitCTask` in `python-multiplatform-gradle-plugin` (#25, step 2) runs + `python -m typedpython emit-c` with declared Gradle inputs (`@InputFiles` `sources` with + `PathSensitivity.RELATIVE` and `@SkipWhenEmpty`, `@Input` `pythonExecutable` and `@Optional @InputFiles` + `pythonPath` with `PathSensitivity.RELATIVE`) and outputs (`@OutputDirectory` `cacheDirectory` and + `@OutputFile` `manifestFile`), fails with `GradleException` naming the stderr diagnostics on exit 1 or 2, + and parses the written v1 manifest into `TypedPythonEmitCManifest` (`nativeModules` mapped to + `ExtensionModuleSource(moduleName, sources, includeDirs, defines)` and `notNativeModules` mapped to + `NotNativeModule(module, kind, items)` with `SkippedItem(name, file, line, message)`). Tests: + `TypedPythonEmitCTaskTest.kt`. - **N-11** TypedPython fixed-layout classes (M2b, 2026-11-21; design §4.3.3). A class compiles to fixed field access only when its layout is fixed **in CPython too**: it declares `__slots__` listing exactly its annotated instance fields, has no base but `object`, no metaclass, no decorator but `@compiled`, diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/TypedPythonEmitCTask.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/TypedPythonEmitCTask.kt new file mode 100644 index 000000000..21f23326b --- /dev/null +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/TypedPythonEmitCTask.kt @@ -0,0 +1,359 @@ +package python.multiplatform.gradle + +import groovy.json.JsonSlurper +import org.gradle.api.DefaultTask +import org.gradle.api.GradleException +import org.gradle.api.file.ConfigurableFileCollection +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.provider.Property +import org.gradle.api.tasks.IgnoreEmptyDirectories +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.InputFiles +import org.gradle.api.tasks.Internal +import org.gradle.api.tasks.Optional +import org.gradle.api.tasks.OutputDirectory +import org.gradle.api.tasks.OutputFile +import org.gradle.api.tasks.PathSensitive +import org.gradle.api.tasks.PathSensitivity +import org.gradle.api.tasks.SkipWhenEmpty +import org.gradle.api.tasks.TaskAction +import java.io.File + +/** `docs/design/typedpython.md` section 4.3.2a (`emit-c` manifest format). */ +const val TYPEDPYTHON_EMIT_C_FORMAT = "typedpython-emit-c" + +/** `docs/design/typedpython.md` section 4.3.2a (`emit-c` manifest version). */ +const val TYPEDPYTHON_EMIT_C_VERSION = 1 + +/** + * One extension module's generated C and header/define metadata for pypackpack's compile slot + * (`pypackpack` `ExtensionCompiler.kt`, `BundleRequest.nativeModules`). + */ +data class ExtensionModuleSource( + val moduleName: String, + val sources: List, + val includeDirs: List = emptyList(), + val defines: Map = emptyMap(), +) + +/** + * One function or class that stayed interpreted, and why (`pypackpack` `docs/design/compile-slot.md` section 2). + * + * @param file path relative to `projectRoot`, `/`-separated. + * @param line 1-based line number, or `null` when unavailable. + */ +data class SkippedItem( + val name: String, + val file: String, + val line: Int?, + val message: String, +) + +/** + * One developer module for which TypedPython produced no C, and why (`pypackpack` `BundleRequest.notNativeModules`). + */ +data class NotNativeModule( + val module: String, + val kind: Kind, + val items: List = emptyList(), +) { + enum class Kind { + NOT_MARKED, + NOTHING_COMPILED, + } +} + +/** + * One entry of `nativeModules` in the `emit-c` manifest (`docs/design/typedpython.md` section 4.3.2a), + * carrying both the compile-slot fields ([toExtensionModuleSource]) and the reporting fields + * ([file], [status], [skippedItems]). + */ +data class TypedPythonNativeModuleEntry( + val moduleName: String, + val file: String, + val sources: List, + val includeDirs: List = emptyList(), + val defines: Map = emptyMap(), + val status: String, + val skippedItems: List = emptyList(), +) { + fun toExtensionModuleSource(): ExtensionModuleSource = + ExtensionModuleSource( + moduleName = moduleName, + sources = sources, + includeDirs = includeDirs, + defines = defines, + ) +} + +/** + * Parsed `typedpython-emit-c` v1 manifest (`docs/design/typedpython.md` section 4.3.2a, SPEC N-10). + */ +data class TypedPythonEmitCManifest( + val format: String, + val version: Int, + val compilerVersion: String, + val projectRoot: File, + val runtimeInclude: File, + val nativeModules: List, + val notNativeModules: List, +) { + /** Maps [nativeModules] onto pypackpack's `BundleRequest.nativeModules`. */ + fun toExtensionModuleSources(): List = + nativeModules.map { it.toExtensionModuleSource() } + + /** Maps [notNativeModules] onto pypackpack's `BundleRequest.notNativeModules`. */ + fun toNotNativeModules(): List = notNativeModules +} + +/** + * Parses a `typedpython-emit-c` v1 JSON manifest (`docs/design/typedpython.md` section 4.3.2a) + * into [TypedPythonEmitCManifest]. + * + * Unknown top-level or entry keys are ignored as specified in section 4.3.2a ("adding keys does not + * bump the version"). Unknown `format`, `version` or `notNativeModules[i].kind` fails with + * [IllegalArgumentException]. + */ +fun parseTypedPythonEmitCManifest(json: String): TypedPythonEmitCManifest { + val root = try { + JsonSlurper().parseText(json) + } catch (e: Exception) { + throw IllegalArgumentException("typedpython emit-c manifest is not valid JSON: ${e.message}", e) + } as? Map<*, *> ?: throw IllegalArgumentException("typedpython emit-c manifest is not a JSON object.") + + val format = root["format"] as? String + require(format == TYPEDPYTHON_EMIT_C_FORMAT) { + "typedpython emit-c manifest has format '$format'; expected '$TYPEDPYTHON_EMIT_C_FORMAT'." + } + val version = (root["version"] as? Number)?.toInt() + require(version == TYPEDPYTHON_EMIT_C_VERSION) { + "typedpython emit-c manifest has version '${root["version"]}'; expected version $TYPEDPYTHON_EMIT_C_VERSION." + } + + fun requireString(map: Map<*, *>, key: String, owner: String): String = + (map[key] as? String)?.takeIf { it.isNotEmpty() } + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner.$key' is missing or empty.") + + fun parseSkippedItems(rawList: Any?, owner: String): List { + val list = rawList as? List<*> + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner' must be a list.") + return list.mapIndexed { idx, elem -> + val item = elem as? Map<*, *> + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner[$idx]' is not an object.") + val lineVal = item["line"] + val line = when (lineVal) { + null -> null + is Number -> lineVal.toInt() + else -> throw IllegalArgumentException( + "typedpython emit-c manifest: '$owner[$idx].line' must be an integer or null, got '$lineVal'.", + ) + } + SkippedItem( + name = requireString(item, "name", "$owner[$idx]"), + file = requireString(item, "file", "$owner[$idx]"), + line = line, + message = (item["message"] as? String) + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner[$idx].message' is missing."), + ) + } + } + + val rawNative = root["nativeModules"] as? List<*> + ?: throw IllegalArgumentException("typedpython emit-c manifest: 'nativeModules' is missing or not a list.") + val nativeModules = rawNative.mapIndexed { idx, elem -> + val owner = "nativeModules[$idx]" + val map = elem as? Map<*, *> + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner' is not an object.") + val sourcesRaw = map["sources"] as? List<*> + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner.sources' is missing or not a list.") + val sources = sourcesRaw.map { path -> + val str = (path as? String)?.takeIf { it.isNotEmpty() } + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner.sources' contains a non-string entry.") + File(str) + } + val includeDirsRaw = map["includeDirs"] as? List<*> + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner.includeDirs' is missing or not a list.") + val includeDirs = includeDirsRaw.map { path -> + val str = (path as? String)?.takeIf { it.isNotEmpty() } + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner.includeDirs' contains a non-string entry.") + File(str) + } + val definesRaw = map["defines"] as? Map<*, *> + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner.defines' is missing or not an object.") + val defines = definesRaw.entries.associate { (k, v) -> + val key = (k as? String)?.takeIf { it.isNotEmpty() } + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner.defines' has a blank key.") + val value = when (v) { + null -> null + is String -> v + else -> v.toString() + } + key to value + } + TypedPythonNativeModuleEntry( + moduleName = requireString(map, "moduleName", owner), + file = requireString(map, "file", owner), + sources = sources, + includeDirs = includeDirs, + defines = defines, + status = requireString(map, "status", owner), + skippedItems = parseSkippedItems(map["skippedItems"] ?: emptyList(), "$owner.skippedItems"), + ) + } + + val rawNotNative = root["notNativeModules"] as? List<*> + ?: throw IllegalArgumentException("typedpython emit-c manifest: 'notNativeModules' is missing or not a list.") + val notNativeModules = rawNotNative.mapIndexed { idx, elem -> + val owner = "notNativeModules[$idx]" + val map = elem as? Map<*, *> + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner' is not an object.") + val kindRaw = requireString(map, "kind", owner) + val kind = try { + NotNativeModule.Kind.valueOf(kindRaw) + } catch (e: IllegalArgumentException) { + throw IllegalArgumentException( + "typedpython emit-c manifest: '$owner.kind' has unknown kind '$kindRaw'; " + + "expected one of ${NotNativeModule.Kind.values().joinToString()}.", + e, + ) + } + NotNativeModule( + module = requireString(map, "module", owner), + kind = kind, + items = parseSkippedItems(map["items"] ?: emptyList(), "$owner.items"), + ) + } + + return TypedPythonEmitCManifest( + format = format, + version = version, + compilerVersion = requireString(root, "compilerVersion", "root"), + projectRoot = File(requireString(root, "projectRoot", "root")), + runtimeInclude = File(requireString(root, "runtimeInclude", "root")), + nativeModules = nativeModules, + notNativeModules = notNativeModules, + ) +} + +/** + * Builds the `python -m typedpython emit-c` command line (`docs/design/typedpython.md` section 4.3.2a). + * Input files are sorted by their path relative to [rootDir] so the manifest module order is deterministic. + */ +fun typedPythonEmitCCommand( + pythonExecutable: String, + files: List, + rootDir: File, + cacheDir: File, + manifestFile: File, +): List { + val rootAbs = rootDir.absoluteFile + val sorted = files.map { it.absoluteFile }.sortedBy { file -> + file.relativeToOrSelf(rootAbs).invariantSeparatorsPath + } + return buildList { + add(pythonExecutable) + add("-m") + add("typedpython") + add("emit-c") + sorted.forEach { add(it.absolutePath) } + add("--root") + add(rootAbs.absolutePath) + add("--cache") + add(cacheDir.absoluteFile.absolutePath) + add("--manifest") + add(manifestFile.absoluteFile.absolutePath) + } +} + +/** + * SPEC N-10 (`python-multiplatform#25` step 2): runs `python -m typedpython emit-c` over [sources] + * and writes the per-module C cache ([cacheDirectory]) and the v1 JSON manifest ([manifestFile]) + * for pypackpack's native compile slot. + */ +abstract class TypedPythonEmitCTask : DefaultTask() { + + /** The developer `.py` files to compile to C. */ + @get:InputFiles + @get:SkipWhenEmpty + @get:IgnoreEmptyDirectories + @get:PathSensitive(PathSensitivity.RELATIVE) + abstract val sources: ConfigurableFileCollection + + /** + * The directory module names are derived relative to (`--root`, the ppp package's `python/` + * or `src/main` root). Fingerprinted through [sources]'s `PathSensitivity.RELATIVE`. + */ + @get:Internal + abstract val rootDirectory: DirectoryProperty + + /** The Python interpreter executable that runs `-m typedpython emit-c` (defaults to `"python3"`). */ + @get:Input + abstract val pythonExecutable: Property + + /** Optional `PYTHONPATH` entries for locating the `typedpython` package when not installed in the interpreter. */ + @get:InputFiles + @get:Optional + @get:PathSensitive(PathSensitivity.RELATIVE) + abstract val pythonPath: ConfigurableFileCollection + + /** `generate_project`'s incremental source-to-C cache directory (`--cache`). */ + @get:OutputDirectory + abstract val cacheDirectory: DirectoryProperty + + /** The JSON manifest written by `emit-c` (`--manifest`). */ + @get:OutputFile + abstract val manifestFile: RegularFileProperty + + init { + pythonExecutable.convention("python3") + } + + /** Parses the manifest written at [manifestFile] into [TypedPythonEmitCManifest]. */ + fun readManifest(): TypedPythonEmitCManifest = + parseTypedPythonEmitCManifest(manifestFile.get().asFile.readText()) + + @TaskAction + fun emitC() { + val pyFiles = sources.files.filter { it.isFile && it.extension == "py" } + val root = rootDirectory.get().asFile + val cache = cacheDirectory.get().asFile + val manifest = manifestFile.get().asFile + val executable = pythonExecutable.getOrElse("python3") + + val command = typedPythonEmitCCommand( + pythonExecutable = executable, + files = pyFiles, + rootDir = root, + cacheDir = cache, + manifestFile = manifest, + ) + logger.info("typedPythonEmitC: ${command.joinToString(" ")}") + + val work = temporaryDir.apply { mkdirs() } + val tmp = File(work, "tmp").apply { mkdirs() } + val stdoutFile = File(work, "emit-c.stdout") + val stderrFile = File(work, "emit-c.stderr") + + val pb = ProcessBuilder(command) + .directory(project.projectDir) + .redirectOutput(stdoutFile) + .redirectError(stderrFile) + pb.environment()["TMPDIR"] = tmp.absolutePath + val extraPythonPath = pythonPath.files.map { it.absolutePath }.filter { it.isNotEmpty() } + if (extraPythonPath.isNotEmpty()) { + val existing = pb.environment()["PYTHONPATH"].orEmpty() + val joined = extraPythonPath.joinToString(File.pathSeparator) + pb.environment()["PYTHONPATH"] = if (existing.isEmpty()) joined else "$joined${File.pathSeparator}$existing" + } + + val exitCode = pb.start().waitFor() + if (exitCode != 0) { + val details = listOf(stderrFile.readText().trim(), stdoutFile.readText().trim()) + .filter { it.isNotEmpty() } + .joinToString("\n") + .ifEmpty { "(no output)" } + throw GradleException("typedpython emit-c failed (exit $exitCode):\n$details") + } + } +} diff --git a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/TypedPythonEmitCTaskTest.kt b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/TypedPythonEmitCTaskTest.kt new file mode 100644 index 000000000..23893acd3 --- /dev/null +++ b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/TypedPythonEmitCTaskTest.kt @@ -0,0 +1,321 @@ +package python.multiplatform.gradle + +import org.gradle.api.GradleException +import org.gradle.api.tasks.IgnoreEmptyDirectories +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.InputFiles +import org.gradle.api.tasks.Internal +import org.gradle.api.tasks.Optional +import org.gradle.api.tasks.OutputDirectory +import org.gradle.api.tasks.OutputFile +import org.gradle.api.tasks.PathSensitive +import org.gradle.api.tasks.PathSensitivity +import org.gradle.api.tasks.SkipWhenEmpty +import org.gradle.testfixtures.ProjectBuilder +import java.io.File +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertNull +import kotlin.test.assertTrue + +/** + * SPEC N-10 (python-multiplatform#25 step 2): Gradle task running `python -m typedpython emit-c` + * with declared inputs/outputs and mapping the v1 manifest (`docs/design/typedpython.md` section 4.3.2a) + * onto pypackpack's `BundleRequest.nativeModules` (`ExtensionModuleSource`) and + * `BundleRequest.notNativeModules` (`NotNativeModule` / `SkippedItem`). + */ +class TypedPythonEmitCTaskTest { + + private val sampleManifestJson = + """ + { + "format": "typedpython-emit-c", + "version": 1, + "compilerVersion": "abc123def456", + "projectRoot": "/abs/proj", + "runtimeInclude": "/abs/site-packages/typedpython/runtime", + "nativeModules": [ + { + "moduleName": "app.physics.nbody", + "file": "app/physics/nbody.py", + "sources": ["/abs/cache/nbody.c"], + "includeDirs": ["/abs/site-packages/typedpython/runtime"], + "defines": {"TP_FLAG": "1", "TP_BARE": null}, + "status": "rebuilt", + "skippedItems": [ + {"name": "main", "file": "app/physics/nbody.py", "line": 30, "message": "not pure"} + ] + } + ], + "notNativeModules": [ + { + "module": "app.util", + "file": "app/util.py", + "kind": "NOT_MARKED", + "items": [] + }, + { + "module": "app.text", + "file": "app/text.py", + "kind": "NOTHING_COMPILED", + "items": [ + {"name": "f", "file": "app/text.py", "line": 2, "message": "str slice is not supported"}, + {"name": "g", "file": "app/text.py", "line": null, "message": "no line"} + ] + } + ] + } + """.trimIndent() + + @Test + fun parsesManifestV1AndMapsToBundleRequestShapes() { + val manifest = parseTypedPythonEmitCManifest(sampleManifestJson) + assertEquals(TYPEDPYTHON_EMIT_C_FORMAT, manifest.format) + assertEquals(TYPEDPYTHON_EMIT_C_VERSION, manifest.version) + assertEquals("abc123def456", manifest.compilerVersion) + assertEquals(File("/abs/proj"), manifest.projectRoot) + assertEquals(File("/abs/site-packages/typedpython/runtime"), manifest.runtimeInclude) + + assertEquals(1, manifest.nativeModules.size) + val nativeEntry = manifest.nativeModules.single() + assertEquals("app.physics.nbody", nativeEntry.moduleName) + assertEquals("app/physics/nbody.py", nativeEntry.file) + assertEquals(listOf(File("/abs/cache/nbody.c")), nativeEntry.sources) + assertEquals(listOf(File("/abs/site-packages/typedpython/runtime")), nativeEntry.includeDirs) + assertEquals(mapOf("TP_FLAG" to "1", "TP_BARE" to null), nativeEntry.defines) + assertEquals("rebuilt", nativeEntry.status) + assertEquals( + listOf(SkippedItem("main", "app/physics/nbody.py", 30, "not pure")), + nativeEntry.skippedItems, + ) + + val mappedNative = manifest.toExtensionModuleSources() + assertEquals( + listOf( + ExtensionModuleSource( + moduleName = "app.physics.nbody", + sources = listOf(File("/abs/cache/nbody.c")), + includeDirs = listOf(File("/abs/site-packages/typedpython/runtime")), + defines = mapOf("TP_FLAG" to "1", "TP_BARE" to null), + ), + ), + mappedNative, + ) + + val mappedNotNative = manifest.toNotNativeModules() + assertEquals(2, mappedNotNative.size) + assertEquals( + NotNativeModule( + module = "app.util", + kind = NotNativeModule.Kind.NOT_MARKED, + items = emptyList(), + ), + mappedNotNative[0], + ) + assertEquals( + NotNativeModule( + module = "app.text", + kind = NotNativeModule.Kind.NOTHING_COMPILED, + items = listOf( + SkippedItem("f", "app/text.py", 2, "str slice is not supported"), + SkippedItem("g", "app/text.py", null, "no line"), + ), + ), + mappedNotNative[1], + ) + assertNull(mappedNotNative[1].items[1].line) + } + + @Test + fun AdditiveTopLevelOrEntryKeysDoNotBreakVersion1Parsing() { + val withExtraKeys = sampleManifestJson.replace( + "\"version\": 1,", + "\"version\": 1, \"futureTopKey\": 42,", + ) + val manifest = parseTypedPythonEmitCManifest(withExtraKeys) + assertEquals(1, manifest.nativeModules.size) + assertEquals(2, manifest.notNativeModules.size) + } + + @Test + fun rejectsUnknownFormatVersionOrKind() { + val wrongFormat = sampleManifestJson.replace("\"typedpython-emit-c\"", "\"other-format\"") + val formatErr = assertFailsWith { parseTypedPythonEmitCManifest(wrongFormat) } + assertTrue("typedpython-emit-c" in formatErr.message.orEmpty()) + + val wrongVersion = sampleManifestJson.replace("\"version\": 1", "\"version\": 2") + val versionErr = assertFailsWith { parseTypedPythonEmitCManifest(wrongVersion) } + assertTrue("version" in versionErr.message.orEmpty()) + + val wrongKind = sampleManifestJson.replace("\"NOT_MARKED\"", "\"UNKNOWN_KIND\"") + val kindErr = assertFailsWith { parseTypedPythonEmitCManifest(wrongKind) } + assertTrue("UNKNOWN_KIND" in kindErr.message.orEmpty()) + } + + @Test + fun buildsDeterministicCommandLineWithSortedInputFiles() { + val root = File("/proj/src/main") + val cache = File("/proj/build/typedpython/c-cache") + val manifest = File("/proj/build/typedpython/emit-c-manifest.json") + val b = File("/proj/src/main/app/b.py") + val a = File("/proj/src/main/app/a.py") + + val cmd = typedPythonEmitCCommand( + pythonExecutable = "/venv/bin/python3", + files = listOf(b, a), + rootDir = root, + cacheDir = cache, + manifestFile = manifest, + ) + assertEquals( + listOf( + "/venv/bin/python3", + "-m", + "typedpython", + "emit-c", + a.absolutePath, + b.absolutePath, + "--root", + root.absolutePath, + "--cache", + cache.absolutePath, + "--manifest", + manifest.absolutePath, + ), + cmd, + ) + } + + @Test + fun taskDeclaresGradleInputsAndOutputsForIncrementalAndBuildCacheSupport() { + val getters = TypedPythonEmitCTask::class.java.methods.associateBy { it.name } + + val getSources = getters.getValue("getSources") + assertTrue(getSources.isAnnotationPresent(InputFiles::class.java)) + assertTrue(getSources.isAnnotationPresent(SkipWhenEmpty::class.java)) + assertTrue(getSources.isAnnotationPresent(IgnoreEmptyDirectories::class.java)) + assertEquals( + PathSensitivity.RELATIVE, + getSources.getAnnotation(PathSensitive::class.java)?.value, + ) + + assertTrue(getters.getValue("getRootDirectory").isAnnotationPresent(Internal::class.java)) + assertTrue(getters.getValue("getPythonExecutable").isAnnotationPresent(Input::class.java)) + + val getPythonPath = getters.getValue("getPythonPath") + assertTrue(getPythonPath.isAnnotationPresent(InputFiles::class.java)) + assertTrue(getPythonPath.isAnnotationPresent(Optional::class.java)) + assertEquals( + PathSensitivity.RELATIVE, + getPythonPath.getAnnotation(PathSensitive::class.java)?.value, + ) + + assertTrue(getters.getValue("getCacheDirectory").isAnnotationPresent(OutputDirectory::class.java)) + assertTrue(getters.getValue("getManifestFile").isAnnotationPresent(OutputFile::class.java)) + } + + @Test + fun taskActionExecutesStubEmitCAndParsesResultingManifest() { + val project = ProjectBuilder.builder().build() + val pkgRoot = project.file("pkg/src/main").apply { mkdirs() } + val pyFile = File(pkgRoot, "app/fast.py").apply { + parentFile.mkdirs() + writeText("@compiled\ndef inc(n: int) -> int:\n return n + 1\n") + } + val fakeStubRoot = project.file("fake-pythonpath").apply { mkdirs() } + val tpPkg = File(fakeStubRoot, "typedpython").apply { mkdirs() } + val runtimeDir = File(tpPkg, "runtime").apply { mkdirs() } + File(runtimeDir, "tp_runtime.h").writeText("/* header */\n") + File(tpPkg, "__init__.py").writeText("") + File(tpPkg, "__main__.py").writeText( + """ + import argparse, json, sys + from pathlib import Path + p = argparse.ArgumentParser() + p.add_argument("cmd") + p.add_argument("paths", nargs="+") + p.add_argument("--root", required=True) + p.add_argument("--cache", required=True) + p.add_argument("--manifest", required=True) + a = p.parse_args() + cache = Path(a.cache).resolve() + cache.mkdir(parents=True, exist_ok=True) + c_file = cache / "fast.c" + c_file.write_text("/* generated C */\n") + runtime = (Path(__file__).resolve().parent / "runtime").as_posix() + doc = { + "format": "typedpython-emit-c", + "version": 1, + "compilerVersion": "stub-1", + "projectRoot": Path(a.root).resolve().as_posix(), + "runtimeInclude": runtime, + "nativeModules": [{ + "moduleName": "app.fast", + "file": "app/fast.py", + "sources": [c_file.as_posix()], + "includeDirs": [runtime], + "defines": {}, + "status": "rebuilt", + "skippedItems": [], + }], + "notNativeModules": [], + } + mf = Path(a.manifest) + mf.parent.mkdir(parents=True, exist_ok=True) + mf.write_text(json.dumps(doc)) + """.trimIndent() + "\n", + ) + + val task = project.tasks.register("emitTypedPythonC", TypedPythonEmitCTask::class.java) { + sources.from(pyFile) + rootDirectory.set(pkgRoot) + pythonExecutable.set("python3") + pythonPath.from(fakeStubRoot) + cacheDirectory.set(project.layout.buildDirectory.dir("typedpython/c-cache")) + manifestFile.set(project.layout.buildDirectory.file("typedpython/emit-c-manifest.json")) + }.get() + + task.emitC() + + val parsed = task.readManifest() + assertEquals(1, parsed.nativeModules.size) + val mapped = parsed.toExtensionModuleSources().single() + assertEquals("app.fast", mapped.moduleName) + assertTrue(mapped.sources.single().isFile) + assertEquals("fast.c", mapped.sources.single().name) + } + + @Test + fun taskActionFailsWithDiagnosticsOnExit1Or2() { + val project = ProjectBuilder.builder().build() + val pkgRoot = project.file("pkg/src/main").apply { mkdirs() } + val pyFile = File(pkgRoot, "app/broken.py").apply { + parentFile.mkdirs() + writeText("def f(x): return x\n") + } + val fakeStubRoot = project.file("fake-failing-tp").apply { mkdirs() } + val tpPkg = File(fakeStubRoot, "typedpython").apply { mkdirs() } + File(tpPkg, "__init__.py").writeText("") + File(tpPkg, "__main__.py").writeText( + """ + import sys + print("typedpython: broken.py does not pass the gate in compiled mode", file=sys.stderr) + sys.exit(1) + """.trimIndent() + "\n", + ) + + val task = project.tasks.register("emitBroken", TypedPythonEmitCTask::class.java) { + sources.from(pyFile) + rootDirectory.set(pkgRoot) + pythonExecutable.set("python3") + pythonPath.from(fakeStubRoot) + cacheDirectory.set(project.layout.buildDirectory.dir("typedpython/c-cache")) + manifestFile.set(project.layout.buildDirectory.file("typedpython/emit-c-manifest.json")) + }.get() + + val err = assertFailsWith { task.emitC() } + assertTrue("exit 1" in err.message.orEmpty(), err.message) + assertTrue("broken.py does not pass the gate" in err.message.orEmpty(), err.message) + } +} From 3c2ed3d3217ab8328501e389f444e69518126578 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 7 Nov 2026 10:27:26 +0900 Subject: [PATCH 23/29] Feat(gradle-plugin): register emitTypedPythonC from pythonBindings DSL (SPEC N-10, #25) --- docs/SPEC.md | 8 +- .../gradle/PythonBindingsPlugin.kt | 47 ++++++++++ .../gradle/TypedPythonEmitCTask.kt | 3 + .../gradle/TypedPythonEmitCTaskTest.kt | 86 +++++++++++++++++++ .../fixtures/app/build.gradle.kts | 19 ++++ .../fixture/app/TypedPythonKotlinCallTest.kt | 13 +++ 6 files changed, 174 insertions(+), 2 deletions(-) diff --git a/docs/SPEC.md b/docs/SPEC.md index 76f1380db..90d07e494 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -822,8 +822,12 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. `@OutputFile` `manifestFile`), fails with `GradleException` naming the stderr diagnostics on exit 1 or 2, and parses the written v1 manifest into `TypedPythonEmitCManifest` (`nativeModules` mapped to `ExtensionModuleSource(moduleName, sources, includeDirs, defines)` and `notNativeModules` mapped to - `NotNativeModule(module, kind, items)` with `SkippedItem(name, file, line, message)`). Tests: - `TypedPythonEmitCTaskTest.kt`. + `NotNativeModule(module, kind, items)` with `SkippedItem(name, file, line, message)`). Setting + `pythonBindings.typedPythonSourceDir` registers `emitTypedPythonC` (`TypedPythonEmitCTask`, with + `typedPythonRootDir` defaulting to `typedPythonSourceDir`, `typedPythonExecutable` defaulting to + `"python3"`, optional `typedPythonPath`, `cacheDirectory` at `build/typedpython/c-cache` and + `manifestFile` at `build/typedpython/emit-c-manifest.json`); leaving `typedPythonSourceDir` unset + registers no task. Tests: `TypedPythonEmitCTaskTest.kt`, `TypedPythonKotlinCallTest.kt`. - **N-11** TypedPython fixed-layout classes (M2b, 2026-11-21; design §4.3.3). A class compiles to fixed field access only when its layout is fixed **in CPython too**: it declares `__slots__` listing exactly its annotated instance fields, has no base but `object`, no metaclass, no decorator but `@compiled`, diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/PythonBindingsPlugin.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/PythonBindingsPlugin.kt index 823db419d..895f42e4e 100644 --- a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/PythonBindingsPlugin.kt +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/PythonBindingsPlugin.kt @@ -241,6 +241,25 @@ interface PythonBindingsExtension { /** The `ios { }` block itself: without it the nested property is reachable only as `ios.x`. */ fun ios(action: org.gradle.api.Action) = action.execute(ios) + + /** + * SPEC N-10 (`python-multiplatform#25` step 2): directory of developer `.py` files to compile + * to C with `python -m typedpython emit-c`. When set, registers [EMIT_TYPEDPYTHON_C_TASK] + * ([TypedPythonEmitCTask]); unset (the default) registers no task. + */ + val typedPythonSourceDir: org.gradle.api.file.DirectoryProperty + + /** + * The directory module names are derived relative to (`--root`). Defaults to + * [typedPythonSourceDir]. + */ + val typedPythonRootDir: org.gradle.api.file.DirectoryProperty + + /** The Python interpreter executable that runs `-m typedpython emit-c` (defaults to `"python3"`). */ + val typedPythonExecutable: Property + + /** Optional `PYTHONPATH` entries for locating the `typedpython` package when not installed in [typedPythonExecutable]. */ + val typedPythonPath: org.gradle.api.file.ConfigurableFileCollection } /** @@ -326,6 +345,34 @@ class PythonBindingsPlugin : Plugin { // `plugins { ... }` and so do not exist while this plugin is being applied. configureArtifactBindings(this, extension) configureWasmProxyExports(this) + configureTypedPythonEmitC(this, extension) + } + } + + /** + * SPEC N-10 (`python-multiplatform#25` step 2): registers [EMIT_TYPEDPYTHON_C_TASK] + * ([TypedPythonEmitCTask]) when [PythonBindingsExtension.typedPythonSourceDir] is set. + * + * A no-op when [PythonBindingsExtension.typedPythonSourceDir] is unset, so projects that do not + * compile developer Python modules with TypedPython get no task. + */ + internal fun configureTypedPythonEmitC(project: Project, extension: PythonBindingsExtension) { + if (!extension.typedPythonSourceDir.isPresent) return + val srcDir = extension.typedPythonSourceDir.get().asFile + project.tasks.register(EMIT_TYPEDPYTHON_C_TASK, TypedPythonEmitCTask::class.java) { + group = "python" + description = "Runs python -m typedpython emit-c over pythonBindings.typedPythonSourceDir and writes the compile-slot manifest." + sources.from( + project.fileTree(srcDir) { + include("**/*.py") + exclude("**/__pycache__/**", "**/.*", "**/build/**") + }, + ) + rootDirectory.set(extension.typedPythonRootDir.orElse(extension.typedPythonSourceDir)) + pythonExecutable.set(extension.typedPythonExecutable.orElse("python3")) + pythonPath.from(extension.typedPythonPath) + cacheDirectory.set(project.layout.buildDirectory.dir("typedpython/c-cache")) + manifestFile.set(project.layout.buildDirectory.file("typedpython/emit-c-manifest.json")) } } diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/TypedPythonEmitCTask.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/TypedPythonEmitCTask.kt index 21f23326b..7e5d2e28f 100644 --- a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/TypedPythonEmitCTask.kt +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/TypedPythonEmitCTask.kt @@ -26,6 +26,9 @@ const val TYPEDPYTHON_EMIT_C_FORMAT = "typedpython-emit-c" /** `docs/design/typedpython.md` section 4.3.2a (`emit-c` manifest version). */ const val TYPEDPYTHON_EMIT_C_VERSION = 1 +/** Task name registered by [PythonBindingsPlugin] when [PythonBindingsExtension.typedPythonSourceDir] is set. */ +const val EMIT_TYPEDPYTHON_C_TASK = "emitTypedPythonC" + /** * One extension module's generated C and header/define metadata for pypackpack's compile slot * (`pypackpack` `ExtensionCompiler.kt`, `BundleRequest.nativeModules`). diff --git a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/TypedPythonEmitCTaskTest.kt b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/TypedPythonEmitCTaskTest.kt index 23893acd3..2ff23ddba 100644 --- a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/TypedPythonEmitCTaskTest.kt +++ b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/TypedPythonEmitCTaskTest.kt @@ -318,4 +318,90 @@ class TypedPythonEmitCTaskTest { assertTrue("exit 1" in err.message.orEmpty(), err.message) assertTrue("broken.py does not pass the gate" in err.message.orEmpty(), err.message) } + + @Test + fun pluginDoesNotRegisterEmitTypedPythonCWhenSourceDirIsUnset() { + val project = ProjectBuilder.builder().build() + val extension = project.extensions.create("pythonBindings", PythonBindingsExtension::class.java) + PythonBindingsPlugin().configureTypedPythonEmitC(project, extension) + + assertEquals(null, project.tasks.findByName(EMIT_TYPEDPYTHON_C_TASK)) + } + + @Test + fun pluginRegistersEmitTypedPythonCFromExtensionAndExecutesIt() { + val project = ProjectBuilder.builder().build() + val pkgRoot = project.file("pkg/src/main").apply { mkdirs() } + val pyFile = File(pkgRoot, "app/fast.py").apply { + parentFile.mkdirs() + writeText("@compiled\ndef f(x: int) -> int:\n return x + 1\n") + } + File(pkgRoot, "app/__init__.pyi").writeText("") + val fakeStubRoot = project.file("fake-tp-plugin").apply { mkdirs() } + val tpPkg = File(fakeStubRoot, "typedpython").apply { mkdirs() } + File(tpPkg, "__init__.py").writeText("") + File(tpPkg, "__main__.py").writeText( + """ + import argparse, json + from pathlib import Path + p = argparse.ArgumentParser() + p.add_argument("cmd") + p.add_argument("paths", nargs="+") + p.add_argument("--root", required=True) + p.add_argument("--cache", required=True) + p.add_argument("--manifest", required=True) + a = p.parse_args() + assert all(not x.endswith(".pyi") for x in a.paths), a.paths + cache = Path(a.cache).resolve() + cache.mkdir(parents=True, exist_ok=True) + c_file = cache / "fast.c" + c_file.write_text("/* generated C */\n") + runtime = (Path(__file__).resolve().parent / "runtime").as_posix() + doc = { + "format": "typedpython-emit-c", + "version": 1, + "compilerVersion": "stub-1", + "projectRoot": Path(a.root).resolve().as_posix(), + "runtimeInclude": runtime, + "nativeModules": [{ + "moduleName": "app.fast", + "file": "app/fast.py", + "sources": [c_file.as_posix()], + "includeDirs": [runtime], + "defines": {}, + "status": "rebuilt", + "skippedItems": [], + }], + "notNativeModules": [], + } + mf = Path(a.manifest) + mf.parent.mkdir(parents=True, exist_ok=True) + mf.write_text(json.dumps(doc)) + """.trimIndent() + "\n", + ) + + val extension = project.extensions.create("pythonBindings", PythonBindingsExtension::class.java) + extension.typedPythonSourceDir.set(pkgRoot) + extension.typedPythonExecutable.set("python3") + extension.typedPythonPath.from(fakeStubRoot) + + PythonBindingsPlugin().configureTypedPythonEmitC(project, extension) + + val task = project.tasks.findByName(EMIT_TYPEDPYTHON_C_TASK) as? TypedPythonEmitCTask + ?: error("expected $EMIT_TYPEDPYTHON_C_TASK to be registered by PythonBindingsPlugin") + assertEquals(listOf(pyFile.canonicalFile), task.sources.files.map { it.canonicalFile }) + assertEquals(pkgRoot.canonicalFile, task.rootDirectory.get().asFile.canonicalFile) + assertEquals( + project.layout.buildDirectory.dir("typedpython/c-cache").get().asFile.canonicalFile, + task.cacheDirectory.get().asFile.canonicalFile, + ) + assertEquals( + project.layout.buildDirectory.file("typedpython/emit-c-manifest.json").get().asFile.canonicalFile, + task.manifestFile.get().asFile.canonicalFile, + ) + + task.emitC() + val parsed = task.readManifest() + assertEquals("app.fast", parsed.toExtensionModuleSources().single().moduleName) + } } diff --git a/python-multiplatform-ksp/fixtures/app/build.gradle.kts b/python-multiplatform-ksp/fixtures/app/build.gradle.kts index 57ec527d8..9cd4ac650 100644 --- a/python-multiplatform-ksp/fixtures/app/build.gradle.kts +++ b/python-multiplatform-ksp/fixtures/app/build.gradle.kts @@ -106,6 +106,19 @@ val typedPythonSkipReason: String? = when { else -> null } +pythonBindings { + typedPythonSourceDir.set(file("src/desktopTest/typedpython")) + typedPythonExecutable.set(typedPythonHost?.absolutePath ?: "python3") + typedPythonPath.from(rootDir.resolve("python-multiplatform-ksp/src/main/python")) +} + +val typedPythonEmitCManifest = layout.buildDirectory.file("typedpython/emit-c-manifest.json") + +tasks.matching { it.name == "emitTypedPythonC" }.configureEach { + val skip = typedPythonSkipReason + onlyIf("TypedPython emit-c: ${skip ?: "enabled"}") { skip == null } +} + @Suppress("UNCHECKED_CAST") val typedPythonIncludeDir: Provider? = typedPythonMacTarget?.let { target -> val includes = project(":python-multiplatform").extensions.getByName("cpythonIncludeDirectories") @@ -156,6 +169,7 @@ val buildTypedPythonBridge by tasks.registering(Exec::class) { tasks.named("desktopTest") { dependsOn(buildTypedPythonBridge) + dependsOn("emitTypedPythonC") // The tests import tp_kotlin_bridge.py from src/; without this the interpreter leaves a // __pycache__ in the source tree. environment("PYTHONDONTWRITEBYTECODE", "1") @@ -165,10 +179,15 @@ tasks.named("desktopTest") { } if (typedPythonSkipReason == null) { inputs.dir(typedPythonBridgeDir).withPropertyName("typedPythonBridge") + inputs.file(typedPythonEmitCManifest).withPropertyName("typedPythonEmitCManifest") systemProperty( "typedpython.bridge.properties", typedPythonBridgeDir.get().file("bridge.properties").asFile.absolutePath, ) + systemProperty( + "typedpython.emitC.manifest", + typedPythonEmitCManifest.get().asFile.absolutePath, + ) } else { systemProperty("typedpython.bridge.skip", typedPythonSkipReason) } diff --git a/python-multiplatform-ksp/fixtures/app/src/desktopTest/kotlin/fixture/app/TypedPythonKotlinCallTest.kt b/python-multiplatform-ksp/fixtures/app/src/desktopTest/kotlin/fixture/app/TypedPythonKotlinCallTest.kt index 101d0d7a6..4a45e5e62 100644 --- a/python-multiplatform-ksp/fixtures/app/src/desktopTest/kotlin/fixture/app/TypedPythonKotlinCallTest.kt +++ b/python-multiplatform-ksp/fixtures/app/src/desktopTest/kotlin/fixture/app/TypedPythonKotlinCallTest.kt @@ -291,6 +291,19 @@ class TypedPythonKotlinCallTest { ) } + @Test + fun emitTypedPythonCTaskWritesManifestForTpKotlinBridge() { + val manifestPath = System.getProperty("typedpython.emitC.manifest") + assertTrue(manifestPath != null, "system property typedpython.emitC.manifest is unset") + val manifestFile = File(manifestPath!!) + assertTrue(manifestFile.isFile, "emitTypedPythonC ran but $manifestFile does not exist") + val text = manifestFile.readText() + assertTrue("\"format\": \"typedpython-emit-c\"" in text, text) + assertTrue("\"moduleName\": \"tp_kotlin_bridge\"" in text, text) + assertTrue("tp_kotlin_bridge.c" in text, text) + assertTrue("\"notNativeModules\": []" in text, text) + } + /** * The watched reference counts once the JVM has collected the `PyObject` wrappers the binder made * and their cleaners have handed the references back. From e01d4223e6e0d5e44e7140f855464ea0b060835d Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 7 Nov 2026 10:42:26 +0900 Subject: [PATCH 24/29] Publish the library, KSP processor and Gradle plugin to Maven Central (#375) Shared gradle/central-publishing.gradle.kts adds POM fields, javadoc and sources jars, in-memory signing when a key is present, and a centralStaging repository. maven-central.yml bundles on macOS and uploads to the Central Portal on a version tag; a pull request touching the setup runs a dry run signed with a throwaway key. desktopNativeMain isDirectory uses opendir instead of stat: st_mode differs in width between macOS and Linux and the shared metadata compilation that publishing runs rejected it. --- .github/scripts/release/central-bundle.sh | 75 +++++++++ .github/scripts/release/central-upload.sh | 68 ++++++++ .github/workflows/maven-central.yml | 140 +++++++++++++++++ build.gradle.kts | 13 ++ docs/guide/getting-started.html | 19 ++- docs/roadmap/ROADMAP.md | 15 +- gradle/central-publishing.gradle.kts | 148 ++++++++++++++++++ .../build.gradle.kts | 2 + python-multiplatform-ksp/build.gradle.kts | 2 + python-multiplatform/build.gradle.kts | 42 +---- .../env/PackagedPythonHome.desktopNative.kt | 22 +-- 11 files changed, 491 insertions(+), 55 deletions(-) create mode 100755 .github/scripts/release/central-bundle.sh create mode 100755 .github/scripts/release/central-upload.sh create mode 100644 .github/workflows/maven-central.yml create mode 100644 gradle/central-publishing.gradle.kts diff --git a/.github/scripts/release/central-bundle.sh b/.github/scripts/release/central-bundle.sh new file mode 100755 index 000000000..55e8f8f2a --- /dev/null +++ b/.github/scripts/release/central-bundle.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# Builds the Maven Central deployment of python-multiplatform: every component, signed, in one zip. +# .github/scripts/release/central-bundle.sh [--throwaway-key] +# Run from the repository root. Reads MAVEN_GPG_PRIVATE_KEY and MAVEN_GPG_PASSPHRASE (Gradle signs +# in memory, gradle/central-publishing.gradle.kts). --throwaway-key makes a key for this run only, +# for a dry run that must prove signing works without the release key. Without either, the bundle is +# unsigned and the script says so; Central would refuse it. +# Writes build/central-staging (a Maven layout) and ; uploads nothing, see central-upload.sh. +set -euo pipefail +out="${1:?usage: central-bundle.sh [--throwaway-key]}" +mode="${2:-}" +root="$(pwd)" +staging="$root/build/central-staging" +version="$(sed -nE 's/^pythonVersion=(.*)$/\1/p' gradle.properties)-$(sed -nE 's/^libraryVersionSuffix=(.*)$/\1/p' gradle.properties)" + +# Not under the repository: gpg-agent's socket path must stay under about 100 characters. +verify_home="$(mktemp -d)" +chmod 700 "$verify_home" +trap 'rm -rf "$verify_home"' EXIT + +if [[ "$mode" == "--throwaway-key" ]]; then + GNUPGHOME="$verify_home" gpg --batch --quiet --passphrase '' --quick-generate-key \ + "python-multiplatform dry run " rsa3072 sign 1d + MAVEN_GPG_PRIVATE_KEY="$(GNUPGHOME="$verify_home" gpg --batch --pinentry-mode loopback --passphrase '' --armor --export-secret-keys)" + export MAVEN_GPG_PRIVATE_KEY MAVEN_GPG_PASSPHRASE="" +elif [[ -n "${MAVEN_GPG_PRIVATE_KEY:-}" ]]; then + GNUPGHOME="$verify_home" gpg --batch --quiet --import <<< "$MAVEN_GPG_PRIVATE_KEY" +fi +signed=false +[[ -n "${MAVEN_GPG_PRIVATE_KEY:-}" ]] && signed=true + +mkdir -p "$root/.cache" +rm -rf "$staging" +# Gradle reads the key from the environment; the plugin build is a separate build but inherits it. +./gradlew --console=plain publishAllToCentralStaging 2>&1 | tee "$root/.cache/central-gradle.log" || { echo "error: gradle failed (pipe status ${PIPESTATUS[0]})" >&2; exit 1; } + +# What Central checks first: every module has pom + jar/klib/zip, sources and javadoc where it is a +# jar publication, and (signed) a good .asc beside every file. +fail=0 +while IFS= read -r pom; do + dir="$(dirname "$pom")"; base="$(basename "$pom" .pom)" + ver="$(basename "$dir")"; art="$(basename "$(dirname "$dir")")" + [[ "$ver" == "$version" ]] || { echo "error: $pom is not at $version" >&2; fail=1; } + grep -q 'https://github.com/thisisthepy/python-multiplatform' "$pom" || { echo "error: $pom has no project url" >&2; fail=1; } + grep -q '' "$pom" && grep -q '' "$pom" && grep -q '' "$pom" || { echo "error: $pom lacks licenses, developers or scm" >&2; fail=1; } + if ! grep -q 'pom' "$pom"; then + [[ -e "$dir/$base-javadoc.jar" ]] || { echo "error: $art has no javadoc jar" >&2; fail=1; } + [[ -e "$dir/$base-sources.jar" ]] || { echo "error: $art has no sources jar" >&2; fail=1; } + fi +done < <(find "$staging" -name '*.pom') +if $signed; then + while IFS= read -r f; do + case "$f" in *.asc|*.md5|*.sha1|*.sha256|*.sha512) continue ;; esac + [[ -e "$f.asc" ]] || { echo "error: $f is not signed" >&2; fail=1; continue; } + GNUPGHOME="$verify_home" gpg --batch --quiet --verify "$f.asc" "$f" 2>/dev/null || { echo "error: bad signature on $f" >&2; fail=1; } + done < <(find "$staging" -type f -not -name 'maven-metadata*') +else + echo "warning: no signing key, the bundle is unsigned and Central will refuse it" >&2 +fi +(( fail == 0 )) || exit 1 + +echo "== left out of the bundle (not built on this machine)" +grep -h "CENTRAL-SKIPPED" "$root/.cache/central-gradle.log" || echo "(none)" +echo "== coordinates in the bundle" +find "$staging" -name '*.pom' | while IFS= read -r pom; do + rel="${pom#"$staging"/}"; ver="$(basename "$(dirname "$rel")")"; art="$(basename "$(dirname "$(dirname "$rel")")")" + grp="$(dirname "$(dirname "$(dirname "$rel")")" | tr / .)" + echo "$grp:$art:$ver" +done | sort + +mkdir -p "$(dirname "$out")" +out="$(cd "$(dirname "$out")" && pwd)/$(basename "$out")" +rm -f "$out" +(cd "$staging" && zip -qr "$out" . -x 'maven-metadata*' '*/maven-metadata*') +echo "== bundle $out ($(find "$staging" -type f | wc -l | tr -d ' ') files)" diff --git a/.github/scripts/release/central-upload.sh b/.github/scripts/release/central-upload.sh new file mode 100755 index 000000000..39c3a85e1 --- /dev/null +++ b/.github/scripts/release/central-upload.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Uploads a bundle made by central_bundle.py to the Sonatype Central Portal and waits for its +# validation. +# +# Usage: central-upload.sh [USER_MANAGED|AUTOMATIC] +# +# Reads CENTRAL_TOKEN_USERNAME and CENTRAL_TOKEN_PASSWORD, the two halves of a Central Portal +# user token (central.sonatype.com, View Account, Generate User Token). The Portal API takes +# them as a bearer token of base64(":"). +# +# USER_MANAGED (the default) stops after validation: the deployment waits in the Portal under +# Deployments until someone presses Publish or Drop, so a person looks at it before it becomes +# permanent. AUTOMATIC publishes as soon as validation passes. Either way nothing published to +# Central can be changed or removed afterwards. +# +# Exits 0 when the deployment is VALIDATED (USER_MANAGED) or PUBLISHING/PUBLISHED (AUTOMATIC), +# 1 when it FAILED, printing the Portal's errors, and 2 on a usage or network error. +set -euo pipefail + +API="https://central.sonatype.com/api/v1/publisher" + +bundle="${1:?usage: central-upload.sh [USER_MANAGED|AUTOMATIC]}" +name="${2:?usage: central-upload.sh [USER_MANAGED|AUTOMATIC]}" +type="${3:-USER_MANAGED}" +case "$type" in USER_MANAGED|AUTOMATIC) ;; *) echo "error: publishing type '$type'" >&2; exit 2 ;; esac +[[ -s "$bundle" ]] || { echo "error: no bundle at $bundle" >&2; exit 2; } +[[ -n "${CENTRAL_TOKEN_USERNAME:-}" && -n "${CENTRAL_TOKEN_PASSWORD:-}" ]] || { + echo "error: CENTRAL_TOKEN_USERNAME and CENTRAL_TOKEN_PASSWORD must hold a Central Portal user token" >&2 + exit 2 +} + +token="$(printf '%s:%s' "$CENTRAL_TOKEN_USERNAME" "$CENTRAL_TOKEN_PASSWORD" | base64 | tr -d '\n')" +auth="Authorization: Bearer $token" + +encoded_name="$(python3 -I -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.argv[1]))' "$name")" +echo "==> uploading $bundle as '$name' ($type)" +id="$(curl --fail-with-body --silent --show-error --retry 3 \ + --header "$auth" \ + --form "bundle=@$bundle" \ + "$API/upload?publishingType=$type&name=$encoded_name")" +echo "deployment $id" + +# Validation of a large bundle takes minutes; give it an hour. +for _ in $(seq 1 120); do + status="$(curl --fail-with-body --silent --show-error --retry 3 --request POST \ + --header "$auth" "$API/status?id=$id")" + state="$(python3 -I -c 'import json, sys; print(json.load(sys.stdin)["deploymentState"])' <<< "$status")" + echo "state $state" + case "$state" in + VALIDATED) + if [[ "$type" == USER_MANAGED ]]; then + echo "validated; publish or drop it at https://central.sonatype.com/publishing/deployments" + exit 0 + fi + ;; + PUBLISHING|PUBLISHED) + echo "publishing; it appears on https://repo1.maven.org/maven2/org/thisisthepy/ within about 30 minutes" + exit 0 + ;; + FAILED) + python3 -I -c 'import json, sys; print(json.dumps(json.load(sys.stdin).get("errors"), indent=2))' <<< "$status" >&2 + exit 1 + ;; + esac + sleep 30 +done +echo "error: deployment $id is still $state after an hour; see https://central.sonatype.com/publishing/deployments" >&2 +exit 2 diff --git a/.github/workflows/maven-central.yml b/.github/workflows/maven-central.yml new file mode 100644 index 000000000..e79994d0b --- /dev/null +++ b/.github/workflows/maven-central.yml @@ -0,0 +1,140 @@ +name: Maven Central + +# Publishes python-multiplatform (library for every target, KSP processor, Gradle plugin and its +# plugin marker) to Maven Central through the Sonatype Central Portal, as ONE deployment. +# +# - A version tag (v-, the tag desktop-release.yml also reads) +# builds, signs with the release key and uploads. +# - A manual run is a dry run unless "dry_run" is turned off: it builds and signs with a key made in +# the job and thrown away, checks the bundle and keeps it as a workflow artifact. Nothing is +# uploaded. With dry_run off, pick the tag to publish in "tag". +# +# macOS is required: the Apple targets (iosArm64, iosSimulatorArm64, iosX64, macosArm64) only build +# there. Secrets read by a real upload, nothing else reads them: +# MAVEN_GPG_PRIVATE_KEY signing key, ASCII armoured (gpg --armor --export-secret-keys) +# MAVEN_GPG_PASSPHRASE its passphrase +# CENTRAL_TOKEN_USERNAME Central Portal user token, username half +# CENTRAL_TOKEN_PASSWORD Central Portal user token, password half +# The upload runs in the environment "maven-central", where required reviewers can be set. + +on: + push: + tags: + - 'v*' + # A pull request that touches the publishing setup runs the dry run, so the bundle is checked + # before the first tag depends on it. + pull_request: + paths: + - '.github/workflows/maven-central.yml' + - '.github/scripts/release/central-*.sh' + - 'gradle/central-publishing.gradle.kts' + workflow_dispatch: + inputs: + dry_run: + description: 'Build and sign with a throwaway key, upload nothing' + type: boolean + default: true + tag: + description: 'Tag to publish when dry_run is off (e.g. v3.15.0-alpha02); ignored for a dry run' + required: false + default: '' + publishing_type: + description: 'USER_MANAGED waits in the Central Portal for someone to press Publish; AUTOMATIC publishes once validated' + type: choice + options: [USER_MANAGED, AUTOMATIC] + default: USER_MANAGED + +permissions: + contents: read + +concurrency: + group: maven-central-${{ github.event.inputs.tag || github.ref }} + cancel-in-progress: false + +jobs: + bundle: + name: Build and sign the bundle (macOS) + runs-on: macos-latest + timeout-minutes: 180 + env: + DRY_RUN: ${{ github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && inputs.dry_run) }} + TAG: ${{ github.event.inputs.tag || github.ref_name }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ (github.event_name == 'push' || inputs.dry_run == false) && (github.event.inputs.tag || github.ref) || github.ref }} + # The tag's own gradle.properties decides the version, not the branch it was cut from. + + - name: Setup Java + uses: actions/setup-java@v4 + with: + java-version: '21' + distribution: 'zulu' + + - name: The tag names this build's version + if: env.DRY_RUN != 'true' + run: | + set -euo pipefail + version="$(sed -nE 's/^pythonVersion=(.*)$/\1/p' gradle.properties)-$(sed -nE 's/^libraryVersionSuffix=(.*)$/\1/p' gradle.properties)" + test "$TAG" = "v$version" || { echo "error: tag $TAG does not match version $version" >&2; exit 1; } + echo "VERSION=$version" >> "$GITHUB_ENV" + + - name: Build and sign the bundle (dry run, throwaway key) + if: env.DRY_RUN == 'true' + run: .github/scripts/release/central-bundle.sh "$RUNNER_TEMP/central-bundle.zip" --throwaway-key + + - name: Build and sign the bundle (release key) + if: env.DRY_RUN != 'true' + env: + MAVEN_GPG_PRIVATE_KEY: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }} + run: | + set -euo pipefail + [[ -n "$MAVEN_GPG_PRIVATE_KEY" ]] || { echo "::error::the MAVEN_GPG_PRIVATE_KEY secret is not set"; exit 1; } + .github/scripts/release/central-bundle.sh "$RUNNER_TEMP/central-bundle.zip" + + - name: Keep the bundle + uses: actions/upload-artifact@v4 + with: + name: central-bundle + path: ${{ runner.temp }}/central-bundle.zip + retention-days: 14 + + + upload: + name: Upload to the Central Portal + needs: bundle + if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.dry_run == false) + runs-on: ubuntu-24.04 + timeout-minutes: 90 + # Required reviewers can be set on this environment, so even a pushed tag waits for a person + # before anything reaches Central. A dry run never gets here. + environment: maven-central + env: + TAG: ${{ github.event.inputs.tag || github.ref_name }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.tag || github.ref }} + sparse-checkout: | + .github/scripts/release + gradle.properties + sparse-checkout-cone-mode: false + + - uses: actions/download-artifact@v4 + with: + name: central-bundle + path: ${{ runner.temp }} + + - name: Upload + env: + CENTRAL_TOKEN_USERNAME: ${{ secrets.CENTRAL_TOKEN_USERNAME }} + CENTRAL_TOKEN_PASSWORD: ${{ secrets.CENTRAL_TOKEN_PASSWORD }} + TYPE: ${{ inputs.publishing_type || 'USER_MANAGED' }} + run: | + set -euo pipefail + version="$(sed -nE 's/^pythonVersion=(.*)$/\1/p' gradle.properties)-$(sed -nE 's/^libraryVersionSuffix=(.*)$/\1/p' gradle.properties)" + .github/scripts/release/central-upload.sh "$RUNNER_TEMP/central-bundle.zip" \ + "python-multiplatform $version" "$TYPE" diff --git a/build.gradle.kts b/build.gradle.kts index 63c01e66a..cad457f0d 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -45,6 +45,19 @@ tasks.register("publishAllToMavenLocal") { dependsOn(gradle.includedBuild("python-multiplatform-gradle-plugin").task(":publishToMavenLocal")) } +// Maven Central (ROADMAP 15.1): one entry point that fills build/central-staging with all three +// components. Signed when MAVEN_GPG_PRIVATE_KEY is set (gradle/central-publishing.gradle.kts). +// It does not clean the directory (the plugin build is another build, so ordering cannot reach +// it): .github/scripts/release/central-bundle.sh empties it first. +tasks.register("publishAllToCentralStaging") { + group = "publishing" + description = "Publishes python-multiplatform, python-multiplatform-ksp and " + + "python-multiplatform-gradle-plugin (with its plugin marker) to build/central-staging." + dependsOn(":python-multiplatform:publishAllToCentralStaging") + dependsOn(":python-multiplatform-ksp:publishAllToCentralStaging") + dependsOn(gradle.includedBuild("python-multiplatform-gradle-plugin").task(":publishAllToCentralStaging")) +} + // The build's single Kotlin/Wasm npm lock (one workspace for every wasm module) lives next to the // version catalog instead of a root `kotlin-js-store/` (issue #110). KGP only allows this on the // root project; the `*Property` forms are the non-deprecated API from Kotlin 2.4 on. Mismatch diff --git a/docs/guide/getting-started.html b/docs/guide/getting-started.html index 61f3d1b07..dd10ff835 100644 --- a/docs/guide/getting-started.html +++ b/docs/guide/getting-started.html @@ -69,7 +69,24 @@

03Your first call

There is no close() in that code, and none is missing: every wrapper releases its Python reference when the Kotlin object is collected.위 코드에는 close() 가 없고, 빠진 것도 아니다. 모든 래퍼는 Kotlin 객체가 수거될 때 Python 참조를 놓는다.

-

04Verify a change변경 검증하기

+

04Use it from Maven CentralMaven Central 에서 쓰기

+ 
// settings.gradle.kts
+pluginManagement { repositories { mavenCentral(); gradlePluginPortal() } }
+dependencyResolutionManagement { repositories { mavenCentral() } }
+
+// build.gradle.kts of a Kotlin Multiplatform module
+plugins {
+    kotlin("multiplatform")
+    id("org.thisisthepy.python.multiplatform.bindings") version "3.15.0-alpha02"
+}
+kotlin {
+    sourceSets.commonMain.dependencies {
+        implementation("org.thisisthepy:python-multiplatform:3.15.0-alpha02")
+    }
+}
+

The library, the KSP processor and the Gradle plugin share one version, $pythonVersion-$libraryVersionSuffix. A version tag publishes them in one deployment through .github/workflows/maven-central.yml; until a release has been published there, resolve from mavenLocal() after ./gradlew publishAllToMavenLocal.라이브러리, KSP 프로세서, Gradle 플러그인은 $pythonVersion-$libraryVersionSuffix 라는 한 버전을 쓴다. 버전 태그를 달면 .github/workflows/maven-central.yml 이 셋을 한 번의 배포로 올린다. 릴리스가 올라가기 전에는 ./gradlew publishAllToMavenLocal 뒤에 mavenLocal() 에서 가져온다.

+ +

05Verify a change변경 검증하기

./gradlew :python-multiplatform:compileKotlinAndroidNativeArm64 --console=plain > .tmp/build.log 2>&1; echo "EXIT=$?"
 ./gradlew :python-multiplatform:desktopTest --rerun --console=plain > .tmp/test.log 2>&1; echo "EXIT=$?"
 ./gradlew :python-multiplatform-ksp:fixtures:app:desktopTest --rerun
diff --git a/docs/roadmap/ROADMAP.md b/docs/roadmap/ROADMAP.md
index bd328f959..029df0d52 100644
--- a/docs/roadmap/ROADMAP.md
+++ b/docs/roadmap/ROADMAP.md
@@ -300,10 +300,17 @@ stripped), the root metadata jar is 382 KB, `PythonBootstrap` stages the Android
 desktop stdlib (`GP/test/kotlin/python/multiplatform/gradle/PythonHomeStagingTest.kt`, SPEC L-4).
 Old §15a, §15b, §15d, §15f, §15g, §15h are closed; §15e item 4 is closed by §15h.
 
-- **15.1 Nothing is published to a remote repository, open.** No publishing repository or signing is
-  configured (`signing { }` is commented out in `python-multiplatform/build.gradle.kts`), and
-  `repo1.maven.org/maven2/io/github/thisisthepy/python-multiplatform/` returns 404 (2026-10-03).
-  Every consumer verification so far used `mavenLocal()`. 10.1 depends on this or on a release asset.
+- **15.1 Maven Central publishing, implemented, first upload pending (#375).** All three components
+  (the library for every target, the KSP processor, the Gradle plugin with its plugin marker) carry
+  Central POM metadata, sources and javadoc jars and in-memory GPG signing
+  (`gradle/central-publishing.gradle.kts`, signing only when `MAVEN_GPG_PRIVATE_KEY` is set, so
+  `publishToMavenLocal` still works without keys). `./gradlew publishAllToCentralStaging` fills
+  `build/central-staging`; `.github/scripts/release/central-bundle.sh` signs, checks and zips it, and
+  `.github/workflows/maven-central.yml` (version tag, or manual with a `dry_run` input) builds on macOS and
+  uploads one deployment to the Central Portal (`central-upload.sh`, environment `maven-central`). The
+  dry run signs with a throwaway key and uploads nothing. Open: the first real upload (needs the
+  secrets, only CI can show Central accepting the deployment), and `repo1.maven.org/maven2/org/thisisthepy/`
+  does not list the artifacts until then. 10.1 depends on this or on a release asset.
 - **15.2 The library, KSP processor and plugin publish under one version, done (#70).** Each build
   derives it from `gradle.properties` (`$pythonVersion-$libraryVersionSuffix`, a `-P` override included);
   the root build fails when a project's version differs, and `PublishedVersionTest` pins the processor
diff --git a/gradle/central-publishing.gradle.kts b/gradle/central-publishing.gradle.kts
new file mode 100644
index 000000000..59b6db55d
--- /dev/null
+++ b/gradle/central-publishing.gradle.kts
@@ -0,0 +1,148 @@
+// Maven Central publishing, applied by the three published components (the library, the KSP
+// processor and the Gradle plugin, which is an included build):
+//
+//     apply(from = rootProject.file("gradle/central-publishing.gradle.kts"))   // root build
+//     apply(from = rootDir.resolve("../gradle/central-publishing.gradle.kts")) // plugin build
+//
+// What it adds to every Maven publication of the project (ROADMAP 15.1):
+//   * the POM fields Central requires: name, description, url, license, developers, scm;
+//   * an empty javadoc jar and, where the publication has none, an empty sources jar (Central
+//     refuses a jar publication without both; pom-only plugin markers need neither);
+//   * a local `centralStaging` repository, a plain Maven layout under /build/central-staging,
+//     which `.github/scripts/release/central-bundle.sh` zips for the Central Portal upload API.
+//     Every component writes the same directory, so one deployment carries all of them;
+//   * in-memory PGP signing of every publication, only when a key is given
+//     (MAVEN_GPG_PRIVATE_KEY / MAVEN_GPG_PASSPHRASE in the environment, or the Gradle properties
+//     signingInMemoryKey / signingInMemoryKeyPassword). Without a key nothing is signed, so
+//     `publishToMavenLocal` keeps working on a machine that has none.
+// Nothing here uploads anything; the upload is .github/scripts/release/central-upload.sh.
+
+import org.gradle.api.publish.maven.MavenPublication
+import org.gradle.api.publish.maven.tasks.AbstractPublishToMaven
+import org.gradle.api.publish.maven.tasks.PublishToMavenLocal
+import org.gradle.api.publish.maven.tasks.PublishToMavenRepository
+import org.gradle.plugins.signing.Sign
+import org.gradle.plugins.signing.SigningExtension
+
+val repoRoot: File = if (rootDir.resolve("gradle.properties").isFile && rootDir.resolve("gradle").isDirectory) {
+    rootDir
+} else {
+    rootDir.parentFile
+}
+val stagingDir: File = (findProperty("centralStagingDir") as String?)?.let(::File)
+    ?: repoRoot.resolve("build/central-staging")
+
+val signingKey: String? = (findProperty("signingInMemoryKey") as String?)
+    ?: System.getenv("MAVEN_GPG_PRIVATE_KEY")?.takeIf { it.isNotBlank() }
+val signingPassword: String = (findProperty("signingInMemoryKeyPassword") as String?)
+    ?: System.getenv("MAVEN_GPG_PASSPHRASE")
+    ?: ""
+
+val projectUrl = "https://github.com/thisisthepy/python-multiplatform"
+
+val emptyJar = { classifier: String ->
+    tasks.register("central${classifier.replaceFirstChar { it.uppercase() }}Jar") {
+        archiveClassifier.set(classifier)
+        destinationDirectory.set(layout.buildDirectory.dir("central-empty-jars"))
+        // Central only checks that the jar exists; the KDoc lives in the sources jar.
+        from(rootProject.file("LICENSE")) { into("META-INF") }
+    }
+}
+
+fun publicationIsBuilt(publication: org.gradle.api.publish.Publication, logger: org.gradle.api.logging.Logger): Boolean {
+    publication as MavenPublication
+    val missing = publication.artifacts.filter { !it.file.exists() }
+    if (missing.isNotEmpty()) {
+        logger.warn("CENTRAL-SKIPPED ${publication.groupId}:${publication.artifactId}: not built here: " +
+            missing.joinToString { it.file.name })
+    }
+    return missing.isEmpty()
+}
+
+pluginManager.withPlugin("maven-publish") {
+    apply(plugin = "signing")
+
+    val publishing = extensions.getByType()
+    publishing.repositories.maven {
+        name = "centralStaging"
+        url = stagingDir.toURI()
+    }
+
+    val javadocJar = emptyJar("javadoc")
+    val sourcesJar = emptyJar("sources")
+
+    publishing.publications.withType().configureEach {
+        val publication = this
+        // A plugin marker is a pom-only publication: nothing to attach.
+        val markerOnly = publication.name.endsWith("PluginMarkerMaven")
+        pom {
+            name.set(provider { publication.artifactId })
+            description.set(
+                "Embeds CPython in Kotlin Multiplatform and provides Kotlin <-> Python interoperability " +
+                    "(${publication.artifactId})."
+            )
+            url.set(projectUrl)
+            licenses {
+                license {
+                    name.set("The Apache License, Version 2.0")
+                    url.set("https://www.apache.org/licenses/LICENSE-2.0.txt")
+                }
+            }
+            developers {
+                developer {
+                    id.set("thisisthepy")
+                    name.set("thisisthepy")
+                    email.set("thisisthepy@gmail.com")
+                }
+            }
+            scm {
+                connection.set("scm:git:https://github.com/thisisthepy/python-multiplatform.git")
+                developerConnection.set("scm:git:ssh://git@github.com/thisisthepy/python-multiplatform.git")
+                url.set(projectUrl)
+            }
+        }
+        if (!markerOnly) {
+            // Attached once every plugin has added its own artifacts (KGP adds the sources jars of
+            // its targets late); a classifier already present is left alone.
+            gradle.projectsEvaluated {
+                val classifiers = publication.artifacts.mapNotNull { it.classifier }.toSet()
+                if ("javadoc" !in classifiers) publication.artifact(javadocJar)
+                if ("sources" !in classifiers) publication.artifact(sourcesJar)
+            }
+        }
+    }
+
+    if (signingKey != null) {
+        extensions.configure {
+            useInMemoryPgpKeys(signingKey, signingPassword)
+            sign(publishing.publications)
+        }
+        // KMP publications are created late and read the .asc files without declaring the tasks
+        // that write them; make every publish task wait for every signing task.
+        tasks.withType().configureEach {
+            dependsOn(tasks.withType())
+        }
+    }
+
+    // A publication whose file is not built on this machine is left out of the staging, loudly,
+    // rather than failing the whole deployment. Today that is only `wasmRuntime`: its zip needs a
+    // local Emscripten CPython build (-PwasmPythonDir) that CI does not make (ROADMAP 10.x), so
+    // `central-bundle.sh` lists what was left out and a release owner decides whether that is fine.
+    tasks.withType().matching { it.name.endsWith("ToCentralStagingRepository") }
+        .configureEach {
+            val task = this
+            onlyIf { publicationIsBuilt(task.publication, task.logger) }
+        }
+    // The same for mavenLocal, so `publishAllToMavenLocal` works in a checkout without the wasm build.
+    tasks.withType().configureEach {
+        val task = this
+        onlyIf { publicationIsBuilt(task.publication, task.logger) }
+    }
+
+    // One task per component that fills the staging directory with every publication.
+    tasks.register("publishAllToCentralStaging") {
+        group = "publishing"
+        description = "Publishes every publication of this project to $stagingDir (signed when a key is given)."
+        dependsOn(tasks.withType().matching { it.name.endsWith("ToCentralStagingRepository") })
+    }
+}
diff --git a/python-multiplatform-gradle-plugin/build.gradle.kts b/python-multiplatform-gradle-plugin/build.gradle.kts
index 384ce7ce4..89fe990e6 100644
--- a/python-multiplatform-gradle-plugin/build.gradle.kts
+++ b/python-multiplatform-gradle-plugin/build.gradle.kts
@@ -418,3 +418,5 @@ tasks.test {
     inputs.files(walkerFixtureKlib).withPropertyName("walkerFixtureKlib").withPathSensitivity(PathSensitivity.NAME_ONLY)
     systemProperty("python.multiplatform.walkerFixtureKlib", walkerFixtureKlib.singleFile.absolutePath)
 }
+
+apply(from = rootDir.resolve("../gradle/central-publishing.gradle.kts"))
diff --git a/python-multiplatform-ksp/build.gradle.kts b/python-multiplatform-ksp/build.gradle.kts
index 67b3f22ac..c50576ae8 100644
--- a/python-multiplatform-ksp/build.gradle.kts
+++ b/python-multiplatform-ksp/build.gradle.kts
@@ -48,3 +48,5 @@ publishing {
         }
     }
 }
+
+apply(from = rootProject.file("gradle/central-publishing.gradle.kts"))
diff --git a/python-multiplatform/build.gradle.kts b/python-multiplatform/build.gradle.kts
index bb3ac4824..30a722feb 100644
--- a/python-multiplatform/build.gradle.kts
+++ b/python-multiplatform/build.gradle.kts
@@ -2789,47 +2789,11 @@ publishing {
             artifact(wasmBrowserRuntimeZip)
         }
     }
-    publications.withType().configureEach {
-        //artifact(javadocJar)
-        groupId = groupId
-        artifactId = artifactId
-        version = version
-
-        pom {
-            name.set("python-multiplatform")
-            description.set("A multiplatform solution to use Python with Kotlin interoperably.")
-            url.set("https://github.com/thisisthepy/python-multiplatform-mobile")
-            licenses {
-                license {
-                    name.set("The Apache License, Version 2.0")
-                    url.set("https://www.apache.org/licenses/LICENSE-2.0.txt")
-                }
-            }
-            developers {
-                developer {
-                    id.set("thisisthepy")
-                    name.set("thisisthepy")
-                    email.set("thisisthepy@gmail.com")
-                }
-            }
-            scm {
-                connection.set("scm:git:github.com/thisisthepy/python-multiplatform-mobile.git")
-                developerConnection.set("scm:git:ssh://github.com/thisisthepy/python-multiplatform-mobile.git")
-                url.set("https://github.com/thisisthepy/python-multiplatform-mobile")
-            }
-        }
-    }
 }
 
-//signing {
-//    useInMemoryPgpKeys(
-//        rootProject.extra["signing_key_id"] as String,
-//        rootProject.extra["signing_secret_key"] as String,
-//        rootProject.extra["signing_password"] as String
-//    )
-//    sign(publishing.publications)
-//}
-
+// POM metadata, the sources and javadoc jars, the centralStaging repository and the in-memory GPG
+// signing (only when MAVEN_GPG_PRIVATE_KEY is set) for Maven Central: ROADMAP 15.1.
+apply(from = rootProject.file("gradle/central-publishing.gradle.kts"))
 
 fun downloadPythonBuilds() {
     // Moved to task `downloadAllPythonBuilds`
diff --git a/python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/env/PackagedPythonHome.desktopNative.kt b/python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/env/PackagedPythonHome.desktopNative.kt
index a338dc59e..ec3d202f4 100644
--- a/python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/env/PackagedPythonHome.desktopNative.kt
+++ b/python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/env/PackagedPythonHome.desktopNative.kt
@@ -1,12 +1,8 @@
 package python.multiplatform.env
 
 import kotlinx.cinterop.ExperimentalForeignApi
-import kotlinx.cinterop.alloc
-import kotlinx.cinterop.memScoped
-import kotlinx.cinterop.ptr
-import platform.posix.S_IFDIR
-import platform.posix.S_IFMT
-import platform.posix.stat
+import platform.posix.closedir
+import platform.posix.opendir
 
 /**
  * The directory containing the running executable, or null when the OS cannot report it.
@@ -67,12 +63,16 @@ internal object DesktopNativePythonHome {
         return null
     }
 
-    /** `stat(2)`'s answer: a directory exists at [path] (a regular file there is not one). */
+    /**
+     * A directory exists at [path] (a regular file there is not one): `opendir(3)` succeeds only on
+     * a directory. Not `stat(2)`: `st_mode` is 16 bits on macOS and 32 on Linux, and the metadata
+     * compilation of this shared source set, which publishing runs, rejects that mismatch.
+     */
     @OptIn(ExperimentalForeignApi::class)
-    internal fun isDirectory(path: String): Boolean = memScoped {
-        val info = alloc()
-        if (stat(path, info.ptr) != 0) return false
-        (info.st_mode.toInt() and S_IFMT) == S_IFDIR
+    internal fun isDirectory(path: String): Boolean {
+        val dir = opendir(path) ?: return false
+        closedir(dir)
+        return true
     }
 }
 

From 8fa5a33ddb8196b3759c4494565337b208d6835d Mon Sep 17 00:00:00 2001
From: BREW <81044362+b-re-w@users.noreply.github.com>
Date: Sat, 7 Nov 2026 11:00:38 +0900
Subject: [PATCH 25/29] maven-central: install uv for the typedpython-gate
 bundle (#375)

---
 .github/workflows/maven-central.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/maven-central.yml b/.github/workflows/maven-central.yml
index e79994d0b..d22eca2ee 100644
--- a/.github/workflows/maven-central.yml
+++ b/.github/workflows/maven-central.yml
@@ -72,6 +72,10 @@ jobs:
           java-version: '21'
           distribution: 'zulu'
 
+      - name: Setup uv
+        # The KSP processor's typedpython-gate bundle is built with uv build.
+        uses: astral-sh/setup-uv@v6
+
       - name: The tag names this build's version
         if: env.DRY_RUN != 'true'
         run: |

From cf909305f7d26be843230660186b31523bb2de17 Mon Sep 17 00:00:00 2001
From: BREW <81044362+b-re-w@users.noreply.github.com>
Date: Sat, 7 Nov 2026 11:24:06 +0900
Subject: [PATCH 26/29] Release: bump to 3.15.0-alpha03, the first Maven
 Central release (#377)

---
 docs/guide/getting-started.html                     | 4 ++--
 gradle.properties                                   | 2 +-
 python-multiplatform-gradle-plugin/build.gradle.kts | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/guide/getting-started.html b/docs/guide/getting-started.html
index dd10ff835..47a73045c 100644
--- a/docs/guide/getting-started.html
+++ b/docs/guide/getting-started.html
@@ -77,11 +77,11 @@ 
04Use it from Maven Central

The library, the KSP processor and the Gradle plugin share one version, $pythonVersion-$libraryVersionSuffix. A version tag publishes them in one deployment through .github/workflows/maven-central.yml; until a release has been published there, resolve from mavenLocal() after ./gradlew publishAllToMavenLocal.라이브러리, KSP 프로세서, Gradle 플러그인은 $pythonVersion-$libraryVersionSuffix 라는 한 버전을 쓴다. 버전 태그를 달면 .github/workflows/maven-central.yml 이 셋을 한 번의 배포로 올린다. 릴리스가 올라가기 전에는 ./gradlew publishAllToMavenLocal 뒤에 mavenLocal() 에서 가져온다.

diff --git a/gradle.properties b/gradle.properties index 000eaca4e..069e2f372 100644 --- a/gradle.properties +++ b/gradle.properties @@ -34,7 +34,7 @@ pythonAppleSupportBuild=b11 # Every published artifact (library, KSP processor, Gradle plugin) is versioned # "$pythonVersion-$libraryVersionSuffix"; each build derives it from these two values and fails if # its own version disagrees (#70). -libraryVersionSuffix=alpha02 +libraryVersionSuffix=alpha03 # GraalVM native-image (sample's nativeCompile task). Liberica NIK is not in a location Gradle's # toolchain auto-detection scans, so it is registered explicitly here. Adding a path only makes a diff --git a/python-multiplatform-gradle-plugin/build.gradle.kts b/python-multiplatform-gradle-plugin/build.gradle.kts index 89fe990e6..5c20b0b6f 100644 --- a/python-multiplatform-gradle-plugin/build.gradle.kts +++ b/python-multiplatform-gradle-plugin/build.gradle.kts @@ -17,7 +17,7 @@ val rootGradleProperties = Properties().apply { } fun rootProperty(name: String, fallback: String): String = findProperty(name)?.toString() ?: rootGradleProperties.getProperty(name) ?: fallback -val publishedVersion = "${rootProperty("pythonVersion", "3.15.0")}-${rootProperty("libraryVersionSuffix", "alpha02")}" +val publishedVersion = "${rootProperty("pythonVersion", "3.15.0")}-${rootProperty("libraryVersionSuffix", "alpha03")}" version = publishedVersion /** From ac5e0ddc1253680f12bdfee636ab2bb5205de651 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 7 Nov 2026 11:55:13 +0900 Subject: [PATCH 27/29] Publish python-multiplatform-compose to Maven Central; drop stale Android libs; runnable getting-started (#379) - python-multiplatform-compose applies central-publishing (common, android, desktop) and joins publishAll* tasks. - copyAndroidPythonBinaries is a Sync; the link task removes stale libmultiplatform_python*.so; central-bundle.sh fails on an AAR that carries a libpython for another major.minor. - getting-started section 04: Kotlin version, desktop target, pythonBindings role app, upcall bootstrap, compose dependency; Central instead of mavenLocal; .tmp/ -> .cache/. --- .github/scripts/release/central-bundle.sh | 8 +++ build.gradle.kts | 15 +++--- docs/guide/getting-started.html | 50 ++++++++++++++++--- gradle/central-publishing.gradle.kts | 4 +- python-multiplatform-compose/build.gradle.kts | 9 ++++ python-multiplatform/build.gradle.kts | 13 ++++- 6 files changed, 83 insertions(+), 16 deletions(-) diff --git a/.github/scripts/release/central-bundle.sh b/.github/scripts/release/central-bundle.sh index 55e8f8f2a..84e0b9585 100755 --- a/.github/scripts/release/central-bundle.sh +++ b/.github/scripts/release/central-bundle.sh @@ -48,6 +48,14 @@ while IFS= read -r pom; do [[ -e "$dir/$base-sources.jar" ]] || { echo "error: $art has no sources jar" >&2; fail=1; } fi done < <(find "$staging" -name '*.pom') +# An AAR must carry the native libraries of this pythonVersion only. A Copy-staged jniLibs tree once +# kept libpython3.14.so beside libpython3.15.so after a version bump. libpython3.so (the stable-ABI +# stub) carries no minor version and is allowed. +py_mm="$(sed -nE 's/^pythonVersion=([0-9]+\.[0-9]+).*$/\1/p' gradle.properties)" +while IFS= read -r aar; do + stale="$(unzip -Z1 "$aar" | grep -E '(^|/)lib(multiplatform_)?python[0-9]+\.[0-9]+[^/]*\.so$' | grep -vE "(^|/)lib(multiplatform_)?python${py_mm//./\\.}(t)?\.so$" || true)" + [[ -z "$stale" ]] || { echo "error: $aar carries native libraries for another python than $py_mm:" >&2; echo "$stale" >&2; fail=1; } +done < <(find "$staging" -name '*.aar') if $signed; then while IFS= read -r f; do case "$f" in *.asc|*.md5|*.sha1|*.sha256|*.sha512) continue ;; esac diff --git a/build.gradle.kts b/build.gradle.kts index cad457f0d..c1a07f833 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -34,27 +34,30 @@ gradle.projectsEvaluated { // (see `pluginManagement { includeBuild(...) }` in settings.gradle.kts), so its tasks never join // the root task graph -- `./gradlew tasks --all` from here lists none of them, and a coordinator // publishing only what the root build shows would silently skip it. This task does not change how -// any of the three components publish (each already has its own `maven-publish` config); it only -// gives the root build one entry point that reaches all three. +// any component publishes (each already has its own `maven-publish` config); it only gives the root +// build one entry point that reaches all of them (the Compose host included). tasks.register("publishAllToMavenLocal") { group = "publishing" - description = "Publishes python-multiplatform, python-multiplatform-ksp, and " + + description = "Publishes python-multiplatform, python-multiplatform-ksp, python-multiplatform-compose and " + "python-multiplatform-gradle-plugin (including its plugin-marker publication) to mavenLocal()." dependsOn(":python-multiplatform:publishToMavenLocal") dependsOn(":python-multiplatform-ksp:publishToMavenLocal") + dependsOn(":python-multiplatform-compose:publishToMavenLocal") dependsOn(gradle.includedBuild("python-multiplatform-gradle-plugin").task(":publishToMavenLocal")) } -// Maven Central (ROADMAP 15.1): one entry point that fills build/central-staging with all three -// components. Signed when MAVEN_GPG_PRIVATE_KEY is set (gradle/central-publishing.gradle.kts). +// Maven Central (ROADMAP 15.1): one entry point that fills build/central-staging with every +// component (library, KSP processor, Compose host, Gradle plugin). Signed when MAVEN_GPG_PRIVATE_KEY +// is set (gradle/central-publishing.gradle.kts). // It does not clean the directory (the plugin build is another build, so ordering cannot reach // it): .github/scripts/release/central-bundle.sh empties it first. tasks.register("publishAllToCentralStaging") { group = "publishing" - description = "Publishes python-multiplatform, python-multiplatform-ksp and " + + description = "Publishes python-multiplatform, python-multiplatform-ksp, python-multiplatform-compose and " + "python-multiplatform-gradle-plugin (with its plugin marker) to build/central-staging." dependsOn(":python-multiplatform:publishAllToCentralStaging") dependsOn(":python-multiplatform-ksp:publishAllToCentralStaging") + dependsOn(":python-multiplatform-compose:publishAllToCentralStaging") dependsOn(gradle.includedBuild("python-multiplatform-gradle-plugin").task(":publishAllToCentralStaging")) } diff --git a/docs/guide/getting-started.html b/docs/guide/getting-started.html index 47a73045c..976e6e60b 100644 --- a/docs/guide/getting-started.html +++ b/docs/guide/getting-started.html @@ -34,7 +34,7 @@
Getting started시작하기

From clone to a running sample클론부터 샘플 실행까지

-

No release is published yet, so you build from source. The sample app is the shortest path to seeing both directions work.아직 배포된 릴리스가 없으므로 소스에서 빌드한다. 두 방향이 동작하는 것을 보는 가장 짧은 길은 샘플 앱이다.

+

Releases are on Maven Central (section 04). To work on the library itself, build from source: the sample app is the shortest path to seeing both directions work.릴리스는 Maven Central 에 있다(04 절). 라이브러리 자체를 고치려면 소스에서 빌드한다. 두 방향이 동작하는 것을 보는 가장 짧은 길은 샘플 앱이다.

01Requirements준비물

@@ -70,25 +70,61 @@

03Your first call

There is no close() in that code, and none is missing: every wrapper releases its Python reference when the Kotlin object is collected.위 코드에는 close() 가 없고, 빠진 것도 아니다. 모든 래퍼는 Kotlin 객체가 수거될 때 Python 참조를 놓는다.

04Use it from Maven CentralMaven Central 에서 쓰기

+

Consumers must build with Kotlin 2.4.20-Beta2, the version the artifacts are compiled with. A desktop app that Python calls back into:소비자는 아티팩트가 컴파일된 Kotlin 2.4.20-Beta2 로 빌드해야 한다. Python 이 다시 호출하는 데스크톱 앱:

// settings.gradle.kts
 pluginManagement { repositories { mavenCentral(); gradlePluginPortal() } }
-dependencyResolutionManagement { repositories { mavenCentral() } }
+dependencyResolutionManagement { repositories { mavenCentral(); google() } }
 
-// build.gradle.kts of a Kotlin Multiplatform module
+// build.gradle.kts of a Kotlin Multiplatform app
 plugins {
-    kotlin("multiplatform")
+    kotlin("multiplatform") version "2.4.20-Beta2"
     id("org.thisisthepy.python.multiplatform.bindings") version "3.15.0-alpha03"
 }
 kotlin {
+    jvm("desktop") {
+        @OptIn(org.jetbrains.kotlin.gradle.ExperimentalKotlinGradlePluginApi::class)
+        binaries { executable { mainClass.set("app.MainKt") } }
+    }
     sourceSets.commonMain.dependencies {
         implementation("org.thisisthepy:python-multiplatform:3.15.0-alpha03")
+        // Compose hosts (PythonAppView, PythonWidget, PythonLauncher):
+        // implementation("org.thisisthepy:python-multiplatform-compose:3.15.0-alpha03")
     }
+}
+// A KMP executable is otherwise treated as a library and gets no FunctionTable.
+pythonBindings { role.set("app") }
+ 
// src/desktopMain/kotlin/app/Main.kt
+package app
+
+import python.multiplatform.ffi.Python3
+import python.multiplatform.ffi.upcall.PythonProxySource
+import python.native.ffi.UpcallStub
+
+class Greeter(val subject: String) {
+    fun greet(times: Long): String = List(times.toInt()) { "hello $subject" }.joinToString(", ")
+}
+
+fun main() {
+    Python3.initialize()
+    // Upcalls: install the generated table, bind the entry points, install the proxies.
+    python.multiplatform.generated.FunctionTable.installInto()
+    Python3.exec(
+        """
+        import ctypes
+        _pm_resolve = ctypes.CFUNCTYPE(ctypes.c_long, ctypes.c_char_p)(${UpcallStub.resolveHandleStubAddr})
+        _pm_invoke = ctypes.CFUNCTYPE(ctypes.py_object, ctypes.c_long, ctypes.py_object)(${UpcallStub.invokeWithArgsStubAddr})
+        _pm_release = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_long)(${UpcallStub.releaseObjectStubAddr})
+        _pm_cancel = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_long)(${UpcallStub.cancelCallStubAddr})
+        """.trimIndent(),
+    )
+    PythonProxySource.install()
+    Python3.exec("from app import Greeter\nprint(Greeter('Kotlin').greet(2), flush=True)")
 }
-

The library, the KSP processor and the Gradle plugin share one version, $pythonVersion-$libraryVersionSuffix. A version tag publishes them in one deployment through .github/workflows/maven-central.yml; until a release has been published there, resolve from mavenLocal() after ./gradlew publishAllToMavenLocal.라이브러리, KSP 프로세서, Gradle 플러그인은 $pythonVersion-$libraryVersionSuffix 라는 한 버전을 쓴다. 버전 태그를 달면 .github/workflows/maven-central.yml 이 셋을 한 번의 배포로 올린다. 릴리스가 올라가기 전에는 ./gradlew publishAllToMavenLocal 뒤에 mavenLocal() 에서 가져온다.

+

The library, the KSP processor, the Compose host and the Gradle plugin share one version, $pythonVersion-$libraryVersionSuffix, and are published together on Maven Central; the current release is 3.15.0-alpha03. google() is needed for Android targets.라이브러리, KSP 프로세서, Compose 호스트, Gradle 플러그인은 $pythonVersion-$libraryVersionSuffix 라는 한 버전을 쓰고 Maven Central 에 함께 올라간다. 현재 릴리스는 3.15.0-alpha03 이다. Android 타깃에는 google() 이 필요하다.

05Verify a change변경 검증하기

- 
./gradlew :python-multiplatform:compileKotlinAndroidNativeArm64 --console=plain > .tmp/build.log 2>&1; echo "EXIT=$?"
-./gradlew :python-multiplatform:desktopTest --rerun --console=plain > .tmp/test.log 2>&1; echo "EXIT=$?"
+  
./gradlew :python-multiplatform:compileKotlinAndroidNativeArm64 --console=plain > .cache/build.log 2>&1; echo "EXIT=$?"
+./gradlew :python-multiplatform:desktopTest --rerun --console=plain > .cache/test.log 2>&1; echo "EXIT=$?"
 ./gradlew :python-multiplatform-ksp:fixtures:app:desktopTest --rerun
 ./gradlew :python-multiplatform-compose:fixtures:compose:desktopTest --rerun
 ./gradlew :python-multiplatform-gradle-plugin-fixtures:artifact:desktopTest --rerun

diff --git a/gradle/central-publishing.gradle.kts b/gradle/central-publishing.gradle.kts
index 59b6db55d..c108f2bb9 100644
--- a/gradle/central-publishing.gradle.kts
+++ b/gradle/central-publishing.gradle.kts
@@ -1,5 +1,5 @@
-// Maven Central publishing, applied by the three published components (the library, the KSP
-// processor and the Gradle plugin, which is an included build):
+// Maven Central publishing, applied by the published components (the library, the KSP processor,
+// the Compose host and the Gradle plugin, which is an included build):
 //
 //     apply(from = rootProject.file("gradle/central-publishing.gradle.kts"))   // root build
 //     apply(from = rootDir.resolve("../gradle/central-publishing.gradle.kts")) // plugin build
diff --git a/python-multiplatform-compose/build.gradle.kts b/python-multiplatform-compose/build.gradle.kts
index 011f95798..353384d43 100644
--- a/python-multiplatform-compose/build.gradle.kts
+++ b/python-multiplatform-compose/build.gradle.kts
@@ -15,10 +15,14 @@ plugins {
     alias(libs.plugins.android.library)
     alias(libs.plugins.jetpack.compose)
     alias(libs.plugins.compose.compiler)
+    id("maven-publish")
 }
 
 kotlin {
     androidTarget {
+        // Without a named variant KMP creates no Android publication (ROADMAP 15d, same as the library).
+        publishLibraryVariants("release")
+
         @OptIn(ExperimentalKotlinGradlePluginApi::class)
         compilerOptions {
             jvmTarget.set(JvmTarget.JVM_17)
@@ -52,3 +56,8 @@ android {
         targetCompatibility = JavaVersion.VERSION_17
     }
 }
+
+// Published to Maven Central beside the library as org.thisisthepy:python-multiplatform-compose, at the
+// root's one version (#70): POM fields, sources/javadoc jars, centralStaging repository and in-memory
+// signing come from the shared script (ROADMAP 15.1).
+apply(from = rootProject.file("gradle/central-publishing.gradle.kts"))
diff --git a/python-multiplatform/build.gradle.kts b/python-multiplatform/build.gradle.kts
index 30a722feb..e9f99602c 100644
--- a/python-multiplatform/build.gradle.kts
+++ b/python-multiplatform/build.gradle.kts
@@ -1067,7 +1067,11 @@ kotlin {
         }
         afterEvaluate {
             val abiList = listOf("arm64-v8a", "x86_64")
-            val copyAndroidPythonBinaries by tasks.creating(Copy::class) {
+            // A Sync, not a Copy: a Copy never removes what it stopped copying, so a jniLibs tree
+            // staged under an older pythonVersion (libpython3.14.so beside libpython3.15.so) kept
+            // shipping in the AAR. Nothing else writes this directory (the link task stages into
+            // `$androidBuildDir//jniLibs`), so the Sync owns all of it.
+            val copyAndroidPythonBinaries by tasks.creating(Sync::class) {
                 dependsOn(
                     tasks.named("linkAndroidNativeArm64"),
                     tasks.named("linkAndroidNativeX64"),
@@ -1301,7 +1305,14 @@ kotlin {
                             // outputFile. That is why a changed .def or source produced an APK with
                             // a stale library and an UnsatisfiedLinkError that looked like a code
                             // bug -- the fix had to be built twice for the second run to pick it up.
+                            // A library linked under an older pythonVersion
+                            // (libmultiplatform_python3.14.so) is removed first, or it keeps
+                            // shipping in the AAR beside the current one.
                             doLast {
+                                val current = outputFile.get().name
+                                dest.listFiles { f ->
+                                    f.name.startsWith("libmultiplatform_python") && f.name.endsWith(".so") && f.name != current
+                                }?.forEach { it.delete() }
                                 copy {
                                     from(outputFile)
                                     into(dest)

From aade0f8d162bacae09e505d38283d74999cff819 Mon Sep 17 00:00:00 2001
From: BREW <81044362+b-re-w@users.noreply.github.com>
Date: Sat, 7 Nov 2026 12:25:55 +0900
Subject: [PATCH 28/29] AGENTS 7: what counts as the owner's decision (shared
 rule)

---
 AGENTS.md | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/AGENTS.md b/AGENTS.md
index 105f61706..8dda6210b 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -120,6 +120,27 @@ never force-push without the user's explicit approval.
 When a rule and backward compatibility conflict, **the rule wins.** List the callers that break and
 fix them; do not keep the forbidden thing "so nothing breaks".
 
+### What counts as the owner's decision
+
+Added after an agent turned the owner's question "should ExecuTorch be the NPU back end?" into an
+offline per-model `.pte` pipeline with its own CI, recorded it as "decided, do not re-discuss" and
+later as project intent. The owner had approved none of it (torchnative, 2026-08 to 2026-11).
+
+- **A question is not a decision.** "X를 쓰는게 좋을까?", "should we use X?" opens a decision. If the
+  owner did not answer, it stays open; never record it as settled.
+- **Approving a goal is not approving a means.** "Make the Snapdragon NPU work, go ahead" approves
+  the goal. The engine, the dependencies, the pipeline, the CI workflow and the artefact types an
+  agent picks to reach it are separate decisions.
+- **Never restate the owner's question as a different one.** When asking for a decision, quote the
+  owner's words and the constraint behind them (for example "NNAPI is deprecated, so what
+  abstraction reaches every vendor's NPU?"), and give options that answer that question.
+- **Ask before adding** a dependency, a pipeline, a CI workflow or a kind of artefact the owner did
+  not name. Show the options and wait.
+- **Docs keep provenance.** A line in INTENT, SPEC, AGENTS or a design doc that states a direction
+  cites the owner's words (date and source). Anything else is marked "agent proposal, not
+  confirmed". Never write "decided", "settled" or "do not re-discuss" for a choice the owner did not
+  make, and keep that mark when a doc is restructured.
+
 ## 8. Verification that can fail
 
 - Never read a build's exit code through a pipe (`| tail`, `| grep`). Redirect to a file, then read

From 9e0718b6801f99c494357dd8fb903bc6a73ecb0d Mon Sep 17 00:00:00 2001
From: BREW <81044362+b-re-w@users.noreply.github.com>
Date: Sun, 8 Nov 2026 03:36:24 +0900
Subject: [PATCH 29/29] AGENTS 3, 10: clean up on merge; agents only check
 compilation

---
 AGENTS.md | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/AGENTS.md b/AGENTS.md
index 8dda6210b..7b7791293 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -50,8 +50,14 @@ build caches, model weights, `node_modules`) into every worktree is how 86 workt
 - Create worktrees under `.worktrees/`.
 - **Symlink** large untracked directories from the main checkout instead of copying or rebuilding
   them. If `tools/worktree-add.sh` exists, use it, it does the linking.
-- Delete a worktree once its branch is merged: `git worktree remove .worktrees/`.
-- Periodically delete `build/` directories inside worktrees; they only grow.
+- **When a branch merges, clean up in the same step**: remove its worktree
+  (`git worktree remove .worktrees/`), delete the local and remote branch, and delete every
+  build output, cache or target directory created for that work (per-branch `build/`, a
+  `CARGO_TARGET_DIR`, a venv, a scratch directory). Work is not finished while any of these remain.
+  This was added after 32 per-agent cargo target directories (33 GB) were left behind in one
+  repository.
+- Never create a per-agent or per-branch build cache (a separate `CARGO_TARGET_DIR`, a copied
+  `.gradle`, a private venv). Use the repository's one canonical location.
 
 ## 4. Branches
 
@@ -181,10 +187,13 @@ to implement", say what you counted against.
   counts, lifetimes, class loaders) gets the strongest tier; work a test will catch can use a
   cheaper one.
 - Give every agent prompt the absolute paths it may write to, and repeat rule 2 in it.
-- **Subagents do not run heavy local builds.** Subagents write code, design, investigate, review
-  and document. Gradle builds, cargo builds, the test gate and model runs are done by the session
-  itself, one at a time on this machine, or by CI (GitHub Actions) on a pushed branch. Several
-  sessions share one machine; parallel local builds slow every one of them.
+- **Agents do not build.** Every agent, teammate sessions and subagents alike, may check that code
+  compiles and nothing more: `cargo check`, a Gradle `compile*` task, `python -m py_compile`, a
+  type checker or linter. Full builds (`cargo build`, `assemble`, wheels, native images, app
+  packages), test suites, gates, benchmarks and model runs go to CI (GitHub Actions) on a pushed
+  branch. The only local exceptions are device tests and measurements the coordinating session
+  runs itself, one at a time, and says why. Several sessions share one machine, and agent builds
+  once filled the disk and slowed every session.
 
 ---