From fb5b531c50bff13c2f672e95604cba253d72abf8 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sun, 4 Oct 2026 00:10:52 +0900 Subject: [PATCH] Bench: Add Cython (pure and typed) and numba columns to the TypedPython benchmarks --- .gitignore | 7 + .../typedpython/cython/binary_trees.pyx | 46 ++ benchmarks/typedpython/cython/build.py | 46 ++ benchmarks/typedpython/cython/fannkuch.pyx | 61 +++ benchmarks/typedpython/cython/nbody.pyx | 80 ++++ .../typedpython/cython/spectral_norm.pyx | 61 +++ benchmarks/typedpython/cython/wordfreq.pyx | 54 +++ benchmarks/typedpython/numba/binary_trees.py | 72 +++ benchmarks/typedpython/numba/fannkuch.py | 62 +++ benchmarks/typedpython/numba/nbody.py | 82 ++++ benchmarks/typedpython/numba/spectral_norm.py | 61 +++ benchmarks/typedpython/numba/wordfreq.py | 69 +++ benchmarks/typedpython/run.py | 436 +++++++++++++++--- 13 files changed, 1083 insertions(+), 54 deletions(-) create mode 100644 benchmarks/typedpython/cython/binary_trees.pyx create mode 100644 benchmarks/typedpython/cython/build.py create mode 100644 benchmarks/typedpython/cython/fannkuch.pyx create mode 100644 benchmarks/typedpython/cython/nbody.pyx create mode 100644 benchmarks/typedpython/cython/spectral_norm.pyx create mode 100644 benchmarks/typedpython/cython/wordfreq.pyx create mode 100644 benchmarks/typedpython/numba/binary_trees.py create mode 100644 benchmarks/typedpython/numba/fannkuch.py create mode 100644 benchmarks/typedpython/numba/nbody.py create mode 100644 benchmarks/typedpython/numba/spectral_norm.py create mode 100644 benchmarks/typedpython/numba/wordfreq.py diff --git a/.gitignore b/.gitignore index 670457d97..c9ffafa6b 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,10 @@ google-services.json # scratch space for agents and scripts; delete when done .tmp/ + +# TypedPython benchmark harness: uv venv, Cython/numba build output and numba caches +benchmarks/typedpython/.venv/ +benchmarks/typedpython/build/ +benchmarks/typedpython/target/ +benchmarks/typedpython/java-out/ +benchmarks/typedpython/tp-build/ diff --git a/benchmarks/typedpython/cython/binary_trees.pyx b/benchmarks/typedpython/cython/binary_trees.pyx new file mode 100644 index 000000000..1f25e4e66 --- /dev/null +++ b/benchmarks/typedpython/cython/binary_trees.pyx @@ -0,0 +1,46 @@ +# cython: language_level=3, boundscheck=False, wraparound=False, cdivision=True, initializedcheck=False +# Same algorithm as py/binary_trees.py: one heap object per node, reference counted, freed by +# refcounting (not an arena). Semantics that differ from CPython: Node is a cdef class (no __dict__, +# attributes not visible from Python), C long counters, recursion uses the C stack (no +# RecursionError), no None checks on typed attributes. +import sys + + +cdef class Node: + cdef Node left + cdef Node right + + def __init__(self, Node left, Node right): + self.left = left + self.right = right + + +cdef Node make(int depth): + if depth > 0: + return Node(make(depth - 1), make(depth - 1)) + return Node(None, None) + + +cdef long check(Node node) noexcept: + cdef Node left = node.left + cdef Node right = node.right + if left is None or right is None: + return 1 + return 1 + check(left) + check(right) + + +def main(): + cdef int n = int(sys.argv[1]) + cdef int min_depth = 4 + cdef int max_depth = max(min_depth + 2, n) + cdef int depth, i + cdef long iterations, chk, total = 0 + cdef long stretch = check(make(max_depth + 1)) + cdef Node long_lived = make(max_depth) + for depth in range(min_depth, max_depth + 1, 2): + iterations = 1 << (max_depth - depth + min_depth) + chk = 0 + for i in range(iterations): + chk += check(make(depth)) + total += chk + print(f"{stretch} {total} {check(long_lived)}") diff --git a/benchmarks/typedpython/cython/build.py b/benchmarks/typedpython/cython/build.py new file mode 100644 index 000000000..1ed61b65a --- /dev/null +++ b/benchmarks/typedpython/cython/build.py @@ -0,0 +1,46 @@ +"""Build the Cython columns. Run with the harness venv's python (see ../README in run.py's docstring). + + .venv/bin/python cython/build.py + +Writes /pure/.so (the unmodified py/*.py compiled in pure Python mode, annotations +as written) and /typed/_typed.so (cython/*.pyx). Not part of any timed run. +Flags: -O3 -ffp-contract=off (no fused multiply-add, so float results match CPython bit for bit). +""" +import shutil +import sys +from pathlib import Path + +from Cython.Build import cythonize +from setuptools import Extension, setup + +HERE = Path(__file__).resolve().parent +PY = HERE.parent / "py" +NAMES = {"nbody": "nbody", "spectral_norm": "spectral_norm", "fannkuch": "fannkuch", + "binary_trees": "binary_trees", "wordfreq": "wordfreq"} +CFLAGS = ["-O3", "-ffp-contract=off"] + + +def build(out: Path, kind: str) -> None: + work = out / f"src-{kind}" + shutil.rmtree(work, ignore_errors=True) + work.mkdir(parents=True) + exts = [] + for name in NAMES: + if kind == "pure": + src = work / f"{name}.py" + shutil.copy(PY / f"{name}.py", src) # unmodified copy + mod = name + else: + mod = f"{name}_typed" + src = work / f"{mod}.pyx" + shutil.copy(HERE / f"{name}.pyx", src) + exts.append(Extension(mod, [str(src)], extra_compile_args=CFLAGS)) + ext_modules = cythonize(exts, language_level=3, quiet=True, build_dir=str(work / "c")) + setup(name=f"tp-bench-{kind}", ext_modules=ext_modules, script_name="setup.py", + script_args=["build_ext", "--build-lib", str(out / kind), "--build-temp", str(work / "obj"), "-q"]) + + +if __name__ == "__main__": + target = Path(sys.argv[1]).resolve() + for k in ("pure", "typed"): + build(target, k) diff --git a/benchmarks/typedpython/cython/fannkuch.pyx b/benchmarks/typedpython/cython/fannkuch.pyx new file mode 100644 index 000000000..9927c47fc --- /dev/null +++ b/benchmarks/typedpython/cython/fannkuch.pyx @@ -0,0 +1,61 @@ +# cython: language_level=3, boundscheck=False, wraparound=False, cdivision=True, initializedcheck=False +# Same algorithm as py/fannkuch.py. Semantics that differ from CPython: C int arrays and C long +# counters (wrap instead of growing), no bounds checks, fixed capacity of 16 elements (n > 16 raises). +import sys + + +cdef (long, long) fannkuch(int n) noexcept: + cdef int perm1[16] + cdef int count[16] + cdef int perm[16] + cdef long max_flips = 0, checksum = 0, perm_count = 0, flips + cdef int r = n, i, k, lo, hi, t, p0 + for i in range(n): + perm1[i] = i + count[i] = 0 + while True: + while r != 1: + count[r - 1] = r + r -= 1 + for i in range(n): + perm[i] = perm1[i] + flips = 0 + k = perm[0] + while k != 0: + lo = 0 + hi = k + while lo < hi: + t = perm[lo] + perm[lo] = perm[hi] + perm[hi] = t + lo += 1 + hi -= 1 + flips += 1 + k = perm[0] + if flips > max_flips: + max_flips = flips + if perm_count % 2 == 0: + checksum += flips + else: + checksum -= flips + while True: + if r == n: + return checksum, max_flips + p0 = perm1[0] + for i in range(r): + perm1[i] = perm1[i + 1] + perm1[r] = p0 + count[r] -= 1 + if count[r] > 0: + break + r += 1 + perm_count += 1 + + +def main(): + cdef int n = int(sys.argv[1]) + if n > 16: + raise ValueError("fannkuch.pyx supports n <= 16") + cdef long checksum, max_flips + checksum, max_flips = fannkuch(n) + print(f"{checksum} {max_flips}") diff --git a/benchmarks/typedpython/cython/nbody.pyx b/benchmarks/typedpython/cython/nbody.pyx new file mode 100644 index 000000000..a12fd632a --- /dev/null +++ b/benchmarks/typedpython/cython/nbody.pyx @@ -0,0 +1,80 @@ +# cython: language_level=3, boundscheck=False, wraparound=False, cdivision=True, initializedcheck=False +# Same algorithm as py/nbody.py. Semantics that differ from CPython: float division does not raise +# ZeroDivisionError (cdivision), no bounds checks (arrays are C arrays), loop counters are C long. +from libc.math cimport sqrt +import sys + +cdef double PI = 3.141592653589793 +cdef double SOLAR_MASS = 4 * PI * PI +cdef double DAYS_PER_YEAR = 365.24 + + +cdef void offset_momentum(double* vx, double* vy, double* vz, double* mass) noexcept nogil: + cdef double px = 0.0, py = 0.0, pz = 0.0 + cdef int i + for i in range(5): + px += vx[i] * mass[i] + py += vy[i] * mass[i] + pz += vz[i] * mass[i] + vx[0] = -px / SOLAR_MASS + vy[0] = -py / SOLAR_MASS + vz[0] = -pz / SOLAR_MASS + + +cdef void advance(double dt, long steps, double* x, double* y, double* z, + double* vx, double* vy, double* vz, double* mass) noexcept nogil: + cdef long s + cdef int i, j + cdef double dx, dy, dz, d2, mag + for s in range(steps): + for i in range(5): + for j in range(i + 1, 5): + dx = x[i] - x[j] + dy = y[i] - y[j] + dz = z[i] - z[j] + d2 = dx * dx + dy * dy + dz * dz + mag = dt / (d2 * sqrt(d2)) + vx[i] -= dx * mass[j] * mag + vy[i] -= dy * mass[j] * mag + vz[i] -= dz * mass[j] * mag + vx[j] += dx * mass[i] * mag + vy[j] += dy * mass[i] * mag + vz[j] += dz * mass[i] * mag + for i in range(5): + x[i] += dt * vx[i] + y[i] += dt * vy[i] + z[i] += dt * vz[i] + + +cdef double energy(double* x, double* y, double* z, double* vx, double* vy, double* vz, + double* mass) noexcept nogil: + cdef double e = 0.0, dx, dy, dz + cdef int i, j + for i in range(5): + e += 0.5 * mass[i] * (vx[i] * vx[i] + vy[i] * vy[i] + vz[i] * vz[i]) + for j in range(i + 1, 5): + dx = x[i] - x[j] + dy = y[i] - y[j] + dz = z[i] - z[j] + e -= mass[i] * mass[j] / sqrt(dx * dx + dy * dy + dz * dz) + return e + + +def main(): + cdef long n = int(sys.argv[1]) + cdef double[5] x = [0.0, 4.84143144246472090e+00, 8.34336671824457987e+00, 1.28943695621391310e+01, 1.53796971148509165e+01] + cdef double[5] y = [0.0, -1.16032004402742839e+00, 4.12479856412430479e+00, -1.51111514016986312e+01, -2.59193146099879641e+01] + cdef double[5] z = [0.0, -1.03622044471123109e-01, -4.03523417114321381e-01, -2.23307578892655734e-01, 1.79258772950371181e-01] + cdef double[5] vx = [0.0, 1.66007664274403694e-03 * DAYS_PER_YEAR, -2.76742510726862411e-03 * DAYS_PER_YEAR, + 2.96460137564761618e-03 * DAYS_PER_YEAR, 2.68067772490389322e-03 * DAYS_PER_YEAR] + cdef double[5] vy = [0.0, 7.69901118419740425e-03 * DAYS_PER_YEAR, 4.99852801234917238e-03 * DAYS_PER_YEAR, + 2.37847173959480950e-03 * DAYS_PER_YEAR, 1.62824170038242295e-03 * DAYS_PER_YEAR] + cdef double[5] vz = [0.0, -6.90460016972063023e-05 * DAYS_PER_YEAR, 2.30417297573763929e-05 * DAYS_PER_YEAR, + -2.96589568540237556e-05 * DAYS_PER_YEAR, -9.51592254519715870e-05 * DAYS_PER_YEAR] + cdef double[5] mass = [SOLAR_MASS, 9.54791938424326609e-04 * SOLAR_MASS, 2.85885980666130812e-04 * SOLAR_MASS, + 4.36624404335156298e-05 * SOLAR_MASS, 5.15138902046611451e-05 * SOLAR_MASS] + offset_momentum(vx, vy, vz, mass) + cdef double before = energy(x, y, z, vx, vy, vz, mass) + advance(0.01, n, x, y, z, vx, vy, vz, mass) + cdef double after = energy(x, y, z, vx, vy, vz, mass) + print(f"{before:.9f} {after:.9f}") diff --git a/benchmarks/typedpython/cython/spectral_norm.pyx b/benchmarks/typedpython/cython/spectral_norm.pyx new file mode 100644 index 000000000..0b73cd8bc --- /dev/null +++ b/benchmarks/typedpython/cython/spectral_norm.pyx @@ -0,0 +1,61 @@ +# cython: language_level=3, boundscheck=False, wraparound=False, cdivision=True, initializedcheck=False +# Same algorithm as py/spectral_norm.py. Semantics that differ from CPython: loop and index +# arithmetic is C long (wraps instead of growing; not reachable at these sizes), cdivision (no +# ZeroDivisionError), no bounds checks (malloc'd C arrays). +from libc.math cimport sqrt +from libc.stdlib cimport malloc, free +import sys + + +cdef inline double eval_a(long i, long j) noexcept nogil: + return 1.0 / ((i + j) * (i + j + 1) // 2 + i + 1) + + +cdef void mul_av(long n, double* v, double* out) noexcept nogil: + cdef long i, j + cdef double s + for i in range(n): + s = 0.0 + for j in range(n): + s += eval_a(i, j) * v[j] + out[i] = s + + +cdef void mul_atv(long n, double* v, double* out) noexcept nogil: + cdef long i, j + cdef double s + for i in range(n): + s = 0.0 + for j in range(n): + s += eval_a(j, i) * v[j] + out[i] = s + + +cdef void mul_atav(long n, double* v, double* out, double* tmp) noexcept nogil: + mul_av(n, v, tmp) + mul_atv(n, tmp, out) + + +def main(): + cdef long n = int(sys.argv[1]) + cdef long i + cdef double* u = malloc(n * sizeof(double)) + cdef double* v = malloc(n * sizeof(double)) + cdef double* tmp = malloc(n * sizeof(double)) + cdef double vbv = 0.0, vv = 0.0 + try: + for i in range(n): + u[i] = 1.0 + v[i] = 0.0 + tmp[i] = 0.0 + for i in range(10): + mul_atav(n, u, v, tmp) + mul_atav(n, v, u, tmp) + for i in range(n): + vbv += u[i] * v[i] + vv += v[i] * v[i] + finally: + free(u) + free(v) + free(tmp) + print(f"{sqrt(vbv / vv):.9f}") diff --git a/benchmarks/typedpython/cython/wordfreq.pyx b/benchmarks/typedpython/cython/wordfreq.pyx new file mode 100644 index 000000000..2b1734a91 --- /dev/null +++ b/benchmarks/typedpython/cython/wordfreq.pyx @@ -0,0 +1,54 @@ +# cython: language_level=3, boundscheck=False, wraparound=False, cdivision=True, initializedcheck=False +# Same algorithm as py/wordfreq.py: the counts stay a Python dict[str, int] (the algorithm's data +# structure), only the arithmetic is typed and words are built in a C buffer. Semantics that differ +# from CPython: C long arithmetic (wraps; the LCG product stays below 2**47 and the weighted sum +# below 2**63 at benchmark size), cdivision. +import sys + +cdef long VOCAB = 50000 +cdef long MOD = 2147483647 + + +cdef inline long next_state(long s) noexcept nogil: + return (s * 48271) % MOD + + +cdef str word_of(long word_id): + cdef char buf[32] + cdef int pos = 32 + cdef long x = word_id + 17576 + while x > 0: + pos -= 1 + buf[pos] = (97 + x % 26) + x //= 26 + return (buf[pos:32]).decode("ascii") + + +def main(): + cdef long n = int(sys.argv[1]) + cdef dict counts = {} + cdef long s = 12345, a, b, i, c, weighted = 0, best_count + cdef str word, w, best_word + cdef list taken = [] + cdef list parts = [] + for i in range(n): + s = next_state(s) + a = s % VOCAB + s = next_state(s) + b = s % VOCAB + word = word_of(a * b // VOCAB) + counts[word] = counts.get(word, 0) + 1 + for w, c in counts.items(): + weighted += c * c + for i in range(5): + best_word = "" + best_count = -1 + for w, c in counts.items(): + if w in taken: + continue + if c > best_count or (c == best_count and w < best_word): + best_word = w + best_count = c + taken.append(best_word) + parts.append(f"{best_word}:{best_count}") + print(f"{len(counts)} {weighted} {','.join(parts)}") diff --git a/benchmarks/typedpython/numba/binary_trees.py b/benchmarks/typedpython/numba/binary_trees.py new file mode 100644 index 000000000..35a4b7fcc --- /dev/null +++ b/benchmarks/typedpython/numba/binary_trees.py @@ -0,0 +1,72 @@ +"""numba version of py/binary_trees.py, using numba.experimental.jitclass. + +Node is a jitclass with a deferred (self-referential) Optional type, the closest numba has to a +heap object per node; allocation and release go through numba's NRT reference counting, like the +CPython program. (An index-based tree in preallocated arrays would be faster but changes the +allocation pattern into an arena, so it is not used.) + +Semantics that differ from CPython: counters are int64, recursion uses the native stack (no +RecursionError). The module must be registered in sys.modules before it runs (run.py does). +""" +from numba import deferred_type, int64, njit, optional +from numba.core.types import UniTuple +from numba.experimental import jitclass + +node_type = deferred_type() + + +@jitclass([("left", optional(node_type)), ("right", optional(node_type))]) +class Node: + def __init__(self, left, right): + self.left = left + self.right = right + + +NodeT = Node.class_type.instance_type +node_type.define(NodeT) + +# cache=True is not usable here: numba raises AttributeError: 'Optional' object has no attribute +# 'type' while hashing the recursive jitclass type for the cache index. So this benchmark has no +# cached run; every process compiles (at import, see below). +OPTS = dict(cache=False) + + +# Explicit signatures: numba cannot infer the return type of the self-recursive make() through +# the jitclass's deferred Optional type (AttributeError: 'Optional' object has no attribute +# 'type'). The consequence is that these two functions compile when this module is imported, not +# on the first call; run.py counts the import time as compile time. +@njit(NodeT(int64), **OPTS) +def make(depth): + if depth > 0: + return Node(make(depth - 1), make(depth - 1)) + return Node(None, None) + + +@njit(int64(optional(NodeT)), **OPTS) +def check(node): + left = node.left + right = node.right + if left is None or right is None: + return 1 + return 1 + check(left) + check(right) + + +@njit(UniTuple(int64, 3)(int64), **OPTS) +def kernel(n): + min_depth = 4 + max_depth = max(min_depth + 2, n) + stretch = check(make(max_depth + 1)) + long_lived = make(max_depth) + total = 0 + for depth in range(min_depth, max_depth + 1, 2): + iterations = 1 << (max_depth - depth + min_depth) + chk = 0 + for _ in range(iterations): + chk += check(make(depth)) + total += chk + return stretch, total, check(long_lived) + + +def run(n: int) -> str: + stretch, total, long_lived = kernel(n) + return f"{stretch} {total} {long_lived}" diff --git a/benchmarks/typedpython/numba/fannkuch.py b/benchmarks/typedpython/numba/fannkuch.py new file mode 100644 index 000000000..5f21f092e --- /dev/null +++ b/benchmarks/typedpython/numba/fannkuch.py @@ -0,0 +1,62 @@ +"""numba @njit version of py/fannkuch.py. `run(n) -> str` is the whole benchmark. + +Semantics that differ from CPython: int64 numpy arrays and int64 counters (wrap instead of +growing), no bounds checks. +""" +import numpy as np +from numba import njit + +OPTS = dict(cache=True, error_model="numpy") + + +@njit(**OPTS) +def fannkuch(n): + perm1 = np.arange(n) + count = np.zeros(n, dtype=np.int64) + perm = np.zeros(n, dtype=np.int64) + max_flips = 0 + checksum = 0 + perm_count = 0 + r = n + while True: + while r != 1: + count[r - 1] = r + r -= 1 + for i in range(n): + perm[i] = perm1[i] + flips = 0 + k = perm[0] + while k != 0: + lo = 0 + hi = k + while lo < hi: + t = perm[lo] + perm[lo] = perm[hi] + perm[hi] = t + lo += 1 + hi -= 1 + flips += 1 + k = perm[0] + if flips > max_flips: + max_flips = flips + if perm_count % 2 == 0: + checksum += flips + else: + checksum -= flips + while True: + if r == n: + return checksum, max_flips + p0 = perm1[0] + for i in range(r): + perm1[i] = perm1[i + 1] + perm1[r] = p0 + count[r] -= 1 + if count[r] > 0: + break + r += 1 + perm_count += 1 + + +def run(n: int) -> str: + checksum, max_flips = fannkuch(n) + return f"{checksum} {max_flips}" diff --git a/benchmarks/typedpython/numba/nbody.py b/benchmarks/typedpython/numba/nbody.py new file mode 100644 index 000000000..74d8b8bee --- /dev/null +++ b/benchmarks/typedpython/numba/nbody.py @@ -0,0 +1,82 @@ +"""numba @njit version of py/nbody.py. Loaded by run.py; `run(n) -> str` is the whole benchmark. + +Semantics that differ from CPython: float64 numpy arrays instead of lists; error_model="numpy" +(float division by zero gives inf instead of raising ZeroDivisionError); no bounds checks. +""" +import math + +import numpy as np +from numba import njit + +PI = 3.141592653589793 +SOLAR_MASS = 4 * PI * PI +DAYS_PER_YEAR = 365.24 +OPTS = dict(cache=True, error_model="numpy") + + +@njit(**OPTS) +def offset_momentum(vx, vy, vz, mass): + px = 0.0 + py = 0.0 + pz = 0.0 + for i in range(5): + px += vx[i] * mass[i] + py += vy[i] * mass[i] + pz += vz[i] * mass[i] + vx[0] = -px / SOLAR_MASS + vy[0] = -py / SOLAR_MASS + vz[0] = -pz / SOLAR_MASS + + +@njit(**OPTS) +def advance(dt, steps, x, y, z, vx, vy, vz, mass): + for _ in range(steps): + for i in range(5): + for j in range(i + 1, 5): + dx = x[i] - x[j] + dy = y[i] - y[j] + dz = z[i] - z[j] + d2 = dx * dx + dy * dy + dz * dz + mag = dt / (d2 * math.sqrt(d2)) + vx[i] -= dx * mass[j] * mag + vy[i] -= dy * mass[j] * mag + vz[i] -= dz * mass[j] * mag + vx[j] += dx * mass[i] * mag + vy[j] += dy * mass[i] * mag + vz[j] += dz * mass[i] * mag + for i in range(5): + x[i] += dt * vx[i] + y[i] += dt * vy[i] + z[i] += dt * vz[i] + + +@njit(**OPTS) +def energy(x, y, z, vx, vy, vz, mass): + e = 0.0 + for i in range(5): + e += 0.5 * mass[i] * (vx[i] * vx[i] + vy[i] * vy[i] + vz[i] * vz[i]) + for j in range(i + 1, 5): + dx = x[i] - x[j] + dy = y[i] - y[j] + dz = z[i] - z[j] + e -= mass[i] * mass[j] / math.sqrt(dx * dx + dy * dy + dz * dz) + return e + + +def run(n: int) -> str: + x = np.array([0.0, 4.84143144246472090e+00, 8.34336671824457987e+00, 1.28943695621391310e+01, 1.53796971148509165e+01]) + y = np.array([0.0, -1.16032004402742839e+00, 4.12479856412430479e+00, -1.51111514016986312e+01, -2.59193146099879641e+01]) + z = np.array([0.0, -1.03622044471123109e-01, -4.03523417114321381e-01, -2.23307578892655734e-01, 1.79258772950371181e-01]) + vx = np.array([0.0, 1.66007664274403694e-03 * DAYS_PER_YEAR, -2.76742510726862411e-03 * DAYS_PER_YEAR, + 2.96460137564761618e-03 * DAYS_PER_YEAR, 2.68067772490389322e-03 * DAYS_PER_YEAR]) + vy = np.array([0.0, 7.69901118419740425e-03 * DAYS_PER_YEAR, 4.99852801234917238e-03 * DAYS_PER_YEAR, + 2.37847173959480950e-03 * DAYS_PER_YEAR, 1.62824170038242295e-03 * DAYS_PER_YEAR]) + vz = np.array([0.0, -6.90460016972063023e-05 * DAYS_PER_YEAR, 2.30417297573763929e-05 * DAYS_PER_YEAR, + -2.96589568540237556e-05 * DAYS_PER_YEAR, -9.51592254519715870e-05 * DAYS_PER_YEAR]) + mass = np.array([SOLAR_MASS, 9.54791938424326609e-04 * SOLAR_MASS, 2.85885980666130812e-04 * SOLAR_MASS, + 4.36624404335156298e-05 * SOLAR_MASS, 5.15138902046611451e-05 * SOLAR_MASS]) + offset_momentum(vx, vy, vz, mass) + before = energy(x, y, z, vx, vy, vz, mass) + advance(0.01, n, x, y, z, vx, vy, vz, mass) + after = energy(x, y, z, vx, vy, vz, mass) + return f"{before:.9f} {after:.9f}" diff --git a/benchmarks/typedpython/numba/spectral_norm.py b/benchmarks/typedpython/numba/spectral_norm.py new file mode 100644 index 000000000..bfa99bae5 --- /dev/null +++ b/benchmarks/typedpython/numba/spectral_norm.py @@ -0,0 +1,61 @@ +"""numba @njit version of py/spectral_norm.py. `run(n) -> str` is the whole benchmark. + +Semantics that differ from CPython: float64 numpy arrays, int64 index arithmetic (wraps), +error_model="numpy" (no ZeroDivisionError), no bounds checks. +""" +import math + +import numpy as np +from numba import njit + +OPTS = dict(cache=True, error_model="numpy") + + +@njit(**OPTS) +def eval_a(i, j): + return 1.0 / ((i + j) * (i + j + 1) // 2 + i + 1) + + +@njit(**OPTS) +def mul_av(n, v, out): + for i in range(n): + s = 0.0 + for j in range(n): + s += eval_a(i, j) * v[j] + out[i] = s + + +@njit(**OPTS) +def mul_atv(n, v, out): + for i in range(n): + s = 0.0 + for j in range(n): + s += eval_a(j, i) * v[j] + out[i] = s + + +@njit(**OPTS) +def mul_atav(n, v, out, tmp): + mul_av(n, v, tmp) + mul_atv(n, tmp, out) + + +@njit(**OPTS) +def kernel(n): + u = np.ones(n) + v = np.zeros(n) + tmp = np.zeros(n) + for _ in range(10): + mul_atav(n, u, v, tmp) + mul_atav(n, v, u, tmp) + vbv = 0.0 + vv = 0.0 + for i in range(n): + vbv += u[i] * v[i] + vv += v[i] * v[i] + return vbv, vv + + +def run(n: int) -> str: + vbv, vv = kernel(n) + return f"{math.sqrt(vbv / vv):.9f}" diff --git a/benchmarks/typedpython/numba/wordfreq.py b/benchmarks/typedpython/numba/wordfreq.py new file mode 100644 index 000000000..5bc8599fb --- /dev/null +++ b/benchmarks/typedpython/numba/wordfreq.py @@ -0,0 +1,69 @@ +"""numba @njit version of py/wordfreq.py: numba.typed.Dict[unicode_type, int64]. + +Semantics that differ from CPython: int64 arithmetic (wraps), words are numba unicode strings, +the counts dict is a numba typed.Dict (hash order differs, but the result does not depend on +iteration order because ties are broken by string comparison). +""" +import numpy as np +from numba import njit, types +from numba.typed import Dict, List + +VOCAB = 50000 +MOD = 2147483647 +OPTS = dict(cache=True, error_model="numpy") + + +@njit(**OPTS) +def next_state(s): + return (s * 48271) % MOD + + +@njit(**OPTS) +def word_of(word_id): + x = word_id + 17576 + w = "" + while x > 0: + w = chr(97 + x % 26) + w + x //= 26 + return w + + +@njit(**OPTS) +def kernel(n): + counts = Dict.empty(key_type=types.unicode_type, value_type=types.int64) + s = 12345 + for _ in range(n): + s = next_state(s) + a = s % VOCAB + s = next_state(s) + b = s % VOCAB + word = word_of(a * b // VOCAB) + if word in counts: + counts[word] += 1 + else: + counts[word] = 1 + weighted = 0 + for w in counts: + c = counts[w] + weighted += c * c + taken = List.empty_list(types.unicode_type) + best_counts = np.zeros(5, dtype=np.int64) + for k in range(5): + best_word = "" + best_count = -1 + for w in counts: + c = counts[w] + if w in taken: + continue + if c > best_count or (c == best_count and w < best_word): + best_word = w + best_count = c + taken.append(best_word) + best_counts[k] = best_count + return len(counts), weighted, taken, best_counts + + +def run(n: int) -> str: + size, weighted, taken, best_counts = kernel(n) + parts = [f"{w}:{c}" for w, c in zip(taken, best_counts)] + return f"{size} {weighted} {','.join(parts)}" diff --git a/benchmarks/typedpython/run.py b/benchmarks/typedpython/run.py index c7fced567..b6ce8a280 100644 --- a/benchmarks/typedpython/run.py +++ b/benchmarks/typedpython/run.py @@ -13,6 +13,27 @@ the `# typedpython: compiled` marker, compiled through `typedpython.pipeline`, and its `main()` is run from the built extension. Without the option the column reports n/a. +Cython and numba columns (issue: compare TypedPython fairly against them). They need a harness venv +that holds Cython, numba and a C compiler (Xcode command line tools), created ONLY with uv: + + uv venv --python 3.14 benchmarks/typedpython/.venv + uv pip install --python benchmarks/typedpython/.venv/bin/python cython setuptools numba + +`--tools-python` (default benchmarks/typedpython/.venv/bin/python) names that interpreter; without it +the three columns report n/a. Columns, each in the form most favourable to it: + "Cython (pure)" py/*.py compiled unmodified by cythonize (annotations as written), same CPython; + "Cython (typed)" cython/*.pyx: cdef types, C arrays, boundscheck/wraparound off, cdef class Node; + "numba" numba/*.py with @njit(cache=True) (binary-trees: jitclass, no cache). The time + reported is the STEADY-STATE call at benchmark size, inside the process, after a + warm-up call that triggered the JIT. The JIT cost is the separate record + "numba compile" (first call: cold cache vs cached; numba import excluded). +Cython is built by cython/build.py before any timing (build/cython/, git-ignored). numba runs in a +fresh process with its own NUMBA_CACHE_DIR (build/numba-cache/, wiped per cold run). +"CPython (tools python)" is the same program under the tools interpreter, and is the baseline for the +Cython/numba ratios in `interleaved` (so those pairs use one interpreter); it reads "same as CPython" +when --python and --tools-python are the same version. Per-column tool version, flags and a semantics +note are written to the JSON ("tools") and printed after the table. + binary-trees declares `__slots__` (TypedPython compiles fixed-layout classes only, SPEC N-11). The speedup is reported against that CPython version; `CPython (no __slots__)` runs the same program with the `__slots__` line removed, so what adding the slots changes in CPython is recorded too. @@ -21,6 +42,7 @@ import datetime import json import os +import shutil import socket import statistics import subprocess @@ -45,7 +67,31 @@ "wordfreq": ("wordfreq.py", "wordfreq.mjs", "Wordfreq", "wordfreq", "100000", "3000000"), } NOSLOTS = "CPython (no __slots__)" -COLUMNS = ["CPython", NOSLOTS, "Node", "Java", "Rust", "TypedPython @compiled"] +CPY_TOOLS = "CPython (tools python)" +CY_PURE = "Cython (pure)" +CY_TYPED = "Cython (typed)" +NUMBA = "numba" +TOOL_COLS = [CY_PURE, CY_TYPED, NUMBA] +COLUMNS = ["CPython", NOSLOTS, CPY_TOOLS, "Node", "Java", "Rust", "TypedPython @compiled", *TOOL_COLS] +TOOLS_PY_DEFAULT = ROOT / ".venv" / "bin" / "python" +CYTHON_OUT = ROOT / "build" / "cython" +NUMBA_CACHE = ROOT / "build" / "numba-cache" +# tiny sizes for numba's warm-up (first) call: the types are the same at any size, so the first +# call is almost all compile time +NUMBA_WARM = {"nbody": "10", "spectral-norm": "4", "fannkuch-redux": "4", "binary-trees": "1", + "wordfreq": "10"} +CYTHON_FLAGS = "cython/build.py: cythonize(language_level=3), cc -O3 -ffp-contract=off (no fused multiply-add)" +SEMANTICS = { + CY_PURE: "py/*.py unmodified; Cython honours the existing annotations (float is a C double, int stays " + "a Python int, list stays a list), so semantics are CPython's apart from float arithmetic on typed locals", + CY_TYPED: "C semantics where declared: C long/int arithmetic wraps instead of growing, cdivision (no " + "ZeroDivisionError), boundscheck/wraparound off on C arrays, cdef class Node (no __dict__), " + "C-stack recursion (no RecursionError); wordfreq keeps a Python dict", + NUMBA: "fixed-width int64/float64 (ints wrap, no Python big ints, no OverflowError), error_model=numpy " + "(no ZeroDivisionError), no bounds checks, numpy arrays for lists, typed.Dict/List for dict/list " + "(wordfreq), jitclass for the tree (binary-trees); reported time is steady state, JIT is separate", +} + SLOTS_LINE = ' __slots__ = ("left", "right")\n' NOSLOTS_BENCHMARKS = {"binary-trees"} NOT_BUILT = "n/a (not built)" @@ -118,8 +164,64 @@ def noslots_source(name: str) -> Path: return out +TOOLS_PROBE = """ +import json, sys +out = {"python": sys.version.split()[0]} +for mod in ("Cython", "numba", "llvmlite", "numpy", "setuptools"): + try: + out[mod] = __import__(mod).__version__ + except Exception as e: + out[mod] = f"missing ({type(e).__name__}: {e})" +print(json.dumps(out)) +""" + + +def _cython_up_to_date() -> bool: + sources = [*(ROOT / "py").glob("*.py"), *(ROOT / "cython").glob("*"), ] + newest = max(p.stat().st_mtime for p in sources) + for name, (py, *_r) in BENCHMARKS.items(): + stem = Path(py).stem + for kind, mod in (("pure", stem), ("typed", stem + "_typed")): + found = list((CYTHON_OUT / kind).glob(f"{mod}.*.so")) + if not found or found[0].stat().st_mtime < newest: + return False + return True + + +def build_tools(tools_python: str | None, python: str) -> dict[str, object]: + """Probe the tools interpreter and cythonize + C-compile the Cython columns (never in a timed run). + Returns {"python", "versions", "na": {column: reason}, "same_python"}.""" + tools: dict[str, object] = {"python": None, "versions": {}, "na": {}, "same_python": True} + na: dict[str, str] = tools["na"] # type: ignore[assignment] + if not tools_python or not Path(tools_python).exists(): + why = ("n/a (no harness venv; create it with: uv venv --python 3.14 benchmarks/typedpython/.venv && " + "uv pip install --python benchmarks/typedpython/.venv/bin/python cython setuptools numba)") + return {**tools, "na": {c: why for c in TOOL_COLS}} + probe = subprocess.run([tools_python, "-c", TOOLS_PROBE], capture_output=True, text=True) + if probe.returncode != 0: + return {**tools, "na": {c: f"n/a (tools python failed: {probe.stderr.strip()[-200:]})" for c in TOOL_COLS}} + versions = json.loads(probe.stdout.strip().splitlines()[-1]) + tools.update(python=tools_python, versions=versions) + base = subprocess.run([python, "-c", "import sys; print(sys.version.split()[0])"], + capture_output=True, text=True).stdout.strip() + tools["same_python"] = base == versions["python"] + tools["baseline_python"] = base + if "missing" in versions["Cython"] or "missing" in versions["setuptools"]: + na[CY_PURE] = na[CY_TYPED] = f"n/a (Cython {versions['Cython']}; setuptools {versions['setuptools']})" + elif not _cython_up_to_date(): + build = subprocess.run([tools_python, str(ROOT / "cython" / "build.py"), str(CYTHON_OUT)], + capture_output=True, text=True) + if build.returncode != 0: + reason = f"n/a (cython build failed: {' | '.join(build.stderr.strip().splitlines()[-3:])})" + na[CY_PURE] = na[CY_TYPED] = reason + if "missing" in versions["numba"] or "missing" in versions["numpy"]: + na[NUMBA] = f"n/a (numba {versions['numba']}; numpy {versions['numpy']})" + return tools + + def commands(name: str, size: str, python: str, java_source: bool, - compiled: dict[str, tuple[str, list[str]] | str] | None = None) -> dict[str, list[str]]: + compiled: dict[str, tuple[str, list[str]] | str] | None = None, + tools: dict[str, object] | None = None) -> dict[str, list[str]]: py, js, cls, rs, _, _ = BENCHMARKS[name] java = ["java", str(ROOT / "java" / f"{cls}.java"), size] if java_source \ else ["java", "-cp", str(JAVA_OUT), cls, size] @@ -134,37 +236,188 @@ def commands(name: str, size: str, python: str, java_source: bool, entry = (compiled or {}).get(name) if isinstance(entry, tuple): cmds["TypedPython @compiled"] = [python, "-c", RUN_SNIPPET, Path(py).stem, entry[0], size] + if tools and tools.get("python"): + tp = str(tools["python"]) + na: dict[str, str] = tools["na"] # type: ignore[assignment] + stem = Path(py).stem + cmds[CPY_TOOLS] = Cmd([tp, str(ROOT / "py" / py), size]) + for col, kind, mod in ((CY_PURE, "pure", stem), (CY_TYPED, "typed", stem + "_typed")): + so = sorted((CYTHON_OUT / kind).glob(f"{mod}.*.so")) + if col not in na and so: + cmd = Cmd([tp, "-c", RUN_SNIPPET, mod, str(so[0]), size]) + cmd.soft = True + cmds[col] = cmd + if NUMBA not in na: + cmd = Cmd([tp, "-c", NUMBA_SNIPPET, str(ROOT / "numba" / py), stem + "_numba", + NUMBA_WARM[name], size]) + cmd.numba = True + cmd.soft = True + cmd.cache_dir = NUMBA_CACHE / name + cmd.env = {"NUMBA_CACHE_DIR": str(cmd.cache_dir)} + cmds[NUMBA] = cmd return cmds -def run_once(cmd: list[str]) -> tuple[float, str]: +class RunFailed(Exception): + """A Cython/numba column failed to run; reported as n/a with the concrete error.""" + + +class Cmd(list): + """argv plus, for the numba column, an environment, a cache directory and the last timing.""" + env: dict[str, str] | None = None + numba = False + cache_dir: Path | None = None + cold = True + last: dict[str, float] | None = None + soft = False + + +NUMBA_SNIPPET = """ +import importlib.util, json, sys, time +import numba, numpy # imported first: import time is not compile time +path, name, warm, size = sys.argv[1:5] +t0 = time.perf_counter() +spec = importlib.util.spec_from_file_location(name, path) +m = importlib.util.module_from_spec(spec) +sys.modules[name] = m # jitclass resolves its module through sys.modules +spec.loader.exec_module(m) # binary-trees (explicit signatures) compiles here +m.run(int(warm)) # first call: JIT compile (or cache load) at a tiny size +t1 = time.perf_counter() +out = m.run(int(size)) # steady state: already compiled +t2 = time.perf_counter() +print(out) +print("#timing " + json.dumps({"first_call_s": t1 - t0, "steady_s": t2 - t1})) +""" + + +def run_once(cmd: list[str], soft: bool = False) -> tuple[float, str]: + """Wall time and stdout. A numba Cmd returns its steady-state seconds instead (see NUMBA_SNIPPET).""" + cache_dir = getattr(cmd, "cache_dir", None) + if cache_dir is not None and getattr(cmd, "cold", True): + shutil.rmtree(cache_dir, ignore_errors=True) + env = dict(os.environ, **cmd.env) if getattr(cmd, "env", None) else None # type: ignore[attr-defined] start = time.perf_counter() - result = subprocess.run(cmd, capture_output=True, text=True) + result = subprocess.run(cmd, capture_output=True, text=True, env=env) elapsed = time.perf_counter() - start if result.returncode != 0: + if soft or getattr(cmd, "soft", False): + last = (result.stderr.strip().splitlines() or ["no stderr"])[-1] + raise RunFailed(f"exit {result.returncode}: {last}") sys.stderr.write(f"failed ({result.returncode}): {' '.join(cmd)}\n{result.stderr}") sys.exit(1) - return elapsed, result.stdout.strip() + out = result.stdout.strip() + if getattr(cmd, "numba", False): + lines = out.splitlines() + timing = json.loads(lines[-1][len("#timing "):]) + cmd.last = dict(timing, wall_s=elapsed) # type: ignore[attr-defined] + out = "\n".join(lines[:-1]).strip() + elapsed = timing["steady_s"] + return elapsed, out + + +def tools_for(args: argparse.Namespace) -> dict[str, object]: + return build_tools(args.tools_python, args.python) + + +def tools_report(tools: dict[str, object]) -> dict[str, object]: + """Per-column tool version, flags and semantics note, for the printout and the JSON.""" + v: dict[str, str] = tools.get("versions") or {} # type: ignore[assignment] + na: dict[str, str] = tools["na"] # type: ignore[assignment] + py = f"CPython {v.get('python', '?')}" + report: dict[str, object] = { + "tools_python": {"interpreter": py, "baseline_python": tools.get("baseline_python"), + "same_interpreter_as_CPython_column": tools.get("same_python")}, + CY_PURE: {"tool": f"Cython {v.get('Cython')} on {py}", "flags": CYTHON_FLAGS, + "semantics": SEMANTICS[CY_PURE]}, + CY_TYPED: {"tool": f"Cython {v.get('Cython')} on {py}", + "flags": CYTHON_FLAGS + "; # cython: boundscheck=False, wraparound=False, cdivision=True, " + "initializedcheck=False in each .pyx", "semantics": SEMANTICS[CY_TYPED]}, + NUMBA: {"tool": f"numba {v.get('numba')}, llvmlite {v.get('llvmlite')}, numpy {v.get('numpy')} on {py}", + "flags": "@njit(cache=True, error_model='numpy'); binary-trees: jitclass + explicit " + "signatures, cache=False; NUMBA_CACHE_DIR per benchmark", + "semantics": SEMANTICS[NUMBA]}, + } + for col in TOOL_COLS: + if col in na: + report[col]["unavailable"] = na[col] # type: ignore[index] + return report + + +def print_tools_report(report: dict[str, object]) -> None: + print("\ntools:") + for key, val in report.items(): + if key == "tools_python": + print(f" tools python: {val['interpreter']}; baseline CPython column: {val['baseline_python']}; " # type: ignore[index] + f"same interpreter: {val['same_interpreter_as_CPython_column']}") # type: ignore[index] + else: + print(f" {key}: {val['tool']}\n flags: {val['flags']}\n semantics: {val['semantics']}") # type: ignore[index] + + +def run_column(col: str, cmd: list[str]) -> tuple[float, str] | str: + """run_once, but a Cython/numba failure becomes the n/a string instead of ending the run.""" + try: + return run_once(cmd) + except RunFailed as e: + return f"n/a ({e})" def verify(args: argparse.Namespace) -> int: build() compiled = build_compiled(Path(args.typedpython), args.typedpython_python) if args.typedpython else None + tools = tools_for(args) bad = 0 for name, spec in BENCHMARKS.items(): - outputs = {col: run_once(cmd)[1] - for col, cmd in commands(name, spec[4], args.python, True, compiled).items()} + outputs: dict[str, str] = {} + notes: dict[str, str] = dict(tools["na"]) # type: ignore[arg-type] + for col, cmd in commands(name, spec[4], args.python, True, compiled, tools).items(): + res = run_column(col, cmd) + if isinstance(res, str): + notes[col] = res + else: + outputs[col] = res[1] distinct = set(outputs.values()) status = "OK " if len(distinct) == 1 else "FAIL" print(f"{status} {name} (size {spec[4]}): {next(iter(outputs.values())) if len(distinct) == 1 else ''}") + print(f" columns: {', '.join(outputs)}") + for col, why in notes.items(): + print(f" {col}: {why}") if len(distinct) != 1: bad += 1 for col, out in outputs.items(): print(f" {col}: {out}") print("all outputs agree" if bad == 0 else f"{bad} benchmark(s) disagree") + print_tools_report(tools_report(tools)) return 1 if bad else 0 +def measure_numba(cmd: Cmd, repeat: int) -> tuple[dict[str, object], dict[str, object]]: + """Cold-cache runs (each wipes the cache dir, so each compiles) give the steady-state time and the + cold compile time; one more run with the cache left in place gives the cached first-call time.""" + steady, outs, cold_first, walls = [], set(), [], [] + for _ in range(repeat): + t, out = run_once(cmd) + steady.append(t) + outs.add(out) + cold_first.append(cmd.last["first_call_s"]) # type: ignore[index] + walls.append(cmd.last["wall_s"]) # type: ignore[index] + cache_files = len([p for p in cmd.cache_dir.rglob("*") if p.is_file()]) if cmd.cache_dir else 0 + cmd.cold = False + try: + t_c, out_c = run_once(cmd) + outs.add(out_c) + cached_first = cmd.last["first_call_s"] # type: ignore[index] + cached_steady = t_c + finally: + cmd.cold = True + column = {"median_s": statistics.median(steady), "runs_s": steady, "output": sorted(outs), + "process_wall_s_runs": walls, "note": "steady-state call at benchmark size, JIT excluded"} + compile_rec = {"cold_first_call_s_median": statistics.median(cold_first), "cold_first_call_s_runs": cold_first, + "cached_first_call_s": cached_first, "cached_steady_s": cached_steady, + "cache_files_written": cache_files, "cache_used": cache_files > 0, + "note": "first call = module import + first run() at a tiny size; numba import excluded"} + return column, compile_rec + + def measure(args: argparse.Namespace) -> int: load = os.getloadavg() print(f"load average: {load[0]:.2f} {load[1]:.2f} {load[2]:.2f}") @@ -174,6 +427,8 @@ def measure(args: argparse.Namespace) -> int: return 2 build() compiled = build_compiled(Path(args.typedpython), args.typedpython_python) if args.typedpython else None + tools = tools_for(args) + report = tools_report(tools) results: dict[str, dict[str, object]] = {} for name, spec in BENCHMARKS.items(): size = args.size.get(name, spec[5]) @@ -181,15 +436,29 @@ def measure(args: argparse.Namespace) -> int: entry = (compiled or {}).get(name, NOT_BUILT) if isinstance(entry, tuple): row["compiled_functions"] = entry[1] - for col, cmd in commands(name, size, args.python, False, compiled).items(): - runs = [run_once(cmd) for _ in range(args.repeat)] - outs = {out for _, out in runs} - row[col] = {"median_s": statistics.median(t for t, _ in runs), - "runs_s": [t for t, _ in runs], "output": sorted(outs)} + for col, cmd in commands(name, size, args.python, False, compiled, tools).items(): + if col == CPY_TOOLS and tools.get("same_python"): + continue + try: + if col == NUMBA: + row[col], row["numba compile"] = measure_numba(cmd, args.repeat) # type: ignore[arg-type] + outs = set(row[col]["output"]) # type: ignore[index] + else: + runs = [run_once(cmd) for _ in range(args.repeat)] + outs = {out for _, out in runs} + row[col] = {"median_s": statistics.median(t for t, _ in runs), + "runs_s": [t for t, _ in runs], "output": sorted(outs)} + except RunFailed as e: + row[col] = f"n/a ({e})" + continue if len(outs) != 1: print(f"warning: {name}/{col} produced differing outputs across runs") if not isinstance(entry, tuple): row["TypedPython @compiled"] = entry + for col, why in tools["na"].items(): # type: ignore[union-attr] + row[col] = why + if tools.get("same_python"): + row[CPY_TOOLS] = "same as CPython" measured = [c for c in COLUMNS if isinstance(row.get(c), dict)] outs_all = {tuple(row[c]["output"]) for c in measured} # type: ignore[index] row["outputs_agree"] = len(outs_all) == 1 @@ -206,6 +475,13 @@ def measure(args: argparse.Namespace) -> int: flag = "" if row["outputs_agree"] else " (OUTPUTS DIFFER)" print(f"| {name}{flag} | {row['size']} | " + " | ".join(cells) + " |") print(f"\nmedian of {args.repeat} runs, wall time; Java excludes javac, includes JVM startup.") + print("numba column = steady-state call inside the process (JIT excluded); its compile cost:") + for name, row in results.items(): + rec = row.get("numba compile") + if isinstance(rec, dict): + print(f" {name}: cold first call {rec['cold_first_call_s_median']:.3f} s, cached first call " + f"{rec['cached_first_call_s']:.3f} s, cache files written {rec['cache_files_written']}") + print_tools_report(report) if args.write: out_dir = ROOT / "results" @@ -214,7 +490,7 @@ def measure(args: argparse.Namespace) -> int: payload = {"date": datetime.datetime.now().isoformat(timespec="seconds"), "host": socket.gethostname(), "load_average": load, "repeat": args.repeat, "python": subprocess.run([args.python, "--version"], capture_output=True, text=True).stdout.strip(), - "results": results} + "tools": report, "results": results} path.write_text(json.dumps(payload, indent=2) + "\n") print(f"wrote {path}") return 0 @@ -224,46 +500,48 @@ def interleaved(args: argparse.Namespace) -> int: """CPython vs TypedPython @compiled, alternated A/B in the same time window (leader decision, 2026-10-03): when the machine is never quiet, load hits both sides of each pair alike, so the per-pair ratio is stable even though absolute times are not. Reports the median ratio, never - absolute times as the result, with the load average recorded at every pair.""" - if not args.typedpython: - print("interleaved needs --typedpython") + absolute times as the result, with the load average recorded at every pair. + + The Cython and numba columns are paired the same way against the CPython run under the tools + interpreter (`CPython (tools python)`), each pair alternating which side runs first. For numba the + ratio uses its in-process steady-state time (JIT excluded, process start included for CPython, a + small bias in numba's favour at these sizes); its first-call (compile) time is recorded per pair.""" + tools = tools_for(args) + if not args.typedpython and not tools.get("python"): + print("interleaved needs --typedpython or a tools python") return 2 - compiled = build_compiled(Path(args.typedpython), args.typedpython_python) + compiled = build_compiled(Path(args.typedpython), args.typedpython_python) if args.typedpython else {} results: dict[str, dict[str, object]] = {} for name, spec in BENCHMARKS.items(): entry = compiled.get(name) - if not isinstance(entry, tuple): - results[name] = {"skipped": entry} - continue size = args.size.get(name, spec[5]) - cmds = commands(name, size, args.python, False, compiled) - a_cmd, b_cmd = cmds["CPython"], cmds["TypedPython @compiled"] - c_cmd = cmds.get(NOSLOTS) - pairs = [] - for i in range(args.pairs): - order = (a_cmd, b_cmd) if i % 2 == 0 else (b_cmd, a_cmd) # alternate which runs first - load = os.getloadavg() - (t1, o1), (t2, o2) = run_once(order[0]), run_once(order[1]) - t_a, o_a, t_b, o_b = (t1, o1, t2, o2) if order[0] is a_cmd else (t2, o2, t1, o1) - if o_a != o_b: - print(f"{name}: OUTPUTS DIFFER in pair {i}: {o_a!r} vs {o_b!r}") - return 1 - pair = {"cpython_s": t_a, "compiled_s": t_b, "ratio": t_a / t_b, "load": load} - if c_cmd is not None: - t_c, o_c = run_once(c_cmd) - if o_c != o_a: - print(f"{name}: OUTPUTS DIFFER in pair {i} (no __slots__): {o_c!r} vs {o_a!r}") - return 1 - pair.update(cpython_noslots_s=t_c, ratio_vs_noslots=t_c / t_b) - pairs.append(pair) - ratios = [p["ratio"] for p in pairs] - results[name] = {"size": size, "compiled_functions": entry[1], "pairs": pairs, - "median_ratio": statistics.median(ratios), "min_ratio": min(ratios), - "max_ratio": max(ratios)} - if c_cmd is not None: - results[name]["median_ratio_vs_noslots"] = statistics.median( - p["ratio_vs_noslots"] for p in pairs) + cmds = commands(name, size, args.python, False, compiled, tools) + r: dict[str, object] = {"size": size} + if args.typedpython: + if isinstance(entry, tuple): + r.update(measure_pair(name, cmds["CPython"], cmds["TypedPython @compiled"], cmds.get(NOSLOTS), + args.pairs, "compiled")) + r["compiled_functions"] = entry[1] + else: + r["skipped"] = entry + extras: dict[str, object] = {} + base = cmds.get(CPY_TOOLS) + for col in TOOL_COLS: + if col in tools["na"]: # type: ignore[operator] + extras[col] = tools["na"][col] # type: ignore[index] + elif col not in cmds or base is None: + extras[col] = "n/a (not built)" + else: + try: + extras[col] = measure_pair(name, base, cmds[col], None, args.pairs, col) + except RunFailed as e: + extras[col] = f"n/a ({e})" + r["tools_pairs"] = extras + results[name] = r print(f"measured {name}", file=sys.stderr) + if any(isinstance(v, dict) and v.get("outputs_differ") for v in [r, *extras.values()]): + print(f"{name}: OUTPUTS DIFFER, see the JSON/table") + return 1 print() print("| benchmark | size | speedup vs CPython (median of pairs) | range | load during pairs (1-min) |") @@ -271,22 +549,69 @@ def interleaved(args: argparse.Namespace) -> int: for name, r in results.items(): if "skipped" in r: print(f"| {name} | - | {r['skipped']} | | |") - continue - loads = [p["load"][0] for p in r["pairs"]] # type: ignore[index] - print(f"| {name} | {r['size']} | {r['median_ratio']:.1f}x | {r['min_ratio']:.1f}–{r['max_ratio']:.1f}x " - f"| {min(loads):.1f}–{max(loads):.1f} |") - print(f"\n{args.pairs} A/B pairs per benchmark, order alternated; ratio = CPython time / compiled time.") + elif "median_ratio" in r: + loads = [p["load"][0] for p in r["pairs"]] # type: ignore[index] + print(f"| {name} | {r['size']} | {r['median_ratio']:.1f}x | {r['min_ratio']:.1f}\u2013{r['max_ratio']:.1f}x " + f"| {min(loads):.1f}\u2013{max(loads):.1f} |") + if args.typedpython: + print(f"\n{args.pairs} A/B pairs per benchmark, order alternated; ratio = CPython time / compiled time.") + print("\n| benchmark | column | speedup vs CPython (tools python), median of pairs | range | extra |") + print("|---|---|---|---|---|") + for name, r in results.items(): + for col, p in r["tools_pairs"].items(): # type: ignore[union-attr] + if isinstance(p, dict): + extra = "" + if col == NUMBA: + extra = (f"first call (compile) median {statistics.median(q['first_call_s'] for q in p['pairs']):.3f} s; " + f"process wall median {statistics.median(q['wall_s'] for q in p['pairs']):.3f} s") + print(f"| {name} | {col} | {p['median_ratio']:.1f}x | {p['min_ratio']:.1f}\u2013{p['max_ratio']:.1f}x | {extra} |") + else: + print(f"| {name} | {col} | {p} | | |") + print(f"\n{args.pairs} pairs per column, order alternated; ratio = CPython (tools python) time / column time.") + report = tools_report(tools) + print_tools_report(report) if args.write: out_dir = ROOT / "results" out_dir.mkdir(exist_ok=True) path = out_dir / f"{datetime.date.today().isoformat()}-{socket.gethostname()}-interleaved.json" path.write_text(json.dumps({"date": datetime.datetime.now().isoformat(timespec="seconds"), "host": socket.gethostname(), "method": "interleaved A/B", - "pairs": args.pairs, "results": results}, indent=2) + "\n") + "pairs": args.pairs, "tools": report, "results": results}, indent=2) + "\n") print(f"wrote {path}") return 0 +def measure_pair(name: str, a_cmd: list[str], b_cmd: list[str], c_cmd: list[str] | None, n_pairs: int, + label: str) -> dict[str, object]: + """n_pairs A/B pairs (A = CPython, B = the column); which runs first alternates every pair.""" + pairs = [] + for i in range(n_pairs): + order = (a_cmd, b_cmd) if i % 2 == 0 else (b_cmd, a_cmd) # alternate which runs first + load = os.getloadavg() + (t1, o1), (t2, o2) = run_once(order[0]), run_once(order[1]) + t_a, o_a, t_b, o_b = (t1, o1, t2, o2) if order[0] is a_cmd else (t2, o2, t1, o1) + if o_a != o_b: + print(f"{name}/{label}: OUTPUTS DIFFER in pair {i}: {o_a!r} vs {o_b!r}") + return {"outputs_differ": True} + pair: dict[str, object] = {"cpython_s": t_a, "compiled_s": t_b, "ratio": t_a / t_b, "load": load} + last = getattr(b_cmd, "last", None) + if getattr(b_cmd, "numba", False) and last: + pair.update(first_call_s=last["first_call_s"], wall_s=last["wall_s"]) + if c_cmd is not None: + t_c, o_c = run_once(c_cmd) + if o_c != o_a: + print(f"{name}: OUTPUTS DIFFER in pair {i} (no __slots__): {o_c!r} vs {o_a!r}") + return {"outputs_differ": True} + pair.update(cpython_noslots_s=t_c, ratio_vs_noslots=t_c / t_b) + pairs.append(pair) + ratios = [p["ratio"] for p in pairs] + out: dict[str, object] = {"pairs": pairs, "median_ratio": statistics.median(ratios), + "min_ratio": min(ratios), "max_ratio": max(ratios)} + if c_cmd is not None: + out["median_ratio_vs_noslots"] = statistics.median(p["ratio_vs_noslots"] for p in pairs) + return out + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = parser.add_subparsers(dest="cmd", required=True) @@ -294,7 +619,7 @@ def main() -> int: v.add_argument("--python", default=sys.executable) m = sub.add_parser("measure") m.add_argument("--python", default=sys.executable) - ab = sub.add_parser("interleaved", help="CPython vs @compiled alternated A/B; reports median ratios") + ab = sub.add_parser("interleaved", help="CPython vs @compiled/Cython/numba alternated A/B; reports median ratios") ab.add_argument("--python", default=sys.executable) ab.add_argument("--pairs", type=int, default=11) ab.add_argument("--write", action="store_true") @@ -304,6 +629,9 @@ def main() -> int: help="directory holding the typedpython package; enables the @compiled column") p.add_argument("--typedpython-python", default=sys.executable, help="interpreter for compiling (needs pyrefly 1.3.2)") + p.add_argument("--tools-python", default=str(TOOLS_PY_DEFAULT), + help="interpreter of the uv venv holding Cython and numba (default: benchmarks/" + "typedpython/.venv/bin/python); enables the Cython and numba columns") m.add_argument("--repeat", type=int, default=5) m.add_argument("--force", action="store_true") m.add_argument("--write", action="store_true")