From c3a74c17c5f381b273c2d80ec75a31c79c51fe23 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sun, 25 Oct 2026 18:47:48 +0900 Subject: [PATCH] Feat: memoryview[float] / memoryview[int] buffer parameters; resize and release during a call raise BufferError (#149) SPEC states that concurrent writes from another thread are unsynchronised, as in CPython. Adds the call-heavy buffer_calls.py benchmark (list vs array.array vs memoryview); not measured yet. --- benchmarks/typedpython/py/buffer_calls.py | 52 ++ docs/SPEC.md | 2 +- .../src/main/python/typedpython/cgen.py | 9 +- .../src/main/python/typedpython/frontend.py | 6 +- .../main/python/typedpython/runtime/API.md | 3 +- .../python/typedpython/runtime/tp_runtime.h | 20 +- .../src/main/python/typedpython/verify.py | 4 +- .../src/test/python/test_buffer_arrays.py | 4 +- .../src/test/python/test_buffer_memoryview.py | 546 ++++++++++++++++++ 9 files changed, 635 insertions(+), 11 deletions(-) create mode 100644 benchmarks/typedpython/py/buffer_calls.py create mode 100644 python-multiplatform-ksp/src/test/python/test_buffer_memoryview.py diff --git a/benchmarks/typedpython/py/buffer_calls.py b/benchmarks/typedpython/py/buffer_calls.py new file mode 100644 index 00000000..c27cf610 --- /dev/null +++ b/benchmarks/typedpython/py/buffer_calls.py @@ -0,0 +1,52 @@ +import array +import sys + + +def bump_list(a: list[float], i: int) -> None: + a[i] += 1.0 + + +def bump_array(a: array.array[float], i: int) -> None: + a[i] += 1.0 + + +def bump_view(a: memoryview[float], i: int) -> None: + a[i] += 1.0 + + +def run_list(a: list[float], calls: int) -> None: + n: int = len(a) + for k in range(calls): + bump_list(a, (k * 7919) % n) + + +def run_array(a: array.array[float], calls: int) -> None: + n: int = len(a) + for k in range(calls): + bump_array(a, (k * 7919) % n) + + +def run_view(a: memoryview[float], calls: int) -> None: + n: int = len(a) + for k in range(calls): + bump_view(a, (k * 7919) % n) + + +def main() -> None: + # argv[1] = number of bump calls, argv[2] = array size (default 1000000). Each call touches one element + # of a large buffer: a list argument is copied in and written back on every call (O(n)), an exact + # array.array / memoryview argument is used in place (SPEC N-8, #149). Output: one checksum per variant. + calls: int = int(sys.argv[1]) + size: int = int(sys.argv[2]) if len(sys.argv) > 2 else 1000000 + as_list: list[float] = [0.0] * size + as_array: array.array[float] = array.array("d", [0.0]) * size + backing: array.array[float] = array.array("d", [0.0]) * size + as_view: memoryview[float] = memoryview(backing) + run_list(as_list, calls) + run_array(as_array, calls) + run_view(as_view, calls) + print(f"{sum(as_list)} {sum(as_array)} {sum(backing)}") + + +if __name__ == "__main__": + main() diff --git a/docs/SPEC.md b/docs/SPEC.md index 86691389..f86497e7 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -688,7 +688,7 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. | Array access in bounds, or a runtime bounds check kept | the check stays (never an unchecked access) | | Array parameters only indexed, stored, `len`-ed; no aliasing (entry guard) | not compiled; diagnostic | | Integer ops proven safe by the verifier's intervals (constants, `len`, `range` variables, guards such as `if x < e` / `while lo < hi`, joined local assignments, `//` and `%` by constants) | emitted unchecked (a shift/mask for `//`/`%` by a power of two of a non-negative value); the sanitizer CI run builds with `TP_CHECK_PROVEN` and turns a wrong proof into a SystemError (#148) | - | `array.array[float]` / `array.array[int]` parameters (exact `array.array`, typecode `d` or `q`/`l`, 8-byte items) are read and written in place: the buffer export is held for the call and released on every exit, and every element access is one relaxed atomic 8-byte load or store, so each access is atomic like the interpreter's `a[i]`; no copy-in and no limit on the number of buffers. Open (#149): while the export is held, another thread's resize of that array raises BufferError. numpy arrays: see the next row | anything else: object path | + | `array.array[float]` / `array.array[int]` parameters (exact `array.array`, typecode `d` or `q`/`l`, 8-byte items) are read and written in place: the buffer export is held for the call and released on every exit, and every element access is one relaxed atomic 8-byte load or store, so each access is atomic like the interpreter's `a[i]`; no copy-in and no limit on the number of buffers. While the export is held, a resize of that array (`append`, `extend`, `del a[i]`, ... from another thread) raises BufferError, exactly as CPython does for any held export; the compiled call never sees a resize. Concurrent writes from another thread to the same elements are not synchronised: as in CPython, no ordering between the two threads' element accesses is guaranteed, only that each access is whole (no torn element). `memoryview[float]` / `memoryview[int]` parameters (#149: exact `memoryview`, 1-D, C-contiguous, format `d` or `q`/`l`, 8-byte aligned, writable when stored to; any exporter: an `array.array`, a numpy array, a `bytearray` cast with `.cast('d')`) take the same path with array.array's element semantics (an element is a Python float / int); the export is taken on the memoryview, so while the call runs `release()` on it raises BufferError, as CPython does for an exported view. An out-of-range index raises memoryview's `IndexError("index out of bounds on dimension 1")`. A released, read-only (when stored to), non-contiguous, multi-dimensional or other-format view deopts: the interpreter raises or runs it. numpy arrays: see the next row | anything else: object path | | numpy `npt.NDArray[np.float64]` / `NDArray[np.int64]` parameters (exact `numpy.ndarray`, 1-D, C-contiguous, native byte order, writable when stored to) take the same buffer path (#220, step 1) when the function accesses the array purely by element (`x[i]`, `len(x)`); a function that uses it as a whole object (`np.sum(x)`, `x.T`, passing `x` on, a slice) compiles as before with the parameter an opaque object. `NDArray[np.bool_]` (#261) is the same path with 1-byte items (buffer format `?`; a `uint8` or other array deopts): an element read is a native bool, boxed as `np.bool` when it escapes; a store compiles only for a bool value (`x[i] = v`, `v: bool`), because numpy converts an int or float by truthiness and that function stays interpreted. Other dtypes are opaque objects. Results follow the interpreter: a value derived from an element is a numpy scalar there, so it is boxed as `np.float64` / `np.int64` / `np.bool` when it escapes (return, `Box`), decided per value at run time by a taint flag; a function declared `-> float` / `int` / `bool` that holds a numpy array returns the interpreter's object. `float(v)` of such a value is a plain float. An out-of-range index raises numpy's `IndexError("index i is out of bounds for axis 0 with size n")`. Arithmetic on a value derived from an element follows numpy's scalar semantics (#220, step 2): float `/`, `//`, `%` never raise ZeroDivisionError (inf / nan, with numpy's result bits and sign rules, `//` and `%` as numpy's `divmod`), int64 `//` and `%` by zero give 0, int64 `/` converts both operands to double. numpy's RuntimeWarning (category and text exactly as numpy's, e.g. `overflow encountered in scalar add`, `divide by zero encountered in scalar divide`, `invalid value encountered in scalar floor_divide`) is emitted once per flagged operation, with the source file and line of the operation. A pure function does not emit it itself: on any floating-point exception (overflow, invalid, divide by zero) or int64 wrap or division by zero it deopts, so the interpreter emits it; an impure function cannot deopt after an effect, so the compiled code emits the warning (`PyErr_WarnExplicitObject`, honouring `warnings` filters and the module's `__warningregistry__`). Float `+ - * / // %` on such values is compiled in both kinds of function, including a store of the result into the array (`x[i] -= d * m`); int64 `/ // %` is compiled in a pure function only. Int64 `+ - *` and `// %` of an element in an impure function (a native INT would have to be a numpy scalar) stay on the object path | anything else: object path | | A function holding an array parameter runs no user code: closed, and no slot read or `C(args)` whose slow path runs user code, in its whole call tree (#203) | not compiled; diagnostic | | OBJ references follow one ownership rule; no IR form for free, borrowed alias or NULL read | not compiled; diagnostic | diff --git a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py index 9bb91967..2d1838fc 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py @@ -186,7 +186,7 @@ class CGenError(Exception): _ARRAY_CT = {Type.F64_ARRAY: "tp_f64_array", Type.I64_ARRAY: "tp_i64_array"} # Buffer arrays (#149): `ArrayParam.buffer` -> the exact type the guard compares against (the index # into the module state's `buf_types`), and the format strings the export must carry. -_BUFFER_KINDS = {None: None, "array": 0, "numpy": 1} +_BUFFER_KINDS = {None: None, "array": 0, "numpy": 1, "memoryview": None} # buf_types slots 2..4: the numpy scalar types a Box.np makes (np.float64, np.int64, np.bool_) _NP_SLOT = {Type.F64: 2, Type.I64: 3, Type.BOOL: 4} _BUFFER_FMT = {Type.F64_ARRAY: ('"d"', "NULL"), Type.I64_ARRAY: ('"q"', '"l"'), Type.BOOL_ARRAY: ('"?"', "NULL")} @@ -2784,9 +2784,10 @@ def wrapper(self) -> str: for i, p in buffers: fmt1, fmt2 = _BUFFER_FMT[p.type] kind = _BUFFER_KINDS[p.buffer] - enter = "tp_buf_enter_np" if p.buffer == "numpy" else "tp_buf_enter" - L.append(f" tp_s = {enter}(tp_args[{i}], " - f"((tp_state *)PyModule_GetState(tp_module))->buf_types[{kind}], " + enter = {"numpy": "tp_buf_enter_np", "memoryview": "tp_buf_enter_mv"}.get(p.buffer, "tp_buf_enter") + exact = ("(PyObject *)&PyMemoryView_Type" if p.buffer == "memoryview" + else f"((tp_state *)PyModule_GetState(tp_module))->buf_types[{kind}]") + L.append(f" tp_s = {enter}(tp_args[{i}], {exact}, " f"{fmt1}, {fmt2}, {int(p.stored)}, &tp_a{i});") L.append(" if (tp_s < 0) goto tp_error;") L.append(" if (tp_s != 0) goto tp_deopt;") diff --git a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py index c4e18f57..7653876f 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py @@ -326,7 +326,7 @@ def _qualified(node: ast.expr, info: _ModuleInfo) -> str | None: def _buffer_kind(annotation: ast.expr, info: _ModuleInfo) -> tuple[Type, str] | None: """The buffer-array reading of an annotation (#149), else None (an ordinary object parameter): `array.array[float]` / `array.array[int]` (any spelling that resolves, through this module's - own imports, to `array.array`). A bare `array.array`, another element type, a numpy + own imports, to `array.array`), and the builtin `memoryview[float]` / `memoryview[int]`. A bare `array.array`, another element type, a numpy `NDArray[...]` (not supported: its element arithmetic is numpy's), or a name bound more than once is not one.""" if not isinstance(annotation, ast.Subscript): @@ -336,6 +336,10 @@ def _buffer_kind(annotation: ast.expr, info: _ModuleInfo) -> tuple[Type, str] | if base == "array.array" and isinstance(arg, ast.Name) and arg.id in ("float", "int") \ and not info.bound(arg.id): return ARRAY_KIND[f"list[{arg.id}]"], "array" + if isinstance(annotation.value, ast.Name) and annotation.value.id == "memoryview" \ + and not info.bound("memoryview") and isinstance(arg, ast.Name) \ + and arg.id in ("float", "int") and not info.bound(arg.id): + return ARRAY_KIND[f"list[{arg.id}]"], "memoryview" if base == "numpy.typing.NDArray": dtype = _qualified(arg, info) if dtype in _NUMPY_DTYPE: diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md index 6691ef7f..30b4cfd6 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md @@ -95,8 +95,9 @@ An exact `array.array` argument (`array.array[float]` / `array.array[int]`; the through the module's own top-level imports) is not copied. typedef struct { Py_buffer view; Py_ssize_t len; void *data; int held; } tp_buf; int tp_buf_enter(PyObject *obj, PyObject *exact, const char *fmt1, const char *fmt2, int writable, tp_buf *b); + int tp_buf_enter_mv(PyObject *obj, PyObject *exact, const char *fmt1, const char *fmt2, int writable, tp_buf *b); /* memoryview (#149): memoryview's IndexError text */ void tp_buf_exit(tp_buf *b); /* PyBuffer_Release if held; zeroes b; idempotent */ - int tp_buf_slot(const tp_buf *b, int64_t i, int store, Py_ssize_t *slot); /* 0 / -1, array.array's IndexError text */ + int tp_buf_slot(const tp_buf *b, int64_t i, int store, Py_ssize_t *slot); /* 0 / -1, array.array's, numpy's or memoryview's IndexError text */ double tp_buf_ld_f64(const tp_buf *b, Py_ssize_t i); void tp_buf_st_f64(const tp_buf *b, Py_ssize_t i, double v); int64_t tp_buf_ld_i64(const tp_buf *b, Py_ssize_t i); void tp_buf_st_i64(const tp_buf *b, Py_ssize_t i, int64_t v); int tp_import_attr(const char *mod, const char *attr, PyObject **out); /* exec slot: new ref or NULL (cleared); -1 MemoryError */ diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h index 73dec7dc..3cca1983 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h @@ -1224,8 +1224,22 @@ static inline int tp_buf_enter_np(PyObject *obj, PyObject *exact, const char *fm return rc; } +/* A memoryview (#149): the same guard on the exact builtin memoryview (the export is taken on the + * view itself, so its release() raises BufferError while the call runs); `np` 2 selects + * memoryview's IndexError text. Elements are Python floats / ints, as for array.array. */ +static inline int tp_buf_enter_mv(PyObject *obj, PyObject *exact, const char *fmt1, const char *fmt2, + int writable, tp_buf *b) +{ + int rc = tp_buf_enter(obj, exact, fmt1, fmt2, writable, b); + if (rc == 0) { + b->np = 2; + } + return rc; +} + /* index: a negative index counts from the end; out of range -> IndexError with the text CPython's - * own array.array uses ("array index out of range" / "array assignment index out of range") */ + * own array.array uses ("array index out of range" / "array assignment index out of range"), + * numpy's (np 1) or memoryview's (np 2) */ static inline int tp_buf_slot(const tp_buf *b, int64_t i, int store, Py_ssize_t *slot) { int64_t orig = i; @@ -1239,6 +1253,10 @@ static inline int tp_buf_slot(const tp_buf *b, int64_t i, int store, Py_ssize_t *slot = (Py_ssize_t)i; return 0; out_of_range: + if (b->np == 2) { + PyErr_SetString(PyExc_IndexError, "index out of bounds on dimension 1"); + return -1; + } if (b->np) { PyErr_Format(PyExc_IndexError, "index %lld is out of bounds for axis 0 with size %zd", (long long)orig, b->len); diff --git a/python-multiplatform-ksp/src/main/python/typedpython/verify.py b/python-multiplatform-ksp/src/main/python/typedpython/verify.py index 10e756cb..6493c8f9 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/verify.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/verify.py @@ -45,7 +45,7 @@ verify/redo `Call.redo` where it is not legal (see below) verify/entry-global-open a function with entry_globals is not closed verify/array-open a function with an ArrayParam runs user code (#203) - verify/buffer a malformed `ArrayParam.buffer` (only None and "array"), or a + verify/buffer a malformed `ArrayParam.buffer` (None, "array", "numpy", "memoryview"), or a CopyArray of a buffer array (#149) verify/par-loop a `ForRange.par` (SPEC N-15, #277) that the verifier does not recompute to the same value: the function is not marked `parallel`, or the loop is @@ -450,7 +450,7 @@ def declarations(self) -> frozenset[str]: self.error(TYPE, f"array parameter '{p.name}' has non-array type {p.type}") elif not isinstance(p.stored, bool): self.error(TYPE, f"array parameter '{p.name}' has a non-bool `stored`") - elif p.buffer not in (None, "array", "numpy"): + elif p.buffer not in (None, "array", "numpy", "memoryview"): self.error(BUFFER, f"array parameter '{p.name}' has buffer kind {p.buffer!r}") else: self.arrays[p.name] = p.type diff --git a/python-multiplatform-ksp/src/test/python/test_buffer_arrays.py b/python-multiplatform-ksp/src/test/python/test_buffer_arrays.py index 5ad603eb..e60d116e 100644 --- a/python-multiplatform-ksp/src/test/python/test_buffer_arrays.py +++ b/python-multiplatform-ksp/src/test/python/test_buffer_arrays.py @@ -472,7 +472,7 @@ def test_verifier_buffer_kind_and_copy_rules(): from test_verify import F64 as VF64, F64A as VF64A, assert_rejected, fn from typedpython.ir import ArrayParam, Assign, Const, CopyArray, Index, Return, Type - for kind in ("ndarray", "memoryview"): # "numpy" is a buffer kind since #220 + for kind in ("ndarray", "bytes"): # "numpy" (#220) and "memoryview" (#149) are buffer kinds assert_rejected("verify/buffer", fn("f", params=[ArrayParam("a", VF64A, False, kind)], returns=VF64, pure=True, body=[Return(Index(VF64, "a", Const(Type.I64, 0)))])) @@ -483,6 +483,8 @@ def test_verifier_buffer_kind_and_copy_rules(): import dataclasses assert_rejected("verify/buffer", dataclasses.replace( copy, params=[ArrayParam("a", VF64A, False, "numpy")])) + assert_rejected("verify/buffer", dataclasses.replace( + copy, params=[ArrayParam("a", VF64A, False, "memoryview")])) def test_verifier_buffer_params_are_not_limited_but_lists_still_are(): diff --git a/python-multiplatform-ksp/src/test/python/test_buffer_memoryview.py b/python-multiplatform-ksp/src/test/python/test_buffer_memoryview.py new file mode 100644 index 00000000..f67cc707 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_buffer_memoryview.py @@ -0,0 +1,546 @@ +"""`memoryview[float]` / `memoryview[int]` parameters on the zero-copy buffer path (issue #149). + +A memoryview parameter takes the same path as `array.array[float]` / `array.array[int]`: the export +is taken on the view, held for the call, released on every exit, and elements are read and written +in place. Anything the path cannot take (released, read-only when stored to, non-contiguous, +multi-dimensional, other format, not a memoryview) goes to the object path and the interpreter +raises or runs it. + +Tests are differential against the interpreted function (`__typedpython_interpreted__`): same +result, same exception type and text, same final contents. numpy cases skip without numpy. +""" +from __future__ import annotations + +import array +import textwrap +import threading +import time + +import pytest + +from typedpython import cbuild +from test_buffer_arrays import (Built, build_source, deopts, interp, load_so, outcome, + same_value) + +SOURCE = textwrap.dedent(''' + # typedpython: compiled + import array + + + def sum_mf(m: memoryview[float]) -> float: + s = 0.0 + for i in range(len(m)): + s += m[i] + return s + + + def sum_mi(m: memoryview[int]) -> int: + s = 0 + for i in range(len(m)): + s += m[i] + return s + + + def get_mf(m: memoryview[float], i: int) -> float: + return m[i] + + + def get_mi(m: memoryview[int], i: int) -> int: + return m[i] + + + def set_mf(m: memoryview[float], i: int, v: float) -> None: + m[i] = v + + + def set_mi(m: memoryview[int], i: int, v: int) -> None: + m[i] = v + + + def length_mf(m: memoryview[float]) -> int: + return len(m) + + + def scale_mf(m: memoryview[float], k: float) -> None: + for i in range(len(m)): + m[i] = m[i] * k + + + def spin_mf(m: memoryview[float], n: int) -> float: + s = 0.0 + L = len(m) + for k in range(n): + s += m[k % L] + return s + + + def spin_af(a: array.array[float], n: int) -> float: + s = 0.0 + L = len(a) + for k in range(n): + s += a[k % L] + return s +''') + +COMPILED = {"sum_mf", "sum_mi", "get_mf", "get_mi", "set_mf", "set_mi", "length_mf", "scale_mf", + "spin_mf", "spin_af"} + +F = [1.5, -2.25, 3.0, 4.5] +I = [3, -5, 7, 11] + + +@pytest.fixture(scope="module") +def built(tmp_path_factory): + d = tmp_path_factory.mktemp("mvmod") + path, proved, diags, c = build_source(d, SOURCE, "mvmod") + so = cbuild.build(c, path.stem, d / "out") + return Built(proved, diags, c, load_so(so)) + + +def MF(v=F): return memoryview(array.array("d", v)) +def MI(v=I): return memoryview(array.array("q", v)) + + +def contents(x): + if isinstance(x, memoryview): + try: + return ("mv", x.tolist()) + except ValueError as e: + return ("released", str(e)) + if isinstance(x, bytearray): + return ("seq", list(x)) + if isinstance(x, (array.array, list)): + return ("seq", list(x)) + return ("other", None) + + +def same_contents(a, b): + if a[0] != b[0]: + return False + if a[0] in ("mv", "seq"): + return len(a[1]) == len(b[1]) and all(same_value(p, q) for p, q in zip(a[1], b[1])) + return a == b + + +def differential(m, name, make, expect_deopt=False): + """Compiled vs interpreted on two fresh argument sets; also compares the final contents of + every argument (viewed through the memoryview, and through its owner when `make` returns a + trailing owners tuple).""" + ca, ia = make(), make() + cargs, cown = ca[:-1], ca[-1] + iargs, iown = ia[:-1], ia[-1] + before = deopts(m) + c = outcome(getattr(m, name), cargs) + after = deopts(m) + i = outcome(interp(m, name), iargs) + assert c[0] == i[0], (c, i) + if c[0] == "ok": + assert same_value(c[1], i[1]), (c, i) + else: + assert c[1:] == i[1:], (c, i) + for x, y in zip(list(cargs) + list(cown), list(iargs) + list(iown)): + assert same_contents(contents(x), contents(y)), (contents(x), contents(y)) + if expect_deopt: + assert after == before + 1, "expected a deopt (an object-path call), got none" + else: + assert after == before, "unexpected deopt: the compiled path did not run" + return c + + +def as_list(owner, fmt): + if isinstance(owner, bytearray): + return memoryview(owner).cast(fmt).tolist() + return list(owner) + + +def mk(view, *rest): + """Argument factory: view() builds (memoryview, owner); rest are plain extra arguments.""" + def make(): + v, owner = view() + return (v,) + rest + ((owner,),) + return make + + +def v_array_d(vals=F): + a = array.array("d", vals) + return memoryview(a), a + + +def v_array_q(vals=I): + a = array.array("q", vals) + return memoryview(a), a + + +def v_array_l(vals=I): + a = array.array("l", vals) + return memoryview(a), a + + +def v_bytearray_d(vals=F): + b = bytearray(array.array("d", vals).tobytes()) + return memoryview(b).cast("d"), b + + +def v_bytearray_q(vals=I): + b = bytearray(array.array("q", vals).tobytes()) + return memoryview(b).cast("q"), b + + +def v_numpy_d(vals=F): + np = pytest.importorskip("numpy") + a = np.array(vals, dtype=np.float64) + return memoryview(a), a + + +def v_numpy_q(vals=I): + np = pytest.importorskip("numpy") + a = np.array(vals, dtype=np.int64) + return memoryview(a), a + + +FLOAT_VIEWS = [v_array_d, v_bytearray_d, v_numpy_d] +INT_VIEWS = [v_array_q, v_array_l, v_bytearray_q, v_numpy_q] + + +# --- what is compiled ------------------------------------------------------------------------------ + +def test_memoryview_params_are_in_the_compiled_set(built): + names = {f.name for f in built.proved.functions} + assert names >= COMPILED, (sorted(COMPILED - names), built.proved.skipped) + + +def test_ir_shape_marks_memoryview_params(built): + by = {f.name: f for f in built.proved.functions} + assert (by["sum_mf"].params[0].buffer, by["sum_mf"].params[0].stored) == ("memoryview", False) + assert (by["sum_mi"].params[0].buffer, by["sum_mi"].params[0].stored) == ("memoryview", False) + assert by["scale_mf"].params[0].stored is True + assert by["set_mi"].params[0].buffer == "memoryview" + assert by["spin_af"].params[0].buffer == "array" + + +# --- reads, len, negative index, writes ------------------------------------------------------------ + +@pytest.mark.parametrize("view", FLOAT_VIEWS) +def test_float_reads_len_and_negative_index(built, view): + m = built.module + differential(m, "sum_mf", mk(view)) + differential(m, "length_mf", mk(view)) + differential(m, "sum_mf", mk(lambda: view([]))) + for i in (0, 2, -1, -4): + differential(m, "get_mf", mk(view, i)) + + +@pytest.mark.parametrize("view", INT_VIEWS) +def test_int_reads_and_negative_index(built, view): + m = built.module + differential(m, "sum_mi", mk(view)) + for i in (0, 3, -1, -4): + differential(m, "get_mi", mk(view, i)) + + +@pytest.mark.parametrize("view", FLOAT_VIEWS) +def test_float_writes_are_visible_to_the_caller(built, view): + m = built.module + differential(m, "set_mf", mk(view, 1, 9.5)) + differential(m, "set_mf", mk(view, -1, 9.5)) + differential(m, "scale_mf", mk(view, 2.0)) + v, owner = view() + before = deopts(m) + m.set_mf(v, 1, 42.0) + m.scale_mf(v, 2.0) + assert deopts(m) == before + assert v.tolist() == [3.0, 84.0, 6.0, 9.0] + assert as_list(owner, "d") == [3.0, 84.0, 6.0, 9.0] + + +@pytest.mark.parametrize("view", INT_VIEWS) +def test_int_writes_are_visible_to_the_caller(built, view): + m = built.module + differential(m, "set_mi", mk(view, 0, -77)) + differential(m, "set_mi", mk(view, 2, 2 ** 40)) + v, owner = view() + m.set_mi(v, 3, 123) + assert as_list(owner, "q") == [3, -5, 7, 123] + + +def test_view_of_a_slice_of_the_owner_is_the_same_memory(built): + a = array.array("d", F) + v = memoryview(a)[1:3] # 1-D, contiguous, offset by one element + differential(built.module, "sum_mf", lambda: (memoryview(array.array("d", F))[1:3], ())) + built.module.set_mf(v, 0, 100.0) + assert a.tolist() == [1.5, 100.0, 3.0, 4.5] + + +# --- index errors ---------------------------------------------------------------------------------- + +@pytest.mark.parametrize("view", FLOAT_VIEWS) +def test_out_of_range_read_and_store_raise_memoryview_indexerror(built, view): + m = built.module + for name, args in (("get_mf", (4,)), ("get_mf", (-5,)), ("set_mf", (4, 1.0)), + ("set_mf", (-5, 1.0))): + c = differential(m, name, mk(view, *args)) + assert c[0] == "raise" and c[1] is IndexError + assert c[2] == "index out of bounds on dimension 1", c + + +def test_out_of_range_on_empty_and_int_views(built): + m = built.module + c = differential(m, "get_mf", mk(lambda: v_array_d([]), 0)) + assert c[1] is IndexError and c[2] == "index out of bounds on dimension 1" + c = differential(m, "get_mi", mk(v_array_q, 99)) + assert c[1] is IndexError and c[2] == "index out of bounds on dimension 1" + c = differential(m, "set_mi", mk(v_array_q, -5, 1)) + assert c[1] is IndexError and c[2] == "index out of bounds on dimension 1" + + +def test_wrong_value_type_matches_the_interpreter(built): + # a memoryview 'd' store of a non-float: the interpreter's TypeError, not a silent conversion + differential(built.module, "set_mf", mk(v_array_d, 0, 1), expect_deopt=True) + + +# --- object path: the interpreter raises or runs it ------------------------------------------------- + +def test_read_only_view_read_is_compiled_and_store_deopts(built): + m = built.module + ro = lambda: (memoryview(bytes(array.array("d", F).tobytes())).cast("d"), ()) # noqa: E731 + differential(m, "sum_mf", ro) # reading needs no write access + c = differential(m, "set_mf", lambda: ro()[:1] + (0, 1.0, ()), expect_deopt=True) + assert c[0] == "raise" and c[1] is TypeError, c + c = differential(m, "scale_mf", lambda: ro()[:1] + (2.0, ()), expect_deopt=True) + assert c[0] == "raise" + # all-zero bytes object, as in the issue + c = differential(m, "set_mf", + lambda: (memoryview(bytes(16)).cast("d"), 0, 1.0, ()), expect_deopt=True) + assert c[0] == "raise" and c[1] is TypeError + + +def test_non_contiguous_view_takes_the_object_path(built): + m = built.module + mkv = lambda: (memoryview(array.array("d", [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]))[::2], ()) # noqa: E731 + differential(m, "sum_mf", mkv, expect_deopt=True) + differential(m, "length_mf", mkv, expect_deopt=True) + differential(m, "get_mf", lambda: mkv()[:1] + (1, ()), expect_deopt=True) + differential(m, "scale_mf", lambda: mkv()[:1] + (3.0, ()), expect_deopt=True) + + +def test_two_dimensional_cast_takes_the_object_path(built): + m = built.module + mkv = lambda: (memoryview(array.array("d", [1.0, 2.0, 3.0, 4.0])).cast("B").cast("d", (2, 2)), ()) # noqa: E731 + c = differential(m, "sum_mf", mkv, expect_deopt=True) + assert c[0] == "raise" + differential(m, "length_mf", mkv, expect_deopt=True) + differential(m, "get_mf", lambda: mkv()[:1] + (0, ()), expect_deopt=True) + + +def test_other_formats_take_the_object_path(built): + m = built.module + f = lambda: (memoryview(array.array("f", [1.0, 2.0])), ()) # noqa: E731 + differential(m, "sum_mf", f, expect_deopt=True) + differential(m, "get_mf", lambda: f()[:1] + (0, ()), expect_deopt=True) + i = lambda: (memoryview(array.array("i", [1, 2])), ()) # noqa: E731 + differential(m, "sum_mi", i, expect_deopt=True) + differential(m, "sum_mi", lambda: (memoryview(array.array("d", F)), ()), expect_deopt=True) + differential(m, "sum_mf", lambda: (memoryview(array.array("q", I)), ()), expect_deopt=True) + differential(m, "sum_mi", lambda: (memoryview(b"abcd"), ()), expect_deopt=True) + + +def test_released_view_takes_the_object_path_and_raises_valueerror(built): + m = built.module + + def released(): + v, owner = v_array_d() + v.release() + return v, (owner,) + for name, extra in (("sum_mf", ()), ("length_mf", ()), ("get_mf", (0,)), ("set_mf", (0, 1.0))): + def make(extra=extra): + v, owner = released() + return (v,) + extra + (owner,) + c = differential(m, name, make, expect_deopt=True) + assert c[0] == "raise" and c[1] is ValueError, c + + +@pytest.mark.parametrize("make_arg", [ + lambda: list(F), + lambda: array.array("d", F), + lambda: bytearray(32), + lambda: bytes(32), + lambda: None, +]) +def test_not_a_memoryview_takes_the_object_path(built, make_arg): + differential(built.module, "sum_mf", lambda: (make_arg(), ()), expect_deopt=True) + differential(built.module, "length_mf", lambda: (make_arg(), ()), expect_deopt=True) + + +def test_memoryview_subclass_like_wrong_kind_in_array_param(built): + # a memoryview where an array.array[float] is expected stays on the object path + differential(built.module, "spin_af", lambda: (MF(), 4, ()), expect_deopt=True) + + +# --- the export is released after ok, error and deopt ---------------------------------------------- + +def can_resize(a): + try: + a.append(0) + except BufferError: + return False + return True + + +def release_ok(v): + try: + v.release() + except BufferError: + return False + return True + + +def test_export_released_after_ok_error_and_deopt(built): + m = built.module + # ok + a = array.array("d", F) + v = memoryview(a) + before = deopts(m) + m.sum_mf(v) + assert deopts(m) == before + assert release_ok(v) and can_resize(a) + # error (index) + a = array.array("d", F) + v = memoryview(a) + with pytest.raises(IndexError): + m.get_mf(v, 99) + with pytest.raises(IndexError): + m.set_mf(v, 99, 1.0) + assert release_ok(v) and can_resize(a) + # deopt after entry: i64 overflow in the sum + a = array.array("q", [2 ** 62, 2 ** 62, 2 ** 62]) + v = memoryview(a) + before = deopts(m) + assert m.sum_mi(v) == 3 * 2 ** 62 + assert deopts(m) == before + 1 + assert release_ok(v) and can_resize(a) + # deopt at the guard: wrong format for the parameter + a = array.array("d", F) + v = memoryview(a) + before = deopts(m) + c = outcome(m.sum_mi, (v,)) + assert deopts(m) == before + 1 + assert c == outcome(interp(m, "sum_mi"), (v,)) + assert release_ok(v) and can_resize(a) + # deopt: read-only store (export of a read-only view must not linger) + ro = memoryview(bytes(16)).cast("d") + with pytest.raises(TypeError): + m.set_mf(ro, 0, 1.0) + assert release_ok(ro) + # the view itself stays usable after the call (it was not released by the call) + a = array.array("d", F) + v = memoryview(a) + m.scale_mf(v, 2.0) + assert v.tolist() == [x * 2 for x in F] + v.release() + a.append(1.0) + + +def test_export_is_held_only_during_the_call(built): + a = array.array("d", F) + v = memoryview(a) + built.module.sum_mf(v) + a2 = array.array("d", F) + built.module.sum_mf(memoryview(a2)) # a temporary view: nothing keeps a2 pinned + a2.append(1.0) + del v + a.append(1.0) + assert len(a) == 5 and len(a2) == 5 + + +# --- another thread while a compiled call runs ------------------------------------------------------ + +N = 1 << 16 +SPIN = 600_000_000 + + +def calibrated_spin(built): + """Pick an iteration count so that the compiled call runs about 0.6 s on this machine.""" + a = array.array("d", [1.0] * N) + t0 = time.perf_counter() + built.module.spin_mf(memoryview(a), 20_000_000) + dt = max(time.perf_counter() - t0, 1e-4) + return int(20_000_000 * 0.6 / dt) + + +def probe_during_call(call, probes, attempts=3): + """Run call() on the main thread while a worker applies each probe in a loop. Returns + (result, {probe_name: (BufferError count, other outcomes)}). Retries `attempts` times when a + probe never observed the call (machine too fast or too loaded).""" + for _ in range(attempts): + go, done = threading.Event(), threading.Event() + seen = {name: [0, []] for name in probes} + + def worker(): + go.wait() + time.sleep(0.05) + while not done.is_set(): + for name, probe in probes.items(): + try: + probe() + except BufferError: + seen[name][0] += 1 + except Exception as e: # noqa: BLE001 + seen[name][1].append(repr(e)) + else: + seen[name][1].append("succeeded") + return + time.sleep(0.001) + + t = threading.Thread(target=worker, daemon=True) + t.start() + go.set() + try: + result = call() + finally: + done.set() + t.join(timeout=30) + assert not t.is_alive() + if all(s[0] > 0 for s in seen.values()): + break + return result, {k: tuple(v) for k, v in seen.items()} + + +def test_other_thread_release_and_append_get_bufferror_during_a_memoryview_call(built): + n = calibrated_spin(built) + a = array.array("d", [1.0] * N) + v = memoryview(a) + expected = interp(built.module, "spin_mf")(memoryview(array.array("d", [1.0] * N)), 1000) + assert expected == 1000.0 + result, seen = probe_during_call( + lambda: built.module.spin_mf(v, n), + {"release": v.release, "append": lambda: a.append(9.0)}) + assert result == float(n) + assert seen["release"][0] > 0 and seen["release"][1] == [], seen + assert seen["append"][0] > 0 and seen["append"][1] == [], seen + assert len(a) == N # nothing resized under the call + assert release_ok(v) and can_resize(a) # held only during the call + + +def test_other_thread_append_gets_bufferror_during_an_array_call(built): + n = calibrated_spin(built) + a = array.array("d", [1.0] * N) + result, seen = probe_during_call( + lambda: built.module.spin_af(a, n), + {"append": lambda: a.append(9.0), "extend": lambda: a.extend([1.0]), + "pop": lambda: a.pop()}) + assert result == float(n) + for name in ("append", "extend", "pop"): + assert seen[name][0] > 0 and seen[name][1] == [], (name, seen) + assert len(a) == N + assert can_resize(a) + + +def test_numpy_backed_view_blocks_release_during_a_call(built): + np = pytest.importorskip("numpy") + n = calibrated_spin(built) + a = np.ones(N, dtype=np.float64) + v = memoryview(a) + result, seen = probe_during_call(lambda: built.module.spin_mf(v, n), {"release": v.release}) + assert result == float(n) + assert seen["release"][0] > 0 and seen["release"][1] == [], seen + assert release_ok(v)