From bd57e839e191f4e606b3023f6fa6daaa1cbd00de Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 17 Oct 2026 23:36:35 +0900 Subject: [PATCH 1/2] WIP: Compile methods of fixed-layout classes (#155), stopped mid-work, unverified --- .../src/main/python/typedpython/cgen.py | 282 +++++- .../src/main/python/typedpython/cost.py | 10 +- .../src/main/python/typedpython/frontend.py | 140 ++- .../src/main/python/typedpython/ir.py | 22 + .../main/python/typedpython/runtime/API.md | 61 ++ .../python/typedpython/runtime/tp_runtime.h | 87 ++ .../src/main/python/typedpython/verify.py | 55 +- .../src/test/python/test_class_methods.py | 927 ++++++++++++++++++ 8 files changed, 1553 insertions(+), 31 deletions(-) create mode 100644 python-multiplatform-ksp/src/test/python/test_class_methods.py diff --git a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py index 0a9bfe0d..dab6b3f4 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py @@ -151,6 +151,21 @@ `__typedpython_class_info__()` reports, per class, the captured and current tags, the slow-path and refresh counters and whether the module dict is watched (introspection for tests). +Methods (SPEC N-11, issue #155). An `ir.Function` with an `owner` is a method of that compiled class. +It has an impl and a wrapper like any function (its `self` is `Param(OBJ, cls=owner)`, guarded through +`TP_SELF_EXACT`, so a `self` that is not exactly the class runs the original), but the module dict is +not touched. The exec slot, after the class captures and for each class that is compiled +(`cls_ok`): takes the class's OWN dict entry (it must be a plain Python function, else the class is +marked not compiled), stores it in `__typedpython_interpreted__["C.m"]`, creates a `tp_method_obj` +(`_TEMPLATE_METHOD_TYPE`: an immutable heap type, vectorcall, `tp_descr_get` like a function, +`__name__`/`__qualname__`/`__doc__`/`__module__` forwarded, `__wrapped__` the original), replaces the +dict entry with it, and finally `tp_class_retag` adopts the class's new version tag as the proved tag +(the fields and `__init__` were not touched). `ir.CheckMethod` (`ev_CheckMethod`) is how a pure caller +reaches the impl directly: `tp_methods_current` = `type(obj) is C` and (one tag compare against +`cls_rt[k].mtag`, else `tp_methods_refresh`) every installed method still is, in C's own dict, the very +descriptor installed (`tp_st->mdescrs`), and the class shape is unchanged; a failure is a deopt (pure +functions only). Module-state fields `mtype` and `mdescrs[]` are written by the exec slot only. + Generated arithmetic is in separate C statements and the file sets `#pragma STDC FP_CONTRACT OFF` (cbuild also passes -ffp-contract=off): a fused multiply-add would round differently from CPython. """ @@ -333,7 +348,7 @@ def _can_deopt(e: ir.Expr, functions: dict[str, ir.Function]) -> bool: return not e.proven # a proven op cannot deopt (no overflow, or a shift / mask) if isinstance(e, ir.UnaryOp) and e.op is ir.UnaryOpKind.NEG and e.operand.type is Type.I64: return not e.proven - if isinstance(e, (ir.Unbox, ir.CheckExact)): + if isinstance(e, (ir.Unbox, ir.CheckExact, ir.CheckMethod)): return True if isinstance(e, ir.Call): if e.redo: @@ -353,7 +368,7 @@ def _has_cls_param(f: ir.Function) -> bool: def _direct_classes(f: ir.Function) -> set[str]: out = {p.cls for p in f.params if isinstance(p, ir.Param) and p.cls is not None} for e in _all_exprs(f): - if isinstance(e, (ir.IsExact, ir.CheckExact, ir.FieldGet, ir.New)): + if isinstance(e, (ir.IsExact, ir.CheckExact, ir.CheckMethod, ir.FieldGet, ir.New)): out.add(e.cls) for s in _walk_stmts(f.body): if isinstance(s, ir.FieldSet): @@ -361,11 +376,13 @@ def _direct_classes(f: ir.Function) -> set[str]: return out -def _param_cls_fail(obj_c: str, k: int, optional: bool) -> str: +def _param_cls_fail(obj_c: str, k: int, optional: bool, method_self: bool = False) -> str: """The C condition under which a `Param.cls` guard FAILS for the borrowed argument `obj_c` (class slot `k` of the module state): not exactly the class, and not None when optional. - Emitted both in the wrapper and at the top of the impl (module docstring).""" - exact = f"tp_is_exact({obj_c}, (PyTypeObject *)tp_st->classes[{k}])" + Emitted both in the wrapper and at the top of the impl (module docstring). `method_self`: the + `self` of a compiled method, guarded through the single mutation point TP_SELF_EXACT.""" + test = "TP_SELF_EXACT" if method_self else "tp_is_exact" + exact = f"{test}({obj_c}, (PyTypeObject *)tp_st->classes[{k}])" return f"!({exact} || {obj_c} == Py_None)" if optional else f"!{exact}" @@ -474,6 +491,27 @@ def __init__(self, module: ir.Module): self.slot_base[cd.name] = total total += len(cd.fields) self.n_slots = total + # Methods (module docstring, "Methods"): compiled functions with an `owner`. They are installed + # in their class's dict, not in the module dict, and have no PyMethodDef. + self.plain = [f for f in module.functions if f.owner is None] + self.methods_of: dict[str, list[ir.Function]] = {cd.name: [] for cd in module.classes} + for f in module.functions: + if f.owner is not None: + if f.owner not in self.classes: + raise CGenError(f"{f.name}: owner {f.owner!r} has no ClassDecl in the module") + if f.name != f"{f.owner}.{f.name[len(f.owner) + 1:]}" or not f.name[len(f.owner) + 1:].isidentifier(): + raise CGenError(f"{f.name}: a method of {f.owner} is named {f.owner}.") + p0 = f.params[0] if f.params else None + if not (isinstance(p0, ir.Param) and p0.type is Type.OBJ and p0.cls == f.owner + and not p0.optional): + raise CGenError(f"{f.name}: the first parameter of a method must be self guarded {f.owner}") + self.methods_of[f.owner].append(f) + self.method_base: dict[str, int] = {} + total_methods = 0 + for cd in module.classes: + self.method_base[cd.name] = total_methods + total_methods += len(self.methods_of[cd.name]) + self.n_methods = total_methods self._uses: dict[str, set[str]] = {} self.bounded = _bounded_functions(self.functions) # name -> k (module docstring, "Fast entry") @@ -529,6 +567,17 @@ def globals_read(self, f: ir.Function) -> list[str]: todo.append(e.function) return sorted(out, key=lambda n: self.class_index.get(n, -1)) + def method(self, fname: str, cls: str, method: str) -> ir.Function: + f = self.functions.get(f"{cls}.{method}") + if f is None or f.owner != cls: + raise CGenError(f"{fname}: CheckMethod names {cls}.{method}, which is not a compiled method") + return f + + def methods_c(self, cls: str) -> tuple[str, int, str]: + """(C array of the installed method names, their count, C descriptor-array pointer) of `cls`.""" + k = self.class_index[cls] + return f"tp_mnames_{k}", len(self.methods_of[cls]), f"&tp_st->mdescrs[{self.method_base[cls]}]" + def watches_globals(self) -> bool: return any(isinstance(e, (ir.New, ir.IsExact)) for f in self.module.functions for e in _all_exprs(f)) @@ -572,6 +621,7 @@ def generate(module: ir.Module, source_path: Path, display_path: str | None = No w(f"/* Generated by typedpython.cgen from {source_path.name}, do not edit. */") w("#define PY_SSIZE_T_CLEAN") w("#include ") + w("#include ") w("#include ") w('#include "tp_runtime.h"') w("") @@ -597,6 +647,8 @@ def generate(module: ir.Module, source_path: Path, display_path: str | None = No w(" int watch_id1; /* dict watcher id + 1 on the module dict; 0 = none */") w(" PyObject *breaker; /* `lambda: None`; calling it runs the eval breaker (#141) */") w(" PyObject *buf_types[1]; /* strong: array.array (buffer arrays, #149); NULL: not importable */") + w(" PyObject *mtype; /* strong: the descriptor type of compiled methods; NULL: none */") + w(f" PyObject *mdescrs[{mg.n_methods + 1}]; /* strong: the installed descriptor of each method */") w("} tp_state;") w("") w("static const char *const tp_name_strings[] = {") @@ -608,9 +660,14 @@ def generate(module: ir.Module, source_path: Path, display_path: str | None = No for cd in module.classes: items = "".join(f"{c_string(x)}, " for x in cd.fields) w(f"static const char *const tp_fields_{mg.class_index[cd.name]}[] = {{{items}NULL}};") + for cd in module.classes: + items = "".join(f"{c_string(f.name[len(cd.name) + 1:])}, " for f in mg.methods_of[cd.name]) + w(f"static const char *const tp_mnames_{mg.class_index[cd.name]}[] = {{{items}NULL}};") names = "".join(f"{c_string(cd.name)}, " for cd in module.classes) w(f"static const char *const tp_class_names[] = {{{names}NULL}};") w("") + if mg.n_methods: + w(_TEMPLATE_METHOD_TYPE) if module.classes and mg.watches_globals(): w("/* Dict watcher of the module dict (tp_globals_watch): a class-name key changed. */") w("static int tp_watch_cb(PyDict_WatchEvent event, PyObject *dict, PyObject *key, PyObject *value)") @@ -629,7 +686,7 @@ def generate(module: ir.Module, source_path: Path, display_path: str | None = No w(b) # method table w("static PyMethodDef tp_methods[] = {") - for f in module.functions: + for f in mg.plain: w(f" {{{c_string(f.name)}, (PyCFunction)(void (*)(void)){_ident('tp_wrap', f.name)}, " f"METH_FASTCALL | METH_KEYWORDS, NULL}},") w("#ifdef TP_TRACE_FAST") @@ -1247,6 +1304,21 @@ def ev_CheckExact(self, e: ir.CheckExact) -> str: self.emit(f"if (!tp_is_exact({o}, (PyTypeObject *)tp_st->classes[{k}])) {{ tp_rc = 1; goto tp_exit; }}") return o + def ev_CheckMethod(self, e: ir.CheckMethod) -> str: + """`obj.method` is still the compiled method: `type(obj) is C` and every installed method of C + is still the object in C's own dict (tp_methods_current); else a deopt (legal in a pure + function only, _check_function). The receiver is checked here, before the call's other + operands are evaluated, where LOAD_ATTR is.""" + k = self._class(e.cls) + self.mg.method(self.f.name, e.cls, e.method) + o = self._obj_operand(e.obj, "CheckMethod") + names_c, n, descrs_c = self.mg.methods_c(e.cls) + self.emit(f"tp_s = tp_methods_current({o}, tp_st->classes[{k}], &tp_st->cls_rt[{k}], " + f"{names_c}, {n}, {descrs_c});") + self.emit("if (tp_s < 0) { tp_rc = -1; goto tp_exit; }") + self.emit(f"if (!tp_s) {{ tp_class_note_slow(&tp_st->cls_rt[{k}]); tp_rc = 1; goto tp_exit; }}") + return o + def ev_FieldGet(self, e: ir.FieldGet) -> str: slot = self.mg.slot(self.f.name, e.cls, e.field) o = self._obj_operand(e.obj, "FieldGet") @@ -1777,7 +1849,7 @@ def impl(self) -> str: for p in f.params: if isinstance(p, ir.Param) and p.cls is not None: k = self._class(p.cls) - out.append(f" if ({_param_cls_fail(_ident('p', p.name), k, p.optional)}) return 1;") + out.append(f" if ({_param_cls_fail(_ident('p', p.name), k, p.optional, f.owner is not None and p is f.params[0])}) return 1;") k_bound = self.mg.bounded.get(f.name) if self.fast: # Fast twin of a bounded function (module docstring, "Fast entry"): the caller proved @@ -1870,7 +1942,7 @@ def wrapper(self) -> str: L.append(" if (!(" + " && ".join(f"tp_st->cls_ok[{self.mg.class_index[n]}]" for n in classes) + ")) goto tp_deopt;") for i, p in cls_params: - L.append(f" if ({_param_cls_fail(f'tp_args[{i}]', self._class(p.cls), p.optional)}) goto tp_deopt;") + L.append(f" if ({_param_cls_fail(f'tp_args[{i}]', self._class(p.cls), p.optional, f.owner is not None and i == 0)}) goto tp_deopt;") if classes and f.entry_globals and not f.pure: # module docstring, rule 8: a closed impure function must not reach a slow path for cname in classes: @@ -1974,6 +2046,162 @@ def wrapper(self) -> str: # --- fixed parts of the module ------------------------------------------------------------------- +_TEMPLATE_METHOD_TYPE = r''' +/* Compiled methods (module docstring, "Methods"). The type of the object that replaces a function + * in a compiled class's dict. IMMUTABLE after tp_exec: the type has Py_TPFLAGS_IMMUTABLETYPE and each + * object's fields are written once, by tp_method_new, before it is published in the class dict. */ +typedef PyObject *(*tp_method_fn)(PyObject *module, PyObject *const *args, Py_ssize_t nargs, + PyObject *kwnames); + +typedef struct { + PyObject_HEAD + vectorcallfunc vectorcall; + PyObject *module; /* strong: the extension module (its state holds the class guards) */ + PyObject *func; /* strong: the original Python function (`__wrapped__`) */ + tp_method_fn fn; /* the generated wrapper: guards, impl, or the interpreted redo */ +} tp_method_obj; + +static PyObject *tp_method_vectorcall(PyObject *callable, PyObject *const *args, size_t nargsf, + PyObject *kwnames) +{ + tp_method_obj *m = (tp_method_obj *)callable; + return m->fn(m->module, args, PyVectorcall_NARGS(nargsf), kwnames); +} + +/* Binds like a function (func_descr_get): C.m is the object itself, obj.m a bound method. */ +static PyObject *tp_method_get(PyObject *self, PyObject *obj, PyObject *type) +{ + (void)type; + if (obj == NULL || obj == Py_None) { + return Py_NewRef(self); + } + return PyMethod_New(self, obj); +} + +/* The attributes a function shows are the original function's (the five of SPEC N-11 / API.md + * "Methods"); `__wrapped__` is a member. Looked up before the type's own `__doc__` / `__module__`. */ +static PyObject *tp_method_getattro(PyObject *self, PyObject *name) +{ + if (PyUnicode_Check(name) + && (PyUnicode_CompareWithASCIIString(name, "__name__") == 0 + || PyUnicode_CompareWithASCIIString(name, "__qualname__") == 0 + || PyUnicode_CompareWithASCIIString(name, "__doc__") == 0 + || PyUnicode_CompareWithASCIIString(name, "__module__") == 0)) { + return PyObject_GetAttr(((tp_method_obj *)self)->func, name); + } + return PyObject_GenericGetAttr(self, name); +} + +static PyObject *tp_method_repr(PyObject *self) +{ + PyObject *q = PyObject_GetAttrString(((tp_method_obj *)self)->func, "__qualname__"), *r; + if (q == NULL) { + return NULL; + } + r = PyUnicode_FromFormat("", q, (void *)self); + Py_DECREF(q); + return r; +} + +static int tp_method_traverse(PyObject *self, visitproc visit, void *arg) +{ + tp_method_obj *m = (tp_method_obj *)self; + Py_VISIT(Py_TYPE(self)); + Py_VISIT(m->module); + Py_VISIT(m->func); + return 0; +} + +static int tp_method_clear(PyObject *self) +{ + tp_method_obj *m = (tp_method_obj *)self; + Py_CLEAR(m->module); + Py_CLEAR(m->func); + return 0; +} + +static void tp_method_dealloc(PyObject *self) +{ + PyTypeObject *tp = Py_TYPE(self); + PyObject_GC_UnTrack(self); + (void)tp_method_clear(self); + tp->tp_free(self); + Py_DECREF(tp); +} + +static PyMemberDef tp_method_members[] = { + {"__vectorcalloffset__", Py_T_PYSSIZET, offsetof(tp_method_obj, vectorcall), Py_READONLY, NULL}, + {"__wrapped__", Py_T_OBJECT_EX, offsetof(tp_method_obj, func), Py_READONLY, NULL}, + {NULL, 0, 0, 0, NULL} +}; + +static PyType_Slot tp_method_slots[] = { + {Py_tp_dealloc, (void *)tp_method_dealloc}, + {Py_tp_traverse, (void *)tp_method_traverse}, + {Py_tp_clear, (void *)tp_method_clear}, + {Py_tp_repr, (void *)tp_method_repr}, + {Py_tp_getattro, (void *)tp_method_getattro}, + {Py_tp_descr_get, (void *)tp_method_get}, + {Py_tp_call, (void *)PyVectorcall_Call}, + {Py_tp_members, (void *)tp_method_members}, + {0, NULL} +}; + +static PyType_Spec tp_method_spec = { + "typedpython.compiled_function", sizeof(tp_method_obj), 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE + | Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_METHOD_DESCRIPTOR + | Py_TPFLAGS_HAVE_VECTORCALL, + tp_method_slots +}; + +/* Install the descriptor for method `name` of `cls`: the class's own entry must be a plain Python + * function. The original is stored in `table` under `qname` whatever it is. 0 = installed (*out is + * the strong reference the module state keeps), 1 = not installed (absent or not a function: the + * class is then not compiled), -1 = error. Init time only. */ +static int tp_method_install(PyObject *type, PyObject *module, PyObject *cls, const char *name, + PyObject *qname, tp_method_fn fn, PyObject *table, PyObject **out) +{ + PyObject *orig = NULL; + tp_method_obj *d = NULL; + int found, rc = -1; + + found = tp_own_attr((PyTypeObject *)cls, name, &orig); + if (found < 0) { + return -1; + } + if (found == 0) { + return 1; + } + if (PyDict_SetItem(table, qname, orig) < 0) { + goto done; + } + if (!PyFunction_Check(orig)) { /* exactly a Python function: no decorator wrapper */ + rc = 1; + goto done; + } + d = PyObject_GC_New(tp_method_obj, (PyTypeObject *)type); + if (d == NULL) { + goto done; + } + d->vectorcall = tp_method_vectorcall; + d->module = Py_NewRef(module); + d->func = orig; + orig = NULL; /* moved into the descriptor */ + d->fn = fn; + PyObject_GC_Track((PyObject *)d); + if (PyObject_SetAttrString(cls, name, (PyObject *)d) < 0) { + Py_DECREF((PyObject *)d); + goto done; + } + *out = (PyObject *)d; /* the module state keeps the reference */ + rc = 0; +done: + Py_XDECREF(orig); + return rc; +} +''' + _TEMPLATE_REDO = r''' /* Deopt: count it, then call __typedpython_interpreted__[name] with the original arguments. */ static PyObject *tp_cg_redo(PyObject *tp_module, int tp_name, PyObject *const *tp_args, @@ -2027,6 +2255,8 @@ def _state_functions(mg: _ModGen) -> str: " size_t i;", " if (tp_st == NULL) return 0;", " Py_VISIT(tp_st->breaker);", + " Py_VISIT(tp_st->mtype);", + " for (i = 0; i < sizeof(tp_st->mdescrs) / sizeof(tp_st->mdescrs[0]); i++) Py_VISIT(tp_st->mdescrs[i]);", " for (i = 0; i < 1; i++) Py_VISIT(tp_st->buf_types[i]);", " for (i = 0; i < sizeof(tp_st->names) / sizeof(tp_st->names[0]); i++) Py_VISIT(tp_st->names[i]);", " for (i = 0; i < sizeof(tp_st->kwnames) / sizeof(tp_st->kwnames[0]); i++) Py_VISIT(tp_st->kwnames[i]);", @@ -2043,6 +2273,8 @@ def _state_functions(mg: _ModGen) -> str: " if (tp_st == NULL) return 0;", " if (tp_st->watch_id1 > 0) tp_globals_unwatch(PyModule_GetDict(tp_module), &tp_st->watch_id1);", " tp_release(&tp_st->breaker);", + " for (i = 0; i < sizeof(tp_st->mdescrs) / sizeof(tp_st->mdescrs[0]); i++) tp_release(&tp_st->mdescrs[i]);", + " tp_release(&tp_st->mtype);", " for (i = 0; i < 1; i++) tp_release(&tp_st->buf_types[i]);", " for (i = 0; i < sizeof(tp_st->names) / sizeof(tp_st->names[0]); i++) tp_release(&tp_st->names[i]);", " for (i = 0; i < sizeof(tp_st->kwnames) / sizeof(tp_st->kwnames[0]); i++) tp_release(&tp_st->kwnames[i]);", @@ -2075,14 +2307,16 @@ def _class_info_function(mg: _ModGen) -> str: k = mg.class_index[cd.name] L += [ f" tp_c = tp_st->classes[{k}];", - " tp_d = Py_BuildValue(\"{s:O,s:I,s:I,s:O,s:n,s:n,s:O}\",", + " tp_d = Py_BuildValue(\"{s:O,s:I,s:I,s:O,s:n,s:n,s:O,s:n,s:O}\",", f" \"compiled\", tp_st->cls_ok[{k}] ? Py_True : Py_False,", f" \"captured_tag\", tp_class_rt_tag(&tp_st->cls_rt[{k}]),", " \"tag\", tp_class_tag(tp_c),", f" \"current\", (tp_c != NULL && tp_class_tag(tp_c) == tp_class_rt_tag(&tp_st->cls_rt[{k}])) ? Py_True : Py_False,", f" \"slow\", tp_class_rt_slow(&tp_st->cls_rt[{k}]), \"refreshed\", " f"tp_class_rt_refreshed(&tp_st->cls_rt[{k}]),", - " \"watched\", tp_st->watch_id1 > 0 ? Py_True : Py_False);", + " \"watched\", tp_st->watch_id1 > 0 ? Py_True : Py_False,", + f" \"methods\", (Py_ssize_t){len(mg.methods_of[cd.name])},", + f" \"methods_current\", (tp_c != NULL && tp_class_tag(tp_c) == tp_atomic_load_u32(&tp_st->cls_rt[{k}].mtag)) ? Py_True : Py_False);", f" if (tp_d == NULL || PyDict_SetItemString(tp_out, {c_string(cd.name)}, tp_d) < 0) {{", " Py_XDECREF(tp_d); Py_DECREF(tp_out); return NULL; }", " Py_DECREF(tp_d);", @@ -2161,6 +2395,30 @@ def _exec_function(mg: _ModGen) -> str: " }", " tp_release(&tp_c);", ] + if mg.n_methods: + L += [ + " tp_st->mtype = PyType_FromModuleAndSpec(tp_module, &tp_method_spec, NULL);", + " if (tp_st->mtype == NULL) goto tp_done;", + ] + for cd in mg.module.classes: + ms = mg.methods_of[cd.name] + if not ms: + continue + j = mg.class_index[cd.name] + L += [f" if (tp_st->cls_ok[{j}]) do {{"] + for i, f in enumerate(ms): + L += [ + f" tp_k = tp_method_install(tp_st->mtype, tp_module, tp_st->classes[{j}], " + f"{c_string(f.name[len(cd.name) + 1:])}, tp_st->names[{mg.name_index(f.name)}], " + f"{_ident('tp_wrap', f.name)}, tp_table, &tp_st->mdescrs[{mg.method_base[cd.name] + i}]);", + " if (tp_k < 0) goto tp_done;", + f" if (tp_k > 0) {{ tp_st->cls_ok[{j}] = 0; break; }}", + ] + L += [ + f" if (!tp_class_retag((PyTypeObject *)tp_st->classes[{j}], &tp_st->cls_rt[{j}])) " + f"tp_st->cls_ok[{j}] = 0;", + " } while (0);", + ] if mg.module.classes: if mg.watches_globals(): # after the source ran: from here a rebinding of a class name moves tp_globals_epoch @@ -2176,7 +2434,7 @@ def _exec_function(mg: _ModGen) -> str: " tp_modname = PyModule_GetNameObject(tp_module);", " if (tp_modname == NULL) goto tp_done;", ] - for j, f in enumerate(mg.module.functions): + for j, f in enumerate(mg.plain): k = mg.name_index(f.name) L += [ f" tp_k = PyDict_GetItemRef(tp_dict, tp_st->names[{k}], &tp_f);", @@ -2189,7 +2447,7 @@ def _exec_function(mg: _ModGen) -> str: ] L += [ "#ifdef TP_TRACE_FAST", - f" tp_cf = PyCFunction_NewEx(&tp_methods[{len(mg.module.functions)}], tp_module, tp_modname);", + f" tp_cf = PyCFunction_NewEx(&tp_methods[{len(mg.plain)}], tp_module, tp_modname);", " if (tp_cf == NULL || PyDict_SetItemString(tp_dict, \"__tp_trace_fast_hits__\", tp_cf) < 0) goto tp_done;", " tp_release(&tp_cf);", "#endif", diff --git a/python-multiplatform-ksp/src/main/python/typedpython/cost.py b/python-multiplatform-ksp/src/main/python/typedpython/cost.py index bd95614a..55fde944 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/cost.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/cost.py @@ -26,6 +26,9 @@ the target of a direct C `Call`, anything that calls a rejected function is rejected too (the same propagation `verify` uses for an unverified callee). +Methods (`ir.Function.owner`, SPEC N-11) are functions here: the same rule decides each one, and a method +that is rejected stays the original Python function in its class (it is simply not installed). + Straight-line impure functions (no loop at all) are kept: they have no loop to lose to the interpreter, and that keeps every function that compiled before #150 compiled (fixed-layout setters, Kotlin calls). The rule targets loops over objects only. @@ -103,8 +106,11 @@ def apply(module: ir.Module, diagnostics: list[Diagnostic]) -> tuple[ir.Module, for fn in module.functions: if fn.name in reasons: continue - bad = sorted({n.function for n in _walk(fn.body) - if isinstance(n, ir.Call) and n.function in reasons}) + # (a CheckMethod names its method too: the descriptors are built from the kept methods) + bad = sorted({n.function if isinstance(n, ir.Call) else f"{n.cls}.{n.method}" + for n in _walk(fn.body) + if (isinstance(n, ir.Call) and n.function in reasons) + or (isinstance(n, ir.CheckMethod) and f"{n.cls}.{n.method}" in reasons)}) if bad: reasons[fn.name] = (f"{RULE}: calls '{bad[0]}', which is not compiled " f"({reasons[bad[0]]})") diff --git a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py index fcffa094..5c852322 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py @@ -42,6 +42,15 @@ `CallObject(Global(C))`. `trivial_init` requires the i-th `__init__` parameter to be stored into the i-th slot (ir.ClassDecl: "`__init__(self, *fields-in-order)`"); other orders stay an object call. +Methods (SPEC N-11, #155). The plain instance methods of a `ClassDecl` class (`_method_targets`: no +decorator, not a dunder, once in the class body, no `super()`/`__class__`, `self` unannotated or +annotated with the class) are lowered like functions under the IR name `C.m` (`ir.Function.owner`), with +`self` as `Param(OBJ, cls=C)`. Every other method of such a class is in `Module.skipped` under `C.m` with +the reason. `obj.m(args)` on a local of class C (`method_call`) is `Call("C.m", (CheckMethod(obj, C, m), +*args))` in a pure function when the callee is a pure compiled method with no entry globals and matching +argument types; in every other case it stays the generic `CallObject(GetAttr(obj, m))`, which reaches the +installed descriptor through CPython's own lookup. + Objects, str and dict code (#150). A function may also use, as object operations (each is the C API call the interpreter's bytecode makes, ir.py says which): `str` constants, `{}` and `[]`, `o[i]` (`GetItem`) and `o[i] = v` (`StoreItem`) on anything that is not a native array, `x in c` / @@ -161,6 +170,12 @@ class _Signature: tuple_returns: tuple[Type, ...] | None = None tuple_ellipsis: bool = False # `tuple[int, ...]`: a variable length return_annotation: str = "" + owner: str | None = None # the compiled class this is a method of (ir.Function.owner) + + @property + def qname(self) -> str: + """The IR name: `f`, or `C.m` for a method (a module function `m` cannot collide).""" + return f"{self.owner}.{self.node.name}" if self.owner else self.node.name def lower(path: Path) -> ir.Module: @@ -183,14 +198,23 @@ def lower(path: Path) -> ir.Module: signatures[node.name] = _signature(node, info.classes, info) except Skip as e: skipped[node.name] = e.reason + for owner, node, reason in _method_targets(tree, info): + q = f"{owner}.{node.name}" + if reason is not None: + skipped[q] = reason + continue + try: + signatures[q] = _signature(node, info.classes, info, owner=owner) + except Skip as e: + skipped[q] = e.reason - kinds = _inferred_kinds(path, source, tree, [s.node for s in signatures.values()]) + kinds = _inferred_kinds(path, source, tree, [(q, s.node) for q, s in signatures.items()]) # Optimistic start: every candidate lowered, pure, not deopting; then iterate to a fixpoint # (purity is the greatest fixpoint through calls, may_deopt the least). env: dict[str, ir.Function] = { name: ir.Function(name=name, params=s.params, returns=s.returns, locals={}, body=(), pure=True, - may_deopt=False, source_line=s.node.lineno) + may_deopt=False, source_line=s.node.lineno, owner=s.owner) for name, s in signatures.items() } results: dict[str, ir.Function | str] = {} @@ -552,8 +576,53 @@ def _targets( return found +def _method_targets( + tree: ast.Module, info: _ModuleInfo, +) -> list[tuple[str, ast.FunctionDef, str | None]]: + """(class, def, reason it stays interpreted or None) for every `def` in the body of a class that + is a `ClassDecl` (SPEC N-11, #155). Only a plain instance method is a candidate: no decorator, + not a dunder (the interpreter runs those: `__init__` is lowered by `trivial_init`, the rest + would change what `repr`, `==`, `del` ... do when compiled), not a name-mangled `__private`, + defined once in the class body, not using `super()` or `__class__` (the compiler creates the + cell for the original function only), and `self` is unannotated or annotated with the class.""" + found: list[tuple[str, ast.FunctionDef, str | None]] = [] + for node in tree.body: + if not isinstance(node, ast.ClassDef) or node.name not in info.classes: + continue + counts: dict[str, int] = {} + for st in node.body: + if isinstance(st, (ast.FunctionDef, ast.AsyncFunctionDef)): + counts[st.name] = counts.get(st.name, 0) + 1 + for st in node.body: + if isinstance(st, ast.AsyncFunctionDef): + found.append((node.name, st, f"line {st.lineno}: async method is not supported")) # type: ignore[arg-type] + continue + if not isinstance(st, ast.FunctionDef): + continue + at = f"line {st.lineno}" + reason = None + if st.decorator_list: + reason = (f"{at}: decorator `@{_src(st.decorator_list[0])}` on a method is not " + "supported (a plain instance method only)") + elif st.name.startswith("__"): + reason = (f"{at}: `{st.name}` is a dunder or name-mangled method; it stays " + "interpreted (the interpreter runs it)") + elif counts[st.name] > 1: + reason = f"{at}: `{st.name}` is defined more than once in the class body" + elif not st.args.args and not st.args.posonlyargs: + reason = f"{at}: a method without a positional `self` parameter" + else: + used = next((n for n in ast.walk(st) if isinstance(n, ast.Name) + and n.id in ("super", "__class__")), None) + if used is not None: + reason = (f"{at}: `{used.id}` needs the class cell of the original function; " + "the method stays interpreted") + found.append((node.name, st, reason)) + return found + + def _signature(fn: ast.FunctionDef, classes: dict[str, ir.ClassDecl] | None = None, - info: _ModuleInfo | None = None) -> _Signature: + info: _ModuleInfo | None = None, owner: str | None = None) -> _Signature: args = fn.args line = fn.lineno if args.posonlyargs: @@ -567,7 +636,14 @@ def _signature(fn: ast.FunctionDef, classes: dict[str, ir.ClassDecl] | None = No if args.defaults: raise Skip(line, "default parameter values are not supported") params: list[ir.Param | ir.ArrayParam] = [] - for a in args.args: + for position, a in enumerate(args.args): + if owner is not None and position == 0: + # `self` (ir.Function.owner): unannotated, or annotated with the class itself + if a.annotation is not None and _annotation_kind(a.annotation).strip("'\"") != owner: + raise Skip(a.lineno, f"`{a.arg}` is annotated `{_src(a.annotation)}`, not the " + f"class `{owner}`") + params.append(ir.Param(a.arg, Type.OBJ, owner, False)) + continue if a.annotation is None: raise Skip(a.lineno, f"parameter `{a.arg}` has no annotation") kind = _annotation_kind(a.annotation) @@ -594,7 +670,7 @@ def _signature(fn: ast.FunctionDef, classes: dict[str, ir.ClassDecl] | None = No else: returns = SCALAR_KIND.get(_annotation_kind(fn.returns), Type.OBJ) elements, ellipsis = _tuple_annotation(fn.returns) - return _Signature(fn, tuple(params), returns, elements, ellipsis, _src(fn.returns)) + return _Signature(fn, tuple(params), returns, elements, ellipsis, _src(fn.returns), owner) def _tuple_annotation(annotation: ast.expr) -> tuple[tuple[Type, ...] | None, bool]: @@ -618,11 +694,12 @@ def _annotation_kind(annotation: ast.expr) -> str: # --- local types from Pyrefly -------------------------------------------------------------------- def _inferred_kinds( - path: Path, source: str, tree: ast.Module, functions: list[ast.FunctionDef], + path: Path, source: str, tree: ast.Module, functions: list[tuple[str, ast.FunctionDef]], ) -> dict[str, dict[str, list[tuple[int, str]]]]: - """function -> local -> [(line, base type)] for every unannotated assignment (Pyrefly).""" + """IR function name -> local -> [(line, base type)] for every unannotated assignment (Pyrefly).""" scopes = rebinding._scopes(tree) - wanted_scopes = {scopes.index(fn) for fn in functions} + keys = {id(fn): q for q, fn in functions} + wanted_scopes = {scopes.index(fn) for _, fn in functions} wanted = [a for a in rebinding.assignments(tree) if a.scope in wanted_scopes] if not wanted: return {} @@ -642,7 +719,7 @@ def _inferred_kinds( spelled = types.get((p.line, p.column), "Unknown") function = scopes[a.scope] assert isinstance(function, ast.FunctionDef) - found.setdefault(function.name, {}).setdefault(p.name, []).append( + found.setdefault(keys[id(function)], {}).setdefault(p.name, []).append( (a.line, rebinding._base(spelled))) return found @@ -653,7 +730,7 @@ def _lower_function( sig: _Signature, env: dict[str, ir.Function], info: _ModuleInfo, kinds: dict[str, dict[str, list[tuple[int, str]]]], ) -> ir.Function | str: - inferred = kinds.get(sig.node.name, {}) + inferred = kinds.get(sig.qname, {}) pure_error: Skip | None = None try: fn = _lower_in_mode(sig, env, info, inferred, impure=False) @@ -823,9 +900,10 @@ def run(self) -> ir.Function: assert pure or not self.deopts or not self.impure_mode, \ "impure mode emitted a deopting node" return ir.Function( - name=self.fn.name, params=params, returns=self.sig.returns, locals=locals_, body=body, + name=self.sig.qname, params=params, returns=self.sig.returns, locals=locals_, body=body, pure=pure, may_deopt=self.deopts, source_line=self.fn.lineno, - entry_globals=tuple(ir.Param(n, t) for n, t in self.entry_used.items())) + entry_globals=tuple(ir.Param(n, t) for n, t in self.entry_used.items()), + owner=self.sig.owner) # --- names and types --- @@ -1205,7 +1283,7 @@ def arg_proved(self, e: ir.Expr, p: ir.Param) -> bool: """The argument `e` is certainly accepted by the callee's entry guard for `p.cls`.""" if isinstance(e, ir.Const) and e.value is None: return p.optional - if isinstance(e, (ir.New, ir.CheckExact)): + if isinstance(e, (ir.New, ir.CheckExact, ir.CheckMethod)): return e.cls == p.cls if isinstance(e, ir.Local) and e.type == Type.OBJ: cls, exact = self.cls_of_name(e.name) @@ -1778,6 +1856,11 @@ def call(self, node: ast.Call) -> ir.Expr: if math is not None: return math + if isinstance(func, ast.Attribute) and plain: + direct = self.method_call(func, node) + if direct is not None: + return direct + if isinstance(func, ast.Name) and func.id in self.arrays: raise self.array_misuse(func.id, node) callee = self.boxed(self.sub(func), func) @@ -1823,6 +1906,37 @@ def direct_call(self, name: str, node: ast.Call) -> ir.Expr | None: for p, e, a in zip(callee.params, args, node.args)) return ir.Call(callee.returns, name, typed, redo=redo) + def method_call(self, func: ast.Attribute, node: ast.Call) -> ir.Expr | None: + """`obj.m(args)` where `obj` is a local of compiled class C and `C.m` a compiled method: the + direct C call `Call("C.m", (CheckMethod(obj, C, m), *args))` (ir.CheckMethod), only in a pure + function and only to a pure callee, because the check is a deopt. None: the generic path + (`CallObject(GetAttr(obj, m), args)`, CPython's own lookup, which finds the descriptor the + module installed) lowers the call, decided before any argument is lowered. A callee with + entry globals is left to the generic path too (its guard reads would need the caller's).""" + if self.impure_mode or not (isinstance(func.value, ast.Name) and self.is_local(func.value.id)): + return None + cls, _ = self.cls_of_name(func.value.id) + callee = self.env.get(f"{cls}.{func.attr}") if cls is not None else None + if callee is None or callee.owner != cls or not callee.pure or callee.entry_globals \ + or len(node.args) != len(callee.params) - 1 \ + or any(isinstance(p, ir.ArrayParam) for p in callee.params) \ + or self.local_type(func.value.id, node.lineno) != Type.OBJ: + return None + assert cls is not None + recv = ir.CheckMethod(Type.OBJ, ir.Local(Type.OBJ, func.value.id), cls, func.attr) + self.hoist_ok = False + args = [recv] + [self.value(a) for a in node.args] + for p, e in zip(callee.params, args): + if p.type != Type.OBJ and e.type != p.type: + # the callee's entry guard would deopt; the impure retry (which takes the generic + # path) keeps CPython's result + raise Skip(node.lineno, f"argument of type {TYPE_NAME[e.type]} for parameter " + f"`{p.name}` of `{cls}.{func.attr}`") + self.deopts = True # the CheckMethod (and a class argument's guard) can deopt + typed = tuple(self.boxed(e, a) if p.type == Type.OBJ else e + for p, e, a in zip(callee.params, args, [func] + list(node.args))) + return ir.Call(callee.returns, f"{cls}.{func.attr}", typed) + def math_call(self, attr: str, node: ast.Call) -> ir.Expr | None: try: func = ir.MathFunc(attr) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/ir.py b/python-multiplatform-ksp/src/main/python/typedpython/ir.py index 6d1c613a..a5cdef90 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/ir.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/ir.py @@ -454,6 +454,23 @@ class CheckExact(Expr): cls: str +@dataclass(frozen=True) +class CheckMethod(Expr): + """`obj` itself (type OBJ), after checking that `obj.method` is the compiled method `cls.method` + CPython would find (SPEC N-11, issue #155): `type(obj) is C` AND every method the module installed + in C is still the very object in C's own dict (nobody rebound `C.method` or any other compiled + method since init). Anything else → deopt, so it is legal only in a pure function, the same rule + as CheckExact; it is how a pure compiled method or function calls `obj.method(args)` as a direct + C call: the front end emits `Call("C.method", (CheckMethod(obj, C, method), *args))`. No user + code runs. The check comes first among the call's operands, before the arguments are evaluated, + where CPython's LOAD_ATTR is. The verifier accepts the result as an exact-C proof (it is that, at + the check). `Module.functions` must hold a Function named `C.method` whose `owner` is C.""" + + obj: Expr + cls: str + method: str + + @dataclass(frozen=True) class FieldGet(Expr): """`obj.field` of a compiled fixed-layout class C whose exact type is proved (a dominating true @@ -709,6 +726,11 @@ class Function: # only to closed functions), so nothing can # rebind the global between entry and use, and reading it once equals reading it at each use. entry_globals: tuple[Param, ...] = () + # A method of the compiled fixed-layout class `owner` (SPEC N-11, #155): `name` is + # "." and params[0] is `self` as Param(OBJ, cls=owner, optional=False). The + # back end does not replace a module global with it; it installs a descriptor in the class dict + # (cgen.py "Methods"). None: a module-level function. + owner: str | None = None @dataclass(frozen=True) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md index 64e181e5..6aa9d66f 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md @@ -241,6 +241,8 @@ Guard state, one per ClassDecl in the module state (zeroed = not compiled): PyObject *init; /* IMMUTABLE after capture. strong: the class's own __init__ at capture, or NULL */ Py_ssize_t slow; /* ATOMIC. failed per-access checks (introspection) */ Py_ssize_t refreshed; /* ATOMIC. new tags adopted by tp_class_refresh (introspection) */ + unsigned int mtag; /* ATOMIC. tp_version_tag at which every installed method was the very object in the class dict (#155); 0 until tp_class_retag */ + unsigned int mbad_tag; /* ATOMIC. a tag at which tp_methods_refresh failed (0: none); not retried */ } tp_class_rt; static uint64_t tp_globals_epoch; /* ATOMIC. moves when a watched module dict changes a class-name key */ /* atomic accessors (relaxed loads, acq_rel adds); every read of cls->tp_version_tag goes through one */ @@ -313,6 +315,61 @@ Slot access and the slow paths: FieldGet's slow path is `tp_getattr`, New's is `tp_global` + `tp_call` (the binding loaded before the arguments), IsExact's compares with `tp_global`; in a pure function every slow path is a deopt. +### Methods (ir.Function.owner / ir.CheckMethod; SPEC N-11, issue #155) + +A plain instance method of a compiled class is an ir.Function named `C.m` with `owner = "C"`. The module +init (after the class guards are captured) replaces the class dict's function by a descriptor object +of a type the extension defines, stores the original function in `__typedpython_interpreted__["C.m"]` +and adopts the class's new version tag: + + typedef PyObject *(*tp_method_fn)(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames); + typedef struct { PyObject_HEAD; vectorcallfunc vectorcall; PyObject *module; PyObject *func; tp_method_fn fn; } tp_method_obj; + /* generated: tp_method_install(type, module, cls, name, qname, fn, table, &out): the class's OWN entry + must be a plain function (PyFunction_Check): 0 installed, 1 not (class not compiled), -1 error */ + int tp_class_retag(PyTypeObject *cls, tp_class_rt *rt); /* init only: adopt the current tag as tag and mtag */ + /* CheckMethod: 1 = type(obj) is cls and every installed method name in `names` is, in cls's own dict, + the very descriptor in `descrs` (mtag fast path, else tp_methods_refresh); 0 = deopt; -1 error */ + int tp_methods_current(PyObject *obj, PyObject *cls, tp_class_rt *rt, const char *const *names, + Py_ssize_t n, PyObject *const *descrs); + int tp_methods_refresh(PyTypeObject *cls, tp_class_rt *rt, const char *const *names, Py_ssize_t n, + PyObject *const *descrs); /* also re-proves tp_class_shape_ok */ + #define TP_SELF_EXACT(o, cls) tp_is_exact(o, cls) /* the one guard of a method's `self` */ + +What the descriptor does (the type is `typedpython.compiled_function`, immutable, not instantiable, +GC-tracked, `Py_TPFLAGS_METHOD_DESCRIPTOR | Py_TPFLAGS_HAVE_VECTORCALL`): + + * `C.m` is the descriptor itself, `obj.m` is a `types.MethodType` over it (`tp_descr_get` is + `func_descr_get`'s rule); a call goes `vectorcall -> tp_wrap_C.m`, the same wrapper a module-level + function has: arity or any keyword, a `self` that is not exactly `C` (a subclass instance, `C.m(5)`), + a failed guard, or a deopt of the impl runs the ORIGINAL function with the same arguments, so + CPython's own result or error comes out. The deopt counter counts it. + * `self` is `Param(OBJ, cls=C)`; its guard is `TP_SELF_EXACT` in the wrapper and at the top of the impl. + * A call `obj.m(args)` in a PURE compiled function whose receiver is a local of class C and whose + callee is a pure compiled method is `Call("C.m", (CheckMethod(obj, C, m), *args))`: a direct C call + guarded by `tp_methods_current` (one type compare, one tag compare). If someone rebound any + compiled method of C, added `__getattribute__` ..., or the receiver is not exactly C, the pure + caller deopts and CPython's own lookup runs in the redo. Impure callers, calls with keywords, + callees with entry globals and argument type mismatches use `CallObject(GetAttr(obj, m))`, which finds + the descriptor through `tp_getattr`. + * Rebinding or deleting `C.m` replaces or removes the descriptor (CPython semantics); other + descriptors of the class are untouched. + +Observable differences from a function (recorded; each is tested): + + * `type(C.__dict__["m"])` is `typedpython.compiled_function` (not `function`); `isinstance(C.m, + types.FunctionType)` is False; `inspect.isfunction` is False, `callable` is True. + * `repr(C.m)` is ``; `repr(obj.m)` is the bound-method repr + (``), `obj.m.__func__ is C.m`. + * Forwarded from the original function on every access: `__name__`, `__qualname__`, `__doc__`, + `__module__`. `__wrapped__` is the original function (so `inspect.signature` and `inspect.unwrap` + see it). NOT forwarded (AttributeError): `__code__`, `__globals__`, `__defaults__`, `__kwdefaults__`, + `__annotations__`, `__closure__`, `__dict__`, `__type_params__`. Nothing can be assigned to it. + * A module-level compiled function is a builtin function (`PyCFunction`) with `__name__` and `__module__` + only: no `__wrapped__`, no forwarded `__doc__`. The two differ on purpose; a method needs the + forwarding for `help()`, `inspect` and pickling a bound method (`__reduce__` reads `__name__`). + * A subclass is a different layout case: a subclass instance never takes the compiled path, even + through an inherited method (guard fails, original runs). + Version tags (CPython 3.14 Objects/typeobject.c): PyType_Modified sets tp_version_tag to 0 for the type and its subclasses; assign_version_tag gives a heap type `NEXT_VERSION_TAG(interp)++`, a per-interpreter counter that only grows and, once it wraps to 0, assigns nothing again; a class @@ -333,6 +390,10 @@ static or module-state field must be added here): | `tp_pc` (loop countdown) | local of each impl | per call | a C local | | `tp_globals_epoch` | tp_runtime.h static | atomic | relaxed loads, acq_rel add in the watcher callback (any thread); the epoch + dict lookup that remember a binding run in the module dict's critical section | | `cls_rt[k].tag`, `.bad_tag`, `.gepoch` | module state | atomic | relaxed `tp_atomic_*`; a stale tag is never reused (tags only grow), so a lost or reordered refresh costs one more refresh | +| `cls_rt[k].mtag`, `.mbad_tag` | module state | atomic | relaxed `tp_atomic_*`, as `tag`; `tp_class_retag` writes `mtag` plainly at init only | +| `mtype` (descriptor type), `mdescrs[]` | module state | immutable after init | written by `tp_exec` (and `tp_clear`); the type is `Py_TPFLAGS_IMMUTABLETYPE` | +| each `tp_method_obj` (`module`, `func`, `fn`, `vectorcall`) | heap object in a class dict | immutable | written once by `tp_method_install` before it is published in the class dict; `tp_method_get` / call read only | +| `tp_mnames_*`, `tp_method_spec`, `tp_method_slots`, `tp_method_members` | generated statics | immutable | never written after import | | `cls_rt[k].slow`, `.refreshed` | module state | atomic | `tp_atomic_add_ssize` (introspection only) | | `cls->tp_version_tag` | CPython type | atomic read | `tp_type_tag` (CPython clears and reassigns it from any thread) | | `cls_rt[k].init`, `names`, `kwnames`, `classes`, `cls_ok`, `slots`, `descrs`, `breaker`, `watch_id1` | module state | immutable after init | written by `tp_exec` (and `tp_clear` at teardown, when no call can run); the `PyMemberDef`s live in the classes the state holds | diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h index 7521ff6d..1bd5bc6f 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h @@ -1521,6 +1521,10 @@ typedef struct { IMMUTABLE after tp_class_capture */ Py_ssize_t slow; /* per-access checks that failed (introspection only). ATOMIC */ Py_ssize_t refreshed; /* new tags adopted by tp_class_refresh (introspection only). ATOMIC */ + unsigned int mtag; /* tp_version_tag at which every compiled method installed in the class was + the very object in the class's own dict (tp_methods_refresh); 0 until + tp_class_retag. ATOMIC */ + unsigned int mbad_tag; /* a tag at which tp_methods_refresh failed (0: none); not retried. ATOMIC */ } tp_class_rt; /* Moves whenever a watched module dict changes a key that is (or may be) a compiled class name, is @@ -1610,6 +1614,8 @@ static inline void tp_class_rt_clear(tp_class_rt *rt, PyObject **descrs, Py_ssiz rt->tag = 0; rt->bad_tag = 0; rt->gepoch = 0; + rt->mtag = 0; + rt->mbad_tag = 0; } /* At module init: the class object must be what ClassDecl promises (tp_class_shape_ok) and every @@ -1749,6 +1755,83 @@ static inline int tp_class_current(PyObject *obj, PyObject *cls_obj, tp_class_rt return tp_class_refresh(cls, rt, fields, n, descrs); } +/* Methods (SPEC N-11, #155). At module init, after the module installed its descriptors in the class + * dict (which moved the class's version tag): adopt the new tag as the proved tag of the slots (the + * fields and __init__ were not touched) and of the methods. Init time, plain stores. 1 = done, 0 = the + * class cannot get a tag (the caller marks it not compiled). */ +static inline int tp_class_retag(PyTypeObject *cls, tp_class_rt *rt) +{ + if (!PyUnstable_Type_AssignVersionTag(cls) || tp_type_tag(cls) == 0) { + return 0; + } + rt->tag = tp_type_tag(cls); + rt->bad_tag = 0; + rt->mtag = rt->tag; + rt->mbad_tag = 0; + return 1; +} + +/* The tag moved (or is 0). Re-prove that `obj.m` still finds the compiled method for every installed + * method: the shape still holds (no __getattribute__ / __setattr__ ... added since: they would run + * instead of the plain lookup), and the class's own dict entry of each name is the very descriptor + * object the module installed (identity, descrs[i] keeps it alive). No user code runs. The tag is + * read BEFORE the dict is examined, as in tp_class_refresh. 1 = current (tag adopted), 0 = not, + * -1 = error (a failed dict lookup). */ +static inline int tp_methods_refresh(PyTypeObject *cls, tp_class_rt *rt, const char *const *names, + Py_ssize_t n, PyObject *const *descrs) +{ + unsigned int tag; + Py_ssize_t i; + PyObject *d; + int found; + + if (!PyUnstable_Type_AssignVersionTag(cls) || tp_type_tag(cls) == 0) { + return 0; + } + tag = tp_type_tag(cls); + if (tag == tp_atomic_load_u32(&rt->mtag)) { + return 1; + } + if (tag == tp_atomic_load_u32(&rt->mbad_tag)) { + return 0; + } + if (!tp_class_shape_ok(cls)) { + goto bad; + } + for (i = 0; i < n; i++) { + found = tp_own_attr(cls, names[i], &d); + if (found < 0) { + return -1; + } + Py_XDECREF(d); + if (d != descrs[i]) { + goto bad; + } + } + tp_atomic_store_u32(&rt->mtag, tag); + return 1; +bad: + tp_atomic_store_u32(&rt->mbad_tag, tag); + return 0; +} + +/* CheckMethod: `type(obj) is cls` and `obj.m` finds the compiled method installed for it, for every + * name in `names`. 1 = yes, 0 = no (deopt), -1 = error. Fast case: one type compare, one tag compare. */ +static inline int tp_methods_current(PyObject *obj, PyObject *cls_obj, tp_class_rt *rt, + const char *const *names, Py_ssize_t n, PyObject *const *descrs) +{ + PyTypeObject *cls = (PyTypeObject *)cls_obj; + unsigned int mtag; + if (cls == NULL || !Py_IS_TYPE(obj, cls)) { + return 0; + } + mtag = tp_atomic_load_u32(&rt->mtag); + if (mtag != 0 && tp_type_tag(cls) == mtag) { /* mtag is 0 until tp_class_retag */ + return 1; + } + return tp_methods_refresh(cls, rt, names, n, descrs); +} + /* LOAD_GLOBAL of a class name, as an identity test: 1 = module_dict[name] is cls, 0 = it is not (or * the name is not a module global: the slow path then asks tp_global), -1 = error. With a dict * watcher on module_dict (`watched`), a 1 is remembered in rt->gepoch until tp_globals_epoch moves, @@ -1899,6 +1982,10 @@ static inline int tp_is_exact(PyObject *obj, PyTypeObject *cls) return Py_IS_TYPE(obj, cls); } +/* The exact-type guard of the `self` of a compiled method (wrapper and impl). A single definition so + * a test can show that it matters (a header where it is always true is caught). */ +#define TP_SELF_EXACT(o, cls) tp_is_exact(o, cls) + /* ------------------------------------------------------------------------------------------ * Call depth (issue #57): compiled frames have no Python frame, so a recursion made of them would * neither raise RecursionError nor stop before the C stack ends. tp_enter_call / tp_leave_call diff --git a/python-multiplatform-ksp/src/main/python/typedpython/verify.py b/python-multiplatform-ksp/src/main/python/typedpython/verify.py index b0611305..b911e81a 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/verify.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/verify.py @@ -94,6 +94,13 @@ "`obj` itself ... after checking": the result is a new reference to the operand (the back end increfs it); the operand's own reference is untouched. +Methods (SPEC N-11, #155). A `Function` with an `owner` must name a valid ClassDecl, be called +`.` (not a slot name) and take `self` as its first parameter, `Param(OBJ, cls=owner, +optional=False)` (`verify/class` otherwise). `CheckMethod(obj, C, m)` is CheckExact plus a callee edge to +`C.m`: it deopts (pure functions only, counts toward `may_deopt`), names a compiled method of C +(`verify/call-unresolved` otherwise; a rejected or removed method rejects the caller) and is an exact-C +proof like CheckExact. + Objects, str and dict code (#150). GetItem, Contains, Format, StoreItem and ForObj run user code (`__getitem__`, `__hash__`, `__eq__`, `__format__`, `__iter__`, `__next__`, `__setitem__`): each is an effect and makes the function not closed (so it can hold no array, no buffer and no entry global), @@ -221,7 +228,7 @@ Local, MathCall, MathFunc, Module, NewArray, ObjToFloat, Or, Param, Return, StoreIndex, ToFloat, Truth, Tuple, Type, Unbox, UnaryOp, UnaryOpKind, While, ) -from .ir import CheckExact, ClassDecl, FieldGet, FieldSet, Is, IsExact, New +from .ir import CheckExact, CheckMethod, ClassDecl, FieldGet, FieldSet, Is, IsExact, New from .ir import BuildString, Contains, ForObj, Format, GetItem, NewDict, NewList, StoreItem DA = "verify/definite-assignment" @@ -260,7 +267,7 @@ class Diagnostic: # the one the back end implements). _EXPRS = frozenset({Const, Local, BinOp, UnaryOp, Compare, And, Or, ToFloat, Box, Unbox, MathCall, Call, Global, GetAttr, CallObject, Truth, CompareObj, ObjToFloat, Len, Index, - NewArray, CopyArray, Tuple, Is, IsExact, CheckExact, FieldGet, New, + NewArray, CopyArray, Tuple, Is, IsExact, CheckExact, CheckMethod, FieldGet, New, GetItem, Contains, NewList, NewDict, Format, BuildString}) _STMTS = frozenset({Assign, StoreIndex, ExprStmt, If, While, ForRange, Return, Break, Continue, FieldSet, StoreItem, ForObj}) @@ -269,7 +276,7 @@ class Diagnostic: # borrows under that rule) and Calls of a compiled function returning OBJ (which returns a new # reference by the same rule). _OBJ_PRODUCERS = frozenset({Box, Global, GetAttr, CallObject, BinOp, Local, Call, Tuple, New, - FieldGet, CheckExact, Const, # an OBJ Const is None or a str (e_Const) + FieldGet, CheckExact, CheckMethod, Const, # an OBJ Const is None or a str (e_Const) GetItem, NewList, NewDict, Format, BuildString}) _CONVERSIONS = (-1, 97, 114, 115) # Format.conversion: none, !a, !r, !s _VALUE_TYPES = (Type.I64, Type.F64, Type.BOOL, Type.OBJ) # what a local may hold @@ -503,7 +510,7 @@ def exact_of(self, e: object) -> str | None: cls = type(e) if cls is Local: return self.facts.get(e.name) or self.new_only.get(e.name) - if cls is CheckExact or cls is New: + if cls is CheckExact or cls is CheckMethod or cls is New: return e.cls if isinstance(e.cls, str) and e.cls in self.classes else None return None @@ -1117,6 +1124,23 @@ def e_CheckExact(self, e: CheckExact, state: _State) -> ir.Expr: self.can_deopt = True # a wrong type deopts, even if the check is redundant return dataclasses.replace(e, obj=obj) + def e_CheckMethod(self, e: CheckMethod, state: _State) -> ir.Expr: + """Methods (SPEC N-11, #155): the check is a deopt, like CheckExact, and names a compiled + method of the class: the callee edge makes a rejected or removed method reject this function + too (the back end's descriptors are built from the methods that are compiled).""" + obj = self.obj_operand(e.obj, state, "operand of CheckMethod") + self.result(e, Type.OBJ) + self.can_deopt = True + if self.known_class(e.cls, "CheckMethod"): + q = f"{e.cls}.{e.method}" if isinstance(e.method, str) else None + callee = self.sigs.get(q) if q is not None else None + if callee is None or callee.owner != e.cls: + self.error(UNRESOLVED, f"CheckMethod names method {e.method!r} of '{e.cls}', which " + f"is not a compiled method of this module") + else: + self.callees.add(q) + return dataclasses.replace(e, obj=obj) + def e_FieldGet(self, e: FieldGet, state: _State) -> ir.Expr: obj = self.obj_operand(e.obj, state, f"object of .{e.field}") self.result(e, Type.OBJ) @@ -1761,6 +1785,10 @@ def reject(fn_name: str, line: int, rule: str, message: str) -> None: f"{len(gs)} times") sigs = {name: gs[0] for name, gs in named.items() if len(gs) == 1} classes, skipped_classes = _check_classes(module, set(named), diags) + for name, g in sigs.items(): + problem = _owner_problem(g, classes) + if problem is not None: + reject(name, g.source_line, CLASS, problem) checked: dict[str, Function] = {} callees: dict[str, set[str]] = {} @@ -1839,6 +1867,25 @@ def reject(fn_name: str, line: int, rule: str, message: str) -> None: return Module(module.name, kept, skipped, tuple(classes.values())), diags +def _owner_problem(g: Function, classes: dict[str, ClassDecl]) -> str | None: + """Why `g.owner` is not a legal method declaration (ir.Function.owner), or None.""" + owner = g.owner + if owner is None: + return None + if not isinstance(owner, str) or owner not in classes: + return f"owner {owner!r} is not a compiled class of this module" + prefix = f"{owner}." + if not (g.name.startswith(prefix) and g.name[len(prefix):].isidentifier()): + return f"a method of '{owner}' must be named '{owner}.', not '{g.name}'" + first = g.params[0] if g.params else None + if not (type(first) is Param and first.type is Type.OBJ and first.cls == owner + and first.optional is False): + return f"the first parameter of method '{g.name}' must be `self` guarded exactly '{owner}'" + if classes[owner].fields and g.name[len(prefix):] in classes[owner].fields: + return f"method '{g.name}' has the name of a slot of '{owner}'" + return None + + def _check_classes(module: Module, function_names: set[str], diags: list[Diagnostic] ) -> tuple[dict[str, ClassDecl], dict[str, str]]: """The valid ClassDecls of the module by name, and the reason for each invalid one.""" diff --git a/python-multiplatform-ksp/src/test/python/test_class_methods.py b/python-multiplatform-ksp/src/test/python/test_class_methods.py new file mode 100644 index 00000000..f0783cb2 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_class_methods.py @@ -0,0 +1,927 @@ +"""TypedPython: methods of fixed-layout classes compile (SPEC N-11, issue #155). + +A plain instance method of an N-11 class (`def m(self, ...)` in the class body: no decorator, not a +dunder, `self` unannotated or annotated with the class) is an IR function named `Class.method`. The +module init replaces the class dict's function by a descriptor object of a type the extension defines; +the original function stays in `__typedpython_interpreted__["Class.method"]`. + +What is shown here, every behavioural test against the interpreted class (the same source loaded as an +ordinary module): + + * frontend: which methods are lowered, which stay interpreted and why; + * results of compiled methods (pure, impure, recursive through methods) equal the interpreter's, + through `obj.m(...)`, `C.m(obj, ...)`, `getattr`, bound methods and `functools.partial`; + * `C.m(5)`, keywords and a wrong arity run the ORIGINAL function (CPython's own errors); + * a subclass instance and an overriding subclass take the interpreted path with correct results; + * rebinding or deleting `C.m` after import, `obj.__class__` assignment: same outcomes as CPython; + * the attributes the descriptor shows, and the observable differences from a function; + * reference counts stay flat, concurrent calls on shared and private instances (free-threaded); + * mutation: a `self` guard that is always true is caught; + * the binary_trees benchmark with `check` as a method of `Node` compiles and matches. + +Scenarios that change class state run in a CHILD process on a fresh instance of the compiled module, +next to the plain-source module, and the two outcome lists are compared. + +Ordering of the red/green record: this file was written before the implementation; with methods +still interpreted every compiled-behaviour test failed (see the report of issue #155). +""" +from __future__ import annotations + +import functools +import gc +import inspect +import json +import shutil +import subprocess +import sys +import textwrap +import types +from pathlib import Path + +import pytest + +from typedpython import cbuild, cgen, cost, frontend, pipeline, verify + +HERE = Path(__file__).resolve().parent +import os # noqa: E402 + +# the package the child processes import (the red run of the issue points it at a pristine copy) +MAIN_PYTHON = Path(os.environ.get("TP_MAIN_PYTHON", HERE.parents[1] / "main" / "python")) +REPO = HERE.parents[3] +BINARY_TREES = REPO / "benchmarks" / "typedpython" / "py" / "binary_trees.py" + +SOURCE = textwrap.dedent('''\ + # typedpython: compiled + """typedpython class-method test module.""" + + + class Node: + __slots__ = ("left", "right") + left: "Node | None" + right: "Node | None" + + def __init__(self, left: "Node | None", right: "Node | None") -> None: + self.left = left + self.right = right + + def check(self) -> int: + """Count the nodes.""" + left: Node | None = self.left + right: Node | None = self.right + if left is None or right is None: + return 1 + return 1 + left.check() + right.check() + + def leaf(self: "Node") -> bool: + return self.left is None and self.right is None + + def height(self) -> int: + h: int = 0 + left: Node | None = self.left + if left is not None: + h = 1 + left.height() + return h + + def scaled(self, k: float) -> float: + return k * 2.0 + + def tag(self) -> int: + return 7 + + def sum_to(self, n: int) -> int: + total: int = 0 + for i in range(n): + total += i + return total + + def set_left(self, v: "Node | None") -> None: + self.left = v + + def swap(self) -> None: + a: Node | None = self.left + b: Node | None = self.right + self.left = b + self.right = a + + def check_swapped(self) -> object: + self.swap() + total: object = self.check() + self.swap() + return total + + def after(self, cb: object) -> object: + cb() + return self.left + + @staticmethod + def st(x: int) -> int: + return x + 1 + + @classmethod + def cm(cls) -> int: + return 1 + + @property + def size(self) -> int: + return self.check() + + def __repr__(self) -> str: + return "Node()" + + def uses_super(self) -> str: + return str(super().__repr__()) + + def no_annotation(self, x) -> int: + return 1 + + + class Plain: + def check(self) -> int: + return 5 + + + def make(depth: int) -> Node: + if depth > 0: + return Node(make(depth - 1), make(depth - 1)) + return Node(None, None) + + + def check_of(n: Node) -> int: + return n.check() + + + def leaf_of(n: Node) -> bool: + return n.leaf() +''') + +COMPILED_METHODS = {"Node.check", "Node.leaf", "Node.height", "Node.scaled", "Node.tag", "Node.sum_to", + "Node.set_left", "Node.swap", "Node.check_swapped", "Node.after"} + + +def build_source(d: Path, source: str, name: str = "methmod", runtime_dir=None): + path = d / f"{name}.py" + path.write_text(source) + proved, diags = verify.verify(frontend.lower(path)) + proved, diags = cost.apply(proved, diags) + c = cgen.generate(proved, path, path.name) + so = cbuild.build(c, path.stem, d / "out", runtime_dir=runtime_dir) + return proved, diags, so + + +@pytest.fixture(scope="module") +def built(tmp_path_factory): + d = tmp_path_factory.mktemp("methmod") + proved, diags, so = build_source(d, SOURCE) + ref = d / "refmod.py" + ref.write_text(SOURCE) + return types.SimpleNamespace(proved=proved, diags=diags, so=so, ref=ref, dir=d, + module=cbuild.load("methmod", so)) + + +def compiled_names(proved): + return {f.name for f in proved.functions} + + +def deopts(m): + return m.__typedpython_deopts__ + + +# --- the front end ------------------------------------------------------------------------------- + +def lower(tmp_path, source): + path = tmp_path / "lowermod.py" + path.write_text(source) + return frontend.lower(path) + + +def test_the_plain_instance_methods_are_lowered_with_qualified_names(tmp_path): + m = lower(tmp_path, SOURCE) + names = {f.name for f in m.functions} + assert COMPILED_METHODS <= names, (COMPILED_METHODS - names, m.skipped) + check = next(f for f in m.functions if f.name == "Node.check") + self_param = check.params[0] + assert (self_param.name, self_param.type.value, self_param.cls, self_param.optional) == \ + ("self", "obj", "Node", False) + assert check.owner == "Node" + assert next(f for f in m.functions if f.name == "make").owner is None + + +@pytest.mark.parametrize("method, why", [ + ("Node.st", "decorator"), + ("Node.cm", "decorator"), + ("Node.size", "decorator"), + ("Node.__repr__", "dunder"), + ("Node.__init__", "dunder"), + ("Node.uses_super", "super"), + ("Node.no_annotation", "no annotation"), +]) +def test_other_methods_stay_interpreted_with_a_reason(tmp_path, method, why): + m = lower(tmp_path, SOURCE) + assert method not in {f.name for f in m.functions} + assert why in m.skipped[method], m.skipped.get(method) + + +def test_the_methods_of_a_class_that_is_not_fixed_layout_are_not_considered(tmp_path): + m = lower(tmp_path, SOURCE) + assert "Plain.check" not in {f.name for f in m.functions} + assert "Plain.check" not in m.skipped # the class has its own entry + assert "Plain" in m.skipped + + +def test_a_method_named_like_a_module_function_does_not_collide(tmp_path): + m = lower(tmp_path, SOURCE + textwrap.dedent('''\ + + def height(n: Node) -> int: + return n.height() + ''')) + names = {f.name for f in m.functions} + assert {"height", "Node.height"} <= names + + +def test_a_method_with_a_foreign_self_annotation_stays_interpreted(tmp_path): + src = textwrap.dedent('''\ + # typedpython: compiled + class A: + __slots__ = ("x",) + x: int + + def __init__(self, x: int) -> None: + self.x = x + + def get(self: int) -> int: + return 1 + ''') + m = lower(tmp_path, src) + assert "A.get" not in {f.name for f in m.functions} + assert "self" in m.skipped["A.get"] + + +def test_a_method_defined_twice_in_the_class_body_stays_interpreted(tmp_path): + src = textwrap.dedent('''\ + # typedpython: compiled + class A: + __slots__ = ("x",) + x: int + + def __init__(self, x: int) -> None: + self.x = x + + def get(self) -> int: + return 1 + + def get(self) -> int: + return 2 + ''') + m = lower(tmp_path, src) + assert "A.get" not in {f.name for f in m.functions} + assert "more than once" in m.skipped["A.get"] + + +def test_an_at_compiled_class_compiles_its_methods_without_the_module_marker(tmp_path): + src = textwrap.dedent('''\ + @compiled + class A: + __slots__ = ("x",) + x: int + + def __init__(self, x: int) -> None: + self.x = x + + def get(self) -> int: + return 1 + ''') + m = lower(tmp_path, src) + assert "A.get" in {f.name for f in m.functions} + + +def test_the_compiled_set_is_the_methods_and_the_functions(built): + assert compiled_names(built.proved) >= COMPILED_METHODS | {"make", "check_of", "leaf_of"} + assert not (compiled_names(built.proved) & {"Node.st", "Node.cm", "Node.size", "Node.__repr__", + "Node.__init__", "Node.uses_super"}) + + +def test_a_method_calling_a_method_is_a_direct_c_call(built): + check = next(f for f in built.proved.functions if f.name == "Node.check") + from typedpython import ir + calls = [] + stack = [check.body] + while stack: + x = stack.pop() + if isinstance(x, (tuple, list)): + stack.extend(x) + elif hasattr(x, "__dataclass_fields__"): + if isinstance(x, ir.Call): + calls.append(x.function) + stack.extend(getattr(x, f) for f in x.__dataclass_fields__) + assert calls.count("Node.check") == 2, calls + assert check.pure + + +# --- the installed descriptor -------------------------------------------------------------------- + +def node(m, depth=3): + return m.make(depth) + + +def test_the_class_dict_holds_a_descriptor_not_a_function(built): + m = built.module + entry = m.Node.__dict__["check"] + assert not isinstance(entry, types.FunctionType) + assert entry is m.Node.check # a function-like descriptor: C.m is itself + orig = m.__typedpython_interpreted__["Node.check"] + assert isinstance(orig, types.FunctionType) + assert entry is not orig + + +def test_methods_that_stay_interpreted_are_still_functions(built): + m = built.module + assert isinstance(m.Node.__dict__["uses_super"], types.FunctionType) + assert isinstance(m.Node.__dict__["__init__"], types.FunctionType) + assert isinstance(m.Node.__dict__["st"], staticmethod) + assert isinstance(m.Node.__dict__["cm"], classmethod) + assert isinstance(m.Node.__dict__["size"], property) + + +def test_an_instance_attribute_is_a_real_bound_method(built): + m = built.module + n = node(m) + b = n.check + assert type(b) is types.MethodType + assert b.__self__ is n + assert b.__func__ is m.Node.check + assert b() == m.__typedpython_interpreted__["Node.check"](n) + assert "Node.check" in repr(b) + assert b == n.check and hash(b) == hash(n.check) + + +def test_the_forwarded_attributes_equal_the_originals(built): + m = built.module + orig = m.__typedpython_interpreted__["Node.check"] + d = m.Node.check + assert d.__name__ == orig.__name__ == "check" + assert d.__qualname__ == orig.__qualname__ == "Node.check" + assert d.__doc__ == orig.__doc__ == "Count the nodes." + assert d.__module__ == orig.__module__ + assert d.__wrapped__ is orig + n = node(m) + assert n.check.__name__ == "check" and n.check.__doc__ == "Count the nodes." + assert n.check.__qualname__ == "Node.check" and n.check.__wrapped__ is orig + assert m.Node.leaf.__doc__ is None + assert inspect.signature(m.Node.check) == inspect.signature(orig) + assert str(inspect.signature(n.check)) == "() -> int" + assert "check" in repr(d) + + +def test_the_attributes_that_are_not_forwarded_are_absent(built): + d = built.module.Node.check + for name in ("__code__", "__globals__", "__defaults__", "__kwdefaults__", "__annotations__", + "__closure__", "__dict__", "__type_params__"): + assert not hasattr(d, name), name + + +def test_a_bound_method_reduces_like_one_of_a_function(built): + m = built.module + n = node(m, 1) + assert n.check.__reduce__() == (getattr, (n, "check")) + assert callable(m.Node.check) and not inspect.isfunction(m.Node.check) + assert inspect.ismethod(n.check) + + +def test_the_descriptor_is_immutable(built): + d = built.module.Node.check + with pytest.raises(AttributeError): + d.__name__ = "other" + with pytest.raises(AttributeError): + d.extra = 1 + with pytest.raises(TypeError): + type(d).extra = 1 + with pytest.raises(TypeError): + type(d)() + + +def test_the_descriptor_type_is_one_type_shared_by_all_methods(built): + m = built.module + assert type(m.Node.check) is type(m.Node.leaf) is type(m.Node.set_left) + + +def test_module_functions_expose_less_than_methods_do(built): + """Recorded difference (API.md): a module-level compiled function is a builtin function and has no + `__wrapped__`; a method forwards `__doc__`, `__wrapped__` ... from the original function.""" + m = built.module + assert not hasattr(m.make, "__wrapped__") + assert hasattr(m.Node.check, "__wrapped__") + + +# --- behaviour ------------------------------------------------------------------------------------ + +def test_pure_recursive_methods_agree_with_the_interpreter(built): + m = built.module + interp = m.__typedpython_interpreted__ + before = deopts(m) + for depth in range(0, 9): + n = node(m, depth) + assert n.check() == interp["Node.check"](n) == (2 ** (depth + 1) - 1) + assert n.height() == interp["Node.height"](n) == depth + assert n.leaf() is interp["Node.leaf"](n) + assert m.check_of(n) == n.check() + assert m.leaf_of(n) is n.leaf() + assert deopts(m) == before, "a compiled method fell back to the interpreter" + + +def test_scalar_and_loop_methods(built): + m = built.module + n = node(m, 1) + before = deopts(m) + assert n.scaled(1.5) == 3.0 and type(n.scaled(1.5)) is float + assert n.sum_to(10) == 45 and n.sum_to(0) == 0 + assert deopts(m) == before + assert n.scaled(2) == 4.0 # an int where a float was annotated: CPython's result + assert n.sum_to(10 ** 6) == sum(range(10 ** 6)) + + +def test_impure_methods_write_fields_like_the_interpreter(built): + m = built.module + before = deopts(m) + a, b = m.Node(None, None), m.Node(None, None) + top = m.Node(a, b) + top.swap() + assert top.left is b and top.right is a + top.set_left(None) + assert top.left is None + assert top.check_swapped() == 1 + assert top.left is None and top.right is a + assert deopts(m) == before + + +def test_the_unbound_forms_and_wrappers(built): + m = built.module + n = node(m, 4) + expected = m.__typedpython_interpreted__["Node.check"](n) + assert m.Node.check(n) == expected + assert getattr(n, "check")() == expected + assert functools.partial(n.check)() == expected + assert functools.partial(m.Node.check, n)() == expected + assert functools.partial(m.Node.scaled, n)(2.0) == 4.0 + assert [f() for f in map(lambda x: x.check, [n, n])] == [expected, expected] + assert (lambda f: f(n))(m.Node.check) == expected + + +def test_a_foreign_self_runs_the_original_function(built): + m = built.module + before = deopts(m) + with pytest.raises(AttributeError) as compiled_error: + m.Node.check(5) + interp = m.__typedpython_interpreted__["Node.check"] + with pytest.raises(AttributeError) as original_error: + interp(5) + assert repr(compiled_error.value) == repr(original_error.value) + assert deopts(m) == before + 1 + assert m.Node.sum_to(5, 4) == 6 # CPython: the original never touched self + + +def test_arity_and_keywords_are_the_originals(built): + m = built.module + n = node(m, 2) + interp = m.__typedpython_interpreted__["Node.check"] + for call in (lambda f: f(n, 1), lambda f: f(), lambda f: f(self=n), lambda f: f(n, x=1)): + outcomes = [] + for f in (m.Node.check, interp): + try: + outcomes.append(("ok", call(f))) + except TypeError as e: + outcomes.append(("raise", str(e))) + assert outcomes[0] == outcomes[1], outcomes + with pytest.raises(TypeError, match="takes 1 positional argument but 2 were given"): + n.check(1) + + +def test_a_subclass_instance_takes_the_interpreted_path(built): + m = built.module + + class Sub(m.Node): + pass + + s = Sub(m.Node(None, None), None) + before = deopts(m) + assert s.check() == 1 + assert deopts(m) == before + 1 + t = Sub(m.Node(None, None), m.Node(None, None)) + assert t.check() == 3 + assert m.Node.check(t) == 3 + assert m.check_of(t) == 3 + + +def test_an_overriding_subclass_and_a_mixed_tree(built): + m = built.module + + class Over(m.Node): + def check(self): + return -1000 + + class Up(m.Node): + def check(self): + return 100 + super().check() + + assert Over(None, None).check() == -1000 + assert Up(m.Node(None, None), m.Node(None, None)).check() == 103 + tree = m.Node(Over(None, None), m.Node(None, None)) + interp = m.__typedpython_interpreted__["Node.check"] + assert tree.check() == interp(tree) == 1 + (-1000) + 1 + assert m.check_of(tree) == -998 + assert m.Node(Up(m.Node(None, None), m.Node(None, None)), m.Node(None, None)).check() == 105 + + +def test_c_to_c_calls_run_without_deopts(built): + m = built.module + n = node(m, 10) + before = deopts(m) + assert n.check() == 2 ** 11 - 1 + assert m.check_of(n) == 2 ** 11 - 1 + assert deopts(m) == before + + +def test_class_info_reports_the_methods(built): + info = built.module.__typedpython_class_info__()["Node"] + assert info["compiled"] is True and info["current"] is True + + +# --- class state changed after import: child processes -------------------------------------------- + +CHILD = ''' +import json, sys +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path +sys.path.insert(0, {main!r}) +from typedpython import cbuild + +comp = cbuild.load("methmod", Path({so!r})) +spec = spec_from_file_location("refmod", {ref!r}) +ref = module_from_spec(spec) +spec.loader.exec_module(ref) + + +def outcome(f): + try: + return ["ok", repr(f())] + except BaseException as e: + return ["raise", type(e).__name__, str(e)] + +{scenario} + +out_c = scenario(comp) +out_r = scenario(ref) +print(json.dumps({{"comp": out_c, "ref": out_r, "deopts": comp.__typedpython_deopts__}})) +''' + + +def run_scenario(built, scenario: str) -> dict: + script = CHILD.format(main=str(MAIN_PYTHON), so=str(built.so), ref=str(built.ref), + scenario=textwrap.dedent(scenario)) + run = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, timeout=300) + assert run.returncode == 0, run.stderr + return json.loads(run.stdout.strip().splitlines()[-1]) + + +def same(built, scenario: str) -> dict: + out = run_scenario(built, scenario) + assert out["comp"] == out["ref"], out + return out + + +def test_rebinding_a_method_after_import(built): + out = same(built, ''' + def scenario(m): + n = m.make(3) + res = [outcome(lambda: m.check_of(n)), outcome(lambda: n.check())] + m.Node.check = lambda self: 99 + res += [outcome(lambda: n.check()), outcome(lambda: m.check_of(n)), + outcome(lambda: n.height()), outcome(lambda: m.Node.check(n))] + m.Node.check = property(lambda self: 7) + res += [outcome(lambda: n.check), outcome(lambda: m.check_of(n))] + del m.Node.check + res += [outcome(lambda: n.check), outcome(lambda: m.check_of(n)), + outcome(lambda: n.height())] + return res + ''') + assert out["comp"][2] == ["ok", "99"] + + +def test_rebinding_a_callee_method_that_a_caller_method_calls(built): + same(built, ''' + def scenario(m): + n = m.Node(m.Node(None, None), m.Node(None, None)) + res = [outcome(lambda: n.check_swapped()), outcome(lambda: n.height())] + m.Node.swap = lambda self: None + res += [outcome(lambda: n.check_swapped()), outcome(lambda: n.height())] + m.Node.check = lambda self: 5 + res += [outcome(lambda: n.check_swapped()), outcome(lambda: n.height()), + outcome(lambda: m.check_of(n))] + return res + ''') + + +def test_rebinding_the_method_through_the_class_dict_of_a_subclass_chain(built): + same(built, ''' + def scenario(m): + class Sub(m.Node): + pass + s = Sub(m.Node(None, None), m.Node(None, None)) + res = [outcome(lambda: s.check()), outcome(lambda: m.check_of(s))] + m.Node.check = lambda self: 11 + res += [outcome(lambda: s.check()), outcome(lambda: m.check_of(s)), + outcome(lambda: m.Node.check(s))] + return res + ''') + + +def test_instance_class_assignment(built): + same(built, ''' + def scenario(m): + class Twin: + __slots__ = ("left", "right") + def check(self): + return 4242 + n = m.make(2) + res = [outcome(lambda: n.check())] + n.__class__ = m.Node + c = m.Node(m.Node(None, None), m.Node(None, None)) + c.left.__class__ = Twin + res += [outcome(lambda: c.check()), outcome(lambda: m.check_of(c)), + outcome(lambda: m.Node.check(c.left)), outcome(lambda: c.height())] + return res + ''') + + +def test_unrelated_selves_and_none_children(built): + same(built, ''' + class Fake: + left = None + right = None + + def scenario(m): + f = Fake() + n = m.Node(None, m.Node(None, None)) + return [outcome(lambda: m.Node.check(f)), outcome(lambda: m.Node.leaf(f)), + outcome(lambda: m.Node.height(None)), outcome(lambda: m.Node.scaled(f, 1.0)), + outcome(lambda: m.Node.set_left(f, 1)), outcome(lambda: m.Node.check(n)), + outcome(lambda: m.Node.after(n, lambda: 3)), + outcome(lambda: m.Node.set_left(n, 5)), outcome(lambda: n.left), + outcome(lambda: n.check()), outcome(lambda: m.check_of(n))] + ''') + + +def test_a_callback_that_rebinds_the_method_mid_body(built): + same(built, ''' + def scenario(m): + n = m.Node(m.Node(None, None), None) + def cb(): + m.Node.check = lambda self: 77 + res = [outcome(lambda: n.after(cb)), outcome(lambda: n.check()), + outcome(lambda: m.check_of(n))] + return res + ''') + + +def test_deep_recursion_raises_what_cpython_raises(built): + out = run_scenario(built, ''' + def scenario(m): + n = m.Node(None, None) + for _ in range(5000): + n = m.Node(n, m.Node(None, None)) + return [outcome(lambda: n.check()), outcome(lambda: n.height())] + ''') + assert out["comp"][0][0] == out["ref"][0][0] + assert out["comp"] == out["ref"] or out["comp"][0][1] == out["ref"][0][1] + + +# --- reference counts ----------------------------------------------------------------------------- + +def test_reference_counts_are_flat_over_many_calls(built): + m = built.module + n = node(m, 5) + other = m.Node(None, None) + desc = m.Node.__dict__["check"] + gc.collect() + base = (sys.getrefcount(n), sys.getrefcount(other), sys.getrefcount(desc), + sys.getrefcount(m.Node), sys.getrefcount(m.__typedpython_interpreted__["Node.check"])) + for _ in range(2000): + n.check() + m.Node.check(n) + n.height() + m.check_of(n) + n.leaf() + n.scaled(1.0) + n.set_left(n.left) + n.swap() + n.swap() + other.set_left(None) + n.check_swapped() + with pytest.raises(AttributeError): + m.Node.check(5) + b = n.check + b() + functools.partial(m.Node.check, n)() + gc.collect() + after = (sys.getrefcount(n), sys.getrefcount(other), sys.getrefcount(desc), + sys.getrefcount(m.Node), sys.getrefcount(m.__typedpython_interpreted__["Node.check"])) + assert after == base, (base, after) + + +# --- free-threaded concurrency -------------------------------------------------------------------- + +THREADS = ''' +import sys, threading +from pathlib import Path +sys.path.insert(0, {main!r}) +from typedpython import cbuild +m = cbuild.load("methmod", Path({so!r})) +shared = m.make(6) +errors = [] +start = threading.Barrier(8) + +def reader(): + try: + start.wait() + for _ in range(300): + assert shared.check() == 127 + assert m.check_of(shared) == 127 + assert shared.height() == 6 + except BaseException as e: + errors.append(repr(e)) + +def private(): + try: + mine = m.make(4) + start.wait() + for _ in range(300): + mine.swap() + assert mine.check_swapped() == 31 + mine.set_left(mine.left) + assert mine.check() == 31 + except BaseException as e: + errors.append(repr(e)) + +def toggler(): + try: + a, b = m.Node(None, None), m.Node(None, None) + top = m.Node(a, b) + box = m.Node(top, None) + start.wait() + for i in range(300): + top.set_left(None if i % 2 else a) + r = top.check() + assert r in (1, 3), r + except BaseException as e: + errors.append(repr(e)) + +threads = [threading.Thread(target=f) for f in [reader] * 4 + [private] * 3 + [toggler]] +for t in threads: t.start() +for t in threads: t.join() +print("GIL", sys._is_gil_enabled() if hasattr(sys, "_is_gil_enabled") else "n/a") +print("ERRORS", errors) +''' + + +def test_concurrent_calls_on_shared_and_private_instances(built): + script = THREADS.format(main=str(MAIN_PYTHON), so=str(built.so)) + run = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, timeout=300) + assert run.returncode == 0, run.stderr + assert "ERRORS []" in run.stdout, run.stdout + if hasattr(sys, "_is_gil_enabled"): + assert "GIL False" in run.stdout, run.stdout # the module keeps the GIL off + + +# --- mutation: the exact-type guard of `self` ------------------------------------------------------- + +GUARD_PROBE = ''' +import sys +from pathlib import Path +sys.path.insert(0, {main!r}) +from typedpython import cbuild +m = cbuild.load("methmod", Path({so!r})) + +class Sub(m.Node): + pass + +s = Sub(m.Node(None, None), m.Node(None, None)) +before = m.__typedpython_deopts__ +results = [s.check(), m.Node.leaf(s), m.Node.height(s), m.Node.tag(s)] +print("DEOPTS", m.__typedpython_deopts__ - before, results) +''' + + +def probe(so) -> str: + script = GUARD_PROBE.format(main=str(MAIN_PYTHON), so=str(so)) + run = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, timeout=300) + assert run.returncode == 0, run.stderr + return run.stdout.strip().splitlines()[-1] + + +def test_the_self_guard_is_one_mutation_point(): + header = (cbuild.RUNTIME_DIR / "tp_runtime.h").read_text() + assert "#define TP_SELF_EXACT(o, cls) tp_is_exact(o, cls)" in header + + +def test_a_self_guard_that_is_always_true_is_caught(built, tmp_path): + good = probe(built.so) + # a subclass instance: four interpreted runs (the guard of `self` decides; `tag` touches no field, + # so nothing else would send it to the interpreter) + assert good == "DEOPTS 4 [3, False, 1, 7]", good + mutant = tmp_path / "runtime" + shutil.copytree(cbuild.RUNTIME_DIR, mutant) + text = (mutant / "tp_runtime.h").read_text() + line = "#define TP_SELF_EXACT(o, cls) tp_is_exact(o, cls)" + (mutant / "tp_runtime.h").write_text(text.replace(line, "#define TP_SELF_EXACT(o, cls) (1)")) + (tmp_path / "src").mkdir() + _, _, so = build_source(tmp_path / "src", SOURCE, runtime_dir=mutant) + bad = probe(so) + assert bad != good, "the mutant (self guard always true) was not caught" + + +# --- the binary_trees benchmark with `check` as a method -------------------------------------------- + +def method_form(text: str) -> str: + text = text.replace(''' def __init__(self, left: "Node | None", right: "Node | None") -> None: + self.left = left + self.right = right +''', ''' def __init__(self, left: "Node | None", right: "Node | None") -> None: + self.left = left + self.right = right + + def check(self) -> int: + left: Node | None = self.left + right: Node | None = self.right + if left is None or right is None: + return 1 + return 1 + left.check() + right.check() +''') + text = text.replace('''def check(node: Node) -> int: + left: Node | None = node.left + right: Node | None = node.right + if left is None or right is None: + return 1 + return 1 + check(left) + check(right) + + +''', "") + text = text.replace("check(make(max_depth + 1))", "make(max_depth + 1).check()") + text = text.replace("check(make(depth))", "make(depth).check()") + text = text.replace("check(long_lived)", "long_lived.check()") + assert "def check(node" not in text and "check(make" not in text + return text + + +def run_main(path_or_module, n): + import contextlib + import io + old = sys.argv + sys.argv = ["binary_trees", str(n)] + buf = io.StringIO() + try: + with contextlib.redirect_stdout(buf): + path_or_module.main() + finally: + sys.argv = old + return buf.getvalue() + + +def test_binary_trees_with_check_as_a_method_compiles_and_matches(tmp_path): + text = "# typedpython: compiled\n" + method_form(BINARY_TREES.read_text()) + path = tmp_path / "bt_method.py" + path.write_text(text) + result = pipeline.compile_module(path, tmp_path / "out") + names = {f.name for f in result.module.functions} + assert {"Node.check", "make"} <= names, (names, result.skipped) + assert result.extension is not None + compiled = pipeline.load(result) + ref_path = tmp_path / "bt_ref.py" + ref_path.write_text(text) + import importlib.util + spec = importlib.util.spec_from_file_location("bt_ref", ref_path) + ref = importlib.util.module_from_spec(spec) + spec.loader.exec_module(ref) + before = compiled.__typedpython_deopts__ + assert run_main(compiled, 6) == run_main(ref, 6) + assert compiled.__typedpython_deopts__ == before + assert isinstance(compiled.Node.__dict__["check"], types.FunctionType) is False + + +# --- the five benchmarks compile as they did -------------------------------------------------------- + +EXPECTED = { + "binary_trees": {"check", "make"}, + "fannkuch": {"fannkuch"}, + "spectral_norm": {"eval_a", "mul_atv", "mul_av"}, + "wordfreq": {"next_state"}, + "nbody": set(), +} + + +@pytest.mark.parametrize("name", sorted(EXPECTED)) +def test_the_benchmarks_compile_the_same_functions_as_before(tmp_path, name): + path = tmp_path / f"{name}.py" + path.write_text("# typedpython: compiled\n" + (REPO / "benchmarks" / "typedpython" / "py" + / f"{name}.py").read_text()) + proved, diags = verify.verify(frontend.lower(path)) + proved, diags = cost.apply(proved, diags) + assert {f.name for f in proved.functions} == EXPECTED[name] From de0a70232bab86648dd67b263b534fd853f076a4 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:23:34 +0900 Subject: [PATCH 2/2] Feat: compile plain instance methods of fixed-layout classes (#155) SPEC N-11 states methods; the refcount test releases its last bound method. --- docs/SPEC.md | 9 +++++++-- .../src/test/python/test_class_methods.py | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/SPEC.md b/docs/SPEC.md index 4076e3fc..ace8748b 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -728,8 +728,13 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. takes the generic path (`getattr`/`setattr`/calling the current global) and a pure one deopts. A class that defines `__getattr__`, `__getattribute__`, `__setattr__` or `__delattr__` is not compiled. - A class outside these rules stays interpreted with a reason. Methods are not compiled yet. - `Status: implemented (module-level functions; methods planned, #155)`. + A class outside these rules stays interpreted with a reason. Plain instance methods of such a class + compile too (#155): `self` is guarded `type(self) is C` like a `Param.cls` parameter, so a subclass + instance or a foreign `self` runs the original function; the class dict holds a descriptor whose + bound method behaves like a function's, and a compiled method calling a method of a compiled class is + a direct C call under the same per-access class checks. Decorated methods (static, class, property), + dunders and methods that use `super` stay interpreted with a reason. + `Status: implemented (module-level functions and plain instance methods)`. - **N-12** TypedPython object cost: virtual objects, refcount elision, region allocation (design §4.3.4). Every execution of compiled code must be indistinguishable from some valid CPython execution. An object of an N-11 class without `__del__` and without a `__weakref__` slot may live as a C struct (a virtual diff --git a/python-multiplatform-ksp/src/test/python/test_class_methods.py b/python-multiplatform-ksp/src/test/python/test_class_methods.py index f0783cb2..90bb4395 100644 --- a/python-multiplatform-ksp/src/test/python/test_class_methods.py +++ b/python-multiplatform-ksp/src/test/python/test_class_methods.py @@ -721,6 +721,7 @@ def test_reference_counts_are_flat_over_many_calls(built): b = n.check b() functools.partial(m.Node.check, n)() + del b # the last bound method holds n and the function gc.collect() after = (sys.getrefcount(n), sys.getrefcount(other), sys.getrefcount(desc), sys.getrefcount(m.Node), sys.getrefcount(m.__typedpython_interpreted__["Node.check"]))