diff --git a/benchmarks/typedpython/results/2026-10-17-fast-entry-ab.json b/benchmarks/typedpython/results/2026-10-17-fast-entry-ab.json new file mode 100644 index 00000000..f7fb1f35 --- /dev/null +++ b/benchmarks/typedpython/results/2026-10-17-fast-entry-ab.json @@ -0,0 +1,314 @@ +{ + "spectral-norm": { + "size": "800", + "pairs": [ + { + "before_s": 1.1141414581798017, + "after_s": 0.798894583247602, + "cpython_s": 3.3166386247612536, + "after_over_before": 0.7170495069385303, + "speedup_before": 2.976855946263522, + "speedup_after": 4.151534751029004, + "load": [ + 2.32373046875, + 2.8388671875, + 3.1533203125 + ] + }, + { + "after_s": 0.05642041703686118, + "before_s": 0.5941873327828944, + "cpython_s": 3.3151200828142464, + "after_over_before": 0.09495392096733946, + "speedup_before": 5.579250683934612, + "speedup_after": 58.757454427328625, + "load": [ + 2.21728515625, + 2.80810546875, + 3.140625 + ] + }, + { + "before_s": 0.5943655422888696, + "after_s": 0.05575770791620016, + "cpython_s": 3.283662290778011, + "after_over_before": 0.09381046502372974, + "speedup_before": 5.524651173641064, + "speedup_after": 58.89162975840255, + "load": [ + 2.19970703125, + 2.79443359375, + 3.1337890625 + ] + }, + { + "after_s": 0.05573229165747762, + "before_s": 0.5970102911815047, + "cpython_s": 3.3194454996846616, + "after_over_before": 0.09335231315222627, + "speedup_before": 5.560114371086234, + "speedup_after": 59.560542029842956, + "load": [ + 2.19970703125, + 2.79443359375, + 3.1337890625 + ] + }, + { + "before_s": 0.5956689999438822, + "after_s": 0.05633941572159529, + "cpython_s": 3.339143875055015, + "after_over_before": 0.0945817488016046, + "speedup_before": 5.605703629649343, + "speedup_after": 59.26834405872438, + "load": [ + 2.103515625, + 2.76416015625, + 3.12109375 + ] + }, + { + "after_s": 0.056803958024829626, + "before_s": 0.6093089170753956, + "cpython_s": 3.3818732076324522, + "after_over_before": 0.09322686150316217, + "speedup_before": 5.550342548513829, + "speedup_after": 59.53587259102964, + "load": [ + 2.25537109375, + 2.78466796875, + 3.1259765625 + ] + }, + { + "before_s": 0.607568624895066, + "after_s": 0.05638662492856383, + "cpython_s": 3.3777862917631865, + "after_over_before": 0.09280700585600918, + "speedup_before": 5.5595140258379345, + "speedup_after": 59.904033909504264, + "load": [ + 2.31494140625, + 2.7880859375, + 3.125 + ] + }, + { + "after_s": 0.05656562466174364, + "before_s": 0.6080640838481486, + "cpython_s": 3.3865731661207974, + "after_over_before": 0.09302576186339882, + "speedup_before": 5.569434630456687, + "speedup_after": 59.86980938285648, + "load": [ + 2.28955078125, + 2.77490234375, + 3.1181640625 + ] + }, + { + "before_s": 0.6106662922538817, + "after_s": 0.059617833234369755, + "cpython_s": 3.3802520409226418, + "after_over_before": 0.09762751602733612, + "speedup_before": 5.5353506224268845, + "speedup_after": 56.6986731576471, + "load": [ + 2.28955078125, + 2.77490234375, + 3.1181640625 + ] + }, + { + "after_s": 0.05642887484282255, + "before_s": 0.6073171659372747, + "cpython_s": 3.380930291954428, + "after_over_before": 0.09291500060884278, + "speedup_before": 5.566992803071236, + "speedup_after": 59.91489820365369, + "load": [ + 2.26611328125, + 2.76171875, + 3.111328125 + ] + }, + { + "before_s": 0.6068534590303898, + "after_s": 0.05705862492322922, + "cpython_s": 3.3896440002135932, + "after_over_before": 0.09402372858580321, + "speedup_before": 5.5856054699423705, + "speedup_after": 59.406338739748186, + "load": [ + 2.32470703125, + 2.765625, + 3.1103515625 + ] + } + ], + "median_after_over_before": 0.09381046502372974, + "min_after_over_before": 0.09280700585600918, + "max_after_over_before": 0.7170495069385303, + "median_speedup_before": 5.560114371086234, + "median_speedup_after": 59.406338739748186 + }, + "nbody": { + "skipped": [ + "n/a (nothing compiled)", + "n/a (nothing compiled)" + ] + }, + "fannkuch-redux": { + "size": "10", + "pairs": [ + { + "before_s": 0.7067045411095023, + "after_s": 0.63105074968189, + "cpython_s": 6.030535667203367, + "after_over_before": 0.8929484855030952, + "speedup_before": 8.533319536528843, + "speedup_after": 9.55634023134167, + "load": [ + 2.53857421875, + 2.80224609375, + 3.12109375 + ] + }, + { + "after_s": 0.2739174999296665, + "before_s": 0.27865454088896513, + "cpython_s": 6.0477433749474585, + "after_over_before": 0.9830003094721281, + "speedup_before": 21.703372769931963, + "speedup_after": 22.078703903548806, + "load": [ + 2.5751953125, + 2.80517578125, + 3.1201171875 + ] + }, + { + "before_s": 0.27599437488242984, + "after_s": 0.2764261248521507, + "cpython_s": 6.042619833257049, + "after_over_before": 1.0015643433671602, + "speedup_before": 21.893996338987453, + "speedup_after": 21.859800105684677, + "load": [ + 2.486328125, + 2.7783203125, + 3.1064453125 + ] + }, + { + "after_s": 0.2808002079837024, + "before_s": 0.2761829160153866, + "cpython_s": 6.013481875415891, + "after_over_before": 1.0167182388937432, + "speedup_before": 21.77354762624373, + "speedup_after": 21.41551788225496, + "load": [ + 2.447265625, + 2.76513671875, + 3.099609375 + ] + }, + { + "before_s": 0.2764340410940349, + "after_s": 0.2888324582017958, + "cpython_s": 6.036349332891405, + "after_over_before": 1.0448512674440964, + "speedup_before": 21.83649057475527, + "speedup_after": 20.899137757827916, + "load": [ + 2.8115234375, + 2.8349609375, + 3.1220703125 + ] + }, + { + "after_s": 0.2795829582028091, + "before_s": 0.2770982920192182, + "cpython_s": 6.064032624941319, + "after_over_before": 1.0089667322215705, + "speedup_before": 21.884049088692134, + "speedup_after": 21.6895645711799, + "load": [ + 2.76611328125, + 2.82373046875, + 3.1142578125 + ] + }, + { + "before_s": 0.27542241709306836, + "after_s": 0.2786844172514975, + "cpython_s": 6.044994332827628, + "after_over_before": 1.011843626211903, + "speedup_before": 21.948083952748682, + "speedup_after": 21.691181704545574, + "load": [ + 3.18505859375, + 2.9091796875, + 3.142578125 + ] + }, + { + "after_s": 0.27875441685318947, + "before_s": 0.27709112502634525, + "cpython_s": 6.081121041905135, + "after_over_before": 1.006002688922953, + "speedup_before": 21.946285870133714, + "speedup_after": 21.81533519918307, + "load": [ + 3.169921875, + 2.91064453125, + 3.1416015625 + ] + }, + { + "before_s": 0.277795874979347, + "after_s": 0.2738394159823656, + "cpython_s": 6.0339908339083195, + "after_over_before": 0.9857576754972495, + "speedup_before": 21.7209518836697, + "speedup_after": 22.034778347237236, + "load": [ + 3.2109375, + 2.92919921875, + 3.1455078125 + ] + }, + { + "after_s": 0.2780231246724725, + "before_s": 0.2764884582720697, + "cpython_s": 6.0378861669451, + "after_over_before": 1.0055505622549086, + "speedup_before": 21.83775121999382, + "speedup_after": 21.717208502199533, + "load": [ + 3.27392578125, + 2.94677734375, + 3.150390625 + ] + }, + { + "before_s": 0.2783553749322891, + "after_s": 0.2733089169487357, + "cpython_s": 6.045158042106777, + "after_over_before": 0.9818704489367918, + "speedup_before": 21.717410858609366, + "speedup_after": 22.11840765971299, + "load": [ + 3.572265625, + 3.013671875, + 3.1728515625 + ] + } + ], + "median_after_over_before": 1.0055505622549086, + "min_after_over_before": 0.8929484855030952, + "max_after_over_before": 1.0448512674440964, + "median_speedup_before": 21.83649057475527, + "median_speedup_after": 21.717208502199533 + } +} diff --git a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py index 2c257d5a..cb99f79d 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py @@ -40,6 +40,19 @@ returns -1 directly: RecursionError set, nothing counted) and `tp_leave_call()` in the one exit block, so ok, error and deopt exits all leave exactly once. +Fast entry (issue #147). A *bounded* function (`_bounded_functions`: no call cycle, every callee +bounded, no node that can run Python code) has a call tree of at most k compiled frames. Its +`tp_impl_f` first tests `tp_depth_room(k)`: with room for all k frames it runs `tp_fimpl_f`, the +twin with the same body and no tp_enter_call / tp_leave_call / entry poll, whose Calls go to the +callees' twins (a `static inline` function in the same unit, so the C compiler can inline it); +without room the counted path runs and raises RecursionError exactly where it always did. Frames +of a fast tree are not in `tp_depth.count`, so every twin takes `int tp_fd`, the number of the +tree's frames active including itself (1 from the precheck, `tp_fd + 1` into a callee twin), and +the two points where a fast tree can run Python code, a `redo` Call and the loop poll's +`tp_poll_slow` (a signal handler, a finalizer, another compiled call), are wrapped in +`tp_depth_py_begin(tp_fd)` / `tp_depth_py_end(tp_fd)`, which add and remove `tp_fd` from the count: +whatever runs there sees the count the counted path would have had. + Ownership rule (ir.py "Safety"), implemented by the `_own_*` emit helpers below and nowhere else: 1. every OBJ local, including each OBJ parameter, which the impl copies with a new reference at entry, owns one strong reference, or is NULL; @@ -331,6 +344,70 @@ def _is_effect(e: ir.Expr) -> bool: return isinstance(e, ir.BinOp) and e.type is Type.OBJ + +# --- bounded functions (fast entry, issue #147) ---------------------------------------------------- + +# Nodes that can run Python code (and so create Python frames the compiled frame count must see): +# their slow paths call getattr, the class, __bool__, __float__, a Python callable, a descriptor. +_RUNS_PYTHON = (ir.CallObject, ir.GetAttr, ir.Truth, ir.CompareObj, ir.ObjToFloat, ir.FieldGet, + ir.FieldSet, ir.New) + + +def _runs_no_python(f: ir.Function) -> bool: + """Conservative: False when `f` has any node that can run Python code, or touches an object + other than a fresh exact int, float or bool made by Box of a scalar (an OBJ parameter, local, + result or any other OBJ expression: releasing one can run a finalizer). A `redo` Call is + allowed (its callee is bounded, checked by the caller): the redo is one of the two points where + a fast tree runs Python code, and `_redo_call` counts the tree's frames around it.""" + if f.returns is Type.OBJ or f.entry_globals and any(g.type is Type.OBJ for g in f.entry_globals): + return False + if any(isinstance(p, ir.Param) and p.type is Type.OBJ for p in f.params): + return False + if any(t is Type.OBJ for t in f.locals.values()): + return False + for s in _walk_stmts(f.body): + if isinstance(s, ir.FieldSet): + return False + for e in _all_exprs(f): + if isinstance(e, _RUNS_PYTHON): + return False + if e.type is Type.OBJ and not (isinstance(e, ir.Box) and e.operand.type in ir.SCALARS): + return False + return True + + +def _bounded_functions(functions: dict[str, ir.Function]) -> dict[str, int]: + """name -> k for every bounded function (module docstring, "Fast entry"): in no call cycle, + every callee bounded, `_runs_no_python`. k(f) = 1 + max(k(callee)), 1 without callees: the most + compiled frames a call of `f` can have in flight at once.""" + callees: dict[str, set[str]] = {} + for name, f in functions.items(): + callees[name] = {e.function for e in _all_exprs(f) if isinstance(e, ir.Call)} + k: dict[str, int | None] = {} # None: not bounded (also while on the DFS stack) + + def visit(name: str) -> int | None: + if name in k: + return k[name] + k[name] = None # on the stack: reaching it again is a cycle + f = functions[name] + if not _runs_no_python(f): + return None + deepest = 0 + for c in callees[name]: + if c not in functions: + return None + kc = visit(c) + if kc is None: + return None + deepest = max(deepest, kc) + k[name] = deepest + 1 + return k[name] + + for name in functions: + visit(name) + return {n: v for n, v in k.items() if v is not None} + + # --- module -------------------------------------------------------------------------------------- class _ModGen: @@ -354,6 +431,7 @@ def __init__(self, module: ir.Module): total += len(cd.fields) self.n_slots = total self._uses: dict[str, set[str]] = {} + self.bounded = _bounded_functions(self.functions) # name -> k (module docstring, "Fast entry") def cls(self, fname: str, name: str) -> ir.ClassDecl: cd = self.classes.get(name) @@ -498,6 +576,8 @@ def generate(module: ir.Module, source_path: Path, display_path: str | None = No w("") for f in module.functions: w(_impl_signature(f) + ";") + if f.name in mg.bounded: + w(_impl_signature(f, fast=True) + ";") w("") w(_TEMPLATE_REDO) for b in bodies: @@ -507,6 +587,9 @@ def generate(module: ir.Module, source_path: Path, display_path: str | None = No for f in module.functions: w(f" {{{c_string(f.name)}, (PyCFunction)(void (*)(void)){_ident('tp_wrap', f.name)}, " f"METH_FASTCALL | METH_KEYWORDS, NULL}},") + w("#ifdef TP_TRACE_FAST") + w(' {"__tp_trace_fast_hits__", (PyCFunction)tp_trace_fast_hits, METH_NOARGS, NULL},') + w("#endif") w(" {NULL, NULL, 0, NULL}") w("};") w("") @@ -628,8 +711,10 @@ def _check_entry_globals(f: ir.Function, functions: dict[str, ir.Function]) -> N "user code, which could rebind the global between entry and use)") -def _impl_signature(f: ir.Function) -> str: +def _impl_signature(f: ir.Function, fast: bool = False) -> str: params = ["PyObject *tp_module"] + if fast: + params.append("int tp_fd") # compiled frames of this fast tree, itself included for p in f.params: if isinstance(p, ir.ArrayParam): params.append(f"{_ARRAY_CT[p.type]} *{_ident('l', p.name)}") @@ -642,14 +727,29 @@ def _impl_signature(f: ir.Function) -> str: if f.returns is not Type.NONE: ct = _CT[f.returns] params.append(f"{ct}{'' if ct.endswith('*') else ' '}*tp_out") + if fast: + return f"static inline int {_ident('tp_fimpl', f.name)}({', '.join(params)})" return f"static int {_ident('tp_impl', f.name)}({', '.join(params)})" +def _impl_call_args(f: ir.Function, fd: str | None = None) -> str: + """The arguments that forward an impl's own parameters to another impl of the same signature + (`fd`: the value of a fast twin's `tp_fd`).""" + args = ["tp_module"] + ([fd] if fd is not None else []) + for p in f.params: + args.append(_ident("l" if isinstance(p, ir.ArrayParam) or p.type is not Type.OBJ else "p", p.name)) + args += [_ident("l", g.name) for g in f.entry_globals] + if f.returns is not Type.NONE: + args.append("tp_out") + return ", ".join(args) + + # --- function bodies ----------------------------------------------------------------------------- class _FnGen: - def __init__(self, mg: _ModGen, f: ir.Function): + def __init__(self, mg: _ModGen, f: ir.Function, fast: bool = False): self.mg, self.f = mg, f + self.fast = fast # the uncounted twin of a bounded function self.lines: list[str] = [] self.depth = 1 self.temps: list[tuple[str, str]] = [] # (C type, name) @@ -947,7 +1047,11 @@ def ev_Call(self, e: ir.Call) -> str | None: self.emit("if (tp_s == 0) {") self.depth += 1 # Depth is counted by the callee itself (tp_enter_call at its entry, issue #57). - self.emit(f"tp_s = {_ident('tp_impl', e.function)}(tp_module{''.join(', ' + x for x in args)});") + if self.fast: + # the fast twin's callees are bounded too (_bounded_functions): their fast twins, uncounted + self.emit(f"tp_s = {_ident('tp_fimpl', e.function)}(tp_module, tp_fd + 1{''.join(', ' + x for x in args)});") + else: + self.emit(f"tp_s = {_ident('tp_impl', e.function)}(tp_module{''.join(', ' + x for x in args)});") self.depth -= 1 self.emit("}") if e.redo: @@ -999,13 +1103,19 @@ def _redo_call(self, callee: ir.Function, arg_exprs, values: list[str], t: str | boxes.append(b) av.append(b) rr = self.tmp("PyObject *") + # In a fast twin the interpreted callee (a Python frame) and whatever it calls must see the + # frames of this fast tree: count them for exactly the span that can run Python code, from + # the call to the release of its result (module docstring, "Fast entry"). + end = "tp_depth_py_end(tp_fd); " if self.fast else "" + if self.fast: + self.emit("tp_depth_py_begin(tp_fd);") if av: self.emit(f"{{ PyObject *tp_av[{len(av)}] = {{{', '.join(av)}}};") self.emit(f" {rr} = tp_cg_redo(tp_module, {k}, tp_av, {len(av)}, NULL); }}") else: self.emit(f"{rr} = tp_cg_redo(tp_module, {k}, NULL, 0, NULL);") self._own_release(*boxes) - self.emit(f"if ({rr} == NULL) {{ tp_rc = -1; goto tp_exit; }}") + self.emit(f"if ({rr} == NULL) {{ {end}tp_rc = -1; goto tp_exit; }}") rt = callee.returns if rt is Type.NONE: self.emit(f"tp_s = ({rr} == Py_None) ? 0 : 1;") @@ -1017,6 +1127,8 @@ def _redo_call(self, callee: ir.Function, arg_exprs, values: list[str], t: str | self.emit(f"tp_s = {_UNBOX[rt]}({rr}, &{t});") what = {Type.F64: "a float", Type.BOOL: "True or False", Type.I64: "an int in i64"}[rt] self._own_release(rr) + if self.fast: + self.emit("tp_depth_py_end(tp_fd);") msg = c_string(f"typedpython: the interpreted redo of {callee.name} did not return {what}") self.emit(f"if (tp_s != 0) {{ PyErr_SetString(PyExc_SystemError, {msg}); tp_rc = -1; goto tp_exit; }}") self.depth -= 1 @@ -1369,6 +1481,12 @@ def poll(self) -> None: if not self.snapshot: # A function-local countdown (kept in a register), not the module counter: a load and # store of a global on every iteration of a short inner loop cost fannkuch ~38%. + if self.fast: + # the poll can run a signal handler or another compiled call: count this fast tree + self.emit("if (tp_pc_tick(&tp_pc)) { tp_depth_py_begin(tp_fd); " + "tp_s = tp_poll_slow(tp_st->breaker); tp_depth_py_end(tp_fd); " + "if (tp_s != 0) { tp_rc = -1; goto tp_exit; } }") + return self.emit("if (tp_pc_tick(&tp_pc) && tp_poll_slow(tp_st->breaker) != 0) " "{ tp_rc = -1; goto tp_exit; }") @@ -1444,7 +1562,7 @@ def st_Continue(self, s) -> None: # whole function - def generate(self) -> str: + def impl(self) -> str: f = self.f prologue = [] for p in f.params: @@ -1459,12 +1577,16 @@ def generate(self) -> str: self.emit("tp_rc = -1;") self.emit("goto tp_exit;") - out = [_impl_signature(f), "{", + out = [_impl_signature(f, fast=self.fast), "{", " int tp_rc = 0;", " int tp_s = 0;", " int tp_pc = TP_POLL_INTERVAL; /* loop poll countdown (#141) */", - " tp_state *tp_st = (tp_state *)PyModule_GetState(tp_module);", - " PyObject *tp_dict = PyModule_GetDict(tp_module); /* borrowed: owned by the module */"] + ] + body_text = "\n".join(self.lines) + if not self.fast or "tp_st" in body_text: + out.append(" tp_state *tp_st = (tp_state *)PyModule_GetState(tp_module);") + if not self.fast or "tp_dict" in body_text: + out.append(" PyObject *tp_dict = PyModule_GetDict(tp_module); /* borrowed: owned by the module */") params = {p.name for p in f.params} | {g.name for g in f.entry_globals} for name, t in self.local_arrays.items(): out.append(f" {_ARRAY_CT[t]} {_ident('l', name)} = {{0}};") @@ -1478,7 +1600,11 @@ def generate(self) -> str: init = "NULL" if ct == "PyObject *" else "{0}" if ct in _CT_ARRAY_REV else "0" sep = "" if ct.endswith("*") else " " out.append(f" {ct}{sep}{n} = {init};") - out.append(" (void)tp_st; (void)tp_dict; (void)tp_s; (void)tp_pc;" + # (a fast twin declares tp_st / tp_dict only when its body reads them: PyModule_GetState and + # PyModule_GetDict are external calls, and a leaf must stay free of them to inline well) + out.append(" (void)tp_s; (void)tp_pc;" + (" (void)tp_fd;" if self.fast else "") + + (" (void)tp_st;" if not self.fast or "tp_st" in body_text else "") + + (" (void)tp_dict;" if not self.fast or "tp_dict" in body_text else "") + "".join(f" (void){_ident('l', n)};" for n in self.types if n not in params) + "".join(f" (void){_ident('l', n)};" for n in self.local_arrays)) # Depth guard (issue #57): nothing is owned yet, so a refusal returns directly and counts @@ -1487,11 +1613,28 @@ def generate(self) -> str: if isinstance(p, ir.Param) and p.cls is not None: k = self._class(p.cls) out.append(f" if ({_param_cls_fail(_ident('p', p.name), k, p.optional)}) return 1;") - out.append(" if (tp_enter_call() != 0) return -1;") + k_bound = self.mg.bounded.get(f.name) + if self.fast: + # Fast twin of a bounded function (module docstring, "Fast entry"): the caller proved + # room for every frame this call tree can have, so there is nothing to count or refuse. + pass + else: + if k_bound is not None: + # Precheck: room for the k frames of this whole call tree, then run its uncounted + # twin. Without room, fall through to the counted path below, which raises + # RecursionError at exactly the call where the interpreter would. + out.append(f" if (tp_depth_room({k_bound})) {{") + out.append(" int tp_fr;") + out.append(" tp_trace_fast_hit();") + out.append(f" tp_fr = {_ident('tp_fimpl', f.name)}({_impl_call_args(f, '1')});") + out.append(" tp_depth_done();") + out.append(" return tp_fr;") + out.append(" }") + out.append(" if (tp_enter_call() != 0) return -1;") # Eval breaker (#141): a snapshot-holding function never polls and defers others' polls. if self.snapshot: out.append(" tp_snapshot_depth++;") - else: + elif not self.fast: out.append(" if (tp_poll(tp_st->breaker) != 0) { tp_rc = -1; goto tp_exit; }") out += prologue out += self.lines @@ -1499,13 +1642,22 @@ def generate(self) -> str: out += self._own_release_all() if self.snapshot: out.append(" tp_snapshot_depth--;") - out.append(" tp_leave_call();") + if not self.fast: + out.append(" tp_leave_call();") out.append(" return tp_rc;") out.append("}") out.append("") - out.append(self.wrapper()) return "\n".join(out) + def generate(self) -> str: + """The impl (preceded by its fast twin when bounded) and the wrapper.""" + parts = [] + if self.f.name in self.mg.bounded: + parts.append(_FnGen(self.mg, self.f, fast=True).impl()) + parts.append(self.impl()) + parts.append(self.wrapper()) + return "\n".join(parts) + def wrapper(self) -> str: f = self.f name_k = self.mg.name_index(f.name) @@ -1847,6 +1999,11 @@ def _exec_function(mg: _ModGen) -> str: " tp_release(&tp_f); tp_release(&tp_cf);", ] L += [ + "#ifdef TP_TRACE_FAST", + f" tp_cf = PyCFunction_NewEx(&tp_methods[{len(mg.module.functions)}], tp_module, tp_modname);", + " if (tp_cf == NULL || PyDict_SetItemString(tp_dict, \"__tp_trace_fast_hits__\", tp_cf) < 0) goto tp_done;", + " tp_release(&tp_cf);", + "#endif", " tp_rc = 0;", "tp_done:", " tp_release(&tp_code); tp_release(&tp_res); tp_release(&tp_table); tp_release(&tp_zero);", diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md index 92dfa751..37295a6d 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md @@ -116,6 +116,29 @@ per module (the header is `static`) and per thread (`TP_THREAD_LOCAL`): frames o compiled module that sit between two frames of this one are not counted, and another thread's frames never are. +## Fast entry (bounded functions; issue #147) + int tp_depth_room(Py_ssize_t k); /* 1 iff tp_python_depth() + count + k <= Py_GetRecursionLimit() */ + void tp_trace_fast_hit(void); /* counts a fast entry; a no-op unless -DTP_TRACE_FAST */ + void tp_depth_py_begin(int fd); /* count += fd, around code a fast tree may run (redo, poll) */ + void tp_depth_py_end(int fd); /* count -= fd; drops the cached frame at count 0 */ + void tp_depth_done(void); /* after a fast tree: drop the cached top frame when count is 0 */ +A function is *bounded* when it is in no call cycle, every function it Calls is bounded, and its body +has no node that can run Python code (CallObject, GetAttr, Truth, CompareObj, ObjToFloat, FieldGet, +FieldSet, New, any OBJ-typed expression except Box of a scalar). A `redo` Call to a bounded callee is allowed. Its k is `1 + max(k(callee))` (1 with no +callees). cgen emits, per bounded `f`, `static inline int tp_fimpl_f(...)` (the impl body without +tp_enter_call / tp_leave_call and without the entry poll, Calls to the callees' `tp_fimpl_*`) and, at +the top of `tp_impl_f` after the Param.cls guards, `if (tp_depth_room(k)) { r = tp_fimpl_f(...); +tp_depth_done(); return r; }`. No room: the counted path runs and raises RecursionError exactly where +it raised before. `tp_enter_call` compares `depth + count + 1`; the fast tree has at most k frames, so +`depth + count + k` is the same comparison at its deepest frame. Py_EnterRecursiveCall is skipped on +the fast path: the tree is bounded, so it adds at most k small C frames. A twin takes `int tp_fd` +(the tree's active frames, itself included; the precheck passes 1, a Call passes `tp_fd + 1`). The +two points where a fast tree can run Python code, a `redo` Call and the loop poll's `tp_poll_slow`, +are wrapped in `tp_depth_py_begin(tp_fd)` / `tp_depth_py_end(tp_fd)`, so everything run from there +(interpreted frames, signal handlers, compiled calls through their precheck) sees the exact count. +`-DTP_TRACE_FAST` +(tests only) adds `__tp_trace_fast_hits__()` to the module: the number of entries that took the fast path. + ## Eval breaker (top of every loop iteration and impl entry; issue #141) int tp_poll(PyObject *breaker); /* 0 continue; -1 exception set (a signal handler raised) */ int tp_poll_slow(PyObject *breaker); /* every TP_POLL_INTERVAL polls: call the breaker */ diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h index db0d16ef..b80a3696 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h @@ -1590,6 +1590,65 @@ static inline void tp_leave_call(void) } } +/* Fast entry (issue #147). A *bounded* compiled function (cgen: no call cycle, bounded callees, no + * node that can run Python code) has a call tree of at most k compiled frames, k known at build time. + * `tp_depth_room(k)` is true exactly when tp_enter_call would accept all k nested frames, the same + * comparison with the deepest frame's count (depth + count + k <= limit); then the whole tree runs + * through its uncounted twins (`tp_fimpl_*`: no tp_enter_call, no tp_leave_call), which the C + * compiler can inline. When it is false the caller takes the counted path, so RecursionError is raised + * at exactly the call where it is raised without the fast entry, never earlier. + * + * Skipping Py_EnterRecursiveCall on the fast path is safe because the tree is bounded: at most k + * small compiled frames (no recursion, no Python callbacks except at the two points below), so they + * use a bounded amount of C stack on top of the entry's. + * + * The frames of a fast tree are not in `tp_depth.count`. A bounded tree runs Python code at exactly + * two points: a `redo` Call (the interpreted original) and the eval-breaker poll in a loop (signal + * handler, finalizer, another compiled call). Around each, the twin brackets the code with + * tp_depth_py_begin(tp_fd) / tp_depth_py_end(tp_fd), `tp_fd` being the tree's active frames (itself + * included), so the count is exactly what the counted path would have there: Python frames and + * compiled calls started from it see the true depth. */ +static inline int tp_depth_room(Py_ssize_t k) +{ + return tp_python_depth() + tp_depth.count + k <= Py_GetRecursionLimit(); +} + +/* After a fast tree: tp_python_depth may have cached the top frame (a strong reference) and, with + * no counted frame in flight, nothing else would drop it, as tp_leave_call does at count 0. */ +static inline void tp_depth_done(void) +{ + if (tp_depth.count == 0) { + Py_CLEAR(tp_depth.frame); + } +} + +/* Around a point of a fast tree that can run Python code: the count includes the tree's `fd` active + * frames. begin/end are balanced. end also drops the cached top frame when no compiled frame is + * counted any more (the code in between may have cached another frame, as tp_leave_call would). */ +static inline void tp_depth_py_begin(int fd) +{ + tp_depth.count += fd; +} + +static inline void tp_depth_py_end(int fd) +{ + tp_depth.count -= fd; + tp_depth_done(); +} + +#ifdef TP_TRACE_FAST +/* Test builds only (-DTP_TRACE_FAST): how many entries took the fast path. */ +static Py_ssize_t tp_trace_fast_hits_count = 0; +static inline void tp_trace_fast_hit(void) { tp_trace_fast_hits_count++; } +static PyObject *tp_trace_fast_hits(PyObject *tp_module, PyObject *tp_unused) +{ + (void)tp_module; (void)tp_unused; + return PyLong_FromSsize_t(tp_trace_fast_hits_count); +} +#else +static inline void tp_trace_fast_hit(void) {} +#endif + /* ------------------------------------------------------------------------------------------ * Eval breaker (issue #141). The interpreter checks between bytecodes for signals, pending calls, * GIL drop requests, async exceptions and scheduled collections; compiled code has no bytecodes. diff --git a/python-multiplatform-ksp/src/test/python/test_fast_entry.py b/python-multiplatform-ksp/src/test/python/test_fast_entry.py new file mode 100644 index 00000000..7adc7d2c --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_fast_entry.py @@ -0,0 +1,395 @@ +"""TypedPython fast entry (issue #147): a depth precheck plus an uncounted entry for *bounded* +compiled functions (no call cycle, bounded callees, no node that can run Python code). + + * the precheck `tp_depth_room(k)` is the exact condition under which the per-call counting of + `tp_enter_call` could not refuse anything, so the point where RecursionError is raised is the + interpreter's, at every depth (differential test, in a child interpreter with a small limit); + * the fast path is really taken when there is room (a hit counter compiled in only with + `-DTP_TRACE_FAST`) and is not taken when there is none; + * functions that are not bounded get no fast impl; + * the generated C shape (the fast impl is `static inline`, calls only fast impls, the normal impl + keeps counting). + +Builds go through the front end, verifier, cgen and cbuild in `.tmp/` of the worktree via tmp_path. +""" +import json +import subprocess +import sys +import textwrap +from pathlib import Path + +import pytest + +from typedpython import cbuild, cgen, frontend, verify + +SOURCE = """\ +# typedpython: compiled + + +def helper(x: int) -> float: + d: dict[str, int] = {} + d['a'] = x + return d['a'] * 2.5 + + +def leaf(i: int) -> float: + return 1.0 / (i + 1) + + +def mid(i: int) -> float: + return leaf(i) + leaf(i + 1) + + +def top(i: int) -> float: + return mid(i) * 2.0 + + +def objy(i: int) -> float: + return float(helper(i)) + leaf(i) + + +def viaobj(i: int) -> float: + return objy(i) + 1.0 + + +def rec(n: int) -> int: + if n == 0: + return 0 + return 1 + rec(n - 1) + + +def callrec(n: int) -> int: + return rec(n) + 1 + + +def loopy(n: int) -> float: + s: float = 0.0 + for i in range(n): + s += leaf(i) + return s + + +def outer(n: int) -> float: + return loopy(n) + 1.0 + + +def cube(x: int) -> float: + return float(x * x * x) + leaf(x) + + +def fill(x: int, out: list[float]) -> None: + out[0] = cube(x) +""" + +BOUNDED = {"leaf": 1, "mid": 2, "top": 3, "loopy": 2, "outer": 3, "cube": 2, "fill": 3} +NOT_BOUNDED = ("objy", "viaobj", "rec", "callrec") + + +def build(d: Path, extra: tuple[str, ...] = ()): + path = d / "fast_mod.py" + path.write_text(SOURCE) + proved, _ = verify.verify(frontend.lower(path)) + c = cgen.generate(proved, path, path.name) + so = cbuild.build(c, path.stem, d / "out", flags=cbuild.DEFAULT_FLAGS + extra) + return proved, c, so + + +@pytest.fixture(scope="module") +def traced(tmp_path_factory): + return build(tmp_path_factory.mktemp("fast_traced"), ("-DTP_TRACE_FAST",)) + + +@pytest.fixture(scope="module") +def plain_build(tmp_path_factory): + return build(tmp_path_factory.mktemp("fast_plain")) + + +def fn_text(c: str, signature_start: str) -> str: + """The text of the C function whose definition starts with `signature_start`.""" + i = c.index(signature_start + "(", c.index("static const char tp_source")) + # skip forward declarations: find the definition (the one followed by a newline and `{`) + while not c[c.index(")", i) + 1:].startswith("\n{"): + i = c.index(signature_start + "(", i + 1) + return c[i:c.index("\n}\n", i)] + + +def test_every_function_compiles(traced): + proved = traced[0] + names = {f.name for f in proved.functions} + assert names == set(BOUNDED) | set(NOT_BOUNDED), proved.skipped + + +def test_bounded_functions_get_a_static_inline_fast_impl(traced): + c = traced[1] + for name in BOUNDED: + assert f"static inline int tp_fimpl_{name}(" in c, name + + +def test_unbounded_functions_get_no_fast_impl(traced): + c = traced[1] + for name in NOT_BOUNDED: + assert f"tp_fimpl_{name}" not in c, name + + +def test_the_fast_impl_calls_only_fast_impls_and_never_counts(traced): + c = traced[1] + fast = fn_text(c, "static inline int tp_fimpl_mid") + assert "tp_fimpl_leaf(" in fast and "tp_impl_leaf(" not in fast + assert "tp_enter_call" not in fast and "tp_leave_call" not in fast + assert "tp_poll(tp_st->breaker)" not in fast # no entry poll + loop = fn_text(c, "static inline int tp_fimpl_loopy") + assert "tp_poll(" in loop or "tp_pc" in loop # loop polls stay + + +def test_the_normal_impl_prechecks_and_still_counts(traced): + c = traced[1] + normal = fn_text(c, "static int tp_impl_mid") + assert "tp_depth_room(2)" in normal + assert normal.index("tp_depth_room(2)") < normal.index("tp_enter_call()") + assert "tp_impl_leaf(" in normal and "tp_fimpl_leaf(" not in normal + top = fn_text(c, "static int tp_impl_top") + assert "tp_depth_room(3)" in top + assert "tp_depth_room" not in fn_text(c, "static int tp_impl_rec") + + +def test_the_fast_impl_is_never_addressed(traced): + c = traced[1] + assert "&tp_fimpl_" not in c and "(tp_fimpl_" not in c + + +LOAD = """\ +import importlib.machinery, importlib.util, json, sys +def load(name, path, ext): + if ext: + loader = importlib.machinery.ExtensionFileLoader(name, path) + spec = importlib.util.spec_from_file_location(name, path, loader=loader) + else: + spec = importlib.util.spec_from_file_location(name, path) + m = importlib.util.module_from_spec(spec) + (loader if ext else spec.loader).exec_module(m) + return m +comp = load("fast_mod", sys.argv[1], True) +plain = load("fast_mod_plain", sys.argv[2], False) +LIMIT = int(sys.argv[3]) +sys.setrecursionlimit(LIMIT) +""" + +DESCEND = """\ +def descend(d, fn, a): + if d > 0: + return descend(d - 1, fn, a) + try: + return ("ok", fn(a)) + except RecursionError: + return ("rec",) + +def run(fn, a, d): + try: + return descend(d, fn, a) + except RecursionError: + return ("deep",) +""" + +CHILD = LOAD + DESCEND + """\ +report = {"mismatch": [], "ok": {}, "rec": {}, "hits": []} +for name in ("leaf", "mid", "top", "loopy"): + arg = 3 + for d in range(1, LIMIT + 6): + before = comp.__tp_trace_fast_hits__() + c = run(getattr(comp, name), arg, d) + hit = comp.__tp_trace_fast_hits__() - before + p = run(getattr(plain, name), arg, d) + if c != p: + report["mismatch"].append([name, d, c, p]) + kind = c[0] + report.setdefault(kind, {}) + report[kind][name] = report[kind].get(name, 0) + 1 + # one hit per entry from Python that found room, none when it ran counted (or never ran) + if hit != (1 if kind == "ok" else 0): + report["hits"].append([name, d, kind, hit]) +print(json.dumps(report)) +""" + + +def test_differential_depth_at_every_depth_around_the_limit(traced, tmp_path): + proved, c, so = traced + plain_src = tmp_path / "fast_mod_plain.py" + plain_src.write_text(SOURCE) + proc = subprocess.run([sys.executable, "-c", CHILD, str(so), str(plain_src), "200"], + capture_output=True, text=True, timeout=300) + assert proc.returncode == 0, proc.stdout + proc.stderr + r = json.loads(proc.stdout.strip().splitlines()[-1]) + assert r["mismatch"] == [], r["mismatch"][:5] + assert r["hits"] == [], r["hits"][:5] + for name in ("leaf", "mid", "top", "loopy"): # the sweep crossed the boundary + assert r["ok"][name] > 100 and r["rec"][name] >= 1, (name, r) + # top needs more room than leaf: it starts raising at a smaller depth + assert r["rec"]["top"] > r["rec"]["leaf"] + + +CHILD_REDO = LOAD + DESCEND + """\ +def mk(mod): + def call(x): + out = [0.0] + mod.fill(x, out) + return out[0] + return call + +def deep_comp(i): + return float(comp.rec(50)) + +def deep_plain(i): + return float(plain.rec(50)) + +report = {"mismatch": [], "ok": {}, "rec": {}, "deopts": 0} +for x in (3, 3_000_000): # 3: no overflow; 3_000_000: x * x * x overflows i64, the redo runs + if x != 3: + # the interpreted cube of the redo looks `leaf` up in its module globals: make that a deep + # call, so that what the redo runs depends on the exact depth it starts from + comp.leaf = deep_comp + plain.leaf = deep_plain + ok = 0 + before = comp.__typedpython_deopts__ + for d in range(1, LIMIT + 6): + c = run(mk(comp), x, d) + p = run(mk(plain), x, d) + if c != p: + report["mismatch"].append([x, d, c, p]) + kind = c[0] + report.setdefault(kind, {}) + report[kind][str(x)] = report[kind].get(str(x), 0) + 1 + ok += kind == "ok" + if x != 3: + report["deopts"] = comp.__typedpython_deopts__ - before + report["deopt_ok"] = ok +print(json.dumps(report)) +""" + +CHILD_SIGNAL = LOAD + DESCEND + """\ +import signal + +class Stop(Exception): + pass + +M = None +res = [] + +def measure(): + lo, hi = 0, 400 + while lo < hi: + mid = (lo + hi + 1) // 2 + try: + M.rec(mid) + lo = mid + except RecursionError: + hi = mid - 1 + return lo + +def handler(sig, frm): + res.append(measure()) + raise Stop + +signal.signal(signal.SIGALRM, handler) + +# interpreted baseline: outer_py -> spin_py -> trip, the handler runs in trip's frame, where the +# compiled run has the breaker lambda (outer and spin are compiled frames with no Python frame) +def trip(): + signal.raise_signal(signal.SIGALRM) + +def spin_py(): + trip() + +def outer_py(a): + spin_py() + +def base(d): + global M + M = plain + res.clear() + try: + descend(d, outer_py, 0) + except Stop: + pass + return list(res) + +def compiled(d): + global M + M = comp + res.clear() + signal.setitimer(signal.ITIMER_REAL, 0.05) + try: + descend(d, comp.outer, 10 ** 9) # outer -> loopy: the poll runs in a fast tree at fd 2 + except Stop: + pass + signal.setitimer(signal.ITIMER_REAL, 0) + return list(res) + +report = {"mismatch": [], "values": []} +for d in range(0, 150, 7): + b, c = base(d), compiled(d) + report["values"].append(b) + if b != c or len(b) != 1: + report["mismatch"].append([d, b, c]) +print(json.dumps(report)) +""" + + +def run_child(script, so, plain_src, limit="200"): + proc = subprocess.run([sys.executable, "-c", script, str(so), str(plain_src), limit], + capture_output=True, text=True, timeout=300) + assert proc.returncode == 0, proc.stdout + proc.stderr + return json.loads(proc.stdout.strip().splitlines()[-1]) + + +def test_a_redo_inside_a_fast_tree_sees_the_exact_depth(traced, tmp_path): + # fill (impure, bounded) -> cube (bounded, deopts on i64 overflow): the redo runs the interpreted + # cube, whose `leaf` (rebound by the child to a call of the compiled recursive rec(50)) goes + # through the counted path; the depth at which that raises RecursionError must be the + # interpreter's, so the redo must be counted with the frames of the fast tree around it + proved, c, so = traced + plain_src = tmp_path / "fast_mod_plain.py" + plain_src.write_text(SOURCE) + r = run_child(CHILD_REDO, so, plain_src) + assert r["mismatch"] == [], r["mismatch"][:5] + for x in ("3", "3000000"): + assert r["ok"][x] > 100 and r["rec"][x] >= 1, (x, r) + assert r["deopts"] >= r["deopt_ok"] > 100 # the redo really ran at every ok depth + assert "tp_depth_py_begin(tp_fd);" in fn_text(c, "static inline int tp_fimpl_fill") + + +def test_a_signal_handler_in_a_fast_tree_poll_sees_the_exact_depth(traced, tmp_path): + # a compiled recursive call made by a signal handler that the poll of a loop two frames deep + # in a fast tree runs reaches RecursionError at the interpreter's depth + proved, c, so = traced + plain_src = tmp_path / "fast_mod_plain.py" + plain_src.write_text(SOURCE) + r = run_child(CHILD_SIGNAL, so, plain_src) + assert r["mismatch"] == [], r["mismatch"][:5] + assert len({v[0] for v in r["values"]}) > 5 # the measured depth tracks the stack depth + loop = fn_text(c, "static inline int tp_fimpl_loopy") + assert "tp_depth_py_begin(tp_fd);" in loop and "tp_depth_py_end(tp_fd);" in loop + + +def test_the_fast_path_is_taken_when_there_is_room(tmp_path): + proved, c, so = build(tmp_path, ("-DTP_TRACE_FAST",)) + m = cbuild.load("fast_mod", so) + assert m.__tp_trace_fast_hits__() == 0 + assert m.top(4) == 2.0 * (1 / 5 + 1 / 6) + assert m.__tp_trace_fast_hits__() == 1 # one precheck, no per-callee counting + assert m.loopy(10) == sum(1.0 / (i + 1) for i in range(10)) + assert m.__tp_trace_fast_hits__() == 2 + assert m.rec(10) == 10 and m.callrec(10) == 11 # not bounded: no precheck + assert m.__tp_trace_fast_hits__() == 2 + assert m.viaobj(3) == 3 * 2.5 + 1 / 4 + 1.0 + assert m.__tp_trace_fast_hits__() == 3 # objy is not bounded; its callee leaf is + assert m.mid(2) == 1 / 3 + 1 / 4 # a bounded callee entered from Python + assert m.__tp_trace_fast_hits__() == 4 + + +def test_the_trace_counter_is_absent_from_a_normal_build(plain_build): + c = plain_build[1] + assert "TP_TRACE_FAST" in c # guarded, so compiled out + so = plain_build[2] + m = cbuild.load("fast_mod", so) + assert not hasattr(m, "__tp_trace_fast_hits__") + assert m.top(1) == 2.0 * (1 / 2 + 1 / 3)