From a24f9ab1a168aad3ae2d51cfad89128c0c959a79 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sun, 25 Oct 2026 07:05:54 +0900 Subject: [PATCH] Perf: one thread-state and one TLS lookup per compiled call (#125) --- .../src/main/python/typedpython/cgen.py | 45 +++++----- .../main/python/typedpython/runtime/API.md | 41 ++++++---- .../python/typedpython/runtime/tp_runtime.h | 82 +++++++++++-------- .../src/test/python/test_cgen.py | 2 +- .../src/test/python/test_fast_entry.py | 12 +-- .../src/test/python/test_recursion.py | 28 ++++++- 6 files changed, 132 insertions(+), 78 deletions(-) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py index 3c10356b8..6f8d023d1 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py @@ -362,7 +362,7 @@ def _all_exprs(f: ir.Function): def _traced_call(k: int) -> str: """tp_traced for the function whose interpreted original is `names[k]` (SPEC N-14). The code object is a strong reference in the module state, filled at module exec.""" - return (f"tp_traced(tp_module, (PyCodeObject *)((tp_state *)PyModule_GetState(tp_module))->orig_code[{k}])") + return (f"tp_traced(tp_module, tp_ts, (PyCodeObject *)((tp_state *)PyModule_GetState(tp_module))->orig_code[{k}])") def _can_deopt(e: ir.Expr, functions: dict[str, ir.Function]) -> bool: @@ -874,6 +874,9 @@ def _impl_signature(f: ir.Function, fast: bool = False) -> str: params = ["PyObject *tp_module"] if fast: params.append("int tp_fd") # compiled frames of this fast tree, itself included + else: + params.append("PyThreadState *tp_ts") # the caller's one thread-state lookup (#125) + params.append("tp_tls *tp_t") # its one TLS-address lookup (#125) for p in f.params: if isinstance(p, ir.ArrayParam): params.append(f"{'tp_buf' if p.buffer else _ARRAY_CT[p.type]} *{_ident('l', p.name)}") @@ -894,7 +897,7 @@ def _impl_signature(f: ir.Function, fast: bool = False) -> str: def _impl_call_args(f: ir.Function, fd: str | None = None) -> str: """The arguments that forward an impl's own parameters to another impl of the same signature (`fd`: the value of a fast twin's `tp_fd`).""" - args = ["tp_module"] + ([fd] if fd is not None else []) + args = ["tp_module"] + ([fd] if fd is not None else []) + ["tp_t"] for p in f.params: args.append(_ident("l" if isinstance(p, ir.ArrayParam) or p.type is not Type.OBJ else "p", p.name)) args += [_ident("l", g.name) for g in f.entry_globals] @@ -1304,9 +1307,9 @@ def ev_Call(self, e: ir.Call) -> str | None: # Depth is counted by the callee itself (tp_enter_call at its entry, issue #57). if self.fast: # the fast twin's callees are bounded too (_bounded_functions): their fast twins, uncounted - self.emit(f"tp_s = {_ident('tp_fimpl', e.function)}(tp_module, tp_fd + 1{''.join(', ' + x for x in args)});") + self.emit(f"tp_s = {_ident('tp_fimpl', e.function)}(tp_module, tp_fd + 1, tp_t{''.join(', ' + x for x in args)});") else: - self.emit(f"tp_s = {_ident('tp_impl', e.function)}(tp_module{''.join(', ' + x for x in args)});") + self.emit(f"tp_s = {_ident('tp_impl', e.function)}(tp_module, tp_ts, tp_t{''.join(', ' + x for x in args)});") self.depth -= 1 self.emit("}") if ti is not None: @@ -1370,9 +1373,9 @@ def _redo_call(self, callee: ir.Function, arg_exprs, values: list[str], t: str | # In a fast twin the interpreted callee (a Python frame) and whatever it calls must see the # frames of this fast tree: count them for exactly the span that can run Python code, from # the call to the release of its result (module docstring, "Fast entry"). - end = "tp_depth_py_end(tp_fd); " if self.fast else "" + end = "tp_depth_py_end(tp_t, tp_fd); " if self.fast else "" if self.fast: - self.emit("tp_depth_py_begin(tp_fd);") + self.emit("tp_depth_py_begin(tp_t, tp_fd);") if av: self.emit(f"{{ PyObject *tp_av[{len(av)}] = {{{', '.join(av)}}};") self.emit(f" {rr} = tp_cg_redo(tp_module, {k}, tp_av, {len(av)}, NULL); }}") @@ -1396,7 +1399,7 @@ def _redo_call(self, callee: ir.Function, arg_exprs, values: list[str], t: str | what = {Type.F64: "a float", Type.BOOL: "True or False", Type.I64: "an int in i64"}[rt] self._own_release(rr) if self.fast: - self.emit("tp_depth_py_end(tp_fd);") + self.emit("tp_depth_py_end(tp_t, tp_fd);") msg = c_string(f"typedpython: the interpreted redo of {callee.name} did not return {what}") self.emit(f"if (tp_s != 0) {{ PyErr_SetString(PyExc_SystemError, {msg}); tp_rc = -1; goto tp_exit; }}") self.depth -= 1 @@ -1884,11 +1887,11 @@ def poll(self) -> None: # store of a global on every iteration of a short inner loop cost fannkuch ~38%. if self.fast: # the poll can run a signal handler or another compiled call: count this fast tree - self.emit("if (tp_pc_tick(&tp_pc)) { tp_depth_py_begin(tp_fd); " - "tp_s = tp_poll_slow(tp_st->breaker); tp_depth_py_end(tp_fd); " + self.emit("if (tp_pc_tick(&tp_pc)) { tp_depth_py_begin(tp_t, tp_fd); " + "tp_s = tp_poll_slow(tp_t, tp_st->breaker); tp_depth_py_end(tp_t, tp_fd); " "if (tp_s != 0) { tp_rc = -1; goto tp_exit; } }") return - self.emit("if (tp_pc_tick(&tp_pc) && tp_poll_slow(tp_st->breaker) != 0) " + self.emit("if (tp_pc_tick(&tp_pc) && tp_poll_slow(tp_t, tp_st->breaker) != 0) " "{ tp_rc = -1; goto tp_exit; }") def st_While(self, s: ir.While) -> None: @@ -2029,7 +2032,7 @@ def impl(self) -> str: out.append(f" {ct}{sep}{n} = {init};") # (a fast twin declares tp_st / tp_dict only when its body reads them: PyModule_GetState and # PyModule_GetDict are external calls, and a leaf must stay free of them to inline well) - out.append(" (void)tp_s; (void)tp_pc;" + (" (void)tp_fd;" if self.fast else "") + out.append(" (void)tp_s; (void)tp_pc;" + (" (void)tp_fd;" if self.fast else " (void)tp_ts;") + " (void)tp_t;" + (" (void)tp_st;" if not self.fast or "tp_st" in body_text else "") + (" (void)tp_dict;" if not self.fast or "tp_dict" in body_text else "") + "".join(f" (void){_ident('l', n)};" for n in self.types if n not in params) @@ -2050,27 +2053,27 @@ def impl(self) -> str: # Precheck: room for the k frames of this whole call tree, then run its uncounted # twin. Without room, fall through to the counted path below, which raises # RecursionError at exactly the call where the interpreter would. - out.append(f" if (tp_depth_room({k_bound})) {{") + out.append(f" if (tp_depth_room(tp_ts, tp_t, {k_bound})) {{") out.append(" int tp_fr;") out.append(" tp_trace_fast_hit();") out.append(f" tp_fr = {_ident('tp_fimpl', f.name)}({_impl_call_args(f, '1')});") - out.append(" tp_depth_done();") + out.append(" tp_depth_done(tp_t);") out.append(" return tp_fr;") out.append(" }") - out.append(" if (tp_enter_call() != 0) return -1;") + out.append(" if (tp_enter_call(tp_ts, tp_t) != 0) return -1;") # Eval breaker (#141): a snapshot-holding function never polls and defers others' polls. if self.snapshot: - out.append(" tp_snapshot_depth++;") + out.append(" tp_t->snapshot_depth++;") elif not self.fast: - out.append(" if (tp_poll(tp_st->breaker) != 0) { tp_rc = -1; goto tp_exit; }") + out.append(" if (tp_poll(tp_t, tp_st->breaker) != 0) { tp_rc = -1; goto tp_exit; }") out += prologue out += self.lines out.append("tp_exit:") out += self._own_release_all() if self.snapshot: - out.append(" tp_snapshot_depth--;") + out.append(" tp_t->snapshot_depth--;") if not self.fast: - out.append(" tp_leave_call();") + out.append(" tp_leave_call(tp_t);") out.append(" return tp_rc;") out.append("}") out.append("") @@ -2091,7 +2094,9 @@ def wrapper(self) -> str: n = len(f.params) L = [f"static PyObject *{_ident('tp_wrap', f.name)}(PyObject *tp_module, PyObject *const *tp_args, " "Py_ssize_t tp_nargs, PyObject *tp_kwnames)", "{", - " int tp_s = 0, tp_x = 0;"] + " int tp_s = 0, tp_x = 0;", + " PyThreadState *tp_ts = PyThreadState_GetUnchecked(); /* the one lookup of this call (#125) */", + " tp_tls *tp_t = tp_tls_get(); /* and its one TLS-address lookup (#125) */"] if f.returns not in (Type.NONE,): init = "NULL" if f.returns is Type.OBJ else "0" L.append(f" {_CT[f.returns]} tp_r = {init};") @@ -2199,7 +2204,7 @@ def wrapper(self) -> str: if gnames: at = len(args) - (1 if f.returns is not Type.NONE else 0) args[at:at] = gnames - L.append(f" tp_s = {_ident('tp_impl', f.name)}(tp_module{''.join(', ' + a for a in args)});") + L.append(f" tp_s = {_ident('tp_impl', f.name)}(tp_module, tp_ts, tp_t{''.join(', ' + a for a in args)});") if f.pure: L.append(" if (tp_s == 1) goto tp_deopt;") else: diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md index a41ce6545..4cf0b7cf9 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md @@ -223,8 +223,20 @@ Same structs as above with `list == NULL` and `dirty == NULL`. 0 ok, -1 MemoryEr PyObject *tp_tuple(PyObject *const *items, Py_ssize_t n); /* new tuple; steals nothing */ ## Call depth (every compiled impl function: enter at entry, leave at every exit; issue #57) - int tp_enter_call(void); /* 0 ok; -1 RecursionError set, nothing counted (the caller must NOT call leave) */ - void tp_leave_call(void); /* undoes one successful tp_enter_call; exactly once per success, on ok, error and deopt exits alike */ + int tp_enter_call(PyThreadState *ts, tp_tls *t); /* 0 ok; -1 RecursionError set, nothing counted (the caller must NOT call leave) */ + void tp_leave_call(tp_tls *t); /* undoes one successful tp_enter_call; exactly once per success, on ok, error and deopt exits alike */ +Thread state (#125): the wrapper calls `PyThreadState_GetUnchecked()` once and passes it as `tp_ts` to the +impl, which passes it to `tp_enter_call`, `tp_depth_room`, `tp_traced` and to its direct compiled callees, +so a compiled call and its C-to-C callees do one TLS lookup of the thread state per call. Likewise the per-thread state (call depth, poll countdown, snapshot depth) is one `__thread` struct +`tp_tls`: the wrapper takes `tp_tls_get()` once and passes `tp_t` down. `Py_EnterRecursiveCall`, +`Py_LeaveRecursiveCall` and `Py_GetRecursionLimit` have no public tstate-taking variant and are kept. + +Thread state (#125): the wrapper calls `PyThreadState_GetUnchecked()` once and passes it as `tp_ts` to the +impl, which passes it to `tp_enter_call`, `tp_depth_room`, `tp_traced` and to its direct compiled callees, +so a compiled call and its C-to-C callees do no further thread-state lookup. Likewise the per-thread state (call depth, poll countdown, snapshot depth) is one `__thread` struct +`tp_tls`: the wrapper takes `tp_tls_get()` once and passes `tp_t` down. `Py_EnterRecursiveCall`, +`Py_LeaveRecursiveCall` and `Py_GetRecursionLimit` have no public tstate-taking variant and are kept. + `tp_enter_call` makes compiled recursion fail where interpreted recursion fails. It counts, per thread, the compiled frames in flight (they have no Python frame), adds the current Python frame depth (`PyThreadState_GetFrame` / `PyFrame_GetBack`, public API only; cached while the top frame is the same @@ -237,34 +249,33 @@ compiled module that sit between two frames of this one are not counted, and ano frames never are. ## Fast entry (bounded functions; issue #147) - int tp_depth_room(Py_ssize_t k); /* 1 iff tp_python_depth() + count + k <= Py_GetRecursionLimit() */ + int tp_depth_room(PyThreadState *ts, tp_tls *t, Py_ssize_t k); /* 1 iff tp_python_depth(ts) + count + k <= Py_GetRecursionLimit() */ void tp_trace_fast_hit(void); /* counts a fast entry; a no-op unless -DTP_TRACE_FAST */ - void tp_depth_py_begin(int fd); /* count += fd, around code a fast tree may run (redo, poll) */ - void tp_depth_py_end(int fd); /* count -= fd; drops the cached frame at count 0 */ - void tp_depth_done(void); /* after a fast tree: drop the cached top frame when count is 0 */ + void tp_depth_py_begin(tp_tls *t, int fd); /* count += fd, around code a fast tree may run (redo, poll) */ + void tp_depth_py_end(tp_tls *t, int fd); /* count -= fd; drops the cached frame at count 0 */ + void tp_depth_done(tp_tls *t); /* after a fast tree: drop the cached top frame when count is 0 */ A function is *bounded* when it is in no call cycle, every function it Calls is bounded, and its body has no node that can run Python code (CallObject, GetAttr, Truth, CompareObj, ObjToFloat, FieldGet, FieldSet, New, any OBJ-typed expression except Box of a scalar). A `redo` Call to a bounded callee is allowed. Its k is `1 + max(k(callee))` (1 with no callees). cgen emits, per bounded `f`, `static inline int tp_fimpl_f(...)` (the impl body without tp_enter_call / tp_leave_call and without the entry poll, Calls to the callees' `tp_fimpl_*`) and, at -the top of `tp_impl_f` after the Param.cls guards, `if (tp_depth_room(k)) { r = tp_fimpl_f(...); +the top of `tp_impl_f` after the Param.cls guards, `if (tp_depth_room(tp_ts, tp_t, k)) { r = tp_fimpl_f(...); tp_depth_done(); return r; }`. No room: the counted path runs and raises RecursionError exactly where it raised before. `tp_enter_call` compares `depth + count + 1`; the fast tree has at most k frames, so `depth + count + k` is the same comparison at its deepest frame. Py_EnterRecursiveCall is skipped on the fast path: the tree is bounded, so it adds at most k small C frames. A twin takes `int tp_fd` (the tree's active frames, itself included; the precheck passes 1, a Call passes `tp_fd + 1`). The two points where a fast tree can run Python code, a `redo` Call and the loop poll's `tp_poll_slow`, -are wrapped in `tp_depth_py_begin(tp_fd)` / `tp_depth_py_end(tp_fd)`, so everything run from there +are wrapped in `tp_depth_py_begin(tp_t, tp_fd)` / `tp_depth_py_end(tp_t, tp_fd)`, so everything run from there (interpreted frames, signal handlers, compiled calls through their precheck) sees the exact count. `-DTP_TRACE_FAST` (tests only) adds `__tp_trace_fast_hits__()` to the module: the number of entries that took the fast path. ## Eval breaker (top of every loop iteration and impl entry; issue #141) - int tp_poll(PyObject *breaker); /* 0 continue; -1 exception set (a signal handler raised) */ - int tp_poll_slow(PyObject *breaker); /* every TP_POLL_INTERVAL polls: call the breaker */ + int tp_poll(tp_tls *t, PyObject *breaker); /* 0 continue; -1 exception set (a signal handler raised) */ + int tp_poll_slow(tp_tls *t, PyObject *breaker); /* every TP_POLL_INTERVAL polls: call the breaker */ int tp_pc_tick(int *pc); /* loop countdown: 1 (and reset) when the slow path is due */ - Py_ssize_t tp_snapshot_depth; /* THREAD-LOCAL; ++ at entry, -- at tp_exit of a snapshot-holding impl */ - int tp_poll_countdown; /* THREAD-LOCAL */ + tp_tls *tp_tls_get(void); /* address of the one THREAD-LOCAL struct: depth, poll_countdown, snapshot_depth (++ at entry, -- at tp_exit of a snapshot-holding impl) */ `breaker` is the module state's `lambda: None`. Calling it runs CPython's own eval-breaker handling in its RESUME: signals, pending calls, stop-the-world requests of other threads (the free-threaded GC), async exceptions and scheduled collections, exactly as between two bytecodes. A function that holds an array copy-in @@ -272,7 +283,7 @@ GC), async exceptions and scheduled collections, exactly as between two bytecode nothing: the event stays pending and is handled after it returns. ## Tracing, profiling and monitoring (entry of every function; issue #142, SPEC N-14) - int tp_traced(PyObject *module, PyCodeObject *code); /* 1: run the interpreted original */ + int tp_traced(PyObject *module, PyThreadState *ts, PyCodeObject *code); /* 1: run the interpreted original */ PyObject *tp_state.orig_code[]; /* GENERATED, module state: STRONG ref to the original's code, per interned name */ `tp_traced` is true when the calling thread has a `sys.settrace` / `sys.setprofile` function (`PyThreadState.c_tracefunc` / `c_profilefunc`), when any `sys.monitoring` tool has a global event @@ -440,8 +451,8 @@ static or module-state field must be added here): | State | Where | Class | How | |---|---|---|---| | `tp_depth` (compiled frames in flight, cached frame) | tp_runtime.h static | thread-local | `TP_THREAD_LOCAL` | -| `tp_poll_countdown` | tp_runtime.h static | thread-local | `TP_THREAD_LOCAL` | -| `tp_snapshot_depth` | tp_runtime.h static | thread-local | `TP_THREAD_LOCAL`: one thread's array function never silences another thread's polls | +| `tp_tls_v.poll_countdown` | tp_runtime.h static | thread-local | `TP_THREAD_LOCAL` struct `tp_tls` | +| `tp_tls_v.snapshot_depth` | tp_runtime.h static | thread-local | `TP_THREAD_LOCAL`: one thread's array function never silences another thread's polls | | `tp_pc` (loop countdown) | local of each impl | per call | a C local | | `tp_globals_epoch` | tp_runtime.h static | atomic | relaxed loads, acq_rel add in the watcher callback (any thread); the epoch + dict lookup that remember a binding run in the module dict's critical section | | `cls_rt[k].tag`, `.bad_tag`, `.gepoch` | module state | atomic | relaxed `tp_atomic_*`; a stale tag is never reused (tags only grow), so a lost or reordered refresh costs one more refresh | diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h index 8ad285dfc..d68972b3f 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h @@ -2603,19 +2603,36 @@ typedef struct { Py_ssize_t pydepth; /* Python frames below and including `frame` */ } tp_depth_state; -static TP_THREAD_LOCAL tp_depth_state tp_depth = {0, NULL, 0}; +#ifndef TP_POLL_INTERVAL +#define TP_POLL_INTERVAL 4096 +#endif + +/* All per-thread state of the compiled code in ONE thread-local, so a compiled call looks its address + * up once (tp_tls_get, in the wrapper) and passes the pointer down (#125). */ +typedef struct { + tp_depth_state depth; /* compiled frames in flight (see above) */ + int poll_countdown; /* eval-breaker poll countdown (#141) */ + Py_ssize_t snapshot_depth; /* snapshot-holding functions in flight (#141) */ +} tp_tls; + +static TP_THREAD_LOCAL tp_tls tp_tls_v = {{0, NULL, 0}, TP_POLL_INTERVAL, 0}; + +static inline tp_tls *tp_tls_get(void) +{ + return &tp_tls_v; +} /* Python frame depth of the running thread: public API only. The walk is O(depth), so it is * cached by the identity of the top frame (kept alive while compiled frames are in flight). */ -static inline Py_ssize_t tp_python_depth(void) +static inline Py_ssize_t tp_python_depth(PyThreadState *ts, tp_tls *t) { - PyFrameObject *top = PyThreadState_GetFrame(PyThreadState_Get()); /* new ref or NULL */ + PyFrameObject *top = PyThreadState_GetFrame(ts); /* new ref or NULL */ if (top == NULL) { return 0; } - if (tp_depth.frame == (PyObject *)top) { + if (t->depth.frame == (PyObject *)top) { Py_DECREF(top); - return tp_depth.pydepth; + return t->depth.pydepth; } Py_ssize_t depth = 0; PyFrameObject *cur = top; @@ -2626,31 +2643,31 @@ static inline Py_ssize_t tp_python_depth(void) Py_DECREF(cur); cur = back; } - PyObject *old = tp_depth.frame; - tp_depth.frame = (PyObject *)top; /* takes the reference from GetFrame */ - tp_depth.pydepth = depth; + PyObject *old = t->depth.frame; + t->depth.frame = (PyObject *)top; /* takes the reference from GetFrame */ + t->depth.pydepth = depth; Py_XDECREF(old); return depth; } -static inline int tp_enter_call(void) +static inline int tp_enter_call(PyThreadState *ts, tp_tls *t) { - if (tp_python_depth() + tp_depth.count + 1 > Py_GetRecursionLimit()) { + if (tp_python_depth(ts, t) + t->depth.count + 1 > Py_GetRecursionLimit()) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded"); return -1; } if (Py_EnterRecursiveCall(" in compiled code")) { return -1; } - tp_depth.count++; + t->depth.count++; return 0; } -static inline void tp_leave_call(void) +static inline void tp_leave_call(tp_tls *t) { Py_LeaveRecursiveCall(); - if (--tp_depth.count == 0) { - Py_CLEAR(tp_depth.frame); + if (--t->depth.count == 0) { + Py_CLEAR(t->depth.frame); } } @@ -2672,32 +2689,32 @@ static inline void tp_leave_call(void) * tp_depth_py_begin(tp_fd) / tp_depth_py_end(tp_fd), `tp_fd` being the tree's active frames (itself * included), so the count is exactly what the counted path would have there: Python frames and * compiled calls started from it see the true depth. */ -static inline int tp_depth_room(Py_ssize_t k) +static inline int tp_depth_room(PyThreadState *ts, tp_tls *t, Py_ssize_t k) { - return tp_python_depth() + tp_depth.count + k <= Py_GetRecursionLimit(); + return tp_python_depth(ts, t) + t->depth.count + k <= Py_GetRecursionLimit(); } /* After a fast tree: tp_python_depth may have cached the top frame (a strong reference) and, with * no counted frame in flight, nothing else would drop it, as tp_leave_call does at count 0. */ -static inline void tp_depth_done(void) +static inline void tp_depth_done(tp_tls *t) { - if (tp_depth.count == 0) { - Py_CLEAR(tp_depth.frame); + if (t->depth.count == 0) { + Py_CLEAR(t->depth.frame); } } /* Around a point of a fast tree that can run Python code: the count includes the tree's `fd` active * frames. begin/end are balanced. end also drops the cached top frame when no compiled frame is * counted any more (the code in between may have cached another frame, as tp_leave_call would). */ -static inline void tp_depth_py_begin(int fd) +static inline void tp_depth_py_begin(tp_tls *t, int fd) { - tp_depth.count += fd; + t->depth.count += fd; } -static inline void tp_depth_py_end(int fd) +static inline void tp_depth_py_end(tp_tls *t, int fd) { - tp_depth.count -= fd; - tp_depth_done(); + t->depth.count -= fd; + tp_depth_done(t); } #ifdef TP_TRACE_FAST @@ -2735,8 +2752,6 @@ static inline void tp_trace_fast_hit(void) {} /* Per thread (#159): a shared counter would make threads poll for each other and, worse, let one * thread's snapshot-holding function suppress another thread's polls. */ -static TP_THREAD_LOCAL int tp_poll_countdown = TP_POLL_INTERVAL; -static TP_THREAD_LOCAL Py_ssize_t tp_snapshot_depth = 0; /* Kept out of line and marked cold: inlined into every loop body it bloated short inner loops * (fannkuch paid ~11% with it inline). */ @@ -2748,11 +2763,11 @@ static TP_THREAD_LOCAL Py_ssize_t tp_snapshot_depth = 0; #define TP_UNLIKELY(x) (x) #endif -static TP_COLD int tp_poll_slow(PyObject *breaker) +static TP_COLD int tp_poll_slow(tp_tls *t, PyObject *breaker) { PyObject *r; - tp_poll_countdown = TP_POLL_INTERVAL; - if (tp_snapshot_depth > 0 || breaker == NULL) { + t->poll_countdown = TP_POLL_INTERVAL; + if (t->snapshot_depth > 0 || breaker == NULL) { return 0; } r = PyObject_CallNoArgs(breaker); @@ -2775,12 +2790,12 @@ static inline int tp_pc_tick(int *pc) } /* 0 = continue, -1 = exception set (e.g. KeyboardInterrupt from a signal handler). */ -static inline int tp_poll(PyObject *breaker) +static inline int tp_poll(tp_tls *t, PyObject *breaker) { - if (!TP_UNLIKELY(--tp_poll_countdown <= 0)) { + if (!TP_UNLIKELY(--t->poll_countdown <= 0)) { return 0; } - return tp_poll_slow(breaker); + return tp_poll_slow(t, breaker); } /* Tracing, profiling and monitoring (issue #142, SPEC N-14). A compiled function has no Python @@ -2843,10 +2858,9 @@ static TP_COLD int tp_mon_scan(PyObject *module) return on; } -static inline int tp_traced(PyObject *module, PyCodeObject *code) +static inline int tp_traced(PyObject *module, PyThreadState *ts, PyCodeObject *code) { uint64_t v = tp_mon_ver; - PyThreadState *ts = PyThreadState_Get(); /* sys.settrace / sys.setprofile (and PyEval_SetTrace / SetProfile): per thread, as in CPython. * c_tracefunc / c_profilefunc are public members of PyThreadState (cpython/pystate.h; coverage's * C tracer and Cython read them), the only internal-looking read here. */ diff --git a/python-multiplatform-ksp/src/test/python/test_cgen.py b/python-multiplatform-ksp/src/test/python/test_cgen.py index d7afecef8..0b13dfd46 100644 --- a/python-multiplatform-ksp/src/test/python/test_cgen.py +++ b/python-multiplatform-ksp/src/test/python/test_cgen.py @@ -1256,7 +1256,7 @@ def test_gen_local_array_scan_calls_only_runtime_helpers(): def test_gen_local_arrays_never_go_through_the_write_back_exit(): text = gen((f_la_build,), name="tp_gen_la2") - impl = text[text.index("static int tp_impl_la_build(PyObject *tp_module, int64_t l_n"):] + impl = text[text.index("static int tp_impl_la_build(PyObject *tp_module, PyThreadState *tp_ts, tp_tls *tp_t, int64_t l_n"):] impl = impl[:impl.index("static PyObject *tp_wrap_la_build")] assert "_array_exit" not in impl and "_array_enter" not in impl assert impl.count("tp_i64_array_free(") >= 3 # a, b, c are freed at the exit block diff --git a/python-multiplatform-ksp/src/test/python/test_fast_entry.py b/python-multiplatform-ksp/src/test/python/test_fast_entry.py index 7adc7d2c9..179e99e02 100644 --- a/python-multiplatform-ksp/src/test/python/test_fast_entry.py +++ b/python-multiplatform-ksp/src/test/python/test_fast_entry.py @@ -136,7 +136,7 @@ def test_the_fast_impl_calls_only_fast_impls_and_never_counts(traced): fast = fn_text(c, "static inline int tp_fimpl_mid") assert "tp_fimpl_leaf(" in fast and "tp_impl_leaf(" not in fast assert "tp_enter_call" not in fast and "tp_leave_call" not in fast - assert "tp_poll(tp_st->breaker)" not in fast # no entry poll + assert "tp_poll(tp_t, tp_st->breaker)" not in fast # no entry poll loop = fn_text(c, "static inline int tp_fimpl_loopy") assert "tp_poll(" in loop or "tp_pc" in loop # loop polls stay @@ -144,11 +144,11 @@ def test_the_fast_impl_calls_only_fast_impls_and_never_counts(traced): def test_the_normal_impl_prechecks_and_still_counts(traced): c = traced[1] normal = fn_text(c, "static int tp_impl_mid") - assert "tp_depth_room(2)" in normal - assert normal.index("tp_depth_room(2)") < normal.index("tp_enter_call()") + assert "tp_depth_room(tp_ts, tp_t, 2)" in normal + assert normal.index("tp_depth_room(tp_ts, tp_t, 2)") < normal.index("tp_enter_call(tp_ts, tp_t)") assert "tp_impl_leaf(" in normal and "tp_fimpl_leaf(" not in normal top = fn_text(c, "static int tp_impl_top") - assert "tp_depth_room(3)" in top + assert "tp_depth_room(tp_ts, tp_t, 3)" in top assert "tp_depth_room" not in fn_text(c, "static int tp_impl_rec") @@ -354,7 +354,7 @@ def test_a_redo_inside_a_fast_tree_sees_the_exact_depth(traced, tmp_path): for x in ("3", "3000000"): assert r["ok"][x] > 100 and r["rec"][x] >= 1, (x, r) assert r["deopts"] >= r["deopt_ok"] > 100 # the redo really ran at every ok depth - assert "tp_depth_py_begin(tp_fd);" in fn_text(c, "static inline int tp_fimpl_fill") + assert "tp_depth_py_begin(tp_t, tp_fd);" in fn_text(c, "static inline int tp_fimpl_fill") def test_a_signal_handler_in_a_fast_tree_poll_sees_the_exact_depth(traced, tmp_path): @@ -367,7 +367,7 @@ def test_a_signal_handler_in_a_fast_tree_poll_sees_the_exact_depth(traced, tmp_p assert r["mismatch"] == [], r["mismatch"][:5] assert len({v[0] for v in r["values"]}) > 5 # the measured depth tracks the stack depth loop = fn_text(c, "static inline int tp_fimpl_loopy") - assert "tp_depth_py_begin(tp_fd);" in loop and "tp_depth_py_end(tp_fd);" in loop + assert "tp_depth_py_begin(tp_t, tp_fd);" in loop and "tp_depth_py_end(tp_t, tp_fd);" in loop def test_the_fast_path_is_taken_when_there_is_room(tmp_path): diff --git a/python-multiplatform-ksp/src/test/python/test_recursion.py b/python-multiplatform-ksp/src/test/python/test_recursion.py index db716afec..289e1de66 100644 --- a/python-multiplatform-ksp/src/test/python/test_recursion.py +++ b/python-multiplatform-ksp/src/test/python/test_recursion.py @@ -304,5 +304,29 @@ def test_every_compiled_function_enters_and_leaves(built): from typedpython import cgen result = built[0] c = cgen.generate(result.module, result.source) - assert c.count("tp_enter_call()") >= len(NAMES) - assert c.count("tp_leave_call()") >= len(NAMES) + assert c.count("tp_enter_call(tp_ts, tp_t)") >= len(NAMES) + assert c.count("tp_leave_call(tp_t)") >= len(NAMES) + + +def test_a_compiled_call_looks_up_the_thread_state_at_most_once(built): + """#125: the wrapper does the one lookup; impls and fast twins only receive tp_ts.""" + import re + from typedpython import cgen + result = built[0] + c = cgen.generate(result.module, result.source) + lookups = re.compile(r"PyThreadState_Get(?:Unchecked)?\s*\(|tp_tls_get\s*\(|\btp_tls_v\b") + tstate = re.compile(r"PyThreadState_Get(?:Unchecked)?\s*\(") + tlsre = re.compile(r"tp_tls_get\s*\(|\btp_tls_v\b") + # strip the comment that mentions the lookup, then count call sites per generated function + bodies = re.split(r"\n(?=static (?:inline )?(?:int|PyObject \*) ?tp_(?:wrap|impl|fimpl)_)", c) + seen = 0 + for b in bodies: + head = b.split("(", 1)[0] + if "tp_wrap_" in head: + seen += 1 + assert len(tstate.findall(b)) <= 1 and len(tlsre.findall(b)) <= 1, head + elif "tp_impl_" in head or "tp_fimpl_" in head: + seen += 1 + assert not lookups.findall(b), head + assert seen >= len(NAMES) + assert "tp_enter_call(tp_ts, tp_t)" in c