diff --git a/benchmarks/typedpython/results/2026-10-17-wordfreq-cost-rule-ab.json b/benchmarks/typedpython/results/2026-10-17-wordfreq-cost-rule-ab.json new file mode 100644 index 00000000..32019a96 --- /dev/null +++ b/benchmarks/typedpython/results/2026-10-17-wordfreq-cost-rule-ab.json @@ -0,0 +1,155 @@ +{ + "wordfreq": { + "size": "3000000", + "pairs": [ + { + "before_s": 2.534866082947701, + "after_s": 2.4698741249740124, + "cpython_s": 2.246576082892716, + "after_over_before": 0.9743607923073743, + "speedup_before": 0.8862701260653014, + "speedup_after": 0.9095913270140251, + "load": [ + 2.22265625, + 2.5498046875, + 3.80224609375 + ] + }, + { + "after_s": 2.054086625110358, + "before_s": 2.0703248330391943, + "cpython_s": 2.226471666712314, + "after_over_before": 0.9921566859124232, + "speedup_before": 1.0754214175385701, + "speedup_after": 1.0839229658061253, + "load": [ + 2.1943359375, + 2.53173828125, + 3.78076171875 + ] + }, + { + "before_s": 2.0203522499650717, + "after_s": 2.0748487496748567, + "cpython_s": 2.2474577920511365, + "after_over_before": 1.0269737614867542, + "speedup_before": 1.112408884188384, + "speedup_after": 1.083191144609133, + "load": [ + 2.0986328125, + 2.505859375, + 3.76416015625 + ] + }, + { + "after_s": 2.057975041680038, + "before_s": 2.072377833072096, + "cpython_s": 2.225645957980305, + "after_over_before": 0.9930501131781035, + "speedup_before": 1.0739576164453584, + "speedup_after": 1.081473736514991, + "load": [ + 2.17041015625, + 2.513671875, + 3.75927734375 + ] + }, + { + "before_s": 2.0653403750620782, + "after_s": 2.0860109999775887, + "cpython_s": 2.24187570810318, + "after_over_before": 1.0100083381727765, + "speedup_before": 1.0854751764758366, + "speedup_after": 1.0747190250325938, + "load": [ + 2.39697265625, + 2.5546875, + 3.76611328125 + ] + }, + { + "after_s": 2.047571041621268, + "before_s": 2.0459034591913223, + "cpython_s": 2.240035249851644, + "after_over_before": 1.0008150836357668, + "speedup_before": 1.0948880504542748, + "speedup_after": 1.093996351930228, + "load": [ + 2.3291015625, + 2.53564453125, + 3.74462890625 + ] + }, + { + "before_s": 2.064202000387013, + "after_s": 2.066363917198032, + "cpython_s": 2.2770254579372704, + "after_over_before": 1.0010473378141354, + "speedup_before": 1.1031020498528514, + "speedup_after": 1.1019479381080624, + "load": [ + 2.22265625, + 2.509765625, + 3.728515625 + ] + }, + { + "after_s": 2.047510417178273, + "before_s": 2.029783667065203, + "cpython_s": 2.238690916914493, + "after_over_before": 1.0087333199102446, + "speedup_before": 1.1029209433689757, + "speedup_after": 1.093372174389076, + "load": [ + 2.12451171875, + 2.484375, + 3.71240234375 + ] + }, + { + "before_s": 2.051607625093311, + "after_s": 2.0582683752290905, + "cpython_s": 2.2774744578637183, + "after_over_before": 1.0032466003997604, + "speedup_before": 1.1100926073815574, + "speedup_after": 1.1065002432494886, + "load": [ + 1.951171875, + 2.43505859375, + 3.68017578125 + ] + }, + { + "after_s": 2.0976750422269106, + "before_s": 2.0433855829760432, + "cpython_s": 2.259623875375837, + "after_over_before": 1.0265683871429683, + "speedup_before": 1.1058235382501125, + "speedup_after": 1.0772039662430268, + "load": [ + 2.03515625, + 2.4443359375, + 3.67578125 + ] + }, + { + "before_s": 2.05512237502262, + "after_s": 2.072784666903317, + "cpython_s": 2.2494797920808196, + "after_over_before": 1.0085942774480778, + "speedup_before": 1.0945721867565479, + "speedup_after": 1.08524528765522, + "load": [ + 1.9521484375, + 2.419921875, + 3.65966796875 + ] + } + ], + "median_after_over_before": 1.0032466003997604, + "min_after_over_before": 0.9743607923073743, + "max_after_over_before": 1.0269737614867542, + "median_speedup_before": 1.0948880504542748, + "median_speedup_after": 1.0839229658061253 + } +} diff --git a/benchmarks/typedpython/results/2026-10-17-wordfreq-object-mode.md b/benchmarks/typedpython/results/2026-10-17-wordfreq-object-mode.md new file mode 100644 index 00000000..519d6cfb --- /dev/null +++ b/benchmarks/typedpython/results/2026-10-17-wordfreq-object-mode.md @@ -0,0 +1,20 @@ +# wordfreq and object-only compiled code (#150, 2026-10-17) + +Quiet windows, interleaved against develop, 11 pairs, CPython 3.15.0rc1t. + +| build | after / before (median, range) | vs CPython | +|---|---|---| +| #150 without the cost rule (`word_of` and `main` compiled) | 1.335 (1.314 to 1.767) | 1.1x to **0.8x** | +| #150 with the cost rule (only `next_state` compiled, as on develop) | 1.003 (0.974 to 1.027) | 1.1x to 1.1x | + +Split, measured in one process: `word_of` compiled 248.6 ms against 95.7 ms interpreted (2.6x slower); +`main` 332 ms against 345 ms. + +Cause (read in the generated C): in an impure function an `int` is a CPython object, so each loop +iteration boxes constants, calls generic `PyNumber_*` / `PyObject_RichCompare`, and looks up the +global `chr` through the module and builtins dicts. CPython 3.15's specialising interpreter +(`BINARY_OP_ADD_INT`, `LOAD_GLOBAL_BUILTIN` caches, the small-int cache) runs that faster. + +Consequence: the cost rule (`typedpython/cost.py`) keeps an impure function interpreted when it has +loops and none of them does native work. The real fix, native ints in impure functions with in-place +promotion on overflow, pre-boxed constants and cached builtins, is #232. diff --git a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py index 08bf18e0..0a9bfe0d 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py @@ -272,6 +272,12 @@ def _exprs_of_stmt(s: ir.Stmt): elif isinstance(s, ir.FieldSet): yield s.value yield s.obj + elif isinstance(s, ir.StoreItem): + yield s.value + yield s.obj + yield s.index + elif isinstance(s, ir.ForObj): + yield s.iter elif isinstance(s, ir.ExprStmt): yield s.value elif isinstance(s, ir.If): @@ -288,7 +294,7 @@ def _exprs_of_stmt(s: ir.Stmt): def _children(s: ir.Stmt): if isinstance(s, ir.If): return s.then + s.orelse - if isinstance(s, (ir.While, ir.ForRange)): + if isinstance(s, (ir.While, ir.ForRange, ir.ForObj)): return s.body return () @@ -376,7 +382,8 @@ def _watch_event_stmt() -> str: def _is_effect(e: ir.Expr) -> bool: - if isinstance(e, (ir.GetAttr, ir.CallObject, ir.Truth, ir.CompareObj, ir.ObjToFloat)): + if isinstance(e, (ir.GetAttr, ir.CallObject, ir.Truth, ir.CompareObj, ir.ObjToFloat, + ir.GetItem, ir.Contains, ir.Format)): return True return isinstance(e, ir.BinOp) and e.type is Type.OBJ @@ -387,7 +394,7 @@ def _is_effect(e: ir.Expr) -> bool: # Nodes that can run Python code (and so create Python frames the compiled frame count must see): # their slow paths call getattr, the class, __bool__, __float__, a Python callable, a descriptor. _RUNS_PYTHON = (ir.CallObject, ir.GetAttr, ir.Truth, ir.CompareObj, ir.ObjToFloat, ir.FieldGet, - ir.FieldSet, ir.New) + ir.FieldSet, ir.New, ir.GetItem, ir.Contains, ir.Format, ir.StoreItem, ir.ForObj) def _runs_no_python(f: ir.Function) -> bool: @@ -403,7 +410,7 @@ def _runs_no_python(f: ir.Function) -> bool: if any(t is Type.OBJ for t in f.locals.values()): return False for s in _walk_stmts(f.body): - if isinstance(s, ir.FieldSet): + if isinstance(s, (ir.FieldSet, ir.StoreItem, ir.ForObj)): return False for e in _all_exprs(f): if isinstance(e, _RUNS_PYTHON): @@ -664,7 +671,7 @@ def _check_function(f: ir.Function, functions: dict[str, ir.Function]) -> None: # a store into a local array is not an effect (ir.py "Local arrays"); into a parameter it is stores = [s for s in _walk_stmts(f.body) if (isinstance(s, ir.StoreIndex) and f.locals.get(s.array) not in ir.ARRAYS) - or isinstance(s, ir.FieldSet)] + or isinstance(s, (ir.FieldSet, ir.StoreItem, ir.ForObj))] if f.pure and (effects or stores): what = type(effects[0]).__name__ if effects else type(stores[0]).__name__ raise CGenError(f"{f.name}: marked pure but contains an effect ({what})") @@ -725,6 +732,8 @@ def _is_closed(f: ir.Function, functions: dict[str, ir.Function], seen=None) -> if f.name in seen: return True seen.add(f.name) + if any(isinstance(s, (ir.StoreItem, ir.ForObj)) for s in _walk_stmts(f.body)): + return False for e in _all_exprs(f): if _is_effect(e): return False @@ -893,6 +902,9 @@ def ev_Const(self, e: ir.Const) -> str: return self._own_copy("Py_None") if isinstance(e.value, bool): return self._own_copy("Py_True" if e.value else "Py_False") + if type(e.value) is str and "\0" not in e.value: + # a str constant: the interned object the module state holds (names table) + return self._own_copy(f"tp_st->names[{self.mg.name_index(e.value)}]") raise CGenError(f"{self.f.name}: OBJ constant {e.value!r} (use Box of a scalar Const)") raise CGenError(f"{self.f.name}: constant of type {e.type}") @@ -934,7 +946,8 @@ def ev_BinOp(self, e: ir.BinOp) -> str: return t if lt is Type.OBJ: t = self.tmp("PyObject *") - self.emit(f"{t} = tp_binop_obj({a}, {b}, {_OBJ_OP[e.op]});") + op = _OBJ_OP[e.op] if not e.inplace else f"TP_BINOP_I{e.op.name}" + self.emit(f"{t} = tp_binop_obj({a}, {b}, {op});") self._own_release(a, b) self.emit(f"if ({t} == NULL) {{ tp_rc = -1; goto tp_exit; }}") return t @@ -1411,6 +1424,54 @@ def ev_Tuple(self, e: ir.Tuple) -> str: self.emit(f"if ({t} == NULL) {{ tp_rc = -1; goto tp_exit; }}") return t + def ev_GetItem(self, e: ir.GetItem) -> str: + o = self._obj_operand(e.obj, "GetItem") # the container, then the index + k = self._obj_operand(e.index, "GetItem index") + t = self.tmp("PyObject *") + self.emit(f"{t} = tp_getitem({o}, {k});") + self._own_release(o, k) + self.emit(f"if ({t} == NULL) {{ tp_rc = -1; goto tp_exit; }}") + return t + + def ev_Contains(self, e: ir.Contains) -> str: + a = self._obj_operand(e.left, "Contains") # CONTAINS_OP: left, then right + b = self._obj_operand(e.right, "Contains") + t = self.tmp("int") + self.emit(f"{t} = tp_contains({b}, {a});") + self._own_release(a, b) + self.emit(f"if ({t} < 0) {{ tp_rc = -1; goto tp_exit; }}") + if e.negate: + self.emit(f"{t} = !{t};") + return t + + def ev_NewList(self, e: ir.NewList) -> str: + return self._own_new("tp_new_list()") + + def ev_NewDict(self, e: ir.NewDict) -> str: + return self._own_new("tp_new_dict()") + + def ev_Format(self, e: ir.Format) -> str: + if e.conversion not in (-1, 97, 114, 115) or (e.spec is not None and "\0" in e.spec): + raise CGenError(f"{self.f.name}: Format conversion {e.conversion!r} / spec {e.spec!r}") + o = self._obj_operand(e.operand, "Format") + spec = "NULL" if e.spec is None else f"tp_st->names[{self.mg.name_index(e.spec)}]" + t = self.tmp("PyObject *") + self.emit(f"{t} = tp_format({o}, {spec}, {e.conversion});") + self._own_release(o) + self.emit(f"if ({t} == NULL) {{ tp_rc = -1; goto tp_exit; }}") + return t + + def ev_BuildString(self, e: ir.BuildString) -> str: + if len(e.elements) < 2: + raise CGenError(f"{self.f.name}: BuildString of {len(e.elements)} elements") + items = [self._obj_operand(x, "BuildString element") for x in e.elements] + t = self.tmp("PyObject *") + self.emit(f"{{ PyObject *tp_av[{len(items)}] = {{{', '.join(items)}}};") + self.emit(f" {t} = tp_build_string(tp_av, {len(items)}); }}") + self._own_release(*items) + self.emit(f"if ({t} == NULL) {{ tp_rc = -1; goto tp_exit; }}") + return t + def _assign_array(self, s: ir.Assign, at: Type) -> None: """`local = NewArray | CopyArray`: build into a zeroed temporary, then (only on success) free the old array and move the temporary in.""" @@ -1489,6 +1550,48 @@ def st_StoreIndex(self, s: ir.StoreIndex) -> None: if not local: # a local array has no list to write back to self.emit(f"{ptr}->dirty[{slot}] = 1;") + def st_StoreItem(self, s: ir.StoreItem) -> None: + v = self._obj_operand(s.value, "StoreItem value") # STORE_SUBSCR: value, container, index + o = self._obj_operand(s.obj, "StoreItem") + k = self._obj_operand(s.index, "StoreItem index") + self.emit(f"tp_s = tp_setitem({o}, {k}, {v});") + self._own_release(o, k, v) + self.emit("if (tp_s < 0) { tp_rc = -1; goto tp_exit; }") + + def st_ForObj(self, s: ir.ForObj) -> None: + if not s.targets or len(set(s.targets)) != len(s.targets) or (not s.unpack and len(s.targets) != 1): + raise CGenError(f"{self.f.name}: ForObj targets {s.targets!r} (unpack={s.unpack})") + for name in s.targets: + if self.types.get(name) is not Type.OBJ: + raise CGenError(f"{self.f.name}: ForObj target {name} must be an OBJ local") + o = self._obj_operand(s.iter, "ForObj iterable") + it = self.tmp("PyObject *") # the iterator: owned by this statement across iterations + self.emit(f"{it} = tp_get_iter({o});") # GET_ITER + self._own_release(o) + self.emit(f"if ({it} == NULL) {{ tp_rc = -1; goto tp_exit; }}") + item = self.tmp("PyObject *") + self.emit("for (;;) {") + self.depth += 1 + self.poll() + self.emit(f"{item} = tp_iter_next({it});") # FOR_ITER (StopIteration ends the loop) + self.emit(f"if ({item} == NULL) {{ if (PyErr_Occurred()) {{ tp_rc = -1; goto tp_exit; }} break; }}") + if not s.unpack: + self._own_move(_ident("l", s.targets[0]), item) + else: + n = len(s.targets) + self.emit(f"{{ PyObject *tp_u[{n}] = {{0}};") + self.emit(f" tp_s = tp_unpack({item}, {n}, tp_u);") # UNPACK_SEQUENCE; nothing owned on error + self._own_release(item) + self.emit(" if (tp_s < 0) { tp_rc = -1; goto tp_exit; }") + for i, name in enumerate(s.targets): + local = _ident("l", name) + self.emit(f" {{ PyObject *tp_old = {local}; {local} = tp_u[{i}]; tp_release(&tp_old); }}") + self.emit("}") + self.stmts(s.body) + self.depth -= 1 + self.emit("}") + self._own_release(it) # after the loop and after a `break`; a return/error exits via tp_exit + def st_FieldSet(self, s: ir.FieldSet) -> None: slot = self.mg.slot(self.f.name, s.cls, s.field) v = self._obj_operand(s.value, "FieldSet value") # CPython: value, then the object diff --git a/python-multiplatform-ksp/src/main/python/typedpython/cost.py b/python-multiplatform-ksp/src/main/python/typedpython/cost.py new file mode 100644 index 00000000..bd95614a --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/cost.py @@ -0,0 +1,120 @@ +"""Cost policy: compile a function only if the C is expected to beat the interpreter (#150). + +This is a policy, not a proof. `verify` proves that the C for a function is safe and keeps CPython's +results; it says nothing about speed, and a function can pass every proof and still be slower +compiled. The case that made this necessary: in an *impure* function a Python `int` is a CPython +object (the front end cannot prove i64 arithmetic free of overflow where a deopt could not be +redone), so every `+`, `//`, `%` and comparison of such code is a generic `PyNumber_*` / +`PyObject_RichCompare` call on a boxed int. CPython 3.15's specialising interpreter runs exactly that +faster than the generic C calls (wordfreq `word_of`: 248.6 ms compiled against 95.7 ms interpreted). +Compiled code wins only when it does native work, so the rule asks for it. + +The rule, applied after verification (so it only ever removes verified functions, never changes +the IR): a function is kept when it is **pure**, or when some loop (`While`, `ForRange`; a `ForObj` +is a loop over objects and does not count by itself, but loops nested inside it do) has a body, nested +statements included, containing a native-typed operation: + + * a `BinOp`, `Compare`, `UnaryOp` or `MathCall` on I64/F64/BOOL operands; + * an `Index` or `StoreIndex` on an array or buffer; + * a `Call` of a compiled function with at least one argument, all of them native. + +A pure function qualifies by construction: it has no object operation at all (no `GetAttr`, +`CallObject`, OBJ `BinOp` ...), so everything it does is native, loop or not (`eval_a`, a pure +straight-line formula, must stay compiled). The rule therefore changes no pure function. + +A rejected function stays interpreted with the diagnostic below. Because an impure function can be +the target of a direct C `Call`, anything that calls a rejected function is rejected too (the same +propagation `verify` uses for an unverified callee). + +Straight-line impure functions (no loop at all) are kept: they have no loop to lose to the +interpreter, and that keeps every function that compiled before #150 compiled (fixed-layout setters, +Kotlin calls). The rule targets loops over objects only. + +The real fix is native ints in impure functions (a separate issue); when it lands, this rule is the +thing to revisit. +""" +from __future__ import annotations + +import dataclasses + +from typedpython import ir +from typedpython.ir import Type +from typedpython.verify import Diagnostic + +RULE = "policy/cost" +REASON = ("only object operations in its loops: compiled C would be slower than the interpreter " + "(#150)") + +_NATIVE = (Type.I64, Type.F64, Type.BOOL) + + +def _walk(x): + if isinstance(x, (tuple, list)): + for y in x: + yield from _walk(y) + elif dataclasses.is_dataclass(x) and not isinstance(x, type): + yield x + for f in dataclasses.fields(x): + yield from _walk(getattr(x, f.name)) + + +def _native_op(n) -> bool: + if isinstance(n, ir.BinOp): + return n.left.type in _NATIVE + if isinstance(n, ir.UnaryOp): + return n.operand.type in _NATIVE + if isinstance(n, ir.Compare): + return n.left.type in _NATIVE and n.right.type in _NATIVE + if isinstance(n, ir.MathCall): + return True + if isinstance(n, (ir.Index, ir.StoreIndex)): + return True + if isinstance(n, ir.Call): + return bool(n.args) and all(a.type in _NATIVE for a in n.args) + return False + + +def _loops(body): + for n in _walk(body): + if isinstance(n, (ir.While, ir.ForRange)): + yield n + + +def does_native_work(fn: ir.Function) -> bool: + if fn.pure: + return True + if not any(isinstance(n, (ir.While, ir.ForRange, ir.ForObj)) for n in _walk(fn.body)): + # straight-line code (a field setter, a Kotlin call): no loop to lose to the interpreter, and + # what compiled before #150 keeps compiling + return True + return any(_native_op(n) for loop in _loops(fn.body) for n in _walk(loop.body)) + + +def apply(module: ir.Module, diagnostics: list[Diagnostic]) -> tuple[ir.Module, list[Diagnostic]]: + """`module` without the functions the rule rejects (they move to `skipped`), and the diagnostics + extended by one per rejected function.""" + reasons: dict[str, str] = {} + for fn in module.functions: + if not does_native_work(fn): + reasons[fn.name] = f"{RULE}: {REASON}" + changed = True + while changed: # a direct Call needs its callee compiled + changed = False + for fn in module.functions: + if fn.name in reasons: + continue + bad = sorted({n.function for n in _walk(fn.body) + if isinstance(n, ir.Call) and n.function in reasons}) + if bad: + reasons[fn.name] = (f"{RULE}: calls '{bad[0]}', which is not compiled " + f"({reasons[bad[0]]})") + changed = True + if not reasons: + return module, diagnostics + diags = list(diagnostics) + lines = {f.name: f.source_line for f in module.functions} + for name, text in reasons.items(): + diags.append(Diagnostic(name, RULE, text.split(": ", 1)[1], lines[name])) + kept = tuple(f for f in module.functions if f.name not in reasons) + skipped = {**module.skipped, **{n: t for n, t in reasons.items()}} + return dataclasses.replace(module, functions=kept, skipped=skipped), diags diff --git a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py index 5e42ddbd..fcffa094 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py @@ -42,6 +42,17 @@ `CallObject(Global(C))`. `trivial_init` requires the i-th `__init__` parameter to be stored into the i-th slot (ir.ClassDecl: "`__init__(self, *fields-in-order)`"); other orders stay an object call. +Objects, str and dict code (#150). A function may also use, as object operations (each is the C API +call the interpreter's bytecode makes, ir.py says which): `str` constants, `{}` and `[]`, `o[i]` +(`GetItem`) and `o[i] = v` (`StoreItem`) on anything that is not a native array, `x in c` / +`x not in c` (`Contains`), f-strings (`Format` / `BuildString`, a constant format spec only), +`for t in obj` and `for a, b in obj` (`ForObj`), and augmented assignment on objects in its in-place +form. `for v in range(n)` whose bound is a CPython int object (an impure function's ints are) is a +`ForObj` over `range(n)` called as an object, and then `v` is an object too (`_ObjRange`). All of +them make the function impure and not closed, so it can hold no array and no entry global. What stays +interpreted: non-empty list/dict displays, slices, `del`, tuple-assignment, comprehensions, +`x[i] += v` on an object, a format spec with a nested field. + Entry globals (ir.Function.entry_globals). A module-level name assigned once at top level, of type float/int/bool (annotated, or literals and arithmetic of such names), never rebound or deleted in the module, and read in a *closed* function (one that runs no user code, decided after lowering) @@ -653,7 +664,12 @@ def _lower_function( try: return _lower_in_mode(sig, env, info, inferred, impure=True) except Skip as e: - return (pure_error or e).reason + # The pure attempt's reason is the natural one, unless it stopped after an impure construct: + # then the function was never going to be pure and the pure mode's types (a native int where + # the impure mode has an object) make its complaint beside the point. + if pure_error is not None and not getattr(pure_error, "impure_seen", False): + return pure_error.reason + return e.reason def _lower_in_mode(sig, env, info, inferred, impure: bool) -> ir.Function: @@ -665,15 +681,39 @@ def _lower_in_mode(sig, env, info, inferred, impure: bool) -> ir.Function: return fn +class _ObjRange(Exception): + """A `for v in range(...)` whose bound is a CPython int object: `v` cannot be an I64 range + index. Raised once per variable; the function is lowered again with `v` an object (a `ForObj`).""" + + def __init__(self, var: str) -> None: + super().__init__(var) + self.var = var + + def _lower_with_bounds(sig, env, info, inferred, impure: bool, candidates: dict[str, Type]) -> ir.Function: + obj_ranges: frozenset[str] = frozenset() + while True: + try: + return _lower_rounds(sig, env, info, inferred, impure, candidates, obj_ranges) + except _ObjRange as r: + assert r.var not in obj_ranges, "an object range variable was asked for twice" + obj_ranges |= {r.var} + + +def _lower_rounds(sig, env, info, inferred, impure: bool, candidates: dict[str, Type], + obj_ranges: frozenset[str]) -> ir.Function: # Range-target intervals: least fixpoint by iteration, widened to TOP if it does not settle. # A local that keeps growing (a counter) is widened to the i64 limit from round 3 on, as the # verifier's own fixpoint does (verify._prove). bounds: dict[str, Interval] = {} for rnd in range(16): - lowerer = _Lowerer(sig, env, info, inferred, impure, bounds, candidates) - fn = lowerer.run() + lowerer = _Lowerer(sig, env, info, inferred, impure, bounds, candidates, obj_ranges) + try: + fn = lowerer.run() + except Skip as e: + e.impure_seen = lowerer.impure # type: ignore[attr-defined] + raise joined = {**bounds} for var, iv in lowerer.found_bounds.items(): new = _join(joined.get(var), iv) @@ -686,7 +726,7 @@ def _lower_with_bounds(sig, env, info, inferred, impure: bool, return fn bounds = joined widened = {var: TOP for var in bounds} - return _Lowerer(sig, env, info, inferred, impure, widened, candidates).run() + return _Lowerer(sig, env, info, inferred, impure, widened, candidates, obj_ranges).run() def _join(a: Interval, b: Interval) -> Interval: @@ -706,8 +746,10 @@ class _Binding: class _Lowerer: def __init__(self, sig: _Signature, env: dict[str, ir.Function], info: _ModuleInfo, inferred: dict[str, list[tuple[int, str]]], impure: bool, - bounds: dict[str, Interval], candidates: dict[str, Type]) -> None: + bounds: dict[str, Interval], candidates: dict[str, Type], + obj_ranges: frozenset[str] = frozenset()) -> None: self.sig = sig + self.obj_ranges = obj_ranges self.candidates = candidates self.entry_used: dict[str, Type] = {} self.fn = sig.node @@ -729,8 +771,11 @@ def __init__(self, sig: _Signature, env: dict[str, ir.Function], info: _ModuleIn self.range_vars = { n for n, bs in self.bindings.items() if n not in self.params and any(b.how == "range" for b in bs) - and all(b.how in ("range", "annotation") for b in bs) + and all(b.how in ("range", "annotation") for b in bs) and n not in self.obj_ranges } + # Locals bound by a loop over an object (`for t in obj`, an object range): always objects. + self.iter_vars = {n for n, bs in self.bindings.items() if any(b.how == "for" for b in bs)} + self.iter_vars |= self.obj_ranges # Locals only ever assigned by plain `=` / `+=` statements (never a loop target): their # interval is the join of what is assigned to them (found_bounds, widened across rounds). self.acc_vars = { @@ -892,6 +937,9 @@ def local_type(self, name: str, line: int) -> Type: if name not in self.annotations: kinds += self.inferred.get(name, []) distinct = sorted({k for _, k in kinds}) + if not distinct and name in self.iter_vars: + self.types[name] = Type.OBJ # what a loop over an object binds: an object + return Type.OBJ if not distinct: raise Skip(self.bindings[name][0].line, f"no type is known for local `{name}`") if len(distinct) > 1: @@ -906,6 +954,9 @@ def local_type(self, name: str, line: int) -> Type: t = Type.OBJ if self.impure_mode and name not in self.range_vars else Type.I64 else: t = SCALAR_KIND.get(kind, Type.OBJ) + if name in self.iter_vars and t != Type.OBJ: + raise Skip(self.bindings[name][0].line, f"loop variable `{name}` is bound by a loop over " + f"an object but is a native {TYPE_NAME[t]}") self.types[name] = t return t @@ -1023,6 +1074,16 @@ def assign(self, target: ast.expr, value_node: ast.expr, node: ast.stmt) -> list value = self.coerce(value, t, value_node, f"local `{name}`") self.note_assign(name, value) return self.take() + [ir.Assign(name, value)] + if isinstance(target, ast.Subscript) and not ( + isinstance(target.value, ast.Name) and target.value.id in self.arrays): + if isinstance(target.slice, ast.Slice): + raise Skip(line, f"slice assignment `{_src(target)} = ...` is not supported") + # STORE_SUBSCR: the value, then the container, then the index. + value = self.boxed(self.value(value_node), value_node) + obj = self.boxed(self.sub(target.value), target.value) + key = self.boxed(self.sub(target.slice), target.slice) + self.impure = True + return self.take() + [ir.StoreItem(obj, key, value)] if isinstance(target, ast.Subscript): array, elem = self.array_of(target) # CPython evaluates the value first, then the index. @@ -1210,7 +1271,7 @@ def aug_assign(self, node: ast.AugAssign) -> list[ir.Stmt]: raise _unsupported(node) t = self.local_type(name, line) current = ir.Local(t, name) - result = self.arith(op, current, self.sub(node.value), node) + result = self.arith(op, current, self.sub(node.value), node, inplace=True) result = self.coerce(result, t, node, f"local `{name}`") self.note_assign(name, result) return self.take() + [ir.Assign(name, result)] @@ -1229,19 +1290,56 @@ def aug_assign(self, node: ast.AugAssign) -> list[ir.Stmt]: raise Skip(line, f"assignment to attribute `{_src(target)}` is not supported") raise _unsupported(target) + def for_object(self, node: ast.For, call: ast.Call | None = None) -> list[ir.Stmt]: + """`for t in obj` / `for a, b in obj` (ir.ForObj); `call` is an object-bounded `range(...)`.""" + line = node.lineno + target = node.target + if isinstance(target, ast.Name): + names, unpack = [target.id], False + elif isinstance(target, (ast.Tuple, ast.List)) and target.elts \ + and all(isinstance(e, ast.Name) for e in target.elts): + names, unpack = [e.id for e in target.elts], True # type: ignore[attr-defined] + else: + raise Skip(line, f"loop target `{_src(target)}` is not supported (names only)") + if len(set(names)) != len(names): + raise Skip(line, f"loop target `{_src(target)}` binds a name twice") + for n in names: + if n in self.arrays: + raise Skip(line, f"loop variable `{n}` is an array") + if self.local_type(n, line) != Type.OBJ: + raise Skip(line, f"loop variable `{n}` is a native {TYPE_NAME[self.local_type(n, line)]}" + ", but a loop over an object binds objects") + if call is not None: # range(...) with an object bound: called as the object it is + callee = ir.Global(Type.OBJ, "range") + self.hoist_ok = False + it: ir.Expr = ir.CallObject(Type.OBJ, callee, tuple( + self.boxed(self.value(a), a) for a in call.args)) + else: + it = self.boxed(self.value(node.iter), node.iter) + self.impure = True + pre = self.take() + killed = _stored_names(node.body) | set(names) + head = {k: v for k, v in self.refined.items() if k not in killed} + self.refined = head + body = self.block(node.body) + self.refined = dict(head) + return pre + [ir.ForObj(tuple(names), it, body, unpack)] + def for_range(self, node: ast.For) -> list[ir.Stmt]: line = node.lineno if node.orelse: raise Skip(line, "`for ... else` is not supported") - if not isinstance(node.target, ast.Name): - raise Skip(line, f"loop target `{_src(node.target)}` is not supported (only a name)") if isinstance(node.iter, ast.Name) and node.iter.id in self.arrays: raise self.array_misuse(node.iter.id, node.iter) if not (_is_range_call(node.iter) and self.builtin("range")): - raise Skip(line, f"`for` over `{_src(node.iter)}` is not supported (only range())") + return self.for_object(node) call = node.iter assert isinstance(call, ast.Call) + if not isinstance(node.target, ast.Name): + raise Skip(line, f"loop target `{_src(node.target)}` is not supported (only a name)") var = node.target.id + if var in self.obj_ranges: + return self.for_object(node, call) if var in self.params: raise Skip(line, f"loop variable `{var}` is a parameter") for inner in ast.walk(ast.Module(body=node.body, type_ignores=[])): @@ -1262,10 +1360,10 @@ def for_range(self, node: ast.For) -> list[ir.Stmt]: step = 1 if len(args) == 1: start: ir.Expr = ir.Const(Type.I64, 0) - stop = self.range_bound(args[0]) + stop = self.range_bound(args[0], var) else: - start = self.range_bound(args[0]) - stop = self.range_bound(args[1]) + start = self.range_bound(args[0], var) + stop = self.range_bound(args[1], var) pre = self.take() s, e = self.interval(start), self.interval(stop) @@ -1283,8 +1381,10 @@ def for_range(self, node: ast.For) -> list[ir.Stmt]: self.refined = dict(head) return pre + [ir.ForRange(var, start, stop, ir.Const(Type.I64, step), body)] - def range_bound(self, node: ast.expr) -> ir.Expr: + def range_bound(self, node: ast.expr, var: str | None = None) -> ir.Expr: e = self.value(node) + if e.type == Type.OBJ and self.impure_mode and var is not None: + raise _ObjRange(var) # a CPython int object: the loop is a ForObj over range(...) if e.type != Type.I64: why = (" (it is a CPython int here: its arithmetic is not proved free of i64 overflow " "in an impure function)" if e.type == Type.OBJ and self.impure_mode else "") @@ -1378,6 +1478,8 @@ def expr(self, node: ast.expr, cond: int = 0) -> ir.Expr: return ir.Const(Type.F64, v) if v is None: return ir.Const(Type.OBJ, None) + if isinstance(v, str): + return self.str_const(v, node) raise Skip(line, f"{type(v).__name__} constant `{_src(node)}` is not supported") if isinstance(node, ast.Name): return self.name(node) @@ -1411,9 +1513,55 @@ def expr(self, node: ast.expr, cond: int = 0) -> ir.Expr: "right-hand side of an assignment is a native copy") array, elem = self.array_of(node) return ir.Index(elem, array, self.index(node.slice)) - raise Skip(line, f"subscript `{_src(node)}` of an object is not supported") + if isinstance(node.slice, ast.Slice): + raise Skip(line, f"slice `{_src(node)}` is not supported") + obj = self.boxed(self.sub(node.value), node.value) # the container, then the index + key = self.boxed(self.sub(node.slice), node.slice) + self.impure = True + return ir.GetItem(Type.OBJ, obj, key) + if isinstance(node, ast.Dict) and not node.keys: + return ir.NewDict(Type.OBJ) + if isinstance(node, ast.List) and not node.elts and isinstance(node.ctx, ast.Load): + return ir.NewList(Type.OBJ) + if isinstance(node, ast.JoinedStr): + return self.fstring(node) raise _unsupported(node) + def str_const(self, v: str, node: ast.AST) -> ir.Expr: + """A `str` constant: the text must be exactly the UTF-8 C string the back end embeds.""" + try: + v.encode("utf-8") + except UnicodeEncodeError: + raise Skip(node.lineno, "a str constant with a lone surrogate is not supported") from None + if "\0" in v: + raise Skip(node.lineno, "a str constant containing NUL is not supported") + return ir.Const(Type.OBJ, v) + + def fstring(self, node: ast.JoinedStr) -> ir.Expr: + """BUILD_STRING of the constant pieces and the `Format` of each field, in source order.""" + parts: list[ir.Expr] = [] + for piece in node.values: + if isinstance(piece, ast.Constant) and isinstance(piece.value, str): + parts.append(self.str_const(piece.value, piece)) + continue + assert isinstance(piece, ast.FormattedValue) + spec = None + if piece.format_spec is not None: + texts = piece.format_spec.values # type: ignore[attr-defined] + if not all(isinstance(t, ast.Constant) and isinstance(t.value, str) for t in texts): + raise Skip(piece.lineno, f"format spec of `{_src(piece)}` has a nested field " + "(only a constant spec is supported)") + spec = "".join(t.value for t in texts) # type: ignore[attr-defined] + self.str_const(spec, piece) + operand = self.boxed(self.value(piece.value), piece.value) + self.impure = True + parts.append(ir.Format(Type.OBJ, operand, spec, piece.conversion)) + if not parts: + return ir.Const(Type.OBJ, "") + if len(parts) == 1: + return parts[0] # the compiler emits no BUILD_STRING for one piece + return ir.BuildString(Type.OBJ, tuple(parts)) + def name(self, node: ast.Name) -> ir.Expr: name = node.id if name in self.arrays: @@ -1460,20 +1608,21 @@ def binop_kind(self, op: ast.operator, node: ast.AST) -> ir.BinOpKind: raise Skip(node.lineno, f"operator `{_src(node)}` is not supported") return kind - def arith(self, op: ir.BinOpKind, left: ir.Expr, right: ir.Expr, node: ast.AST) -> ir.Expr: + def arith(self, op: ir.BinOpKind, left: ir.Expr, right: ir.Expr, node: ast.AST, + inplace: bool = False) -> ir.Expr: lt, rt = left.type, right.type if Type.BOOL in (lt, rt) or Type.NONE in (lt, rt): raise Skip(node.lineno, f"arithmetic on {TYPE_NAME[Type.BOOL if Type.BOOL in (lt, rt) else Type.NONE]} " f"in `{_src(node)}` is not supported") if Type.OBJ in (lt, rt): - return self.obj_binop(op, left, right) + return self.obj_binop(op, left, right, inplace) if lt == Type.I64 and rt == Type.I64: result = Type.F64 if op == ir.BinOpKind.TRUEDIV else Type.I64 e = ir.BinOp(result, op, left, right) if self.recordable(e) and self.proved(e): return dataclasses.replace(e, proven=True) if self.impure_mode: - return self.obj_binop(op, left, right) + return self.obj_binop(op, left, right, inplace) self.deopts = True return e if lt == Type.I64: @@ -1482,9 +1631,10 @@ def arith(self, op: ir.BinOpKind, left: ir.Expr, right: ir.Expr, node: ast.AST) right = ir.ToFloat(Type.F64, right) return ir.BinOp(Type.F64, op, left, right) - def obj_binop(self, op: ir.BinOpKind, left: ir.Expr, right: ir.Expr) -> ir.Expr: + def obj_binop(self, op: ir.BinOpKind, left: ir.Expr, right: ir.Expr, + inplace: bool = False) -> ir.Expr: self.impure = True - return ir.BinOp(Type.OBJ, op, _box(left), _box(right)) + return ir.BinOp(Type.OBJ, op, _box(left), _box(right), inplace=inplace) def unary(self, node: ast.UnaryOp) -> ir.Expr: if isinstance(node.op, ast.Not): @@ -1548,10 +1698,13 @@ def link(self, op: ast.cmpop, left: ir.Expr, right: ir.Expr, obj_ok: bool, f"{TYPE_NAME[left.type if left.type != Type.OBJ else right.type]}" " is not supported (only of objects)") return ir.Is(Type.BOOL, left, right, isinstance(op, ast.IsNot)) + if isinstance(op, (ast.In, ast.NotIn)): + self.impure = True + return ir.Contains(Type.BOOL, _box(left), _box(right), isinstance(op, ast.NotIn)) kind = COMPARE.get(type(op)) if kind is None: raise Skip(node.lineno, f"comparison `{_src(node)}` is not supported " - "(only == != < <= > >=)") + "(only == != < <= > >=, in, not in, is, is not)") lt, rt = left.type, right.type if Type.OBJ in (lt, rt): if not obj_ok: @@ -1570,8 +1723,8 @@ def boolop(self, node: ast.BoolOp, cond: int) -> ir.Expr: for i, v in enumerate(node.values): if i > 0: self.hoist_ok = False # evaluated only if the earlier operands did not decide - e = self.sub(v, 1 if cond == 2 else 0) - if cond == 2: + e = self.sub(v, 1 if cond else 0) + if cond: e = self.truth(e, v) elif e.type != Type.BOOL: raise Skip(node.lineno, f"`{word}` on a non-bool operand `{_src(v)}` outside a " @@ -1903,10 +2056,11 @@ def _walk(x): def _closed(fn: ir.Function, env: dict[str, ir.Function], seen: frozenset[str] = frozenset()) -> bool: - """`fn` runs no user code: no GetAttr, CallObject, Truth, CompareObj, ObjToFloat or OBJ - BinOp, and calls only closed functions.""" + """`fn` runs no user code: no GetAttr, CallObject, Truth, CompareObj, ObjToFloat, GetItem, + Contains, Format, StoreItem, ForObj or OBJ BinOp, and calls only closed functions.""" for n in _walk(fn.body): - if isinstance(n, (ir.GetAttr, ir.CallObject, ir.Truth, ir.CompareObj, ir.ObjToFloat)): + if isinstance(n, (ir.GetAttr, ir.CallObject, ir.Truth, ir.CompareObj, ir.ObjToFloat, + ir.GetItem, ir.Contains, ir.Format, ir.StoreItem, ir.ForObj)): return False if isinstance(n, ir.BinOp) and n.type == Type.OBJ: return False diff --git a/python-multiplatform-ksp/src/main/python/typedpython/ir.py b/python-multiplatform-ksp/src/main/python/typedpython/ir.py index 97992cd1..6d1c613a 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/ir.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/ir.py @@ -32,7 +32,8 @@ - arrays: every `Index`/`StoreIndex` is either proved in bounds (and marked `proven`) or keeps its runtime check; array params are used only by `Index`/`StoreIndex`/`Len`; - ownership: OBJ values are created only by nodes that return a new reference (Box, Global, - GetAttr, CallObject, OBJ BinOp) and the back end releases them by one rule (every OBJ local owns its reference; assigning + GetAttr, CallObject, OBJ BinOp, GetItem, Format, BuildString, NewList, NewDict, a str Const) and + the back end releases them by one rule (every OBJ local owns its reference; assigning releases the old one; every exit releases all); the IR has no node that frees, aliases a borrowed reference, or reads a possibly-NULL object, so use-after-free and NULL dereference have no IR form. @@ -81,9 +82,11 @@ class Expr: @dataclass(frozen=True) class Const(Expr): - """A literal. `value` is a Python int/float/bool/None matching `type`.""" + """A literal. `value` is a Python int/float/bool/None matching `type`. An OBJ constant is None + or a `str` (a new reference to the module-lifetime str the back end holds for it; the text has + no NUL and no lone surrogate, so it is exactly the UTF-8 the C source carries).""" - value: int | float | bool | None + value: int | float | bool | str | None @dataclass(frozen=True) @@ -124,7 +127,9 @@ class BinOp(Expr): FLOORDIV/MOD, CPython's float floor division and modulo (sign of divisor, `-0.0` cases), divisor 0.0 → ZeroDivisionError("float floor division by zero") / ("float modulo by zero"). - OBJ: CPython's generic operation on objects (PyNumber_*), never deopts. + OBJ: CPython's generic operation on objects (PyNumber_*), never deopts. `inplace` (OBJ only) is + the augmented form `a op= b` (PyNumber_InPlace*, which a list's `+=` and a user `__iadd__` + need); an exact `str + str` is PyUnicode_Concat, the function PyNumber_Add ends in for it. """ op: BinOpKind @@ -140,6 +145,7 @@ class BinOp(Expr): # [0, 2**63-1] (Python's floor semantics and C's agree there); the back end emits `>>` / `&`. # Any other op, or an op the verifier cannot prove, is rejected (verify/proven-overflow). proven: bool = False + inplace: bool = False class UnaryOpKind(Enum): @@ -358,6 +364,64 @@ class Tuple(Expr): elements: tuple[Expr, ...] +@dataclass(frozen=True) +class GetItem(Expr): + """`obj[index]` on OBJ operands (obj first, then index): PyObject_GetItem, the call the + interpreter's BINARY_OP [] makes, so `__getitem__`, `__missing__`, `__hash__`/`__eq__` of the + key, `__index__` and every error (KeyError(key), IndexError, TypeError) are CPython's. For an + object that is exactly a `dict` it is PyDict_GetItemRef and, when absent, KeyError(key) built as + dict's own subscript does (runtime `tp_getitem`; the guard is the exact type). Type OBJ. Makes the + function impure and not closed.""" + + obj: Expr + index: Expr + + +@dataclass(frozen=True) +class Contains(Expr): + """`left in right` / `left not in right` (negate) on OBJ operands (left first, then right): + PySequence_Contains(right, left), the call CONTAINS_OP makes, so `__contains__`, iteration and + `__eq__` (with RichCompareBool's identity shortcut) run as in CPython. Type BOOL, always a bool + (so unlike CompareObj it is allowed anywhere). Makes the function impure and not closed.""" + + left: Expr + right: Expr + negate: bool = False + + +@dataclass(frozen=True) +class NewList(Expr): + """`[]`: PyList_New(0). Type OBJ. A fresh object is not an effect.""" + + +@dataclass(frozen=True) +class NewDict(Expr): + """`{}`: PyDict_New(). Type OBJ. A fresh object is not an effect.""" + + +@dataclass(frozen=True) +class Format(Expr): + """One `{value!c:spec}` field of an f-string, exactly as FORMAT_VALUE / CONVERT_VALUE + + FORMAT_SIMPLE / FORMAT_WITH_SPEC do: apply the conversion (`conversion`: -1 none, 115 `!s` + PyObject_Str, 114 `!r` PyObject_Repr, 97 `!a` PyObject_ASCII), then with `spec` (a constant str) + PyObject_Format(value, spec), without it the value itself when it is exactly a str, else + PyObject_Format(value, NULL). Type OBJ (a str unless a user `__format__` returned something + else, which PyObject_Format rejects with CPython's TypeError). Impure and not closed.""" + + operand: Expr + spec: str | None + conversion: int + + +@dataclass(frozen=True) +class BuildString(Expr): + """The f-string itself (BUILD_STRING): the str that joins `elements` (OBJ strs: Const strs and + Format results), evaluated in order, each Format running before the next element is evaluated. + Type OBJ. Runs no user code.""" + + elements: tuple[Expr, ...] + + @dataclass(frozen=True) class Is(Expr): """`a is b` / `a is not b` (negate) on OBJ operands: identity, no user code. Type BOOL. With @@ -474,6 +538,38 @@ class FieldSet(Stmt): value: Expr +@dataclass(frozen=True) +class StoreItem(Stmt): + """`obj[index] = value` on OBJ operands. CPython evaluates the value first, then the container, + then the index: the back end evaluates them in that order (the three operands are expressions + that may be effects). PyObject_SetItem, the call STORE_SUBSCR makes (for an object that is exactly + a `dict`, PyDict_SetItem, which is what dict's `mp_ass_subscript` calls). An effect, and not + closed: `__setitem__`, `__hash__`/`__eq__` of the key and dropping the old value run user code.""" + + obj: Expr + index: Expr + value: Expr + + +@dataclass(frozen=True) +class ForObj(Stmt): + """`for t in iter:` or `for t0, t1, ... in iter:` over any object: `iter` (OBJ) is evaluated once, + GET_ITER is PyObject_GetIter, each step is PyIter_Next (the call FOR_ITER ends in: StopIteration + is the end, any other exception propagates); the item is stored into the OBJ locals `targets` + left to right. With `unpack` the item is unpacked into the targets as UNPACK_SEQUENCE does (an + exact tuple of the right length directly, anything else through its iterator, with CPython's + `cannot unpack non-iterable T object`, `not enough values to unpack (expected n, got k)` and + `too many values to unpack (expected n[, got k])` errors); without it there is exactly one target + and it receives the item. The targets keep their last value after the loop; a loop that ran zero + times leaves them as they were. `break`/`continue` and every exit release the iterator once. + The effect of GetIter/Next is user code, so the function is impure and not closed.""" + + targets: tuple[str, ...] + iter: Expr + body: tuple[Stmt, ...] + unpack: bool = False + + @dataclass(frozen=True) class ExprStmt(Stmt): value: Expr @@ -582,8 +678,8 @@ class Function: Its length never changes after creation, so a bounds proof against Len(local) holds. `pure`: no effect outside its own locals, no stores into anything but locals, no CallObject, - no GetAttr (a property can have effects), no Truth/CompareObj/ObjToFloat/OBJ BinOp (they run - user methods), only `Call`s to pure functions. + no GetAttr (a property can have effects), no Truth/CompareObj/ObjToFloat/OBJ BinOp/GetItem/ + Contains/Format/StoreItem/ForObj (they run user methods), only `Call`s to pure functions. Kotlin interop (maintainer: compiled code must still talk to Kotlin freely): a function that calls into the binder is impure, so its int arithmetic cannot use the deopting I64 path, @@ -609,7 +705,8 @@ class Function: # Module globals of a scalar type read ONCE at entry, as part of the guards: each must be # exactly that type (else deopt, before any effect), and the body reads it as `Local(name)`. # Legal only for a *closed* function, one that runs no user code (no GetAttr, CallObject, - # Truth, CompareObj, ObjToFloat, OBJ BinOp; Calls only to closed functions), so nothing can + # Truth, CompareObj, ObjToFloat, OBJ BinOp, GetItem, Contains, Format, StoreItem, ForObj; Calls + # only to closed functions), so nothing can # rebind the global between entry and use, and reading it once equals reading it at each use. entry_globals: tuple[Param, ...] = () diff --git a/python-multiplatform-ksp/src/main/python/typedpython/pipeline.py b/python-multiplatform-ksp/src/main/python/typedpython/pipeline.py index cd19151b..3f504651 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/pipeline.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/pipeline.py @@ -18,7 +18,7 @@ from pathlib import Path from types import ModuleType -from typedpython import cbuild, cgen, frontend, gate, incremental, ir, verify +from typedpython import cbuild, cgen, cost, frontend, gate, incremental, ir, verify class CompileError(RuntimeError): @@ -54,15 +54,20 @@ def compile_module( ext_suffix: str | None = None, runtime_dir: Path | None = None, project_root: Path | None = None, + cost_rule: bool = True, ) -> Result: """`project_root`: the root the embedded source path is relative to (#112); default: the source's - own directory, so the file name alone is embedded.""" + own directory, so the file name alone is embedded. `cost_rule=False` skips the cost policy + (`cost.py`) and compiles object-only functions too; it exists for tests of the object operations + and is never set by a user-facing path.""" path = Path(path).resolve() root = path.parent if project_root is None else Path(project_root).resolve() _gate_or_raise(path) lowered = frontend.lower(path) proved, diagnostics = verify.verify(lowered) + if cost_rule: + proved, diagnostics = cost.apply(proved, diagnostics) # policy after proof, see cost.py skipped = dict(proved.skipped) if not proved.functions: return Result(path, proved, None, skipped, diagnostics) @@ -168,6 +173,7 @@ def compile_project( python_include: Path | None = None, ext_suffix: str | None = None, runtime_dir: Path | None = None, + cost_rule: bool = True, ) -> ProjectResult: sources = [Path(s).resolve() for s in sources] cache_dir = Path(cache_dir) @@ -178,6 +184,8 @@ def compile_project( def lower(path: Path) -> ir.Module: _gate_or_raise(path) proved, diagnostics = verify.verify(frontend.lower(path)) + if cost_rule: + proved, diagnostics = cost.apply(proved, diagnostics) verified_module_skipped[path] = dict(proved.skipped) verified[path] = skipped_items(proved.skipped, diagnostics, _relative(path, root)) # the VERIFIED module: its signatures are what gets compiled return proved @@ -196,6 +204,8 @@ def compile_c(c_source: str, name: str, out_dir: Path) -> Path: version = COMPILER_VERSION if runtime_dir is not None: version += ":" + _tree_hash(Path(runtime_dir)) + if not cost_rule: + version += ":nocost" suffix = ext_suffix if ext_suffix is not None else sysconfig.get_config_var("EXT_SUFFIX") include = cbuild._python_include(python_include) flags = [*cbuild.DEFAULT_FLAGS, f"-I{include}"] diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md index aa7ead08..64e181e5 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md @@ -136,13 +136,40 @@ All return new references or NULL with CPython's exception set; none deopts. PyObject *tp_global(PyObject *module_dict, PyObject *name); /* globals, then builtins; NameError("name 'x' is not defined") */ PyObject *tp_getattr(PyObject *obj, PyObject *name); /* PyObject_GetAttr */ PyObject *tp_call(PyObject *callee, PyObject *const *args, size_t nargs, PyObject *kwnames); /* PyObject_Vectorcall: nargs = positional count; keyword values follow at args[nargs..] */ - /* tp_binop_obj op codes: TP_BINOP_ADD..TP_BINOP_MOD = 0..5 in ir.BinOpKind declaration order */ + /* tp_binop_obj op codes: TP_BINOP_ADD..TP_BINOP_MOD = 0..5 in ir.BinOpKind declaration order; + TP_BINOP_IADD..TP_BINOP_IMOD = 6..11 are the augmented forms (PyNumber_InPlace*, ir.BinOp.inplace) */ int tp_truth(PyObject *obj); /* 0/1, -1 error (PyObject_IsTrue) */ int tp_compare_bool(PyObject *a, PyObject *b, int op); /* 0/1, -1 error: PyObject_RichCompare then PyObject_IsTrue (no identity shortcut, as `if a == b`) */ int tp_obj_to_f64(PyObject *obj, double *out); /* 0 / -1 (PyNumber_Float, then exact double) */ PyObject *tp_binop_obj(PyObject *a, PyObject *b, int op); /* PyNumber_Add/... for ir.BinOpKind on OBJ */ void tp_release(PyObject **slot); /* Py_CLEAR: the only way generated code drops a reference */ +## Objects: subscript, in, f-strings, iteration (issue #150; ir.GetItem / StoreItem / Contains / NewList / NewDict / Format / BuildString / ForObj / a str Const) +Every helper makes the C API call the interpreter's bytecode makes, so results, exceptions (type, +message, args) and the user `__hash__` / `__eq__` / `__index__` / `__getitem__` / `__format__` / +`__iter__` calls, and their order, are CPython's. New references or NULL / -1 with the exception set; +none deopts. A fast path replaces a generic call only under an exact-type guard, and only with the C +function the generic call ends in for that exact type: the two guards are the macros +`TP_FASTPATH_GUARD_DICT(o)` (`PyDict_CheckExact`) and `TP_FASTPATH_GUARD_STR(o)` +(`PyUnicode_CheckExact`), the single mutation points the tests widen to `PyDict_Check` / +`PyUnicode_Check` and must see fail. A `dict` or `str` subclass (which can override +`__getitem__`, `__setitem__`, `__add__`, ...) always takes the generic call. + PyObject *tp_getitem(PyObject *obj, PyObject *key); /* BINARY_OP []: PyObject_GetItem; exact dict: PyDict_GetItemRef + KeyError(key) as dict_subscript raises it */ + int tp_setitem(PyObject *obj, PyObject *key, PyObject *value); /* STORE_SUBSCR: PyObject_SetItem; exact dict: PyDict_SetItem. 0 / -1 */ + int tp_contains(PyObject *container, PyObject *needle); /* CONTAINS_OP: PySequence_Contains; 1 / 0 / -1 */ + PyObject *tp_new_list(void); /* PyList_New(0) */ + PyObject *tp_new_dict(void); /* PyDict_New() */ + PyObject *tp_format(PyObject *value, PyObject *spec, int conversion); /* one f-string field: conversion -1 / 's' / 'r' / 'a' (PyObject_Str / Repr / ASCII), then spec != NULL: PyObject_Format(v, spec); else an exact str as it is, else PyObject_Format(v, NULL) */ + PyObject *tp_build_string(PyObject *const *parts, Py_ssize_t n); /* BUILD_STRING: PyUnicode_Join("", parts); parts are borrowed */ + PyObject *tp_get_iter(PyObject *obj); /* GET_ITER: PyObject_GetIter */ + PyObject *tp_iter_next(PyObject *iter); /* FOR_ITER: PyIter_Next; NULL without an error at the end */ + int tp_unpack(PyObject *v, Py_ssize_t n, PyObject **out); /* UNPACK_SEQUENCE: an exact tuple of length n directly, else through its iterator with the interpreter's "cannot unpack non-iterable T object" / "not enough values to unpack (expected n, got k)" / "too many values to unpack (expected n[, got k])"; 0 with n new references in out, or -1 with none */ +`str + str` (both exactly str) in `tp_binop_obj` is `PyUnicode_Concat`, which `PyNumber_Add` ends in for two strs. +Free-threaded: containers are read with the strong-reference API (`PyDict_GetItemRef`); an exact +tuple's items may be borrowed because a tuple is immutable; a list or any other object is read +through its iterator, so no borrowed reference into a container outlives the call that returned it. +A str constant is an interned object held in the module state (`names`, immutable after init). + ## Local arrays (ir.NewArray / CopyArray; owned by the function, no list object) Same structs as above with `list == NULL` and `dirty == NULL`. 0 ok, -1 MemoryError. int tp_i64_array_new(tp_i64_array *a, int64_t n, int64_t fill); /* n < 0 → empty */ diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h index e78cb8e5..7521ff6d 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h @@ -143,6 +143,13 @@ static inline void tp_cs_end(tp_cs *c) #define TP_BINOP_TRUEDIV 3 #define TP_BINOP_FLOORDIV 4 #define TP_BINOP_MOD 5 +/* the augmented forms `a op= b` (PyNumber_InPlace*): ir.BinOp.inplace */ +#define TP_BINOP_IADD 6 +#define TP_BINOP_ISUB 7 +#define TP_BINOP_IMUL 8 +#define TP_BINOP_ITRUEDIV 9 +#define TP_BINOP_IFLOORDIV 10 +#define TP_BINOP_IMOD 11 /* ------------------------------------------------------------------------------------------ * Integers (int64_t) @@ -1270,21 +1277,211 @@ static inline int tp_obj_to_f64(PyObject *obj, double *out) return 0; } -/* op is an ir.BinOpKind: TP_BINOP_ADD .. TP_BINOP_MOD (declaration order) */ +/* Fast-path guards (issue #150). A fast path may replace a generic C API call only when the + * operand is exactly the type for which the fast path is what the generic call ends up in: + * subclasses (which can override __getitem__, __setitem__, __add__, ...) always take the generic + * call. These two macros are the only places where "exactly" is decided; the mutation tests in + * test_str_dict.py widen each to PyDict_Check / PyUnicode_Check and must fail. */ +#define TP_FASTPATH_GUARD_DICT(o) PyDict_CheckExact(o) +#define TP_FASTPATH_GUARD_STR(o) PyUnicode_CheckExact(o) + +/* op is an ir.BinOpKind: TP_BINOP_ADD .. TP_BINOP_MOD (declaration order), or its in-place form + * TP_BINOP_IADD .. TP_BINOP_IMOD. `str + str` (both exactly str) is PyUnicode_Concat, which is what + * PyNumber_Add ends in for two strs (str has no nb_add, its sq_concat is PyUnicode_Concat), and + * what the interpreter's BINARY_OP_ADD_UNICODE calls. */ static inline PyObject *tp_binop_obj(PyObject *a, PyObject *b, int op) { switch (op) { - case TP_BINOP_ADD: return PyNumber_Add(a, b); + case TP_BINOP_ADD: + if (TP_FASTPATH_GUARD_STR(a) && TP_FASTPATH_GUARD_STR(b)) { + return PyUnicode_Concat(a, b); + } + return PyNumber_Add(a, b); case TP_BINOP_SUB: return PyNumber_Subtract(a, b); case TP_BINOP_MUL: return PyNumber_Multiply(a, b); case TP_BINOP_TRUEDIV: return PyNumber_TrueDivide(a, b); case TP_BINOP_FLOORDIV: return PyNumber_FloorDivide(a, b); case TP_BINOP_MOD: return PyNumber_Remainder(a, b); + case TP_BINOP_IADD: return PyNumber_InPlaceAdd(a, b); + case TP_BINOP_ISUB: return PyNumber_InPlaceSubtract(a, b); + case TP_BINOP_IMUL: return PyNumber_InPlaceMultiply(a, b); + case TP_BINOP_ITRUEDIV: return PyNumber_InPlaceTrueDivide(a, b); + case TP_BINOP_IFLOORDIV: return PyNumber_InPlaceFloorDivide(a, b); + case TP_BINOP_IMOD: return PyNumber_InPlaceRemainder(a, b); } PyErr_SetString(PyExc_SystemError, "tp_binop_obj: unknown operator"); return NULL; } +/* ------------------------------------------------------------------------------------------ + * str and dict code (issue #150; API.md "Objects: subscript, in, f-strings, iteration"). Each + * helper makes the C API call the interpreter's bytecode makes; the fast paths are guarded by the + * exact type (TP_FASTPATH_GUARD_*). All return new references or NULL / -1 with CPython's exception. + * ------------------------------------------------------------------------------------------ */ + +/* BINARY_OP []: PyObject_GetItem. For an object that is exactly a dict: PyDict_GetItemRef (a strong + * reference, free-threaded safe) and KeyError(key) when absent, which is what dict_subscript does + * for an exact dict (BINARY_OP_SUBSCR_DICT in the interpreter). */ +static inline PyObject *tp_getitem(PyObject *obj, PyObject *key) +{ + if (TP_FASTPATH_GUARD_DICT(obj)) { + PyObject *v = NULL; + int rc = PyDict_GetItemRef(obj, key, &v); + if (rc > 0) { + return v; + } + if (rc == 0) { + PyObject *exc = PyObject_CallOneArg(PyExc_KeyError, key); /* _PyErr_SetKeyError */ + if (exc != NULL) { + PyErr_SetObject((PyObject *)Py_TYPE(exc), exc); + Py_DECREF(exc); + } + } + return NULL; + } + return PyObject_GetItem(obj, key); +} + +/* STORE_SUBSCR: PyObject_SetItem; an exact dict's mp_ass_subscript is PyDict_SetItem. 0 / -1. */ +static inline int tp_setitem(PyObject *obj, PyObject *key, PyObject *value) +{ + if (TP_FASTPATH_GUARD_DICT(obj)) { + return PyDict_SetItem(obj, key, value); + } + return PyObject_SetItem(obj, key, value); +} + +/* CONTAINS_OP: PySequence_Contains(container, needle). 1 / 0 / -1. */ +static inline int tp_contains(PyObject *container, PyObject *needle) +{ + return PySequence_Contains(container, needle); +} + +static inline PyObject *tp_new_list(void) +{ + return PyList_New(0); +} + +static inline PyObject *tp_new_dict(void) +{ + return PyDict_New(); +} + +/* One f-string field (CONVERT_VALUE, then FORMAT_SIMPLE or FORMAT_WITH_SPEC): `conversion` is -1, + * 's', 'r' or 'a'; `spec` is NULL or the format spec str. */ +static inline PyObject *tp_format(PyObject *value, PyObject *spec, int conversion) +{ + PyObject *converted, *result; + switch (conversion) { + case -1: converted = Py_NewRef(value); break; + case 's': converted = PyObject_Str(value); break; + case 'r': converted = PyObject_Repr(value); break; + case 'a': converted = PyObject_ASCII(value); break; + default: + PyErr_SetString(PyExc_SystemError, "tp_format: unknown conversion"); + return NULL; + } + if (converted == NULL) { + return NULL; + } + if (spec == NULL && PyUnicode_CheckExact(converted)) { + return converted; /* FORMAT_SIMPLE: an exact str is its own format */ + } + result = PyObject_Format(converted, spec); + Py_DECREF(converted); + return result; +} + +/* BUILD_STRING: the join of n strs with the empty separator. */ +static inline PyObject *tp_build_string(PyObject *const *parts, Py_ssize_t n) +{ + PyObject *seq = tp_tuple(parts, n); + PyObject *empty, *result; + if (seq == NULL) { + return NULL; + } + empty = PyUnicode_New(0, 0); /* the empty-string singleton */ + if (empty == NULL) { + Py_DECREF(seq); + return NULL; + } + result = PyUnicode_Join(empty, seq); + Py_DECREF(empty); + Py_DECREF(seq); + return result; +} + +/* GET_ITER: PyObject_GetIter. */ +static inline PyObject *tp_get_iter(PyObject *obj) +{ + return PyObject_GetIter(obj); +} + +/* FOR_ITER: the next item (a new reference), or NULL with no error set at the end (a StopIteration + * is the end and is cleared), or NULL with the exception set. */ +static inline PyObject *tp_iter_next(PyObject *iter) +{ + return PyIter_Next(iter); +} + +/* UNPACK_SEQUENCE (no star) of `v` into the n slots `out`, each a new reference. An exact tuple of + * length n is taken directly (immutable, so the items cannot change under us); anything else goes + * through its iterator, with the interpreter's unpack_iterable errors. 0 ok; -1 error, with no + * reference left in `out`. */ +static inline int tp_unpack(PyObject *v, Py_ssize_t n, PyObject **out) +{ + PyObject *it, *w; + Py_ssize_t i; + if (PyTuple_CheckExact(v) && PyTuple_GET_SIZE(v) == n) { + for (i = 0; i < n; i++) { + out[i] = Py_NewRef(PyTuple_GET_ITEM(v, i)); + } + return 0; + } + it = PyObject_GetIter(v); + if (it == NULL) { + if (PyErr_ExceptionMatches(PyExc_TypeError) && Py_TYPE(v)->tp_iter == NULL + && !PySequence_Check(v)) { + PyErr_Format(PyExc_TypeError, "cannot unpack non-iterable %.200s object", + Py_TYPE(v)->tp_name); + } + return -1; + } + for (i = 0; i < n; i++) { + w = PyIter_Next(it); + if (w == NULL) { + if (!PyErr_Occurred()) { + PyErr_Format(PyExc_ValueError, + "not enough values to unpack (expected %zd, got %zd)", n, i); + } + goto fail; + } + out[i] = w; + } + w = PyIter_Next(it); + if (w == NULL) { + if (PyErr_Occurred()) { + goto fail; + } + Py_DECREF(it); + return 0; + } + Py_DECREF(w); + if (PyList_CheckExact(v) || PyTuple_CheckExact(v) || PyDict_CheckExact(v)) { + Py_ssize_t ll = PyDict_CheckExact(v) ? PyDict_Size(v) : Py_SIZE(v); + PyErr_Format(PyExc_ValueError, "too many values to unpack (expected %zd, got %zd)", n, ll); + } + else { + PyErr_Format(PyExc_ValueError, "too many values to unpack (expected %zd)", n); + } +fail: + while (i > 0) { + Py_CLEAR(out[--i]); + } + Py_DECREF(it); + return -1; +} + /* Py_CLEAR: the only way generated code drops a reference */ static inline void tp_release(PyObject **slot) { diff --git a/python-multiplatform-ksp/src/main/python/typedpython/verify.py b/python-multiplatform-ksp/src/main/python/typedpython/verify.py index ec71758a..b0611305 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/verify.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/verify.py @@ -94,6 +94,16 @@ "`obj` itself ... after checking": the result is a new reference to the operand (the back end increfs it); the operand's own reference is untouched. +Objects, str and dict code (#150). GetItem, Contains, Format, StoreItem and ForObj run user code +(`__getitem__`, `__hash__`, `__eq__`, `__format__`, `__iter__`, `__next__`, `__setitem__`): each is an +effect and makes the function not closed (so it can hold no array, no buffer and no entry global), +like CallObject. NewList, NewDict and BuildString run none. They are OBJ producers (new references) +with the OBJ operand slots below; an OBJ Const is None or a plain str (UTF-8, no NUL); a ForObj's +targets are distinct declared OBJ locals, assigned by the loop (so definitely assigned inside the +body, and not after it unless assigned before: a loop of zero iterations binds nothing), killing +any exact-type fact about them; `unpack` false means exactly one target; `BinOp.inplace` only on an +OBJ operation; `Format.conversion` is -1, 97, 114 or 115 and `spec` None or a plain str. + Proven i64 operations. The verifier never trusts `BinOp.proven` / `UnaryOp.proven`: for every I64 expression it computes an interval [lo, hi] (exact Python ints) and accepts `proven=True` only on an I64 ADD/SUB/MUL (BinOp) or NEG (UnaryOp) whose exact result interval lies inside [-2**63, 2**63-1], @@ -212,6 +222,7 @@ Truth, Tuple, Type, Unbox, UnaryOp, UnaryOpKind, While, ) from .ir import CheckExact, ClassDecl, FieldGet, FieldSet, Is, IsExact, New +from .ir import BuildString, Contains, ForObj, Format, GetItem, NewDict, NewList, StoreItem DA = "verify/definite-assignment" MISSING_RETURN = "verify/missing-return" @@ -249,15 +260,18 @@ class Diagnostic: # the one the back end implements). _EXPRS = frozenset({Const, Local, BinOp, UnaryOp, Compare, And, Or, ToFloat, Box, Unbox, MathCall, Call, Global, GetAttr, CallObject, Truth, CompareObj, ObjToFloat, Len, Index, - NewArray, CopyArray, Tuple, Is, IsExact, CheckExact, FieldGet, New}) + NewArray, CopyArray, Tuple, Is, IsExact, CheckExact, FieldGet, New, + GetItem, Contains, NewList, NewDict, Format, BuildString}) _STMTS = frozenset({Assign, StoreIndex, ExprStmt, If, While, ForRange, Return, Break, Continue, - FieldSet}) + FieldSet, StoreItem, ForObj}) # Nodes whose OBJ result is a new reference (ir.py "ownership": Box, Global, GetAttr, CallObject, # OBJ BinOp, Tuple), plus reads of an OBJ local (the local owns its reference; the back end increfs or # borrows under that rule) and Calls of a compiled function returning OBJ (which returns a new # reference by the same rule). _OBJ_PRODUCERS = frozenset({Box, Global, GetAttr, CallObject, BinOp, Local, Call, Tuple, New, - FieldGet, CheckExact, Const}) # an OBJ Const is None (e_Const) + FieldGet, CheckExact, Const, # an OBJ Const is None or a str (e_Const) + GetItem, NewList, NewDict, Format, BuildString}) +_CONVERSIONS = (-1, 97, 114, 115) # Format.conversion: none, !a, !r, !s _VALUE_TYPES = (Type.I64, Type.F64, Type.BOOL, Type.OBJ) # what a local may hold _RETURN_TYPES = _VALUE_TYPES + (Type.NONE,) _ELEMENT = {Type.F64_ARRAY: Type.F64, Type.I64_ARRAY: Type.I64} @@ -574,7 +588,7 @@ def e_Const(self, e: Const, state: _State) -> ir.Expr: or (t is Type.BOOL and type(v) is bool) or (t is Type.NONE and v is None)) if t is Type.OBJ: - ok = v is None # the only OBJ constant: a new reference to the None singleton + ok = v is None or _plain_str(v) # None, or a str the C source can carry exactly if not ok: self.error(TYPE, f"constant {v!r} does not match type {t.value}") return e @@ -705,6 +719,10 @@ def e_BinOp(self, e: BinOp, state: _State) -> ir.Expr: return e left, right = self.expr(e.left, state), self.expr(e.right, state) lt, rt = getattr(left, "type", None), getattr(right, "type", None) + if type(e.inplace) is not bool: + self.error(TYPE, f"BinOp `inplace` {e.inplace!r} is not a bool") + elif e.inplace and lt is not Type.OBJ: + self.error(TYPE, "BinOp `inplace` is set on a non-OBJ operation (it is PyNumber_InPlace*)") proven = False if type(e.proven) is not bool: self.error(TYPE, f"BinOp `proven` {e.proven!r} is not a bool") @@ -1029,6 +1047,55 @@ def in_bounds(self, array: str, idx: ir.Expr) -> bool: # --- fixed-layout classes ----------------------------------------------------------------- + def e_GetItem(self, e: GetItem, state: _State) -> ir.Expr: + obj = self.obj_operand(e.obj, state, "container of GetItem") + index = self.obj_operand(e.index, state, "index of GetItem") + self.result(e, Type.OBJ) + self.effects.append("GetItem (runs __getitem__, __hash__, __eq__, __index__)") + self.opens.append("GetItem") + return dataclasses.replace(e, obj=obj, index=index) + + def e_Contains(self, e: Contains, state: _State) -> ir.Expr: + left = self.obj_operand(e.left, state, "left operand of in") + right = self.obj_operand(e.right, state, "right operand of in") + if type(e.negate) is not bool: + self.error(TYPE, f"Contains `negate` {e.negate!r} is not a bool") + self.result(e, Type.BOOL) + self.effects.append("Contains (runs __contains__, __iter__, __eq__)") + self.opens.append("Contains") + return dataclasses.replace(e, left=left, right=right) + + def e_NewList(self, e: NewList, state: _State) -> ir.Expr: + self.result(e, Type.OBJ) + return e + + def e_NewDict(self, e: NewDict, state: _State) -> ir.Expr: + self.result(e, Type.OBJ) + return e + + def e_Format(self, e: Format, state: _State) -> ir.Expr: + operand = self.obj_operand(e.operand, state, "operand of Format") + if e.spec is not None and not _plain_str(e.spec): + self.error(TYPE, f"Format spec {e.spec!r} is not None or a plain str") + if e.conversion not in _CONVERSIONS or type(e.conversion) is not int: + self.error(TYPE, f"Format conversion {e.conversion!r} is not one of {_CONVERSIONS}") + self.result(e, Type.OBJ) + self.effects.append("Format (runs __format__, __str__, __repr__)") + self.opens.append("Format") + return dataclasses.replace(e, operand=operand) + + def e_BuildString(self, e: BuildString, state: _State) -> ir.Expr: + if not isinstance(e.elements, tuple) or len(e.elements) < 2: + self.error(STRUCTURE, "BuildString elements are not a tuple of at least two") + return e + elements = tuple(self.obj_operand(x, state, f"element {k + 1} of BuildString") + for k, x in enumerate(e.elements)) + for k, x in enumerate(elements): + if not (type(x) is Format or (type(x) is Const and type(x.value) is str)): + self.error(TYPE, f"element {k + 1} of BuildString is not a Format or a str Const") + self.result(e, Type.OBJ) + return dataclasses.replace(e, elements=elements) + def e_Is(self, e: Is, state: _State) -> ir.Expr: left = self.obj_operand(e.left, state, "left operand of is") right = self.obj_operand(e.right, state, "right operand of is") @@ -1243,6 +1310,57 @@ def s_FieldSet(self, s: FieldSet, state: _State): self.opens.append(f"FieldSet .{s.field}") # dropping the old value may run its __del__ return dataclasses.replace(s, obj=obj, value=value), state + def s_StoreItem(self, s: StoreItem, state: _State): + obj = self.obj_operand(s.obj, state, "container of StoreItem") + index = self.obj_operand(s.index, state, "index of StoreItem") + value = self.obj_operand(s.value, state, "value of StoreItem") + self.effects.append("StoreItem (runs __setitem__, __hash__, __eq__)") + self.opens.append("StoreItem") + return dataclasses.replace(s, obj=obj, index=index, value=value), state + + def s_ForObj(self, s: ForObj, state: _State): + it = self.obj_operand(s.iter, state, "iterable of ForObj") + targets = s.targets + ok = (isinstance(targets, tuple) and targets and all(isinstance(t, str) for t in targets) + and len(set(targets)) == len(targets)) + if not ok: + self.error(STRUCTURE, f"ForObj targets {targets!r} are not distinct names") + targets = () + if type(s.unpack) is not bool: + self.error(TYPE, f"ForObj `unpack` {s.unpack!r} is not a bool") + elif not s.unpack and len(targets) != 1 and ok: + self.error(STRUCTURE, "ForObj without unpack has exactly one target") + for t in targets: + if t in self.arrays or t in self.larrays: + self.error(ARRAY_USE, f"ForObj target '{t}' is an array") + elif self.scalars.get(t) is not Type.OBJ: + self.error(TYPE, f"ForObj target '{t}' must be a declared obj local") + if t in self.entry: + self.error(STRUCTURE, f"entry global '{t}' is assigned by the loop") + for outer in self.loops: + if outer.var == t: + self.error(STRUCTURE, f"loop variable '{t}' is assigned in its loop body " + f"(by an inner loop)") + self.effects.append("ForObj (runs __iter__ and __next__)") + self.opens.append("ForObj") + loop = _Loop(None, None) + self.loops.append(loop) + head = self.loop_facts(s.body, tuple(targets)) + self.facts = dict(head) + killed = _assigned_names(s.body) | set(targets) + head_refine = {k: v for k, v in self.refine.items() if k not in killed} + self.refine = head_refine + body, _ = self.block(s.body, None if state is None else state | set(targets)) + self.loops.pop() + self.refine = dict(head_refine) + self.facts = dict(head) + # Zero iterations leave the targets as they were: the state after the loop is the entry + # state, met with every break state (each a superset of it). + after: _State = state + for br in loop.breaks: + after = _meet(after, br) + return dataclasses.replace(s, iter=it, body=body), after + def s_If(self, s: If, state: _State): cond = self.cond(s.cond, state, "if") base = self.facts @@ -1444,7 +1562,7 @@ def _assigned(body: object, counts: dict[str, int] | None = None) -> dict[str, i elif cls is If: _assigned(st.then, counts) _assigned(st.orelse, counts) - elif cls in (While, ForRange): + elif cls in (While, ForRange, ForObj): _assigned(st.body, counts) return counts @@ -1461,6 +1579,10 @@ def _assigned_values(body: object, out: dict[str, list[object]]) -> None: _assigned_values(st.orelse, out) elif cls in (While, ForRange): _assigned_values(st.body, out) + elif cls is ForObj: + for t in st.targets if isinstance(st.targets, tuple) else (): + out.setdefault(t, []).append(st) # not a New: ends form (e) for the target + _assigned_values(st.body, out) def _loop_vars(body: object) -> set[str]: @@ -1470,7 +1592,7 @@ def _loop_vars(body: object) -> set[str]: cls = type(st) if cls is If: out |= _loop_vars(st.then) | _loop_vars(st.orelse) - elif cls is While: + elif cls in (While, ForObj): out |= _loop_vars(st.body) elif cls is ForRange: out.add(st.var) @@ -1490,12 +1612,26 @@ def _assigned_names(body: object) -> set[str]: out |= _assigned_names(st.then) | _assigned_names(st.orelse) elif cls is While: out |= _assigned_names(st.body) + elif cls is ForObj: + out |= set(st.targets) if isinstance(st.targets, tuple) else set() + out |= _assigned_names(st.body) elif cls is ForRange: out.add(st.var) out |= _assigned_names(st.body) return out +def _plain_str(v: object) -> bool: + """A str the generated C source carries exactly: it encodes to UTF-8 and has no NUL.""" + if type(v) is not str or "\0" in v: + return False + try: + v.encode("utf-8") + except UnicodeEncodeError: + return False + return True + + def _meet_interval(a: tuple[int, int], b: tuple[int, int] | None) -> tuple[int, int]: """`a` narrowed by `b`; an empty meet (unreachable code) leaves `a`, which is still sound.""" if b is None: diff --git a/python-multiplatform-ksp/src/test/python/test_cost_rule.py b/python-multiplatform-ksp/src/test/python/test_cost_rule.py new file mode 100644 index 00000000..f7411245 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_cost_rule.py @@ -0,0 +1,135 @@ +"""The cost policy (cost.py, #150): a function is compiled only if it does native work. + +A verified function that only does object operations in its loops (in an impure function an int is a +CPython object, so `+`, `//`, `%` and `>` are generic PyNumber / RichCompare calls) is slower compiled +than interpreted, so it stays interpreted with a diagnostic. Pure functions are never touched. +`pipeline.compile_module(..., cost_rule=False)` is the test-only opt-out. +""" +from __future__ import annotations + +import textwrap + +import pytest + +from typedpython import cost, pipeline + +HEADER = "# typedpython: compiled\n" + +OBJECT_ONLY = HEADER + textwrap.dedent(''' + def word_of(word_id: int) -> str: + x: int = word_id + 17576 + w: str = "" + while x > 0: + w = chr(97 + x % 26) + w + x //= 26 + return w +''') + + +def build(tmp_path, source, **kw): + path = tmp_path / "m.py" + path.write_text(source) + return pipeline.compile_module(path, tmp_path / "out", **kw) + + +def names(result): + return {f.name for f in result.module.functions} + + +def test_an_object_only_loop_stays_interpreted_with_the_reason(tmp_path): + r = build(tmp_path, OBJECT_ONLY) + assert names(r) == set() and r.extension is None + assert "slower than the interpreter" in r.skipped["word_of"] + assert r.skipped["word_of"].startswith("policy/cost: only object operations in its loops") + assert [d.rule for d in r.diagnostics] == ["policy/cost"] + + +def test_the_opt_out_compiles_the_same_function(tmp_path): + r = build(tmp_path, OBJECT_ONLY, cost_rule=False) + assert names(r) == {"word_of"} and r.extension is not None + + +MIXED = HEADER + textwrap.dedent(''' + def mixed(n: int, xs: list[str]) -> float: + total: float = 0.0 + for i in range(n): + total += 1.5 * 2.0 + xs.append("a") + return total + + + def sq(k: int) -> int: + return k * k + + + def pure_no_loop(a: float, b: float) -> float: + return a * b + 1.0 + + + def object_loop_with_native_op_outside(n: int, xs: list[str]) -> float: + total: float = 1.0 + 2.0 + i: int = 0 + while i < n: + xs.append("a") + i += 1 + return total + + + def native_loop_inside_object_loop(rows: list[list[str]]) -> float: + total: float = 0.0 + for row in rows: + for k in range(3): + total += 0.5 * 2.0 + return total +''') + + +def test_one_native_loop_with_object_operations_compiles(tmp_path): + r = build(tmp_path, MIXED) + got = names(r) + assert {"mixed", "pure_no_loop", "native_loop_inside_object_loop"} <= got, r.skipped + assert "object_loop_with_native_op_outside" not in got # the native op is outside the loop + assert "object_loop_with_native_op_outside" in r.skipped + + +def test_a_pure_function_is_never_rejected_even_without_a_loop(tmp_path): + r = build(tmp_path, MIXED) + assert "pure_no_loop" in names(r) and "sq" in names(r) + + +def test_a_caller_of_a_rejected_function_is_rejected_too(tmp_path): + # unit level: the propagation over direct Calls + from typedpython import frontend, ir, verify + src = tmp_path / "p.py" + src.write_text(OBJECT_ONLY) + proved, diags = verify.verify(frontend.lower(src)) + assert [f.name for f in proved.functions] == ["word_of"] + word = proved.functions[0] + caller = ir.Function(name="c", params=(), returns=ir.Type.NONE, locals={}, pure=False, + may_deopt=False, source_line=1, + body=(ir.ForRange("i", ir.Const(ir.Type.I64, 0), ir.Const(ir.Type.I64, 3), + ir.Const(ir.Type.I64, 1), + (ir.ExprStmt(ir.Call(ir.Type.NONE, "word_of", ())),)),)) + module = ir.Module("p", (word, caller)) + out, diags = cost.apply(module, []) + assert out.functions == () # `c` has no native op (an empty-arg Call is not one) + assert "word_of" in out.skipped and "c" in out.skipped + + +def test_the_rule_is_what_keeps_it_interpreted(tmp_path, monkeypatch): + """Mutation: with the rule replaced by 'always native' the first test above must fail.""" + monkeypatch.setattr(cost, "does_native_work", lambda fn: True) + with pytest.raises(AssertionError): + test_an_object_only_loop_stays_interpreted_with_the_reason(tmp_path) + + +STRAIGHT_LINE = HEADER + textwrap.dedent(''' + def label(n: int, name: str) -> str: + return f"{name}:{n + 1}" +''') + + +def test_a_straight_line_impure_function_still_compiles(tmp_path): + # no loop to lose to the interpreter: what compiled before #150 keeps compiling + r = build(tmp_path, STRAIGHT_LINE) + assert names(r) == {"label"}, r.skipped diff --git a/python-multiplatform-ksp/src/test/python/test_free_threaded.py b/python-multiplatform-ksp/src/test/python/test_free_threaded.py index 264f945f..6acf69f6 100644 --- a/python-multiplatform-ksp/src/test/python/test_free_threaded.py +++ b/python-multiplatform-ksp/src/test/python/test_free_threaded.py @@ -161,7 +161,7 @@ def built(tmp_path_factory): d = tmp_path_factory.mktemp("ft") path = d / "ftmod.py" path.write_text(SOURCE) - result = pipeline.compile_module(path, d / "out") + result = pipeline.compile_module(path, d / "out", cost_rule=False) # object-only setters: the cost policy would skip them (#150) assert result.extension is not None, result.skipped compiled = {f.name for f in result.module.functions} wanted = {"geta", "seta", "mk", "total", "fill", "edge", "fill_int", "edge_int", "rotate", "zero2", diff --git a/python-multiplatform-ksp/src/test/python/test_frontend.py b/python-multiplatform-ksp/src/test/python/test_frontend.py index 03217ece..3a51a355 100644 --- a/python-multiplatform-ksp/src/test/python/test_frontend.py +++ b/python-multiplatform-ksp/src/test/python/test_frontend.py @@ -116,7 +116,9 @@ def g(a: list[float], n: int) -> None: assert f.locals == {"m": OBJ} assert f.body[:2] == ( Assign("m", Box(OBJ, Local(I64, "n"))), - Assign("m", BinOp(OBJ, BinOpKind.ADD, Local(OBJ, "m"), Box(OBJ, Const(I64, 1)))), + # `m += 1` on an object is the in-place operation (PyNumber_InPlaceAdd, #150) + Assign("m", BinOp(OBJ, BinOpKind.ADD, Local(OBJ, "m"), Box(OBJ, Const(I64, 1)), + inplace=True)), ) assert not f.pure and not f.may_deopt @@ -709,7 +711,8 @@ def render(n: int, scale: float) -> float: ExprStmt(CallObject(OBJ, Global(OBJ, "draw"), ( Local(OBJ, "canvas"), Box(OBJ, Local(F64, "x")), Box(OBJ, Const(F64, 0.5)), ), ("alpha",))), - Assign("hits", BinOp(OBJ, BinOpKind.ADD, Local(OBJ, "hits"), Box(OBJ, Const(I64, 1)))), + Assign("hits", BinOp(OBJ, BinOpKind.ADD, Local(OBJ, "hits"), Box(OBJ, Const(I64, 1)), + inplace=True)), # `hits += 1` on an object: in place (#150) If(CompareObj(BOOL, CompareKind.GT, Local(OBJ, "hits"), Box(OBJ, Local(I64, "n"))), (Break(),)), ) diff --git a/python-multiplatform-ksp/src/test/python/test_module_report.py b/python-multiplatform-ksp/src/test/python/test_module_report.py index 195b7a43..8625cf18 100644 --- a/python-multiplatform-ksp/src/test/python/test_module_report.py +++ b/python-multiplatform-ksp/src/test/python/test_module_report.py @@ -101,7 +101,7 @@ def ok(n: int) -> int: def greet(name: str) -> str: - return "hi " + name + return name[1:] """) @@ -125,7 +125,7 @@ def test_kind_compiled(tmp_path): def test_kind_nothing_compiled_with_structured_items(tmp_path): _, _, m = build_one(tmp_path, "pkg/plain.py", - "# typedpython: compiled\n\ndef f(s: str) -> str:\n return s + 'x'\n") + "# typedpython: compiled\n\ndef f(s: str) -> str:\n return s[1:]\n") assert m.kind == "nothing_compiled" and m.extension is None [item] = m.skipped_items assert (item.name, item.file, item.line) == ("f", "pkg/plain.py", 4) @@ -138,7 +138,7 @@ def test_kind_not_marked_when_there_is_neither_marker_nor_decorator(tmp_path): def test_a_compiled_decorator_alone_opts_in(tmp_path): - src = "@compiled\ndef f(s: str) -> str:\n return s + 'x'\n" + src = "@compiled\ndef f(s: str) -> str:\n return s[1:]\n" _, _, m = build_one(tmp_path, "pkg/deco.py", src) assert m.kind == "nothing_compiled" assert [(i.name, i.line) for i in m.skipped_items] == [("f", 3)] @@ -170,7 +170,7 @@ def test_structure_round_trips_through_the_cache_for_reused_modules(tmp_path): def test_not_marked_and_nothing_compiled_survive_reuse(tmp_path): root, path, _ = build_one(tmp_path, "pkg/plain.py", - "# typedpython: compiled\n\ndef f(s: str) -> str:\n return s + 'x'\n") + "# typedpython: compiled\n\ndef f(s: str) -> str:\n return s[1:]\n") again = pipeline.compile_project([path], root, tmp_path / "cache").modules[0] assert again.status == "reused" and again.kind == "nothing_compiled" and again.skipped_items root2, path2, _ = build_one(tmp_path, "pkg/un.py", "def f(x: int) -> int:\n return x\n") diff --git a/python-multiplatform-ksp/src/test/python/test_pipeline.py b/python-multiplatform-ksp/src/test/python/test_pipeline.py index 039b39c5..2e43816e 100644 --- a/python-multiplatform-ksp/src/test/python/test_pipeline.py +++ b/python-multiplatform-ksp/src/test/python/test_pipeline.py @@ -54,11 +54,11 @@ def test_unsupported_functions_stay_interpreted_with_a_reason(write, tmp_path): path = write("mixed.py", SCALAR + """ def greet(name: str) -> str: - return "hi " + name + return name[1:] """) result = pipeline.compile_module(path, tmp_path / "out") assert "greet" in result.skipped and result.skipped["greet"] - assert pipeline.load(result).greet("x") == "hi x" + assert pipeline.load(result).greet("xy") == "y" def test_a_module_with_nothing_to_compile_produces_no_extension(write, tmp_path): diff --git a/python-multiplatform-ksp/src/test/python/test_pipeline_incremental.py b/python-multiplatform-ksp/src/test/python/test_pipeline_incremental.py index 9eef0930..dd7cfad8 100644 --- a/python-multiplatform-ksp/src/test/python/test_pipeline_incremental.py +++ b/python-multiplatform-ksp/src/test/python/test_pipeline_incremental.py @@ -133,7 +133,7 @@ def test_rebuilt_and_reused_extensions_compute_the_interpreted_results(project, def test_module_with_nothing_compiled_has_no_extension_and_is_reused(tmp_path): root = tmp_path / "proj" root.mkdir() - (root / "plain.py").write_text(HEADER + "def f(s: str) -> str:\n return s + 'x'\n") + (root / "plain.py").write_text(HEADER + "def f(s: str) -> str:\n return s[1:]\n") cache = tmp_path / "cache" first = pipeline.compile_project([root / "plain.py"], root, cache) m = first.modules[0] diff --git a/python-multiplatform-ksp/src/test/python/test_runtime.py b/python-multiplatform-ksp/src/test/python/test_runtime.py index c213475a..629798e7 100644 --- a/python-multiplatform-ksp/src/test/python/test_runtime.py +++ b/python-multiplatform-ksp/src/test/python/test_runtime.py @@ -1203,4 +1203,4 @@ def test_binop_obj_maps_the_six_kinds_to_the_python_operators(rt): assert rt.binop_obj(I64_MAX, 1, 0) == 2**63 assert rt.binop_obj(I64_MIN, -1, 4) == 2**63 assert exc_of(rt.binop_obj, 1, 0, 4)[1:3] == (ZeroDivisionError, str(exc_of(operator.floordiv, 1, 0)[2])) - assert exc_of(rt.binop_obj, 1, 2, 9)[1] is SystemError + assert exc_of(rt.binop_obj, 1, 2, 12)[1] is SystemError # 6..11 are the in-place forms (#150) diff --git a/python-multiplatform-ksp/src/test/python/test_str_dict.py b/python-multiplatform-ksp/src/test/python/test_str_dict.py new file mode 100644 index 00000000..2362033c --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_str_dict.py @@ -0,0 +1,1005 @@ +"""str and dict code in compiled functions (issue #150, widened; SPEC N-8 / N-9). + +Every operation added here must call the C API the interpreter's bytecode calls, so each test is +differential: the compiled function and `__typedpython_interpreted__[name]` run on two fresh copies +of the arguments and must agree on the result, on the exception (type and repr, so the message and +args), on the final state of every argument and on the log of every user `__hash__` / `__eq__` / +`__index__` / `__format__` / `__repr__` / `__iter__` call (same calls, same order). A test that +expects the compiled path compares `__typedpython_deopts__` before and after, so a function that +silently stayed interpreted fails it. + +Also here: the wordfreq benchmark's `word_of` and `main` compile and print what CPython prints, the +object operations leak no reference, and the exact-type guards of the fast paths are shown to matter +(a mutant header whose guard is "always true" makes a test fail). +""" +from __future__ import annotations + +import contextlib +import dataclasses +import gc +import importlib.util +import io +import shutil +import sys +import textwrap +from pathlib import Path + +import pytest + +from typedpython import cbuild, cgen, frontend, verify + +REPO = Path(__file__).resolve().parents[4] +WORDFREQ = REPO / "benchmarks" / "typedpython" / "py" / "wordfreq.py" + +SOURCE = textwrap.dedent(''' + # typedpython: compiled + from typing import SupportsIndex + + + def get(d: dict[object, object], k: object) -> object: + return d[k] + + + def get_list(xs: list[object], i: int) -> object: + return xs[i] + + + def get_index(xs: list[object], i: SupportsIndex) -> object: + return xs[i] + + + def get_str(s: str, i: int) -> str: + return s[i] + + + def get_nested(d: dict[object, dict[object, object]], a: object, b: object) -> object: + return d[a][b] + + + def put(d: dict[object, object], k: object, v: object) -> None: + d[k] = v + + + def put_list(xs: list[object], i: int, v: object) -> None: + xs[i] = v + + + def has(c: list[object], x: object) -> bool: + return x in c + + + def has_not(c: list[object], x: object) -> bool: + return x not in c + + + def has_dict(c: dict[object, object], x: object) -> bool: + return x in c + + + def has_str(c: str, x: str) -> bool: + return x in c + + + def has_branch(c: list[object], x: object) -> str: + if x in c: + return "yes" + return "no" + + + def has_chain(c: list[object], x: object, y: object) -> bool: + return x in c in y + + + def pairs(xs: list[tuple[object, ...]]) -> list[object]: + out: list[object] = [] + for a, b in xs: + out.append(a) + out.append(b) + return out + + + def triples(xs: list[tuple[object, ...]]) -> list[object]: + out: list[object] = [] + for a, b, c in xs: + out.append(a) + out.append(c) + return out + + + def each(xs: list[object]) -> list[object]: + out: list[object] = [] + for x in xs: + if x is None: + continue + out.append(x) + return out + + + def first_after(xs: list[object], stop: object) -> object: + found: object = None + for x in xs: + if x == stop: + found = x + break + return found + + + def return_inside(xs: list[object]) -> object: + for x in xs: + return x + return None + + + def nested(xs: list[list[object]]) -> list[object]: + out: list[object] = [] + for row in xs: + for x in row: + if x is None: + break + out.append(x) + return out + + + def items_of(d: dict[str, object]) -> list[object]: + out: list[object] = [] + for k, v in d.items(): + out.append(k) + out.append(v) + return out + + + def mutate_while_iterating(d: dict[str, object]) -> object: + n: int = 0 + for k in d: + d["new" + k] = n + n += 1 + return n + + + def fmt(x: object, y: float, n: int) -> str: + return f"{x}|{x!r}|{x!s}|{x!a}|{y:.3f}|{n:05d}|{x:>5}|{n}" + + + def fmt_one(x: object) -> str: + return f"{x}" + + + def fmt_text(x: object) -> str: + return f"a{x}b{x!r}c" + + + def fmt_empty() -> str: + return f"" + + + def fmt_debug(x: object) -> str: + return f"{x=}" + + + def cat(a: str, b: str) -> str: + return a + b + + + def cat_in_place(a: str, b: str) -> str: + s: str = "" + s += a + s += b + return s + + + def grow(xs: list[object], ys: list[object]) -> list[object]: + out: list[object] = [] + out += xs + out += ys + return out + + + def empties() -> object: + d: dict[object, object] = {} + xs: list[object] = [] + xs.append(d) + return xs + + + class Acc: + def __init__(self, log: list[object]) -> None: + self.log = log + + def __iadd__(self, other: int) -> Acc: + self.log.append(("iadd", other)) + return self + + def __isub__(self, other: int) -> Acc: + self.log.append(("isub", other)) + return self + + def __imul__(self, other: int) -> Acc: + self.log.append(("imul", other)) + return self + + def __itruediv__(self, other: int) -> Acc: + self.log.append(("itruediv", other)) + return self + + def __ifloordiv__(self, other: int) -> Acc: + self.log.append(("ifloordiv", other)) + return self + + def __imod__(self, other: int) -> Acc: + self.log.append(("imod", other)) + return self + + + def aug_ops(a: Acc, b: int) -> Acc: + a += b + a -= b + a *= b + a /= b + a //= b + a %= b + return a + + + def aug_list(xs: list[str], n: int) -> list[str]: + xs *= n + xs += xs + return xs + + + def lookup_default(d: dict[str, int], k: str) -> object: + counts: dict[str, int] = {} + counts[k] = counts.get(k, 0) + 1 + counts[k] = counts.get(k, 0) + 1 + return counts[k] + d.get(k, 0) +''') + +COMPILED = { + "get", "get_list", "get_index", "get_str", "get_nested", "put", "put_list", "has", + "has_not", "has_dict", "has_str", "has_branch", "has_chain", "pairs", "triples", "each", + "first_after", "return_inside", "nested", "items_of", "mutate_while_iterating", "fmt", "fmt_one", + "fmt_text", "fmt_empty", "fmt_debug", "cat", "cat_in_place", "grow", "empties", "lookup_default", "aug_ops", "aug_list", +} + + +def build_source(d, source, name="strmod", runtime_dir=None): + path = d / f"{name}.py" + path.write_text(source) + proved, diags = verify.verify(frontend.lower(path)) + c = cgen.generate(proved, path, path.name) + so = cbuild.build(c, path.stem, d / "out", runtime_dir=runtime_dir) + return proved, diags, cbuild.load(so.stem.split(".")[0], so) + + +@dataclasses.dataclass +class Built: + proved: object + diags: list + module: object + + +@pytest.fixture(scope="module") +def built(tmp_path_factory): + d = tmp_path_factory.mktemp("strmod") + proved, diags, m = build_source(d, SOURCE) + return Built(proved, diags, m) + + +def deopts(m): + return m.__typedpython_deopts__ + + +def interp(m, name): + return m.__typedpython_interpreted__[name] + + +def outcome(fn, args): + try: + return ("ok", fn(*args)) + except BaseException as e: # noqa: BLE001 - the type, the message and the args are observable + return ("raise", repr(e)) + + +def state(args): + return repr(args) + + +def differential(m, name, make, expect_deopt=False, logged=False): + """Compiled and interpreted run on two fresh argument lists. With `logged`, `make()` returns + `(args, log)` and the two logs of user-code calls must be equal and not empty.""" + ca, ia = make(), make() + if logged: + (ca, clog), (ia, ilog) = ca, ia + before = deopts(m) + c = outcome(getattr(m, name), ca) + after = deopts(m) + i = outcome(interp(m, name), ia) + assert c == i, (c, i) + assert state(ca) == state(ia), (state(ca), state(ia)) + if logged: + assert clog == ilog and clog, (clog, ilog) + if expect_deopt: + assert after == before + 1, "expected a deopt (the interpreted path), got none" + else: + assert after == before, "unexpected deopt: the compiled path did not run" + return c + + +class K: + """A key whose hash and equality are user code, logging every call. Hash collides on purpose.""" + + def __init__(self, v, log): + self.v, self.log = v, log + + def __hash__(self): + self.log.append(("hash", self.v)) + return hash(self.v) % 3 + + def __eq__(self, other): + self.log.append(("eq", self.v, getattr(other, "v", other))) + return isinstance(other, K) and other.v == self.v + + def __repr__(self): + return f"K({self.v})" + + +class SKey(str): + """A str subclass whose hash and equality are user code.""" + log: list = [] + + def __hash__(self): + SKey.log.append(("hash", str(self))) + return hash(str(self)) + + def __eq__(self, other): + SKey.log.append(("eq", str(self), str(other))) + return str.__eq__(self, other) + + +class EqList: + def __init__(self, v, log): + self.v, self.log = v, log + + def __eq__(self, other): + self.log.append(("eq", self.v, getattr(other, "v", other))) + return getattr(other, "v", other) == self.v + + __hash__ = None + + def __repr__(self): + return f"E({self.v})" + + +class Idx: + def __init__(self, i, log): + self.i, self.log = i, log + + def __index__(self): + self.log.append(("index", self.i)) + return self.i + + def __repr__(self): + return f"Idx({self.i})" + + +class Fmt: + def __init__(self, log): + self.log = log + + def __format__(self, spec): + self.log.append(("format", spec)) + return f"<{spec}>" + + def __repr__(self): + self.log.append(("repr",)) + return "R" + + def __str__(self): + self.log.append(("str",)) + return "S" + + def __eq__(self, other): + return self is other + + __hash__ = object.__hash__ + + +class SubDict(dict): + def __getitem__(self, k): + return ("sub-get", k) + + def __setitem__(self, k, v): + dict.__setitem__(self, k, ("sub-set", v)) + + def __contains__(self, k): + return k == "magic" + + +class MissingDict(dict): + def __missing__(self, k): + return ("missing", k) + + +class SubList(list): + def __getitem__(self, i): + return ("sub-list", i) + + +class SubStr(str): + def __add__(self, other): + return "sub-add" + + def __radd__(self, other): + return "sub-radd" + + +class Three: + def __iter__(self): + return iter([1, 2, 3]) + + def __repr__(self): + return "Three()" + + +class Money: + def __init__(self, v): + self.v = v + + def __add__(self, other): + return ("add", self.v, other) + + def __radd__(self, other): + return ("radd", self.v, other) + + def __repr__(self): + return f"Money({self.v})" + + +def test_compiled_set(built): + names = {f.name for f in built.proved.functions} + assert names >= COMPILED, (sorted(COMPILED - names), + {n: built.proved.skipped.get(n) for n in sorted(COMPILED - names)}) + + +# --- subscript ---------------------------------------------------------------------------------- + +def test_dict_getitem(built): + m = built.module + for key in ("a", "zz", 3, None, (1, 2), 1.0): + differential(m, "get", lambda key=key: [{"a": 1, 3: 4, None: 5, (1, 2): 6, 1: 7}, key]) + differential(m, "get", lambda: [{}, []]) # unhashable key: TypeError + differential(m, "get", lambda: [{}, {"a": 1}]) + differential(m, "get", lambda: [{"a": 1}, SKey("a")]) + + +def test_dict_getitem_user_hash_and_eq_run_the_same_times(built): + m = built.module + for present in ([1], [1, 2, 3, 4], [0, 3, 6, 9]): # colliding hashes: __eq__ runs + for key_value in (1, 3, 5, 9): + def make(present=present, key_value=key_value): + log = [] + d = {K(v, log): v for v in present} + log.clear() + return [d, K(key_value, log)], log + differential(m, "get", make, logged=True) + for same in (True, False): # identity shortcut vs __eq__ + def make(same=same): + log = [] + key = K(1, log) + d = {key: "x", K(4, log): "y"} + log.clear() + return [d, key if same else K(1, log)], log + differential(m, "get", make, logged=True) + + +def test_dict_getitem_subclass_keys_and_dict_subclasses_use_their_own_methods(built): + m = built.module + SKey.log.clear() + differential(m, "get", lambda: [{SKey("a"): 1}, "a"]) + differential(m, "get", lambda: [{"a": 1}, SKey("a")]) + assert SKey.log # the user hash ran + differential(m, "get", lambda: [SubDict({"a": 1}), "a"]) # __getitem__ override + assert m.get(SubDict({"a": 1}), "a") == ("sub-get", "a") + differential(m, "get", lambda: [MissingDict(), "q"]) # __missing__ + assert m.get(MissingDict(), "q") == ("missing", "q") + differential(m, "get_nested", lambda: [{1: {2: 3}}, 1, 2]) + differential(m, "get_nested", lambda: [{1: {2: 3}}, 1, 5]) + differential(m, "get_nested", lambda: [{1: {2: 3}}, 7, 5]) + + +def test_list_and_str_getitem(built): + m = built.module + for i in (0, 1, -1, 2, -3, 3, -4, 100): + differential(m, "get_list", lambda i=i: [[10, 20, 30], i]) + differential(m, "get_str", lambda i=i: ["abc", i]) + differential(m, "get_list", lambda: [SubList([1, 2]), 0]) + differential(m, "get_list", lambda: [(1, 2, 3), 1]) # a tuple is subscriptable too + differential(m, "get_list", lambda: [5, 0]) # TypeError: not subscriptable + differential(m, "get_list", lambda: [None, 0]) + differential(m, "get_list", lambda: [3.5, 0]) + differential(m, "get_list", lambda: [{"a": 1}, 0]) # KeyError(0) + + +def test_getitem_with_user_index(built): + m = built.module + + def make(i, xs=(10, 20, 30)): + def _make(): + log = [] + return [list(xs), Idx(i, log), log] + return _make + for i in (0, 2, -1, 5): + differential(m, "get_index", lambda i=i: make(i)()[:2]) + # the user __index__ call is the same: logged through the argument itself + log_a, log_b = [], [] + a = outcome(m.get_index, [[1, 2, 3], Idx(1, log_a)]) + b = outcome(interp(m, "get_index"), [[1, 2, 3], Idx(1, log_b)]) + assert a == b and log_a == log_b == [("index", 1)] + + +def test_dict_setitem(built): + m = built.module + for key in ("a", "new", 3, None): + differential(m, "put", lambda key=key: [{"a": 1}, key, "v"]) + differential(m, "put", lambda: [{}, [], 1]) # unhashable + differential(m, "put", lambda: [SubDict(), "a", 1]) # __setitem__ override + assert SubDict.__setitem__ is not dict.__setitem__ + d = SubDict() + m.put(d, "a", 1) + assert dict.__getitem__(d, "a") == ("sub-set", 1) + differential(m, "put", lambda: [(1, 2), 0, 1]) # tuple: TypeError + differential(m, "put", lambda: [None, 0, 1]) + differential(m, "put", lambda: [5, 0, 1]) + differential(m, "put_list", lambda: [[1, 2, 3], 1, "x"]) + differential(m, "put_list", lambda: [[1, 2, 3], -3, "x"]) + differential(m, "put_list", lambda: [[1, 2, 3], 3, "x"]) + differential(m, "put_list", lambda: [[1, 2, 3], -4, "x"]) + differential(m, "put_list", lambda: [(1, 2, 3), 0, "x"]) + + +def test_dict_setitem_user_hash_and_eq(built): + m = built.module + for same in (True, False): + def make(same=same): + log = [] + key = K(1, log) + d = {key: "x", K(4, log): "y"} + log.clear() + return [d, key if same else K(1, log), "new"], log + differential(m, "put", make, logged=True) + for v in (7, 4, 10): # new key, colliding key, collision + def make(v=v): + log = [] + d = {K(1, log): 0, K(4, log): 0} + log.clear() + return [d, K(v, log), 1], log + differential(m, "put", make, logged=True) + + +# --- in ---------------------------------------------------------------------------------------- + +def test_contains(built): + m = built.module + for x in (1, 2, 9, None, "a"): + differential(m, "has", lambda x=x: [[1, 2, 3, None, "a"], x]) + differential(m, "has_not", lambda x=x: [[1, 2, 3, None, "a"], x]) + differential(m, "has_dict", lambda x=x: [{1: 1, None: 2, "a": 3}, x]) + differential(m, "has_branch", lambda x=x: [[1, 2, 3], x]) + differential(m, "has_dict", lambda: [{}, []]) # unhashable: TypeError + differential(m, "has", lambda: [5, 1]) # not iterable + differential(m, "has", lambda: [None, 1]) + differential(m, "has_str", lambda: ["hello", "ell"]) + differential(m, "has_str", lambda: ["hello", "xyz"]) + differential(m, "has_str", lambda: ["hello", ""]) + differential(m, "has_dict", lambda: [SubDict(), "magic"]) # __contains__ override + differential(m, "has_dict", lambda: [SubDict(), "other"]) + assert m.has_dict(SubDict(), "magic") is True + differential(m, "has_chain", lambda: [[1, 2], 1, [[1, 2], 5]]) + differential(m, "has_chain", lambda: [[1, 2], 1, [[3], 5]]) + differential(m, "has_chain", lambda: [[1, 2], 9, [[1, 2], 5]]) + + +def test_contains_with_custom_eq_runs_the_same_calls(built): + m = built.module + for needle in (1, 2, 7): + la, lb = [], [] + ca = [EqList(1, la), EqList(2, la), EqList(3, la)] + cb = [EqList(1, lb), EqList(2, lb), EqList(3, lb)] + a = outcome(m.has, [ca, EqList(needle, la)]) + b = outcome(interp(m, "has"), [cb, EqList(needle, lb)]) + assert a == b and la == lb and la, (la, lb) + # identity shortcut: the same object does not call __eq__ (PyObject_RichCompareBool) + la, lb = [], [] + xa, xb = EqList(1, la), EqList(1, lb) + assert outcome(m.has, [[xa], xa]) == outcome(interp(m, "has"), [[xb], xb]) + assert la == lb == [] + nan = float("nan") + assert m.has([nan], nan) is True and interp(m, "has")([nan], nan) is True + + +# --- for --------------------------------------------------------------------------------------- + +def test_for_unpack_and_errors(built): + m = built.module + good = [(1, 2), [3, 4], "ab", {5: 6}.popitem()] + differential(m, "pairs", lambda: [list(good)]) + differential(m, "pairs", lambda: [[]]) + differential(m, "pairs", lambda: [[(1, 2, 3)]]) # too many, exact tuple + differential(m, "pairs", lambda: [[[1, 2, 3]]]) # too many, exact list + differential(m, "pairs", lambda: [["abc"]]) # too many, other type + differential(m, "pairs", lambda: [[{1: 1, 2: 2, 3: 3}]]) # too many, exact dict + differential(m, "pairs", lambda: [[Three()]]) # too many, any iterable + differential(m, "pairs", lambda: [[(1,)]]) # not enough + differential(m, "pairs", lambda: [[()]]) + differential(m, "pairs", lambda: [[""]]) + differential(m, "pairs", lambda: [[5]]) # non-iterable int + differential(m, "pairs", lambda: [[None]]) + differential(m, "pairs", lambda: [[1.5]]) + differential(m, "pairs", lambda: [[(1, 2), (1, 2, 3)]]) # error after one item + differential(m, "triples", lambda: [[(1, 2, 3), (4, 5, 6)]]) + differential(m, "triples", lambda: [[(1, 2)]]) + differential(m, "triples", lambda: [[(1, 2, 3, 4)]]) + differential(m, "triples", lambda: [[{1: 1, 2: 2, 3: 3}]]) + differential(m, "pairs", lambda: [5]) # not iterable: TypeError + differential(m, "pairs", lambda: [None]) + + +def test_for_over_objects_break_continue_return(built): + m = built.module + differential(m, "each", lambda: [[1, None, 2, None, 3]]) + differential(m, "each", lambda: [[]]) + differential(m, "each", lambda: [(1, 2)]) + differential(m, "each", lambda: ["abc"]) + differential(m, "each", lambda: [{1: 1, 2: 2}]) + differential(m, "each", lambda: [range(4)]) + differential(m, "each", lambda: [5]) # TypeError + differential(m, "first_after", lambda: [[1, 2, 3, 4], 3]) + differential(m, "first_after", lambda: [[1, 2, 3, 4], 9]) + differential(m, "return_inside", lambda: [[7, 8]]) + differential(m, "return_inside", lambda: [[]]) + differential(m, "nested", lambda: [[[1, 2, None, 3], [4, None], [5]]]) + differential(m, "nested", lambda: [[[1], 5]]) # inner non-iterable + differential(m, "items_of", lambda: [{"a": 1, "b": 2}]) + differential(m, "items_of", lambda: [{}]) + differential(m, "mutate_while_iterating", lambda: [{"a": 1}]) # RuntimeError: size changed + differential(m, "mutate_while_iterating", lambda: [{}]) + + +def test_for_calls_user_iter_and_next_the_same_times(built): + m = built.module + + class It: + def __init__(self, log, n): + self.log, self.n, self.i = log, n, 0 + + def __iter__(self): + self.log.append("iter") + return self + + def __next__(self): + self.log.append(("next", self.i)) + if self.i >= self.n: + raise StopIteration + self.i += 1 + return self.i + + for n in (0, 3): + la, lb = [], [] + a = outcome(m.each, [It(la, n)]) + b = outcome(interp(m, "each"), [It(lb, n)]) + assert a == b and la == lb and la + + class Boom(It): + def __next__(self): + r = super().__next__() + if r == 2: + raise ValueError("boom") + return r + + la, lb = [], [] + a = outcome(m.each, [Boom(la, 5)]) + b = outcome(interp(m, "each"), [Boom(lb, 5)]) + assert a == b and a[0] == "raise" and la == lb + + +def test_early_exit_releases_the_iterator(built): + m = built.module + died = [] + + class Gen: + def __init__(self, tag): + self.tag = tag + + def __iter__(self): + return iter([1, 2, 3]) + + def __del__(self): + died.append(self.tag) + + class Iter: + def __init__(self, tag): + self.tag = tag + + def __iter__(self): + return self + + def __next__(self): + return 1 + + def __del__(self): + died.append(self.tag) + + for fn, name in ((m.first_after, "c"), (interp(m, "first_after"), "i")): + died.clear() + assert fn(Iter(name), 1) == 1 + gc.collect() + assert died == [name], (name, died) + for fn, name in ((m.return_inside, "c"), (interp(m, "return_inside"), "i")): + died.clear() + fn(Iter(name)) + gc.collect() + assert died == [name] + + +# --- f-strings --------------------------------------------------------------------------------- + +def test_fstring(built): + m = built.module + for x in (1, "s", 2.5, None, True, (1, 2), [1, "a"], b"by", "é", 10**30): + for y, n in ((0.0, 0), (3.14159, 42), (-2.5, -7)): + differential(m, "fmt", lambda x=x, y=y, n=n: [x, y, n]) + differential(m, "fmt_one", lambda: ["str"]) + differential(m, "fmt_one", lambda: [12]) + differential(m, "fmt_one", lambda: [None]) + differential(m, "fmt_text", lambda: [3]) + differential(m, "fmt_empty", lambda: []) + differential(m, "fmt_debug", lambda: [12]) + differential(m, "fmt_debug", lambda: ["q"]) + la, lb = [], [] + a = outcome(m.fmt, [Fmt(la), 1.5, 2]) + b = outcome(interp(m, "fmt"), [Fmt(lb), 1.5, 2]) + assert a == b and la == lb and la, (la, lb) + la, lb = [], [] + outcome(m.fmt_text, [Fmt(la)]) + outcome(interp(m, "fmt_text"), [Fmt(lb)]) + assert la == lb == [("format", ""), ("repr",)] + + +def test_fstring_format_errors(built): + m = built.module + differential(m, "fmt", lambda: [{1: 2}, 1.0, 1]) # bad spec: TypeError + differential(m, "fmt", lambda: [None, 1.0, 1]) + + +# --- str concatenation --------------------------------------------------------------------------- + +def test_str_add(built): + m = built.module + differential(m, "cat", lambda: ["a", "b"]) + differential(m, "cat", lambda: ["", ""]) + differential(m, "cat", lambda: ["é", "ü"]) + differential(m, "cat", lambda: [SubStr("a"), "b"]) # a str subclass: its __add__ + differential(m, "cat", lambda: ["a", SubStr("b")]) # its __radd__ wins (subclass) + assert m.cat(SubStr("a"), "b") == "sub-add" + assert m.cat("a", SubStr("b")) == "sub-radd" + differential(m, "cat", lambda: [1, "b"]) # TypeError text + differential(m, "cat", lambda: ["a", 1]) + differential(m, "cat", lambda: [None, None]) + differential(m, "cat", lambda: [Money(1), "x"]) + differential(m, "cat", lambda: ["x", Money(2)]) + differential(m, "cat", lambda: [b"a", "b"]) + differential(m, "cat", lambda: [1, 2]) # int + int through the object path + differential(m, "cat", lambda: [[1], [2]]) + differential(m, "cat", lambda: [(1,), (2,)]) + differential(m, "cat_in_place", lambda: ["ab", "cd"]) + differential(m, "cat_in_place", lambda: [SubStr("a"), "b"]) + + +def test_augmented_assignment_on_objects_is_in_place(built): + m = built.module + xs, ys = [1], [2] + out = m.grow(xs, ys) + assert out == [1, 2] and xs == [1] and ys == [2] + differential(m, "grow", lambda: [[1], [2]]) + differential(m, "grow", lambda: [(1,), [2]]) # list += tuple extends in place + differential(m, "grow", lambda: [{"a": 1}, "xy"]) # list += dict / str extends too + differential(m, "grow", lambda: [5, [1]]) # TypeError text of the in-place op + differential(m, "grow", lambda: [[1], 5]) + + +def test_every_augmented_operator_on_an_object_is_the_in_place_one(built): + m = built.module + la, lb = [], [] + a, b = m.Acc(la), m.Acc(lb) + before = deopts(m) + assert m.aug_ops(a, 3) is a + assert deopts(m) == before + assert interp(m, "aug_ops")(b, 3) is b + assert la == lb == [("iadd", 3), ("isub", 3), ("imul", 3), ("itruediv", 3), + ("ifloordiv", 3), ("imod", 3)] + xs = ["a", "b"] + assert m.aug_list(xs, 2) is xs and xs == ["a", "b"] * 4 # *= and += extend the same list + differential(m, "aug_list", lambda: [["a", "b"], 3]) + differential(m, "aug_list", lambda: [[], 3]) + differential(m, "aug_list", lambda: [("a", "b"), 2]) # a tuple: *= makes a new one + differential(m, "aug_list", lambda: ["ab", 2]) + + +def test_empty_displays_and_method_calls(built): + m = built.module + differential(m, "empties", lambda: []) + assert m.empties() == [{}] + differential(m, "lookup_default", lambda: [{"a": 5}, "a"]) + differential(m, "lookup_default", lambda: [{"a": 5}, "b"]) + differential(m, "lookup_default", lambda: [{}, "b"]) + + +def test_generated_c_calls_only_documented_helpers(tmp_path): + import re + path = tmp_path / "strmod.py" + path.write_text(SOURCE) + proved, _ = verify.verify(frontend.lower(path)) + c = cgen.generate(proved, path, path.name) + body = re.sub(r"/\*.*?\*/", "", c, flags=re.S) + body = re.sub(r'"(?:\\.|[^"\\])*"', '""', body) + called = {n for n in re.findall(r"\b(tp_[a-z0-9_]+)\s*\(", body)} + api = (cbuild.RUNTIME_DIR / "API.md").read_text() + documented = set(re.findall(r"\b(tp_[a-z0-9_]+)\s*\(", api)) + defined = set(re.findall(r"^static[^\n(]*\b(tp_[a-z0-9_]+)\s*\(", body, flags=re.M)) + for h in ("tp_getitem", "tp_setitem", "tp_contains", "tp_format", "tp_build_string", "tp_unpack", + "tp_get_iter", "tp_iter_next", "tp_new_list", "tp_new_dict"): + assert h in called, f"{h} is not used by the generated C of the test module" + assert sorted(called - documented - defined) == [] + + +# --- no reference leaks -------------------------------------------------------------------------- + +def test_no_reference_is_leaked_or_dropped(built): + m = built.module + key, val, other = object(), object(), object() + obj = type("O", (), {"__eq__": lambda s, o: False, "__hash__": lambda s: 1})() + bag = [obj] + + def churn(): + d = {key: val} + for _ in range(200): + m.get(d, key) + m.put(d, key, val) + m.has([obj], obj) + m.has_not(bag, other) + m.has_dict(d, key) + m.each(bag) + m.pairs([(obj, obj)]) + m.fmt_one(obj) + m.fmt_text(obj) + m.items_of({"a": obj}) + m.grow(bag, bag) + for bad in (lambda: m.get({}, key), lambda: m.pairs([(obj,)]), lambda: m.pairs([(obj,) * 3]), + lambda: m.get_list([obj], 5), lambda: m.put({}, [], obj), + lambda: m.has_dict({}, [obj]), lambda: m.each(5), lambda: m.cat(obj, obj)): + with contextlib.suppress(Exception): + bad() + + churn() + gc.collect() + before = [sys.getrefcount(x) for x in (key, val, other, obj)] + churn() + gc.collect() + after = [sys.getrefcount(x) for x in (key, val, other, obj)] + assert before == after, (before, after) + + +# --- the wordfreq benchmark -------------------------------------------------------------------- + +def run_main(main, n): + old = sys.argv + sys.argv = ["wordfreq", str(n)] + buf = io.StringIO() + try: + with contextlib.redirect_stdout(buf): + main() + finally: + sys.argv = old + return buf.getvalue() + + +@pytest.fixture(scope="module") +def wordfreq(tmp_path_factory): + d = tmp_path_factory.mktemp("wordfreq") + src = "# typedpython: compiled\n" + WORDFREQ.read_text() + proved, diags, m = build_source(d, src, name="wordfreq") + spec = importlib.util.spec_from_file_location("wordfreq_plain", WORDFREQ) + plain = importlib.util.module_from_spec(spec) + spec.loader.exec_module(plain) + return Built(proved, diags, m), plain + + +def test_wordfreq_functions_compile(wordfreq): + built, _ = wordfreq + names = {f.name for f in built.proved.functions} + assert {"next_state", "word_of", "main"} <= names, built.proved.skipped + + +def test_wordfreq_output_equals_cpython(wordfreq): + built, plain = wordfreq + m = built.module + for n in (0, 1, 2, 50, 3000): + compiled = run_main(m.main, n) + assert compiled == run_main(plain.main, n) + assert compiled == run_main(interp(m, "main"), n) + assert deopts(m) == 0 + for i in (0, 1, 25, 26, 17575, 17576, 99999, 10**9): + assert m.word_of(i) == plain.word_of(i) + for bad in (-17576, -20000, True, 1.5, None, "x", 2**70): + assert outcome(m.word_of, [bad]) == outcome(plain.word_of, [bad]) + + +def test_wordfreq_main_n_100000(wordfreq): + built, plain = wordfreq + m = built.module + assert run_main(m.main, 100000) == run_main(interp(m, "main"), 100000) + + +def test_wordfreq_main_errors_equal_cpython(wordfreq): + built, plain = wordfreq + m = built.module + old = sys.argv + try: + for argv in (["w"], ["w", "x"], ["w", "1.5"], ["w", "-3"]): + sys.argv = list(argv) + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + a = outcome(m.main, []) + b = outcome(plain.main, []) + assert a == b, (argv, a, b) + finally: + sys.argv = old + + +# --- the exact-type guards matter --------------------------------------------------------------- + +def mutant_runtime(tmp_path, macro): + """A copy of the runtime header whose fast-path guard `macro` accepts subclasses too.""" + mutant = tmp_path / "runtime" + shutil.copytree(cbuild.RUNTIME_DIR, mutant) + text = (mutant / "tp_runtime.h").read_text() + exact = {"TP_FASTPATH_GUARD_DICT": "PyDict_CheckExact", "TP_FASTPATH_GUARD_STR": "PyUnicode_CheckExact"}[macro] + loose = exact.replace("CheckExact", "Check") + line = f"#define {macro}(o) {exact}(o)" + assert line in text, line + (mutant / "tp_runtime.h").write_text(text.replace(line, f"#define {macro}(o) {loose}(o)")) + return mutant + + +def test_the_fast_path_guards_are_the_single_mutation_points(): + header = (cbuild.RUNTIME_DIR / "tp_runtime.h").read_text() + assert header.count("TP_FASTPATH_GUARD_DICT(") >= 3 # the define and its uses + assert header.count("TP_FASTPATH_GUARD_STR(") >= 2 + + +def test_a_dict_guard_that_accepts_subclasses_is_caught(tmp_path): + """Mutation: the dict fast paths are guarded by the exact type. Built against a header whose + guard is `PyDict_Check`, a subclass overriding __getitem__ / __setitem__ is bypassed, and the + differential tests above fail; here the same module is built both ways to show it.""" + (tmp_path / "good").mkdir() + (tmp_path / "bad").mkdir() + _, _, good = build_source(tmp_path / "good", SOURCE) + _, _, bad = build_source(tmp_path / "bad", SOURCE, + runtime_dir=mutant_runtime(tmp_path, "TP_FASTPATH_GUARD_DICT")) + assert good.get(SubDict({"a": 1}), "a") == ("sub-get", "a") + assert bad.get(SubDict({"a": 1}), "a") == 1 # the mutant skips the override: caught + d, e = SubDict(), SubDict() + good.put(d, "a", 1) + bad.put(e, "a", 1) + assert dict.__getitem__(d, "a") == ("sub-set", 1) + assert dict.__getitem__(e, "a") == 1 + with pytest.raises(AssertionError): + differential(bad, "get", lambda: [SubDict({"a": 1}), "a"]) + + +def test_a_str_guard_that_accepts_subclasses_is_caught(tmp_path): + (tmp_path / "bad").mkdir() + _, _, bad = build_source(tmp_path / "bad", SOURCE, + runtime_dir=mutant_runtime(tmp_path, "TP_FASTPATH_GUARD_STR")) + assert bad.cat(SubStr("a"), "b") == "ab" # the override ("sub-add") is skipped + with pytest.raises(AssertionError): + differential(bad, "cat", lambda: [SubStr("a"), "b"]) diff --git a/python-multiplatform-ksp/src/test/python/test_verify_classes.py b/python-multiplatform-ksp/src/test/python/test_verify_classes.py index 78132918..e08a0413 100644 --- a/python-multiplatform-ksp/src/test/python/test_verify_classes.py +++ b/python-multiplatform-ksp/src/test/python/test_verify_classes.py @@ -513,8 +513,12 @@ def test_the_operand_of_a_field_node_must_be_an_object(): def test_none_is_an_obj_constant_and_nothing_else_is(): rejected(TYPE, getter([If(Is(BOOL, lo("q"), Const(OBJ, 0)), (Return(g_("a")),)), Return(g_("b"))], params=(obj(),))) - rejected(TYPE, getter([If(Is(BOOL, lo("q"), Const(OBJ, "x")), (Return(g_("a")),)), + rejected(TYPE, getter([If(Is(BOOL, lo("q"), Const(OBJ, b"x")), (Return(g_("a")),)), Return(g_("b"))], params=(obj(),))) + # a str is the other OBJ constant (#150); its text must be exactly what the C source carries + accepted(getter([Return(Const(OBJ, "x"))], params=())) + rejected(TYPE, getter([Return(Const(OBJ, "a\0b"))], params=())) + rejected(TYPE, getter([Return(Const(OBJ, "\ud800"))], params=())) # a None constant is a new reference to the singleton wherever an object may flow accepted(getter([Return(none())], params=())) accepted(getter([Assign("t", none()), Return(lo("t"))], params=(), locals_={"t": OBJ}))