diff --git a/docs/SPEC.md b/docs/SPEC.md index e0f0e6eb..c7e4256d 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -753,13 +753,15 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. without being passed it. At such a call every live virtual object is materialised, in allocation order. `is`, `type()` and `isinstance` on virtual objects are decided statically. An increment/decrement pair is elided only when another owned reference keeps the object alive throughout and no unknown code runs in - between, so every refcount that can be observed is exact. **On free-threaded 3.15t (no GIL) both - borrowed field reads and pair elision apply only to objects proven thread-local** (never escaped to a - global, a container, another thread or unknown code; #193, decision D5): another thread could otherwise - write the slot and free the child between the read and its use. Every other field read takes a real - reference, or reads inside CPython's per-object critical section (`Py_BEGIN_CRITICAL_SECTION`) when a + between, so every refcount that can be observed is exact. **On free-threaded 3.15t (no GIL), the only build (#158), no real object is thread-local** (#143 step 2, + #193, decision D5 revisited 2026-10-25): even an object the function has just allocated and never passed + anywhere is reachable from another thread through `gc.get_objects()` or `gc.get_referrers()`, which can + replace its fields and free the old value between a borrowed read and its use. So every field read of a + real object takes a strong reference (`PyMember_GetOne`) and no increment/decrement pair on a real object + is elided; borrowed reads and pair elision apply only to N-12a virtual objects, which have no object a + collector can find. There is no separate GIL-build path. A field read takes a real reference, or reads inside CPython's per-object critical section (`Py_BEGIN_CRITICAL_SECTION`) when a sequence of reads must be consistent. A free-threaded stress test (one thread reading a field, another - replacing it) under ASan/TSan is part of N-12's tests. Virtual objects whose count is unbounded live in + replacing it) under ASan/TSan is part of N-12's tests. A second one has a thread find a function's freshly allocated object through `gc.get_objects()` and replace its field while the function reads it; a borrowed-read mutant of the field read must fail it. Virtual objects whose count is unbounded live in a region tied to one activation (the innermost scope that bounds their lifetime) and are freed together; real objects are never placed in a region. A failed region allocation raises `MemoryError` where CPython would. The verifier proves non-escape, materialisation before every observation point, a keeping diff --git a/python-multiplatform-ksp/src/test/python/test_free_threaded.py b/python-multiplatform-ksp/src/test/python/test_free_threaded.py index 1452afc2..7e9ead81 100644 --- a/python-multiplatform-ksp/src/test/python/test_free_threaded.py +++ b/python-multiplatform-ksp/src/test/python/test_free_threaded.py @@ -68,6 +68,20 @@ def mk(a: object, b: object) -> Cell: return Cell(a, b) +def fresh(a: object, n: int) -> object: + c = Cell(a, None) + keep: object = c # read as a value: c is a real object, not virtual (N-12a) + s: object = None + for i in range(n): + s = c.a + return s + + +def vpick(a: object, b: object) -> object: + c = Cell(a, b) + return c.b + + def total(xs: list[float], rounds: int) -> float: s: float = 0.0 for r in range(rounds): @@ -165,8 +179,10 @@ def built(tmp_path_factory): assert result.extension is not None, result.skipped compiled = {f.name for f in result.module.functions} wanted = {"geta", "seta", "mk", "total", "fill", "edge", "fill_int", "edge_int", "rotate", "zero2", - "pair_edge", "usek"} + "pair_edge", "usek", "fresh", "vpick"} assert wanted <= compiled, {n: result.skipped.get(n) for n in wanted - compiled} + fns = {f.name: f for f in result.module.functions} + assert fns["vpick"].virtual == ("c",) and fns["fresh"].virtual == () # #143: one virtual, one real return result, path, d @@ -667,8 +683,8 @@ def reader(): caught = False for _ in range(3): try: - r = run_child(so, body) - except AssertionError: # the child crashed on a freed object: caught + r = run_child(so, body, timeout=40) + except (AssertionError, subprocess.TimeoutExpired): # crashed or hung on a freed object: caught caught = True break if r["bad"] or r["errs"]: @@ -677,6 +693,85 @@ def reader(): assert caught, "the mutant (borrowed field read) was not caught" +@pytest.mark.race_mutant +def test_mutant_with_a_borrowed_read_of_a_fresh_object_is_caught(built, tmp_path): + """#143 step 2, #193 (SPEC N-12): a real object the function has just allocated and never + passed anywhere is still not thread-local on 3.15t, because another thread reaches it through + gc.get_objects() and replaces its fields. Its field reads must take a strong reference too; the + borrowed-read mutant of tp_field_get must be caught by this test. + + gc.get_objects() stops the world, so the compiled loop must reach a safepoint. A loop polls the + eval breaker once per TP_POLL_INTERVAL chunks of TP_POLL_INTERVAL iterations (4096 * 4096 = 16.7M + iterations by default), which is too rare to find an object in a short test. Both builds below + (the real header and the mutant) therefore use TP_POLL_INTERVAL 32, a poll every 1024 iterations.""" + result, path, _ = built + h = (cbuild.RUNTIME_DIR / "tp_runtime.h").read_text() + good = " return PyMember_GetOne((const char *)obj, m);\n" + assert good in h + racy = (" PyObject *v = *(PyObject *volatile *)((char *)obj + m->offset);\n" + " for (volatile int i = 0; i < 200; i++) { }\n" + " return Py_NewRef(v);\n") + frequent = h.replace("#define TP_POLL_INTERVAL 4096", "#define TP_POLL_INTERVAL 32") + assert frequent != h + c_text = cgen.generate(result.module, path) + rt_real = tmp_path / "rt_real" + rt_real.mkdir() + (rt_real / "tp_runtime.h").write_text(frequent) + real = cbuild.build(c_text, "ftmod", tmp_path / "real", runtime_dir=rt_real) + rt_mut = tmp_path / "rt_mut" + rt_mut.mkdir() + (rt_mut / "tp_runtime.h").write_text(frequent.replace(good, racy)) + so = cbuild.build(c_text, "ftmod", tmp_path / "mut", runtime_dir=rt_mut) + body = """ + import gc + stop = False + bad = [] + hits = [0] + def ok(v): + return (type(v) is tuple and len(v) == 2 and v[0] == "fresh" + and type(v[1]) is list and len(v[1]) == 1 and type(v[1][0]) is int) + def scavenger(): + # gc.get_objects() stops the world; fresh() polls every 4096 loop iterations, so it can. + # Once the Cell is found, keep replacing its field (and recycling the freed memory with + # other shapes) for as long as fresh() is reading it. + while not stop: + for o in gc.get_objects(): + if type(o) is m.Cell and type(o.a) is tuple and o.a and o.a[0] == "fresh": + hits[0] += 1 + # fresh() holds two references (l_c, l_keep): write until it has returned + r0 = sys.getrefcount(o) + t_end = time.time() + 5 + k = 0 + while sys.getrefcount(o) > r0 - 2 and time.time() < t_end: + o.a = ("fresh", [k]) + junk = [("x", k), {k: k}, [k, k], "s%d" % k] + k += 1 + def reader(): + for _ in range(2 * STRESS): + v = m.fresh(("fresh", [0]), NITER) + if not ok(v): + bad.append(1) + st = threading.Thread(target=scavenger); st.start() + errs = run_threads([reader] * 3) + stop = True + st.join() + print(json.dumps({"bad": len(bad), "errs": len(errs), "found": hits[0] > 0})) + """ + r = run_child(real, body.replace("NITER", "20_000_000")) + assert r == {"bad": 0, "errs": 0, "found": True}, r # the real read is sound, and the object was reachable + caught = False + for _ in range(5): # a crash on a freed object counts as caught + try: + r = run_child(so, body.replace("NITER", "300_000"), timeout=40) + except (AssertionError, subprocess.TimeoutExpired): # crashed or hung on a freed object: caught + caught = True + break + if r["bad"] or r["errs"]: + caught = True + break + assert caught, "the mutant (borrowed read of a fresh real object) was not caught" + + # --- ThreadSanitizer (best effort) -------------------------------------------------------------------- # The interpreter is not instrumented, so TSan sees races between two accesses made by generated C or # the header (the epoch, the class state) but NOT a missing list lock (the list is written by CPython). @@ -686,7 +781,7 @@ def reader(): stop = False def hammer(): while not stop: - c = m.mk(1, 2); m.geta(cell); m.seta(cell, 5) + c = m.mk(1, 2); m.geta(cell); m.seta(cell, 5); m.vpick(c, cell); m.fresh(c, 3) ht = threading.Thread(target=lambda: run_threads([hammer] * 3)); ht.start() orig = m.Cell for i in range(300):