N-12 step 2 decision: borrow only virtual objects, gc race test (#143, #193)

docs/SPEC.md

@@ -753,13 +753,15 @@ All in `PM/commonTest`, so they run wherever the interpreter loads.
     without being passed it. At such a call every live virtual object is materialised, in allocation order.
   `is`, `type()` and `isinstance` on virtual objects are decided statically. An increment/decrement pair is
   elided only when another owned reference keeps the object alive throughout and no unknown code runs in
-  between, so every refcount that can be observed is exact. **On free-threaded 3.15t (no GIL) both
-  borrowed field reads and pair elision apply only to objects proven thread-local** (never escaped to a
-  global, a container, another thread or unknown code; #193, decision D5): another thread could otherwise
-  write the slot and free the child between the read and its use. Every other field read takes a real
-  reference, or reads inside CPython's per-object critical section (`Py_BEGIN_CRITICAL_SECTION`) when a
+  between, so every refcount that can be observed is exact. **On free-threaded 3.15t (no GIL), the only build (#158), no real object is thread-local** (#143 step 2,
+  #193, decision D5 revisited 2026-10-25): even an object the function has just allocated and never passed
+  anywhere is reachable from another thread through `gc.get_objects()` or `gc.get_referrers()`, which can
+  replace its fields and free the old value between a borrowed read and its use. So every field read of a
+  real object takes a strong reference (`PyMember_GetOne`) and no increment/decrement pair on a real object
+  is elided; borrowed reads and pair elision apply only to N-12a virtual objects, which have no object a
+  collector can find. There is no separate GIL-build path. A field read takes a real reference, or reads inside CPython's per-object critical section (`Py_BEGIN_CRITICAL_SECTION`) when a
   sequence of reads must be consistent. A free-threaded stress test (one thread reading a field, another
-  replacing it) under ASan/TSan is part of N-12's tests. Virtual objects whose count is unbounded live in
+  replacing it) under ASan/TSan is part of N-12's tests. A second one has a thread find a function's freshly allocated object through `gc.get_objects()` and replace its field while the function reads it; a borrowed-read mutant of the field read must fail it. Virtual objects whose count is unbounded live in
   a region tied to one activation (the innermost scope that bounds their lifetime) and are freed together;
   real objects are never placed in a region. A failed region allocation raises `MemoryError` where CPython
   would. The verifier proves non-escape, materialisation before every observation point, a keeping

python-multiplatform-ksp/src/test/python/test_free_threaded.py

@@ -68,6 +68,20 @@ def mk(a: object, b: object) -> Cell:
     return Cell(a, b)
 
 
+def fresh(a: object, n: int) -> object:
+    c = Cell(a, None)
+    keep: object = c                    # read as a value: c is a real object, not virtual (N-12a)
+    s: object = None
+    for i in range(n):
+        s = c.a
+    return s
+
+
+def vpick(a: object, b: object) -> object:
+    c = Cell(a, b)
+    return c.b
+
+
 def total(xs: list[float], rounds: int) -> float:
     s: float = 0.0
     for r in range(rounds):
@@ -165,8 +179,10 @@ def built(tmp_path_factory):
     assert result.extension is not None, result.skipped
     compiled = {f.name for f in result.module.functions}
     wanted = {"geta", "seta", "mk", "total", "fill", "edge", "fill_int", "edge_int", "rotate", "zero2",
-              "pair_edge", "usek"}
+              "pair_edge", "usek", "fresh", "vpick"}
     assert wanted <= compiled, {n: result.skipped.get(n) for n in wanted - compiled}
+    fns = {f.name: f for f in result.module.functions}
+    assert fns["vpick"].virtual == ("c",) and fns["fresh"].virtual == ()   # #143: one virtual, one real
     return result, path, d
 
 
@@ -667,8 +683,8 @@ def reader():
     caught = False
     for _ in range(3):
         try:
-            r = run_child(so, body)
-        except AssertionError:          # the child crashed on a freed object: caught
+            r = run_child(so, body, timeout=40)
+        except (AssertionError, subprocess.TimeoutExpired):   # crashed or hung on a freed object: caught
             caught = True
             break
         if r["bad"] or r["errs"]:
@@ -677,6 +693,85 @@ def reader():
     assert caught, "the mutant (borrowed field read) was not caught"
 
 
+@pytest.mark.race_mutant
+def test_mutant_with_a_borrowed_read_of_a_fresh_object_is_caught(built, tmp_path):
+    """#143 step 2, #193 (SPEC N-12): a real object the function has just allocated and never
+    passed anywhere is still not thread-local on 3.15t, because another thread reaches it through
+    gc.get_objects() and replaces its fields. Its field reads must take a strong reference too; the
+    borrowed-read mutant of tp_field_get must be caught by this test.
+
+    gc.get_objects() stops the world, so the compiled loop must reach a safepoint. A loop polls the
+    eval breaker once per TP_POLL_INTERVAL chunks of TP_POLL_INTERVAL iterations (4096 * 4096 = 16.7M
+    iterations by default), which is too rare to find an object in a short test. Both builds below
+    (the real header and the mutant) therefore use TP_POLL_INTERVAL 32, a poll every 1024 iterations."""
+    result, path, _ = built
+    h = (cbuild.RUNTIME_DIR / "tp_runtime.h").read_text()
+    good = "    return PyMember_GetOne((const char *)obj, m);\n"
+    assert good in h
+    racy = ("    PyObject *v = *(PyObject *volatile *)((char *)obj + m->offset);\n"
+            "    for (volatile int i = 0; i < 200; i++) { }\n"
+            "    return Py_NewRef(v);\n")
+    frequent = h.replace("#define TP_POLL_INTERVAL 4096", "#define TP_POLL_INTERVAL 32")
+    assert frequent != h
+    c_text = cgen.generate(result.module, path)
+    rt_real = tmp_path / "rt_real"
+    rt_real.mkdir()
+    (rt_real / "tp_runtime.h").write_text(frequent)
+    real = cbuild.build(c_text, "ftmod", tmp_path / "real", runtime_dir=rt_real)
+    rt_mut = tmp_path / "rt_mut"
+    rt_mut.mkdir()
+    (rt_mut / "tp_runtime.h").write_text(frequent.replace(good, racy))
+    so = cbuild.build(c_text, "ftmod", tmp_path / "mut", runtime_dir=rt_mut)
+    body = """
+        import gc
+        stop = False
+        bad = []
+        hits = [0]
+        def ok(v):
+            return (type(v) is tuple and len(v) == 2 and v[0] == "fresh"
+                    and type(v[1]) is list and len(v[1]) == 1 and type(v[1][0]) is int)
+        def scavenger():
+            # gc.get_objects() stops the world; fresh() polls every 4096 loop iterations, so it can.
+            # Once the Cell is found, keep replacing its field (and recycling the freed memory with
+            # other shapes) for as long as fresh() is reading it.
+            while not stop:
+                for o in gc.get_objects():
+                    if type(o) is m.Cell and type(o.a) is tuple and o.a and o.a[0] == "fresh":
+                        hits[0] += 1
+                        # fresh() holds two references (l_c, l_keep): write until it has returned
+                        r0 = sys.getrefcount(o)
+                        t_end = time.time() + 5
+                        k = 0
+                        while sys.getrefcount(o) > r0 - 2 and time.time() < t_end:
+                            o.a = ("fresh", [k])
+                            junk = [("x", k), {k: k}, [k, k], "s%d" % k]
+                            k += 1
+        def reader():
+            for _ in range(2 * STRESS):
+                v = m.fresh(("fresh", [0]), NITER)
+                if not ok(v):
+                    bad.append(1)
+        st = threading.Thread(target=scavenger); st.start()
+        errs = run_threads([reader] * 3)
+        stop = True
+        st.join()
+        print(json.dumps({"bad": len(bad), "errs": len(errs), "found": hits[0] > 0}))
+    """
+    r = run_child(real, body.replace("NITER", "20_000_000"))
+    assert r == {"bad": 0, "errs": 0, "found": True}, r   # the real read is sound, and the object was reachable
+    caught = False
+    for _ in range(5):          # a crash on a freed object counts as caught
+        try:
+            r = run_child(so, body.replace("NITER", "300_000"), timeout=40)
+        except (AssertionError, subprocess.TimeoutExpired):   # crashed or hung on a freed object: caught
+            caught = True
+            break
+        if r["bad"] or r["errs"]:
+            caught = True
+            break
+    assert caught, "the mutant (borrowed read of a fresh real object) was not caught"
+
+
 # --- ThreadSanitizer (best effort) --------------------------------------------------------------------
 # The interpreter is not instrumented, so TSan sees races between two accesses made by generated C or
 # the header (the epoch, the class state) but NOT a missing list lock (the list is written by CPython).
@@ -686,7 +781,7 @@ def reader():
 stop = False
 def hammer():
     while not stop:
-        c = m.mk(1, 2); m.geta(cell); m.seta(cell, 5)
+        c = m.mk(1, 2); m.geta(cell); m.seta(cell, 5); m.vpick(c, cell); m.fresh(c, 3)
 ht = threading.Thread(target=lambda: run_threads([hammer] * 3)); ht.start()
 orig = m.Cell
 for i in range(300):