From 3275ccf945948f5c9a39e5139a67cc27e793c675 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Fri, 20 Nov 2026 19:40:22 -0800 Subject: [PATCH 1/2] fix(cuda.core): pick one owner when IPC pool registrations of one UUID race Follow-up to #3511 (#3510). Two deferred review items: - MP_register could overwrite a re-entrant registration of the same UUID. A garbage-collection finalizer that registered the same pool between the outer call's lookup and its insert left two owners of one driver pool, with the registry pointing at the loser. Build the weak reference before the insert and claim the slot with dict.setdefault so a re-entrant winner is kept, not overwritten; both calls then return that one owner. A plain dict of weak references replaces the WeakValueDictionary so the wrapper allocation happens outside the check-and-set. A one-shot test hook injects the reentry at the vulnerable point for a deterministic regression test. - The re-import race test now checks read/write access on every GPU the pool grants access to, not only the default one, since the final release revokes access one GPU at a time. --- cuda_core/cuda/core/_memory/_ipc.pyi | 2 + cuda_core/cuda/core/_memory/_ipc.pyx | 80 ++++++++++++++++--- cuda_core/tests/helpers/memory.py | 13 +++ .../test_ipc_pool_duplicate_import.py | 54 ++++++++++++- 4 files changed, 134 insertions(+), 15 deletions(-) diff --git a/cuda_core/cuda/core/_memory/_ipc.pyi b/cuda_core/cuda/core/_memory/_ipc.pyi index ebeb839b058..57707557b93 100644 --- a/cuda_core/cuda/core/_memory/_ipc.pyi +++ b/cuda_core/cuda/core/_memory/_ipc.pyi @@ -215,5 +215,7 @@ class VirtualMemoryFabricBufferDescriptor(_VirtualMemoryDescriptor): def __eq__(self, other) -> bool: ... def __hash__(self) -> int: ... +def _set_register_reentry_hook_for_testing(hook): ... +def _registry_remove(key, ref): ... def _reduce_allocation_handle(alloc_handle: IPCAllocationHandle) -> tuple[object, ...]: ... def _reconstruct_allocation_handle(cls: type, df: object, uuid: uuid.UUID | None) -> IPCAllocationHandle: ... diff --git a/cuda_core/cuda/core/_memory/_ipc.pyx b/cuda_core/cuda/core/_memory/_ipc.pyx index 199af8da6cb..727d86eaf5f 100644 --- a/cuda_core/cuda/core/_memory/_ipc.pyx +++ b/cuda_core/cuda/core/_memory/_ipc.pyx @@ -26,6 +26,7 @@ from cuda.core._rt cimport ( from cuda.core._utils.cuda_utils cimport HANDLE_RETURN from cuda.core._utils.cuda_utils import check_multiprocessing_start_method +import functools import multiprocessing import operator import os @@ -41,12 +42,45 @@ if TYPE_CHECKING: __all__ = [] -cdef object registry = weakref.WeakValueDictionary() +# Maps an IPC UUID to a weak reference to its registered resource. A plain dict +# (not a WeakValueDictionary) so MP_register can build the weak reference before +# the check-and-set below and keep the resulting garbage-collection reentry out +# of the critical section; see MP_register. +cdef dict registry = {} # Serializes MP_register. Reentrant: inserting into the registry allocates a # weak reference, which can run a garbage-collection callback or finalizer that # registers another resource on the same thread. cdef object _registry_lock = threading.RLock() +# Test-only hook. When set, MP_register calls it once it has decided to insert +# but before it builds its own weak reference, so a test can inject a re-entrant +# registration of the same UUID at the point a garbage-collection finalizer +# could. See test_reentrant_registration_of_one_uuid_picks_one_winner. +cdef object _register_reentry_hook_for_testing = None + + +def _set_register_reentry_hook_for_testing(hook): + global _register_reentry_hook_for_testing + _register_reentry_hook_for_testing = hook + + +def _registry_remove(key, ref): + # Weak-reference callback: drop a dead resource's entry, but only when a + # newer registration has not already replaced it under the same UUID. + with _registry_lock: + if registry.get(key) is ref: + del registry[key] + + +cdef _MemPool _registry_get(uuid): + # Resolve a UUID to its live registered resource, or None. Lock-free, like + # WeakValueDictionary.get was: callers that must be authoritative (MP_register) + # re-check under _registry_lock. + ref = registry.get(uuid) + if ref is None: + return None + return ref() + cdef cydriver.CUmemAllocationHandleType IPC_HANDLE_TYPE = \ cydriver.CUmemAllocationHandleType.CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR \ @@ -557,7 +591,7 @@ cdef _MemPool MP_from_allocation_handle( # Quick exit for registry hits. uuid = getattr(alloc_handle, 'uuid', None) # no-cython-lint - mr = registry.get(uuid) + mr = _registry_get(uuid) if mr is not None: _check_registered_type(mr, cls, uuid) MP_check_open(<_MemPool>mr) @@ -608,24 +642,29 @@ cdef _MemPool MP_from_allocation_handle( cdef _MemPool MP_from_registry(uuid): - cdef _MemPool mr - try: - mr = registry[uuid] - MP_check_open(mr) - return mr - except KeyError: + cdef _MemPool mr = _registry_get(uuid) + if mr is None: raise RuntimeError(f"Memory resource {uuid} was not found") from None + MP_check_open(mr) + return mr cdef _MemPool MP_register(_MemPool self, uuid): MP_check_open(self) + cdef object existing_ref + cdef object winner_ref + cdef _MemPool winner # The lookup and the insertion must be one step: without the GIL, two # threads can both miss and register different resources for one UUID. with _registry_lock: - existing = registry.get(uuid) - if existing is not None: - MP_check_open(<_MemPool>existing) - return existing + existing_ref = registry.get(uuid) + if existing_ref is not None: + existing = existing_ref() + if existing is not None: + MP_check_open(<_MemPool>existing) + return existing + # A dead entry whose removal callback has not run yet. + del registry[uuid] if not self.is_ipc_enabled: raise RuntimeError("Memory resource is not IPC-enabled") if self.uuid is not None and self.uuid != uuid: @@ -633,7 +672,22 @@ cdef _MemPool MP_register(_MemPool self, uuid): f"Cannot register memory resource with UUID {uuid}: " f"the resource already has UUID {self.uuid}" ) - registry[uuid] = self + if _register_reentry_hook_for_testing is not None: + _register_reentry_hook_for_testing(uuid) + # Allocating the weak reference can start a garbage collection whose + # finalizers re-enter MP_register on this thread (the lock is + # reentrant). It runs before the insert, so a re-entrant registration + # of the same UUID lands in the registry first; setdefault then keeps + # it instead of overwriting it, and both calls return that one winner. + self_ref = weakref.ref(self, functools.partial(_registry_remove, uuid)) + winner_ref = registry.setdefault(uuid, self_ref) + if winner_ref is not self_ref: + winner = winner_ref() + if winner is not None: + MP_check_open(winner) + return winner + # The re-entrant winner already died; take over its dead entry. + registry[uuid] = self_ref self._ipc_data._alloc_handle._uuid = uuid return self diff --git a/cuda_core/tests/helpers/memory.py b/cuda_core/tests/helpers/memory.py index 506e54c3e94..1a479808cf3 100644 --- a/cuda_core/tests/helpers/memory.py +++ b/cuda_core/tests/helpers/memory.py @@ -63,6 +63,19 @@ def import_pool_resource(cls, device, alloc_handle): return DeviceMemoryResource.from_allocation_handle(device, alloc_handle) +def pool_supported_devices(mr_type, device): + """Devices on which an imported pool of ``mr_type`` grants access in this process. + + A pinned (host-NUMA) pool grants device access to every device that supports + host NUMA memory pools; a device pool grants access to its own device. Keep + this in sync with ``_PMR_grant_device_access``. + """ + if mr_type is PinnedMemoryResource: + return [d for d in Device.get_all_devices() if d.properties.host_numa_memory_pools_supported] + assert mr_type is DeviceMemoryResource, mr_type + return [device] + + def pool_access_flags(mr, device): """The ``CUmemAccess_flags`` that ``device`` has on the memory pool of ``mr`` in this process.""" location = driver.CUmemLocation() diff --git a/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py b/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py index 2c5769106b7..e145c189990 100644 --- a/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py +++ b/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py @@ -22,11 +22,12 @@ import pytest from helpers.child_processes import run_child from helpers.contexts import assert_no_cuda_warning -from helpers.memory import import_pool_resource, is_pool_resource, pool_access_flags +from helpers.memory import import_pool_resource, is_pool_resource, pool_access_flags, pool_supported_devices from cuda.bindings import driver from cuda.core import Buffer, Device, DeviceMemoryResource from cuda.core._memory._buffer import _live_owned_mempool_count +from cuda.core._memory._ipc import _set_register_reentry_hook_for_testing from cuda.core._rt import _set_ipc_pool_release_delay_for_testing NBYTES = 64 @@ -144,8 +145,11 @@ def reimport(): raise errors[0] # The release has finished: its cleanup must have left the new import alone. + # The final release revokes access one GPU at a time, so check every GPU the + # pool grants access to, not only the default one. ((mr, buffer),) = reimported - assert pool_access_flags(mr, device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE + for access_device in pool_supported_devices(mr_type, device): + assert pool_access_flags(mr, access_device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE buffer.fill(7, stream=stream) stream.sync() buffer.close() @@ -205,6 +209,47 @@ def register_during_collection(phase, info): assert _live_owned_mempool_count() == baseline +def child_register_reentrant_same_uuid(mr_type, alloc_handle): + device = Device() + device.set_current() + baseline = _live_owned_mempool_count() + + # Two resources that both claim one UUID: the outer call and a re-entrant + # call that the hook injects between the outer call's lookup and insertion, + # where a garbage-collection finalizer could. They share one pool handle, so + # both are valid owners; registration must pick exactly one winner, and the + # outer call must return that winner rather than overwrite it. + first = import_pool_resource(mr_type, device, int(alloc_handle)) + second = import_pool_resource(mr_type, device, int(alloc_handle)) + + target_uuid = uuid.uuid4() + reentrant = [] + + def inject_reentrant_registration(hooked_uuid): + # Fire once: disarm before re-entering so the nested registration does + # not recurse through the hook. + _set_register_reentry_hook_for_testing(None) + reentrant.append(second.register(hooked_uuid)) + + _set_register_reentry_hook_for_testing(inject_reentrant_registration) + try: + outer = first.register(target_uuid) + finally: + _set_register_reentry_hook_for_testing(None) + + assert reentrant, "the reentry hook did not fire" + winner = reentrant[0] + assert winner is first or winner is second + # Both calls agree on the winner, and the registry holds it, so no second + # owner of the pool survives under this UUID. + assert outer is winner + assert mr_type.from_registry(target_uuid) is winner + + first.close() + second.close() + assert _live_owned_mempool_count() == baseline + + def _import_concurrently(mr_type, device, alloc_handle): barrier = threading.Barrier(THREADS) results = [None] * THREADS @@ -303,6 +348,11 @@ def test_registration_reentered_from_garbage_collection(self, ipc_device, pool_r ipc_device.set_current() run_child(child_register_reentered_by_gc, type(pool_resource), pool_resource.allocation_handle) + @pytest.mark.agent_authored(model="claude-opus-5-5") + def test_reentrant_registration_of_one_uuid_picks_one_winner(self, ipc_device, pool_resource): + ipc_device.set_current() + run_child(child_register_reentrant_same_uuid, type(pool_resource), pool_resource.allocation_handle) + @pytest.mark.agent_authored(model="claude-opus-5-5") def test_concurrent_imports_of_one_uuid_share_a_resource(self, ipc_device, pool_resource): ipc_device.set_current() From e04e2786fc6b7827f8fe8f14f1628e04c7af7df3 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Fri, 20 Nov 2026 20:01:22 -0800 Subject: [PATCH 2/2] fix(cuda.core): keep the IPC registry a WeakValueDictionary The previous commit replaced the registry with a plain dict of weak references whose removal callback took _registry_lock. That callback runs wherever the last reference to a registered resource is dropped, including in garbage collection on any thread, so dropping one now waited for any registration in progress on another thread. A thread holding an application lock while dropping a registered resource deadlocked with a registration whose garbage-collection finalizer took the same lock. WeakValueDictionary removes entries without a lock, through an atomic helper, for exactly this reason. Restore the WeakValueDictionary and its lookups unchanged. MP_register now marks the resource it is inserting, under _registry_lock, before it inserts: a registration of the same UUID re-entered from a garbage-collection callback returns the marked resource instead of inserting its own, and one that completed before the mark is found by a second lookup. --- cuda_core/cuda/core/_memory/_ipc.pyi | 1 - cuda_core/cuda/core/_memory/_ipc.pyx | 102 +++++++----------- .../test_ipc_pool_duplicate_import.py | 8 +- 3 files changed, 45 insertions(+), 66 deletions(-) diff --git a/cuda_core/cuda/core/_memory/_ipc.pyi b/cuda_core/cuda/core/_memory/_ipc.pyi index 57707557b93..06599a3e656 100644 --- a/cuda_core/cuda/core/_memory/_ipc.pyi +++ b/cuda_core/cuda/core/_memory/_ipc.pyi @@ -216,6 +216,5 @@ class VirtualMemoryFabricBufferDescriptor(_VirtualMemoryDescriptor): def __hash__(self) -> int: ... def _set_register_reentry_hook_for_testing(hook): ... -def _registry_remove(key, ref): ... def _reduce_allocation_handle(alloc_handle: IPCAllocationHandle) -> tuple[object, ...]: ... def _reconstruct_allocation_handle(cls: type, df: object, uuid: uuid.UUID | None) -> IPCAllocationHandle: ... diff --git a/cuda_core/cuda/core/_memory/_ipc.pyx b/cuda_core/cuda/core/_memory/_ipc.pyx index 727d86eaf5f..f73f6e198a3 100644 --- a/cuda_core/cuda/core/_memory/_ipc.pyx +++ b/cuda_core/cuda/core/_memory/_ipc.pyx @@ -26,7 +26,6 @@ from cuda.core._rt cimport ( from cuda.core._utils.cuda_utils cimport HANDLE_RETURN from cuda.core._utils.cuda_utils import check_multiprocessing_start_method -import functools import multiprocessing import operator import os @@ -42,20 +41,19 @@ if TYPE_CHECKING: __all__ = [] -# Maps an IPC UUID to a weak reference to its registered resource. A plain dict -# (not a WeakValueDictionary) so MP_register can build the weak reference before -# the check-and-set below and keep the resulting garbage-collection reentry out -# of the critical section; see MP_register. -cdef dict registry = {} +cdef object registry = weakref.WeakValueDictionary() # Serializes MP_register. Reentrant: inserting into the registry allocates a # weak reference, which can run a garbage-collection callback or finalizer that # registers another resource on the same thread. cdef object _registry_lock = threading.RLock() - -# Test-only hook. When set, MP_register calls it once it has decided to insert -# but before it builds its own weak reference, so a test can inject a re-entrant -# registration of the same UUID at the point a garbage-collection finalizer -# could. See test_reentrant_registration_of_one_uuid_picks_one_winner. +# The resource each in-progress MP_register call is inserting, by UUID. Accessed +# only under _registry_lock, so only the thread inserting it sees an entry. +cdef dict _registering = {} + +# Test-only hook. When set, MP_register calls it right before it inserts into the +# registry, so a test can register the same UUID re-entrantly at the point where +# a garbage-collection finalizer could. See +# test_reentrant_registration_of_one_uuid_picks_one_winner. cdef object _register_reentry_hook_for_testing = None @@ -64,24 +62,6 @@ def _set_register_reentry_hook_for_testing(hook): _register_reentry_hook_for_testing = hook -def _registry_remove(key, ref): - # Weak-reference callback: drop a dead resource's entry, but only when a - # newer registration has not already replaced it under the same UUID. - with _registry_lock: - if registry.get(key) is ref: - del registry[key] - - -cdef _MemPool _registry_get(uuid): - # Resolve a UUID to its live registered resource, or None. Lock-free, like - # WeakValueDictionary.get was: callers that must be authoritative (MP_register) - # re-check under _registry_lock. - ref = registry.get(uuid) - if ref is None: - return None - return ref() - - cdef cydriver.CUmemAllocationHandleType IPC_HANDLE_TYPE = \ cydriver.CUmemAllocationHandleType.CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR \ if platform.system() == "Linux" else \ @@ -591,7 +571,7 @@ cdef _MemPool MP_from_allocation_handle( # Quick exit for registry hits. uuid = getattr(alloc_handle, 'uuid', None) # no-cython-lint - mr = _registry_get(uuid) + mr = registry.get(uuid) if mr is not None: _check_registered_type(mr, cls, uuid) MP_check_open(<_MemPool>mr) @@ -642,29 +622,28 @@ cdef _MemPool MP_from_allocation_handle( cdef _MemPool MP_from_registry(uuid): - cdef _MemPool mr = _registry_get(uuid) - if mr is None: + cdef _MemPool mr + try: + mr = registry[uuid] + MP_check_open(mr) + return mr + except KeyError: raise RuntimeError(f"Memory resource {uuid} was not found") from None - MP_check_open(mr) - return mr cdef _MemPool MP_register(_MemPool self, uuid): MP_check_open(self) - cdef object existing_ref - cdef object winner_ref - cdef _MemPool winner # The lookup and the insertion must be one step: without the GIL, two # threads can both miss and register different resources for one UUID. with _registry_lock: - existing_ref = registry.get(uuid) - if existing_ref is not None: - existing = existing_ref() - if existing is not None: - MP_check_open(<_MemPool>existing) - return existing - # A dead entry whose removal callback has not run yet. - del registry[uuid] + existing = registry.get(uuid) + if existing is None: + # A garbage-collection callback re-entered MP_register while this + # thread is inserting another resource under this UUID. + existing = _registering.get(uuid) + if existing is not None: + MP_check_open(<_MemPool>existing) + return existing if not self.is_ipc_enabled: raise RuntimeError("Memory resource is not IPC-enabled") if self.uuid is not None and self.uuid != uuid: @@ -672,22 +651,23 @@ cdef _MemPool MP_register(_MemPool self, uuid): f"Cannot register memory resource with UUID {uuid}: " f"the resource already has UUID {self.uuid}" ) - if _register_reentry_hook_for_testing is not None: - _register_reentry_hook_for_testing(uuid) - # Allocating the weak reference can start a garbage collection whose - # finalizers re-enter MP_register on this thread (the lock is - # reentrant). It runs before the insert, so a re-entrant registration - # of the same UUID lands in the registry first; setdefault then keeps - # it instead of overwriting it, and both calls return that one winner. - self_ref = weakref.ref(self, functools.partial(_registry_remove, uuid)) - winner_ref = registry.setdefault(uuid, self_ref) - if winner_ref is not self_ref: - winner = winner_ref() - if winner is not None: - MP_check_open(winner) - return winner - # The re-entrant winner already died; take over its dead entry. - registry[uuid] = self_ref + # The registry and uuid.UUID run Python code, where a collection can + # start and its callbacks can register this UUID on this thread. Once + # this resource is marked, such a registration returns it instead of + # inserting its own, which the insertion below would overwrite. One + # that completed before the mark is found by the second lookup. + _registering[uuid] = self + try: + existing = registry.get(uuid) + if existing is None: + if _register_reentry_hook_for_testing is not None: + _register_reentry_hook_for_testing(uuid) + registry[uuid] = self + finally: + del _registering[uuid] + if existing is not None: + MP_check_open(<_MemPool>existing) + return existing self._ipc_data._alloc_handle._uuid = uuid return self diff --git a/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py b/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py index e145c189990..dc4aa16c792 100644 --- a/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py +++ b/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py @@ -215,10 +215,10 @@ def child_register_reentrant_same_uuid(mr_type, alloc_handle): baseline = _live_owned_mempool_count() # Two resources that both claim one UUID: the outer call and a re-entrant - # call that the hook injects between the outer call's lookup and insertion, - # where a garbage-collection finalizer could. They share one pool handle, so - # both are valid owners; registration must pick exactly one winner, and the - # outer call must return that winner rather than overwrite it. + # call that the hook injects right before the outer call's insertion, where + # a garbage-collection finalizer could. They share one pool handle, so both + # are valid owners; registration must pick exactly one of them, and both + # calls must return it. first = import_pool_resource(mr_type, device, int(alloc_handle)) second = import_pool_resource(mr_type, device, int(alloc_handle))