From c41a3e5729d2c8464f48e9d3053fd1372570b606 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Sat, 21 Nov 2026 07:26:53 -0800 Subject: [PATCH 1/3] test(cuda.core): cover re-importing a pinned pool after a release with a pending free Releasing the last import of a pool while an unsynchronized cuMemFreeAsync is still pending leaves the driver holding the pool. The next import returns the same CUmemoryPool, and its first cuMemPoolSetAccess fails with CUDA_ERROR_ALREADY_MAPPED. The retry added in #3511 handles this, but no test exercised the sequence. The new test imports a pool, fills a buffer, closes the buffer without synchronizing, releases every resource, and imports the pool again. It asserts that the re-import grants read/write access, that a buffer imported through it can be filled, and that no CUDAWarning is issued. It covers a single import, several imports of one raw handle, and concurrent imports of one UUID. Without the retry, the three PinnedMemoryResource cases fail with CUDA_ERROR_ALREADY_MAPPED. The DeviceMemoryResource cases pass either way, since a device pool import grants no access. Closes #3514. Signed-off-by: Ralf Juengling --- .../test_ipc_pool_duplicate_import.py | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py b/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py index 2c5769106b7..6f7d4eb4961 100644 --- a/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py +++ b/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py @@ -36,6 +36,7 @@ RELEASE_DELAY_US = 50_000 RACE_ITERATIONS = 3 REENTRY_ITERATIONS = 20 +REIMPORT_ROUNDS = 3 # these tests spawn new processes and files which fails for very many threads; # child processes and IPC fixtures are shared across workers (#2784); cap at two threads @@ -250,6 +251,60 @@ def child_concurrent_uuid_imports(mr_type, alloc_handle): assert _live_owned_mempool_count() == baseline +def _import_by_path(import_path, mr_type, device, alloc_handle): + """Import the pool as ``import_path`` names; returns every resource the import made.""" + if import_path == "single": + return [import_pool_resource(mr_type, device, alloc_handle)] + if import_path == "raw_handle": + # A raw file descriptor carries no UUID, so each import is a real import. + return [import_pool_resource(mr_type, device, int(alloc_handle)) for _ in range(THREADS)] + assert import_path == "concurrent_uuid", import_path + return _import_concurrently(mr_type, device, alloc_handle) + + +def _import_fill_and_release_unsynchronized(import_path, mr_type, device, alloc_handle, descriptor, stream): + resources = _import_by_path(import_path, mr_type, device, alloc_handle) + buffers = [Buffer.from_ipc_descriptor(mr, descriptor, stream=stream) for mr in resources] + for buffer in buffers: + buffer.fill(7, stream=stream) + stream.sync() + # Closing a buffer enqueues its cuMemFreeAsync without waiting for it, so + # the frees can still be pending when the last resource releases the pool. + for buffer in buffers: + buffer.close() + for mr in resources: + mr.close() + + +def _reimport_and_use(import_path, mr_type, device, alloc_handle, descriptor, stream): + resources = _import_by_path(import_path, mr_type, device, alloc_handle) + for mr in resources: + assert pool_access_flags(mr, device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE + buffer = Buffer.from_ipc_descriptor(resources[0], descriptor, stream=stream) + buffer.fill(9, stream=stream) + stream.sync() + buffer.close() + for mr in resources: + mr.close() + + +def child_reimport_after_release_with_pending_free(mr_type, alloc_handle, descriptor, import_path): + device = Device() + device.set_current() + stream = device.default_stream + baseline = _live_owned_mempool_count() + + # A failed grant is also reported as a warning by the cleanup that follows it. + with assert_no_cuda_warning(): + for _ in range(REIMPORT_ROUNDS): + # The helpers keep no reference to any resource once they return, so + # the registry entry is gone and the next import is a real one. + _import_fill_and_release_unsynchronized(import_path, mr_type, device, alloc_handle, descriptor, stream) + _reimport_and_use(import_path, mr_type, device, alloc_handle, descriptor, stream) + stream.sync() + assert _live_owned_mempool_count() == baseline + + class TestIpcPoolDuplicateImport: """Repeat imports of one pool in a process share the pool and its access settings.""" @@ -307,3 +362,17 @@ def test_registration_reentered_from_garbage_collection(self, ipc_device, pool_r def test_concurrent_imports_of_one_uuid_share_a_resource(self, ipc_device, pool_resource): ipc_device.set_current() run_child(child_concurrent_uuid_imports, type(pool_resource), pool_resource.allocation_handle) + + # Releasing the last import of a pool with unsynchronized frees pending + # leaves the driver holding the pool (#3514); the next import must still + # grant device access. + @pytest.mark.parametrize("import_path", ["single", "raw_handle", "concurrent_uuid"]) + @pytest.mark.agent_authored(model="claude-sonnet-5-5") + def test_reimport_after_release_with_pending_free(self, pool_resource, exported_buffer, import_path): + run_child( + child_reimport_after_release_with_pending_free, + type(pool_resource), + pool_resource.allocation_handle, + exported_buffer.ipc_descriptor, + import_path, + ) From 92f3186ab5036843c2379bc373cc9a2b9886f684 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Sat, 21 Nov 2026 07:41:01 -0800 Subject: [PATCH 2/3] test(cuda.core): release resources when a duplicate-import test fails The child functions in test_ipc_pool_duplicate_import.py closed their resources and buffers at the end of the body, so a failed assertion or a failed second import left them open. They now enter each one into an ExitStack as it is created: closing(resource) for the pool resources, which are not context managers, and the buffer itself. The stack closes in reverse order, so buffers still close before the resources they came from. Closes that are the sequence under test stay explicit, and the stack repeats them as no-ops. _import_twice takes the stack, so the first import is released if the second fails. _import_concurrently closes the imports that succeeded before it raises. The race test also owns the buffer, resource and stream that its re-import thread creates, and still synchronizes the stream before closing it. Signed-off-by: Ralf Juengling --- .../test_ipc_pool_duplicate_import.py | 265 +++++++++--------- 1 file changed, 138 insertions(+), 127 deletions(-) diff --git a/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py b/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py index 6f7d4eb4961..99cc0c1c0bb 100644 --- a/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py +++ b/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py @@ -43,10 +43,10 @@ pytestmark = pytest.mark.parallel_threads_limit(2) -def _import_twice(mr_type, device, alloc_handle): +def _import_twice(stack, mr_type, device, alloc_handle): # A raw file descriptor carries no UUID, so the registry cannot dedupe. - first = import_pool_resource(mr_type, device, int(alloc_handle)) - second = import_pool_resource(mr_type, device, int(alloc_handle)) + first = stack.enter_context(contextlib.closing(import_pool_resource(mr_type, device, int(alloc_handle)))) + second = stack.enter_context(contextlib.closing(import_pool_resource(mr_type, device, int(alloc_handle)))) assert first is not second assert int(first.handle) == int(second.handle) return first, second @@ -58,17 +58,18 @@ def child_close_one_import(mr_type, alloc_handle, descriptor): stream = device.default_stream baseline = _live_owned_mempool_count() - first, second = _import_twice(mr_type, device, alloc_handle) - assert _live_owned_mempool_count() == baseline + 1 + # The stack closes the buffer, then both resources; the first is closed + # early on purpose, and closing it again is a no-op. + with contextlib.ExitStack() as stack: + first, second = _import_twice(stack, mr_type, device, alloc_handle) + assert _live_owned_mempool_count() == baseline + 1 - first.close() - assert pool_access_flags(second, device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE + first.close() + assert pool_access_flags(second, device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE - buffer = Buffer.from_ipc_descriptor(second, descriptor, stream=stream) - buffer.fill(7, stream=stream) - stream.sync() - buffer.close() - second.close() + buffer = stack.enter_context(Buffer.from_ipc_descriptor(second, descriptor, stream=stream)) + buffer.fill(7, stream=stream) + stream.sync() assert _live_owned_mempool_count() == baseline @@ -78,14 +79,16 @@ def child_close_both_imports_with_outstanding_buffer(mr_type, alloc_handle, desc stream = device.default_stream baseline = _live_owned_mempool_count() - with assert_no_cuda_warning(): - first, second = _import_twice(mr_type, device, alloc_handle) + # The closes below are the sequence under test; the stack repeats them as + # no-ops, and releases whatever a failed assertion leaves open. + with assert_no_cuda_warning(), contextlib.ExitStack() as stack: + first, second = _import_twice(stack, mr_type, device, alloc_handle) assert _live_owned_mempool_count() == baseline + 1 # The buffer keeps the pool alive after both resources are closed. - outstanding = Buffer.from_ipc_descriptor(first, descriptor, stream=stream) + outstanding = stack.enter_context(Buffer.from_ipc_descriptor(first, descriptor, stream=stream)) first.close() assert pool_access_flags(second, device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE - buffer = Buffer.from_ipc_descriptor(second, descriptor, stream=stream) + buffer = stack.enter_context(Buffer.from_ipc_descriptor(second, descriptor, stream=stream)) buffer.fill(7, stream=stream) stream.sync() buffer.close() @@ -103,56 +106,65 @@ def _race_reimport_with_last_release(mr_type, device, alloc_handle, descriptor): # The buffer is freed on another thread, which a per-thread default stream # would report as a cross-thread free. stream = device.create_stream() - # The buffer holds the last reference to the pool once its resource is - # closed, so closing the buffer releases the pool. - mr = import_pool_resource(mr_type, device, int(alloc_handle)) - outstanding = Buffer.from_ipc_descriptor(mr, descriptor, stream=stream) - mr.close() - del mr - - barrier = threading.Barrier(2) - reimported = [] - errors = [] - - def release(): - try: - barrier.wait() - outstanding.close() - except Exception as e: - errors.append(e) - barrier.abort() - - def reimport(): - try: - Device().set_current() - barrier.wait() - # Start once the release is under way, inside its delay. - time.sleep(RELEASE_DELAY_US / 4 / 1e6) - # A raw file descriptor carries no UUID, so this is a real import, - # through an object the releasing thread never sees. - mr = import_pool_resource(mr_type, device, int(alloc_handle)) - reimported.append((mr, Buffer.from_ipc_descriptor(mr, descriptor, stream=stream))) - except Exception as e: - errors.append(e) - barrier.abort() - - threads = [threading.Thread(target=release), threading.Thread(target=reimport)] - for thread in threads: - thread.start() - for thread in threads: - thread.join() - if errors: - raise errors[0] - - # The release has finished: its cleanup must have left the new import alone. - ((mr, buffer),) = reimported - assert pool_access_flags(mr, device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE - buffer.fill(7, stream=stream) - stream.sync() - buffer.close() - mr.close() - stream.sync() - stream.close() + # The stack closes in reverse order: the new buffer and resource, then the + # stream after it has synchronized the pending frees. + with contextlib.ExitStack() as stack: + stack.enter_context(contextlib.closing(stream)) + stack.callback(stream.sync) + + # The buffer holds the last reference to the pool once its resource is + # closed, so closing the buffer releases the pool. + with contextlib.closing(import_pool_resource(mr_type, device, int(alloc_handle))) as mr: + outstanding = stack.enter_context(Buffer.from_ipc_descriptor(mr, descriptor, stream=stream)) + del mr + + barrier = threading.Barrier(2) + reimported = [] + reimported_buffers = [] + errors = [] + + def release(): + try: + barrier.wait() + outstanding.close() + except Exception as e: + errors.append(e) + barrier.abort() + + def reimport(): + try: + Device().set_current() + barrier.wait() + # Start once the release is under way, inside its delay. + time.sleep(RELEASE_DELAY_US / 4 / 1e6) + # A raw file descriptor carries no UUID, so this is a real import, + # through an object the releasing thread never sees. + mr = import_pool_resource(mr_type, device, int(alloc_handle)) + reimported.append(mr) + reimported_buffers.append(Buffer.from_ipc_descriptor(mr, descriptor, stream=stream)) + except Exception as e: + errors.append(e) + barrier.abort() + + threads = [threading.Thread(target=release), threading.Thread(target=reimport)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + # Own whatever the thread made before looking at its errors. + for mr in reimported: + stack.enter_context(contextlib.closing(mr)) + for buffer in reimported_buffers: + stack.enter_context(buffer) + if errors: + raise errors[0] + + # The release has finished: its cleanup must have left the new import alone. + (mr,) = reimported + (buffer,) = reimported_buffers + assert pool_access_flags(mr, device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE + buffer.fill(7, stream=stream) + stream.sync() def child_reimport_races_last_release(mr_type, alloc_handle, descriptor): @@ -175,34 +187,30 @@ def child_register_reentered_by_gc(mr_type, alloc_handle): device.set_current() baseline = _live_owned_mempool_count() - nested = import_pool_resource(mr_type, device, int(alloc_handle)) - nested_uuid = uuid.uuid4() - nested.register(nested_uuid) - - # Registering inserts a weak reference into the registry, and allocating - # it can start a collection. A callback that registers again then runs on - # the thread that holds the registry lock. - def register_during_collection(phase, info): - if phase == "start": - nested.register(nested_uuid) - - imports = [] - threshold = gc.get_threshold() - gc.callbacks.append(register_during_collection) - try: - gc.set_threshold(1) - for _ in range(REENTRY_ITERATIONS): - # A raw file descriptor carries no UUID, so each import registers a new one. - mr = import_pool_resource(mr_type, device, int(alloc_handle)) - imports.append(mr) - assert mr.register(uuid.uuid4()) is mr - finally: - gc.set_threshold(*threshold) - gc.callbacks.remove(register_during_collection) - - for mr in imports: - mr.close() - nested.close() + # The stack closes the imports, then the nested resource. + with contextlib.ExitStack() as stack: + nested = stack.enter_context(contextlib.closing(import_pool_resource(mr_type, device, int(alloc_handle)))) + nested_uuid = uuid.uuid4() + nested.register(nested_uuid) + + # Registering inserts a weak reference into the registry, and allocating + # it can start a collection. A callback that registers again then runs on + # the thread that holds the registry lock. + def register_during_collection(phase, info): + if phase == "start": + nested.register(nested_uuid) + + threshold = gc.get_threshold() + gc.callbacks.append(register_during_collection) + try: + gc.set_threshold(1) + for _ in range(REENTRY_ITERATIONS): + # A raw file descriptor carries no UUID, so each import registers a new one. + mr = stack.enter_context(contextlib.closing(import_pool_resource(mr_type, device, int(alloc_handle)))) + assert mr.register(uuid.uuid4()) is mr + finally: + gc.set_threshold(*threshold) + gc.callbacks.remove(register_during_collection) assert _live_owned_mempool_count() == baseline @@ -226,6 +234,10 @@ def importer(index): for thread in threads: thread.join() if errors: + # The imports that succeeded would otherwise stay open. + for mr in results: + if mr is not None: + mr.close() raise errors[0] return results @@ -237,55 +249,54 @@ def child_concurrent_uuid_imports(mr_type, alloc_handle): baseline = _live_owned_mempool_count() for _ in range(ITERATIONS): - results = _import_concurrently(mr_type, device, alloc_handle) - # Every import of the UUID returns the registered resource, whichever - # thread registered it, with its device set before it was registered. - assert all(mr is results[0] for mr in results) - if mr_type is DeviceMemoryResource: - assert results[0].device_id == device.device_id - assert _live_owned_mempool_count() == baseline + 1 - # Dropping the losing imports must not revoke the winner's access. - assert pool_access_flags(results[0], device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE - results[0].close() + with contextlib.ExitStack() as stack: + results = _import_by_path(stack, "concurrent_uuid", mr_type, device, alloc_handle) + # Every import of the UUID returns the registered resource, whichever + # thread registered it, with its device set before it was registered. + assert all(mr is results[0] for mr in results) + if mr_type is DeviceMemoryResource: + assert results[0].device_id == device.device_id + assert _live_owned_mempool_count() == baseline + 1 + # Dropping the losing imports must not revoke the winner's access. + assert pool_access_flags(results[0], device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE del results assert _live_owned_mempool_count() == baseline -def _import_by_path(import_path, mr_type, device, alloc_handle): - """Import the pool as ``import_path`` names; returns every resource the import made.""" +def _import_by_path(stack, import_path, mr_type, device, alloc_handle): + """Import the pool as ``import_path`` names; ``stack`` closes every resource the import made.""" if import_path == "single": - return [import_pool_resource(mr_type, device, alloc_handle)] + return [stack.enter_context(contextlib.closing(import_pool_resource(mr_type, device, alloc_handle)))] if import_path == "raw_handle": # A raw file descriptor carries no UUID, so each import is a real import. - return [import_pool_resource(mr_type, device, int(alloc_handle)) for _ in range(THREADS)] + return [ + stack.enter_context(contextlib.closing(import_pool_resource(mr_type, device, int(alloc_handle)))) + for _ in range(THREADS) + ] assert import_path == "concurrent_uuid", import_path - return _import_concurrently(mr_type, device, alloc_handle) + return [stack.enter_context(contextlib.closing(mr)) for mr in _import_concurrently(mr_type, device, alloc_handle)] def _import_fill_and_release_unsynchronized(import_path, mr_type, device, alloc_handle, descriptor, stream): - resources = _import_by_path(import_path, mr_type, device, alloc_handle) - buffers = [Buffer.from_ipc_descriptor(mr, descriptor, stream=stream) for mr in resources] - for buffer in buffers: - buffer.fill(7, stream=stream) - stream.sync() + # The stack closes in reverse order: the buffers, then the resources. # Closing a buffer enqueues its cuMemFreeAsync without waiting for it, so # the frees can still be pending when the last resource releases the pool. - for buffer in buffers: - buffer.close() - for mr in resources: - mr.close() + with contextlib.ExitStack() as stack: + resources = _import_by_path(stack, import_path, mr_type, device, alloc_handle) + buffers = [stack.enter_context(Buffer.from_ipc_descriptor(mr, descriptor, stream=stream)) for mr in resources] + for buffer in buffers: + buffer.fill(7, stream=stream) + stream.sync() def _reimport_and_use(import_path, mr_type, device, alloc_handle, descriptor, stream): - resources = _import_by_path(import_path, mr_type, device, alloc_handle) - for mr in resources: - assert pool_access_flags(mr, device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE - buffer = Buffer.from_ipc_descriptor(resources[0], descriptor, stream=stream) - buffer.fill(9, stream=stream) - stream.sync() - buffer.close() - for mr in resources: - mr.close() + with contextlib.ExitStack() as stack: + resources = _import_by_path(stack, import_path, mr_type, device, alloc_handle) + for mr in resources: + assert pool_access_flags(mr, device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE + buffer = stack.enter_context(Buffer.from_ipc_descriptor(resources[0], descriptor, stream=stream)) + buffer.fill(9, stream=stream) + stream.sync() def child_reimport_after_release_with_pending_free(mr_type, alloc_handle, descriptor, import_path): From 56274e33703a64f38df333429cc7df4d0c2d2629 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Sat, 21 Nov 2026 07:48:34 -0800 Subject: [PATCH 3/3] test(cuda.core): consolidate duplicate-import test helpers Three helpers remove repetition across the child functions: _enter_import wraps an import in contextlib.closing on an ExitStack, _run_threads starts and joins a list of threads, and the _child_device context manager sets the current device and asserts the owned-pool count returns to its baseline on exit. The child functions use them instead of repeating the setup, the thread start/join loops, and the baseline bookkeeping. Signed-off-by: Ralf Juengling --- .../test_ipc_pool_duplicate_import.py | 139 ++++++++---------- 1 file changed, 63 insertions(+), 76 deletions(-) diff --git a/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py b/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py index 99cc0c1c0bb..2b19226b73d 100644 --- a/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py +++ b/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py @@ -43,24 +43,43 @@ pytestmark = pytest.mark.parallel_threads_limit(2) +def _enter_import(stack, mr_type, device, alloc_handle): + """Import the pool for ``alloc_handle`` and register it to close on ``stack``.""" + return stack.enter_context(contextlib.closing(import_pool_resource(mr_type, device, alloc_handle))) + + +def _run_threads(threads): + """Start every thread, then join every thread.""" + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + +@contextlib.contextmanager +def _child_device(): + """Set the child's current device, and assert the owned-pool count returns to its baseline.""" + device = Device() + device.set_current() + baseline = _live_owned_mempool_count() + yield device, baseline + assert _live_owned_mempool_count() == baseline + + def _import_twice(stack, mr_type, device, alloc_handle): # A raw file descriptor carries no UUID, so the registry cannot dedupe. - first = stack.enter_context(contextlib.closing(import_pool_resource(mr_type, device, int(alloc_handle)))) - second = stack.enter_context(contextlib.closing(import_pool_resource(mr_type, device, int(alloc_handle)))) + first = _enter_import(stack, mr_type, device, int(alloc_handle)) + second = _enter_import(stack, mr_type, device, int(alloc_handle)) assert first is not second assert int(first.handle) == int(second.handle) return first, second def child_close_one_import(mr_type, alloc_handle, descriptor): - device = Device() - device.set_current() - stream = device.default_stream - baseline = _live_owned_mempool_count() - # The stack closes the buffer, then both resources; the first is closed # early on purpose, and closing it again is a no-op. - with contextlib.ExitStack() as stack: + with _child_device() as (device, baseline), contextlib.ExitStack() as stack: + stream = device.default_stream first, second = _import_twice(stack, mr_type, device, alloc_handle) assert _live_owned_mempool_count() == baseline + 1 @@ -70,18 +89,13 @@ def child_close_one_import(mr_type, alloc_handle, descriptor): buffer = stack.enter_context(Buffer.from_ipc_descriptor(second, descriptor, stream=stream)) buffer.fill(7, stream=stream) stream.sync() - assert _live_owned_mempool_count() == baseline def child_close_both_imports_with_outstanding_buffer(mr_type, alloc_handle, descriptor): - device = Device() - device.set_current() - stream = device.default_stream - baseline = _live_owned_mempool_count() - # The closes below are the sequence under test; the stack repeats them as # no-ops, and releases whatever a failed assertion leaves open. - with assert_no_cuda_warning(), contextlib.ExitStack() as stack: + with _child_device() as (device, baseline), assert_no_cuda_warning(), contextlib.ExitStack() as stack: + stream = device.default_stream first, second = _import_twice(stack, mr_type, device, alloc_handle) assert _live_owned_mempool_count() == baseline + 1 # The buffer keeps the pool alive after both resources are closed. @@ -99,7 +113,6 @@ def child_close_both_imports_with_outstanding_buffer(mr_type, alloc_handle, desc stream.sync() outstanding.close() stream.sync() - assert _live_owned_mempool_count() == baseline def _race_reimport_with_last_release(mr_type, device, alloc_handle, descriptor): @@ -146,11 +159,7 @@ def reimport(): errors.append(e) barrier.abort() - threads = [threading.Thread(target=release), threading.Thread(target=reimport)] - for thread in threads: - thread.start() - for thread in threads: - thread.join() + _run_threads([threading.Thread(target=release), threading.Thread(target=reimport)]) # Own whatever the thread made before looking at its errors. for mr in reimported: stack.enter_context(contextlib.closing(mr)) @@ -168,28 +177,20 @@ def reimport(): def child_reimport_races_last_release(mr_type, alloc_handle, descriptor): - device = Device() - device.set_current() - baseline = _live_owned_mempool_count() - - _set_ipc_pool_release_delay_for_testing(RELEASE_DELAY_US) - try: - with assert_no_cuda_warning(): - for _ in range(RACE_ITERATIONS): - _race_reimport_with_last_release(mr_type, device, alloc_handle, descriptor) - finally: - _set_ipc_pool_release_delay_for_testing(0) - assert _live_owned_mempool_count() == baseline + with _child_device() as (device, _baseline): + _set_ipc_pool_release_delay_for_testing(RELEASE_DELAY_US) + try: + with assert_no_cuda_warning(): + for _ in range(RACE_ITERATIONS): + _race_reimport_with_last_release(mr_type, device, alloc_handle, descriptor) + finally: + _set_ipc_pool_release_delay_for_testing(0) def child_register_reentered_by_gc(mr_type, alloc_handle): - device = Device() - device.set_current() - baseline = _live_owned_mempool_count() - # The stack closes the imports, then the nested resource. - with contextlib.ExitStack() as stack: - nested = stack.enter_context(contextlib.closing(import_pool_resource(mr_type, device, int(alloc_handle)))) + with _child_device() as (device, _baseline), contextlib.ExitStack() as stack: + nested = _enter_import(stack, mr_type, device, int(alloc_handle)) nested_uuid = uuid.uuid4() nested.register(nested_uuid) @@ -206,12 +207,11 @@ def register_during_collection(phase, info): gc.set_threshold(1) for _ in range(REENTRY_ITERATIONS): # A raw file descriptor carries no UUID, so each import registers a new one. - mr = stack.enter_context(contextlib.closing(import_pool_resource(mr_type, device, int(alloc_handle)))) + mr = _enter_import(stack, mr_type, device, int(alloc_handle)) assert mr.register(uuid.uuid4()) is mr finally: gc.set_threshold(*threshold) gc.callbacks.remove(register_during_collection) - assert _live_owned_mempool_count() == baseline def _import_concurrently(mr_type, device, alloc_handle): @@ -228,11 +228,7 @@ def importer(index): errors.append(e) barrier.abort() - threads = [threading.Thread(target=importer, args=(i,)) for i in range(THREADS)] - for thread in threads: - thread.start() - for thread in threads: - thread.join() + _run_threads([threading.Thread(target=importer, args=(i,)) for i in range(THREADS)]) if errors: # The imports that succeeded would otherwise stay open. for mr in results: @@ -243,36 +239,32 @@ def importer(index): def child_concurrent_uuid_imports(mr_type, alloc_handle): - device = Device() - device.set_current() - assert alloc_handle.uuid is not None - baseline = _live_owned_mempool_count() - - for _ in range(ITERATIONS): - with contextlib.ExitStack() as stack: - results = _import_by_path(stack, "concurrent_uuid", mr_type, device, alloc_handle) - # Every import of the UUID returns the registered resource, whichever - # thread registered it, with its device set before it was registered. - assert all(mr is results[0] for mr in results) - if mr_type is DeviceMemoryResource: - assert results[0].device_id == device.device_id - assert _live_owned_mempool_count() == baseline + 1 - # Dropping the losing imports must not revoke the winner's access. - assert pool_access_flags(results[0], device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE - del results - assert _live_owned_mempool_count() == baseline + with _child_device() as (device, baseline): + assert alloc_handle.uuid is not None + for _ in range(ITERATIONS): + with contextlib.ExitStack() as stack: + results = _import_by_path(stack, "concurrent_uuid", mr_type, device, alloc_handle) + # Every import of the UUID returns the registered resource, whichever + # thread registered it, with its device set before it was registered. + assert all(mr is results[0] for mr in results) + if mr_type is DeviceMemoryResource: + assert results[0].device_id == device.device_id + assert _live_owned_mempool_count() == baseline + 1 + # Dropping the losing imports must not revoke the winner's access. + assert ( + pool_access_flags(results[0], device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE + ) + del results + assert _live_owned_mempool_count() == baseline def _import_by_path(stack, import_path, mr_type, device, alloc_handle): """Import the pool as ``import_path`` names; ``stack`` closes every resource the import made.""" if import_path == "single": - return [stack.enter_context(contextlib.closing(import_pool_resource(mr_type, device, alloc_handle)))] + return [_enter_import(stack, mr_type, device, alloc_handle)] if import_path == "raw_handle": # A raw file descriptor carries no UUID, so each import is a real import. - return [ - stack.enter_context(contextlib.closing(import_pool_resource(mr_type, device, int(alloc_handle)))) - for _ in range(THREADS) - ] + return [_enter_import(stack, mr_type, device, int(alloc_handle)) for _ in range(THREADS)] assert import_path == "concurrent_uuid", import_path return [stack.enter_context(contextlib.closing(mr)) for mr in _import_concurrently(mr_type, device, alloc_handle)] @@ -300,20 +292,15 @@ def _reimport_and_use(import_path, mr_type, device, alloc_handle, descriptor, st def child_reimport_after_release_with_pending_free(mr_type, alloc_handle, descriptor, import_path): - device = Device() - device.set_current() - stream = device.default_stream - baseline = _live_owned_mempool_count() - # A failed grant is also reported as a warning by the cleanup that follows it. - with assert_no_cuda_warning(): + with _child_device() as (device, _baseline), assert_no_cuda_warning(): + stream = device.default_stream for _ in range(REIMPORT_ROUNDS): # The helpers keep no reference to any resource once they return, so # the registry entry is gone and the next import is a real one. _import_fill_and_release_unsynchronized(import_path, mr_type, device, alloc_handle, descriptor, stream) _reimport_and_use(import_path, mr_type, device, alloc_handle, descriptor, stream) stream.sync() - assert _live_owned_mempool_count() == baseline class TestIpcPoolDuplicateImport: