From c012be1d2c8c0afcd8002e7726626bf1e176f9c6 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 22 Oct 2026 19:32:29 +0000 Subject: [PATCH] Work around Orin IPC export FD exhaustion (nvbug 6977893) On Orin, recognize the driver's successful export with an invalid -EMFILE descriptor as a known bug. Use an explicit non-strict xfail restricted to that outcome, so fixed drivers can pass and unrelated failures still fail. Continue checking descriptor rollback and the successful export/import path before reporting the expected failure. Close all test buffers on partial setup and assertion failures. Avoid closing a negative descriptor in the export rollback so the original invalid-handle error is preserved instead of being masked by EBADF. Signed-off-by: Ralf W. Grosse-Kunstleve --- .../core/_memory/_virtual_memory_resource.pyx | 4 +- cuda_core/tests/memory_ipc/test_vmm_ipc.py | 113 +++++++++++------- 2 files changed, 70 insertions(+), 47 deletions(-) diff --git a/cuda_core/cuda/core/_memory/_virtual_memory_resource.pyx b/cuda_core/cuda/core/_memory/_virtual_memory_resource.pyx index 827677939c1..c3bb0179885 100644 --- a/cuda_core/cuda/core/_memory/_virtual_memory_resource.pyx +++ b/cuda_core/cuda/core/_memory/_virtual_memory_resource.pyx @@ -1124,7 +1124,9 @@ cdef VirtualMemoryIPCBufferDescriptor _export_ipc_descriptor(Buffer buf): try: handle = IPCAllocationHandle._init(fd, None) except: # noqa: E722 rollback-then-raise: the fd is not owned yet - os.close(fd) + # nvbug 6977893: an Orin driver can return a negative fd despite success. + if fd >= 0: + os.close(fd) raise # The handle owns the fd now; if append fails, the handle closes it once. handles.append(handle) diff --git a/cuda_core/tests/memory_ipc/test_vmm_ipc.py b/cuda_core/tests/memory_ipc/test_vmm_ipc.py index 0aba6ad4bea..8bf149b0055 100644 --- a/cuda_core/tests/memory_ipc/test_vmm_ipc.py +++ b/cuda_core/tests/memory_ipc/test_vmm_ipc.py @@ -12,6 +12,7 @@ """ import ctypes +import errno import gc import json import multiprocessing as mp @@ -20,6 +21,7 @@ import pickle import socket import threading +from contextlib import ExitStack import pytest from helpers.child_processes import child_timeout_sec, kill_subprocesses, noop, run_child, warmup_multiprocessing @@ -700,66 +702,85 @@ def child_main(device_id, to_child, from_child): buffer.close() +class _OrinIPCExportFDExhaustionError(Exception): + """nvbug 6977893: Orin export reports success with an invalid -EMFILE fd.""" + + class TestFileDescriptorLimit: @pytest.mark.agent_authored(model="claude-fable-5-1") @pytest.mark.thread_unsafe(reason="changes the process-wide file descriptor limit") - def test_export_near_the_limit(self, vmm_ipc_device): + def test_export_near_the_limit(self, vmm_ipc_device, request): """Under a lowered RLIMIT_NOFILE an export either succeeds or fails with a readable error and no leak.""" import resource # POSIX-only; a module-level import breaks collection on Windows device = vmm_ipc_device + is_orin = device.name == "Orin" + xfail_reason = "Workaround for nvbug 6977893: Orin IPC export mishandles FD exhaustion" + if is_orin: + request.node.add_marker( + pytest.mark.xfail(reason=xfail_reason, raises=_OrinIPCExportFDExhaustionError, strict=False) + ) + orin_export_bug = False stream = device.default_stream mr = _resource(device) - first = mr.allocate(1) - gran = first.size - second = mr.modify_allocation(first, 2 * gran) - buf = mr.modify_allocation(second, 3 * gran) - values = (0x61, 0x62, 0x63) - for i, value in enumerate(values): - _fill(buf, value, offset=i * gran, size=gran) - soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) - fillers = [] - imported = desc = None - try: - # Fill the holes in the descriptor table so that the limit alone decides how many more fit. - highest = max(int(name) for name in os.listdir("/proc/self/fd")) - while True: - fd = os.open(os.devnull, os.O_RDONLY) - if fd > highest: - os.close(fd) - break - fillers.append(fd) - baseline = _settled_open_fds() - # Room for two more descriptors: a three-allocation export cannot complete. - resource.setrlimit(resource.RLIMIT_NOFILE, (highest + 1 + 2, hard)) - failure = None + with ExitStack() as buffers: + first = buffers.enter_context(mr.allocate(1)) + gran = first.size + second = buffers.enter_context(mr.modify_allocation(first, 2 * gran)) + buf = buffers.enter_context(mr.modify_allocation(second, 3 * gran)) + values = (0x61, 0x62, 0x63) + for i, value in enumerate(values): + _fill(buf, value, offset=i * gran, size=gran) + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + fillers = [] + desc = None try: + # Fill the holes in the descriptor table so that the limit alone decides how many more fit. + highest = max(int(name) for name in os.listdir("/proc/self/fd")) + while True: + fd = os.open(os.devnull, os.O_RDONLY) + if fd > highest: + os.close(fd) + break + fillers.append(fd) + baseline = _settled_open_fds() + # Room for two more descriptors: a three-allocation export cannot complete. + resource.setrlimit(resource.RLIMIT_NOFILE, (highest + 1 + 2, hard)) + export_failed = False + try: + desc = buf.ipc_descriptor + except CUDAError as exc: + if not _has_note(exc, "file descriptor limit"): + raise + export_failed = True + except ValueError as exc: + if not is_orin or str(exc) != ( + f"Invalid allocation handle (fd) {-errno.EMFILE}: must be non-negative" + ): + raise + orin_export_bug = export_failed = True + if export_failed: + assert _open_fds() == baseline, "a failed export must close the descriptors it already took" + else: + # Acceptable only if the driver did not need one descriptor per allocation. + assert len(desc.fds) == 3 + desc = None + gc.collect() + # Room for the three descriptors and the listing: export and import both succeed. + resource.setrlimit(resource.RLIMIT_NOFILE, (highest + 1 + 6, hard)) desc = buf.ipc_descriptor - except CUDAError as exc: - failure = exc - if failure is not None: - assert _has_note(failure, "file descriptor limit") - assert _open_fds() == baseline, "a failed export must close the descriptors it already took" - else: - # Acceptable only if the driver did not need one descriptor per allocation. assert len(desc.fds) == 3 + imported = buffers.enter_context(Buffer.from_ipc_descriptor(mr, desc, stream=stream)) + assert _open_fds() == baseline + 3, "the import must not take file descriptors of its own" + finally: + resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard)) + for fd in fillers: + os.close(fd) desc = None gc.collect() - # Room for the three descriptors and the listing: export and import both succeed. - resource.setrlimit(resource.RLIMIT_NOFILE, (highest + 1 + 6, hard)) - desc = buf.ipc_descriptor - assert len(desc.fds) == 3 - imported = Buffer.from_ipc_descriptor(mr, desc, stream=stream) - assert _open_fds() == baseline + 3, "the import must not take file descriptors of its own" - finally: - resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard)) - for fd in fillers: - os.close(fd) - assert [_read(imported, i * gran, 16) for i in range(3)] == [bytes([value]) * 16 for value in values] - del desc - gc.collect() - for buffer in (imported, buf, second, first): - buffer.close() + assert [_read(imported, i * gran, 16) for i in range(3)] == [bytes([value]) * 16 for value in values] + if orin_export_bug: + raise _OrinIPCExportFDExhaustionError(xfail_reason) class TestRawFileDescriptorTransport: