From e5302e67eb5ed3faaa4812aa3fa5d135fcad38ce Mon Sep 17 00:00:00 2001 From: Ivar Refsdal Date: Mon, 11 Nov 2024 13:25:40 +0100 Subject: [PATCH 01/14] gh-126434: Use multiprocessing.Value for multiprocessing.Event to avoid deadlock when there is reentrant usage of `set` from `is_set`, e.g. when handling system signals --- Lib/multiprocessing/synchronize.py | 26 +++++++++---------- ...-11-05-11-11-29.gh-issue-126434.Hw3wdI.rst | 1 + 2 files changed, 14 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-11-05-11-11-29.gh-issue-126434.Hw3wdI.rst diff --git a/Lib/multiprocessing/synchronize.py b/Lib/multiprocessing/synchronize.py index 3ccbfe311c71f37..4456fa7a9f18814 100644 --- a/Lib/multiprocessing/synchronize.py +++ b/Lib/multiprocessing/synchronize.py @@ -329,34 +329,34 @@ class Event(object): def __init__(self, *, ctx): self._cond = ctx.Condition(ctx.Lock()) - self._flag = ctx.Semaphore(0) + self._flag = ctx.Value('i', 0) def is_set(self): - with self._cond: - if self._flag.acquire(False): - self._flag.release() - return True - return False + return self._flag.value == 1 def set(self): + assert not self._cond._lock._semlock._is_mine(), \ + 'multiprocessing.Event is not reentrant for clear(), set() and wait()' with self._cond: - self._flag.acquire(False) - self._flag.release() + self._flag.value = 1 self._cond.notify_all() def clear(self): + assert not self._cond._lock._semlock._is_mine(), \ + 'multiprocessing.Event is not reentrant for clear(), set() and wait()' with self._cond: - self._flag.acquire(False) + self._flag.value = 0 def wait(self, timeout=None): + assert not self._cond._lock._semlock._is_mine(), \ + 'multiprocessing.Event is not reentrant for clear(), set() and wait()' with self._cond: - if self._flag.acquire(False): - self._flag.release() + if self._flag.value == 1: + return True else: self._cond.wait(timeout) - if self._flag.acquire(False): - self._flag.release() + if self._flag.value == 1: return True return False diff --git a/Misc/NEWS.d/next/Library/2024-11-05-11-11-29.gh-issue-126434.Hw3wdI.rst b/Misc/NEWS.d/next/Library/2024-11-05-11-11-29.gh-issue-126434.Hw3wdI.rst new file mode 100644 index 000000000000000..41b4ccd2eb530c8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-05-11-11-29.gh-issue-126434.Hw3wdI.rst @@ -0,0 +1 @@ +Use :mod:`multiprocessing` ``Value`` for :mod:`multiprocessing` ``Event`` to avoid deadlock when reetrant usage of `set` from `is_set`, e.g. when handling system signals From cbecc76752fa21c43a44aef81a4e5c21617b5923 Mon Sep 17 00:00:00 2001 From: Ivar Refsdal Date: Tue, 12 Nov 2024 09:43:36 +0100 Subject: [PATCH 02/14] gh-126434: Add test --- Lib/test/multiprocessingdata/event_signal.py | 33 +++++++++++++++ .../test_multiprocessing_event/__init__.py | 5 +++ .../test_multiprocessing_event/test_event.py | 41 +++++++++++++++++++ Makefile.pre.in | 2 + 4 files changed, 81 insertions(+) create mode 100644 Lib/test/multiprocessingdata/event_signal.py create mode 100644 Lib/test/test_multiprocessing_event/__init__.py create mode 100644 Lib/test/test_multiprocessing_event/test_event.py diff --git a/Lib/test/multiprocessingdata/event_signal.py b/Lib/test/multiprocessingdata/event_signal.py new file mode 100644 index 000000000000000..5592070e55090c2 --- /dev/null +++ b/Lib/test/multiprocessingdata/event_signal.py @@ -0,0 +1,33 @@ +import multiprocessing +import os +import signal +import concurrent.futures + + +def send_sigint(pid): + print('send_sigint running') + os.kill(pid, signal.SIGINT) + + +def run_signal_handler_test(): + shutdown_event = multiprocessing.Event() + + def sigterm_handler(_signo, _stack_frame): + try: + print('sigterm_handler running') + shutdown_event.set() + finally: + print('sigterm_handler done') + + signal.signal(signal.SIGINT, sigterm_handler) + + with concurrent.futures.ProcessPoolExecutor() as executor: + f = executor.submit(send_sigint, os.getpid()) + while not shutdown_event.is_set(): + pass + print('Received shutdown event') + f.result() + + +if __name__ == '__main__': + run_signal_handler_test() diff --git a/Lib/test/test_multiprocessing_event/__init__.py b/Lib/test/test_multiprocessing_event/__init__.py new file mode 100644 index 000000000000000..b90f7b4cbd9a486 --- /dev/null +++ b/Lib/test/test_multiprocessing_event/__init__.py @@ -0,0 +1,5 @@ +import os.path +from test import support + +def load_tests(*args): + return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_multiprocessing_event/test_event.py b/Lib/test/test_multiprocessing_event/test_event.py new file mode 100644 index 000000000000000..d7db60b9513c9e4 --- /dev/null +++ b/Lib/test/test_multiprocessing_event/test_event.py @@ -0,0 +1,41 @@ +import unittest +from test import support +import sys +import signal +import os + + +try: + import multiprocessing + from concurrent.futures import ProcessPoolExecutor + _have_multiprocessing = True +except (NotImplementedError, ModuleNotFoundError): + _have_multiprocessing = False + + +class TestEventSignalHandling(unittest.TestCase): + @unittest.skipUnless(_have_multiprocessing, + "requires multiprocessing") + @unittest.skipUnless(hasattr(signal, 'signal'), + "Requires signal.signal") + @unittest.skipUnless(hasattr(signal, 'SIGINT'), + "Requires signal.SIGINT") + @unittest.skipUnless(hasattr(os, 'kill'), + "Requires os.kill") + @unittest.skipUnless(hasattr(os, 'getppid'), + "Requires os.getppid") + @support.requires_subprocess() + def test_event_signal_handling(self): + import subprocess + script = support.findfile("event_signal.py", subdir="multiprocessingdata") + for x in range(10): + try: + exit_code = subprocess.call([sys.executable, script], stdout=subprocess.DEVNULL, timeout=3) + assert exit_code == 0 + except subprocess.TimeoutExpired: + assert False, 'subprocess.Timeoutexpired for event_signal.py' + + +if __name__ == '__main__': + unittest.main() + diff --git a/Makefile.pre.in b/Makefile.pre.in index c650ecaf7be1373..5296de1d7ea7fd0 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2449,6 +2449,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/leakers \ test/libregrtest \ test/mathdata \ + test/multiprocessingdata \ test/regrtestdata \ test/regrtestdata/import_from_tests \ test/regrtestdata/import_from_tests/test_regrtest_b \ @@ -2519,6 +2520,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_interpreters \ test/test_json \ test/test_module \ + test/test_multiprocessing_event \ test/test_multiprocessing_fork \ test/test_multiprocessing_forkserver \ test/test_multiprocessing_spawn \ From 2f90fb293596649004dd0fb8d775cc81665fd5cc Mon Sep 17 00:00:00 2001 From: Ivar Refsdal Date: Tue, 12 Nov 2024 13:52:21 +0100 Subject: [PATCH 03/14] gh-126434: Remove print statements used/useful for debugging --- Lib/test/multiprocessingdata/event_signal.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/Lib/test/multiprocessingdata/event_signal.py b/Lib/test/multiprocessingdata/event_signal.py index 5592070e55090c2..b97ea22a1e643ca 100644 --- a/Lib/test/multiprocessingdata/event_signal.py +++ b/Lib/test/multiprocessingdata/event_signal.py @@ -5,7 +5,6 @@ def send_sigint(pid): - print('send_sigint running') os.kill(pid, signal.SIGINT) @@ -13,11 +12,7 @@ def run_signal_handler_test(): shutdown_event = multiprocessing.Event() def sigterm_handler(_signo, _stack_frame): - try: - print('sigterm_handler running') - shutdown_event.set() - finally: - print('sigterm_handler done') + shutdown_event.set() signal.signal(signal.SIGINT, sigterm_handler) @@ -25,7 +20,6 @@ def sigterm_handler(_signo, _stack_frame): f = executor.submit(send_sigint, os.getpid()) while not shutdown_event.is_set(): pass - print('Received shutdown event') f.result() From 0ca4a855f08abc29eebee1631d24796509ef4da4 Mon Sep 17 00:00:00 2001 From: Ivar Refsdal Date: Tue, 12 Nov 2024 13:55:53 +0100 Subject: [PATCH 04/14] gh-126434: Support systems with high load (no accidental timeouts) --- Lib/test/test_multiprocessing_event/test_event.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_multiprocessing_event/test_event.py b/Lib/test/test_multiprocessing_event/test_event.py index d7db60b9513c9e4..0d537bc2b94669a 100644 --- a/Lib/test/test_multiprocessing_event/test_event.py +++ b/Lib/test/test_multiprocessing_event/test_event.py @@ -30,7 +30,7 @@ def test_event_signal_handling(self): script = support.findfile("event_signal.py", subdir="multiprocessingdata") for x in range(10): try: - exit_code = subprocess.call([sys.executable, script], stdout=subprocess.DEVNULL, timeout=3) + exit_code = subprocess.call([sys.executable, script], stdout=subprocess.DEVNULL, timeout=30) assert exit_code == 0 except subprocess.TimeoutExpired: assert False, 'subprocess.Timeoutexpired for event_signal.py' From f9307f6b7c91cf647a37dcda1730b65781ee53ce Mon Sep 17 00:00:00 2001 From: Ivar Refsdal Date: Wed, 13 Nov 2024 14:48:45 +0100 Subject: [PATCH 05/14] gh-126434: Detect when .set() is called by a thread already .wait()-ing. Raise an exception if that is the case. Fix race condition in multiprocessing.Event.wait() as described in https://github.com/python/cpython/issues/95826 --- Lib/multiprocessing/synchronize.py | 15 ++++--- .../multiprocessingdata/set_clear_race.py | 34 ++++++++++++++++ .../{event_signal.py => set_is_set.py} | 6 ++- .../multiprocessingdata/wait_set_throws.py | 34 ++++++++++++++++ .../test_multiprocessing_event/test_event.py | 39 ++++++++++++------- 5 files changed, 106 insertions(+), 22 deletions(-) create mode 100644 Lib/test/multiprocessingdata/set_clear_race.py rename Lib/test/multiprocessingdata/{event_signal.py => set_is_set.py} (82%) create mode 100644 Lib/test/multiprocessingdata/wait_set_throws.py diff --git a/Lib/multiprocessing/synchronize.py b/Lib/multiprocessing/synchronize.py index 4456fa7a9f18814..cc9ffcca615e883 100644 --- a/Lib/multiprocessing/synchronize.py +++ b/Lib/multiprocessing/synchronize.py @@ -329,6 +329,7 @@ class Event(object): def __init__(self, *, ctx): self._cond = ctx.Condition(ctx.Lock()) + self._threadlocal_wait_lock = None self._flag = ctx.Value('i', 0) def is_set(self): @@ -338,6 +339,9 @@ def set(self): assert not self._cond._lock._semlock._is_mine(), \ 'multiprocessing.Event is not reentrant for clear(), set() and wait()' with self._cond: + if self._threadlocal_wait_lock is not None: + assert not self._threadlocal_wait_lock.v.locked(), \ + 'multiprocessing.Event.set() cannot be called from a thread that is already wait()-ing' self._flag.value = 1 self._cond.notify_all() @@ -351,14 +355,15 @@ def wait(self, timeout=None): assert not self._cond._lock._semlock._is_mine(), \ 'multiprocessing.Event is not reentrant for clear(), set() and wait()' with self._cond: - if self._flag.value == 1: - return True - else: - self._cond.wait(timeout) + if self._threadlocal_wait_lock is None: + self._threadlocal_wait_lock = threading.local() + self._threadlocal_wait_lock.v = threading.Lock() if self._flag.value == 1: return True - return False + else: + with self._threadlocal_wait_lock.v: + return self._cond.wait(timeout) def __repr__(self) -> str: set_status = 'set' if self.is_set() else 'unset' diff --git a/Lib/test/multiprocessingdata/set_clear_race.py b/Lib/test/multiprocessingdata/set_clear_race.py new file mode 100644 index 000000000000000..370622782384df4 --- /dev/null +++ b/Lib/test/multiprocessingdata/set_clear_race.py @@ -0,0 +1,34 @@ +import multiprocessing +import sys + + +# Reproduction code copied and modified from https://github.com/python/cpython/issues/95826 +class SimpleRepro: + def __init__(self): + self.heartbeat_event = multiprocessing.Event() + self.shutdown_event = multiprocessing.Event() + self.child_proc = multiprocessing.Process(target=self.child_process, daemon=True) + self.child_proc.start() + + def child_process(self): + while True: + if self.shutdown_event.is_set(): + return + self.heartbeat_event.set() + self.heartbeat_event.clear() + + def test_heartbeat(self): + exit_code = 0 + for i in range(2000): + success = self.heartbeat_event.wait(100) + if not success: + exit_code = 1 + break + self.shutdown_event.set() + self.child_proc.join() + sys.exit(exit_code) + + +if __name__ == '__main__': + foo = SimpleRepro() + foo.test_heartbeat() diff --git a/Lib/test/multiprocessingdata/event_signal.py b/Lib/test/multiprocessingdata/set_is_set.py similarity index 82% rename from Lib/test/multiprocessingdata/event_signal.py rename to Lib/test/multiprocessingdata/set_is_set.py index b97ea22a1e643ca..31d6d18e0c6c7a1 100644 --- a/Lib/test/multiprocessingdata/event_signal.py +++ b/Lib/test/multiprocessingdata/set_is_set.py @@ -2,13 +2,15 @@ import os import signal import concurrent.futures +import time def send_sigint(pid): + time.sleep(1) os.kill(pid, signal.SIGINT) -def run_signal_handler_test(): +def run_signal_handler_set_is_set_test(): shutdown_event = multiprocessing.Event() def sigterm_handler(_signo, _stack_frame): @@ -24,4 +26,4 @@ def sigterm_handler(_signo, _stack_frame): if __name__ == '__main__': - run_signal_handler_test() + run_signal_handler_set_is_set_test() diff --git a/Lib/test/multiprocessingdata/wait_set_throws.py b/Lib/test/multiprocessingdata/wait_set_throws.py new file mode 100644 index 000000000000000..0adbbb80c39eb86 --- /dev/null +++ b/Lib/test/multiprocessingdata/wait_set_throws.py @@ -0,0 +1,34 @@ +import multiprocessing +import signal +import concurrent.futures +import time +import sys +import os + + +def send_sigint(pid): + time.sleep(1) + os.kill(pid, signal.SIGINT) + + +def run_signal_handler_wait_set_test(): + shutdown_event = multiprocessing.Event() + + def sigterm_handler(_signo, _stack_frame): + shutdown_event.set() + + signal.signal(signal.SIGINT, sigterm_handler) + + with concurrent.futures.ProcessPoolExecutor() as executor: + f = executor.submit(send_sigint, os.getpid()) + shutdown_event.wait() + f.result() + + +if __name__ == '__main__': + try: + run_signal_handler_wait_set_test() + sys.exit(1) + except AssertionError as e: + assert 'multiprocessing.Event.set() cannot be called from a thread that is already wait()-ing' in str(e) + sys.exit(0) diff --git a/Lib/test/test_multiprocessing_event/test_event.py b/Lib/test/test_multiprocessing_event/test_event.py index 0d537bc2b94669a..9787bf8f2aee26f 100644 --- a/Lib/test/test_multiprocessing_event/test_event.py +++ b/Lib/test/test_multiprocessing_event/test_event.py @@ -13,27 +13,36 @@ _have_multiprocessing = False -class TestEventSignalHandling(unittest.TestCase): - @unittest.skipUnless(_have_multiprocessing, - "requires multiprocessing") - @unittest.skipUnless(hasattr(signal, 'signal'), - "Requires signal.signal") - @unittest.skipUnless(hasattr(signal, 'SIGINT'), - "Requires signal.SIGINT") - @unittest.skipUnless(hasattr(os, 'kill'), +@unittest.skipUnless(_have_multiprocessing, + "requires multiprocessing") +@unittest.skipUnless(hasattr(signal, 'signal'), + "Requires signal.signal") +@unittest.skipUnless(hasattr(signal, 'SIGINT'), + "Requires signal.SIGINT") +@unittest.skipUnless(hasattr(os, 'kill'), "Requires os.kill") - @unittest.skipUnless(hasattr(os, 'getppid'), +@unittest.skipUnless(hasattr(os, 'getppid'), "Requires os.getppid") - @support.requires_subprocess() - def test_event_signal_handling(self): +@support.requires_subprocess() +class TestEventSignalHandling(unittest.TestCase): + def test_no_race_for_set_is_set(self): import subprocess - script = support.findfile("event_signal.py", subdir="multiprocessingdata") + script = support.findfile("set_is_set.py", subdir="multiprocessingdata") for x in range(10): try: - exit_code = subprocess.call([sys.executable, script], stdout=subprocess.DEVNULL, timeout=30) - assert exit_code == 0 + assert 0 == subprocess.call([sys.executable, script], timeout=60) except subprocess.TimeoutExpired: - assert False, 'subprocess.Timeoutexpired for event_signal.py' + assert False, 'subprocess.Timeoutexpired for set_is_set.py' + + def test_no_race_set_clear(self): + import subprocess + script = support.findfile("set_clear_race.py", subdir="multiprocessingdata") + assert 0 == subprocess.call([sys.executable, script]) + + def test_wait_set_throws(self): + import subprocess + script = support.findfile("wait_set_throws.py", subdir="multiprocessingdata") + assert 0 == subprocess.call([sys.executable, script]) if __name__ == '__main__': From 25236fe4781e7130fc367bd1fa6874590a90786e Mon Sep 17 00:00:00 2001 From: Ivar Refsdal Date: Thu, 14 Nov 2024 10:49:05 +0100 Subject: [PATCH 06/14] gh-126434: multiprocessing.Event is fully reentrant, i.e. can be called from a signal handler in any combination. --- Lib/multiprocessing/synchronize.py | 70 +++++++++++++------ .../{set_is_set.py => is_set_set.py} | 0 .../multiprocessingdata/set_clear_race.py | 2 + ..._set_throws.py => wait_set_no_deadlock.py} | 12 ++-- .../test_multiprocessing_event/test_event.py | 33 +++++++-- 5 files changed, 81 insertions(+), 36 deletions(-) rename Lib/test/multiprocessingdata/{set_is_set.py => is_set_set.py} (100%) rename Lib/test/multiprocessingdata/{wait_set_throws.py => wait_set_no_deadlock.py} (67%) diff --git a/Lib/multiprocessing/synchronize.py b/Lib/multiprocessing/synchronize.py index cc9ffcca615e883..4e1b9b659c60dba 100644 --- a/Lib/multiprocessing/synchronize.py +++ b/Lib/multiprocessing/synchronize.py @@ -328,46 +328,70 @@ def wait_for(self, predicate, timeout=None): class Event(object): def __init__(self, *, ctx): - self._cond = ctx.Condition(ctx.Lock()) - self._threadlocal_wait_lock = None self._flag = ctx.Value('i', 0) + # Allocate a ctypes.c_ulonglong to hold the set_id: + # Represents the C unsigned long long datatype. + # The constructor accepts an optional integer initializer; no overflow checking is done. + # From https://docs.python.org/3/library/ctypes.html#ctypes.c_ulonglong + # See multiprocessing/sharedctypes.py for typecode to ctypes definitions + self._set_id = ctx.Value('Q', 0) def is_set(self): - return self._flag.value == 1 + # From https://docs.python.org/3/library/multiprocessing.html#multiprocessing.Value : + # If lock is True (the default) then a new recursive lock object is created to synchronize access to the value. + with self._flag: + return self._flag.value == 1 def set(self): - assert not self._cond._lock._semlock._is_mine(), \ - 'multiprocessing.Event is not reentrant for clear(), set() and wait()' - with self._cond: - if self._threadlocal_wait_lock is not None: - assert not self._threadlocal_wait_lock.v.locked(), \ - 'multiprocessing.Event.set() cannot be called from a thread that is already wait()-ing' - self._flag.value = 1 - self._cond.notify_all() + # If `set` is called from a signal handler, this is fine as the lock is recursive (i.e. it won't deadlock). + # If the thread interrupted by the signal handler is wait()-ing and the signal handler calls set(), + # this is fine as wait() spins on the value. Thus, after the signal handler is done, the thread will + # return from wait() + # Fixes https://github.com/python/cpython/issues/85772 + with self._flag: + with self._set_id: + if self._flag.value == 0: + # There is a theoretical chance of race here. It requires the following conditions: + # The interrupted thread must be wait()ing. + # Then set must be called reentrant for the maximum value of c_ulonglong times, + # and all interruptions must happen exactly after `if self._flag.value == 0:`. + # The _set_id value will then wrap around. Then clear() must be called + # before the original wait() code continues. The wait() code will then continue + # to (incorrectly) wait. I think this case is safe to ignore. The stack + # will grow too large before there is any chance of this actually happening. + + self._flag.value = 1 + self._set_id.value += 1 + # There is no race here by reentrant set when reaching the maximum value for `self._set_id.value`. + # ctypes.c_ulonglong will overflow without any exception: + # https://docs.python.org/3/library/ctypes.html#ctypes.c_ulonglong + # > no overflow checking is done. + # This means that we do not need to check if some maximum value is reached: + # C will wrap around the value for us. def clear(self): - assert not self._cond._lock._semlock._is_mine(), \ - 'multiprocessing.Event is not reentrant for clear(), set() and wait()' - with self._cond: + with self._flag: self._flag.value = 0 def wait(self, timeout=None): - assert not self._cond._lock._semlock._is_mine(), \ - 'multiprocessing.Event is not reentrant for clear(), set() and wait()' - with self._cond: - if self._threadlocal_wait_lock is None: - self._threadlocal_wait_lock = threading.local() - self._threadlocal_wait_lock.v = threading.Lock() - + start_time = time.monotonic() + set_id = self._set_id.value + while True: if self._flag.value == 1: return True + elif set_id != self._set_id.value: + return True # flag is unset, but set_id changed, so there must have been a `set` followed by a `clear` + # during `time.sleep()`. Fixes https://github.com/python/cpython/issues/95826 + elif timeout is not None and (time.monotonic() - start_time) > timeout: + return False else: - with self._threadlocal_wait_lock.v: - return self._cond.wait(timeout) + # Fixes https://github.com/python/cpython/issues/85772 by spinning and sleeping. + time.sleep(0.000001) # sleep one microsecond def __repr__(self) -> str: set_status = 'set' if self.is_set() else 'unset' return f"<{type(self).__qualname__} at {id(self):#x} {set_status}>" + # # Barrier # diff --git a/Lib/test/multiprocessingdata/set_is_set.py b/Lib/test/multiprocessingdata/is_set_set.py similarity index 100% rename from Lib/test/multiprocessingdata/set_is_set.py rename to Lib/test/multiprocessingdata/is_set_set.py diff --git a/Lib/test/multiprocessingdata/set_clear_race.py b/Lib/test/multiprocessingdata/set_clear_race.py index 370622782384df4..47ce99b3a36dcb0 100644 --- a/Lib/test/multiprocessingdata/set_clear_race.py +++ b/Lib/test/multiprocessingdata/set_clear_race.py @@ -3,6 +3,8 @@ # Reproduction code copied and modified from https://github.com/python/cpython/issues/95826 +# Fixes the issue above + class SimpleRepro: def __init__(self): self.heartbeat_event = multiprocessing.Event() diff --git a/Lib/test/multiprocessingdata/wait_set_throws.py b/Lib/test/multiprocessingdata/wait_set_no_deadlock.py similarity index 67% rename from Lib/test/multiprocessingdata/wait_set_throws.py rename to Lib/test/multiprocessingdata/wait_set_no_deadlock.py index 0adbbb80c39eb86..26160a3f2278059 100644 --- a/Lib/test/multiprocessingdata/wait_set_throws.py +++ b/Lib/test/multiprocessingdata/wait_set_no_deadlock.py @@ -2,12 +2,13 @@ import signal import concurrent.futures import time -import sys import os +# Shows that https://github.com/python/cpython/issues/85772 is fixed + def send_sigint(pid): - time.sleep(1) + time.sleep(1) # Make sure shutdown_event.wait() is called os.kill(pid, signal.SIGINT) @@ -26,9 +27,4 @@ def sigterm_handler(_signo, _stack_frame): if __name__ == '__main__': - try: - run_signal_handler_wait_set_test() - sys.exit(1) - except AssertionError as e: - assert 'multiprocessing.Event.set() cannot be called from a thread that is already wait()-ing' in str(e) - sys.exit(0) + run_signal_handler_wait_set_test() diff --git a/Lib/test/test_multiprocessing_event/test_event.py b/Lib/test/test_multiprocessing_event/test_event.py index 9787bf8f2aee26f..3e8088d3ad6e499 100644 --- a/Lib/test/test_multiprocessing_event/test_event.py +++ b/Lib/test/test_multiprocessing_event/test_event.py @@ -25,25 +25,48 @@ "Requires os.getppid") @support.requires_subprocess() class TestEventSignalHandling(unittest.TestCase): - def test_no_race_for_set_is_set(self): + def test_no_race_for_is_set_set(self): import subprocess - script = support.findfile("set_is_set.py", subdir="multiprocessingdata") + script = support.findfile("is_set_set.py", subdir="multiprocessingdata") for x in range(10): try: assert 0 == subprocess.call([sys.executable, script], timeout=60) except subprocess.TimeoutExpired: - assert False, 'subprocess.Timeoutexpired for set_is_set.py' + assert False, 'subprocess.Timeoutexpired for is_set_set.py' def test_no_race_set_clear(self): import subprocess script = support.findfile("set_clear_race.py", subdir="multiprocessingdata") assert 0 == subprocess.call([sys.executable, script]) - def test_wait_set_throws(self): + def test_wait_set_no_deadlock(self): import subprocess - script = support.findfile("wait_set_throws.py", subdir="multiprocessingdata") + script = support.findfile("wait_set_no_deadlock.py", subdir="multiprocessingdata") assert 0 == subprocess.call([sys.executable, script]) + def test_wait_timeout(self): + event = multiprocessing.Event() + # https://docs.python.org/3/library/multiprocessing.html#multiprocessing.Event + # multiprocessing.Event: A clone of threading.Event. + + # threading.Event: https://docs.python.org/3/library/threading.html#threading.Event + + # threading.Event.wait(): https://docs.python.org/3/library/threading.html#threading.Event.wait + # Block as long as the internal flag is false and the timeout, if given, has not expired. + # The return value represents the reason that this blocking method returned; + # True if returning because the internal flag is set to true, or + # False if a timeout is given and the internal flag did not become true within the given wait time. + + # When the timeout argument is present and not None, it should be a floating-point number + # specifying a timeout for the operation in seconds, or fractions thereof. + + # wait() supports both integer and float: + flag_set = event.wait(1) + assert flag_set == False + + flag_set = event.wait(0.1) + assert flag_set == False + if __name__ == '__main__': unittest.main() From b1103b2dfde16e372ef75e863c321d54ba3bf80e Mon Sep 17 00:00:00 2001 From: Ivar Refsdal Date: Thu, 14 Nov 2024 11:14:41 +0100 Subject: [PATCH 07/14] gh-126434: Sleep longer. --- Lib/multiprocessing/synchronize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/multiprocessing/synchronize.py b/Lib/multiprocessing/synchronize.py index 4e1b9b659c60dba..8d7f5245d57bd64 100644 --- a/Lib/multiprocessing/synchronize.py +++ b/Lib/multiprocessing/synchronize.py @@ -386,7 +386,7 @@ def wait(self, timeout=None): return False else: # Fixes https://github.com/python/cpython/issues/85772 by spinning and sleeping. - time.sleep(0.000001) # sleep one microsecond + time.sleep(0.010) # sleep 10 milliseconds def __repr__(self) -> str: set_status = 'set' if self.is_set() else 'unset' From bec90700aab0d65f72f6b06f6dc271bfc10872e5 Mon Sep 17 00:00:00 2001 From: Ivar Refsdal Date: Thu, 14 Nov 2024 13:50:01 +0100 Subject: [PATCH 08/14] gh-126434: Update blurb. --- .../next/Library/2024-11-05-11-11-29.gh-issue-126434.Hw3wdI.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2024-11-05-11-11-29.gh-issue-126434.Hw3wdI.rst b/Misc/NEWS.d/next/Library/2024-11-05-11-11-29.gh-issue-126434.Hw3wdI.rst index 41b4ccd2eb530c8..2333aab7eea7b75 100644 --- a/Misc/NEWS.d/next/Library/2024-11-05-11-11-29.gh-issue-126434.Hw3wdI.rst +++ b/Misc/NEWS.d/next/Library/2024-11-05-11-11-29.gh-issue-126434.Hw3wdI.rst @@ -1 +1 @@ -Use :mod:`multiprocessing` ``Value`` for :mod:`multiprocessing` ``Event`` to avoid deadlock when reetrant usage of `set` from `is_set`, e.g. when handling system signals +All of :mod:`multiprocessing` ``Event`` is now reentrant and thread safe, and can thus be used from signal handlers. From c15a45a73172527451ca9e2427496cc0be0716cb Mon Sep 17 00:00:00 2001 From: Ivar Refsdal Date: Thu, 14 Nov 2024 15:04:25 +0100 Subject: [PATCH 09/14] gh-126434: Document 64 bit example of how many times reentrant set() call must be made to reach race condition possibility. --- Lib/multiprocessing/synchronize.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Lib/multiprocessing/synchronize.py b/Lib/multiprocessing/synchronize.py index 8d7f5245d57bd64..b2134c1923e8faf 100644 --- a/Lib/multiprocessing/synchronize.py +++ b/Lib/multiprocessing/synchronize.py @@ -343,7 +343,7 @@ def is_set(self): return self._flag.value == 1 def set(self): - # If `set` is called from a signal handler, this is fine as the lock is recursive (i.e. it won't deadlock). + # If set() is called from a signal handler, this is fine as the lock is recursive (i.e. it won't deadlock). # If the thread interrupted by the signal handler is wait()-ing and the signal handler calls set(), # this is fine as wait() spins on the value. Thus, after the signal handler is done, the thread will # return from wait() @@ -351,18 +351,20 @@ def set(self): with self._flag: with self._set_id: if self._flag.value == 0: - # There is a theoretical chance of race here. It requires the following conditions: + # There is a theoretical chance of a race here. It requires the following conditions: # The interrupted thread must be wait()ing. - # Then set must be called reentrant for the maximum value of c_ulonglong times, - # and all interruptions must happen exactly after `if self._flag.value == 0:`. + # Then set() must be called reentrant for the maximum value of c_ulonglong + 1 times, i.e. + # ./python.exe -c 'from ctypes import *; print(256**sizeof(c_ulonglong()))', which + # evaluates to 18446744073709551616 on a 64 bit machine. + # All interruptions must happen exactly after `if self._flag.value == 0:`. # The _set_id value will then wrap around. Then clear() must be called # before the original wait() code continues. The wait() code will then continue - # to (incorrectly) wait. I think this case is safe to ignore. The stack - # will grow too large before there is any chance of this actually happening. + # to (incorrectly) wait. This case should be safe to ignore. The stack + # will probably grow too large before there is any chance of this actually happening. self._flag.value = 1 self._set_id.value += 1 - # There is no race here by reentrant set when reaching the maximum value for `self._set_id.value`. + # There is no race here by reentrant set() when reaching the maximum value for `self._set_id.value`. # ctypes.c_ulonglong will overflow without any exception: # https://docs.python.org/3/library/ctypes.html#ctypes.c_ulonglong # > no overflow checking is done. From c308344675c182ae5937840224b72ffabfdb2d6b Mon Sep 17 00:00:00 2001 From: Ivar Refsdal Date: Thu, 28 Nov 2024 12:29:11 +0100 Subject: [PATCH 10/14] gh-126434: Process signals on dedicated thread --- Lib/multiprocessing/synchronize.py | 81 +++++++++--------------------- Lib/signal.py | 70 +++++++++++++++++++++++--- 2 files changed, 89 insertions(+), 62 deletions(-) diff --git a/Lib/multiprocessing/synchronize.py b/Lib/multiprocessing/synchronize.py index b2134c1923e8faf..4772e95e485b1eb 100644 --- a/Lib/multiprocessing/synchronize.py +++ b/Lib/multiprocessing/synchronize.py @@ -9,7 +9,7 @@ __all__ = [ 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Condition', 'Event' - ] +] import threading import sys @@ -157,7 +157,7 @@ def __repr__(self): except Exception: value = 'unknown' return '<%s(value=%s, maxvalue=%s)>' % \ - (self.__class__.__name__, value, self._semlock.maxvalue) + (self.__class__.__name__, value, self._semlock.maxvalue) # # Non-recursive lock @@ -253,7 +253,7 @@ def __repr__(self): def wait(self, timeout=None): assert self._lock._semlock._is_mine(), \ - 'must acquire() condition before using wait()' + 'must acquire() condition before using wait()' # indicate that this thread is going to sleep self._sleeping_count.release() @@ -328,72 +328,41 @@ def wait_for(self, predicate, timeout=None): class Event(object): def __init__(self, *, ctx): - self._flag = ctx.Value('i', 0) - # Allocate a ctypes.c_ulonglong to hold the set_id: - # Represents the C unsigned long long datatype. - # The constructor accepts an optional integer initializer; no overflow checking is done. - # From https://docs.python.org/3/library/ctypes.html#ctypes.c_ulonglong - # See multiprocessing/sharedctypes.py for typecode to ctypes definitions - self._set_id = ctx.Value('Q', 0) + self._cond = ctx.Condition(ctx.Lock()) + self._flag = ctx.Semaphore(0) def is_set(self): - # From https://docs.python.org/3/library/multiprocessing.html#multiprocessing.Value : - # If lock is True (the default) then a new recursive lock object is created to synchronize access to the value. - with self._flag: - return self._flag.value == 1 + with self._cond: + if self._flag.acquire(False): + self._flag.release() + return True + return False def set(self): - # If set() is called from a signal handler, this is fine as the lock is recursive (i.e. it won't deadlock). - # If the thread interrupted by the signal handler is wait()-ing and the signal handler calls set(), - # this is fine as wait() spins on the value. Thus, after the signal handler is done, the thread will - # return from wait() - # Fixes https://github.com/python/cpython/issues/85772 - with self._flag: - with self._set_id: - if self._flag.value == 0: - # There is a theoretical chance of a race here. It requires the following conditions: - # The interrupted thread must be wait()ing. - # Then set() must be called reentrant for the maximum value of c_ulonglong + 1 times, i.e. - # ./python.exe -c 'from ctypes import *; print(256**sizeof(c_ulonglong()))', which - # evaluates to 18446744073709551616 on a 64 bit machine. - # All interruptions must happen exactly after `if self._flag.value == 0:`. - # The _set_id value will then wrap around. Then clear() must be called - # before the original wait() code continues. The wait() code will then continue - # to (incorrectly) wait. This case should be safe to ignore. The stack - # will probably grow too large before there is any chance of this actually happening. - - self._flag.value = 1 - self._set_id.value += 1 - # There is no race here by reentrant set() when reaching the maximum value for `self._set_id.value`. - # ctypes.c_ulonglong will overflow without any exception: - # https://docs.python.org/3/library/ctypes.html#ctypes.c_ulonglong - # > no overflow checking is done. - # This means that we do not need to check if some maximum value is reached: - # C will wrap around the value for us. + with self._cond: + self._flag.acquire(False) + self._flag.release() + self._cond.notify_all() def clear(self): - with self._flag: - self._flag.value = 0 + with self._cond: + self._flag.acquire(False) def wait(self, timeout=None): - start_time = time.monotonic() - set_id = self._set_id.value - while True: - if self._flag.value == 1: - return True - elif set_id != self._set_id.value: - return True # flag is unset, but set_id changed, so there must have been a `set` followed by a `clear` - # during `time.sleep()`. Fixes https://github.com/python/cpython/issues/95826 - elif timeout is not None and (time.monotonic() - start_time) > timeout: - return False + with self._cond: + if self._flag.acquire(False): + self._flag.release() else: - # Fixes https://github.com/python/cpython/issues/85772 by spinning and sleeping. - time.sleep(0.010) # sleep 10 milliseconds + self._cond.wait(timeout) + + if self._flag.acquire(False): + self._flag.release() + return True + return False def __repr__(self) -> str: set_status = 'set' if self.is_set() else 'unset' return f"<{type(self).__qualname__} at {id(self):#x} {set_status}>" - # # Barrier # diff --git a/Lib/signal.py b/Lib/signal.py index c8cd3d4f597ca50..fa29fc1553ca9b7 100644 --- a/Lib/signal.py +++ b/Lib/signal.py @@ -1,6 +1,9 @@ import _signal +import os from _signal import * from enum import IntEnum as _IntEnum +import threading +import queue _globals = globals() @@ -42,6 +45,43 @@ def _enum_to_int(value): except (ValueError, TypeError): return value +_init_lock = threading.Lock() +_signal_queue = queue.SimpleQueue() # SimpleQueue has reentrant put, so it can safely be called from signal handlers. https://github.com/python/cpython/issues/59181 +_bubble_queue = queue.SimpleQueue() +_signal_thread = None +_signo_to_handler = {} + +def _signal_queue_handler(): + assert threading.current_thread() is not threading.main_thread() + global _signal_queue, _signo_to_handler, _bubble_queue + while True: + (signo, stack_frame) = _signal_queue.get() + try: + handler = _signo_to_handler.get(signo, None) + handler(signo, stack_frame) + except Exception as e: + _bubble_queue.put(e) + # _signal.raise_signal(SIGTERM) # does not work when using event.wait() + # _thread.interrupt_main(SIGTERM) # does not work when using event.wait() + os.kill(os.getpid(), signo) + +def _init_signal_thread(): + assert threading.current_thread() is threading.main_thread() + global _signal_thread, _init_lock + with _init_lock: + if _signal_thread is None: + _signal_thread = threading.Thread(target=_signal_queue_handler, daemon=True) + _signal_thread.start() + +def _push_signal_to_queue_handler(signo, stack_frame): + assert threading.current_thread() is threading.main_thread() + try: + global _bubble_queue + bubble_exception = _bubble_queue.get(block=False) + raise bubble_exception + except queue.Empty: + global _signal_queue + _signal_queue.put((signo, stack_frame)) # Similar to functools.wraps(), but only assign __doc__. # __module__ should be preserved, @@ -53,16 +93,34 @@ def decorator(wrapper): return wrapper return decorator -@_wraps(_signal.signal) -def signal(signalnum, handler): - handler = _signal.signal(_enum_to_int(signalnum), _enum_to_int(handler)) - return _int_to_enum(handler, Handlers) +def signal(signalnum, handler, use_dedicated_thread=True): + assert threading.current_thread() is threading.main_thread() + global _signo_to_handler + signal_int = _enum_to_int(signalnum) + old_handler = _signo_to_handler.get(signal_int, None) + if use_dedicated_thread and callable(handler): + assert callable(handler) + global _signal_thread + if _signal_thread is None: + _init_signal_thread() + _signo_to_handler[signal_int] = handler + handler = _signal.signal(signal_int, _enum_to_int(_push_signal_to_queue_handler)) + return old_handler or _int_to_enum(handler, Handlers) + else: + if signal_int in _signo_to_handler: + del _signo_to_handler[signal_int] + handler = _signal.signal(signal_int, _enum_to_int(handler)) + return old_handler or _int_to_enum(handler, Handlers) @_wraps(_signal.getsignal) def getsignal(signalnum): - handler = _signal.getsignal(signalnum) - return _int_to_enum(handler, Handlers) + global _signo_to_handler + if signalnum in _signo_to_handler: + return _signo_to_handler[signalnum] + else: + handler = _signal.getsignal(signalnum) + return _int_to_enum(handler, Handlers) if 'pthread_sigmask' in _globals: From 5a43697d5b316c524aafeb5110f4454f968ca906 Mon Sep 17 00:00:00 2001 From: Ivar Refsdal Date: Tue, 3 Dec 2024 11:43:16 +0100 Subject: [PATCH 11/14] gh-126434: Continue process signals on dedicated thread, don't bubble exception on main thread --- Lib/signal.py | 85 ++++++++++++------- .../test_multiprocessing_event/__init__.py | 5 -- .../test_multiprocessing_event/test_event.py | 73 ---------------- Makefile.pre.in | 1 - bug.py | 45 ++++++++++ 5 files changed, 99 insertions(+), 110 deletions(-) delete mode 100644 Lib/test/test_multiprocessing_event/__init__.py delete mode 100644 Lib/test/test_multiprocessing_event/test_event.py create mode 100755 bug.py diff --git a/Lib/signal.py b/Lib/signal.py index fa29fc1553ca9b7..cda720e2bccc93e 100644 --- a/Lib/signal.py +++ b/Lib/signal.py @@ -1,9 +1,9 @@ import _signal -import os from _signal import * from enum import IntEnum as _IntEnum import threading import queue +import traceback _globals = globals() @@ -45,43 +45,65 @@ def _enum_to_int(value): except (ValueError, TypeError): return value -_init_lock = threading.Lock() _signal_queue = queue.SimpleQueue() # SimpleQueue has reentrant put, so it can safely be called from signal handlers. https://github.com/python/cpython/issues/59181 -_bubble_queue = queue.SimpleQueue() _signal_thread = None _signo_to_handler = {} -def _signal_queue_handler(): - assert threading.current_thread() is not threading.main_thread() - global _signal_queue, _signo_to_handler, _bubble_queue - while True: - (signo, stack_frame) = _signal_queue.get() - try: - handler = _signo_to_handler.get(signo, None) - handler(signo, stack_frame) - except Exception as e: - _bubble_queue.put(e) - # _signal.raise_signal(SIGTERM) # does not work when using event.wait() - # _thread.interrupt_main(SIGTERM) # does not work when using event.wait() - os.kill(os.getpid(), signo) - def _init_signal_thread(): assert threading.current_thread() is threading.main_thread() - global _signal_thread, _init_lock - with _init_lock: - if _signal_thread is None: - _signal_thread = threading.Thread(target=_signal_queue_handler, daemon=True) - _signal_thread.start() + global _signal_thread + if _signal_thread is None: + _signal_thread = threading.Thread(target=_signal_queue_handler, daemon=True) + _signal_thread.name = 'SignalHandlerThread' + _signal_thread.start() -def _push_signal_to_queue_handler(signo, stack_frame): +def _push_signal_to_queue_handler(signo, _stack_frame): assert threading.current_thread() is threading.main_thread() + global _signal_queue + _signal_queue.put(signo) + +def _sigint_to_str(signo): + for x in valid_signals(): + if x == signo: + return x.name + raise RuntimeError('Could not find signal name') + +def _log_missing_signal_handler(signo): + import logging + logger = logging.getLogger(__name__) + str_name = '' + for x in valid_signals(): + if x == signo: + str_name = x.name + logger.warning('Handler for signal.%s (%d) was not found.', str_name, signo) + +def _stop_signal_thread(): + global _signal_thread, _signal_queue + if _signal_thread is not None: + _signal_queue.put('STOP_SIGNAL_HANDLER') + +def _signal_queue_handler(): try: - global _bubble_queue - bubble_exception = _bubble_queue.get(block=False) - raise bubble_exception - except queue.Empty: - global _signal_queue - _signal_queue.put((signo, stack_frame)) + assert threading.current_thread() is not threading.main_thread() + global _signal_queue, _signo_to_handler + while True: + signo = _signal_queue.get() + if signo == 'STOP_SIGNAL_HANDLER': + break + try: + handler = _signo_to_handler.get(signo, None) + if handler is not None: + handler(signo, None) + else: + _log_missing_signal_handler(signo) + except Exception: + traceback.print_exc() + pass + except: + pass + finally: + global _signal_thread + _signal_thread = None # Similar to functools.wraps(), but only assign __doc__. # __module__ should be preserved, @@ -93,8 +115,9 @@ def decorator(wrapper): return wrapper return decorator -def signal(signalnum, handler, use_dedicated_thread=True): - assert threading.current_thread() is threading.main_thread() +def signal(signalnum, handler, use_dedicated_thread=False): + if use_dedicated_thread: + assert threading.current_thread() is threading.main_thread() global _signo_to_handler signal_int = _enum_to_int(signalnum) old_handler = _signo_to_handler.get(signal_int, None) diff --git a/Lib/test/test_multiprocessing_event/__init__.py b/Lib/test/test_multiprocessing_event/__init__.py deleted file mode 100644 index b90f7b4cbd9a486..000000000000000 --- a/Lib/test/test_multiprocessing_event/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -import os.path -from test import support - -def load_tests(*args): - return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_multiprocessing_event/test_event.py b/Lib/test/test_multiprocessing_event/test_event.py deleted file mode 100644 index 3e8088d3ad6e499..000000000000000 --- a/Lib/test/test_multiprocessing_event/test_event.py +++ /dev/null @@ -1,73 +0,0 @@ -import unittest -from test import support -import sys -import signal -import os - - -try: - import multiprocessing - from concurrent.futures import ProcessPoolExecutor - _have_multiprocessing = True -except (NotImplementedError, ModuleNotFoundError): - _have_multiprocessing = False - - -@unittest.skipUnless(_have_multiprocessing, - "requires multiprocessing") -@unittest.skipUnless(hasattr(signal, 'signal'), - "Requires signal.signal") -@unittest.skipUnless(hasattr(signal, 'SIGINT'), - "Requires signal.SIGINT") -@unittest.skipUnless(hasattr(os, 'kill'), - "Requires os.kill") -@unittest.skipUnless(hasattr(os, 'getppid'), - "Requires os.getppid") -@support.requires_subprocess() -class TestEventSignalHandling(unittest.TestCase): - def test_no_race_for_is_set_set(self): - import subprocess - script = support.findfile("is_set_set.py", subdir="multiprocessingdata") - for x in range(10): - try: - assert 0 == subprocess.call([sys.executable, script], timeout=60) - except subprocess.TimeoutExpired: - assert False, 'subprocess.Timeoutexpired for is_set_set.py' - - def test_no_race_set_clear(self): - import subprocess - script = support.findfile("set_clear_race.py", subdir="multiprocessingdata") - assert 0 == subprocess.call([sys.executable, script]) - - def test_wait_set_no_deadlock(self): - import subprocess - script = support.findfile("wait_set_no_deadlock.py", subdir="multiprocessingdata") - assert 0 == subprocess.call([sys.executable, script]) - - def test_wait_timeout(self): - event = multiprocessing.Event() - # https://docs.python.org/3/library/multiprocessing.html#multiprocessing.Event - # multiprocessing.Event: A clone of threading.Event. - - # threading.Event: https://docs.python.org/3/library/threading.html#threading.Event - - # threading.Event.wait(): https://docs.python.org/3/library/threading.html#threading.Event.wait - # Block as long as the internal flag is false and the timeout, if given, has not expired. - # The return value represents the reason that this blocking method returned; - # True if returning because the internal flag is set to true, or - # False if a timeout is given and the internal flag did not become true within the given wait time. - - # When the timeout argument is present and not None, it should be a floating-point number - # specifying a timeout for the operation in seconds, or fractions thereof. - - # wait() supports both integer and float: - flag_set = event.wait(1) - assert flag_set == False - - flag_set = event.wait(0.1) - assert flag_set == False - - -if __name__ == '__main__': - unittest.main() - diff --git a/Makefile.pre.in b/Makefile.pre.in index 5296de1d7ea7fd0..c7ff502ec7c7b67 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2520,7 +2520,6 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_interpreters \ test/test_json \ test/test_module \ - test/test_multiprocessing_event \ test/test_multiprocessing_fork \ test/test_multiprocessing_forkserver \ test/test_multiprocessing_spawn \ diff --git a/bug.py b/bug.py new file mode 100755 index 000000000000000..d35f7505ae50869 --- /dev/null +++ b/bug.py @@ -0,0 +1,45 @@ +import os +import signal +import threading +import time +import multiprocessing + +def sigint_self(): + time.sleep(1) + print(f'{threading.current_thread().name}: Stopping PID {os.getpid()} with SIGINT') + os.kill(os.getpid(), signal.SIGINT) + +def sigkill_self(): + time.sleep(5) + print(f'{threading.current_thread().name}: Killing PID {os.getpid()} with SIGKILL') + os.kill(os.getpid(), signal.SIGKILL) + +def run_signal_handler_dedicated_thread(): + event = multiprocessing.Event() + def sigint_handler(_signo, _stack_frame): + try: + x = 1 / 0 + print(f'{threading.current_thread().name}: sigint_handler is setting event') + event.set() + finally: + print(f'{threading.current_thread().name}: sigint_handler is done') + + def sigterm_handler(_signo, _stack_frame): + print(f'{threading.current_thread().name}: sigterm_handler is running') + pass + + signal.signal(signal.SIGTERM, sigterm_handler) + signal.signal(signal.SIGINT, sigint_handler) + + threading.Thread(target=sigint_self, daemon=True).start() + threading.Thread(target=sigkill_self, daemon=True).start() # Used for debugging only. + + print(f'{threading.current_thread().name}: Waiting on event. PID = {os.getpid()}') + event.wait() + print(f'{threading.current_thread().name}: Waiting is done') + +if __name__ == '__main__': + try: + run_signal_handler_dedicated_thread() + finally: + print(f'{threading.current_thread().name}: Exiting') From 574e91c6bbb845d7c9266aede79b578aa851a8a8 Mon Sep 17 00:00:00 2001 From: Ivar Refsdal Date: Tue, 3 Dec 2024 12:11:52 +0100 Subject: [PATCH 12/14] =?UTF-8?q?gh-126434:=20Start=20fix=20tests=20when?= =?UTF-8?q?=20using=20use=5Fdedicated=5Fthread=3DTrue=20by=20default=CB=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Lib/signal.py | 4 ++-- Lib/test/test_asyncio/test_base_events.py | 2 ++ Lib/test/test_asyncio/test_context.py | 2 ++ Lib/test/test_asyncio/test_futures2.py | 2 ++ Lib/test/test_asyncio/test_locks.py | 3 ++- Lib/test/test_asyncio/test_queues.py | 3 ++- Lib/test/test_asyncio/test_runners.py | 1 + Lib/test/test_asyncio/test_server.py | 2 ++ Lib/test/test_asyncio/test_staggered.py | 2 ++ Lib/test/test_asyncio/test_taskgroups.py | 4 ++-- Lib/test/test_asyncio/test_threads.py | 3 ++- Lib/test/test_asyncio/test_timeouts.py | 3 ++- 12 files changed, 23 insertions(+), 8 deletions(-) diff --git a/Lib/signal.py b/Lib/signal.py index cda720e2bccc93e..c2064fce64c5e53 100644 --- a/Lib/signal.py +++ b/Lib/signal.py @@ -77,7 +77,7 @@ def _log_missing_signal_handler(signo): str_name = x.name logger.warning('Handler for signal.%s (%d) was not found.', str_name, signo) -def _stop_signal_thread(): +def stop_signal_thread(): global _signal_thread, _signal_queue if _signal_thread is not None: _signal_queue.put('STOP_SIGNAL_HANDLER') @@ -115,7 +115,7 @@ def decorator(wrapper): return wrapper return decorator -def signal(signalnum, handler, use_dedicated_thread=False): +def signal(signalnum, handler, use_dedicated_thread=True): if use_dedicated_thread: assert threading.current_thread() is threading.main_thread() global _signo_to_handler diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index c14a0bb180d79bd..6bd4df41c553f45 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -4,6 +4,7 @@ import errno import math import platform +import signal import socket import sys import threading @@ -26,6 +27,7 @@ def tearDownModule(): asyncio.set_event_loop_policy(None) + signal.stop_signal_thread() def mock_socket_module(): diff --git a/Lib/test/test_asyncio/test_context.py b/Lib/test/test_asyncio/test_context.py index 6b80721873d95cb..54770f926a8fe9e 100644 --- a/Lib/test/test_asyncio/test_context.py +++ b/Lib/test/test_asyncio/test_context.py @@ -1,10 +1,12 @@ import asyncio import decimal import unittest +import signal def tearDownModule(): asyncio.set_event_loop_policy(None) + signal.stop_signal_thread() @unittest.skipUnless(decimal.HAVE_CONTEXTVAR, "decimal is built with a thread-local context") diff --git a/Lib/test/test_asyncio/test_futures2.py b/Lib/test/test_asyncio/test_futures2.py index b7cfffb76bd8f17..1f1ef25c049c1a1 100644 --- a/Lib/test/test_asyncio/test_futures2.py +++ b/Lib/test/test_asyncio/test_futures2.py @@ -3,11 +3,13 @@ import contextvars import traceback import unittest +import signal from asyncio import tasks def tearDownModule(): asyncio.set_event_loop_policy(None) + signal.stop_signal_thread() class FutureTests: diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index c3bff760f7307e3..2b16bc52a2aab0f 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -1,5 +1,5 @@ """Tests for locks.py""" - +import signal import unittest from unittest import mock import re @@ -21,6 +21,7 @@ def tearDownModule(): asyncio.set_event_loop_policy(None) + signal.stop_signal_thread() class LockTests(unittest.IsolatedAsyncioTestCase): diff --git a/Lib/test/test_asyncio/test_queues.py b/Lib/test/test_asyncio/test_queues.py index 5019e9a293525d1..f5e5586c3a8abcf 100644 --- a/Lib/test/test_asyncio/test_queues.py +++ b/Lib/test/test_asyncio/test_queues.py @@ -1,13 +1,14 @@ """Tests for queues.py""" import asyncio +import signal import unittest from types import GenericAlias def tearDownModule(): asyncio.set_event_loop_policy(None) - + signal.stop_signal_thread() class QueueBasicTests(unittest.IsolatedAsyncioTestCase): diff --git a/Lib/test/test_asyncio/test_runners.py b/Lib/test/test_asyncio/test_runners.py index 45f70d09a2083a0..476aa05e2fbd88b 100644 --- a/Lib/test/test_asyncio/test_runners.py +++ b/Lib/test/test_asyncio/test_runners.py @@ -13,6 +13,7 @@ def tearDownModule(): asyncio.set_event_loop_policy(None) + signal.stop_signal_thread() def interrupt_self(): diff --git a/Lib/test/test_asyncio/test_server.py b/Lib/test/test_asyncio/test_server.py index 60a40cc8349fed5..04d86ce9dfef611 100644 --- a/Lib/test/test_asyncio/test_server.py +++ b/Lib/test/test_asyncio/test_server.py @@ -1,5 +1,6 @@ import asyncio import os +import signal import socket import time import threading @@ -12,6 +13,7 @@ def tearDownModule(): asyncio.set_event_loop_policy(None) + signal.stop_signal_thread() class BaseStartServer(func_tests.FunctionalTestCaseMixin): diff --git a/Lib/test/test_asyncio/test_staggered.py b/Lib/test/test_asyncio/test_staggered.py index 74941f704c48901..ef1394d1be53782 100644 --- a/Lib/test/test_asyncio/test_staggered.py +++ b/Lib/test/test_asyncio/test_staggered.py @@ -1,4 +1,5 @@ import asyncio +import signal import unittest from asyncio.staggered import staggered_race @@ -9,6 +10,7 @@ def tearDownModule(): asyncio.set_event_loop_policy(None) + signal.stop_signal_thread() class StaggeredTests(unittest.IsolatedAsyncioTestCase): diff --git a/Lib/test/test_asyncio/test_taskgroups.py b/Lib/test/test_asyncio/test_taskgroups.py index 1b4de96a572fb94..7c2697190b17464 100644 --- a/Lib/test/test_asyncio/test_taskgroups.py +++ b/Lib/test/test_asyncio/test_taskgroups.py @@ -1,6 +1,6 @@ # Adapted with permission from the EdgeDB project; # license: PSFL. - +import signal import sys import gc import asyncio @@ -15,7 +15,7 @@ # To prevent a warning "test altered the execution environment" def tearDownModule(): asyncio.set_event_loop_policy(None) - + signal.stop_signal_thread() class MyExc(Exception): pass diff --git a/Lib/test/test_asyncio/test_threads.py b/Lib/test/test_asyncio/test_threads.py index 774380270a7d70a..39c5d4a49296ee8 100644 --- a/Lib/test/test_asyncio/test_threads.py +++ b/Lib/test/test_asyncio/test_threads.py @@ -1,6 +1,7 @@ """Tests for asyncio/threads.py""" import asyncio +import signal import unittest from contextvars import ContextVar @@ -9,7 +10,7 @@ def tearDownModule(): asyncio.set_event_loop_policy(None) - + signal.stop_signal_thread() class ToThreadTests(unittest.IsolatedAsyncioTestCase): async def test_to_thread(self): diff --git a/Lib/test/test_asyncio/test_timeouts.py b/Lib/test/test_asyncio/test_timeouts.py index f5543e191d07ff9..283391daa29482b 100644 --- a/Lib/test/test_asyncio/test_timeouts.py +++ b/Lib/test/test_asyncio/test_timeouts.py @@ -1,5 +1,5 @@ """Tests for asyncio/timeouts.py""" - +import signal import unittest import time @@ -10,6 +10,7 @@ def tearDownModule(): asyncio.set_event_loop_policy(None) + signal.stop_signal_thread() class TimeoutTests(unittest.IsolatedAsyncioTestCase): From 25f4c7e2995dad5a5c37be81f54e3510a6c486f7 Mon Sep 17 00:00:00 2001 From: Ivar Refsdal Date: Tue, 3 Dec 2024 19:57:50 +0100 Subject: [PATCH 13/14] gh-126434: What should be done if sys.exit() is called from a signal handler (on a non-main thread)? --- Lib/signal.py | 15 +++++++++++---- Lib/test/support/threading_helper.py | 3 +++ Lib/test/test_asyncio/test_unix_events.py | 1 + bug.py | 14 +++++++++----- 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/Lib/signal.py b/Lib/signal.py index c2064fce64c5e53..65e591a1dabee74 100644 --- a/Lib/signal.py +++ b/Lib/signal.py @@ -81,6 +81,8 @@ def stop_signal_thread(): global _signal_thread, _signal_queue if _signal_thread is not None: _signal_queue.put('STOP_SIGNAL_HANDLER') + _signal_thread.join() + _signal_thread = None def _signal_queue_handler(): try: @@ -98,12 +100,15 @@ def _signal_queue_handler(): _log_missing_signal_handler(signo) except Exception: traceback.print_exc() - pass + except SystemExit: + pass # TODO: what should be done in the event of a handler calling `sys.exit()`? except: - pass + traceback.print_exc() + # import _thread + # _thread.interrupt_main() + # print(dir(threading.main_thread())) finally: - global _signal_thread - _signal_thread = None + pass # Similar to functools.wraps(), but only assign __doc__. # __module__ should be preserved, @@ -132,6 +137,8 @@ def signal(signalnum, handler, use_dedicated_thread=True): else: if signal_int in _signo_to_handler: del _signo_to_handler[signal_int] + if 0 == len(_signo_to_handler): + stop_signal_thread() handler = _signal.signal(signal_int, _enum_to_int(handler)) return old_handler or _int_to_enum(handler, Handlers) diff --git a/Lib/test/support/threading_helper.py b/Lib/test/support/threading_helper.py index afa25a76f638295..ab8608df93f139e 100644 --- a/Lib/test/support/threading_helper.py +++ b/Lib/test/support/threading_helper.py @@ -5,6 +5,7 @@ import threading import time import unittest +import signal from test import support @@ -28,6 +29,8 @@ def threading_setup(): def threading_cleanup(*original_values): orig_count, orig_ndangling = original_values + signal.stop_signal_thread() + timeout = 1.0 for _ in support.sleeping_retry(timeout, error=False): # Copy the thread list to get a consistent output. threading._dangling diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index 021f45478d6f484..e09a196ddc971e4 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -34,6 +34,7 @@ def tearDownModule(): asyncio.set_event_loop_policy(None) + signal.stop_signal_thread() MOCK_ANY = mock.ANY diff --git a/bug.py b/bug.py index d35f7505ae50869..ec333e8180ecac7 100755 --- a/bug.py +++ b/bug.py @@ -3,6 +3,7 @@ import threading import time import multiprocessing +import sys def sigint_self(): time.sleep(1) @@ -18,9 +19,9 @@ def run_signal_handler_dedicated_thread(): event = multiprocessing.Event() def sigint_handler(_signo, _stack_frame): try: - x = 1 / 0 print(f'{threading.current_thread().name}: sigint_handler is setting event') - event.set() + #event.set() + sys.exit() finally: print(f'{threading.current_thread().name}: sigint_handler is done') @@ -28,14 +29,17 @@ def sigterm_handler(_signo, _stack_frame): print(f'{threading.current_thread().name}: sigterm_handler is running') pass - signal.signal(signal.SIGTERM, sigterm_handler) - signal.signal(signal.SIGINT, sigint_handler) + signal.signal(signal.SIGINT, sigint_handler, True) threading.Thread(target=sigint_self, daemon=True).start() threading.Thread(target=sigkill_self, daemon=True).start() # Used for debugging only. print(f'{threading.current_thread().name}: Waiting on event. PID = {os.getpid()}') - event.wait() + while True: + if event.is_set(): + break + else: + time.sleep(0.1) print(f'{threading.current_thread().name}: Waiting is done') if __name__ == '__main__': From 88b45b92a7450efc671b6bf08d60ea976ba63973 Mon Sep 17 00:00:00 2001 From: Ivar Refsdal Date: Tue, 3 Dec 2024 21:22:29 +0100 Subject: [PATCH 14/14] gh-126434: Propagate sys.exit from signal handler to main thread --- Lib/signal.py | 57 ++++++++++++++++++++++++++++----------------------- bug.py | 13 +++++++++--- 2 files changed, 41 insertions(+), 29 deletions(-) diff --git a/Lib/signal.py b/Lib/signal.py index 65e591a1dabee74..2b2381b73c6c8a6 100644 --- a/Lib/signal.py +++ b/Lib/signal.py @@ -1,4 +1,5 @@ import _signal +import sys from _signal import * from enum import IntEnum as _IntEnum import threading @@ -46,6 +47,7 @@ def _enum_to_int(value): return value _signal_queue = queue.SimpleQueue() # SimpleQueue has reentrant put, so it can safely be called from signal handlers. https://github.com/python/cpython/issues/59181 +_sys_exit_queue = queue.SimpleQueue() _signal_thread = None _signo_to_handler = {} @@ -59,8 +61,12 @@ def _init_signal_thread(): def _push_signal_to_queue_handler(signo, _stack_frame): assert threading.current_thread() is threading.main_thread() - global _signal_queue - _signal_queue.put(signo) + global _signal_queue, _sys_exit_queue + try: + exit_code = _sys_exit_queue.get(block=False) + sys.exit(exit_code) + except queue.Empty: + _signal_queue.put(signo) def _sigint_to_str(signo): for x in valid_signals(): @@ -85,30 +91,29 @@ def stop_signal_thread(): _signal_thread = None def _signal_queue_handler(): - try: - assert threading.current_thread() is not threading.main_thread() - global _signal_queue, _signo_to_handler - while True: - signo = _signal_queue.get() - if signo == 'STOP_SIGNAL_HANDLER': - break - try: - handler = _signo_to_handler.get(signo, None) - if handler is not None: - handler(signo, None) - else: - _log_missing_signal_handler(signo) - except Exception: - traceback.print_exc() - except SystemExit: - pass # TODO: what should be done in the event of a handler calling `sys.exit()`? - except: - traceback.print_exc() - # import _thread - # _thread.interrupt_main() - # print(dir(threading.main_thread())) - finally: - pass + assert threading.current_thread() is not threading.main_thread() + global _signal_queue, _signo_to_handler + while True: + signo = _signal_queue.get() + if signo == 'STOP_SIGNAL_HANDLER': + break + raise_systemexit = False + exitcode = 'NOTSET' + try: + handler = _signo_to_handler.get(signo, None) + if handler is not None: + handler(signo, None) + else: + _log_missing_signal_handler(signo) + except SystemExit as se: + exitcode = se.code + raise_systemexit = True + except Exception: + traceback.print_exc() + if raise_systemexit: + global _sys_exit_queue + _sys_exit_queue.put(exitcode) + raise_signal(signo) # Similar to functools.wraps(), but only assign __doc__. # __module__ should be preserved, diff --git a/bug.py b/bug.py index ec333e8180ecac7..317fd50f02b087d 100755 --- a/bug.py +++ b/bug.py @@ -19,17 +19,24 @@ def run_signal_handler_dedicated_thread(): event = multiprocessing.Event() def sigint_handler(_signo, _stack_frame): try: - print(f'{threading.current_thread().name}: sigint_handler is setting event') - #event.set() + # print(f'{threading.current_thread().name}: sigint_handler raising SIGUSR1 ...') + # signal.raise_signal(signal.SIGUSR1) + # print(f'{threading.current_thread().name}: sigint_handler raising SIGUSR1 ... OK') sys.exit() finally: - print(f'{threading.current_thread().name}: sigint_handler is done') + print(f'{threading.current_thread().name}: sigint_handler exiting ...') + + def sigusr1_handler(_signo, _): + print(f'{threading.current_thread().name}: USR1 running') def sigterm_handler(_signo, _stack_frame): print(f'{threading.current_thread().name}: sigterm_handler is running') pass signal.signal(signal.SIGINT, sigint_handler, True) + signal.signal(signal.SIGUSR1, sigusr1_handler, True) + + # signal.raise_signal(signal.SIGINT) threading.Thread(target=sigint_self, daemon=True).start() threading.Thread(target=sigkill_self, daemon=True).start() # Used for debugging only.