From eab1df122787acc287d80b7feedb7fdf5feaa619 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Fri, 25 Aug 2023 11:53:16 +0100 Subject: [PATCH 1/5] Try to fix stubtest --- mypy/stubtest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index 34bb985b702eb..704598c820cf3 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -1724,8 +1724,10 @@ def get_importable_stdlib_modules() -> set[str]: try: silent_import_module(submodule_name) + except KeyboardInterrupt: + raise # importing multiprocessing.popen_forkserver on Windows raises AttributeError... - except Exception: + except BaseException: continue else: importable_stdlib_modules.add(submodule_name) From ee2b44fc8290a0cccb3ae92630277637aaff7331 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Fri, 25 Aug 2023 12:24:53 +0100 Subject: [PATCH 2/5] fix stdlib-finding logic --- mypy/stubtest.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index 704598c820cf3..85c380ce88a0e 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -23,6 +23,7 @@ import typing import typing_extensions import warnings +from collections import defaultdict from contextlib import redirect_stderr, redirect_stdout from functools import singledispatch from pathlib import Path @@ -1679,16 +1680,16 @@ def get_importable_stdlib_modules() -> set[str]: all_stdlib_modules = sys.stdlib_module_names else: all_stdlib_modules = set(sys.builtin_module_names) - python_exe_dir = Path(sys.executable).parent + modules_by_finder: defaultdict[importlib.machinery.FileFinder, set[str]] = defaultdict(set) for m in pkgutil.iter_modules(): - finder = m.module_finder - if isinstance(finder, importlib.machinery.FileFinder): - finder_path = Path(finder.path) - if ( - python_exe_dir in finder_path.parents - and "site-packages" not in finder_path.parts - ): - all_stdlib_modules.add(m.name) + if isinstance(m.module_finder, importlib.machinery.FileFinder): + modules_by_finder[m.module_finder].add(m.name) + for finder, module_group in modules_by_finder.items(): + if ( + "site-packages" not in Path(finder.path).parents + and {"json", "_json", "_queue"} & module_group + ): + all_stdlib_modules.update(module_group) importable_stdlib_modules: set[str] = set() for module_name in all_stdlib_modules: From 1e5929622a8ee6ed8e8dfdfbcd855f4175dda8cb Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Fri, 25 Aug 2023 12:30:20 +0100 Subject: [PATCH 3/5] More --- mypy/stubtest.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index 85c380ce88a0e..bca37153bd5c3 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -1720,7 +1720,13 @@ def get_importable_stdlib_modules() -> set[str]: # The idlelib.* submodules are similarly annoying in opening random tkinter windows, # and we're unlikely to ever add stubs for idlelib in typeshed # (see discussion in https://github.com/python/typeshed/pull/9193) - if submodule_name.endswith(".__main__") or submodule_name.startswith("idlelib."): + # + # test.* modules do weird things like raising unraisable exceptions in __del__ methods + if ( + submodule_name.endswith(".__main__") + or submodule_name.startswith("idlelib.") + or submodule_name.startswith("test.") + ): continue try: From 258fca78fba2f50fedcd54a52fd793468806c459 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Fri, 25 Aug 2023 12:38:46 +0100 Subject: [PATCH 4/5] improve comment --- mypy/stubtest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index bca37153bd5c3..414d8474a18fd 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -1721,7 +1721,9 @@ def get_importable_stdlib_modules() -> set[str]: # and we're unlikely to ever add stubs for idlelib in typeshed # (see discussion in https://github.com/python/typeshed/pull/9193) # - # test.* modules do weird things like raising unraisable exceptions in __del__ methods + # test.* modules do weird things like raising exceptions in __del__ methods, + # leading to unraisable exceptions being logged to the terminal + # as a warning at the end of the stubtest run if ( submodule_name.endswith(".__main__") or submodule_name.startswith("idlelib.") From bf9fdad2ae7542fb85ae9ffba2b488924240769a Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Fri, 25 Aug 2023 17:11:13 +0100 Subject: [PATCH 5/5] Add comments --- mypy/stubtest.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index 414d8474a18fd..a804835a632bf 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -1687,7 +1687,13 @@ def get_importable_stdlib_modules() -> set[str]: for finder, module_group in modules_by_finder.items(): if ( "site-packages" not in Path(finder.path).parents - and {"json", "_json", "_queue"} & module_group + # if "_queue" is present, it's most likely the module finder + # for stdlib extension modules; + # if "queue" is present, it's most likely the module finder + # for pure-Python stdlib modules. + # In either case, we'll want to add all the modules that the finder has to offer us. + # This is a bit hacky, but seems to work well in a cross-platform way. + and {"_queue", "queue"} & module_group ): all_stdlib_modules.update(module_group) @@ -1736,6 +1742,8 @@ def get_importable_stdlib_modules() -> set[str]: except KeyboardInterrupt: raise # importing multiprocessing.popen_forkserver on Windows raises AttributeError... + # some submodules also appear to raise SystemExit as well on some Python versions + # (not sure exactly which) except BaseException: continue else: