From fc31dfe2e9d787082d5cccb88b91c85359295bc5 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Sun, 30 Jun 2019 18:01:54 +0200 Subject: [PATCH 1/3] modulefinder: use only dirs containing no __init__.py as namespace package When used with --namespace-packages, a directory containing no __init__.pyi was previously added as a candidate to near_misses even if it contained a __init__.py. This logic doesn't seem sensible, particularly as this directory was then chosen as the result of find_module() instead of the __init__.py inside the directory. This should cure #5854. --- mypy/modulefinder.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index bd331c7dbe78d..1cbdf7a04604a 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -212,6 +212,7 @@ def _find_module(self, id: str) -> Optional[str]: near_misses = [] # Collect near misses for namespace mode (see below). for base_dir, verify in candidate_base_dirs: base_path = base_dir + seplast # so e.g. '/usr/lib/python3.4/foo/bar/baz' + has_init = False dir_prefix = base_dir for _ in range(len(components) - 1): dir_prefix = os.path.dirname(dir_prefix) @@ -220,6 +221,7 @@ def _find_module(self, id: str) -> Optional[str]: path = base_path + sepinit + extension path_stubs = base_path + '-stubs' + sepinit + extension if fscache.isfile_case(path, dir_prefix): + has_init = True if verify and not verify_module(fscache, id, path, dir_prefix): near_misses.append((path, dir_prefix)) continue @@ -229,8 +231,12 @@ def _find_module(self, id: str) -> Optional[str]: near_misses.append((path_stubs, dir_prefix)) continue return path_stubs - elif self.options and self.options.namespace_packages and fscache.isdir(base_path): + + # In namespace mode, register a potential namespace package + if self.options and self.options.namespace_packages: + if fscache.isdir(base_path) and not has_init: near_misses.append((base_path, dir_prefix)) + # No package, look for module. for extension in PYTHON_EXTENSIONS: path = base_path + extension From 3bc56d2c9aa93efd3f73d008d6d5b2ca9c784bcd Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Sun, 30 Jun 2019 18:04:00 +0200 Subject: [PATCH 2/3] tests/modulefinder: tests for FindModuleCache.find_module() Those tests go hand in hand with the directory structure that was created in test-data/packages/modulefinder. --- mypy/test/testmodulefinder.py | 127 ++++++++++++++++++ .../modulefinder/nsx-pkg1/nsx/a/__init__.py | 0 .../modulefinder/nsx-pkg2/nsx/b/__init__.py | 0 .../packages/modulefinder/nsx-pkg3/nsx/c/c.py | 0 .../modulefinder/nsy-pkg1/nsy/a/__init__.py | 0 .../modulefinder/nsy-pkg1/nsy/a/__init__.pyi | 0 .../packages/modulefinder/nsy-pkg2/nsy/b.pyi | 0 .../modulefinder/nsy-pkg2/nsy/b/__init__.py | 0 .../packages/modulefinder/nsy-pkg2/nsy/c.py | 0 .../packages/modulefinder/nsy-pkg2/nsy/c.pyi | 0 test-data/packages/modulefinder/pkg4/a.py | 0 .../packages/modulefinder/pkg5/b/__init__.py | 0 test-data/packages/modulefinder/readme.txt | 4 + 13 files changed, 131 insertions(+) create mode 100644 mypy/test/testmodulefinder.py create mode 100644 test-data/packages/modulefinder/nsx-pkg1/nsx/a/__init__.py create mode 100644 test-data/packages/modulefinder/nsx-pkg2/nsx/b/__init__.py create mode 100644 test-data/packages/modulefinder/nsx-pkg3/nsx/c/c.py create mode 100644 test-data/packages/modulefinder/nsy-pkg1/nsy/a/__init__.py create mode 100644 test-data/packages/modulefinder/nsy-pkg1/nsy/a/__init__.pyi create mode 100644 test-data/packages/modulefinder/nsy-pkg2/nsy/b.pyi create mode 100644 test-data/packages/modulefinder/nsy-pkg2/nsy/b/__init__.py create mode 100644 test-data/packages/modulefinder/nsy-pkg2/nsy/c.py create mode 100644 test-data/packages/modulefinder/nsy-pkg2/nsy/c.pyi create mode 100644 test-data/packages/modulefinder/pkg4/a.py create mode 100644 test-data/packages/modulefinder/pkg5/b/__init__.py create mode 100644 test-data/packages/modulefinder/readme.txt diff --git a/mypy/test/testmodulefinder.py b/mypy/test/testmodulefinder.py new file mode 100644 index 0000000000000..46d4e9952eee0 --- /dev/null +++ b/mypy/test/testmodulefinder.py @@ -0,0 +1,127 @@ +import os + +from mypy.options import Options +from mypy.modulefinder import FindModuleCache, SearchPaths + +from mypy.test.helpers import Suite, assert_equal +from mypy.test.config import package_path +data_path = os.path.relpath(os.path.join(package_path, "modulefinder")) + + +class ModuleFinderSuite(Suite): + + def setUp(self) -> None: + self.search_paths = SearchPaths( + python_path=(), + mypy_path=( + os.path.join(data_path, "nsx-pkg1"), + os.path.join(data_path, "nsx-pkg2"), + os.path.join(data_path, "nsx-pkg3"), + os.path.join(data_path, "nsy-pkg1"), + os.path.join(data_path, "nsy-pkg2"), + os.path.join(data_path, "pkg4"), + os.path.join(data_path, "pkg5"), + ), + package_path=(), + typeshed_path=(), + ) + options = Options() + options.namespace_packages = True + self.fmc_ns = FindModuleCache(self.search_paths, options=options) + + options = Options() + options.namespace_packages = False + self.fmc_nons = FindModuleCache(self.search_paths, options=options) + + def test__no_namespace_packages__nsx(self) -> None: + """ + If namespace_packages is False, we shouldn't find nsx + """ + found_module = self.fmc_nons.find_module("nsx") + self.assertIsNone(found_module) + + def test__no_namespace_packages__nsx_a(self) -> None: + """ + If namespace_packages is False, we shouldn't find nsx.a. + """ + found_module = self.fmc_nons.find_module("nsx.a") + self.assertIsNone(found_module) + + def test__no_namespace_packages__find_a_in_pkg4(self) -> None: + """ + Find find pkg4/a.py for "a" with namespace_packages False. + """ + found_module = self.fmc_nons.find_module("a") + expected = os.path.join(data_path, "pkg4", "a.py") + assert_equal(expected, found_module) + + def test__no_namespace_packages__find_b_in_pkg5(self) -> None: + found_module = self.fmc_ns.find_module("b") + expected = os.path.join(data_path, "pkg5", "b", "__init__.py") + assert_equal(expected, found_module) + + def test__find_nsx_as_namespace_pkg_in_pkg1(self) -> None: + """ + There's no __init__.py in any of the nsx dirs, return + the path to the first one found in mypypath. + """ + found_module = self.fmc_ns.find_module("nsx") + expected = os.path.join(data_path, "nsx-pkg1", "nsx") + assert_equal(expected, found_module) + + def test__find_nsx_a_init_in_pkg1(self) -> None: + found_module = self.fmc_ns.find_module("nsx.a") + expected = os.path.join(data_path, "nsx-pkg1", "nsx", "a", "__init__.py") + assert_equal(expected, found_module) + + def test__find_nsx_b_init_in_pkg2(self) -> None: + found_module = self.fmc_ns.find_module("nsx.b") + expected = os.path.join(data_path, "nsx-pkg2", "nsx", "b", "__init__.py") + assert_equal(expected, found_module) + + def test__find_nsx_c_c_in_pkg3(self) -> None: + found_module = self.fmc_ns.find_module("nsx.c.c") + expected = os.path.join(data_path, "nsx-pkg3", "nsx", "c", "c.py") + assert_equal(expected, found_module) + + def test__find_nsy_a__init_pyi(self) -> None: + """ + Prefer a __init__.pyi file over a __init__.py + """ + found_module = self.fmc_ns.find_module("nsy.a") + expected = os.path.join(data_path, "nsy-pkg1", "nsy", "a", "__init__.pyi") + assert_equal(expected, found_module) + + def test__find_nsy_b__init_py(self) -> None: + """ + There is a nsy-pkg2/nsy/b.pyi, but also a nsy-pkg2/nsy/b/__init__.py. + We expect to find the latter when looking up "nsy.b" as + a package is preferred over a module. + """ + found_module = self.fmc_ns.find_module("nsy.b") + expected = os.path.join(data_path, "nsy-pkg2", "nsy", "b", "__init__.py") + assert_equal(expected, found_module) + + def test__find_nsy_c_pyi(self) -> None: + """ + There is a nsy-pkg2/nsy/c.pyi and nsy-pkg2/nsy/c.py + We expect to find the former when looking up "nsy.b" as + .pyi is preferred over .py. + """ + found_module = self.fmc_ns.find_module("nsy.c") + expected = os.path.join(data_path, "nsy-pkg2", "nsy", "c.pyi") + assert_equal(expected, found_module) + + def test__find_a_in_pkg4(self) -> None: + found_module = self.fmc_ns.find_module("a") + expected = os.path.join(data_path, "pkg4", "a.py") + assert_equal(expected, found_module) + + def test__find_b_init_in_pkg5(self) -> None: + found_module = self.fmc_ns.find_module("b") + expected = os.path.join(data_path, "pkg5", "b", "__init__.py") + assert_equal(expected, found_module) + + def test__find_d_nowhere(self) -> None: + found_module = self.fmc_ns.find_module("d") + self.assertIsNone(found_module) diff --git a/test-data/packages/modulefinder/nsx-pkg1/nsx/a/__init__.py b/test-data/packages/modulefinder/nsx-pkg1/nsx/a/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/test-data/packages/modulefinder/nsx-pkg2/nsx/b/__init__.py b/test-data/packages/modulefinder/nsx-pkg2/nsx/b/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/test-data/packages/modulefinder/nsx-pkg3/nsx/c/c.py b/test-data/packages/modulefinder/nsx-pkg3/nsx/c/c.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/test-data/packages/modulefinder/nsy-pkg1/nsy/a/__init__.py b/test-data/packages/modulefinder/nsy-pkg1/nsy/a/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/test-data/packages/modulefinder/nsy-pkg1/nsy/a/__init__.pyi b/test-data/packages/modulefinder/nsy-pkg1/nsy/a/__init__.pyi new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/test-data/packages/modulefinder/nsy-pkg2/nsy/b.pyi b/test-data/packages/modulefinder/nsy-pkg2/nsy/b.pyi new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/test-data/packages/modulefinder/nsy-pkg2/nsy/b/__init__.py b/test-data/packages/modulefinder/nsy-pkg2/nsy/b/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/test-data/packages/modulefinder/nsy-pkg2/nsy/c.py b/test-data/packages/modulefinder/nsy-pkg2/nsy/c.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/test-data/packages/modulefinder/nsy-pkg2/nsy/c.pyi b/test-data/packages/modulefinder/nsy-pkg2/nsy/c.pyi new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/test-data/packages/modulefinder/pkg4/a.py b/test-data/packages/modulefinder/pkg4/a.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/test-data/packages/modulefinder/pkg5/b/__init__.py b/test-data/packages/modulefinder/pkg5/b/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/test-data/packages/modulefinder/readme.txt b/test-data/packages/modulefinder/readme.txt new file mode 100644 index 0000000000000..669bcfcd2663d --- /dev/null +++ b/test-data/packages/modulefinder/readme.txt @@ -0,0 +1,4 @@ +Samples for testing modulefinder.FindModuleCache. + +Contains three packages for the `nsx` namespace, and two packages +providing `a` and `b`. From 6a4cab9d91c67d5ac0abaf6ca2e8311985dbc442 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Mon, 1 Jul 2019 13:08:46 +0200 Subject: [PATCH 3/3] testmodulefinder: rename pkg{4,5} to pkg{1,2} ...and some more explaining comments. --- mypy/test/testmodulefinder.py | 33 ++++++++++++------- .../packages/modulefinder/{pkg4 => pkg1}/a.py | 0 .../modulefinder/{pkg5 => pkg2}/b/__init__.py | 0 3 files changed, 21 insertions(+), 12 deletions(-) rename test-data/packages/modulefinder/{pkg4 => pkg1}/a.py (100%) rename test-data/packages/modulefinder/{pkg5 => pkg2}/b/__init__.py (100%) diff --git a/mypy/test/testmodulefinder.py b/mypy/test/testmodulefinder.py index 46d4e9952eee0..ab1a0d8e67f4a 100644 --- a/mypy/test/testmodulefinder.py +++ b/mypy/test/testmodulefinder.py @@ -19,8 +19,8 @@ def setUp(self) -> None: os.path.join(data_path, "nsx-pkg3"), os.path.join(data_path, "nsy-pkg1"), os.path.join(data_path, "nsy-pkg2"), - os.path.join(data_path, "pkg4"), - os.path.join(data_path, "pkg5"), + os.path.join(data_path, "pkg1"), + os.path.join(data_path, "pkg2"), ), package_path=(), typeshed_path=(), @@ -47,17 +47,17 @@ def test__no_namespace_packages__nsx_a(self) -> None: found_module = self.fmc_nons.find_module("nsx.a") self.assertIsNone(found_module) - def test__no_namespace_packages__find_a_in_pkg4(self) -> None: + def test__no_namespace_packages__find_a_in_pkg1(self) -> None: """ - Find find pkg4/a.py for "a" with namespace_packages False. + Find find pkg1/a.py for "a" with namespace_packages False. """ found_module = self.fmc_nons.find_module("a") - expected = os.path.join(data_path, "pkg4", "a.py") + expected = os.path.join(data_path, "pkg1", "a.py") assert_equal(expected, found_module) - def test__no_namespace_packages__find_b_in_pkg5(self) -> None: + def test__no_namespace_packages__find_b_in_pkg2(self) -> None: found_module = self.fmc_ns.find_module("b") - expected = os.path.join(data_path, "pkg5", "b", "__init__.py") + expected = os.path.join(data_path, "pkg2", "b", "__init__.py") assert_equal(expected, found_module) def test__find_nsx_as_namespace_pkg_in_pkg1(self) -> None: @@ -70,23 +70,32 @@ def test__find_nsx_as_namespace_pkg_in_pkg1(self) -> None: assert_equal(expected, found_module) def test__find_nsx_a_init_in_pkg1(self) -> None: + """ + Find nsx-pkg1/nsx/a/__init__.py for "nsx.a" in namespace mode. + """ found_module = self.fmc_ns.find_module("nsx.a") expected = os.path.join(data_path, "nsx-pkg1", "nsx", "a", "__init__.py") assert_equal(expected, found_module) def test__find_nsx_b_init_in_pkg2(self) -> None: + """ + Find nsx-pkg2/nsx/b/__init__.py for "nsx.b" in namespace mode. + """ found_module = self.fmc_ns.find_module("nsx.b") expected = os.path.join(data_path, "nsx-pkg2", "nsx", "b", "__init__.py") assert_equal(expected, found_module) def test__find_nsx_c_c_in_pkg3(self) -> None: + """ + Find nsx-pkg3/nsx/c/c.py for "nsx.c.c" in namespace mode. + """ found_module = self.fmc_ns.find_module("nsx.c.c") expected = os.path.join(data_path, "nsx-pkg3", "nsx", "c", "c.py") assert_equal(expected, found_module) def test__find_nsy_a__init_pyi(self) -> None: """ - Prefer a __init__.pyi file over a __init__.py + Prefer nsy-pkg1/a/__init__.pyi file over __init__.py. """ found_module = self.fmc_ns.find_module("nsy.a") expected = os.path.join(data_path, "nsy-pkg1", "nsy", "a", "__init__.pyi") @@ -112,14 +121,14 @@ def test__find_nsy_c_pyi(self) -> None: expected = os.path.join(data_path, "nsy-pkg2", "nsy", "c.pyi") assert_equal(expected, found_module) - def test__find_a_in_pkg4(self) -> None: + def test__find_a_in_pkg1(self) -> None: found_module = self.fmc_ns.find_module("a") - expected = os.path.join(data_path, "pkg4", "a.py") + expected = os.path.join(data_path, "pkg1", "a.py") assert_equal(expected, found_module) - def test__find_b_init_in_pkg5(self) -> None: + def test__find_b_init_in_pkg2(self) -> None: found_module = self.fmc_ns.find_module("b") - expected = os.path.join(data_path, "pkg5", "b", "__init__.py") + expected = os.path.join(data_path, "pkg2", "b", "__init__.py") assert_equal(expected, found_module) def test__find_d_nowhere(self) -> None: diff --git a/test-data/packages/modulefinder/pkg4/a.py b/test-data/packages/modulefinder/pkg1/a.py similarity index 100% rename from test-data/packages/modulefinder/pkg4/a.py rename to test-data/packages/modulefinder/pkg1/a.py diff --git a/test-data/packages/modulefinder/pkg5/b/__init__.py b/test-data/packages/modulefinder/pkg2/b/__init__.py similarity index 100% rename from test-data/packages/modulefinder/pkg5/b/__init__.py rename to test-data/packages/modulefinder/pkg2/b/__init__.py