From 462e9a563f8510150630ea57404a89c3fc42b787 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 24 Mar 2020 17:57:04 +0000 Subject: [PATCH 1/9] Filter the set of suppressed modules to speed things up --- mypy/dmypy_server.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mypy/dmypy_server.py b/mypy/dmypy_server.py index c7e3be575e236..6f889245b4942 100644 --- a/mypy/dmypy_server.py +++ b/mypy/dmypy_server.py @@ -671,6 +671,10 @@ def find_added_suppressed(self, for module, state in graph.items(): all_suppressed |= state.suppressed_set + # Filter out things that shouldn't actually be considered suppressed. + # TODO: Figure out why these are treated as suppressed + all_suppressed = {module for module in all_suppressed if module not in graph} + # TODO: Namespace packages # TODO: Handle seen? From 19404517ade40696c47e7a5d7b69aa31280a4131 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 25 Mar 2020 11:48:26 +0000 Subject: [PATCH 2/9] Fix mypy daemon issue with following imports and import inside function --- mypy/server/update.py | 7 ++++++- .../unit/fine-grained-follow-imports.test | 21 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/mypy/server/update.py b/mypy/server/update.py index 6269b5c0cbfa9..e76832296edea 100644 --- a/mypy/server/update.py +++ b/mypy/server/update.py @@ -1145,8 +1145,13 @@ def refresh_suppressed_submodules( trigger = make_trigger(submodule) if trigger in deps: for dep in deps[trigger]: - # TODO: <...> deps, imports in functions, etc. + # TODO: <...> deps, etc. state = graph.get(dep) + if not state: + # Maybe it's a non-top-level target. We only care about the module. + dep_module = module_prefix(graph, dep) + if dep_module is not None: + state = graph.get(dep_module) if state: tree = state.tree assert tree # TODO: What if doesn't exist? diff --git a/test-data/unit/fine-grained-follow-imports.test b/test-data/unit/fine-grained-follow-imports.test index 6066d86442089..f96092c4cd216 100644 --- a/test-data/unit/fine-grained-follow-imports.test +++ b/test-data/unit/fine-grained-follow-imports.test @@ -542,3 +542,24 @@ main.py:3: error: Too few arguments for "f" == main.py:3: error: Too few arguments for "f" main.py:4: error: Too many arguments for "f" + +[case testFollowImportsNormalSubmoduleCreatedWithImportInFunction] +# flags: --follow-imports=normal +# cmd: mypy main.py + +[file main.py] +def f() -> None: + from p import m + +[file p/__init__.py.2] +1() + +[file p/m.py.2] +''() + +[out] +main.py:2: error: Cannot find implementation or library stub for module named 'p' +main.py:2: note: See https://mypy.readthedocs.io/en/latest/running_mypy.html#missing-imports +== +p/m.py:1: error: "str" not callable +p/__init__.py:1: error: "int" not callable From c10ca2d9b9f3ff5200b751dc298b6bc265cce600 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 25 Mar 2020 12:25:27 +0000 Subject: [PATCH 3/9] dmypy following imports: Fix Windows paths and stub packages Allow alternate path separate on Windows. Recognize `__init__.pyi` files. --- mypy/server/update.py | 15 +++++++++-- .../unit/fine-grained-follow-imports.test | 26 +++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/mypy/server/update.py b/mypy/server/update.py index e76832296edea..56523d28d1443 100644 --- a/mypy/server/update.py +++ b/mypy/server/update.py @@ -113,6 +113,7 @@ """ import os +import sys import time from typing import ( Dict, List, Set, Tuple, Union, Optional, NamedTuple, Sequence @@ -1111,6 +1112,17 @@ def target_from_node(module: str, return '%s.%s' % (module, node.name) +if sys.platform != 'win32': + INIT_SUFFIXES = ('/__init__.py', '/__init__.pyi') # type: Final +else: + INIT_SUFFIXES = ( + os.sep + '__init__.py', + os.sep + '__init__.pyi', + os.altsep + '__init__.py', + os.altsep + '__init__.pyi', + ) # type: Final + + def refresh_suppressed_submodules( module: str, path: Optional[str], @@ -1129,8 +1141,7 @@ def refresh_suppressed_submodules( module: target package in which to look for submodules path: path of the module """ - # TODO: Windows paths - if path is None or not path.endswith(os.sep + '__init__.py'): + if path is None or not path.endswith(INIT_SUFFIXES): # Only packages have submodules. return # Find any submodules present in the directory. diff --git a/test-data/unit/fine-grained-follow-imports.test b/test-data/unit/fine-grained-follow-imports.test index f96092c4cd216..d8d861d33a9c5 100644 --- a/test-data/unit/fine-grained-follow-imports.test +++ b/test-data/unit/fine-grained-follow-imports.test @@ -563,3 +563,29 @@ main.py:2: note: See https://mypy.readthedocs.io/en/latest/running_mypy.html#mis == p/m.py:1: error: "str" not callable p/__init__.py:1: error: "int" not callable + +[case testFollowImportsNormalPackageInitFileStub] +# flags: --follow-imports=normal +# cmd: mypy main.py + +[file main.py] +from p import m + +[file p/__init__.pyi.2] +1() + +[file p/m.pyi.2] +''() + +[file p/mm.pyi.3] +x x x + +[out] +main.py:1: error: Cannot find implementation or library stub for module named 'p' +main.py:1: note: See https://mypy.readthedocs.io/en/latest/running_mypy.html#missing-imports +== +p/m.pyi:1: error: "str" not callable +p/__init__.pyi:1: error: "int" not callable +== +p/m.pyi:1: error: "str" not callable +p/__init__.pyi:1: error: "int" not callable From 6263d9a8cfd938ab7ea8795966ced6a5d5a9f461 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 25 Mar 2020 12:45:26 +0000 Subject: [PATCH 4/9] Add test case --- .../unit/fine-grained-follow-imports.test | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/test-data/unit/fine-grained-follow-imports.test b/test-data/unit/fine-grained-follow-imports.test index d8d861d33a9c5..0cc4b3ed79c70 100644 --- a/test-data/unit/fine-grained-follow-imports.test +++ b/test-data/unit/fine-grained-follow-imports.test @@ -543,6 +543,34 @@ main.py:3: error: Too few arguments for "f" main.py:3: error: Too few arguments for "f" main.py:4: error: Too many arguments for "f" +[case testFollowImportsNormalPackageInitFile4-only_when_cache] +# flags: --follow-imports=normal +# cmd: mypy main.py + +[file main.py] +import p1.m # type: ignore +from p2 import m # type: ignore + +[file p1/__init__.py.2] +1() + +[file p1/m.py.2] +''() + +[file p2/__init__.py.3] +''() + +[file p2/m.py.3] + +[out] +== +p1/__init__.py:1: error: "int" not callable +p1/m.py:1: error: "str" not callable +== +p1/__init__.py:1: error: "int" not callable +p1/m.py:1: error: "str" not callable +p2/__init__.py:1: error: "str" not callable + [case testFollowImportsNormalSubmoduleCreatedWithImportInFunction] # flags: --follow-imports=normal # cmd: mypy main.py From 6ed694f3e52f17dc9630b52c1b0d877ca4b4fa42 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 25 Mar 2020 14:46:42 +0000 Subject: [PATCH 5/9] dmypy following imports: Minimal namespace package support --- mypy/dmypy_server.py | 3 ++ .../unit/fine-grained-follow-imports.test | 28 +++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/mypy/dmypy_server.py b/mypy/dmypy_server.py index 6f889245b4942..a5855d64d62f0 100644 --- a/mypy/dmypy_server.py +++ b/mypy/dmypy_server.py @@ -724,6 +724,9 @@ def pretty_messages(self, messages: List[str], n_sources: int, def update_sources(self, sources: List[BuildSource]) -> None: paths = [source.path for source in sources if source.path is not None] + if self.following_imports(): + # Filter out directories (used for namespace packages). + paths = [path for path in paths if self.fscache.isfile(path)] self.fswatcher.add_watched_paths(paths) def update_changed(self, diff --git a/test-data/unit/fine-grained-follow-imports.test b/test-data/unit/fine-grained-follow-imports.test index 0cc4b3ed79c70..1c95667a4e724 100644 --- a/test-data/unit/fine-grained-follow-imports.test +++ b/test-data/unit/fine-grained-follow-imports.test @@ -617,3 +617,31 @@ p/__init__.pyi:1: error: "int" not callable == p/m.pyi:1: error: "str" not callable p/__init__.pyi:1: error: "int" not callable + +[case testFollowImportsNormalNamespacePackages] +# flags: --follow-imports=normal --namespace-packages +# cmd: mypy main.py + +[file main.py] +import p1.m1 +import p2.m2 + +[file p1/m1.py] +1() + +[file p2/m2.py.2] +''() + +[delete p2/m2.py.3] + +[out] +p1/m1.py:1: error: "int" not callable +main.py:2: error: Cannot find implementation or library stub for module named 'p2.m2' +main.py:2: note: See https://mypy.readthedocs.io/en/latest/running_mypy.html#missing-imports +== +p2/m2.py:1: error: "str" not callable +p1/m1.py:1: error: "int" not callable +== +main.py:2: error: Cannot find implementation or library stub for module named 'p2.m2' +main.py:2: note: See https://mypy.readthedocs.io/en/latest/running_mypy.html#missing-imports +p1/m1.py:1: error: "int" not callable From 28c77e5f5f2d830b552ff9aac8b286e49a6bc24e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 25 Mar 2020 15:54:20 +0000 Subject: [PATCH 6/9] Improve exception message --- mypy/test/testcheck.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index e2a916a19c54d..49a85861b6e32 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -336,7 +336,7 @@ def parse_module(self, cache = FindModuleCache(search_paths) for module_name in module_names.split(' '): path = cache.find_module(module_name) - assert isinstance(path, str), "Can't find ad hoc case file" + assert isinstance(path, str), "Can't find ad hoc case file: %s" % module_name with open(path, encoding='utf8') as f: program_text = f.read() out.append((module_name, path, program_text)) From 64f9e81857f5369ece59a3e5375db23c5af4a896 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 25 Mar 2020 16:36:07 +0000 Subject: [PATCH 7/9] dmypy following imports: implement logging --- mypy/dmypy_server.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/mypy/dmypy_server.py b/mypy/dmypy_server.py index a5855d64d62f0..f2db4fd6e1811 100644 --- a/mypy/dmypy_server.py +++ b/mypy/dmypy_server.py @@ -524,10 +524,11 @@ def fine_grained_increment(self, def fine_grained_increment_follow_imports(self, sources: List[BuildSource]) -> List[str]: """Like fine_grained_increment, but follow imports.""" + t0 = time.time() + # TODO: # - file events # - search path updates - # - logging changed_paths = self.fswatcher.find_changed() @@ -542,6 +543,9 @@ def fine_grained_increment_follow_imports(self, sources: List[BuildSource]) -> L # updating sources after finding changed, necessary? self.update_sources(sources) + t1 = time.time() + manager.log("fine-grained increment: find_changed: {:.3f}s".format(t1 - t0)) + sources_set = {source.module for source in sources} # Find changed modules reachable from roots (or in roots) already in graph. @@ -570,10 +574,14 @@ def fine_grained_increment_follow_imports(self, sources: List[BuildSource]) -> L # TODO: Removed? worklist.extend(changed) + t2 = time.time() + for module_id, state in graph.items(): refresh_suppressed_submodules(module_id, state.path, fine_grained_manager.deps, graph, self.fscache) + t3 = time.time() + # There may be new files that became available, currently treated as # suppressed imports. Process them. seen_suppressed = set() # type: Set[str] @@ -593,6 +601,8 @@ def fine_grained_increment_follow_imports(self, sources: List[BuildSource]) -> L refresh_suppressed_submodules(module_id, path, fine_grained_manager.deps, graph, self.fscache) + t4 = time.time() + # Find all original modules in graph that were not reached -- they are deleted. to_delete = [] for module_id in orig_modules: @@ -612,6 +622,17 @@ def fine_grained_increment_follow_imports(self, sources: List[BuildSource]) -> L self.previous_sources = find_all_sources_in_build(graph) self.update_sources(self.previous_sources) + + t5 = time.time() + + manager.log("fine-grained increment: update: {:.3f}s".format(t5 - t1)) + manager.add_stats( + find_changes_time=t1 - t0, + fg_update_time=t2 - t1, + refresh_suppressed_time=t3 - t2, + find_added_supressed_time=t4 - t3, + cleanup_time=t5 - t4) + return messages def follow_imports(self, From c5d0baea3827d3aabe8406b192c6461633e9c0d9 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 25 Mar 2020 18:08:28 +0000 Subject: [PATCH 8/9] dmypy following imports: Update search path and allow changes to command line --- mypy/dmypy_server.py | 23 ++++---- .../unit/fine-grained-follow-imports.test | 55 +++++++++++++++++++ 2 files changed, 66 insertions(+), 12 deletions(-) diff --git a/mypy/dmypy_server.py b/mypy/dmypy_server.py index f2db4fd6e1811..57955fd08821c 100644 --- a/mypy/dmypy_server.py +++ b/mypy/dmypy_server.py @@ -423,7 +423,7 @@ def initialize_fine_grained(self, sources: List[BuildSource], self.fine_grained_manager = FineGrainedBuildManager(result) if self.following_imports(): - sources = find_all_sources_in_build(self.fine_grained_manager.graph) + sources = find_all_sources_in_build(self.fine_grained_manager.graph, sources) self.update_sources(sources) self.previous_sources = sources @@ -523,14 +523,9 @@ def fine_grained_increment(self, def fine_grained_increment_follow_imports(self, sources: List[BuildSource]) -> List[str]: """Like fine_grained_increment, but follow imports.""" - t0 = time.time() - # TODO: - # - file events - # - search path updates - - changed_paths = self.fswatcher.find_changed() + # TODO: Support file events assert self.fine_grained_manager is not None fine_grained_manager = self.fine_grained_manager @@ -539,9 +534,10 @@ def fine_grained_increment_follow_imports(self, sources: List[BuildSource]) -> L orig_modules = list(graph.keys()) - # TODO: Are the differences from fine_grained_increment(), such as - # updating sources after finding changed, necessary? + # TODO: Are the differences from fine_grained_increment(), necessary? self.update_sources(sources) + changed_paths = self.fswatcher.find_changed() + manager.search_paths = compute_search_paths(sources, manager.options, manager.data_dir) t1 = time.time() manager.log("fine-grained increment: find_changed: {:.3f}s".format(t1 - t0)) @@ -856,10 +852,13 @@ def get_meminfo() -> Dict[str, Any]: return res -def find_all_sources_in_build(graph: mypy.build.Graph) -> List[BuildSource]: - result = [] +def find_all_sources_in_build(graph: mypy.build.Graph, + extra: Sequence[BuildSource] = ()) -> List[BuildSource]: + result = list(extra) + seen = set(source.module for source in result) for module, state in graph.items(): - result.append(BuildSource(state.path, module)) + if module not in seen: + result.append(BuildSource(state.path, module)) return result diff --git a/test-data/unit/fine-grained-follow-imports.test b/test-data/unit/fine-grained-follow-imports.test index 1c95667a4e724..e3368329d4502 100644 --- a/test-data/unit/fine-grained-follow-imports.test +++ b/test-data/unit/fine-grained-follow-imports.test @@ -645,3 +645,58 @@ p1/m1.py:1: error: "int" not callable main.py:2: error: Cannot find implementation or library stub for module named 'p2.m2' main.py:2: note: See https://mypy.readthedocs.io/en/latest/running_mypy.html#missing-imports p1/m1.py:1: error: "int" not callable + +[case testFollowImportsNormalNewFileOnCommandLine] +# flags: --follow-imports=normal +# cmd: mypy main.py +# cmd2: mypy main.py x.py + +[file main.py] +1() + +[file x.py.2] +''() + +[out] +main.py:1: error: "int" not callable +== +x.py:1: error: "str" not callable +main.py:1: error: "int" not callable + +[case testFollowImportsNormalSearchPathUpdate-only_when_nocache] +# flags: --follow-imports=normal +# cmd: mypy main.py +# cmd2: mypy main.py src/foo.py + +[file main.py] + +[file src/foo.py.2] +import bar +''() + +[file src/bar.py.2] +1() + +[out] +== +src/bar.py:1: error: "int" not callable +src/foo.py:2: error: "str" not callable + +[case testFollowImportsNormalSearchPathUpdate-only_when_cache] +# flags: --follow-imports=normal +# cmd: mypy main.py +# cmd2: mypy main.py src/foo.py + +[file main.py] + +[file src/foo.py.2] +import bar +''() + +[file src/bar.py.2] +1() + +[out] +== +src/foo.py:2: error: "str" not callable +src/bar.py:1: error: "int" not callable From 9c55947147e77b2fa730d5ce338b880801af882a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 25 Mar 2020 18:50:51 +0000 Subject: [PATCH 9/9] Include base_dir in the repr of BuildSource --- mypy/modulefinder.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index 33b9b7394ec84..55d226cb647ae 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -77,9 +77,11 @@ def __init__(self, path: Optional[str], module: Optional[str], self.base_dir = base_dir # Directory where the package is rooted (e.g. 'xxx/yyy') def __repr__(self) -> str: - return '' % (self.path, - self.module, - self.text is not None) + return '' % ( + self.path, + self.module, + self.text is not None, + self.base_dir) class FindModuleCache: