From cd5605c4e0fe2bf352ac507aad452ef4aaf4351c Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 28 Oct 2023 21:23:41 +0100 Subject: [PATCH 1/4] Fix file reloading in dmypy with --export-types --- mypy/dmypy_server.py | 11 +++++++++++ test-data/unit/daemon.test | 26 ++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/mypy/dmypy_server.py b/mypy/dmypy_server.py index 9cc0888fc2088..0d18a2675755d 100644 --- a/mypy/dmypy_server.py +++ b/mypy/dmypy_server.py @@ -557,6 +557,12 @@ def fine_grained_increment( # Use the remove/update lists to update fswatcher. # This avoids calling stat() for unchanged files. changed, removed = self.update_changed(sources, remove or [], update or []) + if self.options.export_types: + # If --export-types is given, we need to force full re-checking of all + # explicitly passed files, since we need to visit each expression. + changed += [ + (bs.module, bs.path) for bs in sources if (bs.module, bs.path) not in changed + ] changed += self.find_added_suppressed( self.fine_grained_manager.graph, set(), manager.search_paths ) @@ -601,6 +607,11 @@ def fine_grained_increment_follow_imports(self, sources: list[BuildSource]) -> l changed, new_files = self.find_reachable_changed_modules( sources, graph, seen, changed_paths ) + if self.options.export_types: + # Same as in fine_grained_increment(). + changed += [ + (bs.module, bs.path) for bs in sources if (bs.module, bs.path) not in changed + ] sources.extend(new_files) # Process changes directly reachable from roots. diff --git a/test-data/unit/daemon.test b/test-data/unit/daemon.test index 18a03a92207d2..5eb3d1ac21920 100644 --- a/test-data/unit/daemon.test +++ b/test-data/unit/daemon.test @@ -360,6 +360,32 @@ def bar() -> None: x = foo('abc') # type: str foo(arg='xyz') +[case testDaemonInspectCheck] +$ dmypy start +Daemon started +$ dmypy check foo.py +Success: no issues found in 40 source files +$ dmypy check foo.py --export-types +Success: no issues found in 1 source file +$ dmypy inspect foo.py:1:1 +[file foo.py] +x = 1 + +[case testDaemonInspectRun] +$ dmypy run test1.py +Daemon started +Success: no issues found in 40 source files +$ dmypy run test2.py +Success: no issues found in 1 source file +$ dmypy run test1.py --export-types +Success: no issues found in 1 source file +$ dmypy inspect test1.py:1:1 +"int" +[file test1.py] +a: int +[file test2.py] +a: str + [case testDaemonGetType] $ dmypy start --log-file log.txt -- --follow-imports=error --no-error-summary --python-version 3.8 Daemon started From 786e6db234dbc8bea0c9dabe449098f39f81b32f Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 28 Oct 2023 22:12:34 +0100 Subject: [PATCH 2/4] Some fixes --- mypy/dmypy_server.py | 45 ++++++++++++++++++++++++++++---------- test-data/unit/daemon.test | 5 +++-- 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/mypy/dmypy_server.py b/mypy/dmypy_server.py index 0d18a2675755d..6e801792bbc34 100644 --- a/mypy/dmypy_server.py +++ b/mypy/dmypy_server.py @@ -393,15 +393,21 @@ def cmd_recheck( t1 = time.time() manager = self.fine_grained_manager.manager manager.log(f"fine-grained increment: cmd_recheck: {t1 - t0:.3f}s") - self.options.export_types = export_types + old_export_types = self.options.export_types + self.options.export_types = self.options.export_types or export_types if not self.following_imports(): - messages = self.fine_grained_increment(sources, remove, update) + messages = self.fine_grained_increment( + sources, remove, update, explicit_export_types=export_types + ) else: assert remove is None and update is None - messages = self.fine_grained_increment_follow_imports(sources) + messages = self.fine_grained_increment_follow_imports( + sources, explicit_export_types=export_types + ) res = self.increment_output(messages, sources, is_tty, terminal_width) self.flush_caches() self.update_stats(res) + self.options.export_types = old_export_types return res def check( @@ -412,17 +418,21 @@ def check( If is_tty is True format the output nicely with colors and summary line (unless disabled in self.options). Also pass the terminal_width to formatter. """ - self.options.export_types = export_types + old_export_types = self.options.export_types + self.options.export_types = self.options.export_types or export_types if not self.fine_grained_manager: res = self.initialize_fine_grained(sources, is_tty, terminal_width) else: if not self.following_imports(): - messages = self.fine_grained_increment(sources) + messages = self.fine_grained_increment(sources, explicit_export_types=export_types) else: - messages = self.fine_grained_increment_follow_imports(sources) + messages = self.fine_grained_increment_follow_imports( + sources, explicit_export_types=export_types + ) res = self.increment_output(messages, sources, is_tty, terminal_width) self.flush_caches() self.update_stats(res) + self.options.export_types = old_export_types return res def flush_caches(self) -> None: @@ -461,6 +471,7 @@ def initialize_fine_grained( messages = result.errors self.fine_grained_manager = FineGrainedBuildManager(result) + original_sources_len = len(sources) if self.following_imports(): sources = find_all_sources_in_build(self.fine_grained_manager.graph, sources) self.update_sources(sources) @@ -525,7 +536,8 @@ def initialize_fine_grained( __, n_notes, __ = count_stats(messages) status = 1 if messages and n_notes < len(messages) else 0 - messages = self.pretty_messages(messages, len(sources), is_tty, terminal_width) + # We use explicit sources length to match the logic in non-incremental mode. + messages = self.pretty_messages(messages, original_sources_len, is_tty, terminal_width) return {"out": "".join(s + "\n" for s in messages), "err": "", "status": status} def fine_grained_increment( @@ -533,6 +545,7 @@ def fine_grained_increment( sources: list[BuildSource], remove: list[str] | None = None, update: list[str] | None = None, + explicit_export_types: bool = False, ) -> list[str]: """Perform a fine-grained type checking increment. @@ -543,6 +556,8 @@ def fine_grained_increment( sources: sources passed on the command line remove: paths of files that have been removed update: paths of files that have been changed or created + explicit_export_types: --export-type was passed in a check command + (as opposite to being set in dmypy start) """ assert self.fine_grained_manager is not None manager = self.fine_grained_manager.manager @@ -557,11 +572,13 @@ def fine_grained_increment( # Use the remove/update lists to update fswatcher. # This avoids calling stat() for unchanged files. changed, removed = self.update_changed(sources, remove or [], update or []) - if self.options.export_types: + if explicit_export_types: # If --export-types is given, we need to force full re-checking of all # explicitly passed files, since we need to visit each expression. changed += [ - (bs.module, bs.path) for bs in sources if (bs.module, bs.path) not in changed + (bs.module, bs.path) + for bs in sources + if bs.path and (bs.module, bs.path) not in changed ] changed += self.find_added_suppressed( self.fine_grained_manager.graph, set(), manager.search_paths @@ -581,7 +598,9 @@ def fine_grained_increment( self.previous_sources = sources return messages - def fine_grained_increment_follow_imports(self, sources: list[BuildSource]) -> list[str]: + def fine_grained_increment_follow_imports( + self, sources: list[BuildSource], explicit_export_types: bool = False + ) -> list[str]: """Like fine_grained_increment, but follow imports.""" t0 = time.time() @@ -607,10 +626,12 @@ def fine_grained_increment_follow_imports(self, sources: list[BuildSource]) -> l changed, new_files = self.find_reachable_changed_modules( sources, graph, seen, changed_paths ) - if self.options.export_types: + if explicit_export_types: # Same as in fine_grained_increment(). changed += [ - (bs.module, bs.path) for bs in sources if (bs.module, bs.path) not in changed + (bs.module, bs.path) + for bs in sources + if bs.path and (bs.module, bs.path) not in changed ] sources.extend(new_files) diff --git a/test-data/unit/daemon.test b/test-data/unit/daemon.test index 5eb3d1ac21920..99dcf1ccd9411 100644 --- a/test-data/unit/daemon.test +++ b/test-data/unit/daemon.test @@ -364,17 +364,18 @@ def bar() -> None: $ dmypy start Daemon started $ dmypy check foo.py -Success: no issues found in 40 source files +Success: no issues found in 1 source file $ dmypy check foo.py --export-types Success: no issues found in 1 source file $ dmypy inspect foo.py:1:1 +"int" [file foo.py] x = 1 [case testDaemonInspectRun] $ dmypy run test1.py Daemon started -Success: no issues found in 40 source files +Success: no issues found in 1 source file $ dmypy run test2.py Success: no issues found in 1 source file $ dmypy run test1.py --export-types From 5df3f05a732bea83cf3b2f1c7766ea73491b9e7e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 28 Oct 2023 22:24:47 +0100 Subject: [PATCH 3/4] Fix fine-grained tests --- mypy/test/testfinegrained.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypy/test/testfinegrained.py b/mypy/test/testfinegrained.py index ba0526d325588..3000cd66edc24 100644 --- a/mypy/test/testfinegrained.py +++ b/mypy/test/testfinegrained.py @@ -149,6 +149,7 @@ def get_options(self, source: str, testcase: DataDrivenTestCase, build_cache: bo options.use_fine_grained_cache = self.use_cache and not build_cache options.cache_fine_grained = self.use_cache options.local_partial_types = True + options.export_types = True options.enable_incomplete_feature = [TYPE_VAR_TUPLE, UNPACK] # Treat empty bodies safely for these test cases. options.allow_empty_bodies = not testcase.name.endswith("_no_empty") @@ -164,7 +165,7 @@ def get_options(self, source: str, testcase: DataDrivenTestCase, build_cache: bo return options def run_check(self, server: Server, sources: list[BuildSource]) -> list[str]: - response = server.check(sources, export_types=True, is_tty=False, terminal_width=-1) + response = server.check(sources, export_types=False, is_tty=False, terminal_width=-1) out = response["out"] or response["err"] assert isinstance(out, str) return out.splitlines() From ac5c9648c2425f2a4e7b5fa7ef3c622431f86073 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 30 Oct 2023 19:46:03 +0000 Subject: [PATCH 4/4] Address CR --- mypy/dmypy_server.py | 28 ++++++++++++++++++---------- mypy/test/testfinegrained.py | 2 +- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/mypy/dmypy_server.py b/mypy/dmypy_server.py index 6e801792bbc34..42236497f275c 100644 --- a/mypy/dmypy_server.py +++ b/mypy/dmypy_server.py @@ -575,11 +575,7 @@ def fine_grained_increment( if explicit_export_types: # If --export-types is given, we need to force full re-checking of all # explicitly passed files, since we need to visit each expression. - changed += [ - (bs.module, bs.path) - for bs in sources - if bs.path and (bs.module, bs.path) not in changed - ] + add_all_sources_to_changed(sources, changed) changed += self.find_added_suppressed( self.fine_grained_manager.graph, set(), manager.search_paths ) @@ -628,11 +624,7 @@ def fine_grained_increment_follow_imports( ) if explicit_export_types: # Same as in fine_grained_increment(). - changed += [ - (bs.module, bs.path) - for bs in sources - if bs.path and (bs.module, bs.path) not in changed - ] + add_all_sources_to_changed(sources, changed) sources.extend(new_files) # Process changes directly reachable from roots. @@ -1041,6 +1033,22 @@ def find_all_sources_in_build( return result +def add_all_sources_to_changed(sources: list[BuildSource], changed: list[tuple[str, str]]) -> None: + """Add all (explicit) sources to the list changed files in place. + + Use this when re-processing of unchanged files is needed (e.g. for + the purpose of exporting types for inspections). + """ + changed_set = set(changed) + changed.extend( + [ + (bs.module, bs.path) + for bs in sources + if bs.path and (bs.module, bs.path) not in changed_set + ] + ) + + def fix_module_deps(graph: mypy.build.Graph) -> None: """After an incremental update, update module dependencies to reflect the new state. diff --git a/mypy/test/testfinegrained.py b/mypy/test/testfinegrained.py index 3f66b5890f483..f61a58c425fc6 100644 --- a/mypy/test/testfinegrained.py +++ b/mypy/test/testfinegrained.py @@ -149,7 +149,7 @@ def get_options(self, source: str, testcase: DataDrivenTestCase, build_cache: bo options.use_fine_grained_cache = self.use_cache and not build_cache options.cache_fine_grained = self.use_cache options.local_partial_types = True - options.export_types = True + options.export_types = "inspect" in testcase.file # Treat empty bodies safely for these test cases. options.allow_empty_bodies = not testcase.name.endswith("_no_empty") if re.search("flags:.*--follow-imports", source) is None: