From d87d66b33d95ec8e503308ffa88a8f9b77c08b88 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Thu, 25 Aug 2016 16:12:52 -0700 Subject: [PATCH 01/18] Change tests to reflect expected behavior This commit modifies the incremental tests (and in particular, the 'rechecked' and 'stale' annotations) to more accurately describe what the expected behavior should be after making interfaces changes propagate only one step out. --- test-data/unit/check-incremental.test | 276 ++++++++++++++++++++++++-- test-data/unit/fixtures/args.pyi | 1 + test-data/unit/fixtures/module.pyi | 12 +- 3 files changed, 270 insertions(+), 19 deletions(-) diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 00e52a1b967b9..273b42d7ed391 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -122,8 +122,7 @@ class Parent: pass class A(Parent): pass [rechecked mod1, mod2] -[stale mod1, mod2] - +[stale mod2] [case testIncrementalPartialInterfaceChange] import mod1 @@ -143,8 +142,8 @@ def func3() -> None: pass [file mod3.py.next] def func3() -> int: return 2 -[rechecked mod1, mod2, mod3] -[stale mod1, mod2, mod3] +[rechecked mod2, mod3] +[stale mod3] [case testIncrementalInternalFunctionDefinitionChange] import mod1 @@ -220,7 +219,7 @@ class Foo: return "a" [rechecked mod1, mod2] -[stale mod1, mod2] +[stale mod2] [case testIncrementalBaseClassChange] import mod1 @@ -242,7 +241,7 @@ class Bad: pass class Child(Bad): pass [rechecked mod1, mod2] -[stale mod1, mod2] +[stale mod2] [out2] main:1: note: In module imported here: tmp/mod1.py:2: error: "Child" has no attribute "good_method" @@ -270,7 +269,7 @@ C = 3 C = "A" [rechecked mod1, mod2, mod3, mod4] -[stale mod1, mod2, mod3, mod4] +[stale mod2, mod3, mod4] [out2] main:1: note: In module imported here: tmp/mod1.py:3: error: Argument 1 to "accepts_int" has incompatible type "str"; expected "int" @@ -296,7 +295,70 @@ const = 3 # Import to mod4 is gone! [rechecked mod1, mod2, mod3] -[stale mod1, mod2, mod3] +[stale mod3] +[builtins fixtures/module.pyi] +[out2] +main:1: note: In module imported here: +tmp/mod1.py:3: error: "module" has no attribute "mod4" + +[case testIncrementalLongBrokenCascade] +import mod1 + +[file mod1.py] +import mod2 +def accept_int(a: int) -> int: return a +accept_int(mod2.mod3.mod4.mod5.mod6.mod7.const) + +[file mod2.py] +import mod3 + +[file mod3.py] +import mod4 + +[file mod4.py] +import mod5 + +[file mod5.py] +import mod6 + +[file mod6.py] +import mod7 + +[file mod7.py] +const = 3 + +[file mod6.py.next] +# Import to mod7 is gone! + +[rechecked mod1, mod5, mod6] +[stale mod6] +[builtins fixtures/module.pyi] +[out2] +main:1: note: In module imported here: +tmp/mod1.py:3: error: "module" has no attribute "mod7" + +[case testIncrementalNestedBrokenCascade] +import mod1 + +[file mod1.py] +import mod2 +def accept_int(a: int) -> int: return a +accept_int(mod2.mod3.mod4.const) + +[file mod2/__init__.py] +import mod2.mod3 as mod3 + +[file mod2/mod3/__init__.py] +import mod2.mod3.mod4 as mod4 + +[file mod2/mod3/mod4.py] +const = 3 + +[file mod2/mod3/__init__.py.next] +# Import is gone! + +[rechecked mod1, mod2, mod2.mod3] +[stale mod2.mod3] [builtins fixtures/module.pyi] [out2] main:1: note: In module imported here: @@ -322,7 +384,8 @@ const = 3 [file mod4.py.next] const = "foo" -[stale mod1, mod2, mod3, mod4] +[rechecked mod1, mod3, mod4] +[stale mod4] [out2] main:1: note: In module imported here: tmp/mod1.py:3: error: Argument 1 to "accepts_int" has incompatible type "str"; expected "int" @@ -345,12 +408,67 @@ def func2() -> str: return "foo" [rechecked mod1, mod2] -[stale mod1, mod2] +[stale mod2] [out2] main:1: note: In module imported here: tmp/mod1.py: note: In function "func1": tmp/mod1.py:4: error: Incompatible return value type (got "str", expected "int") +[case testIncrementalBadChangeWithSave] +import mod0 + +[file mod0.py] +import mod1 +A = mod1.func2() + +[file mod1.py] +from mod2 import func2 + +def func1() -> int: + return func2() + +[file mod2.py] +def func2() -> int: + return 1 + +[file mod2.py.next] +def func2() -> str: + return "foo" + +[rechecked mod0, mod1, mod2] +[stale mod2] +[out2] +tmp/mod0.py:1: note: In module imported here, +main:1: note: ... from here: +tmp/mod1.py: note: In function "func1": +tmp/mod1.py:4: error: Incompatible return value type (got "str", expected "int") + +[case testIncrementalOkChangeWithSave] +import mod0 + +[file mod0.py] +import mod1 +A = mod1.func2() + +[file mod1.py] +from mod2 import func2 + +def func1() -> int: + func2() + return 1 + +[file mod2.py] +def func2() -> int: + return 1 + +[file mod2.py.next] +def func2() -> str: + return "foo" + +[rechecked mod0, mod1, mod2] +[stale mod0, mod2] +[out2] + [case testIncrementalWithComplexDictExpression] import mod1 @@ -370,7 +488,7 @@ my_dict = { } [rechecked mod1, mod1_private] -[stale mod1, mod1_private] +[stale mod1_private] [builtins fixtures/dict.pyi] [case testIncrementalWithComplexConstantExpressionNoAnnotation] @@ -428,7 +546,7 @@ def some_func(a: int) -> str: return "a" [rechecked mod1, mod1_private] -[stale mod1, mod1_private] +[stale mod1_private] [builtins fixtures/ops.pyi] [out2] main:1: note: In module imported here: @@ -466,7 +584,7 @@ def stringify(f: Callable[[int], int]) -> Callable[[int], str]: def some_func(a: int) -> int: return a + 2 [rechecked mod1, mod1_private] -[stale mod1, mod1_private] +[stale mod1_private] [builtins fixtures/ops.pyi] [out2] main:1: note: In module imported here: @@ -488,7 +606,7 @@ class Foo: A = "hello" [rechecked mod1, mod2] -[stale mod1, mod2] +[stale mod2] [case testIncrementalChangingFields] import mod1 @@ -498,6 +616,28 @@ import mod2 f = mod2.Foo() f.A +[file mod2.py] +class Foo: + def __init__(self) -> None: + self.A = 3 + +[file mod2.py.next] +class Foo: + def __init__(self) -> None: + self.A = "hello" + +[rechecked mod1, mod2] +[stale mod2] +[out2] + +[case testIncrementalChangingFieldsWithAssignment] +import mod1 + +[file mod1.py] +import mod2 +f = mod2.Foo() +B = f.A + [file mod2.py] class Foo: def __init__(self) -> None: @@ -531,7 +671,7 @@ class Foo: self.A = "hello" [rechecked mod1, mod2] -[stale mod1, mod2] +[stale mod2] [out2] main:1: note: In module imported here: tmp/mod1.py:4: error: Argument 1 to "accept_int" has incompatible type "str"; expected "int" @@ -555,7 +695,7 @@ class Foo: attr = "foo" [rechecked mod1, mod2] -[stale mod1, mod2] +[stale mod2] [case testIncrementalSimpleBranchingModules] import mod1 @@ -600,6 +740,72 @@ class Bar: [rechecked] [stale] +[case testIncrementalSubmoduleWithAttr] +import mod.child +x = mod.child.Foo() +x.bar() + +[file mod/__init__.py] + +[file mod/child.py] +class Foo: + def bar(self) -> None: pass +[builtins fixtures/module.pyi] +[rechecked] +[stale] + +[case testIncrementalNestedSubmoduleImportFromWithAttr] +from mod1.mod2 import mod3 +def accept_int(a: int) -> None: pass + +accept_int(mod3.val3) + +[file mod1/__init__.py] +val1 = 1 + +[file mod1/mod2/__init__.py] +val2 = 1 + +[file mod1/mod2/mod3.py] +val3 = 1 + +[builtins fixtures/module.pyi] +[rechecked] +[stale] + +[case testIncrementalNestedSubmoduleWithAttr] +import mod1.mod2.mod3 +def accept_int(a: int) -> None: pass + +accept_int(mod1.mod2.mod3.val3) +accept_int(mod1.mod2.val2) +accept_int(mod1.val1) + +[file mod1/__init__.py] +val1 = 1 + +[file mod1/mod2/__init__.py] +val2 = 1 + +[file mod1/mod2/mod3.py] +val3 = 1 + +[builtins fixtures/module.pyi] +[rechecked] +[stale] + +[case testIncrementalSubmoduleParentWithImportFrom] +import parent + +[file parent/__init__.py] +from parent import a + +[file parent/a.py] +val = 3 + +[builtins fixtures/args.pyi] +[stale] + [case testIncrementalSubmoduleParentBackreference] import parent @@ -775,7 +981,7 @@ class Class: pass [builtins fixtures/args.pyi] [rechecked collections, main, package.subpackage.mod1] -[stale collections, main, package.subpackage.mod1] +[stale collections, package.subpackage.mod1] [out2] tmp/main.py: note: In function "handle": tmp/main.py:4: error: "Class" has no attribute "some_attribute" @@ -783,6 +989,7 @@ tmp/main.py:4: error: "Class" has no attribute "some_attribute" [case testIncrementalWithIgnores] import foo # type: ignore +[builtins fixtures/module.pyi] [stale] [case testIncrementalWithSilentImportsAndIgnore] @@ -833,7 +1040,7 @@ class A: class A: pass [rechecked m, n] -[stale m, n] +[stale n] [out2] main:2: error: "A" has no attribute "bar" @@ -908,3 +1115,36 @@ foo(3) [out2] main:1: note: In module imported here: tmp/client.py:4: error: Argument 1 to "foo" has incompatible type "int"; expected "str" + +[case testIncrementalChangingAlias] +import m1, m2, m3, m4, m5 + +[file m1.py] +from m2 import A +def accepts_int(x: int) -> None: pass +accepts_int(A()) + +[file m2.py] +from m3 import A + +[file m3.py] +from m4 import B +A = B + +[file m3.py.next] +from m5 import C +A = C + +[file m4.py] +def B() -> int: + return 42 + +[file m5.py] +def C() -> str: + return "hello" + +[rechecked m1, m2, m3] +[stale m3] +[out2] +main:1: note: In module imported here: +tmp/m1.py:3: error: Argument 1 to "accepts_int" has incompatible type "str"; expected "int" diff --git a/test-data/unit/fixtures/args.pyi b/test-data/unit/fixtures/args.pyi index b084fc6c68e50..e4a6ffe1f33b4 100644 --- a/test-data/unit/fixtures/args.pyi +++ b/test-data/unit/fixtures/args.pyi @@ -26,3 +26,4 @@ class int: class str: pass class bool: pass class function: pass +class module: pass diff --git a/test-data/unit/fixtures/module.pyi b/test-data/unit/fixtures/module.pyi index e79ca284de225..fb2a4c2753eb5 100644 --- a/test-data/unit/fixtures/module.pyi +++ b/test-data/unit/fixtures/module.pyi @@ -1,8 +1,18 @@ +from typing import Any, Dict, Generic, TypeVar + +T = TypeVar('T') +S = TypeVar('S') + class object: def __init__(self) -> None: pass -class module: pass +class module: + __name__ = ... # type: str + __file__ = ... # type: str + __dict__ = ... # type: Dict[str, Any] class type: pass class function: pass class int: pass class str: pass class bool: pass +class tuple: pass +class dict(Generic[T, S]): pass From db9a4c7cd6cf9534dfc4f20a2356396d3182f5f9 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Thu, 25 Aug 2016 16:18:03 -0700 Subject: [PATCH 02/18] Record module name in TypeInfo This commit modifies TypeInfo so it records the name of the module in which the class (or NewType or namedtuple) was defined in. This does not modify the behavior of mypy, but will make it easier to perform introspection and analysis at later stages. This information is also a part of the TypeInfo's fullname, but is difficult to extract in the case of nested classes. For example, if we have a fully qualified name of a.b.c.d, it's not always obvious whether or not c is a module name or is a class name, and so forth. --- mypy/nodes.py | 17 ++++++++++++----- mypy/semanal.py | 12 +++++++----- mypy/typefixture.py | 9 ++++++++- 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index 259d8f0bd0a9b..72833fc35bdfe 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1779,6 +1779,10 @@ class is generic then it will be a type constructor of higher kind. """ _fullname = None # type: str # Fully qualified name + # Fully qualified name for the module this type was defined in. This + # information is also in the fullname, but is harder to extract in the + # case of nested class definitions. + module_name = None # type: str defn = None # type: ClassDef # Corresponding ClassDef # Method Resolution Order: the order of looking up attributes. The first # value always to refers to this class. @@ -1830,10 +1834,11 @@ class is generic then it will be a type constructor of higher kind. # Alternative to fullname() for 'anonymous' classes. alt_fullname = None # type: Optional[str] - def __init__(self, names: 'SymbolTable', defn: ClassDef) -> None: + def __init__(self, names: 'SymbolTable', defn: ClassDef, module_name: str) -> None: """Initialize a TypeInfo.""" self.names = names self.defn = defn + self.module_name = module_name self.subtypes = set() self.type_vars = [] self.bases = [] @@ -1987,6 +1992,7 @@ def __str__(self) -> str: def serialize(self) -> Union[str, JsonDict]: # NOTE: This is where all ClassDefs originate, so there shouldn't be duplicates. data = {'.class': 'TypeInfo', + 'module_name': self.module_name, 'fullname': self.fullname(), 'alt_fullname': self.alt_fullname, 'names': self.names.serialize(self.alt_fullname or self.fullname()), @@ -2008,7 +2014,8 @@ def serialize(self) -> Union[str, JsonDict]: def deserialize(cls, data: JsonDict) -> 'TypeInfo': names = SymbolTable.deserialize(data['names']) defn = ClassDef.deserialize(data['defn']) - ti = TypeInfo(names, defn) + module_name = data['module_name'] + ti = TypeInfo(names, defn, module_name) ti._fullname = data['fullname'] ti.alt_fullname = data['alt_fullname'] # TODO: Is there a reason to reconstruct ti.subtypes? @@ -2027,9 +2034,9 @@ def deserialize(cls, data: JsonDict) -> 'TypeInfo': return ti -def namedtuple_type_info(tup: 'mypy.types.TupleType', - names: 'SymbolTable', defn: ClassDef) -> TypeInfo: - info = TypeInfo(names, defn) +def namedtuple_type_info(tup: 'mypy.types.TupleType', names: 'SymbolTable', + defn: ClassDef, module_name: str) -> TypeInfo: + info = TypeInfo(names, defn, module_name) info.tuple_type = tup info.bases = [tup.fallback] info.is_named_tuple = True diff --git a/mypy/semanal.py b/mypy/semanal.py index 7d496676fde7b..9b1d541f2bc74 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -729,7 +729,7 @@ def analyze_unbound_tvar(self, t: Type) -> Tuple[str, TypeVarExpr]: def setup_class_def_analysis(self, defn: ClassDef) -> None: """Prepare for the analysis of a class definition.""" if not defn.info: - defn.info = TypeInfo(SymbolTable(), defn) + defn.info = TypeInfo(SymbolTable(), defn, self.cur_mod_id) defn.info._fullname = defn.info.name() if self.is_func_scope() or self.type: kind = MDEF @@ -1416,7 +1416,7 @@ def build_newtype_typeinfo(self, name: str, old_type: Type, base_type: Instance) class_def.fullname = self.qualified_name(name) symbols = SymbolTable() - info = TypeInfo(symbols, class_def) + info = TypeInfo(symbols, class_def, self.cur_mod_id) info.mro = [info] + base_type.type.mro info.bases = [base_type] info.is_newtype = True @@ -1704,7 +1704,8 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], symbols = SymbolTable() class_def = ClassDef(name, Block([])) class_def.fullname = fullname - info = namedtuple_type_info(TupleType(types, fallback), symbols, class_def) + info = namedtuple_type_info(TupleType(types, fallback), symbols, + class_def, self.cur_mod_id) def add_field(var: Var, is_initialized_in_class: bool = False, is_property: bool = False) -> None: @@ -2641,7 +2642,7 @@ def visit_overloaded_func_def(self, func: OverloadedFuncDef) -> None: def visit_class_def(self, cdef: ClassDef) -> None: self.sem.check_no_global(cdef.name, cdef) cdef.fullname = self.sem.qualified_name(cdef.name) - info = TypeInfo(SymbolTable(), cdef) + info = TypeInfo(SymbolTable(), cdef, self.sem.cur_mod_id) info.set_line(cdef.line) cdef.info = info self.sem.globals[cdef.name] = SymbolTableNode(GDEF, info, @@ -2651,11 +2652,12 @@ def visit_class_def(self, cdef: ClassDef) -> None: def process_nested_classes(self, outer_def: ClassDef) -> None: for node in outer_def.defs.body: if isinstance(node, ClassDef): - node.info = TypeInfo(SymbolTable(), node) + node.info = TypeInfo(SymbolTable(), node, self.sem.cur_mod_id) if outer_def.fullname: node.info._fullname = outer_def.fullname + '.' + node.info.name() else: node.info._fullname = node.info.name() + node.fullname = node.info._fullname symbol = SymbolTableNode(MDEF, node.info) outer_def.info.names[node.name] = symbol self.process_nested_classes(node) diff --git a/mypy/typefixture.py b/mypy/typefixture.py index 59ffeea1046d4..a5fac17300dd3 100644 --- a/mypy/typefixture.py +++ b/mypy/typefixture.py @@ -193,6 +193,7 @@ def callable_var_arg(self, min_args, *a): a[-1], self.function) def make_type_info(self, name: str, + module_name: str = None, is_abstract: bool = False, mro: List[TypeInfo] = None, bases: List[Instance] = None, @@ -203,6 +204,12 @@ def make_type_info(self, name: str, class_def = ClassDef(name, Block([]), None, []) class_def.fullname = name + if module_name is None: + if '.' in name: + module_name = name.rsplit('.', 1)[0] + else: + module_name = '__main__' + if typevars: v = [] # type: List[TypeVarDef] for id, n in enumerate(typevars, 1): @@ -213,7 +220,7 @@ def make_type_info(self, name: str, v.append(TypeVarDef(n, id, None, self.o, variance=variance)) class_def.type_vars = v - info = TypeInfo(SymbolTable(), class_def) + info = TypeInfo(SymbolTable(), class_def, module_name) if mro is None: mro = [] if name != 'builtins.object': From ec2fabd51eae10351b2c81fd9ad757a435123789 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Thu, 25 Aug 2016 16:25:33 -0700 Subject: [PATCH 03/18] Modify typechecker to record all module references This commit modifies the typechecking stage to record all module references that a particular module makes. This information will be used in a future commit. The motivation for this change is that it turns out that the set of all module references is often larger then the set of explicitly imported modules, and having this information would allow us to make incremental mode more aggressive. --- mypy/checker.py | 12 ++++++++++++ mypy/checkexpr.py | 47 +++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index efc2985d7513f..9231cd73b0a68 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -87,6 +87,8 @@ class TypeChecker(NodeVisitor[Type]): msg = None # type: MessageBuilder # Types of type checked nodes type_map = None # type: Dict[Node, Type] + # Types of type checked nodes within this specific module + module_type_map = None # type: Dict[Node, Type] # Helper for managing conditional types binder = None # type: ConditionalTypeBinder @@ -121,6 +123,10 @@ class TypeChecker(NodeVisitor[Type]): suppress_none_errors = False options = None # type: Options + # The set of all dependencies (suppressed or not) that this module accesses, either + # directly or indirectly. + module_refs = None # type: Set[str] + def __init__(self, errors: Errors, modules: Dict[str, MypyFile], options: Options) -> None: """Construct a type checker. @@ -131,6 +137,7 @@ def __init__(self, errors: Errors, modules: Dict[str, MypyFile], options: Option self.options = options self.msg = MessageBuilder(errors, modules) self.type_map = {} + self.module_type_map = {} self.binder = ConditionalTypeBinder() self.expr_checker = mypy.checkexpr.ExpressionChecker(self, self.msg) self.return_types = [] @@ -142,6 +149,7 @@ def __init__(self, errors: Errors, modules: Dict[str, MypyFile], options: Option self.deferred_nodes = [] self.pass_num = 0 self.current_node_deferred = False + self.module_refs = set() def visit_file(self, file_node: MypyFile, path: str) -> None: """Type check a mypy file with the given path.""" @@ -152,6 +160,8 @@ def visit_file(self, file_node: MypyFile, path: str) -> None: self.weak_opts = file_node.weak_opts self.enter_partial_types() self.is_typeshed_stub = self.errors.is_typeshed_file(path) + self.module_type_map = {} + self.module_refs = set() if self.options.strict_optional_whitelist is None: self.suppress_none_errors = False else: @@ -2211,6 +2221,8 @@ def check_type_equivalency(self, t1: Type, t2: Type, node: Context, def store_type(self, node: Node, typ: Type) -> None: """Store the type of a node in the type map.""" self.type_map[node] = typ + if typ is not None: + self.module_type_map[node] = typ def typing_mode_none(self) -> bool: if self.is_dynamic_function() and not self.options.check_untyped_defs: diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index bf69f3cd13920..b75334cf5c328 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1,6 +1,6 @@ """Expression type checker. This file is conceptually part of TypeChecker.""" -from typing import cast, Dict, List, Tuple, Callable, Union, Optional +from typing import cast, Dict, Set, List, Iterable, Tuple, Callable, Union, Optional from mypy.types import ( Type, AnyType, CallableType, Overloaded, NoneTyp, Void, TypeVarDef, @@ -16,7 +16,7 @@ ListComprehension, GeneratorExpr, SetExpr, MypyFile, Decorator, ConditionalExpr, ComparisonExpr, TempNode, SetComprehension, DictionaryComprehension, ComplexExpr, EllipsisExpr, StarExpr, - TypeAliasExpr, BackquoteExpr, ARG_POS, ARG_NAMED, ARG_STAR2 + TypeAliasExpr, BackquoteExpr, ARG_POS, ARG_NAMED, ARG_STAR2, MODULE_REF, ) from mypy.nodes import function_type from mypy import nodes @@ -45,6 +45,43 @@ None] +def split_module_names(mod_name: str) -> Iterable[str]: + """Yields the module and all parent module names. + + So, if `mod_name` is 'a.b.c', this function will yield + ['a.b.c', 'a.b', and 'a'].""" + yield mod_name + while '.' in mod_name: + mod_name = mod_name.rsplit('.', 1)[0] + yield mod_name + + +def extract_refexpr_names(expr: RefExpr) -> Set[str]: + """Recursively extracts all module references from a reference expression. + + Note that currently, the only two subclasses of RefExpr are NameExpr and + MemberExpr.""" + output = set() # type: Set[str] + while expr.kind == MODULE_REF or expr.fullname is not None: + if expr.kind == MODULE_REF: + output.add(expr.fullname) + elif expr.fullname is not None and '.' in expr.fullname: + output.add(expr.fullname.rsplit('.', 1)[0]) + + if isinstance(expr, NameExpr): + if expr.info is not None: + output.update(split_module_names(expr.info.module_name)) + break + elif isinstance(expr, MemberExpr): + if isinstance(expr.expr, RefExpr): + expr = expr.expr + else: + break + else: + raise AssertionError("Unknown RefExpr subclass: {}".format(type(expr))) + return output + + class Finished(Exception): """Raised if we can terminate overload argument check early (no match).""" @@ -70,11 +107,16 @@ def __init__(self, self.msg = msg self.strfrm_checker = StringFormatterChecker(self, self.chk, self.msg) + def _visit_typeinfo(self, info: nodes.TypeInfo) -> None: + if info is not None: + self.chk.module_refs.update(split_module_names(info.module_name)) + def visit_name_expr(self, e: NameExpr) -> Type: """Type check a name expression. It can be of any kind: local, member or global. """ + self.chk.module_refs.update(extract_refexpr_names(e)) result = self.analyze_ref_expr(e) return self.chk.narrow_type_from_binder(e, result) @@ -861,6 +903,7 @@ def apply_generic_arguments2(self, overload: Overloaded, types: List[Type], def visit_member_expr(self, e: MemberExpr) -> Type: """Visit member expression (of form e.id).""" + self.chk.module_refs.update(extract_refexpr_names(e)) result = self.analyze_ordinary_member_access(e, False) return self.chk.narrow_type_from_binder(e, result) From 7cb5505e431bfe1fea2f812b426a5ff51c182a4e Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Thu, 25 Aug 2016 16:36:34 -0700 Subject: [PATCH 04/18] Add module reference extractor for types This commit adds a new visitor to extract all module references from arbitrary types. This visitor should be considered provisional -- it will most likely be merged in with the typechecker at a later stage to make it a less expensive call. --- mypy/checkexpr.py | 12 +----- mypy/indirection.py | 101 ++++++++++++++++++++++++++++++++++++++++++++ mypy/util.py | 13 +++++- 3 files changed, 114 insertions(+), 12 deletions(-) create mode 100644 mypy/indirection.py diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index b75334cf5c328..cb63ba843a189 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -36,6 +36,7 @@ from mypy.constraints import get_actual_type from mypy.checkstrformat import StringFormatterChecker from mypy.expandtype import expand_type +from mypy.util import split_module_names from mypy import experiments @@ -45,17 +46,6 @@ None] -def split_module_names(mod_name: str) -> Iterable[str]: - """Yields the module and all parent module names. - - So, if `mod_name` is 'a.b.c', this function will yield - ['a.b.c', 'a.b', and 'a'].""" - yield mod_name - while '.' in mod_name: - mod_name = mod_name.rsplit('.', 1)[0] - yield mod_name - - def extract_refexpr_names(expr: RefExpr) -> Set[str]: """Recursively extracts all module references from a reference expression. diff --git a/mypy/indirection.py b/mypy/indirection.py new file mode 100644 index 0000000000000..c5250b8cda06d --- /dev/null +++ b/mypy/indirection.py @@ -0,0 +1,101 @@ +from typing import Optional, Set, Iterable +from abc import abstractmethod + +from mypy.visitor import NodeVisitor +from mypy.types import TypeVisitor +from mypy.nodes import MODULE_REF +import mypy.nodes as nodes +import mypy.types as types +from mypy.util import split_module_names + + +def extract_module_names(symbol_name: str) -> Iterable[str]: + """Returns the module and parent modules of a fully qualified symbol name.""" + if symbol_name is not None: + while '.' in symbol_name: + symbol_name = symbol_name.rsplit('.', 1)[0] + yield symbol_name + + +class TypeIndirectionVisitor(TypeVisitor[Set[str]]): + """Returns all module references within a particular type.""" + + def __init__(self) -> None: + self.cache = {} # type: Dict[types.Type, Set[str]] + + def find_modules(self, typs: Iterable[types.Type]) -> Set[str]: + return self._visit(*typs) + + def _visit(self, *typs: types.Type) -> Set[str]: + output = set() # type: Set[str] + for typ in typs: + if typ in self.cache: + modules = self.cache[typ] + else: + modules = typ.accept(self) + self.cache[typ] = set(modules) + output.update(modules) + return output + + def visit_unbound_type(self, t: types.UnboundType) -> Set[str]: + return self._visit(*t.args) + + def visit_type_list(self, t: types.TypeList) -> Set[str]: + return self._visit(*t.items) + + def visit_error_type(self, t: types.ErrorType) -> Set[str]: + return set() + + def visit_any(self, t: types.AnyType) -> Set[str]: + return set() + + def visit_void(self, t: types.Void) -> Set[str]: + return set() + + def visit_none_type(self, t: types.NoneTyp) -> Set[str]: + return set() + + def visit_uninhabited_type(self, t: types.UninhabitedType) -> Set[str]: + return set() + + def visit_erased_type(self, t: types.ErasedType) -> Set[str]: + return set() + + def visit_deleted_type(self, t: types.DeletedType) -> Set[str]: + return set() + + def visit_type_var(self, t: types.TypeVarType) -> Set[str]: + return self._visit(*t.values) | self._visit(t.upper_bound) + + def visit_instance(self, t: types.Instance) -> Set[str]: + out = self._visit(*t.args) + if t.type is not None: + out.update(split_module_names(t.type.module_name)) + return out + + def visit_callable_type(self, t: types.CallableType) -> Set[str]: + out = self._visit(*t.arg_types) | self._visit(t.ret_type) + if t.definition is not None: + out.update(extract_module_names(t.definition.fullname())) + return out + + def visit_overloaded(self, t: types.Overloaded) -> Set[str]: + return self._visit(*t.items()) | self._visit(t.fallback) + + def visit_tuple_type(self, t: types.TupleType) -> Set[str]: + return self._visit(*t.items) | self._visit(t.fallback) + + def visit_star_type(self, t: types.StarType) -> Set[str]: + return set() + + def visit_union_type(self, t: types.UnionType) -> Set[str]: + return self._visit(*t.items) + + def visit_partial_type(self, t: types.PartialType) -> Set[str]: + return set() + + def visit_ellipsis_type(self, t: types.EllipsisType) -> Set[str]: + return set() + + def visit_type_type(self, t: types.TypeType) -> Set[str]: + return self._visit(t.item) diff --git a/mypy/util.py b/mypy/util.py index d8b10b8f0a496..2b3005b017d67 100644 --- a/mypy/util.py +++ b/mypy/util.py @@ -2,7 +2,7 @@ import re import subprocess -from typing import TypeVar, List, Any, Tuple, Optional +from typing import TypeVar, List, Any, Tuple, Optional, Iterable T = TypeVar('T') @@ -12,6 +12,17 @@ default_python2_interpreter = ['python2', 'python', '/usr/bin/python'] +def split_module_names(mod_name: str) -> Iterable[str]: + """Yields the module and all parent module names. + + So, if `mod_name` is 'a.b.c', this function will yield + ['a.b.c', 'a.b', and 'a'].""" + yield mod_name + while '.' in mod_name: + mod_name = mod_name.rsplit('.', 1)[0] + yield mod_name + + def short_type(obj: object) -> str: """Return the last component of the type name of an object. From aeffa82b228b09cfce8432e1d74f2b0d6aad2f1c Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Thu, 25 Aug 2016 16:39:24 -0700 Subject: [PATCH 05/18] Add implicit module reference data to dependencies This commit modifies the build process to record all implicit module references a given file makes when writing cache data. Previously, the cache data would only save all explicit dependencies (i.e. modules used within a direct import). This posed a problem because it turned out it's trivially easy to construct a series of interlinked files wherein changing the interface of one module produces or fixes an error in a seemingly unrelated file (which did NOT import anything from the first module). This made optimizing incremental mode harder since an interface change could have unknown effects on the codebase as a whole. This change modifies how dependencies are recorded so BOTH explicit (directly imported) and implicit (indirectly reliant) dependences are recorded for each file. This means that checking to see if any of our dependents' interfaces have changed requires simply checking every module in the list of dependencies (moving one "step") rather then having to potentially crawl the entire graph. In order to avoid accidentally perturbing the import graph in unpredictable ways, all implicit dependences will be given a lower priority then regular imports (lower then PRI_LOW). --- mypy/build.py | 46 +++++++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 524cc206eb53d..32443b97462e5 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -29,6 +29,7 @@ SymbolTableNode, MODULE_REF) from mypy.semanal import FirstPass, SemanticAnalyzer, ThirdPass from mypy.checker import TypeChecker +from mypy.indirection import TypeIndirectionVisitor from mypy.errors import Errors, CompileError, DecodeError, report_internal_error from mypy import fixup from mypy.report import Reports @@ -306,6 +307,7 @@ def default_lib_path(data_dir: str, pyversion: Tuple[int, int]) -> List[str]: PRI_HIGH = 5 # top-level "from X import blah" PRI_MED = 10 # top-level "import X" PRI_LOW = 20 # either form inside a function +PRI_INDIRECT = 30 # an indirect dependency PRI_ALL = 99 # include all priorities @@ -352,6 +354,7 @@ def __init__(self, data_dir: str, self.modules = self.semantic_analyzer.modules self.semantic_analyzer_pass3 = ThirdPass(self.modules, self.errors) self.type_checker = TypeChecker(self.errors, self.modules, options=options) + self.indirection_detector = TypeIndirectionVisitor() self.missing_modules = set() # type: Set[str] self.stale_modules = set() # type: Set[str] self.rechecked_modules = set() # type: Set[str] @@ -1422,11 +1425,35 @@ def type_check(self) -> None: return with self.wrap_context(): manager.type_checker.visit_file(self.tree, self.xpath) + + if manager.options.incremental: + self._patch_indirect_dependencies(manager.type_checker.module_refs) + if manager.options.dump_inference_stats: dump_type_stats(self.tree, self.xpath, inferred=True, typemap=manager.type_checker.type_map) manager.report_file(self.tree) + def _patch_indirect_dependencies(self, module_refs: Set[str]) -> None: + types = self.manager.type_checker.module_type_map.values() + valid = self.valid_references() + + encountered = self.manager.indirection_detector.find_modules(types) | module_refs + extra = encountered - valid + + for dep in sorted(extra): + self.dependencies.append(dep) + self.priorities[dep] = PRI_INDIRECT + + def valid_references(self) -> Set[str]: + valid_refs = set(self.dependencies + self.suppressed + self.ancestors) + valid_refs .add(self.id) + + if "os" in valid_refs: + valid_refs.add("os.path") + + return valid_refs + def write_cache(self) -> None: if self.path and self.manager.options.incremental and not self.manager.errors.is_errors(): dep_prios = [self.priorities.get(dep, PRI_HIGH) for dep in self.dependencies] @@ -1605,25 +1632,6 @@ def process_graph(graph: Graph, manager: BuildManager) -> None: else: process_stale_scc(graph, scc) - # TODO: This is a workaround to get around the "chaining imports" problem - # with the interface checks. - # - # That is, if we have a file named `module_a.py` which does: - # - # import module_b - # module_b.module_c.foo(3) - # - # ...and if the type signature of `module_c.foo(...)` were to change, - # module_a_ would not be rechecked since the interface of `module_b` - # would not be considered changed. - # - # As a workaround, this check will force a module's interface to be - # considered stale if anything it imports has a stale interface, - # which ensures these changes are caught and propagated. - if len(stale_deps) > 0: - for id in scc: - graph[id].mark_interface_stale() - def order_ascc(graph: Graph, ascc: AbstractSet[str], pri_max: int = PRI_ALL) -> List[str]: """Come up with the ideal processing order within an SCC. From 3776416163409dacbe7af903f32f8bf8a10c75e5 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Mon, 29 Aug 2016 20:27:24 -0700 Subject: [PATCH 06/18] Do not store silenced indirect dependencies Previously, the indirect dependency patcher would always add in indirect dependencies, even if they were silenced or ignored. This commit makes the patcher more discriminating. --- mypy/build.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 32443b97462e5..a3bac211648e7 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -1442,8 +1442,11 @@ def _patch_indirect_dependencies(self, module_refs: Set[str]) -> None: extra = encountered - valid for dep in sorted(extra): - self.dependencies.append(dep) - self.priorities[dep] = PRI_INDIRECT + if dep not in self.suppressed and dep not in self.manager.missing_modules: + self.dependencies.append(dep) + self.priorities[dep] = PRI_INDIRECT + elif dep not in self.suppressed and dep in self.manager.missing_modules: + self.suppressed.append(dep) def valid_references(self) -> Set[str]: valid_refs = set(self.dependencies + self.suppressed + self.ancestors) From 54e932a0cfe42545268c86f39749924f0082ef45 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Mon, 29 Aug 2016 20:29:43 -0700 Subject: [PATCH 07/18] Keep track of if a Var is an import or not This commit modifies Vars and adds an extra field to indicate if they refer to imports or not. Normally, modules are not represented as Var nodes, except in the case where that particular module was silenced or suppressed in some manner. This commit will make it easier to perform analysis on the AST during the typechecking phase. --- mypy/nodes.py | 5 +++++ mypy/semanal.py | 7 ++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index 72833fc35bdfe..2d21d616dc1ed 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -583,6 +583,9 @@ class Var(SymbolNode, Statement): is_classmethod = False is_property = False is_settable_property = False + # Set to true when this variable refers to a module we were unable to + # parse for some reason (eg a silenced module) + is_import = False def __init__(self, name: str, type: 'mypy.types.Type' = None) -> None: self._name = name @@ -613,6 +616,7 @@ def serialize(self) -> JsonDict: 'is_classmethod': self.is_classmethod, 'is_property': self.is_property, 'is_settable_property': self.is_settable_property, + 'is_import': self.is_import, } # type: JsonDict return data @@ -629,6 +633,7 @@ def deserialize(cls, data: JsonDict) -> 'Var': v.is_classmethod = data['is_classmethod'] v.is_property = data['is_property'] v.is_settable_property = data['is_settable_property'] + v.is_import = data['is_import'] return v diff --git a/mypy/semanal.py b/mypy/semanal.py index 9b1d541f2bc74..93d4423eb4a0c 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -923,7 +923,7 @@ def add_module_symbol(self, id: str, as_id: str, module_public: bool, self.add_symbol(as_id, SymbolTableNode(MODULE_REF, m, self.cur_mod_id, module_public=module_public), context) else: - self.add_unknown_symbol(as_id, context) + self.add_unknown_symbol(as_id, context, is_import=True) def visit_import_from(self, imp: ImportFrom) -> None: import_id = self.correct_relative_import(imp) @@ -969,7 +969,7 @@ def visit_import_from(self, imp: ImportFrom) -> None: else: # Missing module. for id, as_id in imp.names: - self.add_unknown_symbol(as_id or id, imp) + self.add_unknown_symbol(as_id or id, imp, is_import=True) def process_import_over_existing_name(self, imported_id: str, existing_symbol: SymbolTableNode, @@ -1039,7 +1039,7 @@ def visit_import_all(self, i: ImportAll) -> None: # Don't add any dummy symbols for 'from x import *' if 'x' is unknown. pass - def add_unknown_symbol(self, name: str, context: Context) -> None: + def add_unknown_symbol(self, name: str, context: Context, is_import: bool = False) -> None: var = Var(name) if self.type: var._fullname = self.type.fullname() + "." + name @@ -1047,6 +1047,7 @@ def add_unknown_symbol(self, name: str, context: Context) -> None: var._fullname = self.qualified_name(name) var.is_ready = True var.type = AnyType() + var.is_import = is_import self.add_symbol(name, SymbolTableNode(GDEF, var, self.cur_mod_id), context) # From 678f391c06b7231c1c563624ee383043b91cedae Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Mon, 29 Aug 2016 20:32:47 -0700 Subject: [PATCH 08/18] Handle locally imported silenced modules Previously, the RefExpr module analyzer did not correctly handle the case where a silenced import was used inside of a class. For example, suppose we have a file named `mod1.py`: ```python class Foo: def bar(self) -> None: import unrelated # this is suppressed self.val = unrelated.test() ``` ...the analyzer would end up reporting that this file indirectly relied on module `mod1.Foo`, which is clearly incorrect. This commit attempts to handle this case. --- mypy/checkexpr.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index cb63ba843a189..91a377628257f 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -56,7 +56,8 @@ def extract_refexpr_names(expr: RefExpr) -> Set[str]: if expr.kind == MODULE_REF: output.add(expr.fullname) elif expr.fullname is not None and '.' in expr.fullname: - output.add(expr.fullname.rsplit('.', 1)[0]) + if not (isinstance(expr.node, Var) and expr.node.is_import): + output.add(expr.fullname.rsplit('.', 1)[0]) if isinstance(expr, NameExpr): if expr.info is not None: From 6b1808063bf6632d8b684ed47c5dc660c1afadc7 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Mon, 29 Aug 2016 20:35:31 -0700 Subject: [PATCH 09/18] Add incomplete tests to handle two previous bugs --- test-data/unit/check-incremental.test | 52 +++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 273b42d7ed391..e0f8d5b9698fa 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -1148,3 +1148,55 @@ def C() -> str: [out2] main:1: note: In module imported here: tmp/m1.py:3: error: Argument 1 to "accepts_int" has incompatible type "str"; expected "int" + +[case testIncrementalSilentImportsWithInnerImports] +# cmd: mypy -m main foo +# options: silent_imports + +[file main.py] +from foo import MyClass +m = MyClass() + +[file main.py.next] +from foo import MyClass +m = MyClass() +reveal_type(m.val) + +[file foo.py] +class MyClass: + def __init__(self) -> None: + import unrelated + self.val = unrelated.test() + +[rechecked main] +[stale] +[out] +tmp/main.py:3: error: Revealed type is 'Any' + +[case testIncrementalSilentImportsWithInnerImportsAndNewFile] +# cmd: mypy -m main foo +# cmd2: mypy -m main foo unrelated +# options: silent_imports + +[file main.py] +from foo import MyClass +m = MyClass() + +[file main.py.next] +from foo import MyClass +m = MyClass() +reveal_type(m.val) + +[file foo.py] +class MyClass: + def __init__(self) -> None: + import unrelated + self.val = unrelated.test() + +[file unrelated.py] +def test() -> str: return "foo" + +[rechecked main, foo, unrelated] +[stale foo, unrelated] +[out] +tmp/main.py:3: error: Revealed type is 'builtins.str' From 51edc2dbf04fc31f3682d0d9c0df5102f06c6fc1 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Tue, 30 Aug 2016 10:11:00 -0700 Subject: [PATCH 10/18] Modify tests to work with new '# flag' directive The `# options: ...` directive was replaced with the `# flag: ...` directive. --- test-data/unit/check-incremental.test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index e0f8d5b9698fa..d7a9fdef716fb 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -1151,7 +1151,7 @@ tmp/m1.py:3: error: Argument 1 to "accepts_int" has incompatible type "str"; exp [case testIncrementalSilentImportsWithInnerImports] # cmd: mypy -m main foo -# options: silent_imports +# flags: --silent-imports [file main.py] from foo import MyClass @@ -1176,7 +1176,7 @@ tmp/main.py:3: error: Revealed type is 'Any' [case testIncrementalSilentImportsWithInnerImportsAndNewFile] # cmd: mypy -m main foo # cmd2: mypy -m main foo unrelated -# options: silent_imports +# flags: --silent-imports [file main.py] from foo import MyClass From bf89e5f2ce4936e3a09ace79491d9b62042234f4 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Tue, 30 Aug 2016 13:46:51 -0700 Subject: [PATCH 11/18] Add test cases to handle nested classes --- mypy/checkexpr.py | 10 +++++++--- test-data/unit/check-incremental.test | 14 ++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 91a377628257f..1eb2046972376 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -55,13 +55,17 @@ def extract_refexpr_names(expr: RefExpr) -> Set[str]: while expr.kind == MODULE_REF or expr.fullname is not None: if expr.kind == MODULE_REF: output.add(expr.fullname) - elif expr.fullname is not None and '.' in expr.fullname: - if not (isinstance(expr.node, Var) and expr.node.is_import): - output.add(expr.fullname.rsplit('.', 1)[0]) if isinstance(expr, NameExpr): + is_silenced_import = isinstance(expr.node, Var) and expr.node.is_import if expr.info is not None: + # Reference to regular type output.update(split_module_names(expr.info.module_name)) + elif isinstance(expr.node, TypeInfo): + # Nested class + output.update(split_module_names(expr.node.module_name)) + elif expr.fullname is not None and '.' in expr.fullname and not is_silenced_import: + output.add(expr.fullname.rsplit('.', 1)[0]) break elif isinstance(expr, MemberExpr): if isinstance(expr.expr, RefExpr): diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index d7a9fdef716fb..ed671b1d24abb 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -1200,3 +1200,17 @@ def test() -> str: return "foo" [stale foo, unrelated] [out] tmp/main.py:3: error: Revealed type is 'builtins.str' + +[case testIncrementalWorksWithNestedClasses] +import foo + +[file foo.py] +class MyClass: + class NestedClass: + pass + + class_attr = NestedClass() + +[rechecked] +[stale] +[out] From 17cea09870caebc793aa2282b00438c6ecaed898 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Tue, 30 Aug 2016 14:31:12 -0700 Subject: [PATCH 12/18] Add extra check to filter out non-modules This commit adds an (ad-hoc) check to the indirect dependencies patcher to filter out computed dependencies that are not, in fact, actually modules. Ideally, this ad-hoc check shouldn't be necessary but there are enough weird edge cases that it seems like a useful thing to have to prevent unexpected behavior. --- mypy/build.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mypy/build.py b/mypy/build.py index a3bac211648e7..4acc740bffe62 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -1442,6 +1442,8 @@ def _patch_indirect_dependencies(self, module_refs: Set[str]) -> None: extra = encountered - valid for dep in sorted(extra): + if dep not in self.manager.modules: + continue if dep not in self.suppressed and dep not in self.manager.missing_modules: self.dependencies.append(dep) self.priorities[dep] = PRI_INDIRECT From 6812ae33701d034e491bbdcce381f10d27838996 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Tue, 30 Aug 2016 19:55:02 -0700 Subject: [PATCH 13/18] Test interactions with silent imports and errors This commit adds several more tests to try and check if mypy behaves correctly under a variety silent import related scenarios, and if it behaves correctly starting from an initial error. --- test-data/unit/check-incremental.test | 215 +++++++++++++++++++++++++- 1 file changed, 211 insertions(+), 4 deletions(-) diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index ed671b1d24abb..fc78a22e55e11 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -1058,7 +1058,7 @@ class A: class A: def bar(self): pass [rechecked m, n] -[stale m, n] +[stale n] [out1] main:2: error: "A" has no attribute "bar" @@ -1149,6 +1149,214 @@ def C() -> str: main:1: note: In module imported here: tmp/m1.py:3: error: Argument 1 to "accepts_int" has incompatible type "str"; expected "int" +[case testIncrementalSilentImportsWithBlatantError] +# cmd: mypy -m main +# flags: --silent-imports + +[file main.py] +from evil import Hello + +[file main.py.next] +from evil import Hello +reveal_type(Hello()) + +[file evil.py] +def accept_int(x: int) -> None: pass +accept_int("not an int") + +[rechecked main] +[stale] +[out2] +tmp/main.py:2: error: Revealed type is 'Any' + +[case testIncrementalImportIsNewlySilenced] +# cmd: mypy -m main foo +# cmd2: mypy -m main +# flags: --silent-imports + +[file main.py] +from foo import bar +def accept_int(x: int) -> None: pass +accept_int(bar) + +[file foo.py] +bar = 3 + +[file foo.py.next] +# Empty! + +[rechecked main] +[stale main] + +[case testIncrementalSilencedModuleNoLongerCausesError] +# cmd: mypy -m main evil +# cmd2: mypy -m main +# flags: --silent-imports + +[file main.py] +from evil import bar +def accept_int(x: int) -> None: pass +accept_int(bar) +reveal_type(bar) + +[file evil.py] +bar = "str" + +[rechecked main] +[stale] +[out1] +tmp/main.py:3: error: Argument 1 to "accept_int" has incompatible type "str"; expected "int" +tmp/main.py:4: error: Revealed type is 'builtins.str' +[out2] +tmp/main.py:4: error: Revealed type is 'Any' + +[case testIncrementalFixedBugCausesPropagation] +import mod1 + +[file mod1.py] +from mod2 import A +val = A().makeB().makeC().foo() +reveal_type(val) + +[file mod2.py] +from mod3 import B +class A: + def makeB(self) -> B: return B() + +[file mod3.py] +from mod4 import C +class B: + def makeC(self) -> C: + val = 3 # type: int + val = "str" # deliberately triggering error + return C() + +[file mod3.py.next] +from mod4 import C +class B: + def makeC(self) -> C: return C() + +[file mod4.py] +class C: + def foo(self) -> int: return 1 + +[rechecked mod3, mod2, mod1] +[stale mod3, mod2] +[out1] +tmp/mod2.py:1: note: In module imported here, +tmp/mod1.py:1: note: ... from here, +main:1: note: ... from here: +tmp/mod3.py: note: In member "makeC" of class "B": +tmp/mod3.py:5: error: Incompatible types in assignment (expression has type "str", variable has type "int") +main:1: note: In module imported here: +tmp/mod1.py: note: At top level: +tmp/mod1.py:3: error: Revealed type is 'builtins.int' + +[out2] +main:1: note: In module imported here: +tmp/mod1.py:3: error: Revealed type is 'builtins.int' + +[case testIncrementalIncidentalChangeWithBugCausesPropagation] +import mod1 + +[file mod1.py] +from mod2 import A +val = A().makeB().makeC().foo() +reveal_type(val) + +[file mod2.py] +from mod3 import B +class A: + def makeB(self) -> B: return B() + +[file mod3.py] +from mod4 import C +class B: + def makeC(self) -> C: + val = 3 # type: int + val = "str" # deliberately triggering error + return C() + +[file mod4.py] +class C: + def foo(self) -> int: return 1 + +[file mod4.py.next] +class C: + def foo(self) -> str: return 'a' + +[rechecked mod4, mod3, mod2, mod1] +[stale mod4] +[out1] +tmp/mod2.py:1: note: In module imported here, +tmp/mod1.py:1: note: ... from here, +main:1: note: ... from here: +tmp/mod3.py: note: In member "makeC" of class "B": +tmp/mod3.py:5: error: Incompatible types in assignment (expression has type "str", variable has type "int") +main:1: note: In module imported here: +tmp/mod1.py: note: At top level: +tmp/mod1.py:3: error: Revealed type is 'builtins.int' + +[out2] +tmp/mod2.py:1: note: In module imported here, +tmp/mod1.py:1: note: ... from here, +main:1: note: ... from here: +tmp/mod3.py: note: In member "makeC" of class "B": +tmp/mod3.py:5: error: Incompatible types in assignment (expression has type "str", variable has type "int") +main:1: note: In module imported here: +tmp/mod1.py: note: At top level: +tmp/mod1.py:3: error: Revealed type is 'builtins.str' + +[case testIncrementalIncidentalChangeWithBugFixCausesPropagation] +import mod1 + +[file mod1.py] +from mod2 import A +val = A().makeB().makeC().foo() +reveal_type(val) + +[file mod2.py] +from mod3 import B +class A: + def makeB(self) -> B: return B() + +[file mod3.py] +from mod4 import C +class B: + def makeC(self) -> C: + val = 3 # type: int + val = "str" # deliberately triggering error + return C() + +[file mod3.py.next] +from mod4 import C +class B: + def makeC(self) -> C: return C() + +[file mod4.py] +class C: + def foo(self) -> int: return 1 + +[file mod4.py.next] +class C: + def foo(self) -> str: return 'a' + +[rechecked mod4, mod3, mod2, mod1] +[stale mod4, mod3, mod2] +[out1] +tmp/mod2.py:1: note: In module imported here, +tmp/mod1.py:1: note: ... from here, +main:1: note: ... from here: +tmp/mod3.py: note: In member "makeC" of class "B": +tmp/mod3.py:5: error: Incompatible types in assignment (expression has type "str", variable has type "int") +main:1: note: In module imported here: +tmp/mod1.py: note: At top level: +tmp/mod1.py:3: error: Revealed type is 'builtins.int' + +[out2] +main:1: note: In module imported here: +tmp/mod1.py:3: error: Revealed type is 'builtins.str' + [case testIncrementalSilentImportsWithInnerImports] # cmd: mypy -m main foo # flags: --silent-imports @@ -1170,7 +1378,7 @@ class MyClass: [rechecked main] [stale] -[out] +[out2] tmp/main.py:3: error: Revealed type is 'Any' [case testIncrementalSilentImportsWithInnerImportsAndNewFile] @@ -1198,7 +1406,7 @@ def test() -> str: return "foo" [rechecked main, foo, unrelated] [stale foo, unrelated] -[out] +[out2] tmp/main.py:3: error: Revealed type is 'builtins.str' [case testIncrementalWorksWithNestedClasses] @@ -1213,4 +1421,3 @@ class MyClass: [rechecked] [stale] -[out] From 247512c6f7f024adf4327d8c61655f65c0c2ecdf Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Wed, 31 Aug 2016 15:06:51 -0700 Subject: [PATCH 14/18] Add tests to stress test type references --- mypy/indirection.py | 2 +- test-data/unit/check-incremental.test | 82 ++++++++++++++++++++++++++- 2 files changed, 81 insertions(+), 3 deletions(-) diff --git a/mypy/indirection.py b/mypy/indirection.py index c5250b8cda06d..f6f70a26f95b0 100644 --- a/mypy/indirection.py +++ b/mypy/indirection.py @@ -10,7 +10,7 @@ def extract_module_names(symbol_name: str) -> Iterable[str]: - """Returns the module and parent modules of a fully qualified symbol name.""" + """Return the module and parent modules of a fully qualified symbol name.""" if symbol_name is not None: while '.' in symbol_name: symbol_name = symbol_name.rsplit('.', 1)[0] diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index fc78a22e55e11..ea60d5d17f6d4 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -351,18 +351,96 @@ import mod2.mod3 as mod3 [file mod2/mod3/__init__.py] import mod2.mod3.mod4 as mod4 +[file mod2/mod3/__init__.py.next] +# Import is gone! + [file mod2/mod3/mod4.py] const = 3 +[rechecked mod1, mod2, mod2.mod3] +[stale mod2.mod3] +[builtins fixtures/module.pyi] +[out2] +main:1: note: In module imported here: +tmp/mod1.py:3: error: "module" has no attribute "mod4" + +[case testIncrementalNestedBrokenCascadeWithType1] +import mod1, mod2.mod3.mod5 + +[file mod1.py] +import mod2 +def accept_int(x: int) -> None: pass +def produce() -> mod2.CustomType: + return mod2.CustomType() +a = produce() +accept_int(a.foo()) + +[file mod2/__init__.py] +from mod2.mod3 import CustomType + +[file mod2/mod3/__init__.py] +from mod2.mod3.mod4 import CustomType + [file mod2/mod3/__init__.py.next] -# Import is gone! +# Import a different class that also happens to be called 'CustomType' +from mod2.mod3.mod5 import CustomType +def produce() -> CustomType: + return CustomType() + +[file mod2/mod3/mod4.py] +class CustomType: + def foo(self) -> int: return 1 + +[file mod2/mod3/mod5.py] +class CustomType: + def foo(self) -> str: return "a" + +[rechecked mod1, mod2, mod2.mod3] +[stale mod2, mod2.mod3] +[builtins fixtures/module.pyi] +[out1] +[out2] +main:1: note: In module imported here: +tmp/mod1.py:6: error: Argument 1 to "accept_int" has incompatible type "str"; expected "int" + +[case testIncrementalNestedBrokenCascadeWithType2] +import mod1, mod2.mod3.mod5 + +[file mod1.py] +from mod2 import produce +def accept_int(x: int) -> None: pass +a = produce() +accept_int(a.foo()) + +[file mod2/__init__.py] +from mod2.mod3 import produce + +[file mod2/mod3/__init__.py] +from mod2.mod3.mod4 import CustomType +def produce() -> CustomType: + return CustomType() + +[file mod2/mod3/__init__.py.next] +# Import a different class that also happens to be called 'CustomType' +from mod2.mod3.mod5 import CustomType +def produce() -> CustomType: + return CustomType() + +[file mod2/mod3/mod4.py] +class CustomType: + def foo(self) -> int: return 1 + +[file mod2/mod3/mod5.py] +class CustomType: + def foo(self) -> str: return "a" [rechecked mod1, mod2, mod2.mod3] [stale mod2.mod3] [builtins fixtures/module.pyi] +[out1] [out2] main:1: note: In module imported here: -tmp/mod1.py:3: error: "module" has no attribute "mod4" +tmp/mod1.py:4: error: Argument 1 to "accept_int" has incompatible type "str"; expected "int" [case testIncrementalRemoteChange] import mod1 From 4611cbdface475244edf3bd46b88aac5ff73645a Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Wed, 31 Aug 2016 15:39:01 -0700 Subject: [PATCH 15/18] Clarify docstring for extract_module_names --- mypy/indirection.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mypy/indirection.py b/mypy/indirection.py index f6f70a26f95b0..b106cf4f3dd7a 100644 --- a/mypy/indirection.py +++ b/mypy/indirection.py @@ -9,12 +9,12 @@ from mypy.util import split_module_names -def extract_module_names(symbol_name: str) -> Iterable[str]: - """Return the module and parent modules of a fully qualified symbol name.""" - if symbol_name is not None: - while '.' in symbol_name: - symbol_name = symbol_name.rsplit('.', 1)[0] - yield symbol_name +def extract_module_names(type_name: Optional[str]) -> Iterable[str]: + """Returns the module name of a fully qualified type name.""" + if type_name is not None: + while '.' in type_name: + type_name = type_name.rsplit('.', 1)[0] + yield type_name class TypeIndirectionVisitor(TypeVisitor[Set[str]]): From b012ebb6be91e432c2ff1e0ce05b146393366d5d Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Fri, 2 Sep 2016 11:23:37 -0700 Subject: [PATCH 16/18] Remove unused NameExpr.info property It seems like the `info` attribute inside NameExpr is not actually set or used anywhere. This commit removes it altogether. --- mypy/checkexpr.py | 12 +++++------- mypy/nodes.py | 2 -- mypy/treetransform.py | 1 - 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 1eb2046972376..c0c427a3af1d9 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -57,14 +57,12 @@ def extract_refexpr_names(expr: RefExpr) -> Set[str]: output.add(expr.fullname) if isinstance(expr, NameExpr): - is_silenced_import = isinstance(expr.node, Var) and expr.node.is_import - if expr.info is not None: - # Reference to regular type - output.update(split_module_names(expr.info.module_name)) - elif isinstance(expr.node, TypeInfo): - # Nested class + is_suppressed_import = isinstance(expr.node, Var) and expr.node.is_suppressed_import + if isinstance(expr.node, TypeInfo): + # Reference to a class or a nested class output.update(split_module_names(expr.node.module_name)) - elif expr.fullname is not None and '.' in expr.fullname and not is_silenced_import: + elif expr.fullname is not None and '.' in expr.fullname and not is_suppressed_import: + # Everything else (that is not a silenced import within a class) output.add(expr.fullname.rsplit('.', 1)[0]) break elif isinstance(expr, MemberExpr): diff --git a/mypy/nodes.py b/mypy/nodes.py index 2d21d616dc1ed..13710a2b546da 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1109,8 +1109,6 @@ class NameExpr(RefExpr): """ name = None # type: str # Name referred to (may be qualified) - # TypeInfo of class surrounding expression (may be None) - info = None # type: TypeInfo literal = LITERAL_TYPE diff --git a/mypy/treetransform.py b/mypy/treetransform.py index 798ce8fbeeb48..b6143d4a7fefb 100644 --- a/mypy/treetransform.py +++ b/mypy/treetransform.py @@ -340,7 +340,6 @@ def duplicate_name(self, node: NameExpr) -> NameExpr: # This method is used when the transform result must be a NameExpr. # visit_name_expr() is used when there is no such restriction. new = NameExpr(node.name) - new.info = node.info self.copy_ref(new, node) return new From dd5c3ddfcedf911a78550ec12f5f9c4a4d5a06a1 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Fri, 2 Sep 2016 11:29:44 -0700 Subject: [PATCH 17/18] Respond to first wave (wavelet?) of feedback This commit contains mainly more fixups and refactoring in response to the first wave[let] of feedback. --- mypy/checkexpr.py | 4 ---- mypy/indirection.py | 14 ++++++++------ mypy/nodes.py | 6 +++--- mypy/semanal.py | 2 +- mypy/util.py | 16 +++++++++------- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index c0c427a3af1d9..24a42b17993bc 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -100,10 +100,6 @@ def __init__(self, self.msg = msg self.strfrm_checker = StringFormatterChecker(self, self.chk, self.msg) - def _visit_typeinfo(self, info: nodes.TypeInfo) -> None: - if info is not None: - self.chk.module_refs.update(split_module_names(info.module_name)) - def visit_name_expr(self, e: NameExpr) -> Type: """Type check a name expression. diff --git a/mypy/indirection.py b/mypy/indirection.py index b106cf4f3dd7a..77c5a59e88e3c 100644 --- a/mypy/indirection.py +++ b/mypy/indirection.py @@ -1,4 +1,4 @@ -from typing import Optional, Set, Iterable +from typing import Dict, Iterable, List, Optional, Set from abc import abstractmethod from mypy.visitor import NodeVisitor @@ -9,12 +9,14 @@ from mypy.util import split_module_names -def extract_module_names(type_name: Optional[str]) -> Iterable[str]: - """Returns the module name of a fully qualified type name.""" +def extract_module_names(type_name: Optional[str]) -> List[str]: + """Returns the module names of a fully qualified type name.""" if type_name is not None: - while '.' in type_name: - type_name = type_name.rsplit('.', 1)[0] - yield type_name + # Discard the first one, which is just the qualified name of the type + possible_module_names = split_module_names(type_name) + return possible_module_names[1:] + else: + return [] class TypeIndirectionVisitor(TypeVisitor[Set[str]]): diff --git a/mypy/nodes.py b/mypy/nodes.py index 13710a2b546da..764d947714160 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -585,7 +585,7 @@ class Var(SymbolNode, Statement): is_settable_property = False # Set to true when this variable refers to a module we were unable to # parse for some reason (eg a silenced module) - is_import = False + is_suppressed_import = False def __init__(self, name: str, type: 'mypy.types.Type' = None) -> None: self._name = name @@ -616,7 +616,7 @@ def serialize(self) -> JsonDict: 'is_classmethod': self.is_classmethod, 'is_property': self.is_property, 'is_settable_property': self.is_settable_property, - 'is_import': self.is_import, + 'is_suppressed_import': self.is_suppressed_import, } # type: JsonDict return data @@ -633,7 +633,7 @@ def deserialize(cls, data: JsonDict) -> 'Var': v.is_classmethod = data['is_classmethod'] v.is_property = data['is_property'] v.is_settable_property = data['is_settable_property'] - v.is_import = data['is_import'] + v.is_suppressed_import = data['is_suppressed_import'] return v diff --git a/mypy/semanal.py b/mypy/semanal.py index 93d4423eb4a0c..fd48ef0583d27 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1047,7 +1047,7 @@ def add_unknown_symbol(self, name: str, context: Context, is_import: bool = Fals var._fullname = self.qualified_name(name) var.is_ready = True var.type = AnyType() - var.is_import = is_import + var.is_suppressed_import = is_import self.add_symbol(name, SymbolTableNode(GDEF, var, self.cur_mod_id), context) # diff --git a/mypy/util.py b/mypy/util.py index 2b3005b017d67..c5b635ed232e3 100644 --- a/mypy/util.py +++ b/mypy/util.py @@ -2,7 +2,7 @@ import re import subprocess -from typing import TypeVar, List, Any, Tuple, Optional, Iterable +from typing import TypeVar, List, Any, Tuple, Optional T = TypeVar('T') @@ -12,15 +12,17 @@ default_python2_interpreter = ['python2', 'python', '/usr/bin/python'] -def split_module_names(mod_name: str) -> Iterable[str]: - """Yields the module and all parent module names. +def split_module_names(mod_name: str) -> List[str]: + """Return the module and all parent module names. - So, if `mod_name` is 'a.b.c', this function will yield - ['a.b.c', 'a.b', and 'a'].""" - yield mod_name + So, if `mod_name` is 'a.b.c', this function will return + ['a.b.c', 'a.b', and 'a']. + """ + out = [mod_name] while '.' in mod_name: mod_name = mod_name.rsplit('.', 1)[0] - yield mod_name + out.append(mod_name) + return out def short_type(obj: object) -> str: From fcbdd4b15875a7e491865e8eb6a72a28d7b695f0 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Fri, 2 Sep 2016 11:50:09 -0700 Subject: [PATCH 18/18] Verify namedtuple works with incremental --- test-data/unit/check-incremental.test | 68 +++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index ea60d5d17f6d4..37e52560f1ed0 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -1499,3 +1499,71 @@ class MyClass: [rechecked] [stale] + +[case testIncrementalWorksWithNamedTuple] +import foo + +[file foo.py] +from mid import MyTuple +def accept_int(x: int) -> None: pass +accept_int(MyTuple(1, "b", "c").a) + +[file mid.py] +from bar import MyTuple + +[file bar.py] +from typing import NamedTuple +MyTuple = NamedTuple('MyTuple', [ + ('a', int), + ('b', str), + ('c', str) +]) + +[file bar.py.next] +from typing import NamedTuple +MyTuple = NamedTuple('MyTuple', [ + ('b', int), # a and b are swapped + ('a', str), + ('c', str) +]) + +[rechecked bar, mid, foo] +[stale bar] +[out2] +main:1: note: In module imported here: +tmp/foo.py:3: error: Argument 1 to "accept_int" has incompatible type "str"; expected "int" + +[case testIncrementalWorksWithNestedNamedTuple] +import foo + +[file foo.py] +from mid import Outer +def accept_int(x: int) -> None: pass +accept_int(Outer.MyTuple(1, "b", "c").a) + +[file mid.py] +from bar import Outer + +[file bar.py] +from typing import NamedTuple +class Outer: + MyTuple = NamedTuple('MyTuple', [ + ('a', int), + ('b', str), + ('c', str) + ]) + +[file bar.py.next] +from typing import NamedTuple +class Outer: + MyTuple = NamedTuple('MyTuple', [ + ('b', int), # a and b are swapped + ('a', str), + ('c', str) + ]) + +[rechecked bar, mid, foo] +[stale bar] +[out2] +main:1: note: In module imported here: +tmp/foo.py:3: error: Argument 1 to "accept_int" has incompatible type "str"; expected "int"