From 252f94ed1d96e2b243f397251f80a5bcc07caafe Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 13 Oct 2017 21:36:47 +0200 Subject: [PATCH 1/2] Fix recent crashes in forward refs --- mypy/nodes.py | 1 + mypy/semanal_pass3.py | 32 ++++++++++++- mypy/typeanal.py | 6 ++- test-data/unit/check-classes.test | 49 ++++++++++++++++++++ test-data/unit/check-incremental.test | 65 +++++++++++++++++++++++++++ 5 files changed, 150 insertions(+), 3 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index 761f93620de1a..7d331e8bf8141 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1793,6 +1793,7 @@ class NewTypeExpr(Expression): def __init__(self, name: str, old_type: 'mypy.types.Type', line: int) -> None: self.name = name self.old_type = old_type + self.line = line def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_newtype_expr(self) diff --git a/mypy/semanal_pass3.py b/mypy/semanal_pass3.py index 9e9ba0d0deffa..585a9d56b5cac 100644 --- a/mypy/semanal_pass3.py +++ b/mypy/semanal_pass3.py @@ -17,7 +17,7 @@ Node, Expression, MypyFile, FuncDef, FuncItem, Decorator, RefExpr, Context, TypeInfo, ClassDef, Block, TypedDictExpr, NamedTupleExpr, AssignmentStmt, IndexExpr, TypeAliasExpr, NameExpr, CallExpr, NewTypeExpr, ForStmt, WithStmt, CastExpr, TypeVarExpr, TypeApplication, Lvalue, - TupleExpr, RevealTypeExpr, SymbolTableNode, Var, ARG_POS + TupleExpr, RevealTypeExpr, SymbolTableNode, Var, ARG_POS, OverloadedFuncDef ) from mypy.types import ( Type, Instance, AnyType, TypeOfAny, CallableType, TupleType, TypeVarType, TypedDictType, @@ -85,6 +85,10 @@ def visit_func_def(self, fdef: FuncDef) -> None: super().visit_func_def(fdef) self.errors.pop_function() + def visit_overloaded_func_def(self, fdef: OverloadedFuncDef) -> None: + self.analyze(fdef.type, fdef) + super().visit_overloaded_func_def(fdef) + def visit_class_def(self, tdef: ClassDef) -> None: # NamedTuple base classes are validated in check_namedtuple_classdef; we don't have to # check them again here. @@ -189,6 +193,15 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: analyzed = s.rvalue.analyzed if isinstance(analyzed, NewTypeExpr): self.analyze(analyzed.old_type, analyzed) + if analyzed.info: + for name in analyzed.info.names: + sym = analyzed.info.names[name] + # Currently NewTypes only have __init__, but to be future proof, + # we analyze all symbols. + if isinstance(sym.node, (FuncDef, Decorator)): + self.accept(sym.node) + if isinstance(sym.node, Var): + self.analyze(sym.node.type, sym.node) if analyzed.info and analyzed.info.mro: analyzed.info.mro = [] # Force recomputation mypy.semanal.calculate_class_mro(analyzed.info.defn, self.fail_blocker) @@ -253,10 +266,25 @@ def perform_transform(self, node: Union[Node, SymbolTableNode], for n in node.target: if isinstance(n, NameExpr) and isinstance(n.node, Var) and n.node.type: n.node.type = transform(n.node.type) - if isinstance(node, (FuncDef, CastExpr, AssignmentStmt, TypeAliasExpr, Var)): + if isinstance(node, (FuncDef, OverloadedFuncDef, CastExpr, AssignmentStmt, + TypeAliasExpr, Var)): node.type = transform(node.type) if isinstance(node, NewTypeExpr): node.old_type = transform(node.old_type) + if node.info: + new_bases = [] # type: List[Instance] + for b in node.info.bases: + new_b = transform(b) + # TODO: this code can be combined with code in second pass. + if isinstance(new_b, Instance): + new_bases.append(new_b) + elif isinstance(new_b, TupleType): + new_bases.append(new_b.fallback) + else: + self.fail("Argument 2 to NewType(...) must be subclassable" + " (got {})".format(new_b), node) + new_bases.append(self.builtin_type('object')) + node.info.bases = new_bases if isinstance(node, TypeVarExpr): if node.upper_bound: node.upper_bound = transform(node.upper_bound) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 6de9b1e1dd956..5d9ca143ffa3a 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -14,7 +14,7 @@ Type, UnboundType, TypeVarType, TupleType, TypedDictType, UnionType, Instance, AnyType, CallableType, NoneTyp, DeletedType, TypeList, TypeVarDef, TypeVisitor, SyntheticTypeVisitor, StarType, PartialType, EllipsisType, UninhabitedType, TypeType, get_typ_args, set_typ_args, - CallableArgument, get_type_vars, TypeQuery, union_items, TypeOfAny, ForwardRef + CallableArgument, get_type_vars, TypeQuery, union_items, TypeOfAny, ForwardRef, Overloaded ) from mypy.nodes import ( @@ -759,6 +759,10 @@ def visit_callable_type(self, t: CallableType) -> None: for arg_type in t.arg_types: arg_type.accept(self) + def visit_overloaded(self, t: Overloaded) -> None: + for item in t.items(): + item.accept(self) + def visit_tuple_type(self, t: TupleType) -> None: for item in t.items: item.accept(self) diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index f5ea5d24a0716..047e0e13f2943 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -3620,6 +3620,55 @@ def parse_ast(name_dict: NameDict) -> None: [builtins fixtures/isinstancelist.pyi] [out] +[case testNoCrashForwardRefToBrokenDoubleNewType] +from typing import Any, Dict, List, NewType + +Foo = NewType('NotFoo', int) # E: String argument 1 'NotFoo' to NewType(...) does not match variable name 'Foo' +Foos = NewType('Foos', List[Foo]) # type: ignore + +def frob(foos: Dict[Any, Foos]) -> None: + foo = foos.get(1) + dict(foo) +[builtins fixtures/dict.pyi] +[out] + +[case testNoCrashForwardRefToBrokenDoubleNewTypeClass] +from typing import Any, Dict, List, NewType + +Foo = NewType('NotFoo', int) # type: ignore +Foos = NewType('Foos', List[Foo]) # type: ignore + +x: C +class C: + def frob(self, foos: Dict[Any, Foos]) -> None: + foo = foos.get(1) + dict(foo) + +reveal_type(x.frob) # E: Revealed type is 'def (foos: builtins.dict[Any, __main__.Foos])' +[builtins fixtures/dict.pyi] +[out] + +[case testNewTypeFromForwardNamedTuple] +from typing import NewType, NamedTuple, Tuple + +NT = NewType('NT', N) +class N(NamedTuple): + x: int + +x: NT = N(1) # E: Incompatible types in assignment (expression has type "N", variable has type "NT") +x = NT(N(1)) +[out] + +[case testNewTypeFromForwardTypedDict] +from typing import NewType, Tuple +from mypy_extensions import TypedDict + +NT = NewType('NT', N) # E: Argument 2 to NewType(...) must be subclassable (got TypedDict('__main__.N', {'x': builtins.int})) +class N(TypedDict): + x: int +[builtins fixtures/dict.pyi] +[out] + [case testCorrectAttributeInForwardRefToNamedTuple] from typing import NamedTuple proc: Process diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 5c952c86a92d5..c412226d4680c 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -3047,6 +3047,71 @@ class Pair(NamedTuple): Person(name=Pair(first="John", last="Doe")) [out] +[case testNoCrashForwardRefToBrokenDoubleNewTypeIncremental] +from typing import Any, List, NewType + +Foo = NewType('NotFoo', int) # type: ignore +Foos = NewType('Foos', List[Foo]) # type: ignore + +def frob(foos: List[Foos]) -> None: + pass +[builtins fixtures/list.pyi] +[out] + +[case testNoCrashForwardRefOverloadIncremental] +from typing import overload, List + +@overload +def f(x: int) -> int: ... +@overload +def f(x: F) -> F: ... +def f(x): + pass + +F = List[int] +[builtins fixtures/list.pyi] +[out] + +[case testNoCrashForwardRefOverloadIncrementalClass] +from typing import overload, Tuple, NamedTuple + +x: C +class C: + @overload + def f(self, x: str) -> N: pass + @overload + def f(self, x: int) -> int: pass + def f(self, x): + pass + +class N(NamedTuple): + x: A +A = Tuple[int] +[builtins fixtures/tuple.pyi] +[out] + +[case testNewTypeFromForwardNamedTupleIncremental] +from typing import NewType, NamedTuple, Tuple + +NT = NewType('NT', N) +class N(NamedTuple): + x: int + +x: NT = N(1) # type: ignore +x = NT(N(1)) +[out] + +[case testNewTypeFromForwardTypedDictIncremental] +from typing import NewType, Tuple, Dict +from mypy_extensions import TypedDict + +NT = NewType('NT', N) # type: ignore +class N(TypedDict): + x: A +A = Dict[str, int] +[builtins fixtures/dict.pyi] +[out] + -- Some crazy selef-referential named tuples, types dicts, and aliases -- to be sure that everything can be _serialized_ (i.e. ForwardRef's are removed). -- For this reason errors are silenced (tests with # type: ignore have equivalents in other files) From 0b0dc25ca529e1a59c7211e3ab514afa195b22f2 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 25 Oct 2017 09:59:11 +0200 Subject: [PATCH 2/2] Avoid code duplication --- mypy/semanal_pass3.py | 34 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/mypy/semanal_pass3.py b/mypy/semanal_pass3.py index 36e5c247fc375..52c5bde83dbd4 100644 --- a/mypy/semanal_pass3.py +++ b/mypy/semanal_pass3.py @@ -120,12 +120,7 @@ def visit_class_def(self, tdef: ClassDef) -> None: self.analyze(tdef.analyzed.info.typeddict_type, tdef.analyzed, warn=True) elif isinstance(tdef.analyzed, NamedTupleExpr): self.analyze(tdef.analyzed.info.tuple_type, tdef.analyzed, warn=True) - for name in tdef.analyzed.info.names: - sym = tdef.analyzed.info.names[name] - if isinstance(sym.node, (FuncDef, Decorator)): - self.accept(sym.node) - if isinstance(sym.node, Var): - self.analyze(sym.node.type, sym.node) + self.analyze_info(tdef.analyzed.info) super().visit_class_def(tdef) def visit_decorator(self, dec: Decorator) -> None: @@ -194,14 +189,9 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: if isinstance(analyzed, NewTypeExpr): self.analyze(analyzed.old_type, analyzed) if analyzed.info: - for name in analyzed.info.names: - sym = analyzed.info.names[name] - # Currently NewTypes only have __init__, but to be future proof, - # we analyze all symbols. - if isinstance(sym.node, (FuncDef, Decorator)): - self.accept(sym.node) - if isinstance(sym.node, Var): - self.analyze(sym.node.type, sym.node) + # Currently NewTypes only have __init__, but to be future proof, + # we analyze all symbols. + self.analyze_info(analyzed.info) if analyzed.info and analyzed.info.mro: analyzed.info.mro = [] # Force recomputation mypy.semanal.calculate_class_mro(analyzed.info.defn, self.fail_blocker) @@ -216,12 +206,7 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: self.analyze(analyzed.info.typeddict_type, analyzed, warn=True) if isinstance(analyzed, NamedTupleExpr): self.analyze(analyzed.info.tuple_type, analyzed, warn=True) - for name in analyzed.info.names: - sym = analyzed.info.names[name] - if isinstance(sym.node, (FuncDef, Decorator)): - self.accept(sym.node) - if isinstance(sym.node, Var): - self.analyze(sym.node.type, sym.node) + self.analyze_info(analyzed.info) # We need to pay additional attention to assignments that define a type alias. # The resulting type is also stored in the 'type_override' attribute of # the corresponding SymbolTableNode. @@ -367,6 +352,15 @@ def patch() -> None: node, warn=False))) self.patches.append(patch) + def analyze_info(self, info: TypeInfo) -> None: + # Similar to above but for nodes with synthetic TypeInfos (NamedTuple and NewType). + for name in info.names: + sym = info.names[name] + if isinstance(sym.node, (FuncDef, Decorator)): + self.accept(sym.node) + if isinstance(sym.node, Var): + self.analyze(sym.node.type, sym.node) + def make_type_analyzer(self, indicator: Dict[str, bool]) -> TypeAnalyserPass3: return TypeAnalyserPass3(self.sem.lookup_qualified, self.sem.lookup_fully_qualified,