diff --git a/mypy/build.py b/mypy/build.py index 99effc5e49aa5..f4e5b2b2d746d 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -2833,7 +2833,7 @@ def process_stale_scc(graph: Graph, scc: List[str], manager: BuildManager) -> No if not manager.options.new_semantic_analyzer: manager.semantic_analyzer.add_builtin_aliases(typing_mod) if manager.options.new_semantic_analyzer: - semantic_analysis_for_scc(graph, scc) + semantic_analysis_for_scc(graph, scc, manager.errors) else: for id in stale: graph[id].semantic_analysis() diff --git a/mypy/newsemanal/semanal.py b/mypy/newsemanal/semanal.py index 140b1c669e8f5..b504d96b7919c 100644 --- a/mypy/newsemanal/semanal.py +++ b/mypy/newsemanal/semanal.py @@ -937,7 +937,6 @@ def analyze_class_body_common(self, defn: ClassDef) -> None: """Parts of class body analysis that are common to all kinds of class defs.""" self.enter_class(defn.info) defn.defs.accept(self) - self.calculate_abstract_status(defn.info) self.apply_class_plugin_hooks(defn) self.leave_class() @@ -1028,56 +1027,6 @@ def analyze_class_decorator(self, defn: ClassDef, decorator: Expression) -> None 'typing_extensions.final'): defn.info.is_final = True - def calculate_abstract_status(self, typ: TypeInfo) -> None: - """Calculate abstract status of a class. - - Set is_abstract of the type to True if the type has an unimplemented - abstract attribute. Also compute a list of abstract attributes. - """ - concrete = set() # type: Set[str] - abstract = [] # type: List[str] - abstract_in_this_class = [] # type: List[str] - for base in typ.mro: - for name, symnode in base.names.items(): - node = symnode.node - if isinstance(node, OverloadedFuncDef): - # Unwrap an overloaded function definition. We can just - # check arbitrarily the first overload item. If the - # different items have a different abstract status, there - # should be an error reported elsewhere. - func = node.items[0] # type: Optional[Node] - else: - func = node - if isinstance(func, Decorator): - fdef = func.func - if fdef.is_abstract and name not in concrete: - typ.is_abstract = True - abstract.append(name) - if base is typ: - abstract_in_this_class.append(name) - elif isinstance(node, Var): - if node.is_abstract_var and name not in concrete: - typ.is_abstract = True - abstract.append(name) - if base is typ: - abstract_in_this_class.append(name) - concrete.add(name) - # In stubs, abstract classes need to be explicitly marked because it is too - # easy to accidentally leave a concrete class abstract by forgetting to - # implement some methods. - typ.abstract_attributes = sorted(abstract) - if not self.is_stub_file: - return - if (typ.declared_metaclass and typ.declared_metaclass.type.fullname() == 'abc.ABCMeta'): - return - if typ.is_protocol: - return - if abstract and not abstract_in_this_class: - attrs = ", ".join('"{}"'.format(attr) for attr in sorted(abstract)) - self.fail("Class {} has abstract attributes {}".format(typ.fullname(), attrs), typ) - self.note("If it is meant to be abstract, add 'abc.ABCMeta' as an explicit metaclass", - typ) - def setup_type_promotion(self, defn: ClassDef) -> None: """Setup extra, ad-hoc subtyping relationships between classes (promotion). diff --git a/mypy/newsemanal/semanal_abstract.py b/mypy/newsemanal/semanal_abstract.py new file mode 100644 index 0000000000000..10a2901a1813c --- /dev/null +++ b/mypy/newsemanal/semanal_abstract.py @@ -0,0 +1,80 @@ +"""Calculate the abstract status of classes. + +This happens after semantic analysis and before type checking. +""" + +from typing import List, Set, Optional + +from mypy.nodes import Node, MypyFile, SymbolTable, TypeInfo, Var, Decorator, OverloadedFuncDef +from mypy.errors import Errors + + +def calculate_abstract_status(file: MypyFile, errors: Errors) -> None: + """Calculate the abstract status of all classes in the symbol table in file. + + Also check that ABCMeta is used correctly. + """ + process(file.names, file.is_stub, file.fullname(), errors) + + +def process(names: SymbolTable, is_stub_file: bool, prefix: str, errors: Errors) -> None: + for name, symnode in names.items(): + node = symnode.node + if isinstance(node, TypeInfo) and node.fullname().startswith(prefix): + calculate_class_abstract_status(node, is_stub_file, errors) + new_prefix = prefix + '.' + node.name() + process(node.names, is_stub_file, new_prefix, errors) + + +def calculate_class_abstract_status(typ: TypeInfo, is_stub_file: bool, errors: Errors) -> None: + """Calculate abstract status of a class. + + Set is_abstract of the type to True if the type has an unimplemented + abstract attribute. Also compute a list of abstract attributes. + Report error is required ABCMeta metaclass is missing. + """ + concrete = set() # type: Set[str] + abstract = [] # type: List[str] + abstract_in_this_class = [] # type: List[str] + for base in typ.mro: + for name, symnode in base.names.items(): + node = symnode.node + if isinstance(node, OverloadedFuncDef): + # Unwrap an overloaded function definition. We can just + # check arbitrarily the first overload item. If the + # different items have a different abstract status, there + # should be an error reported elsewhere. + func = node.items[0] # type: Optional[Node] + else: + func = node + if isinstance(func, Decorator): + fdef = func.func + if fdef.is_abstract and name not in concrete: + typ.is_abstract = True + abstract.append(name) + if base is typ: + abstract_in_this_class.append(name) + elif isinstance(node, Var): + if node.is_abstract_var and name not in concrete: + typ.is_abstract = True + abstract.append(name) + if base is typ: + abstract_in_this_class.append(name) + concrete.add(name) + # In stubs, abstract classes need to be explicitly marked because it is too + # easy to accidentally leave a concrete class abstract by forgetting to + # implement some methods. + typ.abstract_attributes = sorted(abstract) + if is_stub_file: + if typ.declared_metaclass and typ.declared_metaclass.type.fullname() == 'abc.ABCMeta': + return + if typ.is_protocol: + return + if abstract and not abstract_in_this_class: + def report(message: str, severity: str) -> None: + errors.report(typ.line, typ.column, message, severity=severity) + + attrs = ", ".join('"{}"'.format(attr) for attr in sorted(abstract)) + report("Class {} has abstract attributes {}".format(typ.fullname(), attrs), 'error') + report("If it is meant to be abstract, add 'abc.ABCMeta' as an explicit metaclass", + 'note') diff --git a/mypy/newsemanal/semanal_infer.py b/mypy/newsemanal/semanal_infer.py new file mode 100644 index 0000000000000..94e2d23390c95 --- /dev/null +++ b/mypy/newsemanal/semanal_infer.py @@ -0,0 +1,114 @@ +"""Simple type inference for decorated functions during semantic analysis.""" + +from typing import Optional + +from mypy.nodes import Expression, Decorator, CallExpr, FuncDef, RefExpr, Var, ARG_POS +from mypy.types import Type, CallableType, AnyType, TypeOfAny, TypeVarType, function_type +from mypy.typevars import has_no_typevars +from mypy.newsemanal.semanal_shared import SemanticAnalyzerInterface + + +def infer_decorator_signature_if_simple(dec: Decorator, + analyzer: SemanticAnalyzerInterface) -> None: + """Try to infer the type of the decorated function. + + This lets us resolve additional references to decorated functions + during type checking. Otherwise the type might not be available + when we need it, since module top levels can't be deferred. + + This basically uses a simple special-purpose type inference + engine just for decorators. + """ + if dec.var.is_property: + # Decorators are expected to have a callable type (it's a little odd). + if dec.func.type is None: + dec.var.type = CallableType( + [AnyType(TypeOfAny.special_form)], + [ARG_POS], + [None], + AnyType(TypeOfAny.special_form), + analyzer.named_type('__builtins__.function'), + name=dec.var.name()) + elif isinstance(dec.func.type, CallableType): + dec.var.type = dec.func.type + return + decorator_preserves_type = True + for expr in dec.decorators: + preserve_type = False + if isinstance(expr, RefExpr) and isinstance(expr.node, FuncDef): + if expr.node.type and is_identity_signature(expr.node.type): + preserve_type = True + if not preserve_type: + decorator_preserves_type = False + break + if decorator_preserves_type: + # No non-identity decorators left. We can trivially infer the type + # of the function here. + dec.var.type = function_type(dec.func, analyzer.named_type('__builtins__.function')) + if dec.decorators: + return_type = calculate_return_type(dec.decorators[0]) + if return_type and isinstance(return_type, AnyType): + # The outermost decorator will return Any so we know the type of the + # decorated function. + dec.var.type = AnyType(TypeOfAny.from_another_any, source_any=return_type) + sig = find_fixed_callable_return(dec.decorators[0]) + if sig: + # The outermost decorator always returns the same kind of function, + # so we know that this is the type of the decorated function. + orig_sig = function_type(dec.func, analyzer.named_type('__builtins__.function')) + sig.name = orig_sig.items()[0].name + dec.var.type = sig + + +def is_identity_signature(sig: Type) -> bool: + """Is type a callable of form T -> T (where T is a type variable)?""" + if isinstance(sig, CallableType) and sig.arg_kinds == [ARG_POS]: + if isinstance(sig.arg_types[0], TypeVarType) and isinstance(sig.ret_type, TypeVarType): + return sig.arg_types[0].id == sig.ret_type.id + return False + + +def calculate_return_type(expr: Expression) -> Optional[Type]: + """Return the return type if we can calculate it. + + This only uses information available during semantic analysis so this + will sometimes return None because of insufficient information (as + type inference hasn't run yet). + """ + if isinstance(expr, RefExpr): + if isinstance(expr.node, FuncDef): + typ = expr.node.type + if typ is None: + # No signature -> default to Any. + return AnyType(TypeOfAny.unannotated) + # Explicit Any return? + if isinstance(typ, CallableType): + return typ.ret_type + return None + elif isinstance(expr.node, Var): + return expr.node.type + elif isinstance(expr, CallExpr): + return calculate_return_type(expr.callee) + return None + + +def find_fixed_callable_return(expr: Expression) -> Optional[CallableType]: + """Return the return type, if expression refers to a callable that returns a callable. + + But only do this if the return type has no type variables. Return None otherwise. + This approximates things a lot as this is supposed to be called before type checking + when full type information is not available yet. + """ + if isinstance(expr, RefExpr): + if isinstance(expr.node, FuncDef): + typ = expr.node.type + if typ: + if isinstance(typ, CallableType) and has_no_typevars(typ.ret_type): + if isinstance(typ.ret_type, CallableType): + return typ.ret_type + elif isinstance(expr, CallExpr): + t = find_fixed_callable_return(expr.callee) + if t: + if isinstance(t.ret_type, CallableType): + return t.ret_type + return None diff --git a/mypy/newsemanal/semanal_main.py b/mypy/newsemanal/semanal_main.py index ab681e4e317c9..ce8586651ce44 100644 --- a/mypy/newsemanal/semanal_main.py +++ b/mypy/newsemanal/semanal_main.py @@ -31,11 +31,14 @@ ) from mypy.newsemanal.semanal_typeargs import TypeArgumentAnalyzer from mypy.state import strict_optional_set +from mypy.newsemanal.semanal import NewSemanticAnalyzer +from mypy.newsemanal.semanal_abstract import calculate_abstract_status +from mypy.errors import Errors +from mypy.newsemanal.semanal_infer import infer_decorator_signature_if_simple MYPY = False if MYPY: from mypy.build import Graph, State - from mypy.newsemanal.semanal import NewSemanticAnalyzer # Perform up to this many semantic analysis iterations until giving up trying to bind all names. @@ -46,7 +49,7 @@ core_modules = ['typing', 'builtins', 'abc', 'collections'] -def semantic_analysis_for_scc(graph: 'Graph', scc: List[str]) -> None: +def semantic_analysis_for_scc(graph: 'Graph', scc: List[str], errors: Errors) -> None: """Perform semantic analysis for all modules in a SCC (import cycle). Assume that reachability analysis has already been performed. @@ -56,7 +59,8 @@ def semantic_analysis_for_scc(graph: 'Graph', scc: List[str]) -> None: # before functions. This limitation is unlikely to go away soon. process_top_levels(graph, scc) process_functions(graph, scc) - check_type_arguments(graph, scc) + check_type_arguments(graph, scc, errors) + process_abstract_status(graph, scc, errors) def process_top_levels(graph: 'Graph', scc: List[str]) -> None: @@ -183,22 +187,31 @@ def semantic_analyze_target(target: str, fnam=tree.path, options=state.options, active_type=active_type): - if isinstance(node, Decorator): + refresh_node = node + if isinstance(refresh_node, Decorator): # Decorator expressions will be processed as part of the module top level. - node = node.func - analyzer.refresh_partial(node, [], final_iteration) + refresh_node = refresh_node.func + analyzer.refresh_partial(refresh_node, [], final_iteration) + if isinstance(node, Decorator): + infer_decorator_signature_if_simple(node, analyzer) if analyzer.deferred: return [target], analyzer.incomplete else: return [], analyzer.incomplete -def check_type_arguments(graph: 'Graph', scc: List[str]) -> None: +def check_type_arguments(graph: 'Graph', scc: List[str], errors: Errors) -> None: for module in scc: state = graph[module] - errors = state.manager.errors assert state.tree analyzer = TypeArgumentAnalyzer(errors) with state.wrap_context(): with strict_optional_set(state.options.strict_optional): state.tree.accept(analyzer) + + +def process_abstract_status(graph: 'Graph', scc: List[str], errors: Errors) -> None: + for module in scc: + tree = graph[module].tree + assert tree + calculate_abstract_status(tree, errors) diff --git a/mypy/newsemanal/semanal_pass3.py b/mypy/newsemanal/semanal_pass3.py index bdb13a0065076..5c47e47ead972 100644 --- a/mypy/newsemanal/semanal_pass3.py +++ b/mypy/newsemanal/semanal_pass3.py @@ -214,46 +214,6 @@ def visit_decorator(self, dec: Decorator) -> None: decorator.accept(self) if self.recurse_into_functions: dec.func.accept(self) - if dec.var.is_property: - # Decorators are expected to have a callable type (it's a little odd). - if dec.func.type is None: - dec.var.type = CallableType( - [AnyType(TypeOfAny.special_form)], - [ARG_POS], - [None], - AnyType(TypeOfAny.special_form), - self.builtin_type('function'), - name=dec.var.name()) - elif isinstance(dec.func.type, CallableType): - dec.var.type = dec.func.type - self.analyze(dec.var.type, dec.var) - return - decorator_preserves_type = True - for expr in dec.decorators: - preserve_type = False - if isinstance(expr, RefExpr) and isinstance(expr.node, FuncDef): - if expr.node.type and is_identity_signature(expr.node.type): - preserve_type = True - if not preserve_type: - decorator_preserves_type = False - break - if decorator_preserves_type: - # No non-identity decorators left. We can trivially infer the type - # of the function here. - dec.var.type = function_type(dec.func, self.builtin_type('function')) - if dec.decorators: - return_type = calculate_return_type(dec.decorators[0]) - if return_type and isinstance(return_type, AnyType): - # The outermost decorator will return Any so we know the type of the - # decorated function. - dec.var.type = AnyType(TypeOfAny.from_another_any, source_any=return_type) - sig = find_fixed_callable_return(dec.decorators[0]) - if sig: - # The outermost decorator always returns the same kind of function, - # so we know that this is the type of the decorated function. - orig_sig = function_type(dec.func, self.builtin_type('function')) - sig.name = orig_sig.items()[0].name - dec.var.type = sig self.analyze(dec.var.type, dec.var) def visit_assignment_stmt(self, s: AssignmentStmt) -> None: @@ -532,54 +492,6 @@ def builtin_type(self, name: str, args: Optional[List[Type]] = None) -> Instance return Instance(node, [any_type] * len(node.defn.type_vars)) -def is_identity_signature(sig: Type) -> bool: - """Is type a callable of form T -> T (where T is a type variable)?""" - if isinstance(sig, CallableType) and sig.arg_kinds == [ARG_POS]: - if isinstance(sig.arg_types[0], TypeVarType) and isinstance(sig.ret_type, TypeVarType): - return sig.arg_types[0].id == sig.ret_type.id - return False - - -def calculate_return_type(expr: Expression) -> Optional[Type]: - """Return the return type if we can calculate it. - - This only uses information available during semantic analysis so this - will sometimes return None because of insufficient information (as - type inference hasn't run yet). - """ - if isinstance(expr, RefExpr): - if isinstance(expr.node, FuncDef): - typ = expr.node.type - if typ is None: - # No signature -> default to Any. - return AnyType(TypeOfAny.unannotated) - # Explicit Any return? - if isinstance(typ, CallableType): - return typ.ret_type - return None - elif isinstance(expr.node, Var): - return expr.node.type - elif isinstance(expr, CallExpr): - return calculate_return_type(expr.callee) - return None - - -def find_fixed_callable_return(expr: Expression) -> Optional[CallableType]: - if isinstance(expr, RefExpr): - if isinstance(expr.node, FuncDef): - typ = expr.node.type - if typ: - if isinstance(typ, CallableType) and has_no_typevars(typ.ret_type): - if isinstance(typ.ret_type, CallableType): - return typ.ret_type - elif isinstance(expr, CallExpr): - t = find_fixed_callable_return(expr.callee) - if t: - if isinstance(t.ret_type, CallableType): - return t.ret_type - return None - - class ForwardReferenceResolver(TypeTranslator): """Visitor to replace previously detected forward reference to synthetic types. diff --git a/mypy/test/hacks.py b/mypy/test/hacks.py index fc556f3639454..74a3456761777 100644 --- a/mypy/test/hacks.py +++ b/mypy/test/hacks.py @@ -7,7 +7,6 @@ # Files to not run with new semantic analyzer. new_semanal_blacklist = [ - 'check-abstract.test', 'check-async-await.test', 'check-attr.test', 'check-bound.test', diff --git a/test-data/unit/check-abstract.test b/test-data/unit/check-abstract.test index 583275dcdf165..a75674cc3d15b 100644 --- a/test-data/unit/check-abstract.test +++ b/test-data/unit/check-abstract.test @@ -870,3 +870,21 @@ class A(metaclass=ABCMeta): @x.setter def x(self, x): pass [out] + + +-- Special cases +-- ------------- + + +[case testNestedAbstractClass] +from abc import abstractmethod, ABCMeta + +class A: + class B(metaclass=ABCMeta): + @abstractmethod + def f(self) -> None: pass + + class C(B): pass + +A.B() # E: Cannot instantiate abstract class 'B' with abstract attribute 'f' +A.C() # E: Cannot instantiate abstract class 'C' with abstract attribute 'f' diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index d2b400dc1c8e3..5f79886ece764 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -1367,8 +1367,6 @@ reveal_type(x['test'][0]) [out] -- N.B: These errors are *wrong* and the new semantic analyzer can't yet -- fully analyze typeshed -typing.pyi:251: error: Class typing.Sequence has abstract attributes "__len__" -typing.pyi:251: note: If it is meant to be abstract, add 'abc.ABCMeta' as an explicit metaclass ast.pyi:31: error: Name 'PyCF_ONLY_AST' already defined (possibly by an import) codecs.pyi:49: error: Cannot overwrite NamedTuple attribute "__init__" _testNewAnalyzerBasicTypeshed_newsemanal.py:4: error: Revealed type is 'builtins.int*'