New semantic analyzer: abstract classes

mypy/build.py

@@ -2833,7 +2833,7 @@ def process_stale_scc(graph: Graph, scc: List[str], manager: BuildManager) -> No
         if not manager.options.new_semantic_analyzer:
             manager.semantic_analyzer.add_builtin_aliases(typing_mod)
     if manager.options.new_semantic_analyzer:
-        semantic_analysis_for_scc(graph, scc)
+        semantic_analysis_for_scc(graph, scc, manager.errors)
     else:
         for id in stale:
             graph[id].semantic_analysis()

mypy/newsemanal/semanal.py

@@ -937,7 +937,6 @@ def analyze_class_body_common(self, defn: ClassDef) -> None:
         """Parts of class body analysis that are common to all kinds of class defs."""
         self.enter_class(defn.info)
         defn.defs.accept(self)
-        self.calculate_abstract_status(defn.info)
         self.apply_class_plugin_hooks(defn)
         self.leave_class()
 
@@ -1028,56 +1027,6 @@ def analyze_class_decorator(self, defn: ClassDef, decorator: Expression) -> None
                                         'typing_extensions.final'):
                 defn.info.is_final = True
 
-    def calculate_abstract_status(self, typ: TypeInfo) -> None:
-        """Calculate abstract status of a class.
-
-        Set is_abstract of the type to True if the type has an unimplemented
-        abstract attribute.  Also compute a list of abstract attributes.
-        """
-        concrete = set()  # type: Set[str]
-        abstract = []  # type: List[str]
-        abstract_in_this_class = []  # type: List[str]
-        for base in typ.mro:
-            for name, symnode in base.names.items():
-                node = symnode.node
-                if isinstance(node, OverloadedFuncDef):
-                    # Unwrap an overloaded function definition. We can just
-                    # check arbitrarily the first overload item. If the
-                    # different items have a different abstract status, there
-                    # should be an error reported elsewhere.
-                    func = node.items[0]  # type: Optional[Node]
-                else:
-                    func = node
-                if isinstance(func, Decorator):
-                    fdef = func.func
-                    if fdef.is_abstract and name not in concrete:
-                        typ.is_abstract = True
-                        abstract.append(name)
-                        if base is typ:
-                            abstract_in_this_class.append(name)
-                elif isinstance(node, Var):
-                    if node.is_abstract_var and name not in concrete:
-                        typ.is_abstract = True
-                        abstract.append(name)
-                        if base is typ:
-                            abstract_in_this_class.append(name)
-                concrete.add(name)
-        # In stubs, abstract classes need to be explicitly marked because it is too
-        # easy to accidentally leave a concrete class abstract by forgetting to
-        # implement some methods.
-        typ.abstract_attributes = sorted(abstract)
-        if not self.is_stub_file:
-            return
-        if (typ.declared_metaclass and typ.declared_metaclass.type.fullname() == 'abc.ABCMeta'):
-            return
-        if typ.is_protocol:
-            return
-        if abstract and not abstract_in_this_class:
-            attrs = ", ".join('"{}"'.format(attr) for attr in sorted(abstract))
-            self.fail("Class {} has abstract attributes {}".format(typ.fullname(), attrs), typ)
-            self.note("If it is meant to be abstract, add 'abc.ABCMeta' as an explicit metaclass",
-                      typ)
-
     def setup_type_promotion(self, defn: ClassDef) -> None:
         """Setup extra, ad-hoc subtyping relationships between classes (promotion).
 

mypy/newsemanal/semanal_abstract.py

@@ -0,0 +1,80 @@
+"""Calculate the abstract status of classes.
+
+This happens after semantic analysis and before type checking.
+"""
+
+from typing import List, Set, Optional
+
+from mypy.nodes import Node, MypyFile, SymbolTable, TypeInfo, Var, Decorator, OverloadedFuncDef
+from mypy.errors import Errors
+
+
+def calculate_abstract_status(file: MypyFile, errors: Errors) -> None:
+    """Calculate the abstract status of all classes in the symbol table in file.
+
+    Also check that ABCMeta is used correctly.
+    """
+    process(file.names, file.is_stub, file.fullname(), errors)
+
+
+def process(names: SymbolTable, is_stub_file: bool, prefix: str, errors: Errors) -> None:
+    for name, symnode in names.items():
+        node = symnode.node
+        if isinstance(node, TypeInfo) and node.fullname().startswith(prefix):
+            calculate_class_abstract_status(node, is_stub_file, errors)
+            new_prefix = prefix + '.' + node.name()
+            process(node.names, is_stub_file, new_prefix, errors)
+
+
+def calculate_class_abstract_status(typ: TypeInfo, is_stub_file: bool, errors: Errors) -> None:
+    """Calculate abstract status of a class.
+
+    Set is_abstract of the type to True if the type has an unimplemented
+    abstract attribute.  Also compute a list of abstract attributes.
+    Report error is required ABCMeta metaclass is missing.
+    """
+    concrete = set()  # type: Set[str]
+    abstract = []  # type: List[str]
+    abstract_in_this_class = []  # type: List[str]
+    for base in typ.mro:
+        for name, symnode in base.names.items():
+            node = symnode.node
+            if isinstance(node, OverloadedFuncDef):
+                # Unwrap an overloaded function definition. We can just
+                # check arbitrarily the first overload item. If the
+                # different items have a different abstract status, there
+                # should be an error reported elsewhere.
+                func = node.items[0]  # type: Optional[Node]
+            else:
+                func = node
+            if isinstance(func, Decorator):
+                fdef = func.func
+                if fdef.is_abstract and name not in concrete:
+                    typ.is_abstract = True
+                    abstract.append(name)
+                    if base is typ:
+                        abstract_in_this_class.append(name)
+            elif isinstance(node, Var):
+                if node.is_abstract_var and name not in concrete:
+                    typ.is_abstract = True
+                    abstract.append(name)
+                    if base is typ:
+                        abstract_in_this_class.append(name)
+            concrete.add(name)
+    # In stubs, abstract classes need to be explicitly marked because it is too
+    # easy to accidentally leave a concrete class abstract by forgetting to
+    # implement some methods.
+    typ.abstract_attributes = sorted(abstract)
+    if is_stub_file:
+        if typ.declared_metaclass and typ.declared_metaclass.type.fullname() == 'abc.ABCMeta':
+            return
+        if typ.is_protocol:
+            return
+        if abstract and not abstract_in_this_class:
+            def report(message: str, severity: str) -> None:
+                errors.report(typ.line, typ.column, message, severity=severity)
+
+            attrs = ", ".join('"{}"'.format(attr) for attr in sorted(abstract))
+            report("Class {} has abstract attributes {}".format(typ.fullname(), attrs), 'error')
+            report("If it is meant to be abstract, add 'abc.ABCMeta' as an explicit metaclass",
+                   'note')

mypy/newsemanal/semanal_infer.py

@@ -0,0 +1,114 @@
+"""Simple type inference for decorated functions during semantic analysis."""
+
+from typing import Optional
+
+from mypy.nodes import Expression, Decorator, CallExpr, FuncDef, RefExpr, Var, ARG_POS
+from mypy.types import Type, CallableType, AnyType, TypeOfAny, TypeVarType, function_type
+from mypy.typevars import has_no_typevars
+from mypy.newsemanal.semanal_shared import SemanticAnalyzerInterface
+
+
+def infer_decorator_signature_if_simple(dec: Decorator,
+                                        analyzer: SemanticAnalyzerInterface) -> None:
+    """Try to infer the type of the decorated function.
+
+    This lets us resolve additional references to decorated functions
+    during type checking. Otherwise the type might not be available
+    when we need it, since module top levels can't be deferred.
+
+    This basically uses a simple special-purpose type inference
+    engine just for decorators.
+    """
+    if dec.var.is_property:
+        # Decorators are expected to have a callable type (it's a little odd).
+        if dec.func.type is None:
+            dec.var.type = CallableType(
+                [AnyType(TypeOfAny.special_form)],
+                [ARG_POS],
+                [None],
+                AnyType(TypeOfAny.special_form),
+                analyzer.named_type('__builtins__.function'),
+                name=dec.var.name())
+        elif isinstance(dec.func.type, CallableType):
+            dec.var.type = dec.func.type
+        return
+    decorator_preserves_type = True
+    for expr in dec.decorators:
+        preserve_type = False
+        if isinstance(expr, RefExpr) and isinstance(expr.node, FuncDef):
+            if expr.node.type and is_identity_signature(expr.node.type):
+                preserve_type = True
+        if not preserve_type:
+            decorator_preserves_type = False
+            break
+    if decorator_preserves_type:
+        # No non-identity decorators left. We can trivially infer the type
+        # of the function here.
+        dec.var.type = function_type(dec.func, analyzer.named_type('__builtins__.function'))
+    if dec.decorators:
+        return_type = calculate_return_type(dec.decorators[0])
+        if return_type and isinstance(return_type, AnyType):
+            # The outermost decorator will return Any so we know the type of the
+            # decorated function.
+            dec.var.type = AnyType(TypeOfAny.from_another_any, source_any=return_type)
+        sig = find_fixed_callable_return(dec.decorators[0])
+        if sig:
+            # The outermost decorator always returns the same kind of function,
+            # so we know that this is the type of the decorated function.
+            orig_sig = function_type(dec.func, analyzer.named_type('__builtins__.function'))
+            sig.name = orig_sig.items()[0].name
+            dec.var.type = sig
+
+
+def is_identity_signature(sig: Type) -> bool:
+    """Is type a callable of form T -> T (where T is a type variable)?"""
+    if isinstance(sig, CallableType) and sig.arg_kinds == [ARG_POS]:
+        if isinstance(sig.arg_types[0], TypeVarType) and isinstance(sig.ret_type, TypeVarType):
+            return sig.arg_types[0].id == sig.ret_type.id
+    return False
+
+
+def calculate_return_type(expr: Expression) -> Optional[Type]:
+    """Return the return type if we can calculate it.
+
+    This only uses information available during semantic analysis so this
+    will sometimes return None because of insufficient information (as
+    type inference hasn't run yet).
+    """
+    if isinstance(expr, RefExpr):
+        if isinstance(expr.node, FuncDef):
+            typ = expr.node.type
+            if typ is None:
+                # No signature -> default to Any.
+                return AnyType(TypeOfAny.unannotated)
+            # Explicit Any return?
+            if isinstance(typ, CallableType):
+                return typ.ret_type
+            return None
+        elif isinstance(expr.node, Var):
+            return expr.node.type
+    elif isinstance(expr, CallExpr):
+        return calculate_return_type(expr.callee)
+    return None
+
+
+def find_fixed_callable_return(expr: Expression) -> Optional[CallableType]:
+    """Return the return type, if expression refers to a callable that returns a callable.
+
+    But only do this if the return type has no type variables. Return None otherwise.
+    This approximates things a lot as this is supposed to be called before type checking
+    when full type information is not available yet.
+    """
+    if isinstance(expr, RefExpr):
+        if isinstance(expr.node, FuncDef):
+            typ = expr.node.type
+            if typ:
+                if isinstance(typ, CallableType) and has_no_typevars(typ.ret_type):
+                    if isinstance(typ.ret_type, CallableType):
+                        return typ.ret_type
+    elif isinstance(expr, CallExpr):
+        t = find_fixed_callable_return(expr.callee)
+        if t:
+            if isinstance(t.ret_type, CallableType):
+                return t.ret_type
+    return None

mypy/newsemanal/semanal_main.py

@@ -31,11 +31,14 @@
 )
 from mypy.newsemanal.semanal_typeargs import TypeArgumentAnalyzer
 from mypy.state import strict_optional_set
+from mypy.newsemanal.semanal import NewSemanticAnalyzer
+from mypy.newsemanal.semanal_abstract import calculate_abstract_status
+from mypy.errors import Errors
+from mypy.newsemanal.semanal_infer import infer_decorator_signature_if_simple
 
 MYPY = False
 if MYPY:
     from mypy.build import Graph, State
-    from mypy.newsemanal.semanal import NewSemanticAnalyzer
 
 
 # Perform up to this many semantic analysis iterations until giving up trying to bind all names.
@@ -46,7 +49,7 @@
 core_modules = ['typing', 'builtins', 'abc', 'collections']
 
 
-def semantic_analysis_for_scc(graph: 'Graph', scc: List[str]) -> None:
+def semantic_analysis_for_scc(graph: 'Graph', scc: List[str], errors: Errors) -> None:
     """Perform semantic analysis for all modules in a SCC (import cycle).
 
     Assume that reachability analysis has already been performed.
@@ -56,7 +59,8 @@ def semantic_analysis_for_scc(graph: 'Graph', scc: List[str]) -> None:
     # before functions. This limitation is unlikely to go away soon.
     process_top_levels(graph, scc)
     process_functions(graph, scc)
-    check_type_arguments(graph, scc)
+    check_type_arguments(graph, scc, errors)
+    process_abstract_status(graph, scc, errors)
 
 
 def process_top_levels(graph: 'Graph', scc: List[str]) -> None:
@@ -183,22 +187,31 @@ def semantic_analyze_target(target: str,
                                    fnam=tree.path,
                                    options=state.options,
                                    active_type=active_type):
-            if isinstance(node, Decorator):
+            refresh_node = node
+            if isinstance(refresh_node, Decorator):
                 # Decorator expressions will be processed as part of the module top level.
-                node = node.func
-            analyzer.refresh_partial(node, [], final_iteration)
+                refresh_node = refresh_node.func
+            analyzer.refresh_partial(refresh_node, [], final_iteration)
+            if isinstance(node, Decorator):
+                infer_decorator_signature_if_simple(node, analyzer)
     if analyzer.deferred:
         return [target], analyzer.incomplete
     else:
         return [], analyzer.incomplete
 
 
-def check_type_arguments(graph: 'Graph', scc: List[str]) -> None:
+def check_type_arguments(graph: 'Graph', scc: List[str], errors: Errors) -> None:
     for module in scc:
         state = graph[module]
-        errors = state.manager.errors
         assert state.tree
         analyzer = TypeArgumentAnalyzer(errors)
         with state.wrap_context():
             with strict_optional_set(state.options.strict_optional):
                 state.tree.accept(analyzer)
+
+
+def process_abstract_status(graph: 'Graph', scc: List[str], errors: Errors) -> None:
+    for module in scc:
+        tree = graph[module].tree
+        assert tree
+        calculate_abstract_status(tree, errors)