From ecca142f86e00e8a4af4c1484debd2b13c2e3473 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Wed, 6 Jul 2016 03:32:07 +0300 Subject: [PATCH 01/74] namedtuple: add (shaky) support for _replace, _asdict + naive test --- mypy/semanal.py | 38 +++++++++++--- mypy/test/testcheck.py | 74 ++++++++++++++-------------- test-data/unit/check-namedtuple.test | 15 ++++++ 3 files changed, 83 insertions(+), 44 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index c0aea9585c2bb..962c37ad44abb 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -59,7 +59,7 @@ SymbolTableNode, BOUND_TVAR, UNBOUND_TVAR, ListComprehension, GeneratorExpr, FuncExpr, MDEF, FuncBase, Decorator, SetExpr, TypeVarExpr, StrExpr, BytesExpr, PrintStmt, ConditionalExpr, PromoteExpr, - ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, MroError, type_aliases, + ComparisonExpr, StarExpr, ARG_POS, ARG_OPT, ARG_STAR, ARG_NAMED, MroError, type_aliases, YieldFromExpr, NamedTupleExpr, NonlocalDecl, SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr, YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, COVARIANT, CONTRAVARIANT, @@ -1534,9 +1534,13 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], var.info = info var.type = typ symbols[item] = SymbolTableNode(MDEF, var) - # Add a __init__ method. - init = self.make_namedtuple_init(info, items, types) + # Add a __init__, _replace, asdict methods. + init = self.make_namedtuple_create('__init__', NoneTyp(), info, items, types, ARG_POS) symbols['__init__'] = SymbolTableNode(MDEF, init) + replace = self.make_namedtuple_create('_replace', AnyType(), info, items, types, ARG_NAMED, EllipsisType) + symbols['_replace'] = SymbolTableNode(MDEF, replace) + asdict = self.make_namedtuple_asdict(info) + symbols['_asdict'] = SymbolTableNode(MDEF, asdict) info.tuple_type = TupleType(types, self.named_type('__builtins__.tuple', [AnyType()])) info.is_named_tuple = True info.mro = [info] + info.tuple_type.fallback.type.mro @@ -1546,19 +1550,39 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], def make_argument(self, name: str, type: Type) -> Argument: return Argument(Var(name), type, None, ARG_POS) - def make_namedtuple_init(self, info: TypeInfo, items: List[str], - types: List[Type]) -> FuncDef: + def make_namedtuple_create(self, funcname, ret: Type, info: TypeInfo, items: List[str], + types: List[Type], kind, initializer=None) -> FuncDef: args = [self.make_argument(item, type) for item, type in zip(items, types)] + for arg in args: + if initializer is not None: + arg.initializer = initializer + arg.kind = kind # TODO: Make sure that the self argument name is not visible? args = [Argument(Var('__self'), NoneTyp(), None, ARG_POS)] + args arg_kinds = [arg.kind for arg in args] signature = CallableType([cast(Type, None)] + types, arg_kinds, ['__self'] + items, - NoneTyp(), + ret, self.named_type('__builtins__.function'), name=info.name()) - func = FuncDef('__init__', + func = FuncDef(funcname, + args, + Block([]), + typ=signature) + func.info = info + return func + + def make_namedtuple_asdict(self, info: TypeInfo) -> FuncDef: + args = [Argument(Var('__self'), NoneTyp(), None, ARG_POS)] + arg_kinds = [arg.kind for arg in args] + signature = CallableType([cast(Type, None)], + arg_kinds, + ['__self'], + AnyType(), + self.named_type('__builtins__.function'), + name=info.name()) + func = FuncDef('_asdict', args, Block([]), typ=signature) diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 0c9ed87cdc185..7b41c0969308b 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -24,44 +24,44 @@ # List of files that contain test case descriptions. files = [ - 'check-basic.test', - 'check-classes.test', - 'check-expressions.test', - 'check-statements.test', - 'check-generics.test', - 'check-tuples.test', - 'check-dynamic-typing.test', - 'check-weak-typing.test', - 'check-functions.test', - 'check-inference.test', - 'check-inference-context.test', - 'check-varargs.test', - 'check-kwargs.test', - 'check-overloading.test', - 'check-type-checks.test', - 'check-abstract.test', - 'check-multiple-inheritance.test', - 'check-super.test', - 'check-modules.test', - 'check-generic-subtyping.test', - 'check-typevar-values.test', - 'check-python2.test', - 'check-unsupported.test', - 'check-unreachable-code.test', - 'check-unions.test', - 'check-isinstance.test', - 'check-lists.test', + # 'check-basic.test', + # 'check-classes.test', + # 'check-expressions.test', + # 'check-statements.test', + # 'check-generics.test', + # 'check-tuples.test', + # 'check-dynamic-typing.test', + # 'check-weak-typing.test', + # 'check-functions.test', + # 'check-inference.test', + # 'check-inference-context.test', + # 'check-varargs.test', + # 'check-kwargs.test', + # 'check-overloading.test', + # 'check-type-checks.test', + # 'check-abstract.test', + # 'check-multiple-inheritance.test', + # 'check-super.test', + # 'check-modules.test', + # 'check-generic-subtyping.test', + # 'check-typevar-values.test', + # 'check-python2.test', + # 'check-unsupported.test', + # 'check-unreachable-code.test', + # 'check-unions.test', + # 'check-isinstance.test', + # 'check-lists.test', 'check-namedtuple.test', - 'check-type-aliases.test', - 'check-ignore.test', - 'check-type-promotion.test', - 'check-semanal-error.test', - 'check-flags.test', - 'check-incremental.test', - 'check-bound.test', - 'check-optional.test', - 'check-fastparse.test', - 'check-warnings.test', + # 'check-type-aliases.test', + # 'check-ignore.test', + # 'check-type-promotion.test', + # 'check-semanal-error.test', + # 'check-flags.test', + # 'check-incremental.test', + # 'check-bound.test', + # 'check-optional.test', + # 'check-fastparse.test', + # 'check-warnings.test', ] diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index d51f078c8fd47..ed635b973bd3e 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -185,3 +185,18 @@ class C(B): pass B(1).b C(2).b [builtins fixtures/property.py] + +[case testNamedTupleAsDict] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +a = x._asdict() + +[case testNamedTupleAsDict] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +a = x._replace(y=5) + From 8ef445f4a2106762c8a042955be2e6f7e239661a Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Wed, 6 Jul 2016 11:14:19 +0300 Subject: [PATCH 02/74] namedtuple: add tests and refine types --- mypy/semanal.py | 44 +++++++++++----------------- test-data/unit/check-namedtuple.test | 21 +++++++++++-- 2 files changed, 36 insertions(+), 29 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 962c37ad44abb..13dfd21571f4f 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1534,13 +1534,20 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], var.info = info var.type = typ symbols[item] = SymbolTableNode(MDEF, var) - # Add a __init__, _replace, asdict methods. - init = self.make_namedtuple_create('__init__', NoneTyp(), info, items, types, ARG_POS) + # Add a __init__, _replace, _asdict methods. + init = self.make_namedtuple_create('__init__', info, items, types, + NoneTyp(), ARG_POS) symbols['__init__'] = SymbolTableNode(MDEF, init) - replace = self.make_namedtuple_create('_replace', AnyType(), info, items, types, ARG_NAMED, EllipsisType) + replace = self.make_namedtuple_create('_replace', info, items, types, + AnyType(), ARG_NAMED, initializer=EllipsisType) symbols['_replace'] = SymbolTableNode(MDEF, replace) - asdict = self.make_namedtuple_asdict(info) + asdict = self.make_namedtuple_method(info, '_asdict', AnyType()) symbols['_asdict'] = SymbolTableNode(MDEF, asdict) + fields_type = TupleType([self.builtin_type('builtins.str') for _ in items], + self.builtin_type('builtins.tuple')) + fields = self.make_namedtuple_method(info, '_fields', fields_type) + fields.is_static = True + symbols['_fields'] = SymbolTableNode(MDEF, fields) info.tuple_type = TupleType(types, self.named_type('__builtins__.tuple', [AnyType()])) info.is_named_tuple = True info.mro = [info] + info.tuple_type.fallback.type.mro @@ -1550,13 +1557,12 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], def make_argument(self, name: str, type: Type) -> Argument: return Argument(Var(name), type, None, ARG_POS) - def make_namedtuple_create(self, funcname, ret: Type, info: TypeInfo, items: List[str], - types: List[Type], kind, initializer=None) -> FuncDef: - args = [self.make_argument(item, type) for item, type in zip(items, types)] - for arg in args: - if initializer is not None: - arg.initializer = initializer - arg.kind = kind + def make_namedtuple_create(self, funcname, info: TypeInfo, items: List[str], types: List[Type], + ret: Type, kind, initializer=None) -> FuncDef: + args = [Argument(Var(item), type, initializer, kind) for item, type in zip(items, types)] + return self.make_namedtuple_method(info, funcname, ret, args, types, items) + + def make_namedtuple_method(self, info: TypeInfo, funcname: str, ret, args=[], types=[], items=[]) -> FuncDef: # TODO: Make sure that the self argument name is not visible? args = [Argument(Var('__self'), NoneTyp(), None, ARG_POS)] + args arg_kinds = [arg.kind for arg in args] @@ -1572,22 +1578,6 @@ def make_namedtuple_create(self, funcname, ret: Type, info: TypeInfo, items: Lis typ=signature) func.info = info return func - - def make_namedtuple_asdict(self, info: TypeInfo) -> FuncDef: - args = [Argument(Var('__self'), NoneTyp(), None, ARG_POS)] - arg_kinds = [arg.kind for arg in args] - signature = CallableType([cast(Type, None)], - arg_kinds, - ['__self'], - AnyType(), - self.named_type('__builtins__.function'), - name=info.name()) - func = FuncDef('_asdict', - args, - Block([]), - typ=signature) - func.info = info - return func def analyze_types(self, items: List[Node]) -> List[Type]: result = [] # type: List[Type] diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index ed635b973bd3e..efd6ed94f58e5 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -193,10 +193,27 @@ X = namedtuple('X', ['x', 'y']) x = None # type: X a = x._asdict() -[case testNamedTupleAsDict] +[case testNamedTupleReplace] from collections import namedtuple X = namedtuple('X', ['x', 'y']) x = None # type: X -a = x._replace(y=5) +x = x._replace(y=5) + +[case testNamedTupleReplaceTyped] +from typing import NamedTuple + +X = NamedTuple('X', [('x', int), ('y', str)]) +x = None # type: X +x = x._replace(x=5) +x = x._replace(y=5) # E: Argument 1 to X has incompatible type "int"; expected "str" +[case testNamedTupleFields] +from typing import Tuple +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +a = None # type: Tuple[str, str] +a = x._fields() +[builtins fixtures/tuple.py] From 5e429cda69530a33c6e8c1529f83577799472dc2 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Wed, 6 Jul 2016 12:38:34 +0300 Subject: [PATCH 03/74] minor cleanup --- mypy/semanal.py | 40 +++++++++++------------ mypy/test/testcheck.py | 74 +++++++++++++++++++++--------------------- 2 files changed, 56 insertions(+), 58 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 13dfd21571f4f..20f7b876a6a33 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -64,7 +64,7 @@ SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr, YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, COVARIANT, CONTRAVARIANT, IntExpr, FloatExpr, UnicodeExpr, - INVARIANT, UNBOUND_IMPORTED + INVARIANT, UNBOUND_IMPORTED, Expression ) from mypy.visitor import NodeVisitor from mypy.traverser import TraverserVisitor @@ -1534,18 +1534,20 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], var.info = info var.type = typ symbols[item] = SymbolTableNode(MDEF, var) - # Add a __init__, _replace, _asdict methods. - init = self.make_namedtuple_create('__init__', info, items, types, - NoneTyp(), ARG_POS) + # Add a __init__, _replace, _asdict methods and _fields static field + init = self.make_namedtuple_method('__init__', info, NoneTyp(), + self.make_factory_args(items, types, ARG_POS)) symbols['__init__'] = SymbolTableNode(MDEF, init) - replace = self.make_namedtuple_create('_replace', info, items, types, - AnyType(), ARG_NAMED, initializer=EllipsisType) + #TODO: refine return type to type(self) + replace = self.make_namedtuple_method('_replace', info, AnyType(), + self.make_factory_args(items, types, ARG_NAMED, initializer=EllipsisType)) symbols['_replace'] = SymbolTableNode(MDEF, replace) - asdict = self.make_namedtuple_method(info, '_asdict', AnyType()) + #TODO: refine to OrderedDict[Union[types]] + asdict = self.make_namedtuple_method('_asdict', info, AnyType()) symbols['_asdict'] = SymbolTableNode(MDEF, asdict) fields_type = TupleType([self.builtin_type('builtins.str') for _ in items], self.builtin_type('builtins.tuple')) - fields = self.make_namedtuple_method(info, '_fields', fields_type) + fields = self.make_namedtuple_method('_fields', info, fields_type) fields.is_static = True symbols['_fields'] = SymbolTableNode(MDEF, fields) info.tuple_type = TupleType(types, self.named_type('__builtins__.tuple', [AnyType()])) @@ -1554,22 +1556,18 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], info.bases = [info.tuple_type.fallback] return info - def make_argument(self, name: str, type: Type) -> Argument: - return Argument(Var(name), type, None, ARG_POS) - - def make_namedtuple_create(self, funcname, info: TypeInfo, items: List[str], types: List[Type], - ret: Type, kind, initializer=None) -> FuncDef: - args = [Argument(Var(item), type, initializer, kind) for item, type in zip(items, types)] - return self.make_namedtuple_method(info, funcname, ret, args, types, items) + def make_factory_args(self, items: List[str], types: List[Type], + kind: int, initializer: Expression = None) -> FuncDef: + return [Argument(Var(item), type, initializer, kind) for item, type in zip(items, types)] - def make_namedtuple_method(self, info: TypeInfo, funcname: str, ret, args=[], types=[], items=[]) -> FuncDef: + def make_namedtuple_method(self, funcname: str, info: TypeInfo, + ret: Type, args: List[Argument] = []) -> FuncDef: + types = [cast(Type, None)] + [arg.type_annotation for arg in args] + items = ['__self'] + [arg.variable.name() for arg in args] + arg_kinds = [ARG_POS] + [arg.kind for arg in args] # TODO: Make sure that the self argument name is not visible? args = [Argument(Var('__self'), NoneTyp(), None, ARG_POS)] + args - arg_kinds = [arg.kind for arg in args] - signature = CallableType([cast(Type, None)] + types, - arg_kinds, - ['__self'] + items, - ret, + signature = CallableType(types, arg_kinds, items, ret, self.named_type('__builtins__.function'), name=info.name()) func = FuncDef(funcname, diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 7b41c0969308b..0c9ed87cdc185 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -24,44 +24,44 @@ # List of files that contain test case descriptions. files = [ - # 'check-basic.test', - # 'check-classes.test', - # 'check-expressions.test', - # 'check-statements.test', - # 'check-generics.test', - # 'check-tuples.test', - # 'check-dynamic-typing.test', - # 'check-weak-typing.test', - # 'check-functions.test', - # 'check-inference.test', - # 'check-inference-context.test', - # 'check-varargs.test', - # 'check-kwargs.test', - # 'check-overloading.test', - # 'check-type-checks.test', - # 'check-abstract.test', - # 'check-multiple-inheritance.test', - # 'check-super.test', - # 'check-modules.test', - # 'check-generic-subtyping.test', - # 'check-typevar-values.test', - # 'check-python2.test', - # 'check-unsupported.test', - # 'check-unreachable-code.test', - # 'check-unions.test', - # 'check-isinstance.test', - # 'check-lists.test', + 'check-basic.test', + 'check-classes.test', + 'check-expressions.test', + 'check-statements.test', + 'check-generics.test', + 'check-tuples.test', + 'check-dynamic-typing.test', + 'check-weak-typing.test', + 'check-functions.test', + 'check-inference.test', + 'check-inference-context.test', + 'check-varargs.test', + 'check-kwargs.test', + 'check-overloading.test', + 'check-type-checks.test', + 'check-abstract.test', + 'check-multiple-inheritance.test', + 'check-super.test', + 'check-modules.test', + 'check-generic-subtyping.test', + 'check-typevar-values.test', + 'check-python2.test', + 'check-unsupported.test', + 'check-unreachable-code.test', + 'check-unions.test', + 'check-isinstance.test', + 'check-lists.test', 'check-namedtuple.test', - # 'check-type-aliases.test', - # 'check-ignore.test', - # 'check-type-promotion.test', - # 'check-semanal-error.test', - # 'check-flags.test', - # 'check-incremental.test', - # 'check-bound.test', - # 'check-optional.test', - # 'check-fastparse.test', - # 'check-warnings.test', + 'check-type-aliases.test', + 'check-ignore.test', + 'check-type-promotion.test', + 'check-semanal-error.test', + 'check-flags.test', + 'check-incremental.test', + 'check-bound.test', + 'check-optional.test', + 'check-fastparse.test', + 'check-warnings.test', ] From 0214e3cd7db62e4c3a125fc3930a28829b4a57dc Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Wed, 6 Jul 2016 12:38:34 +0300 Subject: [PATCH 04/74] minor cleanup --- mypy/semanal.py | 40 +++++++++++------------ mypy/test/testcheck.py | 74 +++++++++++++++++++++--------------------- 2 files changed, 56 insertions(+), 58 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 13dfd21571f4f..d65cf37cd69ee 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -64,7 +64,7 @@ SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr, YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, COVARIANT, CONTRAVARIANT, IntExpr, FloatExpr, UnicodeExpr, - INVARIANT, UNBOUND_IMPORTED + INVARIANT, UNBOUND_IMPORTED, Expression ) from mypy.visitor import NodeVisitor from mypy.traverser import TraverserVisitor @@ -1534,18 +1534,20 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], var.info = info var.type = typ symbols[item] = SymbolTableNode(MDEF, var) - # Add a __init__, _replace, _asdict methods. - init = self.make_namedtuple_create('__init__', info, items, types, - NoneTyp(), ARG_POS) + # Add a __init__, _replace, _asdict methods and _fields static field + init = self.make_namedtuple_method('__init__', info, NoneTyp(), + self.make_factory_args(items, types, ARG_POS)) symbols['__init__'] = SymbolTableNode(MDEF, init) - replace = self.make_namedtuple_create('_replace', info, items, types, - AnyType(), ARG_NAMED, initializer=EllipsisType) + # TODO: refine return type to type(self) + replace = self.make_namedtuple_method('_replace', info, AnyType(), + self.make_factory_args(items, types, ARG_NAMED, initializer=EllipsisType)) symbols['_replace'] = SymbolTableNode(MDEF, replace) - asdict = self.make_namedtuple_method(info, '_asdict', AnyType()) + # TODO: refine to OrderedDict[Union[types]] + asdict = self.make_namedtuple_method('_asdict', info, AnyType()) symbols['_asdict'] = SymbolTableNode(MDEF, asdict) fields_type = TupleType([self.builtin_type('builtins.str') for _ in items], self.builtin_type('builtins.tuple')) - fields = self.make_namedtuple_method(info, '_fields', fields_type) + fields = self.make_namedtuple_method('_fields', info, fields_type) fields.is_static = True symbols['_fields'] = SymbolTableNode(MDEF, fields) info.tuple_type = TupleType(types, self.named_type('__builtins__.tuple', [AnyType()])) @@ -1554,22 +1556,18 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], info.bases = [info.tuple_type.fallback] return info - def make_argument(self, name: str, type: Type) -> Argument: - return Argument(Var(name), type, None, ARG_POS) + def make_factory_args(self, items: List[str], types: List[Type], + kind: int, initializer: Expression = None) -> FuncDef: + return [Argument(Var(item), type, initializer, kind) for item, type in zip(items, types)] - def make_namedtuple_create(self, funcname, info: TypeInfo, items: List[str], types: List[Type], - ret: Type, kind, initializer=None) -> FuncDef: - args = [Argument(Var(item), type, initializer, kind) for item, type in zip(items, types)] - return self.make_namedtuple_method(info, funcname, ret, args, types, items) - - def make_namedtuple_method(self, info: TypeInfo, funcname: str, ret, args=[], types=[], items=[]) -> FuncDef: + def make_namedtuple_method(self, funcname: str, info: TypeInfo, + ret: Type, args: List[Argument] = []) -> FuncDef: + types = [cast(Type, None)] + [arg.type_annotation for arg in args] + items = ['__self'] + [arg.variable.name() for arg in args] + arg_kinds = [ARG_POS] + [arg.kind for arg in args] # TODO: Make sure that the self argument name is not visible? args = [Argument(Var('__self'), NoneTyp(), None, ARG_POS)] + args - arg_kinds = [arg.kind for arg in args] - signature = CallableType([cast(Type, None)] + types, - arg_kinds, - ['__self'] + items, - ret, + signature = CallableType(types, arg_kinds, items, ret, self.named_type('__builtins__.function'), name=info.name()) func = FuncDef(funcname, diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 7b41c0969308b..0c9ed87cdc185 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -24,44 +24,44 @@ # List of files that contain test case descriptions. files = [ - # 'check-basic.test', - # 'check-classes.test', - # 'check-expressions.test', - # 'check-statements.test', - # 'check-generics.test', - # 'check-tuples.test', - # 'check-dynamic-typing.test', - # 'check-weak-typing.test', - # 'check-functions.test', - # 'check-inference.test', - # 'check-inference-context.test', - # 'check-varargs.test', - # 'check-kwargs.test', - # 'check-overloading.test', - # 'check-type-checks.test', - # 'check-abstract.test', - # 'check-multiple-inheritance.test', - # 'check-super.test', - # 'check-modules.test', - # 'check-generic-subtyping.test', - # 'check-typevar-values.test', - # 'check-python2.test', - # 'check-unsupported.test', - # 'check-unreachable-code.test', - # 'check-unions.test', - # 'check-isinstance.test', - # 'check-lists.test', + 'check-basic.test', + 'check-classes.test', + 'check-expressions.test', + 'check-statements.test', + 'check-generics.test', + 'check-tuples.test', + 'check-dynamic-typing.test', + 'check-weak-typing.test', + 'check-functions.test', + 'check-inference.test', + 'check-inference-context.test', + 'check-varargs.test', + 'check-kwargs.test', + 'check-overloading.test', + 'check-type-checks.test', + 'check-abstract.test', + 'check-multiple-inheritance.test', + 'check-super.test', + 'check-modules.test', + 'check-generic-subtyping.test', + 'check-typevar-values.test', + 'check-python2.test', + 'check-unsupported.test', + 'check-unreachable-code.test', + 'check-unions.test', + 'check-isinstance.test', + 'check-lists.test', 'check-namedtuple.test', - # 'check-type-aliases.test', - # 'check-ignore.test', - # 'check-type-promotion.test', - # 'check-semanal-error.test', - # 'check-flags.test', - # 'check-incremental.test', - # 'check-bound.test', - # 'check-optional.test', - # 'check-fastparse.test', - # 'check-warnings.test', + 'check-type-aliases.test', + 'check-ignore.test', + 'check-type-promotion.test', + 'check-semanal-error.test', + 'check-flags.test', + 'check-incremental.test', + 'check-bound.test', + 'check-optional.test', + 'check-fastparse.test', + 'check-warnings.test', ] From fbb75c3b6820201d02e96d5074dcaf4fcc88fb6d Mon Sep 17 00:00:00 2001 From: Elazar Date: Wed, 6 Jul 2016 13:44:05 +0300 Subject: [PATCH 05/74] flake8-ing --- mypy/semanal.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 20f7b876a6a33..d65cf37cd69ee 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1538,11 +1538,11 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], init = self.make_namedtuple_method('__init__', info, NoneTyp(), self.make_factory_args(items, types, ARG_POS)) symbols['__init__'] = SymbolTableNode(MDEF, init) - #TODO: refine return type to type(self) + # TODO: refine return type to type(self) replace = self.make_namedtuple_method('_replace', info, AnyType(), self.make_factory_args(items, types, ARG_NAMED, initializer=EllipsisType)) symbols['_replace'] = SymbolTableNode(MDEF, replace) - #TODO: refine to OrderedDict[Union[types]] + # TODO: refine to OrderedDict[Union[types]] asdict = self.make_namedtuple_method('_asdict', info, AnyType()) symbols['_asdict'] = SymbolTableNode(MDEF, asdict) fields_type = TupleType([self.builtin_type('builtins.str') for _ in items], @@ -1557,9 +1557,9 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], return info def make_factory_args(self, items: List[str], types: List[Type], - kind: int, initializer: Expression = None) -> FuncDef: + kind: int, initializer: Expression = None) -> FuncDef: return [Argument(Var(item), type, initializer, kind) for item, type in zip(items, types)] - + def make_namedtuple_method(self, funcname: str, info: TypeInfo, ret: Type, args: List[Argument] = []) -> FuncDef: types = [cast(Type, None)] + [arg.type_annotation for arg in args] From 17d7f8bff72a5c4aa699098bc87ae0b370e2e23b Mon Sep 17 00:00:00 2001 From: Elazar Date: Wed, 6 Jul 2016 13:50:11 +0300 Subject: [PATCH 06/74] fix type errors --- mypy/semanal.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index d65cf37cd69ee..2fe405e05c978 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -64,7 +64,7 @@ SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr, YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, COVARIANT, CONTRAVARIANT, IntExpr, FloatExpr, UnicodeExpr, - INVARIANT, UNBOUND_IMPORTED, Expression + INVARIANT, UNBOUND_IMPORTED, Expression, EllipsisExpr ) from mypy.visitor import NodeVisitor from mypy.traverser import TraverserVisitor @@ -1540,7 +1540,7 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], symbols['__init__'] = SymbolTableNode(MDEF, init) # TODO: refine return type to type(self) replace = self.make_namedtuple_method('_replace', info, AnyType(), - self.make_factory_args(items, types, ARG_NAMED, initializer=EllipsisType)) + self.make_factory_args(items, types, ARG_NAMED, initializer=EllipsisExpr())) symbols['_replace'] = SymbolTableNode(MDEF, replace) # TODO: refine to OrderedDict[Union[types]] asdict = self.make_namedtuple_method('_asdict', info, AnyType()) @@ -1557,7 +1557,7 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], return info def make_factory_args(self, items: List[str], types: List[Type], - kind: int, initializer: Expression = None) -> FuncDef: + kind: int, initializer: Expression = None) -> List[Argument]: return [Argument(Var(item), type, initializer, kind) for item, type in zip(items, types)] def make_namedtuple_method(self, funcname: str, info: TypeInfo, From 6dd370ce864a28ac46cfa669ca4bcf0e7e400485 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Wed, 6 Jul 2016 13:51:35 +0300 Subject: [PATCH 07/74] fix type errors --- mypy/semanal.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index d65cf37cd69ee..2fe405e05c978 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -64,7 +64,7 @@ SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr, YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, COVARIANT, CONTRAVARIANT, IntExpr, FloatExpr, UnicodeExpr, - INVARIANT, UNBOUND_IMPORTED, Expression + INVARIANT, UNBOUND_IMPORTED, Expression, EllipsisExpr ) from mypy.visitor import NodeVisitor from mypy.traverser import TraverserVisitor @@ -1540,7 +1540,7 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], symbols['__init__'] = SymbolTableNode(MDEF, init) # TODO: refine return type to type(self) replace = self.make_namedtuple_method('_replace', info, AnyType(), - self.make_factory_args(items, types, ARG_NAMED, initializer=EllipsisType)) + self.make_factory_args(items, types, ARG_NAMED, initializer=EllipsisExpr())) symbols['_replace'] = SymbolTableNode(MDEF, replace) # TODO: refine to OrderedDict[Union[types]] asdict = self.make_namedtuple_method('_asdict', info, AnyType()) @@ -1557,7 +1557,7 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], return info def make_factory_args(self, items: List[str], types: List[Type], - kind: int, initializer: Expression = None) -> FuncDef: + kind: int, initializer: Expression = None) -> List[Argument]: return [Argument(Var(item), type, initializer, kind) for item, type in zip(items, types)] def make_namedtuple_method(self, funcname: str, info: TypeInfo, From 560eb3ea01806d1fc9f6c02ebaaa6b0125813c10 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Wed, 6 Jul 2016 18:25:03 +0300 Subject: [PATCH 08/74] _fields will not be supported --- mypy/semanal.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 2fe405e05c978..a4c7ff0429b85 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1545,11 +1545,6 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], # TODO: refine to OrderedDict[Union[types]] asdict = self.make_namedtuple_method('_asdict', info, AnyType()) symbols['_asdict'] = SymbolTableNode(MDEF, asdict) - fields_type = TupleType([self.builtin_type('builtins.str') for _ in items], - self.builtin_type('builtins.tuple')) - fields = self.make_namedtuple_method('_fields', info, fields_type) - fields.is_static = True - symbols['_fields'] = SymbolTableNode(MDEF, fields) info.tuple_type = TupleType(types, self.named_type('__builtins__.tuple', [AnyType()])) info.is_named_tuple = True info.mro = [info] + info.tuple_type.fallback.type.mro From 2627511de994a477eb2d2ea9aacdeac08dc668fe Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Wed, 6 Jul 2016 20:15:33 +0300 Subject: [PATCH 09/74] slight refactoring --- mypy/semanal.py | 26 +++-- test-data/unit/check-namedtuple.test | 152 --------------------------- 2 files changed, 12 insertions(+), 166 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index a4c7ff0429b85..0e7013ce7f149 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -59,7 +59,7 @@ SymbolTableNode, BOUND_TVAR, UNBOUND_TVAR, ListComprehension, GeneratorExpr, FuncExpr, MDEF, FuncBase, Decorator, SetExpr, TypeVarExpr, StrExpr, BytesExpr, PrintStmt, ConditionalExpr, PromoteExpr, - ComparisonExpr, StarExpr, ARG_POS, ARG_OPT, ARG_STAR, ARG_NAMED, MroError, type_aliases, + ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, MroError, type_aliases, YieldFromExpr, NamedTupleExpr, NonlocalDecl, SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr, YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, COVARIANT, CONTRAVARIANT, @@ -1534,17 +1534,15 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], var.info = info var.type = typ symbols[item] = SymbolTableNode(MDEF, var) - # Add a __init__, _replace, _asdict methods and _fields static field - init = self.make_namedtuple_method('__init__', info, NoneTyp(), + # Add __init__, _replace, _asdict methods and _fields static field + self.add_namedtuple_method('__init__', symbols, info, NoneTyp(), self.make_factory_args(items, types, ARG_POS)) - symbols['__init__'] = SymbolTableNode(MDEF, init) # TODO: refine return type to type(self) - replace = self.make_namedtuple_method('_replace', info, AnyType(), + self.add_namedtuple_method('_replace', symbols, info, AnyType(), self.make_factory_args(items, types, ARG_NAMED, initializer=EllipsisExpr())) - symbols['_replace'] = SymbolTableNode(MDEF, replace) # TODO: refine to OrderedDict[Union[types]] - asdict = self.make_namedtuple_method('_asdict', info, AnyType()) - symbols['_asdict'] = SymbolTableNode(MDEF, asdict) + self.add_namedtuple_method('_asdict', symbols, info, AnyType()) + info.tuple_type = TupleType(types, self.named_type('__builtins__.tuple', [AnyType()])) info.is_named_tuple = True info.mro = [info] + info.tuple_type.fallback.type.mro @@ -1555,13 +1553,13 @@ def make_factory_args(self, items: List[str], types: List[Type], kind: int, initializer: Expression = None) -> List[Argument]: return [Argument(Var(item), type, initializer, kind) for item, type in zip(items, types)] - def make_namedtuple_method(self, funcname: str, info: TypeInfo, - ret: Type, args: List[Argument] = []) -> FuncDef: + def add_namedtuple_method(self, funcname: str, symbols: SymbolTable, info: TypeInfo, + ret: Type, args: List[Argument] = []) -> None: types = [cast(Type, None)] + [arg.type_annotation for arg in args] - items = ['__self'] + [arg.variable.name() for arg in args] - arg_kinds = [ARG_POS] + [arg.kind for arg in args] - # TODO: Make sure that the self argument name is not visible? args = [Argument(Var('__self'), NoneTyp(), None, ARG_POS)] + args + items = [arg.variable.name() for arg in args] + arg_kinds = [arg.kind for arg in args] + # TODO: Make sure that the self argument name is not visible? signature = CallableType(types, arg_kinds, items, ret, self.named_type('__builtins__.function'), name=info.name()) @@ -1570,7 +1568,7 @@ def make_namedtuple_method(self, funcname: str, info: TypeInfo, Block([]), typ=signature) func.info = info - return func + symbols[funcname] = SymbolTableNode(MDEF, func) def analyze_types(self, items: List[Node]) -> List[Type]: result = [] # type: List[Type] diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index efd6ed94f58e5..d9f252e765720 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -65,155 +65,3 @@ n = N('x', 'x') # E: Argument 1 to "N" has incompatible type "str"; expected "in n = N(1, b=2) # E: Argument 2 to "N" has incompatible type "int"; expected "str" N(1, 'x') N(b='x', a=1) - -[case testNamedTupleAsBaseClass] -from typing import NamedTuple -N = NamedTuple('N', [('a', int), - ('b', str)]) -class X(N): - pass -x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" -s = '' -i = 0 -s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") -i, s = x -s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") - -[case testNamedTupleAsBaseClass2] -from typing import NamedTuple -class X(NamedTuple('N', [('a', int), - ('b', str)])): - pass -x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" -s = '' -i = 0 -s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") -i, s = x -s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") - -[case testTwoNamedTuplesAsBaseClasses] -from typing import NamedTuple -A = NamedTuple('A', [('a', int)]) -B = NamedTuple('B', [('a', int)]) -class X(A, B): # E: Class has two incompatible bases derived from tuple - pass - -[case testTwoNamedTuplesAsBaseClasses2] -from typing import NamedTuple -A = NamedTuple('A', [('a', int)]) -class X(A, NamedTuple('B', [('a', int)])): # E: Class has two incompatible bases derived from tuple - pass - -[case testSelfTypeWithNamedTupleAsBase] -from typing import NamedTuple -A = NamedTuple('A', [('a', int), ('b', str)]) -class B(A): - def f(self, x: int) -> None: - self.f(self.a) - self.f(self.b) # E: Argument 1 to "f" of "B" has incompatible type "str"; expected "int" - i = 0 - s = '' - i, s = self - i, i = self # E: Incompatible types in assignment (expression has type "str", \ - variable has type "int") -[out] -main: note: In member "f" of class "B": - -[case testTypeReferenceToClassDerivedFromNamedTuple] -from typing import NamedTuple -A = NamedTuple('A', [('a', int), ('b', str)]) -class B(A): - def f(self, x: 'B') -> None: - i = 0 - s = '' - self = x - i, s = x - i, s = x.a, x.b - i, s = x.a, x.a # E: Incompatible types in assignment (expression has type "int", \ - variable has type "str") - i, i = self # E: Incompatible types in assignment (expression has type "str", \ - variable has type "int") -[out] -main: note: In member "f" of class "B": - -[case testNamedTupleSubtyping] -from typing import NamedTuple, Tuple -A = NamedTuple('A', [('a', int), ('b', str)]) -class B(A): pass -a = A(1, '') -b = B(1, '') -t = None # type: Tuple[int, str] -b = a # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "A") -b = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "B") -t = a -t = (1, '') -t = b -a = b - -[case testNamedTupleSimpleTypeInference] -from typing import NamedTuple, Tuple -A = NamedTuple('A', [('a', int)]) -l = [A(1), A(2)] -a = A(1) -a = l[0] -(i,) = l[0] -i, i = l[0] # E: Need more than 1 value to unpack (2 expected) -l = [A(1)] -a = (1,) # E: Incompatible types in assignment (expression has type "Tuple[int]", \ - variable has type "A") -[builtins fixtures/list.py] - -[case testNamedTupleMissingClassAttribute] -import collections -MyNamedTuple = collections.namedtuple('MyNamedTuple', ['spam', 'eggs']) -MyNamedTuple.x # E: "MyNamedTuple" has no attribute "x" - -[case testNamedTupleEmptyItems] -from typing import NamedTuple -A = NamedTuple('A', []) -[builtins fixtures/list.py] - -[case testNamedTupleProperty] -from typing import NamedTuple -A = NamedTuple('A', [('a', int)]) -class B(A): - @property - def b(self) -> int: - return self.a -class C(B): pass -B(1).b -C(2).b -[builtins fixtures/property.py] - -[case testNamedTupleAsDict] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -a = x._asdict() - -[case testNamedTupleReplace] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -x = x._replace(y=5) - -[case testNamedTupleReplaceTyped] -from typing import NamedTuple - -X = NamedTuple('X', [('x', int), ('y', str)]) -x = None # type: X -x = x._replace(x=5) -x = x._replace(y=5) # E: Argument 1 to X has incompatible type "int"; expected "str" - -[case testNamedTupleFields] -from typing import Tuple -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -a = None # type: Tuple[str, str] -a = x._fields() -[builtins fixtures/tuple.py] From 46475d4eaee6039cb0bb54535dfc918a66965573 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Wed, 6 Jul 2016 20:19:28 +0300 Subject: [PATCH 10/74] restore tests for _replace and _asdict --- test-data/unit/check-namedtuple.test | 142 +++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index d9f252e765720..9c2d6c4a86d8f 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -65,3 +65,145 @@ n = N('x', 'x') # E: Argument 1 to "N" has incompatible type "str"; expected "in n = N(1, b=2) # E: Argument 2 to "N" has incompatible type "int"; expected "str" N(1, 'x') N(b='x', a=1) + +[case testNamedTupleAsBaseClass] +from typing import NamedTuple +N = NamedTuple('N', [('a', int), + ('b', str)]) +class X(N): + pass +x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" +s = '' +i = 0 +s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") +i, s = x +s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testNamedTupleAsBaseClass2] +from typing import NamedTuple +class X(NamedTuple('N', [('a', int), + ('b', str)])): + pass +x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" +s = '' +i = 0 +s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") +i, s = x +s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testTwoNamedTuplesAsBaseClasses] +from typing import NamedTuple +A = NamedTuple('A', [('a', int)]) +B = NamedTuple('B', [('a', int)]) +class X(A, B): # E: Class has two incompatible bases derived from tuple + pass + +[case testTwoNamedTuplesAsBaseClasses2] +from typing import NamedTuple +A = NamedTuple('A', [('a', int)]) +class X(A, NamedTuple('B', [('a', int)])): # E: Class has two incompatible bases derived from tuple + pass + +[case testSelfTypeWithNamedTupleAsBase] +from typing import NamedTuple +A = NamedTuple('A', [('a', int), ('b', str)]) +class B(A): + def f(self, x: int) -> None: + self.f(self.a) + self.f(self.b) # E: Argument 1 to "f" of "B" has incompatible type "str"; expected "int" + i = 0 + s = '' + i, s = self + i, i = self # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +[out] +main: note: In member "f" of class "B": + +[case testTypeReferenceToClassDerivedFromNamedTuple] +from typing import NamedTuple +A = NamedTuple('A', [('a', int), ('b', str)]) +class B(A): + def f(self, x: 'B') -> None: + i = 0 + s = '' + self = x + i, s = x + i, s = x.a, x.b + i, s = x.a, x.a # E: Incompatible types in assignment (expression has type "int", \ + variable has type "str") + i, i = self # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +[out] +main: note: In member "f" of class "B": + +[case testNamedTupleSubtyping] +from typing import NamedTuple, Tuple +A = NamedTuple('A', [('a', int), ('b', str)]) +class B(A): pass +a = A(1, '') +b = B(1, '') +t = None # type: Tuple[int, str] +b = a # E: Incompatible types in assignment (expression has type "A", variable has type "B") +a = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "A") +b = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "B") +t = a +t = (1, '') +t = b +a = b + +[case testNamedTupleSimpleTypeInference] +from typing import NamedTuple, Tuple +A = NamedTuple('A', [('a', int)]) +l = [A(1), A(2)] +a = A(1) +a = l[0] +(i,) = l[0] +i, i = l[0] # E: Need more than 1 value to unpack (2 expected) +l = [A(1)] +a = (1,) # E: Incompatible types in assignment (expression has type "Tuple[int]", \ + variable has type "A") +[builtins fixtures/list.py] + +[case testNamedTupleMissingClassAttribute] +import collections +MyNamedTuple = collections.namedtuple('MyNamedTuple', ['spam', 'eggs']) +MyNamedTuple.x # E: "MyNamedTuple" has no attribute "x" + +[case testNamedTupleEmptyItems] +from typing import NamedTuple +A = NamedTuple('A', []) +[builtins fixtures/list.py] + +[case testNamedTupleProperty] +from typing import NamedTuple +A = NamedTuple('A', [('a', int)]) +class B(A): + @property + def b(self) -> int: + return self.a +class C(B): pass +B(1).b +C(2).b +[builtins fixtures/property.py] + +[case testNamedTupleAsDict] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +a = x._asdict() + +[case testNamedTupleReplace] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +x = x._replace(y=5) + +[case testNamedTupleReplaceTyped] +from typing import NamedTuple + +X = NamedTuple('X', [('x', int), ('y', str)]) +x = None # type: X +x = x._replace(x=5) +x = x._replace(y=5) # E: Argument 1 to X has incompatible type "int"; expected "str" From c1dc1c769c0c1455e8a4c411cbd9337dfe409911 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Sun, 10 Jul 2016 02:52:21 +0300 Subject: [PATCH 11/74] NamedTupleTypeInfo --- mypy/nodes.py | 12 + mypy/semanal.py | 5672 +++++++++++++------------- test-data/unit/check-namedtuple.test | 419 +- 3 files changed, 3053 insertions(+), 3050 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index b068715d38162..f4c26634ba25b 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1976,6 +1976,18 @@ def deserialize(cls, data: JsonDict) -> 'TypeInfo': return ti +class NamedTupleTypeInfo(TypeInfo): + is_named_tuple = False + def __init__(self, tup, *args): + super().__init__(*args) + tup.fullname = lambda : '__builtins__.tuple' + self.tuple_type = tup + self.bases = [tup] + self.mro = [self, tup] + # self._promote = tup + + + class SymbolTableNode: # Kind of node. Possible values: # - LDEF: local definition (of any kind) diff --git a/mypy/semanal.py b/mypy/semanal.py index 0e7013ce7f149..50bd040f91536 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1,2841 +1,2831 @@ -"""The semantic analyzer. - -Bind names to definitions and do various other simple consistency -checks. For example, consider this program: - - x = 1 - y = x - -Here semantic analysis would detect that the assignment 'x = 1' -defines a new variable, the type of which is to be inferred (in a -later pass; type inference or type checking is not part of semantic -analysis). Also, it would bind both references to 'x' to the same -module-level variable node. The second assignment would also be -analyzed, and the type of 'y' marked as being inferred. - -Semantic analysis is the first analysis pass after parsing, and it is -subdivided into three passes: - - * FirstPass looks up externally visible names defined in a module but - ignores imports and local definitions. It helps enable (some) - cyclic references between modules, such as module 'a' that imports - module 'b' and used names defined in b *and* vice versa. The first - pass can be performed before dependent modules have been processed. - - * SemanticAnalyzer is the second pass. It does the bulk of the work. - It assumes that dependent modules have been semantically analyzed, - up to the second pass, unless there is a import cycle. - - * ThirdPass checks that type argument counts are valid; for example, - it will reject Dict[int]. We don't do this in the second pass, - since we infer the type argument counts of classes during this - pass, and it is possible to refer to classes defined later in a - file, which would not have the type argument count set yet. This - pass also recomputes the method resolution order of each class, in - case one of its bases belongs to a module involved in an import - loop. - -Semantic analysis of types is implemented in module mypy.typeanal. - -TODO: Check if the third pass slows down type checking significantly. - We could probably get rid of it -- for example, we could collect all - analyzed types in a collection and check them without having to - traverse the entire AST. -""" - -from typing import ( - List, Dict, Set, Tuple, cast, Any, overload, TypeVar, Union, Optional, Callable -) - -from mypy.nodes import ( - MypyFile, TypeInfo, Node, AssignmentStmt, FuncDef, OverloadedFuncDef, - ClassDef, Var, GDEF, MODULE_REF, FuncItem, Import, - ImportFrom, ImportAll, Block, LDEF, NameExpr, MemberExpr, - IndexExpr, TupleExpr, ListExpr, ExpressionStmt, ReturnStmt, - RaiseStmt, AssertStmt, OperatorAssignmentStmt, WhileStmt, - ForStmt, BreakStmt, ContinueStmt, IfStmt, TryStmt, WithStmt, DelStmt, - GlobalDecl, SuperExpr, DictExpr, CallExpr, RefExpr, OpExpr, UnaryExpr, - SliceExpr, CastExpr, RevealTypeExpr, TypeApplication, Context, SymbolTable, - SymbolTableNode, BOUND_TVAR, UNBOUND_TVAR, ListComprehension, GeneratorExpr, - FuncExpr, MDEF, FuncBase, Decorator, SetExpr, TypeVarExpr, - StrExpr, BytesExpr, PrintStmt, ConditionalExpr, PromoteExpr, - ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, MroError, type_aliases, - YieldFromExpr, NamedTupleExpr, NonlocalDecl, - SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr, - YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, COVARIANT, CONTRAVARIANT, - IntExpr, FloatExpr, UnicodeExpr, - INVARIANT, UNBOUND_IMPORTED, Expression, EllipsisExpr -) -from mypy.visitor import NodeVisitor -from mypy.traverser import TraverserVisitor -from mypy.errors import Errors, report_internal_error -from mypy.types import ( - NoneTyp, CallableType, Overloaded, Instance, Type, TypeVarType, AnyType, - FunctionLike, UnboundType, TypeList, ErrorType, TypeVarDef, - replace_leading_arg_type, TupleType, UnionType, StarType, EllipsisType -) -from mypy.nodes import function_type, implicit_module_attrs -from mypy.typeanal import TypeAnalyser, TypeAnalyserPass3, analyze_type_alias -from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError -from mypy.lex import lex -from mypy.parsetype import parse_type -from mypy.sametypes import is_same_type -from mypy.erasetype import erase_typevars -from mypy import defaults -from mypy.options import Options - - -T = TypeVar('T') - - -# Inferred value of an expression. -ALWAYS_TRUE = 0 -ALWAYS_FALSE = 1 -TRUTH_VALUE_UNKNOWN = 2 - -# Map from obsolete name to the current spelling. -obsolete_name_mapping = { - 'typing.Function': 'typing.Callable', - 'typing.typevar': 'typing.TypeVar', -} - -# Hard coded type promotions (shared between all Python versions). -# These add extra ad-hoc edges to the subtyping relation. For example, -# int is considered a subtype of float, even though there is no -# subclass relationship. -TYPE_PROMOTIONS = { - 'builtins.int': 'builtins.float', - 'builtins.float': 'builtins.complex', -} - -# Hard coded type promotions for Python 3. -# -# Note that the bytearray -> bytes promotion is a little unsafe -# as some functions only accept bytes objects. Here convenience -# trumps safety. -TYPE_PROMOTIONS_PYTHON3 = TYPE_PROMOTIONS.copy() -TYPE_PROMOTIONS_PYTHON3.update({ - 'builtins.bytearray': 'builtins.bytes', -}) - -# Hard coded type promotions for Python 2. -# -# These promotions are unsafe, but we are doing them anyway -# for convenience and also for Python 3 compatibility -# (bytearray -> str). -TYPE_PROMOTIONS_PYTHON2 = TYPE_PROMOTIONS.copy() -TYPE_PROMOTIONS_PYTHON2.update({ - 'builtins.str': 'builtins.unicode', - 'builtins.bytearray': 'builtins.str', -}) - -# When analyzing a function, should we analyze the whole function in one go, or -# should we only perform one phase of the analysis? The latter is used for -# nested functions. In the first phase we add the function to the symbol table -# but don't process body. In the second phase we process function body. This -# way we can have mutually recursive nested functions. -FUNCTION_BOTH_PHASES = 0 # Everthing in one go -FUNCTION_FIRST_PHASE_POSTPONE_SECOND = 1 # Add to symbol table but postpone body -FUNCTION_SECOND_PHASE = 2 # Only analyze body - - -class SemanticAnalyzer(NodeVisitor): - """Semantically analyze parsed mypy files. - - The analyzer binds names and does various consistency checks for a - parse tree. Note that type checking is performed as a separate - pass. - - This is the second phase of semantic analysis. - """ - - # Library search paths - lib_path = None # type: List[str] - # Module name space - modules = None # type: Dict[str, MypyFile] - # Global name space for current module - globals = None # type: SymbolTable - # Names declared using "global" (separate set for each scope) - global_decls = None # type: List[Set[str]] - # Names declated using "nonlocal" (separate set for each scope) - nonlocal_decls = None # type: List[Set[str]] - # Local names of function scopes; None for non-function scopes. - locals = None # type: List[SymbolTable] - # Nested block depths of scopes - block_depth = None # type: List[int] - # TypeInfo of directly enclosing class (or None) - type = None # type: TypeInfo - # Stack of outer classes (the second tuple item contains tvars). - type_stack = None # type: List[TypeInfo] - # Type variables that are bound by the directly enclosing class - bound_tvars = None # type: List[SymbolTableNode] - # Stack of type variables that were bound by outer classess - tvar_stack = None # type: List[List[SymbolTableNode]] - # Do weak type checking in this file - weak_opts = set() # type: Set[str] - - # Stack of functions being analyzed - function_stack = None # type: List[FuncItem] - # Stack of next available function type variable ids - next_function_tvar_id_stack = None # type: List[int] - - # Status of postponing analysis of nested function bodies. By using this we - # can have mutually recursive nested functions. Values are FUNCTION_x - # constants. Note that separate phasea are not used for methods. - postpone_nested_functions_stack = None # type: List[int] - # Postponed functions collected if - # postpone_nested_functions_stack[-1] == FUNCTION_FIRST_PHASE_POSTPONE_SECOND. - postponed_functions_stack = None # type: List[List[Node]] - - loop_depth = 0 # Depth of breakable loops - cur_mod_id = '' # Current module id (or None) (phase 2) - is_stub_file = False # Are we analyzing a stub file? - imports = None # type: Set[str] # Imported modules (during phase 2 analysis) - errors = None # type: Errors # Keeps track of generated errors - - def __init__(self, - lib_path: List[str], - errors: Errors, - options: Options) -> None: - """Construct semantic analyzer. - - Use lib_path to search for modules, and report analysis errors - using the Errors instance. - """ - self.locals = [None] - self.imports = set() - self.type = None - self.type_stack = [] - self.bound_tvars = None - self.tvar_stack = [] - self.function_stack = [] - self.next_function_tvar_id_stack = [-1] - self.block_depth = [0] - self.loop_depth = 0 - self.lib_path = lib_path - self.errors = errors - self.modules = {} - self.options = options - self.postpone_nested_functions_stack = [FUNCTION_BOTH_PHASES] - self.postponed_functions_stack = [] - self.all_exports = set() # type: Set[str] - - def visit_file(self, file_node: MypyFile, fnam: str) -> None: - self.errors.set_file(fnam) - self.cur_mod_node = file_node - self.cur_mod_id = file_node.fullname() - self.is_stub_file = fnam.lower().endswith('.pyi') - self.globals = file_node.names - self.weak_opts = file_node.weak_opts - - if 'builtins' in self.modules: - self.globals['__builtins__'] = SymbolTableNode( - MODULE_REF, self.modules['builtins'], self.cur_mod_id) - - for name in implicit_module_attrs: - v = self.globals[name].node - if isinstance(v, Var): - v.type = self.anal_type(v.type) - v.is_ready = True - - defs = file_node.defs - for d in defs: - self.accept(d) - - if self.cur_mod_id == 'builtins': - remove_imported_names_from_symtable(self.globals, 'builtins') - - if '__all__' in self.globals: - for name, g in self.globals.items(): - if name not in self.all_exports: - g.module_public = False - - def visit_func_def(self, defn: FuncDef) -> None: - phase_info = self.postpone_nested_functions_stack[-1] - if phase_info != FUNCTION_SECOND_PHASE: - self.function_stack.append(defn) - # First phase of analysis for function. - self.errors.push_function(defn.name()) - self.update_function_type_variables(defn) - self.errors.pop_function() - self.function_stack.pop() - - defn.is_conditional = self.block_depth[-1] > 0 - - # TODO(jukka): Figure out how to share the various cases. It doesn't - # make sense to have (almost) duplicate code (here and elsewhere) for - # 3 cases: module-level, class-level and local names. Maybe implement - # a common stack of namespaces. As the 3 kinds of namespaces have - # different semantics, this wouldn't always work, but it might still - # be a win. - if self.is_class_scope(): - # Method definition - defn.info = self.type - if not defn.is_decorated and not defn.is_overload: - if defn.name() in self.type.names: - # Redefinition. Conditional redefinition is okay. - n = self.type.names[defn.name()].node - if self.is_conditional_func(n, defn): - defn.original_def = cast(FuncDef, n) - else: - self.name_already_defined(defn.name(), defn) - self.type.names[defn.name()] = SymbolTableNode(MDEF, defn) - self.prepare_method_signature(defn) - elif self.is_func_scope(): - # Nested function - if not defn.is_decorated and not defn.is_overload: - if defn.name() in self.locals[-1]: - # Redefinition. Conditional redefinition is okay. - n = self.locals[-1][defn.name()].node - if self.is_conditional_func(n, defn): - defn.original_def = cast(FuncDef, n) - else: - self.name_already_defined(defn.name(), defn) - else: - self.add_local(defn, defn) - else: - # Top-level function - if not defn.is_decorated and not defn.is_overload: - symbol = self.globals.get(defn.name()) - if isinstance(symbol.node, FuncDef) and symbol.node != defn: - # This is redefinition. Conditional redefinition is okay. - original_def = symbol.node - if self.is_conditional_func(original_def, defn): - # Conditional function definition -- multiple defs are ok. - defn.original_def = original_def - else: - # Report error. - self.check_no_global(defn.name(), defn, True) - if phase_info == FUNCTION_FIRST_PHASE_POSTPONE_SECOND: - # Postpone this function (for the second phase). - self.postponed_functions_stack[-1].append(defn) - return - if phase_info != FUNCTION_FIRST_PHASE_POSTPONE_SECOND: - # Second phase of analysis for function. - self.errors.push_function(defn.name()) - self.analyze_function(defn) - self.errors.pop_function() - - def prepare_method_signature(self, func: FuncDef) -> None: - """Check basic signature validity and tweak annotation of self/cls argument.""" - # Only non-static methods are special. - if not func.is_static: - if not func.arguments: - self.fail('Method must have at least one argument', func) - elif func.type: - sig = cast(FunctionLike, func.type) - if func.is_class: - leading_type = self.class_type(self.type) - else: - leading_type = self_type(self.type) - func.type = replace_implicit_first_type(sig, leading_type) - - def is_conditional_func(self, previous: Node, new: FuncDef) -> bool: - """Does 'new' conditionally redefine 'previous'? - - We reject straight redefinitions of functions, as they are usually - a programming error. For example: - - . def f(): ... - . def f(): ... # Error: 'f' redefined - """ - return isinstance(previous, (FuncDef, Var)) and new.is_conditional - - def update_function_type_variables(self, defn: FuncDef) -> None: - """Make any type variables in the signature of defn explicit. - - Update the signature of defn to contain type variable definitions - if defn is generic. - """ - if defn.type: - functype = cast(CallableType, defn.type) - typevars = self.infer_type_variables(functype) - # Do not define a new type variable if already defined in scope. - typevars = [(name, tvar) for name, tvar in typevars - if not self.is_defined_type_var(name, defn)] - if typevars: - next_tvar_id = self.next_function_tvar_id() - defs = [TypeVarDef(tvar[0], next_tvar_id - i, - tvar[1].values, tvar[1].upper_bound, - tvar[1].variance) - for i, tvar in enumerate(typevars)] - functype.variables = defs - - def infer_type_variables(self, - type: CallableType) -> List[Tuple[str, TypeVarExpr]]: - """Return list of unique type variables referred to in a callable.""" - names = [] # type: List[str] - tvars = [] # type: List[TypeVarExpr] - for arg in type.arg_types + [type.ret_type]: - for name, tvar_expr in self.find_type_variables_in_type(arg): - if name not in names: - names.append(name) - tvars.append(tvar_expr) - return list(zip(names, tvars)) - - def find_type_variables_in_type( - self, type: Type) -> List[Tuple[str, TypeVarExpr]]: - """Return a list of all unique type variable references in type. - - This effectively does partial name binding, results of which are mostly thrown away. - """ - result = [] # type: List[Tuple[str, TypeVarExpr]] - if isinstance(type, UnboundType): - name = type.name - node = self.lookup_qualified(name, type) - if node and node.kind == UNBOUND_TVAR: - result.append((name, cast(TypeVarExpr, node.node))) - for arg in type.args: - result.extend(self.find_type_variables_in_type(arg)) - elif isinstance(type, TypeList): - for item in type.items: - result.extend(self.find_type_variables_in_type(item)) - elif isinstance(type, UnionType): - for item in type.items: - result.extend(self.find_type_variables_in_type(item)) - elif isinstance(type, AnyType): - pass - elif isinstance(type, EllipsisType) or isinstance(type, TupleType): - pass - else: - assert False, 'Unsupported type %s' % type - return result - - def is_defined_type_var(self, tvar: str, context: Node) -> bool: - return self.lookup_qualified(tvar, context).kind == BOUND_TVAR - - def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: - t = [] # type: List[CallableType] - for i, item in enumerate(defn.items): - # TODO support decorated overloaded functions properly - item.is_overload = True - item.func.is_overload = True - item.accept(self) - t.append(cast(CallableType, function_type(item.func, - self.builtin_type('builtins.function')))) - if item.func.is_property and i == 0: - # This defines a property, probably with a setter and/or deleter. - self.analyze_property_with_multi_part_definition(defn) - break - if not [dec for dec in item.decorators - if refers_to_fullname(dec, 'typing.overload')]: - self.fail("'overload' decorator expected", item) - - defn.type = Overloaded(t) - defn.type.line = defn.line - - if self.is_class_scope(): - self.type.names[defn.name()] = SymbolTableNode(MDEF, defn, - typ=defn.type) - defn.info = self.type - elif self.is_func_scope(): - self.add_local(defn, defn) - - def analyze_property_with_multi_part_definition(self, defn: OverloadedFuncDef) -> None: - """Analyze a property defined using multiple methods (e.g., using @x.setter). - - Assume that the first method (@property) has already been analyzed. - """ - defn.is_property = True - items = defn.items - for item in items[1:]: - if len(item.decorators) == 1: - node = item.decorators[0] - if isinstance(node, MemberExpr): - if node.name == 'setter': - # The first item represents the entire property. - defn.items[0].var.is_settable_property = True - # Get abstractness from the original definition. - item.func.is_abstract = items[0].func.is_abstract - else: - self.fail("Decorated property not supported", item) - item.func.accept(self) - - def next_function_tvar_id(self) -> int: - return self.next_function_tvar_id_stack[-1] - - def analyze_function(self, defn: FuncItem) -> None: - is_method = self.is_class_scope() - - tvarnodes = self.add_func_type_variables_to_symbol_table(defn) - next_function_tvar_id = min([self.next_function_tvar_id()] + - [n.tvar_def.id.raw_id - 1 for n in tvarnodes]) - self.next_function_tvar_id_stack.append(next_function_tvar_id) - - if defn.type: - # Signature must be analyzed in the surrounding scope so that - # class-level imported names and type variables are in scope. - defn.type = self.anal_type(defn.type) - self.check_function_signature(defn) - if isinstance(defn, FuncDef): - defn.type = set_callable_name(defn.type, defn) - for arg in defn.arguments: - if arg.initializer: - arg.initializer.accept(self) - self.function_stack.append(defn) - self.enter() - for arg in defn.arguments: - self.add_local(arg.variable, defn) - for arg in defn.arguments: - if arg.initialization_statement: - lvalue = arg.initialization_statement.lvalues[0] - lvalue.accept(self) - - # The first argument of a non-static, non-class method is like 'self' - # (though the name could be different), having the enclosing class's - # instance type. - if is_method and not defn.is_static and not defn.is_class and defn.arguments: - defn.arguments[0].variable.is_self = True - - # First analyze body of the function but ignore nested functions. - self.postpone_nested_functions_stack.append(FUNCTION_FIRST_PHASE_POSTPONE_SECOND) - self.postponed_functions_stack.append([]) - defn.body.accept(self) - - # Analyze nested functions (if any) as a second phase. - self.postpone_nested_functions_stack[-1] = FUNCTION_SECOND_PHASE - for postponed in self.postponed_functions_stack[-1]: - postponed.accept(self) - self.postpone_nested_functions_stack.pop() - self.postponed_functions_stack.pop() - - self.next_function_tvar_id_stack.pop() - disable_typevars(tvarnodes) - - self.leave() - self.function_stack.pop() - - def add_func_type_variables_to_symbol_table( - self, defn: FuncItem) -> List[SymbolTableNode]: - nodes = [] # type: List[SymbolTableNode] - if defn.type: - tt = defn.type - names = self.type_var_names() - items = cast(CallableType, tt).variables - for item in items: - name = item.name - if name in names: - self.name_already_defined(name, defn) - node = self.bind_type_var(name, item, defn) - nodes.append(node) - names.add(name) - return nodes - - def type_var_names(self) -> Set[str]: - if not self.type: - return set() - else: - return set(self.type.type_vars) - - def bind_type_var(self, fullname: str, tvar_def: TypeVarDef, - context: Context) -> SymbolTableNode: - node = self.lookup_qualified(fullname, context) - node.kind = BOUND_TVAR - node.tvar_def = tvar_def - return node - - def check_function_signature(self, fdef: FuncItem) -> None: - sig = cast(CallableType, fdef.type) - if len(sig.arg_types) < len(fdef.arguments): - self.fail('Type signature has too few arguments', fdef) - # Add dummy Any arguments to prevent crashes later. - extra_anys = [AnyType()] * (len(fdef.arguments) - len(sig.arg_types)) - sig.arg_types.extend(extra_anys) - elif len(sig.arg_types) > len(fdef.arguments): - self.fail('Type signature has too many arguments', fdef) - - def visit_class_def(self, defn: ClassDef) -> None: - self.clean_up_bases_and_infer_type_variables(defn) - self.setup_class_def_analysis(defn) - - self.bind_class_type_vars(defn) - - self.analyze_base_classes(defn) - self.analyze_metaclass(defn) - - for decorator in defn.decorators: - self.analyze_class_decorator(defn, decorator) - - self.enter_class(defn) - - self.setup_is_builtinclass(defn) - - # Analyze class body. - defn.defs.accept(self) - - self.calculate_abstract_status(defn.info) - self.setup_type_promotion(defn) - - self.leave_class() - self.unbind_class_type_vars() - - def enter_class(self, defn: ClassDef) -> None: - # Remember previous active class - self.type_stack.append(self.type) - self.locals.append(None) # Add class scope - self.block_depth.append(-1) # The class body increments this to 0 - self.postpone_nested_functions_stack.append(FUNCTION_BOTH_PHASES) - self.type = defn.info - - def leave_class(self) -> None: - """ Restore analyzer state. """ - self.postpone_nested_functions_stack.pop() - self.block_depth.pop() - self.locals.pop() - self.type = self.type_stack.pop() - - def bind_class_type_vars(self, defn: ClassDef) -> None: - """ Unbind type variables of previously active class and bind - the type variables for the active class. - """ - if self.bound_tvars: - disable_typevars(self.bound_tvars) - self.tvar_stack.append(self.bound_tvars) - self.bound_tvars = self.bind_class_type_variables_in_symbol_table(defn.info) - - def unbind_class_type_vars(self) -> None: - """ Unbind the active class' type vars and rebind the - type vars of the previously active class. - """ - disable_typevars(self.bound_tvars) - self.bound_tvars = self.tvar_stack.pop() - if self.bound_tvars: - enable_typevars(self.bound_tvars) - - def analyze_class_decorator(self, defn: ClassDef, decorator: Node) -> None: - decorator.accept(self) - - def setup_is_builtinclass(self, defn: ClassDef): - for decorator in defn.decorators: - if refers_to_fullname(decorator, 'typing.builtinclass'): - defn.is_builtinclass = True - if defn.fullname == 'builtins.object': - # Only 'object' is marked as a built-in class, as otherwise things elsewhere - # would break. We need a better way of dealing with built-in classes. - defn.is_builtinclass = True - - def calculate_abstract_status(self, typ: TypeInfo) -> None: - """Calculate abstract status of a class. - - Set is_abstract of the type to True if the type has an unimplemented - abstract attribute. Also compute a list of abstract attributes. - """ - concrete = set() # type: Set[str] - abstract = [] # type: List[str] - for base in typ.mro: - for name, symnode in base.names.items(): - node = symnode.node - if isinstance(node, OverloadedFuncDef): - # Unwrap an overloaded function definition. We can just - # check arbitrarily the first overload item. If the - # different items have a different abstract status, there - # should be an error reported elsewhere. - func = node.items[0] # type: Node - else: - func = node - if isinstance(func, Decorator): - fdef = func.func - if fdef.is_abstract and name not in concrete: - typ.is_abstract = True - abstract.append(name) - concrete.add(name) - typ.abstract_attributes = sorted(abstract) - - def setup_type_promotion(self, defn: ClassDef) -> None: - """Setup extra, ad-hoc subtyping relationships between classes (promotion). - - This includes things like 'int' being compatible with 'float'. - """ - promote_target = None # type: Type - for decorator in defn.decorators: - if isinstance(decorator, CallExpr): - analyzed = decorator.analyzed - if isinstance(analyzed, PromoteExpr): - # _promote class decorator (undocumented faeture). - promote_target = analyzed.type - if not promote_target: - promotions = (TYPE_PROMOTIONS_PYTHON3 if self.options.python_version[0] >= 3 - else TYPE_PROMOTIONS_PYTHON2) - if defn.fullname in promotions: - promote_target = self.named_type_or_none(promotions[defn.fullname]) - defn.info._promote = promote_target - - def clean_up_bases_and_infer_type_variables(self, defn: ClassDef) -> None: - """Remove extra base classes such as Generic and infer type vars. - - For example, consider this class: - - . class Foo(Bar, Generic[T]): ... - - Now we will remove Generic[T] from bases of Foo and infer that the - type variable 'T' is a type argument of Foo. - - Note that this is performed *before* semantic analysis. - """ - removed = [] # type: List[int] - type_vars = [] # type: List[TypeVarDef] - for i, base_expr in enumerate(defn.base_type_exprs): - try: - base = expr_to_unanalyzed_type(base_expr) - except TypeTranslationError: - # This error will be caught later. - continue - tvars = self.analyze_typevar_declaration(base) - if tvars is not None: - if type_vars: - self.fail('Duplicate Generic in bases', defn) - removed.append(i) - for j, (name, tvar_expr) in enumerate(tvars): - type_vars.append(TypeVarDef(name, j + 1, tvar_expr.values, - tvar_expr.upper_bound, tvar_expr.variance)) - if type_vars: - defn.type_vars = type_vars - if defn.info: - defn.info.type_vars = [tv.name for tv in type_vars] - for i in reversed(removed): - del defn.base_type_exprs[i] - - def analyze_typevar_declaration(self, t: Type) -> Optional[List[Tuple[str, TypeVarExpr]]]: - if not isinstance(t, UnboundType): - return None - unbound = t - sym = self.lookup_qualified(unbound.name, unbound) - if sym is None or sym.node is None: - return None - if sym.node.fullname() == 'typing.Generic': - tvars = [] # type: List[Tuple[str, TypeVarExpr]] - for arg in unbound.args: - tvar = self.analyze_unbound_tvar(arg) - if tvar: - tvars.append(tvar) - else: - self.fail('Free type variable expected in %s[...]' % - sym.node.name(), t) - return tvars - return None - - def analyze_unbound_tvar(self, t: Type) -> Tuple[str, TypeVarExpr]: - if not isinstance(t, UnboundType): - return None - unbound = t - sym = self.lookup_qualified(unbound.name, unbound) - if sym is not None and sym.kind == UNBOUND_TVAR: - return unbound.name, cast(TypeVarExpr, sym.node) - return None - - def setup_class_def_analysis(self, defn: ClassDef) -> None: - """Prepare for the analysis of a class definition.""" - if not defn.info: - defn.info = TypeInfo(SymbolTable(), defn) - defn.info._fullname = defn.info.name() - if self.is_func_scope() or self.type: - kind = MDEF - if self.is_func_scope(): - kind = LDEF - self.add_symbol(defn.name, SymbolTableNode(kind, defn.info), defn) - - def analyze_base_classes(self, defn: ClassDef) -> None: - """Analyze and set up base classes. - - This computes several attributes on the corresponding TypeInfo defn.info - related to the base classes: defn.info.bases, defn.info.mro, and - miscellaneous others (at least tuple_type, fallback_to_any, and is_enum.) - """ - - base_types = [] # type: List[Instance] - for base_expr in defn.base_type_exprs: - try: - base = self.expr_to_analyzed_type(base_expr) - except TypeTranslationError: - self.fail('Invalid base class', base_expr) - defn.info.fallback_to_any = True - continue - - if isinstance(base, TupleType): - if defn.info.tuple_type: - self.fail("Class has two incompatible bases derived from tuple", defn) - if (not self.is_stub_file - and not defn.info.is_named_tuple - and base.fallback.type.fullname() == 'builtins.tuple'): - self.fail("Tuple[...] not supported as a base class outside a stub file", defn) - defn.info.tuple_type = base - base_types.append(base.fallback) - elif isinstance(base, Instance): - base_types.append(base) - elif isinstance(base, AnyType): - defn.info.fallback_to_any = True - else: - self.fail('Invalid base class', base_expr) - defn.info.fallback_to_any = True - - # Add 'object' as implicit base if there is no other base class. - if (not base_types and defn.fullname != 'builtins.object'): - base_types.append(self.object_type()) - - defn.info.bases = base_types - - # Calculate the MRO. It might be incomplete at this point if - # the bases of defn include classes imported from other - # modules in an import loop. We'll recompute it in ThirdPass. - if not self.verify_base_classes(defn): - defn.info.mro = [] - return - calculate_class_mro(defn, self.fail) - # If there are cyclic imports, we may be missing 'object' in - # the MRO. Fix MRO if needed. - if defn.info.mro and defn.info.mro[-1].fullname() != 'builtins.object': - defn.info.mro.append(self.object_type().type) - - def expr_to_analyzed_type(self, expr: Node) -> Type: - if isinstance(expr, CallExpr): - expr.accept(self) - info = self.check_namedtuple(expr) - if info is None: - # Some form of namedtuple is the only valid type that looks like a call - # expression. This isn't a valid type. - raise TypeTranslationError() - fallback = Instance(info, []) - return TupleType(info.tuple_type.items, fallback=fallback) - typ = expr_to_unanalyzed_type(expr) - return self.anal_type(typ) - - def verify_base_classes(self, defn: ClassDef) -> bool: - info = defn.info - for base in info.bases: - baseinfo = base.type - if self.is_base_class(info, baseinfo): - self.fail('Cycle in inheritance hierarchy', defn) - # Clear bases to forcefully get rid of the cycle. - info.bases = [] - if baseinfo.fullname() == 'builtins.bool': - self.fail("'%s' is not a valid base class" % - baseinfo.name(), defn) - return False - dup = find_duplicate(info.direct_base_classes()) - if dup: - self.fail('Duplicate base class "%s"' % dup.name(), defn) - return False - return True - - def is_base_class(self, t: TypeInfo, s: TypeInfo) -> bool: - """Determine if t is a base class of s (but do not use mro).""" - # Search the base class graph for t, starting from s. - worklist = [s] - visited = {s} - while worklist: - nxt = worklist.pop() - if nxt == t: - return True - for base in nxt.bases: - if base.type not in visited: - worklist.append(base.type) - visited.add(base.type) - return False - - def analyze_metaclass(self, defn: ClassDef) -> None: - if defn.metaclass: - sym = self.lookup_qualified(defn.metaclass, defn) - if sym is not None and not isinstance(sym.node, TypeInfo): - self.fail("Invalid metaclass '%s'" % defn.metaclass, defn) - - def object_type(self) -> Instance: - return self.named_type('__builtins__.object') - - def class_type(self, info: TypeInfo) -> Type: - # Construct a function type whose fallback is cls. - from mypy import checkmember # To avoid import cycle. - leading_type = checkmember.type_object_type(info, self.builtin_type) - if isinstance(leading_type, Overloaded): - # Overloaded __init__ is too complex to handle. Plus it's stubs only. - return AnyType() - else: - return leading_type - - def named_type(self, qualified_name: str, args: List[Type] = None) -> Instance: - sym = self.lookup_qualified(qualified_name, None) - return Instance(cast(TypeInfo, sym.node), args or []) - - def named_type_or_none(self, qualified_name: str) -> Instance: - sym = self.lookup_fully_qualified_or_none(qualified_name) - if not sym: - return None - return Instance(cast(TypeInfo, sym.node), []) - - def is_instance_type(self, t: Type) -> bool: - return isinstance(t, Instance) - - def bind_class_type_variables_in_symbol_table( - self, info: TypeInfo) -> List[SymbolTableNode]: - nodes = [] # type: List[SymbolTableNode] - for var, binder in zip(info.type_vars, info.defn.type_vars): - node = self.bind_type_var(var, binder, info) - nodes.append(node) - return nodes - - def visit_import(self, i: Import) -> None: - for id, as_id in i.ids: - if as_id is not None: - self.add_module_symbol(id, as_id, module_public=True, context=i) - else: - # Modules imported in a stub file without using 'as x' won't get exported when - # doing 'from m import *'. - module_public = not self.is_stub_file - base = id.split('.')[0] - self.add_module_symbol(base, base, module_public=module_public, - context=i) - - def add_module_symbol(self, id: str, as_id: str, module_public: bool, - context: Context) -> None: - if id in self.modules: - m = self.modules[id] - self.add_symbol(as_id, SymbolTableNode(MODULE_REF, m, self.cur_mod_id, - module_public=module_public), context) - else: - self.add_unknown_symbol(as_id, context) - - def visit_import_from(self, imp: ImportFrom) -> None: - import_id = self.correct_relative_import(imp) - if import_id in self.modules: - module = self.modules[import_id] - for id, as_id in imp.names: - node = module.names.get(id) - if node and node.kind != UNBOUND_IMPORTED: - node = self.normalize_type_alias(node, imp) - if not node: - return - imported_id = as_id or id - existing_symbol = self.globals.get(imported_id) - if existing_symbol: - # Import can redefine a variable. They get special treatment. - if self.process_import_over_existing_name( - imported_id, existing_symbol, node, imp): - continue - # 'from m import x as x' exports x in a stub file. - module_public = not self.is_stub_file or as_id is not None - symbol = SymbolTableNode(node.kind, node.node, - self.cur_mod_id, - node.type_override, - module_public=module_public) - self.add_symbol(imported_id, symbol, imp) - else: - message = "Module has no attribute '{}'".format(id) - extra = self.undefined_name_extra_info('{}.{}'.format(import_id, id)) - if extra: - message += " {}".format(extra) - self.fail(message, imp) - else: - # Missing module. - for id, as_id in imp.names: - self.add_unknown_symbol(as_id or id, imp) - - def process_import_over_existing_name(self, - imported_id: str, existing_symbol: SymbolTableNode, - module_symbol: SymbolTableNode, - import_node: ImportBase) -> bool: - if (existing_symbol.kind in (LDEF, GDEF, MDEF) and - isinstance(existing_symbol.node, (Var, FuncDef))): - # This is a valid import over an existing definition in the file. Construct a dummy - # assignment that we'll use to type check the import. - lvalue = NameExpr(imported_id) - lvalue.kind = existing_symbol.kind - lvalue.node = existing_symbol.node - rvalue = NameExpr(imported_id) - rvalue.kind = module_symbol.kind - rvalue.node = module_symbol.node - assignment = AssignmentStmt([lvalue], rvalue) - for node in assignment, lvalue, rvalue: - node.set_line(import_node) - import_node.assignments.append(assignment) - return True - return False - - def normalize_type_alias(self, node: SymbolTableNode, - ctx: Context) -> SymbolTableNode: - if node.fullname in type_aliases: - # Node refers to an aliased type such as typing.List; normalize. - node = self.lookup_qualified(type_aliases[node.fullname], ctx) - if node.fullname == 'typing.DefaultDict': - self.add_module_symbol('collections', '__mypy_collections__', False, ctx) - node = self.lookup_qualified('__mypy_collections__.defaultdict', ctx) - return node - - def correct_relative_import(self, node: Union[ImportFrom, ImportAll]) -> str: - if node.relative == 0: - return node.id - - parts = self.cur_mod_id.split(".") - cur_mod_id = self.cur_mod_id - - rel = node.relative - if self.cur_mod_node.is_package_init_file(): - rel -= 1 - if len(parts) < rel: - self.fail("Relative import climbs too many namespaces", node) - if rel != 0: - cur_mod_id = ".".join(parts[:-rel]) - - return cur_mod_id + (("." + node.id) if node.id else "") - - def visit_import_all(self, i: ImportAll) -> None: - i_id = self.correct_relative_import(i) - if i_id in self.modules: - m = self.modules[i_id] - for name, node in m.names.items(): - node = self.normalize_type_alias(node, i) - if not name.startswith('_') and node.module_public: - existing_symbol = self.globals.get(name) - if existing_symbol: - # Import can redefine a variable. They get special treatment. - if self.process_import_over_existing_name( - name, existing_symbol, node, i): - continue - self.add_symbol(name, SymbolTableNode(node.kind, node.node, - self.cur_mod_id), i) - else: - # Don't add any dummy symbols for 'from x import *' if 'x' is unknown. - pass - - def add_unknown_symbol(self, name: str, context: Context) -> None: - var = Var(name) - var._fullname = self.qualified_name(name) - var.is_ready = True - var.type = AnyType() - self.add_symbol(name, SymbolTableNode(GDEF, var, self.cur_mod_id), context) - - # - # Statements - # - - def visit_block(self, b: Block) -> None: - if b.is_unreachable: - return - self.block_depth[-1] += 1 - for s in b.body: - self.accept(s) - self.block_depth[-1] -= 1 - - def visit_block_maybe(self, b: Block) -> None: - if b: - self.visit_block(b) - - def anal_type(self, t: Type, allow_tuple_literal: bool = False) -> Type: - if t: - if allow_tuple_literal: - # Types such as (t1, t2, ...) only allowed in assignment statements. They'll - # generate errors elsewhere, and Tuple[t1, t2, ...] must be used instead. - if isinstance(t, TupleType): - # Unlike TypeAnalyser, also allow implicit tuple types (without Tuple[...]). - star_count = sum(1 for item in t.items if isinstance(item, StarType)) - if star_count > 1: - self.fail('At most one star type allowed in a tuple', t) - return TupleType([AnyType() for _ in t.items], - self.builtin_type('builtins.tuple'), t.line) - items = [self.anal_type(item, True) - for item in t.items] - return TupleType(items, self.builtin_type('builtins.tuple'), t.line) - a = TypeAnalyser(self.lookup_qualified, - self.lookup_fully_qualified, - self.fail) - return t.accept(a) - else: - return None - - def visit_assignment_stmt(self, s: AssignmentStmt) -> None: - for lval in s.lvalues: - self.analyze_lvalue(lval, explicit_type=s.type is not None) - s.rvalue.accept(self) - if s.type: - allow_tuple_literal = isinstance(s.lvalues[-1], (TupleExpr, ListExpr)) - s.type = self.anal_type(s.type, allow_tuple_literal) - else: - # For simple assignments, allow binding type aliases. - # Also set the type if the rvalue is a simple literal. - if (s.type is None and len(s.lvalues) == 1 and - isinstance(s.lvalues[0], NameExpr)): - if s.lvalues[0].is_def: - s.type = self.analyze_simple_literal_type(s.rvalue) - res = analyze_type_alias(s.rvalue, - self.lookup_qualified, - self.lookup_fully_qualified, - self.fail) - if res and (not isinstance(res, Instance) or cast(Instance, res).args): - # TODO: What if this gets reassigned? - name = s.lvalues[0] - node = self.lookup(name.name, name) - node.kind = TYPE_ALIAS - node.type_override = res - if isinstance(s.rvalue, IndexExpr): - s.rvalue.analyzed = TypeAliasExpr(res) - if s.type: - # Store type into nodes. - for lvalue in s.lvalues: - self.store_declared_types(lvalue, s.type) - self.check_and_set_up_type_alias(s) - self.process_typevar_declaration(s) - self.process_namedtuple_definition(s) - - if (len(s.lvalues) == 1 and isinstance(s.lvalues[0], NameExpr) and - s.lvalues[0].name == '__all__' and s.lvalues[0].kind == GDEF and - isinstance(s.rvalue, (ListExpr, TupleExpr))): - self.add_exports(*s.rvalue.items) - - def analyze_simple_literal_type(self, rvalue: Node) -> Optional[Type]: - """Return builtins.int if rvalue is an int literal, etc.""" - if self.weak_opts or self.options.semantic_analysis_only or self.function_stack: - # Skip this if any weak options are set. - # Also skip if we're only doing the semantic analysis pass. - # This is mostly to avoid breaking unit tests. - # Also skip inside a function; this is to avoid confusing - # the code that handles dead code due to isinstance() - # inside type variables with value restrictions (like - # AnyStr). - return None - if isinstance(rvalue, IntExpr): - return self.named_type_or_none('builtins.int') - if isinstance(rvalue, FloatExpr): - return self.named_type_or_none('builtins.float') - if isinstance(rvalue, StrExpr): - return self.named_type_or_none('builtins.str') - if isinstance(rvalue, BytesExpr): - return self.named_type_or_none('builtins.bytes') - if isinstance(rvalue, UnicodeExpr): - return self.named_type_or_none('builtins.unicode') - return None - - def check_and_set_up_type_alias(self, s: AssignmentStmt) -> None: - """Check if assignment creates a type alias and set it up as needed.""" - # For now, type aliases only work at the top level of a module. - if (len(s.lvalues) == 1 and not self.is_func_scope() and not self.type - and not s.type): - lvalue = s.lvalues[0] - if isinstance(lvalue, NameExpr): - if not lvalue.is_def: - # Only a definition can create a type alias, not regular assignment. - return - rvalue = s.rvalue - if isinstance(rvalue, RefExpr): - node = rvalue.node - if isinstance(node, TypeInfo): - # TODO: We should record the fact that this is a variable - # that refers to a type, rather than making this - # just an alias for the type. - self.globals[lvalue.name].node = node - - def analyze_lvalue(self, lval: Node, nested: bool = False, - add_global: bool = False, - explicit_type: bool = False) -> None: - """Analyze an lvalue or assignment target. - - Only if add_global is True, add name to globals table. If nested - is true, the lvalue is within a tuple or list lvalue expression. - """ - - if isinstance(lval, NameExpr): - # Top-level definitions within some statements (at least while) are - # not handled in the first pass, so they have to be added now. - nested_global = (not self.is_func_scope() and - self.block_depth[-1] > 0 and - not self.type) - if (add_global or nested_global) and lval.name not in self.globals: - # Define new global name. - v = Var(lval.name) - v._fullname = self.qualified_name(lval.name) - v.is_ready = False # Type not inferred yet - lval.node = v - lval.is_def = True - lval.kind = GDEF - lval.fullname = v._fullname - self.globals[lval.name] = SymbolTableNode(GDEF, v, - self.cur_mod_id) - elif isinstance(lval.node, Var) and lval.is_def: - # Since the is_def flag is set, this must have been analyzed - # already in the first pass and added to the symbol table. - assert lval.node.name() in self.globals - elif (self.is_func_scope() and lval.name not in self.locals[-1] and - lval.name not in self.global_decls[-1] and - lval.name not in self.nonlocal_decls[-1]): - # Define new local name. - v = Var(lval.name) - lval.node = v - lval.is_def = True - lval.kind = LDEF - lval.fullname = lval.name - self.add_local(v, lval) - elif not self.is_func_scope() and (self.type and - lval.name not in self.type.names): - # Define a new attribute within class body. - v = Var(lval.name) - v.info = self.type - v.is_initialized_in_class = True - v.set_line(lval) - lval.node = v - lval.is_def = True - lval.kind = MDEF - lval.fullname = lval.name - self.type.names[lval.name] = SymbolTableNode(MDEF, v) - else: - # Bind to an existing name. - if explicit_type: - self.name_already_defined(lval.name, lval) - lval.accept(self) - self.check_lvalue_validity(lval.node, lval) - elif isinstance(lval, MemberExpr): - if not add_global: - self.analyze_member_lvalue(lval) - if explicit_type and not self.is_self_member_ref(lval): - self.fail('Type cannot be declared in assignment to non-self ' - 'attribute', lval) - elif isinstance(lval, IndexExpr): - if explicit_type: - self.fail('Unexpected type declaration', lval) - if not add_global: - lval.accept(self) - elif (isinstance(lval, TupleExpr) or - isinstance(lval, ListExpr)): - items = cast(Any, lval).items - if len(items) == 0 and isinstance(lval, TupleExpr): - self.fail("Can't assign to ()", lval) - self.analyze_tuple_or_list_lvalue(cast(Union[ListExpr, TupleExpr], lval), - add_global, explicit_type) - elif isinstance(lval, StarExpr): - if nested: - self.analyze_lvalue(lval.expr, nested, add_global, explicit_type) - else: - self.fail('Starred assignment target must be in a list or tuple', lval) - else: - self.fail('Invalid assignment target', lval) - - def analyze_tuple_or_list_lvalue(self, lval: Union[ListExpr, TupleExpr], - add_global: bool = False, - explicit_type: bool = False) -> None: - """Analyze an lvalue or assignment target that is a list or tuple.""" - items = lval.items - star_exprs = [cast(StarExpr, item) - for item in items - if isinstance(item, StarExpr)] - - if len(star_exprs) > 1: - self.fail('Two starred expressions in assignment', lval) - else: - if len(star_exprs) == 1: - star_exprs[0].valid = True - for i in items: - self.analyze_lvalue(i, nested=True, add_global=add_global, - explicit_type = explicit_type) - - def analyze_member_lvalue(self, lval: MemberExpr) -> None: - lval.accept(self) - if (self.is_self_member_ref(lval) and - self.type.get(lval.name) is None): - # Implicit attribute definition in __init__. - lval.is_def = True - v = Var(lval.name) - v.info = self.type - v.is_ready = False - lval.def_var = v - lval.node = v - self.type.names[lval.name] = SymbolTableNode(MDEF, v) - self.check_lvalue_validity(lval.node, lval) - - def is_self_member_ref(self, memberexpr: MemberExpr) -> bool: - """Does memberexpr to refer to an attribute of self?""" - if not isinstance(memberexpr.expr, NameExpr): - return False - node = memberexpr.expr.node - return isinstance(node, Var) and node.is_self - - def check_lvalue_validity(self, node: Node, ctx: Context) -> None: - if isinstance(node, (TypeInfo, TypeVarExpr)): - self.fail('Invalid assignment target', ctx) - - def store_declared_types(self, lvalue: Node, typ: Type) -> None: - if isinstance(typ, StarType) and not isinstance(lvalue, StarExpr): - self.fail('Star type only allowed for starred expressions', lvalue) - if isinstance(lvalue, RefExpr): - lvalue.is_def = False - if isinstance(lvalue.node, Var): - var = lvalue.node - var.type = typ - var.is_ready = True - # If node is not a variable, we'll catch it elsewhere. - elif isinstance(lvalue, TupleExpr): - if isinstance(typ, TupleType): - if len(lvalue.items) != len(typ.items): - self.fail('Incompatible number of tuple items', lvalue) - return - for item, itemtype in zip(lvalue.items, typ.items): - self.store_declared_types(item, itemtype) - else: - self.fail('Tuple type expected for multiple variables', - lvalue) - elif isinstance(lvalue, StarExpr): - if isinstance(typ, StarType): - self.store_declared_types(lvalue.expr, typ.type) - else: - self.fail('Star type expected for starred expression', lvalue) - else: - # This has been flagged elsewhere as an error, so just ignore here. - pass - - def process_typevar_declaration(self, s: AssignmentStmt) -> None: - """Check if s declares a TypeVar; it yes, store it in symbol table.""" - call = self.get_typevar_declaration(s) - if not call: - return - - lvalue = cast(NameExpr, s.lvalues[0]) - name = lvalue.name - if not lvalue.is_def: - if s.type: - self.fail("Cannot declare the type of a type variable", s) - else: - self.fail("Cannot redefine '%s' as a type variable" % name, s) - return - - if not self.check_typevar_name(call, name, s): - return - - # Constraining types - n_values = call.arg_kinds[1:].count(ARG_POS) - values = self.analyze_types(call.args[1:1 + n_values]) - - res = self.process_typevar_parameters(call.args[1 + n_values:], - call.arg_names[1 + n_values:], - call.arg_kinds[1 + n_values:], - bool(values), - s) - if res is None: - return - variance, upper_bound = res - - # Yes, it's a valid type variable definition! Add it to the symbol table. - node = self.lookup(name, s) - node.kind = UNBOUND_TVAR - TypeVar = TypeVarExpr(name, node.fullname, values, upper_bound, variance) - TypeVar.line = call.line - call.analyzed = TypeVar - node.node = TypeVar - - def check_typevar_name(self, call: CallExpr, name: str, context: Context) -> bool: - if len(call.args) < 1: - self.fail("Too few arguments for TypeVar()", context) - return False - if not isinstance(call.args[0], (StrExpr, BytesExpr)) or not call.arg_kinds[0] == ARG_POS: - self.fail("TypeVar() expects a string literal as first argument", context) - return False - if cast(StrExpr, call.args[0]).value != name: - self.fail("Unexpected TypeVar() argument value", context) - return False - return True - - def get_typevar_declaration(self, s: AssignmentStmt) -> Optional[CallExpr]: - """Returns the TypeVar() call expression if `s` is a type var declaration - or None otherwise. - """ - if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): - return None - if not isinstance(s.rvalue, CallExpr): - return None - call = s.rvalue - if not isinstance(call.callee, RefExpr): - return None - callee = call.callee - if callee.fullname != 'typing.TypeVar': - return None - return call - - def process_typevar_parameters(self, args: List[Node], - names: List[Optional[str]], - kinds: List[int], - has_values: bool, - context: Context) -> Optional[Tuple[int, Type]]: - covariant = False - contravariant = False - upper_bound = self.object_type() # type: Type - for param_value, param_name, param_kind in zip(args, names, kinds): - if not param_kind == ARG_NAMED: - self.fail("Unexpected argument to TypeVar()", context) - return None - if param_name == 'covariant': - if isinstance(param_value, NameExpr): - if param_value.name == 'True': - covariant = True - else: - self.fail("TypeVar 'covariant' may only be 'True'", context) - return None - else: - self.fail("TypeVar 'covariant' may only be 'True'", context) - return None - elif param_name == 'contravariant': - if isinstance(param_value, NameExpr): - if param_value.name == 'True': - contravariant = True - else: - self.fail("TypeVar 'contravariant' may only be 'True'", context) - return None - else: - self.fail("TypeVar 'contravariant' may only be 'True'", context) - return None - elif param_name == 'bound': - if has_values: - self.fail("TypeVar cannot have both values and an upper bound", context) - return None - try: - upper_bound = self.expr_to_analyzed_type(param_value) - except TypeTranslationError: - self.fail("TypeVar 'bound' must be a type", param_value) - return None - elif param_name == 'values': - # Probably using obsolete syntax with values=(...). Explain the current syntax. - self.fail("TypeVar 'values' argument not supported", context) - self.fail("Use TypeVar('T', t, ...) instead of TypeVar('T', values=(t, ...))", - context) - return None - else: - self.fail("Unexpected argument to TypeVar(): {}".format(param_name), context) - return None - - if covariant and contravariant: - self.fail("TypeVar cannot be both covariant and contravariant", context) - return None - elif covariant: - variance = COVARIANT - elif contravariant: - variance = CONTRAVARIANT - else: - variance = INVARIANT - return (variance, upper_bound) - - def process_namedtuple_definition(self, s: AssignmentStmt) -> None: - """Check if s defines a namedtuple; if yes, store the definition in symbol table.""" - if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): - return - lvalue = cast(NameExpr, s.lvalues[0]) - name = lvalue.name - named_tuple = self.check_namedtuple(s.rvalue, name) - if named_tuple is None: - return - # Yes, it's a valid namedtuple definition. Add it to the symbol table. - node = self.lookup(name, s) - node.kind = GDEF # TODO locally defined namedtuple - # TODO call.analyzed - node.node = named_tuple - - def check_namedtuple(self, node: Node, var_name: str = None) -> TypeInfo: - """Check if a call defines a namedtuple. - - The optional var_name argument is the name of the variable to - which this is assigned, if any. - - If it does, return the corresponding TypeInfo. Return None otherwise. - - If the definition is invalid but looks like a namedtuple, - report errors but return (some) TypeInfo. - """ - if not isinstance(node, CallExpr): - return None - call = node - if not isinstance(call.callee, RefExpr): - return None - callee = call.callee - fullname = callee.fullname - if fullname not in ('collections.namedtuple', 'typing.NamedTuple'): - return None - items, types, ok = self.parse_namedtuple_args(call, fullname) - if not ok: - # Error. Construct dummy return value. - return self.build_namedtuple_typeinfo('namedtuple', [], []) - else: - # Give it a unique name derived from the line number. - name = cast(StrExpr, call.args[0]).value - if name != var_name: - name += '@' + str(call.line) - info = self.build_namedtuple_typeinfo(name, items, types) - # Store it as a global just in case it would remain anonymous. - self.globals[name] = SymbolTableNode(GDEF, info, self.cur_mod_id) - call.analyzed = NamedTupleExpr(info).set_line(call.line) - return info - - def parse_namedtuple_args(self, call: CallExpr, - fullname: str) -> Tuple[List[str], List[Type], bool]: - # TODO Share code with check_argument_count in checkexpr.py? - args = call.args - if len(args) < 2: - return self.fail_namedtuple_arg("Too few arguments for namedtuple()", call) - if len(args) > 2: - return self.fail_namedtuple_arg("Too many arguments for namedtuple()", call) - if call.arg_kinds != [ARG_POS, ARG_POS]: - return self.fail_namedtuple_arg("Unexpected arguments to namedtuple()", call) - if not isinstance(args[0], (StrExpr, BytesExpr)): - return self.fail_namedtuple_arg( - "namedtuple() expects a string literal as the first argument", call) - types = [] # type: List[Type] - ok = True - if not isinstance(args[1], ListExpr): - if fullname == 'collections.namedtuple' and isinstance(args[1], (StrExpr, BytesExpr)): - str_expr = cast(StrExpr, args[1]) - items = str_expr.value.split() - else: - return self.fail_namedtuple_arg( - "List literal expected as the second argument to namedtuple()", call) - else: - listexpr = args[1] - if fullname == 'collections.namedtuple': - # The fields argument contains just names, with implicit Any types. - if any(not isinstance(item, (StrExpr, BytesExpr)) for item in listexpr.items): - return self.fail_namedtuple_arg("String literal expected as namedtuple() item", - call) - items = [cast(StrExpr, item).value for item in listexpr.items] - else: - # The fields argument contains (name, type) tuples. - items, types, ok = self.parse_namedtuple_fields_with_types(listexpr.items, call) - if not types: - types = [AnyType() for _ in items] - return items, types, ok - - def parse_namedtuple_fields_with_types(self, nodes: List[Node], - context: Context) -> Tuple[List[str], List[Type], bool]: - items = [] # type: List[str] - types = [] # type: List[Type] - for item in nodes: - if isinstance(item, TupleExpr): - if len(item.items) != 2: - return self.fail_namedtuple_arg("Invalid NamedTuple field definition", - item) - name, type_node = item.items - if isinstance(name, (StrExpr, BytesExpr)): - items.append(name.value) - else: - return self.fail_namedtuple_arg("Invalid NamedTuple() field name", item) - try: - type = expr_to_unanalyzed_type(type_node) - except TypeTranslationError: - return self.fail_namedtuple_arg('Invalid field type', type_node) - types.append(self.anal_type(type)) - else: - return self.fail_namedtuple_arg("Tuple expected as NamedTuple() field", item) - return items, types, True - - def fail_namedtuple_arg(self, message: str, - context: Context) -> Tuple[List[str], List[Type], bool]: - self.fail(message, context) - return [], [], False - - def build_namedtuple_typeinfo(self, name: str, items: List[str], - types: List[Type]) -> TypeInfo: - symbols = SymbolTable() - class_def = ClassDef(name, Block([])) - class_def.fullname = self.qualified_name(name) - info = TypeInfo(symbols, class_def) - # Add named tuple items as attributes. - # TODO: Make them read-only. - for item, typ in zip(items, types): - var = Var(item) - var.info = info - var.type = typ - symbols[item] = SymbolTableNode(MDEF, var) - # Add __init__, _replace, _asdict methods and _fields static field - self.add_namedtuple_method('__init__', symbols, info, NoneTyp(), - self.make_factory_args(items, types, ARG_POS)) - # TODO: refine return type to type(self) - self.add_namedtuple_method('_replace', symbols, info, AnyType(), - self.make_factory_args(items, types, ARG_NAMED, initializer=EllipsisExpr())) - # TODO: refine to OrderedDict[Union[types]] - self.add_namedtuple_method('_asdict', symbols, info, AnyType()) - - info.tuple_type = TupleType(types, self.named_type('__builtins__.tuple', [AnyType()])) - info.is_named_tuple = True - info.mro = [info] + info.tuple_type.fallback.type.mro - info.bases = [info.tuple_type.fallback] - return info - - def make_factory_args(self, items: List[str], types: List[Type], - kind: int, initializer: Expression = None) -> List[Argument]: - return [Argument(Var(item), type, initializer, kind) for item, type in zip(items, types)] - - def add_namedtuple_method(self, funcname: str, symbols: SymbolTable, info: TypeInfo, - ret: Type, args: List[Argument] = []) -> None: - types = [cast(Type, None)] + [arg.type_annotation for arg in args] - args = [Argument(Var('__self'), NoneTyp(), None, ARG_POS)] + args - items = [arg.variable.name() for arg in args] - arg_kinds = [arg.kind for arg in args] - # TODO: Make sure that the self argument name is not visible? - signature = CallableType(types, arg_kinds, items, ret, - self.named_type('__builtins__.function'), - name=info.name()) - func = FuncDef(funcname, - args, - Block([]), - typ=signature) - func.info = info - symbols[funcname] = SymbolTableNode(MDEF, func) - - def analyze_types(self, items: List[Node]) -> List[Type]: - result = [] # type: List[Type] - for node in items: - try: - result.append(self.anal_type(expr_to_unanalyzed_type(node))) - except TypeTranslationError: - self.fail('Type expected', node) - result.append(AnyType()) - return result - - def visit_decorator(self, dec: Decorator) -> None: - for d in dec.decorators: - d.accept(self) - removed = [] # type: List[int] - no_type_check = False - for i, d in enumerate(dec.decorators): - # A bunch of decorators are special cased here. - if refers_to_fullname(d, 'abc.abstractmethod'): - removed.append(i) - dec.func.is_abstract = True - self.check_decorated_function_is_method('abstractmethod', dec) - elif refers_to_fullname(d, 'asyncio.tasks.coroutine'): - removed.append(i) - elif refers_to_fullname(d, 'builtins.staticmethod'): - removed.append(i) - dec.func.is_static = True - dec.var.is_staticmethod = True - self.check_decorated_function_is_method('staticmethod', dec) - elif refers_to_fullname(d, 'builtins.classmethod'): - removed.append(i) - dec.func.is_class = True - dec.var.is_classmethod = True - self.check_decorated_function_is_method('classmethod', dec) - elif (refers_to_fullname(d, 'builtins.property') or - refers_to_fullname(d, 'abc.abstractproperty')): - removed.append(i) - dec.func.is_property = True - dec.var.is_property = True - if refers_to_fullname(d, 'abc.abstractproperty'): - dec.func.is_abstract = True - self.check_decorated_function_is_method('property', dec) - if len(dec.func.arguments) > 1: - self.fail('Too many arguments', dec.func) - elif refers_to_fullname(d, 'typing.no_type_check'): - dec.var.type = AnyType() - no_type_check = True - for i in reversed(removed): - del dec.decorators[i] - if not dec.is_overload or dec.var.is_property: - if self.is_func_scope(): - self.add_symbol(dec.var.name(), SymbolTableNode(LDEF, dec), - dec) - elif self.type: - dec.var.info = self.type - dec.var.is_initialized_in_class = True - self.add_symbol(dec.var.name(), SymbolTableNode(MDEF, dec), - dec) - if not no_type_check: - dec.func.accept(self) - if dec.decorators and dec.var.is_property: - self.fail('Decorated property not supported', dec) - - def check_decorated_function_is_method(self, decorator: str, - context: Context) -> None: - if not self.type or self.is_func_scope(): - self.fail("'%s' used with a non-method" % decorator, context) - - def visit_expression_stmt(self, s: ExpressionStmt) -> None: - s.expr.accept(self) - - def visit_return_stmt(self, s: ReturnStmt) -> None: - if not self.is_func_scope(): - self.fail("'return' outside function", s) - if s.expr: - s.expr.accept(self) - - def visit_raise_stmt(self, s: RaiseStmt) -> None: - if s.expr: - s.expr.accept(self) - if s.from_expr: - s.from_expr.accept(self) - - def visit_assert_stmt(self, s: AssertStmt) -> None: - if s.expr: - s.expr.accept(self) - - def visit_operator_assignment_stmt(self, - s: OperatorAssignmentStmt) -> None: - s.lvalue.accept(self) - s.rvalue.accept(self) - if (isinstance(s.lvalue, NameExpr) and s.lvalue.name == '__all__' and - s.lvalue.kind == GDEF and isinstance(s.rvalue, (ListExpr, TupleExpr))): - self.add_exports(*s.rvalue.items) - - def visit_while_stmt(self, s: WhileStmt) -> None: - s.expr.accept(self) - self.loop_depth += 1 - s.body.accept(self) - self.loop_depth -= 1 - self.visit_block_maybe(s.else_body) - - def visit_for_stmt(self, s: ForStmt) -> None: - s.expr.accept(self) - - # Bind index variables and check if they define new names. - self.analyze_lvalue(s.index) - - self.loop_depth += 1 - self.visit_block(s.body) - self.loop_depth -= 1 - - self.visit_block_maybe(s.else_body) - - def visit_break_stmt(self, s: BreakStmt) -> None: - if self.loop_depth == 0: - self.fail("'break' outside loop", s, True, blocker=True) - - def visit_continue_stmt(self, s: ContinueStmt) -> None: - if self.loop_depth == 0: - self.fail("'continue' outside loop", s, True, blocker=True) - - def visit_if_stmt(self, s: IfStmt) -> None: - infer_reachability_of_if_statement(s, pyversion=self.options.python_version) - for i in range(len(s.expr)): - s.expr[i].accept(self) - self.visit_block(s.body[i]) - self.visit_block_maybe(s.else_body) - - def visit_try_stmt(self, s: TryStmt) -> None: - self.analyze_try_stmt(s, self) - - def analyze_try_stmt(self, s: TryStmt, visitor: NodeVisitor, - add_global: bool = False) -> None: - s.body.accept(visitor) - for type, var, handler in zip(s.types, s.vars, s.handlers): - if type: - type.accept(visitor) - if var: - self.analyze_lvalue(var, add_global=add_global) - handler.accept(visitor) - if s.else_body: - s.else_body.accept(visitor) - if s.finally_body: - s.finally_body.accept(visitor) - - def visit_with_stmt(self, s: WithStmt) -> None: - for e, n in zip(s.expr, s.target): - e.accept(self) - if n: - self.analyze_lvalue(n) - self.visit_block(s.body) - - def visit_del_stmt(self, s: DelStmt) -> None: - s.expr.accept(self) - if not self.is_valid_del_target(s.expr): - self.fail('Invalid delete target', s) - - def is_valid_del_target(self, s: Node) -> bool: - if isinstance(s, (IndexExpr, NameExpr, MemberExpr)): - return True - elif isinstance(s, TupleExpr): - return all(self.is_valid_del_target(item) for item in s.items) - - def visit_global_decl(self, g: GlobalDecl) -> None: - for name in g.names: - if name in self.nonlocal_decls[-1]: - self.fail("Name '{}' is nonlocal and global".format(name), g) - self.global_decls[-1].add(name) - - def visit_nonlocal_decl(self, d: NonlocalDecl) -> None: - if not self.is_func_scope(): - self.fail("nonlocal declaration not allowed at module level", d) - else: - for name in d.names: - for table in reversed(self.locals[:-1]): - if table is not None and name in table: - break - else: - self.fail("No binding for nonlocal '{}' found".format(name), d) - - if self.locals[-1] is not None and name in self.locals[-1]: - self.fail("Name '{}' is already defined in local " - "scope before nonlocal declaration".format(name), d) - - if name in self.global_decls[-1]: - self.fail("Name '{}' is nonlocal and global".format(name), d) - self.nonlocal_decls[-1].add(name) - - def visit_print_stmt(self, s: PrintStmt) -> None: - for arg in s.args: - arg.accept(self) - if s.target: - s.target.accept(self) - - def visit_exec_stmt(self, s: ExecStmt) -> None: - s.expr.accept(self) - if s.variables1: - s.variables1.accept(self) - if s.variables2: - s.variables2.accept(self) - - # - # Expressions - # - - def visit_name_expr(self, expr: NameExpr) -> None: - n = self.lookup(expr.name, expr) - if n: - if n.kind == BOUND_TVAR: - self.fail("'{}' is a type variable and only valid in type " - "context".format(expr.name), expr) - else: - expr.kind = n.kind - expr.node = n.node - expr.fullname = n.fullname - - def visit_super_expr(self, expr: SuperExpr) -> None: - if not self.type: - self.fail('"super" used outside class', expr) - return - expr.info = self.type - - def visit_tuple_expr(self, expr: TupleExpr) -> None: - for item in expr.items: - item.accept(self) - - def visit_list_expr(self, expr: ListExpr) -> None: - for item in expr.items: - item.accept(self) - - def visit_set_expr(self, expr: SetExpr) -> None: - for item in expr.items: - item.accept(self) - - def visit_dict_expr(self, expr: DictExpr) -> None: - for key, value in expr.items: - key.accept(self) - value.accept(self) - - def visit_star_expr(self, expr: StarExpr) -> None: - if not expr.valid: - self.fail('Can use starred expression only as assignment target', expr) - else: - expr.expr.accept(self) - - def visit_yield_from_expr(self, e: YieldFromExpr) -> None: - if not self.is_func_scope(): # not sure - self.fail("'yield from' outside function", e, True, blocker=True) - else: - self.function_stack[-1].is_generator = True - if e.expr: - e.expr.accept(self) - - def visit_call_expr(self, expr: CallExpr) -> None: - """Analyze a call expression. - - Some call expressions are recognized as special forms, including - cast(...) and Any(...). - """ - expr.callee.accept(self) - if refers_to_fullname(expr.callee, 'typing.cast'): - # Special form cast(...). - if not self.check_fixed_args(expr, 2, 'cast'): - return - # Translate first argument to an unanalyzed type. - try: - target = expr_to_unanalyzed_type(expr.args[0]) - except TypeTranslationError: - self.fail('Cast target is not a type', expr) - return - # Piggyback CastExpr object to the CallExpr object; it takes - # precedence over the CallExpr semantics. - expr.analyzed = CastExpr(expr.args[1], target) - expr.analyzed.line = expr.line - expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'builtins.reveal_type'): - if not self.check_fixed_args(expr, 1, 'reveal_type'): - return - expr.analyzed = RevealTypeExpr(expr.args[0]) - expr.analyzed.line = expr.line - expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'typing.Any'): - # Special form Any(...). - if not self.check_fixed_args(expr, 1, 'Any'): - return - expr.analyzed = CastExpr(expr.args[0], AnyType()) - expr.analyzed.line = expr.line - expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'typing._promote'): - # Special form _promote(...). - if not self.check_fixed_args(expr, 1, '_promote'): - return - # Translate first argument to an unanalyzed type. - try: - target = expr_to_unanalyzed_type(expr.args[0]) - except TypeTranslationError: - self.fail('Argument 1 to _promote is not a type', expr) - return - expr.analyzed = PromoteExpr(target) - expr.analyzed.line = expr.line - expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'builtins.dict'): - expr.analyzed = self.translate_dict_call(expr) - else: - # Normal call expression. - for a in expr.args: - a.accept(self) - - if (isinstance(expr.callee, MemberExpr) and - isinstance(expr.callee.expr, NameExpr) and - expr.callee.expr.name == '__all__' and - expr.callee.expr.kind == GDEF and - expr.callee.name in ('append', 'extend')): - if expr.callee.name == 'append' and expr.args: - self.add_exports(expr.args[0]) - elif (expr.callee.name == 'extend' and expr.args and - isinstance(expr.args[0], (ListExpr, TupleExpr))): - self.add_exports(*expr.args[0].items) - - def translate_dict_call(self, call: CallExpr) -> Optional[DictExpr]: - """Translate 'dict(x=y, ...)' to {'x': y, ...}. - - For other variants of dict(...), return None. - """ - if not call.args: - return None - if not all(kind == ARG_NAMED for kind in call.arg_kinds): - # Must still accept those args. - for a in call.args: - a.accept(self) - return None - expr = DictExpr([(StrExpr(key), value) - for key, value in zip(call.arg_names, call.args)]) - expr.set_line(call) - expr.accept(self) - return expr - - def check_fixed_args(self, expr: CallExpr, numargs: int, - name: str) -> bool: - """Verify that expr has specified number of positional args. - - Return True if the arguments are valid. - """ - s = 's' - if numargs == 1: - s = '' - if len(expr.args) != numargs: - self.fail("'%s' expects %d argument%s" % (name, numargs, s), - expr) - return False - if expr.arg_kinds != [ARG_POS] * numargs: - self.fail("'%s' must be called with %s positional argument%s" % - (name, numargs, s), expr) - return False - return True - - def visit_member_expr(self, expr: MemberExpr) -> None: - base = expr.expr - base.accept(self) - # Bind references to module attributes. - if isinstance(base, RefExpr) and base.kind == MODULE_REF: - file = cast(MypyFile, base.node) - n = file.names.get(expr.name, None) if file is not None else None - if n: - n = self.normalize_type_alias(n, expr) - if not n: - return - expr.kind = n.kind - expr.fullname = n.fullname - expr.node = n.node - else: - # We only catch some errors here; the rest will be - # catched during type checking. - # - # This way we can report a larger number of errors in - # one type checker run. If we reported errors here, - # the build would terminate after semantic analysis - # and we wouldn't be able to report any type errors. - full_name = '%s.%s' % (file.fullname() if file is not None else None, expr.name) - if full_name in obsolete_name_mapping: - self.fail("Module has no attribute %r (it's now called %r)" % ( - expr.name, obsolete_name_mapping[full_name]), expr) - - def visit_op_expr(self, expr: OpExpr) -> None: - expr.left.accept(self) - expr.right.accept(self) - - def visit_comparison_expr(self, expr: ComparisonExpr) -> None: - for operand in expr.operands: - operand.accept(self) - - def visit_unary_expr(self, expr: UnaryExpr) -> None: - expr.expr.accept(self) - - def visit_index_expr(self, expr: IndexExpr) -> None: - expr.base.accept(self) - if refers_to_class_or_function(expr.base): - # Special form -- type application. - # Translate index to an unanalyzed type. - types = [] # type: List[Type] - if isinstance(expr.index, TupleExpr): - items = expr.index.items - else: - items = [expr.index] - for item in items: - try: - typearg = expr_to_unanalyzed_type(item) - except TypeTranslationError: - self.fail('Type expected within [...]', expr) - return - typearg = self.anal_type(typearg) - types.append(typearg) - expr.analyzed = TypeApplication(expr.base, types) - expr.analyzed.line = expr.line - else: - expr.index.accept(self) - - def visit_slice_expr(self, expr: SliceExpr) -> None: - if expr.begin_index: - expr.begin_index.accept(self) - if expr.end_index: - expr.end_index.accept(self) - if expr.stride: - expr.stride.accept(self) - - def visit_cast_expr(self, expr: CastExpr) -> None: - expr.expr.accept(self) - expr.type = self.anal_type(expr.type) - - def visit_reveal_type_expr(self, expr: RevealTypeExpr) -> None: - expr.expr.accept(self) - - def visit_type_application(self, expr: TypeApplication) -> None: - expr.expr.accept(self) - for i in range(len(expr.types)): - expr.types[i] = self.anal_type(expr.types[i]) - - def visit_list_comprehension(self, expr: ListComprehension) -> None: - expr.generator.accept(self) - - def visit_set_comprehension(self, expr: SetComprehension) -> None: - expr.generator.accept(self) - - def visit_dictionary_comprehension(self, expr: DictionaryComprehension) -> None: - self.enter() - self.analyze_comp_for(expr) - expr.key.accept(self) - expr.value.accept(self) - self.leave() - self.analyze_comp_for_2(expr) - - def visit_generator_expr(self, expr: GeneratorExpr) -> None: - self.enter() - self.analyze_comp_for(expr) - expr.left_expr.accept(self) - self.leave() - self.analyze_comp_for_2(expr) - - def analyze_comp_for(self, expr: Union[GeneratorExpr, - DictionaryComprehension]) -> None: - """Analyses the 'comp_for' part of comprehensions (part 1). - - That is the part after 'for' in (x for x in l if p). This analyzes - variables and conditions which are analyzed in a local scope. - """ - for i, (index, sequence, conditions) in enumerate(zip(expr.indices, - expr.sequences, - expr.condlists)): - if i > 0: - sequence.accept(self) - # Bind index variables. - self.analyze_lvalue(index) - for cond in conditions: - cond.accept(self) - - def analyze_comp_for_2(self, expr: Union[GeneratorExpr, - DictionaryComprehension]) -> None: - """Analyses the 'comp_for' part of comprehensions (part 2). - - That is the part after 'for' in (x for x in l if p). This analyzes - the 'l' part which is analyzed in the surrounding scope. - """ - expr.sequences[0].accept(self) - - def visit_func_expr(self, expr: FuncExpr) -> None: - self.analyze_function(expr) - - def visit_conditional_expr(self, expr: ConditionalExpr) -> None: - expr.if_expr.accept(self) - expr.cond.accept(self) - expr.else_expr.accept(self) - - def visit_backquote_expr(self, expr: BackquoteExpr) -> None: - expr.expr.accept(self) - - def visit__promote_expr(self, expr: PromoteExpr) -> None: - expr.type = self.anal_type(expr.type) - - def visit_yield_expr(self, expr: YieldExpr) -> None: - if not self.is_func_scope(): - self.fail("'yield' outside function", expr, True, blocker=True) - else: - self.function_stack[-1].is_generator = True - if expr.expr: - expr.expr.accept(self) - - # - # Helpers - # - - def lookup(self, name: str, ctx: Context) -> SymbolTableNode: - """Look up an unqualified name in all active namespaces.""" - # 1a. Name declared using 'global x' takes precedence - if name in self.global_decls[-1]: - if name in self.globals: - return self.globals[name] - else: - self.name_not_defined(name, ctx) - return None - # 1b. Name declared using 'nonlocal x' takes precedence - if name in self.nonlocal_decls[-1]: - for table in reversed(self.locals[:-1]): - if table is not None and name in table: - return table[name] - else: - self.name_not_defined(name, ctx) - return None - # 2. Class attributes (if within class definition) - if self.is_class_scope() and name in self.type.names: - return self.type[name] - # 3. Local (function) scopes - for table in reversed(self.locals): - if table is not None and name in table: - return table[name] - # 4. Current file global scope - if name in self.globals: - return self.globals[name] - # 5. Builtins - b = self.globals.get('__builtins__', None) - if b: - table = cast(MypyFile, b.node).names - if name in table: - if name[0] == "_" and name[1] != "_": - self.name_not_defined(name, ctx) - return None - node = table[name] - return node - # Give up. - self.name_not_defined(name, ctx) - self.check_for_obsolete_short_name(name, ctx) - return None - - def check_for_obsolete_short_name(self, name: str, ctx: Context) -> None: - matches = [obsolete_name - for obsolete_name in obsolete_name_mapping - if obsolete_name.rsplit('.', 1)[-1] == name] - if len(matches) == 1: - self.note("(Did you mean '{}'?)".format(obsolete_name_mapping[matches[0]]), ctx) - - def lookup_qualified(self, name: str, ctx: Context) -> SymbolTableNode: - if '.' not in name: - return self.lookup(name, ctx) - else: - parts = name.split('.') - n = self.lookup(parts[0], ctx) # type: SymbolTableNode - if n: - for i in range(1, len(parts)): - if isinstance(n.node, TypeInfo): - if n.node.mro is None: - # We haven't yet analyzed the class `n.node`. Fall back to direct - # lookup in the names declared directly under it, without its base - # classes. This can happen when we have a forward reference to a - # nested class, and the reference is bound before the outer class - # has been fully semantically analyzed. - # - # A better approach would be to introduce a new analysis pass or - # to move things around between passes, but this unblocks a common - # use case even though this is a little limited in case there is - # inheritance involved. - result = n.node.names.get(parts[i]) - else: - result = n.node.get(parts[i]) - n = result - elif isinstance(n.node, MypyFile): - n = n.node.names.get(parts[i], None) - # TODO: What if node is Var or FuncDef? - if not n: - self.name_not_defined(name, ctx) - break - if n: - n = self.normalize_type_alias(n, ctx) - return n - - def builtin_type(self, fully_qualified_name: str) -> Instance: - node = self.lookup_fully_qualified(fully_qualified_name) - info = cast(TypeInfo, node.node) - return Instance(info, []) - - def lookup_fully_qualified(self, name: str) -> SymbolTableNode: - """Lookup a fully qualified name. - - Assume that the name is defined. This happens in the global namespace -- the local - module namespace is ignored. - """ - assert '.' in name - parts = name.split('.') - n = self.modules[parts[0]] - for i in range(1, len(parts) - 1): - n = cast(MypyFile, n.names[parts[i]].node) - return n.names[parts[-1]] - - def lookup_fully_qualified_or_none(self, name: str) -> SymbolTableNode: - """Lookup a fully qualified name. - - Assume that the name is defined. This happens in the global namespace -- the local - module namespace is ignored. - """ - assert '.' in name - parts = name.split('.') - n = self.modules[parts[0]] - for i in range(1, len(parts) - 1): - next_sym = n.names.get(parts[i]) - if not next_sym: - return None - n = cast(MypyFile, next_sym.node) - return n.names.get(parts[-1]) - - def qualified_name(self, n: str) -> str: - return self.cur_mod_id + '.' + n - - def enter(self) -> None: - self.locals.append(SymbolTable()) - self.global_decls.append(set()) - self.nonlocal_decls.append(set()) - # -1 since entering block will increment this to 0. - self.block_depth.append(-1) - - def leave(self) -> None: - self.locals.pop() - self.global_decls.pop() - self.nonlocal_decls.pop() - self.block_depth.pop() - - def is_func_scope(self) -> bool: - return self.locals[-1] is not None - - def is_class_scope(self) -> bool: - return self.type is not None and not self.is_func_scope() - - def add_symbol(self, name: str, node: SymbolTableNode, - context: Context) -> None: - if self.is_func_scope(): - if name in self.locals[-1]: - # Flag redefinition unless this is a reimport of a module. - if not (node.kind == MODULE_REF and - self.locals[-1][name].node == node.node): - self.name_already_defined(name, context) - self.locals[-1][name] = node - elif self.type: - self.type.names[name] = node - else: - existing = self.globals.get(name) - if existing and (not isinstance(node.node, MypyFile) or - existing.node != node.node) and existing.kind != UNBOUND_IMPORTED: - # Modules can be imported multiple times to support import - # of multiple submodules of a package (e.g. a.x and a.y). - ok = False - # Only report an error if the symbol collision provides a different type. - if existing.type and node.type and is_same_type(existing.type, node.type): - ok = True - if not ok: - self.name_already_defined(name, context) - self.globals[name] = node - - def add_var(self, v: Var, ctx: Context) -> None: - if self.is_func_scope(): - self.add_local(v, ctx) - else: - self.globals[v.name()] = SymbolTableNode(GDEF, v, self.cur_mod_id) - v._fullname = self.qualified_name(v.name()) - - def add_local(self, node: Union[Var, FuncBase], ctx: Context) -> None: - name = node.name() - if name in self.locals[-1]: - self.name_already_defined(name, ctx) - node._fullname = name - self.locals[-1][name] = SymbolTableNode(LDEF, node) - - def add_exports(self, *exps: Node) -> None: - for exp in exps: - if isinstance(exp, StrExpr): - self.all_exports.add(exp.value) - - def check_no_global(self, n: str, ctx: Context, - is_overloaded_func: bool = False) -> None: - if n in self.globals: - prev_is_overloaded = isinstance(self.globals[n], OverloadedFuncDef) - if is_overloaded_func and prev_is_overloaded: - self.fail("Nonconsecutive overload {} found".format(n), ctx) - elif prev_is_overloaded: - self.fail("Definition of '{}' missing 'overload'".format(n), ctx) - else: - self.name_already_defined(n, ctx) - - def name_not_defined(self, name: str, ctx: Context) -> None: - message = "Name '{}' is not defined".format(name) - extra = self.undefined_name_extra_info(name) - if extra: - message += ' {}'.format(extra) - self.fail(message, ctx) - - def name_already_defined(self, name: str, ctx: Context) -> None: - self.fail("Name '{}' already defined".format(name), ctx) - - def fail(self, msg: str, ctx: Context, serious: bool = False, *, - blocker: bool = False) -> None: - if (not serious and - not self.options.check_untyped_defs and - self.function_stack and - self.function_stack[-1].is_dynamic()): - return - self.errors.report(ctx.get_line(), msg, blocker=blocker) - - def note(self, msg: str, ctx: Context) -> None: - if (not self.options.check_untyped_defs and - self.function_stack and - self.function_stack[-1].is_dynamic()): - return - self.errors.report(ctx.get_line(), msg, severity='note') - - def undefined_name_extra_info(self, fullname: str) -> Optional[str]: - if fullname in obsolete_name_mapping: - return "(it's now called '{}')".format(obsolete_name_mapping[fullname]) - else: - return None - - def accept(self, node: Node) -> None: - try: - node.accept(self) - except Exception as err: - report_internal_error(err, self.errors.file, node.line) - - -class FirstPass(NodeVisitor): - """First phase of semantic analysis. - - See docstring of 'analyze()' below for a description of what this does. - """ - - def __init__(self, sem: SemanticAnalyzer) -> None: - self.sem = sem - self.pyversion = sem.options.python_version - - def analyze(self, file: MypyFile, fnam: str, mod_id: str) -> None: - """Perform the first analysis pass. - - Populate module global table. Resolve the full names of - definitions not nested within functions and construct type - info structures, but do not resolve inter-definition - references such as base classes. - - Also add implicit definitions such as __name__. - - In this phase we don't resolve imports. For 'from ... import', - we generate dummy symbol table nodes for the imported names, - and these will get resolved in later phases of semantic - analysis. - """ - sem = self.sem - sem.cur_mod_id = mod_id - sem.errors.set_file(fnam) - sem.globals = SymbolTable() - sem.global_decls = [set()] - sem.nonlocal_decls = [set()] - sem.block_depth = [0] - - defs = file.defs - - # Add implicit definitions of module '__name__' etc. - for name, t in implicit_module_attrs.items(): - v = Var(name, UnboundType(t)) - v._fullname = self.sem.qualified_name(name) - self.sem.globals[name] = SymbolTableNode(GDEF, v, self.sem.cur_mod_id) - - for d in defs: - d.accept(self) - - # Add implicit definition of literals/keywords to builtins, as we - # cannot define a variable with them explicitly. - if mod_id == 'builtins': - literal_types = [ - ('None', NoneTyp()), - # reveal_type is a mypy-only function that gives an error with the type of its arg - ('reveal_type', AnyType()), - ] # type: List[Tuple[str, Type]] - - # TODO(ddfisher): This guard is only needed because mypy defines - # fake builtins for its tests which often don't define bool. If - # mypy is fast enough that we no longer need those, this - # conditional check should be removed. - if 'bool' in self.sem.globals: - bool_type = self.sem.named_type('bool') - literal_types.extend([ - ('True', bool_type), - ('False', bool_type), - ('__debug__', bool_type), - ]) - - for name, typ in literal_types: - v = Var(name, typ) - v._fullname = self.sem.qualified_name(name) - self.sem.globals[name] = SymbolTableNode(GDEF, v, self.sem.cur_mod_id) - - def visit_block(self, b: Block) -> None: - if b.is_unreachable: - return - self.sem.block_depth[-1] += 1 - for node in b.body: - node.accept(self) - self.sem.block_depth[-1] -= 1 - - def visit_assignment_stmt(self, s: AssignmentStmt) -> None: - for lval in s.lvalues: - self.analyze_lvalue(lval, explicit_type=s.type is not None) - - def visit_func_def(self, func: FuncDef) -> None: - sem = self.sem - func.is_conditional = sem.block_depth[-1] > 0 - func._fullname = sem.qualified_name(func.name()) - if func.name() in sem.globals: - # Already defined in this module. - original_sym = sem.globals[func.name()] - if original_sym.kind == UNBOUND_IMPORTED: - # Ah this is an imported name. We can't resolve them now, so we'll postpone - # this until the main phase of semantic analysis. - return - original_def = original_sym.node - if sem.is_conditional_func(original_def, func): - # Conditional function definition -- multiple defs are ok. - func.original_def = cast(FuncDef, original_def) - else: - # Report error. - sem.check_no_global(func.name(), func) - else: - sem.globals[func.name()] = SymbolTableNode(GDEF, func, sem.cur_mod_id) - - def visit_overloaded_func_def(self, func: OverloadedFuncDef) -> None: - self.sem.check_no_global(func.name(), func, True) - func._fullname = self.sem.qualified_name(func.name()) - self.sem.globals[func.name()] = SymbolTableNode(GDEF, func, - self.sem.cur_mod_id) - - def visit_class_def(self, cdef: ClassDef) -> None: - self.sem.check_no_global(cdef.name, cdef) - cdef.fullname = self.sem.qualified_name(cdef.name) - info = TypeInfo(SymbolTable(), cdef) - info.set_line(cdef.line) - cdef.info = info - self.sem.globals[cdef.name] = SymbolTableNode(GDEF, info, - self.sem.cur_mod_id) - self.process_nested_classes(cdef) - - def process_nested_classes(self, outer_def: ClassDef) -> None: - for node in outer_def.defs.body: - if isinstance(node, ClassDef): - node.info = TypeInfo(SymbolTable(), node) - if outer_def.fullname: - node.info._fullname = outer_def.fullname + '.' + node.info.name() - else: - node.info._fullname = node.info.name() - symbol = SymbolTableNode(MDEF, node.info) - outer_def.info.names[node.name] = symbol - self.process_nested_classes(node) - - def visit_import_from(self, node: ImportFrom) -> None: - # We can't bind module names during the first pass, as the target module might be - # unprocessed. However, we add dummy unbound imported names to the symbol table so - # that we at least know that the name refers to a module. - node.is_top_level = True - for name, as_name in node.names: - imported_name = as_name or name - if imported_name not in self.sem.globals: - self.sem.add_symbol(imported_name, SymbolTableNode(UNBOUND_IMPORTED, None), node) - - def visit_import(self, node: Import) -> None: - node.is_top_level = True - # This is similar to visit_import_from -- see the comment there. - for id, as_id in node.ids: - imported_id = as_id or id - if imported_id not in self.sem.globals: - self.sem.add_symbol(imported_id, SymbolTableNode(UNBOUND_IMPORTED, None), node) - else: - # If the previous symbol is a variable, this should take precedence. - self.sem.globals[imported_id] = SymbolTableNode(UNBOUND_IMPORTED, None) - - def visit_import_all(self, node: ImportAll) -> None: - node.is_top_level = True - - def visit_while_stmt(self, s: WhileStmt) -> None: - s.body.accept(self) - if s.else_body: - s.else_body.accept(self) - - def visit_for_stmt(self, s: ForStmt) -> None: - self.analyze_lvalue(s.index) - s.body.accept(self) - if s.else_body: - s.else_body.accept(self) - - def visit_with_stmt(self, s: WithStmt) -> None: - for n in s.target: - if n: - self.analyze_lvalue(n) - s.body.accept(self) - - def visit_decorator(self, d: Decorator) -> None: - d.var._fullname = self.sem.qualified_name(d.var.name()) - self.sem.add_symbol(d.var.name(), SymbolTableNode(GDEF, d.var), d) - - def visit_if_stmt(self, s: IfStmt) -> None: - infer_reachability_of_if_statement(s, pyversion=self.pyversion) - for node in s.body: - node.accept(self) - if s.else_body: - s.else_body.accept(self) - - def visit_try_stmt(self, s: TryStmt) -> None: - self.sem.analyze_try_stmt(s, self, add_global=True) - - def analyze_lvalue(self, lvalue: Node, explicit_type: bool = False) -> None: - self.sem.analyze_lvalue(lvalue, add_global=True, explicit_type=explicit_type) - - -class ThirdPass(TraverserVisitor): - """The third and final pass of semantic analysis. - - Check type argument counts and values of generic types, and perform some - straightforward type inference. - """ - - def __init__(self, modules: Dict[str, MypyFile], errors: Errors) -> None: - self.modules = modules - self.errors = errors - - def visit_file(self, file_node: MypyFile, fnam: str) -> None: - self.errors.set_file(fnam) - self.accept(file_node) - - def accept(self, node: Node) -> None: - try: - node.accept(self) - except Exception as err: - report_internal_error(err, self.errors.file, node.line) - - def visit_block(self, b: Block) -> None: - if b.is_unreachable: - return - super().visit_block(b) - - def visit_func_def(self, fdef: FuncDef) -> None: - self.errors.push_function(fdef.name()) - self.analyze(fdef.type) - super().visit_func_def(fdef) - self.errors.pop_function() - - def visit_class_def(self, tdef: ClassDef) -> None: - for type in tdef.info.bases: - self.analyze(type) - # Recompute MRO now that we have analyzed all modules, to pick - # up superclasses of bases imported from other modules in an - # import loop. (Only do so if we succeeded the first time.) - if tdef.info.mro: - tdef.info.mro = [] # Force recomputation - calculate_class_mro(tdef, self.fail) - super().visit_class_def(tdef) - - def visit_decorator(self, dec: Decorator) -> None: - """Try to infer the type of the decorated function. - - This lets us resolve references to decorated functions during - type checking when there are cyclic imports, as otherwise the - type might not be available when we need it. - - This basically uses a simple special-purpose type inference - engine just for decorators. - """ - super().visit_decorator(dec) - if dec.var.is_property: - # Decorators are expected to have a callable type (it's a little odd). - if dec.func.type is None: - dec.var.type = CallableType( - [AnyType()], - [ARG_POS], - [None], - AnyType(), - self.builtin_type('function'), - name=dec.var.name()) - elif isinstance(dec.func.type, CallableType): - dec.var.type = dec.func.type - return - decorator_preserves_type = True - for expr in dec.decorators: - preserve_type = False - if isinstance(expr, RefExpr) and isinstance(expr.node, FuncDef): - if is_identity_signature(expr.node.type): - preserve_type = True - if not preserve_type: - decorator_preserves_type = False - break - if decorator_preserves_type: - # No non-identity decorators left. We can trivially infer the type - # of the function here. - dec.var.type = function_type(dec.func, self.builtin_type('function')) - if dec.decorators: - if returns_any_if_called(dec.decorators[0]): - # The outermost decorator will return Any so we know the type of the - # decorated function. - dec.var.type = AnyType() - sig = find_fixed_callable_return(dec.decorators[0]) - if sig: - # The outermost decorator always returns the same kind of function, - # so we know that this is the type of the decoratored function. - orig_sig = function_type(dec.func, self.builtin_type('function')) - sig.name = orig_sig.items()[0].name - dec.var.type = sig - - def visit_assignment_stmt(self, s: AssignmentStmt) -> None: - self.analyze(s.type) - super().visit_assignment_stmt(s) - - def visit_cast_expr(self, e: CastExpr) -> None: - self.analyze(e.type) - super().visit_cast_expr(e) - - def visit_reveal_type_expr(self, e: RevealTypeExpr) -> None: - super().visit_reveal_type_expr(e) - - def visit_type_application(self, e: TypeApplication) -> None: - for type in e.types: - self.analyze(type) - super().visit_type_application(e) - - # Helpers - - def analyze(self, type: Type) -> None: - if type: - analyzer = TypeAnalyserPass3(self.fail) - type.accept(analyzer) - - def fail(self, msg: str, ctx: Context) -> None: - self.errors.report(ctx.get_line(), msg) - - def builtin_type(self, name: str, args: List[Type] = None) -> Instance: - names = self.modules['builtins'] - sym = names.names[name] - assert isinstance(sym.node, TypeInfo) - return Instance(sym.node, args or []) - - -def self_type(typ: TypeInfo) -> Union[Instance, TupleType]: - """For a non-generic type, return instance type representing the type. - For a generic G type with parameters T1, .., Tn, return G[T1, ..., Tn]. - """ - tv = [] # type: List[Type] - for i in range(len(typ.type_vars)): - tv.append(TypeVarType(typ.defn.type_vars[i])) - inst = Instance(typ, tv) - if typ.tuple_type is None: - return inst - else: - return TupleType(typ.tuple_type.items, inst) - - -def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: - if isinstance(sig, CallableType): - return replace_leading_arg_type(sig, new) - else: - sig = cast(Overloaded, sig) - return Overloaded([cast(CallableType, replace_implicit_first_type(i, new)) - for i in sig.items()]) - - -def set_callable_name(sig: Type, fdef: FuncDef) -> Type: - if isinstance(sig, FunctionLike): - if fdef.info: - return sig.with_name( - '"{}" of "{}"'.format(fdef.name(), fdef.info.name())) - else: - return sig.with_name('"{}"'.format(fdef.name())) - else: - return sig - - -def refers_to_fullname(node: Node, fullname: str) -> bool: - """Is node a name or member expression with the given full name?""" - return isinstance(node, RefExpr) and node.fullname == fullname - - -def refers_to_class_or_function(node: Node) -> bool: - """Does semantically analyzed node refer to a class?""" - return (isinstance(node, RefExpr) and - isinstance(node.node, (TypeInfo, FuncDef, OverloadedFuncDef))) - - -def calculate_class_mro(defn: ClassDef, fail: Callable[[str, Context], None]) -> None: - try: - defn.info.calculate_mro() - except MroError: - fail("Cannot determine consistent method resolution order " - '(MRO) for "%s"' % defn.name, defn) - defn.info.mro = [] - # The property of falling back to Any is inherited. - defn.info.fallback_to_any = any(baseinfo.fallback_to_any for baseinfo in defn.info.mro) - - -def find_duplicate(list: List[T]) -> T: - """If the list has duplicates, return one of the duplicates. - - Otherwise, return None. - """ - for i in range(1, len(list)): - if list[i] in list[:i]: - return list[i] - return None - - -def disable_typevars(nodes: List[SymbolTableNode]) -> None: - for node in nodes: - assert node.kind in (BOUND_TVAR, UNBOUND_TVAR) - node.kind = UNBOUND_TVAR - - -def enable_typevars(nodes: List[SymbolTableNode]) -> None: - for node in nodes: - assert node.kind in (BOUND_TVAR, UNBOUND_TVAR) - node.kind = BOUND_TVAR - - -def remove_imported_names_from_symtable(names: SymbolTable, - module: str) -> None: - """Remove all imported names from the symbol table of a module.""" - removed = [] # type: List[str] - for name, node in names.items(): - if node.node is None: - continue - fullname = node.node.fullname() - prefix = fullname[:fullname.rfind('.')] - if prefix != module: - removed.append(name) - for name in removed: - del names[name] - - -def infer_reachability_of_if_statement(s: IfStmt, - pyversion: Tuple[int, int]) -> None: - for i in range(len(s.expr)): - result = infer_if_condition_value(s.expr[i], pyversion) - if result == ALWAYS_FALSE: - # The condition is always false, so we skip the if/elif body. - mark_block_unreachable(s.body[i]) - elif result == ALWAYS_TRUE: - # This condition is always true, so all of the remaining - # elif/else bodies will never be executed. - for body in s.body[i + 1:]: - mark_block_unreachable(body) - if s.else_body: - mark_block_unreachable(s.else_body) - break - - -def infer_if_condition_value(expr: Node, pyversion: Tuple[int, int]) -> int: - """Infer whether if condition is always true/false. - - Return ALWAYS_TRUE if always true, ALWAYS_FALSE if always false, - and TRUTH_VALUE_UNKNOWN otherwise. - """ - name = '' - negated = False - alias = expr - if isinstance(alias, UnaryExpr): - if alias.op == 'not': - expr = alias.expr - negated = True - if isinstance(expr, NameExpr): - name = expr.name - elif isinstance(expr, MemberExpr): - name = expr.name - result = TRUTH_VALUE_UNKNOWN - if name == 'PY2': - result = ALWAYS_TRUE if pyversion[0] == 2 else ALWAYS_FALSE - elif name == 'PY3': - result = ALWAYS_TRUE if pyversion[0] == 3 else ALWAYS_FALSE - elif name == 'MYPY': - result = ALWAYS_TRUE - if negated: - if result == ALWAYS_TRUE: - result = ALWAYS_FALSE - elif result == ALWAYS_FALSE: - result = ALWAYS_TRUE - return result - - -def mark_block_unreachable(block: Block) -> None: - block.is_unreachable = True - block.accept(MarkImportsUnreachableVisitor()) - - -class MarkImportsUnreachableVisitor(TraverserVisitor): - """Visitor that flags all imports nested within a node as unreachable.""" - - def visit_import(self, node: Import) -> None: - node.is_unreachable = True - - def visit_import_from(self, node: ImportFrom) -> None: - node.is_unreachable = True - - def visit_import_all(self, node: ImportAll) -> None: - node.is_unreachable = True - - -def is_identity_signature(sig: Type) -> bool: - """Is type a callable of form T -> T (where T is a type variable)?""" - if isinstance(sig, CallableType) and sig.arg_kinds == [ARG_POS]: - if isinstance(sig.arg_types[0], TypeVarType) and isinstance(sig.ret_type, TypeVarType): - return sig.arg_types[0].id == sig.ret_type.id - return False - - -def returns_any_if_called(expr: Node) -> bool: - """Return True if we can predict that expr will return Any if called. - - This only uses information available during semantic analysis so this - will sometimes return False because of insufficient information (as - type inference hasn't run yet). - """ - if isinstance(expr, RefExpr): - if isinstance(expr.node, FuncDef): - typ = expr.node.type - if typ is None: - # No signature -> default to Any. - return True - # Explicit Any return? - return isinstance(typ, CallableType) and isinstance(typ.ret_type, AnyType) - elif isinstance(expr.node, Var): - typ = expr.node.type - return typ is None or isinstance(typ, AnyType) - elif isinstance(expr, CallExpr): - return returns_any_if_called(expr.callee) - return False - - -def find_fixed_callable_return(expr: Node) -> Optional[CallableType]: - if isinstance(expr, RefExpr): - if isinstance(expr.node, FuncDef): - typ = expr.node.type - if typ: - if isinstance(typ, CallableType) and has_no_typevars(typ.ret_type): - if isinstance(typ.ret_type, CallableType): - return typ.ret_type - elif isinstance(expr, CallExpr): - t = find_fixed_callable_return(expr.callee) - if t: - if isinstance(t.ret_type, CallableType): - return t.ret_type - return None - - -def has_no_typevars(typ: Type) -> bool: - return is_same_type(typ, erase_typevars(typ)) +"""The semantic analyzer. + +Bind names to definitions and do various other simple consistency +checks. For example, consider this program: + + x = 1 + y = x + +Here semantic analysis would detect that the assignment 'x = 1' +defines a new variable, the type of which is to be inferred (in a +later pass; type inference or type checking is not part of semantic +analysis). Also, it would bind both references to 'x' to the same +module-level variable node. The second assignment would also be +analyzed, and the type of 'y' marked as being inferred. + +Semantic analysis is the first analysis pass after parsing, and it is +subdivided into three passes: + + * FirstPass looks up externally visible names defined in a module but + ignores imports and local definitions. It helps enable (some) + cyclic references between modules, such as module 'a' that imports + module 'b' and used names defined in b *and* vice versa. The first + pass can be performed before dependent modules have been processed. + + * SemanticAnalyzer is the second pass. It does the bulk of the work. + It assumes that dependent modules have been semantically analyzed, + up to the second pass, unless there is a import cycle. + + * ThirdPass checks that type argument counts are valid; for example, + it will reject Dict[int]. We don't do this in the second pass, + since we infer the type argument counts of classes during this + pass, and it is possible to refer to classes defined later in a + file, which would not have the type argument count set yet. This + pass also recomputes the method resolution order of each class, in + case one of its bases belongs to a module involved in an import + loop. + +Semantic analysis of types is implemented in module mypy.typeanal. + +TODO: Check if the third pass slows down type checking significantly. + We could probably get rid of it -- for example, we could collect all + analyzed types in a collection and check them without having to + traverse the entire AST. +""" + +from typing import ( + List, Dict, Set, Tuple, cast, Any, overload, TypeVar, Union, Optional, Callable +) + +from mypy.nodes import ( + MypyFile, TypeInfo, Node, AssignmentStmt, FuncDef, OverloadedFuncDef, + ClassDef, Var, GDEF, MODULE_REF, FuncItem, Import, + ImportFrom, ImportAll, Block, LDEF, NameExpr, MemberExpr, + IndexExpr, TupleExpr, ListExpr, ExpressionStmt, ReturnStmt, + RaiseStmt, AssertStmt, OperatorAssignmentStmt, WhileStmt, + ForStmt, BreakStmt, ContinueStmt, IfStmt, TryStmt, WithStmt, DelStmt, + GlobalDecl, SuperExpr, DictExpr, CallExpr, RefExpr, OpExpr, UnaryExpr, + SliceExpr, CastExpr, RevealTypeExpr, TypeApplication, Context, SymbolTable, + SymbolTableNode, BOUND_TVAR, UNBOUND_TVAR, ListComprehension, GeneratorExpr, + FuncExpr, MDEF, FuncBase, Decorator, SetExpr, TypeVarExpr, + StrExpr, BytesExpr, PrintStmt, ConditionalExpr, PromoteExpr, + ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, MroError, type_aliases, + YieldFromExpr, NamedTupleExpr, NonlocalDecl, + SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr, + YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, COVARIANT, CONTRAVARIANT, + IntExpr, FloatExpr, UnicodeExpr, + INVARIANT, UNBOUND_IMPORTED, Expression, EllipsisExpr, NamedTupleTypeInfo +) +from mypy.visitor import NodeVisitor +from mypy.traverser import TraverserVisitor +from mypy.errors import Errors, report_internal_error +from mypy.types import ( + NoneTyp, CallableType, Overloaded, Instance, Type, TypeVarType, AnyType, + FunctionLike, UnboundType, TypeList, ErrorType, TypeVarDef, + replace_leading_arg_type, TupleType, UnionType, StarType, EllipsisType +) +from mypy.nodes import function_type, implicit_module_attrs +from mypy.typeanal import TypeAnalyser, TypeAnalyserPass3, analyze_type_alias +from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError +from mypy.lex import lex +from mypy.parsetype import parse_type +from mypy.sametypes import is_same_type +from mypy.erasetype import erase_typevars +from mypy import defaults +from mypy.options import Options + + +T = TypeVar('T') + + +# Inferred value of an expression. +ALWAYS_TRUE = 0 +ALWAYS_FALSE = 1 +TRUTH_VALUE_UNKNOWN = 2 + +# Map from obsolete name to the current spelling. +obsolete_name_mapping = { + 'typing.Function': 'typing.Callable', + 'typing.typevar': 'typing.TypeVar', +} + +# Hard coded type promotions (shared between all Python versions). +# These add extra ad-hoc edges to the subtyping relation. For example, +# int is considered a subtype of float, even though there is no +# subclass relationship. +TYPE_PROMOTIONS = { + 'builtins.int': 'builtins.float', + 'builtins.float': 'builtins.complex', +} + +# Hard coded type promotions for Python 3. +# +# Note that the bytearray -> bytes promotion is a little unsafe +# as some functions only accept bytes objects. Here convenience +# trumps safety. +TYPE_PROMOTIONS_PYTHON3 = TYPE_PROMOTIONS.copy() +TYPE_PROMOTIONS_PYTHON3.update({ + 'builtins.bytearray': 'builtins.bytes', +}) + +# Hard coded type promotions for Python 2. +# +# These promotions are unsafe, but we are doing them anyway +# for convenience and also for Python 3 compatibility +# (bytearray -> str). +TYPE_PROMOTIONS_PYTHON2 = TYPE_PROMOTIONS.copy() +TYPE_PROMOTIONS_PYTHON2.update({ + 'builtins.str': 'builtins.unicode', + 'builtins.bytearray': 'builtins.str', +}) + +# When analyzing a function, should we analyze the whole function in one go, or +# should we only perform one phase of the analysis? The latter is used for +# nested functions. In the first phase we add the function to the symbol table +# but don't process body. In the second phase we process function body. This +# way we can have mutually recursive nested functions. +FUNCTION_BOTH_PHASES = 0 # Everthing in one go +FUNCTION_FIRST_PHASE_POSTPONE_SECOND = 1 # Add to symbol table but postpone body +FUNCTION_SECOND_PHASE = 2 # Only analyze body + + +class SemanticAnalyzer(NodeVisitor): + """Semantically analyze parsed mypy files. + + The analyzer binds names and does various consistency checks for a + parse tree. Note that type checking is performed as a separate + pass. + + This is the second phase of semantic analysis. + """ + + # Library search paths + lib_path = None # type: List[str] + # Module name space + modules = None # type: Dict[str, MypyFile] + # Global name space for current module + globals = None # type: SymbolTable + # Names declared using "global" (separate set for each scope) + global_decls = None # type: List[Set[str]] + # Names declated using "nonlocal" (separate set for each scope) + nonlocal_decls = None # type: List[Set[str]] + # Local names of function scopes; None for non-function scopes. + locals = None # type: List[SymbolTable] + # Nested block depths of scopes + block_depth = None # type: List[int] + # TypeInfo of directly enclosing class (or None) + type = None # type: TypeInfo + # Stack of outer classes (the second tuple item contains tvars). + type_stack = None # type: List[TypeInfo] + # Type variables that are bound by the directly enclosing class + bound_tvars = None # type: List[SymbolTableNode] + # Stack of type variables that were bound by outer classess + tvar_stack = None # type: List[List[SymbolTableNode]] + # Do weak type checking in this file + weak_opts = set() # type: Set[str] + + # Stack of functions being analyzed + function_stack = None # type: List[FuncItem] + # Stack of next available function type variable ids + next_function_tvar_id_stack = None # type: List[int] + + # Status of postponing analysis of nested function bodies. By using this we + # can have mutually recursive nested functions. Values are FUNCTION_x + # constants. Note that separate phasea are not used for methods. + postpone_nested_functions_stack = None # type: List[int] + # Postponed functions collected if + # postpone_nested_functions_stack[-1] == FUNCTION_FIRST_PHASE_POSTPONE_SECOND. + postponed_functions_stack = None # type: List[List[Node]] + + loop_depth = 0 # Depth of breakable loops + cur_mod_id = '' # Current module id (or None) (phase 2) + is_stub_file = False # Are we analyzing a stub file? + imports = None # type: Set[str] # Imported modules (during phase 2 analysis) + errors = None # type: Errors # Keeps track of generated errors + + def __init__(self, + lib_path: List[str], + errors: Errors, + options: Options) -> None: + """Construct semantic analyzer. + + Use lib_path to search for modules, and report analysis errors + using the Errors instance. + """ + self.locals = [None] + self.imports = set() + self.type = None + self.type_stack = [] + self.bound_tvars = None + self.tvar_stack = [] + self.function_stack = [] + self.next_function_tvar_id_stack = [-1] + self.block_depth = [0] + self.loop_depth = 0 + self.lib_path = lib_path + self.errors = errors + self.modules = {} + self.options = options + self.postpone_nested_functions_stack = [FUNCTION_BOTH_PHASES] + self.postponed_functions_stack = [] + self.all_exports = set() # type: Set[str] + + def visit_file(self, file_node: MypyFile, fnam: str) -> None: + self.errors.set_file(fnam) + self.cur_mod_node = file_node + self.cur_mod_id = file_node.fullname() + self.is_stub_file = fnam.lower().endswith('.pyi') + self.globals = file_node.names + self.weak_opts = file_node.weak_opts + + if 'builtins' in self.modules: + self.globals['__builtins__'] = SymbolTableNode( + MODULE_REF, self.modules['builtins'], self.cur_mod_id) + + for name in implicit_module_attrs: + v = self.globals[name].node + if isinstance(v, Var): + v.type = self.anal_type(v.type) + v.is_ready = True + + defs = file_node.defs + for d in defs: + self.accept(d) + + if self.cur_mod_id == 'builtins': + remove_imported_names_from_symtable(self.globals, 'builtins') + + if '__all__' in self.globals: + for name, g in self.globals.items(): + if name not in self.all_exports: + g.module_public = False + + def visit_func_def(self, defn: FuncDef) -> None: + phase_info = self.postpone_nested_functions_stack[-1] + if phase_info != FUNCTION_SECOND_PHASE: + self.function_stack.append(defn) + # First phase of analysis for function. + self.errors.push_function(defn.name()) + self.update_function_type_variables(defn) + self.errors.pop_function() + self.function_stack.pop() + + defn.is_conditional = self.block_depth[-1] > 0 + + # TODO(jukka): Figure out how to share the various cases. It doesn't + # make sense to have (almost) duplicate code (here and elsewhere) for + # 3 cases: module-level, class-level and local names. Maybe implement + # a common stack of namespaces. As the 3 kinds of namespaces have + # different semantics, this wouldn't always work, but it might still + # be a win. + if self.is_class_scope(): + # Method definition + defn.info = self.type + if not defn.is_decorated and not defn.is_overload: + if defn.name() in self.type.names: + # Redefinition. Conditional redefinition is okay. + n = self.type.names[defn.name()].node + if self.is_conditional_func(n, defn): + defn.original_def = cast(FuncDef, n) + else: + self.name_already_defined(defn.name(), defn) + self.type.names[defn.name()] = SymbolTableNode(MDEF, defn) + self.prepare_method_signature(defn) + elif self.is_func_scope(): + # Nested function + if not defn.is_decorated and not defn.is_overload: + if defn.name() in self.locals[-1]: + # Redefinition. Conditional redefinition is okay. + n = self.locals[-1][defn.name()].node + if self.is_conditional_func(n, defn): + defn.original_def = cast(FuncDef, n) + else: + self.name_already_defined(defn.name(), defn) + else: + self.add_local(defn, defn) + else: + # Top-level function + if not defn.is_decorated and not defn.is_overload: + symbol = self.globals.get(defn.name()) + if isinstance(symbol.node, FuncDef) and symbol.node != defn: + # This is redefinition. Conditional redefinition is okay. + original_def = symbol.node + if self.is_conditional_func(original_def, defn): + # Conditional function definition -- multiple defs are ok. + defn.original_def = original_def + else: + # Report error. + self.check_no_global(defn.name(), defn, True) + if phase_info == FUNCTION_FIRST_PHASE_POSTPONE_SECOND: + # Postpone this function (for the second phase). + self.postponed_functions_stack[-1].append(defn) + return + if phase_info != FUNCTION_FIRST_PHASE_POSTPONE_SECOND: + # Second phase of analysis for function. + self.errors.push_function(defn.name()) + self.analyze_function(defn) + self.errors.pop_function() + + def prepare_method_signature(self, func: FuncDef) -> None: + """Check basic signature validity and tweak annotation of self/cls argument.""" + # Only non-static methods are special. + if not func.is_static: + if not func.arguments: + self.fail('Method must have at least one argument', func) + elif func.type: + sig = cast(FunctionLike, func.type) + if func.is_class: + leading_type = self.class_type(self.type) + else: + leading_type = self_type(self.type) + func.type = replace_implicit_first_type(sig, leading_type) + + def is_conditional_func(self, previous: Node, new: FuncDef) -> bool: + """Does 'new' conditionally redefine 'previous'? + + We reject straight redefinitions of functions, as they are usually + a programming error. For example: + + . def f(): ... + . def f(): ... # Error: 'f' redefined + """ + return isinstance(previous, (FuncDef, Var)) and new.is_conditional + + def update_function_type_variables(self, defn: FuncDef) -> None: + """Make any type variables in the signature of defn explicit. + + Update the signature of defn to contain type variable definitions + if defn is generic. + """ + if defn.type: + functype = cast(CallableType, defn.type) + typevars = self.infer_type_variables(functype) + # Do not define a new type variable if already defined in scope. + typevars = [(name, tvar) for name, tvar in typevars + if not self.is_defined_type_var(name, defn)] + if typevars: + next_tvar_id = self.next_function_tvar_id() + defs = [TypeVarDef(tvar[0], next_tvar_id - i, + tvar[1].values, tvar[1].upper_bound, + tvar[1].variance) + for i, tvar in enumerate(typevars)] + functype.variables = defs + + def infer_type_variables(self, + type: CallableType) -> List[Tuple[str, TypeVarExpr]]: + """Return list of unique type variables referred to in a callable.""" + names = [] # type: List[str] + tvars = [] # type: List[TypeVarExpr] + for arg in type.arg_types + [type.ret_type]: + for name, tvar_expr in self.find_type_variables_in_type(arg): + if name not in names: + names.append(name) + tvars.append(tvar_expr) + return list(zip(names, tvars)) + + def find_type_variables_in_type( + self, type: Type) -> List[Tuple[str, TypeVarExpr]]: + """Return a list of all unique type variable references in type. + + This effectively does partial name binding, results of which are mostly thrown away. + """ + result = [] # type: List[Tuple[str, TypeVarExpr]] + if isinstance(type, UnboundType): + name = type.name + node = self.lookup_qualified(name, type) + if node and node.kind == UNBOUND_TVAR: + result.append((name, cast(TypeVarExpr, node.node))) + for arg in type.args: + result.extend(self.find_type_variables_in_type(arg)) + elif isinstance(type, TypeList): + for item in type.items: + result.extend(self.find_type_variables_in_type(item)) + elif isinstance(type, UnionType): + for item in type.items: + result.extend(self.find_type_variables_in_type(item)) + elif isinstance(type, AnyType): + pass + elif isinstance(type, EllipsisType) or isinstance(type, TupleType): + pass + else: + assert False, 'Unsupported type %s' % type + return result + + def is_defined_type_var(self, tvar: str, context: Node) -> bool: + return self.lookup_qualified(tvar, context).kind == BOUND_TVAR + + def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: + t = [] # type: List[CallableType] + for i, item in enumerate(defn.items): + # TODO support decorated overloaded functions properly + item.is_overload = True + item.func.is_overload = True + item.accept(self) + t.append(cast(CallableType, function_type(item.func, + self.builtin_type('builtins.function')))) + if item.func.is_property and i == 0: + # This defines a property, probably with a setter and/or deleter. + self.analyze_property_with_multi_part_definition(defn) + break + if not [dec for dec in item.decorators + if refers_to_fullname(dec, 'typing.overload')]: + self.fail("'overload' decorator expected", item) + + defn.type = Overloaded(t) + defn.type.line = defn.line + + if self.is_class_scope(): + self.type.names[defn.name()] = SymbolTableNode(MDEF, defn, + typ=defn.type) + defn.info = self.type + elif self.is_func_scope(): + self.add_local(defn, defn) + + def analyze_property_with_multi_part_definition(self, defn: OverloadedFuncDef) -> None: + """Analyze a property defined using multiple methods (e.g., using @x.setter). + + Assume that the first method (@property) has already been analyzed. + """ + defn.is_property = True + items = defn.items + for item in items[1:]: + if len(item.decorators) == 1: + node = item.decorators[0] + if isinstance(node, MemberExpr): + if node.name == 'setter': + # The first item represents the entire property. + defn.items[0].var.is_settable_property = True + # Get abstractness from the original definition. + item.func.is_abstract = items[0].func.is_abstract + else: + self.fail("Decorated property not supported", item) + item.func.accept(self) + + def next_function_tvar_id(self) -> int: + return self.next_function_tvar_id_stack[-1] + + def analyze_function(self, defn: FuncItem) -> None: + is_method = self.is_class_scope() + + tvarnodes = self.add_func_type_variables_to_symbol_table(defn) + next_function_tvar_id = min([self.next_function_tvar_id()] + + [n.tvar_def.id.raw_id - 1 for n in tvarnodes]) + self.next_function_tvar_id_stack.append(next_function_tvar_id) + + if defn.type: + # Signature must be analyzed in the surrounding scope so that + # class-level imported names and type variables are in scope. + defn.type = self.anal_type(defn.type) + self.check_function_signature(defn) + if isinstance(defn, FuncDef): + defn.type = set_callable_name(defn.type, defn) + for arg in defn.arguments: + if arg.initializer: + arg.initializer.accept(self) + self.function_stack.append(defn) + self.enter() + for arg in defn.arguments: + self.add_local(arg.variable, defn) + for arg in defn.arguments: + if arg.initialization_statement: + lvalue = arg.initialization_statement.lvalues[0] + lvalue.accept(self) + + # The first argument of a non-static, non-class method is like 'self' + # (though the name could be different), having the enclosing class's + # instance type. + if is_method and not defn.is_static and not defn.is_class and defn.arguments: + defn.arguments[0].variable.is_self = True + + # First analyze body of the function but ignore nested functions. + self.postpone_nested_functions_stack.append(FUNCTION_FIRST_PHASE_POSTPONE_SECOND) + self.postponed_functions_stack.append([]) + defn.body.accept(self) + + # Analyze nested functions (if any) as a second phase. + self.postpone_nested_functions_stack[-1] = FUNCTION_SECOND_PHASE + for postponed in self.postponed_functions_stack[-1]: + postponed.accept(self) + self.postpone_nested_functions_stack.pop() + self.postponed_functions_stack.pop() + + self.next_function_tvar_id_stack.pop() + disable_typevars(tvarnodes) + + self.leave() + self.function_stack.pop() + + def add_func_type_variables_to_symbol_table( + self, defn: FuncItem) -> List[SymbolTableNode]: + nodes = [] # type: List[SymbolTableNode] + if defn.type: + tt = defn.type + names = self.type_var_names() + items = cast(CallableType, tt).variables + for item in items: + name = item.name + if name in names: + self.name_already_defined(name, defn) + node = self.bind_type_var(name, item, defn) + nodes.append(node) + names.add(name) + return nodes + + def type_var_names(self) -> Set[str]: + if not self.type: + return set() + else: + return set(self.type.type_vars) + + def bind_type_var(self, fullname: str, tvar_def: TypeVarDef, + context: Context) -> SymbolTableNode: + node = self.lookup_qualified(fullname, context) + node.kind = BOUND_TVAR + node.tvar_def = tvar_def + return node + + def check_function_signature(self, fdef: FuncItem) -> None: + sig = cast(CallableType, fdef.type) + if len(sig.arg_types) < len(fdef.arguments): + self.fail('Type signature has too few arguments', fdef) + # Add dummy Any arguments to prevent crashes later. + extra_anys = [AnyType()] * (len(fdef.arguments) - len(sig.arg_types)) + sig.arg_types.extend(extra_anys) + elif len(sig.arg_types) > len(fdef.arguments): + self.fail('Type signature has too many arguments', fdef) + + def visit_class_def(self, defn: ClassDef) -> None: + self.clean_up_bases_and_infer_type_variables(defn) + self.setup_class_def_analysis(defn) + + self.bind_class_type_vars(defn) + + self.analyze_base_classes(defn) + self.analyze_metaclass(defn) + + for decorator in defn.decorators: + self.analyze_class_decorator(defn, decorator) + + self.enter_class(defn) + + self.setup_is_builtinclass(defn) + + # Analyze class body. + defn.defs.accept(self) + + self.calculate_abstract_status(defn.info) + self.setup_type_promotion(defn) + + self.leave_class() + self.unbind_class_type_vars() + + def enter_class(self, defn: ClassDef) -> None: + # Remember previous active class + self.type_stack.append(self.type) + self.locals.append(None) # Add class scope + self.block_depth.append(-1) # The class body increments this to 0 + self.postpone_nested_functions_stack.append(FUNCTION_BOTH_PHASES) + self.type = defn.info + + def leave_class(self) -> None: + """ Restore analyzer state. """ + self.postpone_nested_functions_stack.pop() + self.block_depth.pop() + self.locals.pop() + self.type = self.type_stack.pop() + + def bind_class_type_vars(self, defn: ClassDef) -> None: + """ Unbind type variables of previously active class and bind + the type variables for the active class. + """ + if self.bound_tvars: + disable_typevars(self.bound_tvars) + self.tvar_stack.append(self.bound_tvars) + self.bound_tvars = self.bind_class_type_variables_in_symbol_table(defn.info) + + def unbind_class_type_vars(self) -> None: + """ Unbind the active class' type vars and rebind the + type vars of the previously active class. + """ + disable_typevars(self.bound_tvars) + self.bound_tvars = self.tvar_stack.pop() + if self.bound_tvars: + enable_typevars(self.bound_tvars) + + def analyze_class_decorator(self, defn: ClassDef, decorator: Node) -> None: + decorator.accept(self) + + def setup_is_builtinclass(self, defn: ClassDef): + for decorator in defn.decorators: + if refers_to_fullname(decorator, 'typing.builtinclass'): + defn.is_builtinclass = True + if defn.fullname == 'builtins.object': + # Only 'object' is marked as a built-in class, as otherwise things elsewhere + # would break. We need a better way of dealing with built-in classes. + defn.is_builtinclass = True + + def calculate_abstract_status(self, typ: TypeInfo) -> None: + """Calculate abstract status of a class. + + Set is_abstract of the type to True if the type has an unimplemented + abstract attribute. Also compute a list of abstract attributes. + """ + concrete = set() # type: Set[str] + abstract = [] # type: List[str] + for base in typ.mro: + for name, symnode in base.names.items(): + node = symnode.node + if isinstance(node, OverloadedFuncDef): + # Unwrap an overloaded function definition. We can just + # check arbitrarily the first overload item. If the + # different items have a different abstract status, there + # should be an error reported elsewhere. + func = node.items[0] # type: Node + else: + func = node + if isinstance(func, Decorator): + fdef = func.func + if fdef.is_abstract and name not in concrete: + typ.is_abstract = True + abstract.append(name) + concrete.add(name) + typ.abstract_attributes = sorted(abstract) + + def setup_type_promotion(self, defn: ClassDef) -> None: + """Setup extra, ad-hoc subtyping relationships between classes (promotion). + + This includes things like 'int' being compatible with 'float'. + """ + promote_target = None # type: Type + for decorator in defn.decorators: + if isinstance(decorator, CallExpr): + analyzed = decorator.analyzed + if isinstance(analyzed, PromoteExpr): + # _promote class decorator (undocumented faeture). + promote_target = analyzed.type + if not promote_target: + promotions = (TYPE_PROMOTIONS_PYTHON3 if self.options.python_version[0] >= 3 + else TYPE_PROMOTIONS_PYTHON2) + if defn.fullname in promotions: + promote_target = self.named_type_or_none(promotions[defn.fullname]) + defn.info._promote = promote_target + + def clean_up_bases_and_infer_type_variables(self, defn: ClassDef) -> None: + """Remove extra base classes such as Generic and infer type vars. + + For example, consider this class: + + . class Foo(Bar, Generic[T]): ... + + Now we will remove Generic[T] from bases of Foo and infer that the + type variable 'T' is a type argument of Foo. + + Note that this is performed *before* semantic analysis. + """ + removed = [] # type: List[int] + type_vars = [] # type: List[TypeVarDef] + for i, base_expr in enumerate(defn.base_type_exprs): + try: + base = expr_to_unanalyzed_type(base_expr) + except TypeTranslationError: + # This error will be caught later. + continue + tvars = self.analyze_typevar_declaration(base) + if tvars is not None: + if type_vars: + self.fail('Duplicate Generic in bases', defn) + removed.append(i) + for j, (name, tvar_expr) in enumerate(tvars): + type_vars.append(TypeVarDef(name, j + 1, tvar_expr.values, + tvar_expr.upper_bound, tvar_expr.variance)) + if type_vars: + defn.type_vars = type_vars + if defn.info: + defn.info.type_vars = [tv.name for tv in type_vars] + for i in reversed(removed): + del defn.base_type_exprs[i] + + def analyze_typevar_declaration(self, t: Type) -> Optional[List[Tuple[str, TypeVarExpr]]]: + if not isinstance(t, UnboundType): + return None + unbound = t + sym = self.lookup_qualified(unbound.name, unbound) + if sym is None or sym.node is None: + return None + if sym.node.fullname() == 'typing.Generic': + tvars = [] # type: List[Tuple[str, TypeVarExpr]] + for arg in unbound.args: + tvar = self.analyze_unbound_tvar(arg) + if tvar: + tvars.append(tvar) + else: + self.fail('Free type variable expected in %s[...]' % + sym.node.name(), t) + return tvars + return None + + def analyze_unbound_tvar(self, t: Type) -> Tuple[str, TypeVarExpr]: + if not isinstance(t, UnboundType): + return None + unbound = t + sym = self.lookup_qualified(unbound.name, unbound) + if sym is not None and sym.kind == UNBOUND_TVAR: + return unbound.name, cast(TypeVarExpr, sym.node) + return None + + def setup_class_def_analysis(self, defn: ClassDef) -> None: + """Prepare for the analysis of a class definition.""" + if not defn.info: + defn.info = TypeInfo(SymbolTable(), defn) + defn.info._fullname = defn.info.name() + if self.is_func_scope() or self.type: + kind = MDEF + if self.is_func_scope(): + kind = LDEF + self.add_symbol(defn.name, SymbolTableNode(kind, defn.info), defn) + + def analyze_base_classes(self, defn: ClassDef) -> None: + """Analyze and set up base classes. + + This computes several attributes on the corresponding TypeInfo defn.info + related to the base classes: defn.info.bases, defn.info.mro, and + miscellaneous others (at least tuple_type, fallback_to_any, and is_enum.) + """ + + base_types = [] # type: List[Instance] + for base_expr in defn.base_type_exprs: + try: + base = self.expr_to_analyzed_type(base_expr) + except TypeTranslationError: + self.fail('Invalid base class', base_expr) + defn.info.fallback_to_any = True + continue + + if isinstance(base, TupleType): + if defn.info.tuple_type: + self.fail("Class has two incompatible bases derived from tuple", defn) + if (not self.is_stub_file + and not defn.info.is_named_tuple + and base.fallback.type.fullname() == 'builtins.tuple'): + self.fail("Tuple[...] not supported as a base class outside a stub file", defn) + defn.info.tuple_type = base + base_types.append(base.fallback) + elif isinstance(base, Instance): + base_types.append(base) + elif isinstance(base, AnyType): + defn.info.fallback_to_any = True + else: + self.fail('Invalid base class', base_expr) + defn.info.fallback_to_any = True + + # Add 'object' as implicit base if there is no other base class. + if (not base_types and defn.fullname != 'builtins.object'): + base_types.append(self.object_type()) + + defn.info.bases = base_types + + # Calculate the MRO. It might be incomplete at this point if + # the bases of defn include classes imported from other + # modules in an import loop. We'll recompute it in ThirdPass. + if not self.verify_base_classes(defn): + defn.info.mro = [] + return + calculate_class_mro(defn, self.fail) + # If there are cyclic imports, we may be missing 'object' in + # the MRO. Fix MRO if needed. + if defn.info.mro and defn.info.mro[-1].fullname() != 'builtins.object': + defn.info.mro.append(self.object_type().type) + + def expr_to_analyzed_type(self, expr: Node) -> Type: + if isinstance(expr, CallExpr): + expr.accept(self) + info = self.check_namedtuple(expr) + if info is None: + # Some form of namedtuple is the only valid type that looks like a call + # expression. This isn't a valid type. + raise TypeTranslationError() + fallback = Instance(info, []) + return TupleType(info.tuple_type.items, fallback=fallback) + typ = expr_to_unanalyzed_type(expr) + return self.anal_type(typ) + + def verify_base_classes(self, defn: ClassDef) -> bool: + info = defn.info + for base in info.bases: + baseinfo = base.type + if self.is_base_class(info, baseinfo): + self.fail('Cycle in inheritance hierarchy', defn) + # Clear bases to forcefully get rid of the cycle. + info.bases = [] + if baseinfo.fullname() == 'builtins.bool': + self.fail("'%s' is not a valid base class" % + baseinfo.name(), defn) + return False + dup = find_duplicate(info.direct_base_classes()) + if dup: + self.fail('Duplicate base class "%s"' % dup.name(), defn) + return False + return True + + def is_base_class(self, t: TypeInfo, s: TypeInfo) -> bool: + """Determine if t is a base class of s (but do not use mro).""" + # Search the base class graph for t, starting from s. + worklist = [s] + visited = {s} + while worklist: + nxt = worklist.pop() + if nxt == t: + return True + for base in nxt.bases: + if base.type not in visited: + worklist.append(base.type) + visited.add(base.type) + return False + + def analyze_metaclass(self, defn: ClassDef) -> None: + if defn.metaclass: + sym = self.lookup_qualified(defn.metaclass, defn) + if sym is not None and not isinstance(sym.node, TypeInfo): + self.fail("Invalid metaclass '%s'" % defn.metaclass, defn) + + def object_type(self) -> Instance: + return self.named_type('__builtins__.object') + + def class_type(self, info: TypeInfo) -> Type: + # Construct a function type whose fallback is cls. + from mypy import checkmember # To avoid import cycle. + leading_type = checkmember.type_object_type(info, self.builtin_type) + if isinstance(leading_type, Overloaded): + # Overloaded __init__ is too complex to handle. Plus it's stubs only. + return AnyType() + else: + return leading_type + + def named_type(self, qualified_name: str, args: List[Type] = None) -> Instance: + sym = self.lookup_qualified(qualified_name, None) + return Instance(cast(TypeInfo, sym.node), args or []) + + def named_type_or_none(self, qualified_name: str) -> Instance: + sym = self.lookup_fully_qualified_or_none(qualified_name) + if not sym: + return None + return Instance(cast(TypeInfo, sym.node), []) + + def is_instance_type(self, t: Type) -> bool: + return isinstance(t, Instance) + + def bind_class_type_variables_in_symbol_table( + self, info: TypeInfo) -> List[SymbolTableNode]: + nodes = [] # type: List[SymbolTableNode] + for var, binder in zip(info.type_vars, info.defn.type_vars): + node = self.bind_type_var(var, binder, info) + nodes.append(node) + return nodes + + def visit_import(self, i: Import) -> None: + for id, as_id in i.ids: + if as_id is not None: + self.add_module_symbol(id, as_id, module_public=True, context=i) + else: + # Modules imported in a stub file without using 'as x' won't get exported when + # doing 'from m import *'. + module_public = not self.is_stub_file + base = id.split('.')[0] + self.add_module_symbol(base, base, module_public=module_public, + context=i) + + def add_module_symbol(self, id: str, as_id: str, module_public: bool, + context: Context) -> None: + if id in self.modules: + m = self.modules[id] + self.add_symbol(as_id, SymbolTableNode(MODULE_REF, m, self.cur_mod_id, + module_public=module_public), context) + else: + self.add_unknown_symbol(as_id, context) + + def visit_import_from(self, imp: ImportFrom) -> None: + import_id = self.correct_relative_import(imp) + if import_id in self.modules: + module = self.modules[import_id] + for id, as_id in imp.names: + node = module.names.get(id) + if node and node.kind != UNBOUND_IMPORTED: + node = self.normalize_type_alias(node, imp) + if not node: + return + imported_id = as_id or id + existing_symbol = self.globals.get(imported_id) + if existing_symbol: + # Import can redefine a variable. They get special treatment. + if self.process_import_over_existing_name( + imported_id, existing_symbol, node, imp): + continue + # 'from m import x as x' exports x in a stub file. + module_public = not self.is_stub_file or as_id is not None + symbol = SymbolTableNode(node.kind, node.node, + self.cur_mod_id, + node.type_override, + module_public=module_public) + self.add_symbol(imported_id, symbol, imp) + else: + message = "Module has no attribute '{}'".format(id) + extra = self.undefined_name_extra_info('{}.{}'.format(import_id, id)) + if extra: + message += " {}".format(extra) + self.fail(message, imp) + else: + # Missing module. + for id, as_id in imp.names: + self.add_unknown_symbol(as_id or id, imp) + + def process_import_over_existing_name(self, + imported_id: str, existing_symbol: SymbolTableNode, + module_symbol: SymbolTableNode, + import_node: ImportBase) -> bool: + if (existing_symbol.kind in (LDEF, GDEF, MDEF) and + isinstance(existing_symbol.node, (Var, FuncDef))): + # This is a valid import over an existing definition in the file. Construct a dummy + # assignment that we'll use to type check the import. + lvalue = NameExpr(imported_id) + lvalue.kind = existing_symbol.kind + lvalue.node = existing_symbol.node + rvalue = NameExpr(imported_id) + rvalue.kind = module_symbol.kind + rvalue.node = module_symbol.node + assignment = AssignmentStmt([lvalue], rvalue) + for node in assignment, lvalue, rvalue: + node.set_line(import_node) + import_node.assignments.append(assignment) + return True + return False + + def normalize_type_alias(self, node: SymbolTableNode, + ctx: Context) -> SymbolTableNode: + if node.fullname in type_aliases: + # Node refers to an aliased type such as typing.List; normalize. + node = self.lookup_qualified(type_aliases[node.fullname], ctx) + if node.fullname == 'typing.DefaultDict': + self.add_module_symbol('collections', '__mypy_collections__', False, ctx) + node = self.lookup_qualified('__mypy_collections__.defaultdict', ctx) + return node + + def correct_relative_import(self, node: Union[ImportFrom, ImportAll]) -> str: + if node.relative == 0: + return node.id + + parts = self.cur_mod_id.split(".") + cur_mod_id = self.cur_mod_id + + rel = node.relative + if self.cur_mod_node.is_package_init_file(): + rel -= 1 + if len(parts) < rel: + self.fail("Relative import climbs too many namespaces", node) + if rel != 0: + cur_mod_id = ".".join(parts[:-rel]) + + return cur_mod_id + (("." + node.id) if node.id else "") + + def visit_import_all(self, i: ImportAll) -> None: + i_id = self.correct_relative_import(i) + if i_id in self.modules: + m = self.modules[i_id] + for name, node in m.names.items(): + node = self.normalize_type_alias(node, i) + if not name.startswith('_') and node.module_public: + existing_symbol = self.globals.get(name) + if existing_symbol: + # Import can redefine a variable. They get special treatment. + if self.process_import_over_existing_name( + name, existing_symbol, node, i): + continue + self.add_symbol(name, SymbolTableNode(node.kind, node.node, + self.cur_mod_id), i) + else: + # Don't add any dummy symbols for 'from x import *' if 'x' is unknown. + pass + + def add_unknown_symbol(self, name: str, context: Context) -> None: + var = Var(name) + var._fullname = self.qualified_name(name) + var.is_ready = True + var.type = AnyType() + self.add_symbol(name, SymbolTableNode(GDEF, var, self.cur_mod_id), context) + + # + # Statements + # + + def visit_block(self, b: Block) -> None: + if b.is_unreachable: + return + self.block_depth[-1] += 1 + for s in b.body: + self.accept(s) + self.block_depth[-1] -= 1 + + def visit_block_maybe(self, b: Block) -> None: + if b: + self.visit_block(b) + + def anal_type(self, t: Type, allow_tuple_literal: bool = False) -> Type: + if t: + if allow_tuple_literal: + # Types such as (t1, t2, ...) only allowed in assignment statements. They'll + # generate errors elsewhere, and Tuple[t1, t2, ...] must be used instead. + if isinstance(t, TupleType): + # Unlike TypeAnalyser, also allow implicit tuple types (without Tuple[...]). + star_count = sum(1 for item in t.items if isinstance(item, StarType)) + if star_count > 1: + self.fail('At most one star type allowed in a tuple', t) + return TupleType([AnyType() for _ in t.items], + self.builtin_type('builtins.tuple'), t.line) + items = [self.anal_type(item, True) + for item in t.items] + return TupleType(items, self.builtin_type('builtins.tuple'), t.line) + a = TypeAnalyser(self.lookup_qualified, + self.lookup_fully_qualified, + self.fail) + return t.accept(a) + else: + return None + + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + for lval in s.lvalues: + self.analyze_lvalue(lval, explicit_type=s.type is not None) + s.rvalue.accept(self) + if s.type: + allow_tuple_literal = isinstance(s.lvalues[-1], (TupleExpr, ListExpr)) + s.type = self.anal_type(s.type, allow_tuple_literal) + else: + # For simple assignments, allow binding type aliases. + # Also set the type if the rvalue is a simple literal. + if (s.type is None and len(s.lvalues) == 1 and + isinstance(s.lvalues[0], NameExpr)): + if s.lvalues[0].is_def: + s.type = self.analyze_simple_literal_type(s.rvalue) + res = analyze_type_alias(s.rvalue, + self.lookup_qualified, + self.lookup_fully_qualified, + self.fail) + if res and (not isinstance(res, Instance) or cast(Instance, res).args): + # TODO: What if this gets reassigned? + name = s.lvalues[0] + node = self.lookup(name.name, name) + node.kind = TYPE_ALIAS + node.type_override = res + if isinstance(s.rvalue, IndexExpr): + s.rvalue.analyzed = TypeAliasExpr(res) + if s.type: + # Store type into nodes. + for lvalue in s.lvalues: + self.store_declared_types(lvalue, s.type) + self.check_and_set_up_type_alias(s) + self.process_typevar_declaration(s) + self.process_namedtuple_definition(s) + + if (len(s.lvalues) == 1 and isinstance(s.lvalues[0], NameExpr) and + s.lvalues[0].name == '__all__' and s.lvalues[0].kind == GDEF and + isinstance(s.rvalue, (ListExpr, TupleExpr))): + self.add_exports(*s.rvalue.items) + + def analyze_simple_literal_type(self, rvalue: Node) -> Optional[Type]: + """Return builtins.int if rvalue is an int literal, etc.""" + if self.weak_opts or self.options.semantic_analysis_only or self.function_stack: + # Skip this if any weak options are set. + # Also skip if we're only doing the semantic analysis pass. + # This is mostly to avoid breaking unit tests. + # Also skip inside a function; this is to avoid confusing + # the code that handles dead code due to isinstance() + # inside type variables with value restrictions (like + # AnyStr). + return None + if isinstance(rvalue, IntExpr): + return self.named_type_or_none('builtins.int') + if isinstance(rvalue, FloatExpr): + return self.named_type_or_none('builtins.float') + if isinstance(rvalue, StrExpr): + return self.named_type_or_none('builtins.str') + if isinstance(rvalue, BytesExpr): + return self.named_type_or_none('builtins.bytes') + if isinstance(rvalue, UnicodeExpr): + return self.named_type_or_none('builtins.unicode') + return None + + def check_and_set_up_type_alias(self, s: AssignmentStmt) -> None: + """Check if assignment creates a type alias and set it up as needed.""" + # For now, type aliases only work at the top level of a module. + if (len(s.lvalues) == 1 and not self.is_func_scope() and not self.type + and not s.type): + lvalue = s.lvalues[0] + if isinstance(lvalue, NameExpr): + if not lvalue.is_def: + # Only a definition can create a type alias, not regular assignment. + return + rvalue = s.rvalue + if isinstance(rvalue, RefExpr): + node = rvalue.node + if isinstance(node, TypeInfo): + # TODO: We should record the fact that this is a variable + # that refers to a type, rather than making this + # just an alias for the type. + self.globals[lvalue.name].node = node + + def analyze_lvalue(self, lval: Node, nested: bool = False, + add_global: bool = False, + explicit_type: bool = False) -> None: + """Analyze an lvalue or assignment target. + + Only if add_global is True, add name to globals table. If nested + is true, the lvalue is within a tuple or list lvalue expression. + """ + + if isinstance(lval, NameExpr): + # Top-level definitions within some statements (at least while) are + # not handled in the first pass, so they have to be added now. + nested_global = (not self.is_func_scope() and + self.block_depth[-1] > 0 and + not self.type) + if (add_global or nested_global) and lval.name not in self.globals: + # Define new global name. + v = Var(lval.name) + v._fullname = self.qualified_name(lval.name) + v.is_ready = False # Type not inferred yet + lval.node = v + lval.is_def = True + lval.kind = GDEF + lval.fullname = v._fullname + self.globals[lval.name] = SymbolTableNode(GDEF, v, + self.cur_mod_id) + elif isinstance(lval.node, Var) and lval.is_def: + # Since the is_def flag is set, this must have been analyzed + # already in the first pass and added to the symbol table. + assert lval.node.name() in self.globals + elif (self.is_func_scope() and lval.name not in self.locals[-1] and + lval.name not in self.global_decls[-1] and + lval.name not in self.nonlocal_decls[-1]): + # Define new local name. + v = Var(lval.name) + lval.node = v + lval.is_def = True + lval.kind = LDEF + lval.fullname = lval.name + self.add_local(v, lval) + elif not self.is_func_scope() and (self.type and + lval.name not in self.type.names): + # Define a new attribute within class body. + v = Var(lval.name) + v.info = self.type + v.is_initialized_in_class = True + v.set_line(lval) + lval.node = v + lval.is_def = True + lval.kind = MDEF + lval.fullname = lval.name + self.type.names[lval.name] = SymbolTableNode(MDEF, v) + else: + # Bind to an existing name. + if explicit_type: + self.name_already_defined(lval.name, lval) + lval.accept(self) + self.check_lvalue_validity(lval.node, lval) + elif isinstance(lval, MemberExpr): + if not add_global: + self.analyze_member_lvalue(lval) + if explicit_type and not self.is_self_member_ref(lval): + self.fail('Type cannot be declared in assignment to non-self ' + 'attribute', lval) + elif isinstance(lval, IndexExpr): + if explicit_type: + self.fail('Unexpected type declaration', lval) + if not add_global: + lval.accept(self) + elif (isinstance(lval, TupleExpr) or + isinstance(lval, ListExpr)): + items = cast(Any, lval).items + if len(items) == 0 and isinstance(lval, TupleExpr): + self.fail("Can't assign to ()", lval) + self.analyze_tuple_or_list_lvalue(cast(Union[ListExpr, TupleExpr], lval), + add_global, explicit_type) + elif isinstance(lval, StarExpr): + if nested: + self.analyze_lvalue(lval.expr, nested, add_global, explicit_type) + else: + self.fail('Starred assignment target must be in a list or tuple', lval) + else: + self.fail('Invalid assignment target', lval) + + def analyze_tuple_or_list_lvalue(self, lval: Union[ListExpr, TupleExpr], + add_global: bool = False, + explicit_type: bool = False) -> None: + """Analyze an lvalue or assignment target that is a list or tuple.""" + items = lval.items + star_exprs = [cast(StarExpr, item) + for item in items + if isinstance(item, StarExpr)] + + if len(star_exprs) > 1: + self.fail('Two starred expressions in assignment', lval) + else: + if len(star_exprs) == 1: + star_exprs[0].valid = True + for i in items: + self.analyze_lvalue(i, nested=True, add_global=add_global, + explicit_type = explicit_type) + + def analyze_member_lvalue(self, lval: MemberExpr) -> None: + lval.accept(self) + if (self.is_self_member_ref(lval) and + self.type.get(lval.name) is None): + # Implicit attribute definition in __init__. + lval.is_def = True + v = Var(lval.name) + v.info = self.type + v.is_ready = False + lval.def_var = v + lval.node = v + self.type.names[lval.name] = SymbolTableNode(MDEF, v) + self.check_lvalue_validity(lval.node, lval) + + def is_self_member_ref(self, memberexpr: MemberExpr) -> bool: + """Does memberexpr to refer to an attribute of self?""" + if not isinstance(memberexpr.expr, NameExpr): + return False + node = memberexpr.expr.node + return isinstance(node, Var) and node.is_self + + def check_lvalue_validity(self, node: Node, ctx: Context) -> None: + if isinstance(node, (TypeInfo, TypeVarExpr)): + self.fail('Invalid assignment target', ctx) + + def store_declared_types(self, lvalue: Node, typ: Type) -> None: + if isinstance(typ, StarType) and not isinstance(lvalue, StarExpr): + self.fail('Star type only allowed for starred expressions', lvalue) + if isinstance(lvalue, RefExpr): + lvalue.is_def = False + if isinstance(lvalue.node, Var): + var = lvalue.node + var.type = typ + var.is_ready = True + # If node is not a variable, we'll catch it elsewhere. + elif isinstance(lvalue, TupleExpr): + if isinstance(typ, TupleType): + if len(lvalue.items) != len(typ.items): + self.fail('Incompatible number of tuple items', lvalue) + return + for item, itemtype in zip(lvalue.items, typ.items): + self.store_declared_types(item, itemtype) + else: + self.fail('Tuple type expected for multiple variables', + lvalue) + elif isinstance(lvalue, StarExpr): + if isinstance(typ, StarType): + self.store_declared_types(lvalue.expr, typ.type) + else: + self.fail('Star type expected for starred expression', lvalue) + else: + # This has been flagged elsewhere as an error, so just ignore here. + pass + + def process_typevar_declaration(self, s: AssignmentStmt) -> None: + """Check if s declares a TypeVar; it yes, store it in symbol table.""" + call = self.get_typevar_declaration(s) + if not call: + return + + lvalue = cast(NameExpr, s.lvalues[0]) + name = lvalue.name + if not lvalue.is_def: + if s.type: + self.fail("Cannot declare the type of a type variable", s) + else: + self.fail("Cannot redefine '%s' as a type variable" % name, s) + return + + if not self.check_typevar_name(call, name, s): + return + + # Constraining types + n_values = call.arg_kinds[1:].count(ARG_POS) + values = self.analyze_types(call.args[1:1 + n_values]) + + res = self.process_typevar_parameters(call.args[1 + n_values:], + call.arg_names[1 + n_values:], + call.arg_kinds[1 + n_values:], + bool(values), + s) + if res is None: + return + variance, upper_bound = res + + # Yes, it's a valid type variable definition! Add it to the symbol table. + node = self.lookup(name, s) + node.kind = UNBOUND_TVAR + TypeVar = TypeVarExpr(name, node.fullname, values, upper_bound, variance) + TypeVar.line = call.line + call.analyzed = TypeVar + node.node = TypeVar + + def check_typevar_name(self, call: CallExpr, name: str, context: Context) -> bool: + if len(call.args) < 1: + self.fail("Too few arguments for TypeVar()", context) + return False + if not isinstance(call.args[0], (StrExpr, BytesExpr)) or not call.arg_kinds[0] == ARG_POS: + self.fail("TypeVar() expects a string literal as first argument", context) + return False + if cast(StrExpr, call.args[0]).value != name: + self.fail("Unexpected TypeVar() argument value", context) + return False + return True + + def get_typevar_declaration(self, s: AssignmentStmt) -> Optional[CallExpr]: + """Returns the TypeVar() call expression if `s` is a type var declaration + or None otherwise. + """ + if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): + return None + if not isinstance(s.rvalue, CallExpr): + return None + call = s.rvalue + if not isinstance(call.callee, RefExpr): + return None + callee = call.callee + if callee.fullname != 'typing.TypeVar': + return None + return call + + def process_typevar_parameters(self, args: List[Node], + names: List[Optional[str]], + kinds: List[int], + has_values: bool, + context: Context) -> Optional[Tuple[int, Type]]: + covariant = False + contravariant = False + upper_bound = self.object_type() # type: Type + for param_value, param_name, param_kind in zip(args, names, kinds): + if not param_kind == ARG_NAMED: + self.fail("Unexpected argument to TypeVar()", context) + return None + if param_name == 'covariant': + if isinstance(param_value, NameExpr): + if param_value.name == 'True': + covariant = True + else: + self.fail("TypeVar 'covariant' may only be 'True'", context) + return None + else: + self.fail("TypeVar 'covariant' may only be 'True'", context) + return None + elif param_name == 'contravariant': + if isinstance(param_value, NameExpr): + if param_value.name == 'True': + contravariant = True + else: + self.fail("TypeVar 'contravariant' may only be 'True'", context) + return None + else: + self.fail("TypeVar 'contravariant' may only be 'True'", context) + return None + elif param_name == 'bound': + if has_values: + self.fail("TypeVar cannot have both values and an upper bound", context) + return None + try: + upper_bound = self.expr_to_analyzed_type(param_value) + except TypeTranslationError: + self.fail("TypeVar 'bound' must be a type", param_value) + return None + elif param_name == 'values': + # Probably using obsolete syntax with values=(...). Explain the current syntax. + self.fail("TypeVar 'values' argument not supported", context) + self.fail("Use TypeVar('T', t, ...) instead of TypeVar('T', values=(t, ...))", + context) + return None + else: + self.fail("Unexpected argument to TypeVar(): {}".format(param_name), context) + return None + + if covariant and contravariant: + self.fail("TypeVar cannot be both covariant and contravariant", context) + return None + elif covariant: + variance = COVARIANT + elif contravariant: + variance = CONTRAVARIANT + else: + variance = INVARIANT + return (variance, upper_bound) + + def process_namedtuple_definition(self, s: AssignmentStmt) -> None: + """Check if s defines a namedtuple; if yes, store the definition in symbol table.""" + if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): + return + lvalue = cast(NameExpr, s.lvalues[0]) + name = lvalue.name + named_tuple = self.check_namedtuple(s.rvalue, name) + if named_tuple is None: + return + # Yes, it's a valid namedtuple definition. Add it to the symbol table. + node = self.lookup(name, s) + node.kind = TYPE_ALIAS # TODO locally defined namedtuple + # TODO call.analyzed + node.node = named_tuple + + def check_namedtuple(self, node: Node, var_name: str = None) -> TypeInfo: + """Check if a call defines a namedtuple. + + The optional var_name argument is the name of the variable to + which this is assigned, if any. + + If it does, return the corresponding TypeInfo. Return None otherwise. + + If the definition is invalid but looks like a namedtuple, + report errors but return (some) TypeInfo. + """ + if not isinstance(node, CallExpr): + return None + call = node + if not isinstance(call.callee, RefExpr): + return None + callee = call.callee + fullname = callee.fullname + if fullname not in ('collections.namedtuple', 'typing.NamedTuple'): + return None + items, types, ok = self.parse_namedtuple_args(call, fullname) + if not ok: + # Error. Construct dummy return value. + return self.build_namedtuple_typeinfo('namedtuple', [], []) + else: + # Give it a unique name derived from the line number. + name = cast(StrExpr, call.args[0]).value + if name != var_name: + name += '@' + str(call.line) + info = self.build_namedtuple_typeinfo(name, items, types) + # Store it as a global just in case it would remain anonymous. + self.globals[name] = SymbolTableNode(GDEF, info, self.cur_mod_id) + call.analyzed = NamedTupleExpr(info).set_line(call.line) + return info + + def parse_namedtuple_args(self, call: CallExpr, + fullname: str) -> Tuple[List[str], List[Type], bool]: + # TODO Share code with check_argument_count in checkexpr.py? + args = call.args + if len(args) < 2: + return self.fail_namedtuple_arg("Too few arguments for namedtuple()", call) + if len(args) > 2: + return self.fail_namedtuple_arg("Too many arguments for namedtuple()", call) + if call.arg_kinds != [ARG_POS, ARG_POS]: + return self.fail_namedtuple_arg("Unexpected arguments to namedtuple()", call) + if not isinstance(args[0], (StrExpr, BytesExpr)): + return self.fail_namedtuple_arg( + "namedtuple() expects a string literal as the first argument", call) + types = [] # type: List[Type] + ok = True + if not isinstance(args[1], ListExpr): + if fullname == 'collections.namedtuple' and isinstance(args[1], (StrExpr, BytesExpr)): + str_expr = cast(StrExpr, args[1]) + items = str_expr.value.split() + else: + return self.fail_namedtuple_arg( + "List literal expected as the second argument to namedtuple()", call) + else: + listexpr = args[1] + if fullname == 'collections.namedtuple': + # The fields argument contains just names, with implicit Any types. + if any(not isinstance(item, (StrExpr, BytesExpr)) for item in listexpr.items): + return self.fail_namedtuple_arg("String literal expected as namedtuple() item", + call) + items = [cast(StrExpr, item).value for item in listexpr.items] + else: + # The fields argument contains (name, type) tuples. + items, types, ok = self.parse_namedtuple_fields_with_types(listexpr.items, call) + if not types: + types = [AnyType() for _ in items] + return items, types, ok + + def parse_namedtuple_fields_with_types(self, nodes: List[Node], + context: Context) -> Tuple[List[str], List[Type], bool]: + items = [] # type: List[str] + types = [] # type: List[Type] + for item in nodes: + if isinstance(item, TupleExpr): + if len(item.items) != 2: + return self.fail_namedtuple_arg("Invalid NamedTuple field definition", + item) + name, type_node = item.items + if isinstance(name, (StrExpr, BytesExpr)): + items.append(name.value) + else: + return self.fail_namedtuple_arg("Invalid NamedTuple() field name", item) + try: + type = expr_to_unanalyzed_type(type_node) + except TypeTranslationError: + return self.fail_namedtuple_arg('Invalid field type', type_node) + types.append(self.anal_type(type)) + else: + return self.fail_namedtuple_arg("Tuple expected as NamedTuple() field", item) + return items, types, True + + def fail_namedtuple_arg(self, message: str, + context: Context) -> Tuple[List[str], List[Type], bool]: + self.fail(message, context) + return [], [], False + + def build_namedtuple_typeinfo(self, name: str, items: List[str], + types: List[Type]) -> TypeInfo: + symbols = SymbolTable() + tup = TupleType(types, self.named_type('__builtins__.tuple', [AnyType()])) + class_def = ClassDef(name, Block([])) + class_def.fullname = self.qualified_name(name) + info = NamedTupleTypeInfo(tup, symbols, class_def) + vars = [Var(item, typ) for item, typ in zip(items, types)] + # Add named tuple items as attributes. + # TODO: Make them read-only. + for var in vars: + var.info = info + symbols[var.name] = SymbolTableNode(MDEF, var) + # Add __init__, _replace, _asdict methods and _fields static field + self.add_namedtuple_method('__init__', symbols, info, NoneTyp(), + self.make_factory_args(vars, ARG_POS)) + self.add_namedtuple_method('_replace', symbols, info, Instance(info, []), + self.make_factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) + # TODO: refine to OrderedDict[Union[types]] + self.add_namedtuple_method('_asdict', symbols, info, AnyType()) + return info + + def make_factory_args(self, vars: List[Var], kind: int, + initializer: Expression = None) -> List[Argument]: + return [Argument(var, var.type, initializer, kind) for var in vars] + + def add_namedtuple_method(self, funcname: str, symbols: SymbolTable, info: TypeInfo, + ret: Type, args: List[Argument] = []) -> None: + types = [cast(Type, None)] + [arg.type_annotation for arg in args] + args = [Argument(Var('__self'), NoneTyp(), None, ARG_POS)] + args + items = [arg.variable.name() for arg in args] + arg_kinds = [arg.kind for arg in args] + # TODO: Make sure that the self argument name is not visible? + signature = CallableType(types, arg_kinds, items, ret, + self.named_type('__builtins__.function'), + name=info.name()) + func = FuncDef(funcname, + args, + Block([]), + typ=signature) + func.info = info + symbols[funcname] = SymbolTableNode(MDEF, func) + + def analyze_types(self, items: List[Node]) -> List[Type]: + result = [] # type: List[Type] + for node in items: + try: + result.append(self.anal_type(expr_to_unanalyzed_type(node))) + except TypeTranslationError: + self.fail('Type expected', node) + result.append(AnyType()) + return result + + def visit_decorator(self, dec: Decorator) -> None: + for d in dec.decorators: + d.accept(self) + removed = [] # type: List[int] + no_type_check = False + for i, d in enumerate(dec.decorators): + # A bunch of decorators are special cased here. + if refers_to_fullname(d, 'abc.abstractmethod'): + removed.append(i) + dec.func.is_abstract = True + self.check_decorated_function_is_method('abstractmethod', dec) + elif refers_to_fullname(d, 'asyncio.tasks.coroutine'): + removed.append(i) + elif refers_to_fullname(d, 'builtins.staticmethod'): + removed.append(i) + dec.func.is_static = True + dec.var.is_staticmethod = True + self.check_decorated_function_is_method('staticmethod', dec) + elif refers_to_fullname(d, 'builtins.classmethod'): + removed.append(i) + dec.func.is_class = True + dec.var.is_classmethod = True + self.check_decorated_function_is_method('classmethod', dec) + elif (refers_to_fullname(d, 'builtins.property') or + refers_to_fullname(d, 'abc.abstractproperty')): + removed.append(i) + dec.func.is_property = True + dec.var.is_property = True + if refers_to_fullname(d, 'abc.abstractproperty'): + dec.func.is_abstract = True + self.check_decorated_function_is_method('property', dec) + if len(dec.func.arguments) > 1: + self.fail('Too many arguments', dec.func) + elif refers_to_fullname(d, 'typing.no_type_check'): + dec.var.type = AnyType() + no_type_check = True + for i in reversed(removed): + del dec.decorators[i] + if not dec.is_overload or dec.var.is_property: + if self.is_func_scope(): + self.add_symbol(dec.var.name(), SymbolTableNode(LDEF, dec), + dec) + elif self.type: + dec.var.info = self.type + dec.var.is_initialized_in_class = True + self.add_symbol(dec.var.name(), SymbolTableNode(MDEF, dec), + dec) + if not no_type_check: + dec.func.accept(self) + if dec.decorators and dec.var.is_property: + self.fail('Decorated property not supported', dec) + + def check_decorated_function_is_method(self, decorator: str, + context: Context) -> None: + if not self.type or self.is_func_scope(): + self.fail("'%s' used with a non-method" % decorator, context) + + def visit_expression_stmt(self, s: ExpressionStmt) -> None: + s.expr.accept(self) + + def visit_return_stmt(self, s: ReturnStmt) -> None: + if not self.is_func_scope(): + self.fail("'return' outside function", s) + if s.expr: + s.expr.accept(self) + + def visit_raise_stmt(self, s: RaiseStmt) -> None: + if s.expr: + s.expr.accept(self) + if s.from_expr: + s.from_expr.accept(self) + + def visit_assert_stmt(self, s: AssertStmt) -> None: + if s.expr: + s.expr.accept(self) + + def visit_operator_assignment_stmt(self, + s: OperatorAssignmentStmt) -> None: + s.lvalue.accept(self) + s.rvalue.accept(self) + if (isinstance(s.lvalue, NameExpr) and s.lvalue.name == '__all__' and + s.lvalue.kind == GDEF and isinstance(s.rvalue, (ListExpr, TupleExpr))): + self.add_exports(*s.rvalue.items) + + def visit_while_stmt(self, s: WhileStmt) -> None: + s.expr.accept(self) + self.loop_depth += 1 + s.body.accept(self) + self.loop_depth -= 1 + self.visit_block_maybe(s.else_body) + + def visit_for_stmt(self, s: ForStmt) -> None: + s.expr.accept(self) + + # Bind index variables and check if they define new names. + self.analyze_lvalue(s.index) + + self.loop_depth += 1 + self.visit_block(s.body) + self.loop_depth -= 1 + + self.visit_block_maybe(s.else_body) + + def visit_break_stmt(self, s: BreakStmt) -> None: + if self.loop_depth == 0: + self.fail("'break' outside loop", s, True, blocker=True) + + def visit_continue_stmt(self, s: ContinueStmt) -> None: + if self.loop_depth == 0: + self.fail("'continue' outside loop", s, True, blocker=True) + + def visit_if_stmt(self, s: IfStmt) -> None: + infer_reachability_of_if_statement(s, pyversion=self.options.python_version) + for i in range(len(s.expr)): + s.expr[i].accept(self) + self.visit_block(s.body[i]) + self.visit_block_maybe(s.else_body) + + def visit_try_stmt(self, s: TryStmt) -> None: + self.analyze_try_stmt(s, self) + + def analyze_try_stmt(self, s: TryStmt, visitor: NodeVisitor, + add_global: bool = False) -> None: + s.body.accept(visitor) + for type, var, handler in zip(s.types, s.vars, s.handlers): + if type: + type.accept(visitor) + if var: + self.analyze_lvalue(var, add_global=add_global) + handler.accept(visitor) + if s.else_body: + s.else_body.accept(visitor) + if s.finally_body: + s.finally_body.accept(visitor) + + def visit_with_stmt(self, s: WithStmt) -> None: + for e, n in zip(s.expr, s.target): + e.accept(self) + if n: + self.analyze_lvalue(n) + self.visit_block(s.body) + + def visit_del_stmt(self, s: DelStmt) -> None: + s.expr.accept(self) + if not self.is_valid_del_target(s.expr): + self.fail('Invalid delete target', s) + + def is_valid_del_target(self, s: Node) -> bool: + if isinstance(s, (IndexExpr, NameExpr, MemberExpr)): + return True + elif isinstance(s, TupleExpr): + return all(self.is_valid_del_target(item) for item in s.items) + + def visit_global_decl(self, g: GlobalDecl) -> None: + for name in g.names: + if name in self.nonlocal_decls[-1]: + self.fail("Name '{}' is nonlocal and global".format(name), g) + self.global_decls[-1].add(name) + + def visit_nonlocal_decl(self, d: NonlocalDecl) -> None: + if not self.is_func_scope(): + self.fail("nonlocal declaration not allowed at module level", d) + else: + for name in d.names: + for table in reversed(self.locals[:-1]): + if table is not None and name in table: + break + else: + self.fail("No binding for nonlocal '{}' found".format(name), d) + + if self.locals[-1] is not None and name in self.locals[-1]: + self.fail("Name '{}' is already defined in local " + "scope before nonlocal declaration".format(name), d) + + if name in self.global_decls[-1]: + self.fail("Name '{}' is nonlocal and global".format(name), d) + self.nonlocal_decls[-1].add(name) + + def visit_print_stmt(self, s: PrintStmt) -> None: + for arg in s.args: + arg.accept(self) + if s.target: + s.target.accept(self) + + def visit_exec_stmt(self, s: ExecStmt) -> None: + s.expr.accept(self) + if s.variables1: + s.variables1.accept(self) + if s.variables2: + s.variables2.accept(self) + + # + # Expressions + # + + def visit_name_expr(self, expr: NameExpr) -> None: + n = self.lookup(expr.name, expr) + if n: + if n.kind == BOUND_TVAR: + self.fail("'{}' is a type variable and only valid in type " + "context".format(expr.name), expr) + else: + expr.kind = n.kind + expr.node = n.node + expr.fullname = n.fullname + + def visit_super_expr(self, expr: SuperExpr) -> None: + if not self.type: + self.fail('"super" used outside class', expr) + return + expr.info = self.type + + def visit_tuple_expr(self, expr: TupleExpr) -> None: + for item in expr.items: + item.accept(self) + + def visit_list_expr(self, expr: ListExpr) -> None: + for item in expr.items: + item.accept(self) + + def visit_set_expr(self, expr: SetExpr) -> None: + for item in expr.items: + item.accept(self) + + def visit_dict_expr(self, expr: DictExpr) -> None: + for key, value in expr.items: + key.accept(self) + value.accept(self) + + def visit_star_expr(self, expr: StarExpr) -> None: + if not expr.valid: + self.fail('Can use starred expression only as assignment target', expr) + else: + expr.expr.accept(self) + + def visit_yield_from_expr(self, e: YieldFromExpr) -> None: + if not self.is_func_scope(): # not sure + self.fail("'yield from' outside function", e, True, blocker=True) + else: + self.function_stack[-1].is_generator = True + if e.expr: + e.expr.accept(self) + + def visit_call_expr(self, expr: CallExpr) -> None: + """Analyze a call expression. + + Some call expressions are recognized as special forms, including + cast(...) and Any(...). + """ + expr.callee.accept(self) + if refers_to_fullname(expr.callee, 'typing.cast'): + # Special form cast(...). + if not self.check_fixed_args(expr, 2, 'cast'): + return + # Translate first argument to an unanalyzed type. + try: + target = expr_to_unanalyzed_type(expr.args[0]) + except TypeTranslationError: + self.fail('Cast target is not a type', expr) + return + # Piggyback CastExpr object to the CallExpr object; it takes + # precedence over the CallExpr semantics. + expr.analyzed = CastExpr(expr.args[1], target) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, 'builtins.reveal_type'): + if not self.check_fixed_args(expr, 1, 'reveal_type'): + return + expr.analyzed = RevealTypeExpr(expr.args[0]) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, 'typing.Any'): + # Special form Any(...). + if not self.check_fixed_args(expr, 1, 'Any'): + return + expr.analyzed = CastExpr(expr.args[0], AnyType()) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, 'typing._promote'): + # Special form _promote(...). + if not self.check_fixed_args(expr, 1, '_promote'): + return + # Translate first argument to an unanalyzed type. + try: + target = expr_to_unanalyzed_type(expr.args[0]) + except TypeTranslationError: + self.fail('Argument 1 to _promote is not a type', expr) + return + expr.analyzed = PromoteExpr(target) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, 'builtins.dict'): + expr.analyzed = self.translate_dict_call(expr) + else: + # Normal call expression. + for a in expr.args: + a.accept(self) + + if (isinstance(expr.callee, MemberExpr) and + isinstance(expr.callee.expr, NameExpr) and + expr.callee.expr.name == '__all__' and + expr.callee.expr.kind == GDEF and + expr.callee.name in ('append', 'extend')): + if expr.callee.name == 'append' and expr.args: + self.add_exports(expr.args[0]) + elif (expr.callee.name == 'extend' and expr.args and + isinstance(expr.args[0], (ListExpr, TupleExpr))): + self.add_exports(*expr.args[0].items) + + def translate_dict_call(self, call: CallExpr) -> Optional[DictExpr]: + """Translate 'dict(x=y, ...)' to {'x': y, ...}. + + For other variants of dict(...), return None. + """ + if not call.args: + return None + if not all(kind == ARG_NAMED for kind in call.arg_kinds): + # Must still accept those args. + for a in call.args: + a.accept(self) + return None + expr = DictExpr([(StrExpr(key), value) + for key, value in zip(call.arg_names, call.args)]) + expr.set_line(call) + expr.accept(self) + return expr + + def check_fixed_args(self, expr: CallExpr, numargs: int, + name: str) -> bool: + """Verify that expr has specified number of positional args. + + Return True if the arguments are valid. + """ + s = 's' + if numargs == 1: + s = '' + if len(expr.args) != numargs: + self.fail("'%s' expects %d argument%s" % (name, numargs, s), + expr) + return False + if expr.arg_kinds != [ARG_POS] * numargs: + self.fail("'%s' must be called with %s positional argument%s" % + (name, numargs, s), expr) + return False + return True + + def visit_member_expr(self, expr: MemberExpr) -> None: + base = expr.expr + base.accept(self) + # Bind references to module attributes. + if isinstance(base, RefExpr) and base.kind == MODULE_REF: + file = cast(MypyFile, base.node) + n = file.names.get(expr.name, None) if file is not None else None + if n: + n = self.normalize_type_alias(n, expr) + if not n: + return + expr.kind = n.kind + expr.fullname = n.fullname + expr.node = n.node + else: + # We only catch some errors here; the rest will be + # catched during type checking. + # + # This way we can report a larger number of errors in + # one type checker run. If we reported errors here, + # the build would terminate after semantic analysis + # and we wouldn't be able to report any type errors. + full_name = '%s.%s' % (file.fullname() if file is not None else None, expr.name) + if full_name in obsolete_name_mapping: + self.fail("Module has no attribute %r (it's now called %r)" % ( + expr.name, obsolete_name_mapping[full_name]), expr) + + def visit_op_expr(self, expr: OpExpr) -> None: + expr.left.accept(self) + expr.right.accept(self) + + def visit_comparison_expr(self, expr: ComparisonExpr) -> None: + for operand in expr.operands: + operand.accept(self) + + def visit_unary_expr(self, expr: UnaryExpr) -> None: + expr.expr.accept(self) + + def visit_index_expr(self, expr: IndexExpr) -> None: + expr.base.accept(self) + if refers_to_class_or_function(expr.base): + # Special form -- type application. + # Translate index to an unanalyzed type. + types = [] # type: List[Type] + if isinstance(expr.index, TupleExpr): + items = expr.index.items + else: + items = [expr.index] + for item in items: + try: + typearg = expr_to_unanalyzed_type(item) + except TypeTranslationError: + self.fail('Type expected within [...]', expr) + return + typearg = self.anal_type(typearg) + types.append(typearg) + expr.analyzed = TypeApplication(expr.base, types) + expr.analyzed.line = expr.line + else: + expr.index.accept(self) + + def visit_slice_expr(self, expr: SliceExpr) -> None: + if expr.begin_index: + expr.begin_index.accept(self) + if expr.end_index: + expr.end_index.accept(self) + if expr.stride: + expr.stride.accept(self) + + def visit_cast_expr(self, expr: CastExpr) -> None: + expr.expr.accept(self) + expr.type = self.anal_type(expr.type) + + def visit_reveal_type_expr(self, expr: RevealTypeExpr) -> None: + expr.expr.accept(self) + + def visit_type_application(self, expr: TypeApplication) -> None: + expr.expr.accept(self) + for i in range(len(expr.types)): + expr.types[i] = self.anal_type(expr.types[i]) + + def visit_list_comprehension(self, expr: ListComprehension) -> None: + expr.generator.accept(self) + + def visit_set_comprehension(self, expr: SetComprehension) -> None: + expr.generator.accept(self) + + def visit_dictionary_comprehension(self, expr: DictionaryComprehension) -> None: + self.enter() + self.analyze_comp_for(expr) + expr.key.accept(self) + expr.value.accept(self) + self.leave() + self.analyze_comp_for_2(expr) + + def visit_generator_expr(self, expr: GeneratorExpr) -> None: + self.enter() + self.analyze_comp_for(expr) + expr.left_expr.accept(self) + self.leave() + self.analyze_comp_for_2(expr) + + def analyze_comp_for(self, expr: Union[GeneratorExpr, + DictionaryComprehension]) -> None: + """Analyses the 'comp_for' part of comprehensions (part 1). + + That is the part after 'for' in (x for x in l if p). This analyzes + variables and conditions which are analyzed in a local scope. + """ + for i, (index, sequence, conditions) in enumerate(zip(expr.indices, + expr.sequences, + expr.condlists)): + if i > 0: + sequence.accept(self) + # Bind index variables. + self.analyze_lvalue(index) + for cond in conditions: + cond.accept(self) + + def analyze_comp_for_2(self, expr: Union[GeneratorExpr, + DictionaryComprehension]) -> None: + """Analyses the 'comp_for' part of comprehensions (part 2). + + That is the part after 'for' in (x for x in l if p). This analyzes + the 'l' part which is analyzed in the surrounding scope. + """ + expr.sequences[0].accept(self) + + def visit_func_expr(self, expr: FuncExpr) -> None: + self.analyze_function(expr) + + def visit_conditional_expr(self, expr: ConditionalExpr) -> None: + expr.if_expr.accept(self) + expr.cond.accept(self) + expr.else_expr.accept(self) + + def visit_backquote_expr(self, expr: BackquoteExpr) -> None: + expr.expr.accept(self) + + def visit__promote_expr(self, expr: PromoteExpr) -> None: + expr.type = self.anal_type(expr.type) + + def visit_yield_expr(self, expr: YieldExpr) -> None: + if not self.is_func_scope(): + self.fail("'yield' outside function", expr, True, blocker=True) + else: + self.function_stack[-1].is_generator = True + if expr.expr: + expr.expr.accept(self) + + # + # Helpers + # + + def lookup(self, name: str, ctx: Context) -> SymbolTableNode: + """Look up an unqualified name in all active namespaces.""" + # 1a. Name declared using 'global x' takes precedence + if name in self.global_decls[-1]: + if name in self.globals: + return self.globals[name] + else: + self.name_not_defined(name, ctx) + return None + # 1b. Name declared using 'nonlocal x' takes precedence + if name in self.nonlocal_decls[-1]: + for table in reversed(self.locals[:-1]): + if table is not None and name in table: + return table[name] + else: + self.name_not_defined(name, ctx) + return None + # 2. Class attributes (if within class definition) + if self.is_class_scope() and name in self.type.names: + return self.type[name] + # 3. Local (function) scopes + for table in reversed(self.locals): + if table is not None and name in table: + return table[name] + # 4. Current file global scope + if name in self.globals: + return self.globals[name] + # 5. Builtins + b = self.globals.get('__builtins__', None) + if b: + table = cast(MypyFile, b.node).names + if name in table: + if name[0] == "_" and name[1] != "_": + self.name_not_defined(name, ctx) + return None + node = table[name] + return node + # Give up. + self.name_not_defined(name, ctx) + self.check_for_obsolete_short_name(name, ctx) + return None + + def check_for_obsolete_short_name(self, name: str, ctx: Context) -> None: + matches = [obsolete_name + for obsolete_name in obsolete_name_mapping + if obsolete_name.rsplit('.', 1)[-1] == name] + if len(matches) == 1: + self.note("(Did you mean '{}'?)".format(obsolete_name_mapping[matches[0]]), ctx) + + def lookup_qualified(self, name: str, ctx: Context) -> SymbolTableNode: + if '.' not in name: + return self.lookup(name, ctx) + else: + parts = name.split('.') + n = self.lookup(parts[0], ctx) # type: SymbolTableNode + if n: + for i in range(1, len(parts)): + if isinstance(n.node, TypeInfo): + if n.node.mro is None: + # We haven't yet analyzed the class `n.node`. Fall back to direct + # lookup in the names declared directly under it, without its base + # classes. This can happen when we have a forward reference to a + # nested class, and the reference is bound before the outer class + # has been fully semantically analyzed. + # + # A better approach would be to introduce a new analysis pass or + # to move things around between passes, but this unblocks a common + # use case even though this is a little limited in case there is + # inheritance involved. + result = n.node.names.get(parts[i]) + else: + result = n.node.get(parts[i]) + n = result + elif isinstance(n.node, MypyFile): + n = n.node.names.get(parts[i], None) + # TODO: What if node is Var or FuncDef? + if not n: + self.name_not_defined(name, ctx) + break + if n: + n = self.normalize_type_alias(n, ctx) + return n + + def builtin_type(self, fully_qualified_name: str) -> Instance: + node = self.lookup_fully_qualified(fully_qualified_name) + info = cast(TypeInfo, node.node) + return Instance(info, []) + + def lookup_fully_qualified(self, name: str) -> SymbolTableNode: + """Lookup a fully qualified name. + + Assume that the name is defined. This happens in the global namespace -- the local + module namespace is ignored. + """ + assert '.' in name + parts = name.split('.') + n = self.modules[parts[0]] + for i in range(1, len(parts) - 1): + n = cast(MypyFile, n.names[parts[i]].node) + return n.names[parts[-1]] + + def lookup_fully_qualified_or_none(self, name: str) -> SymbolTableNode: + """Lookup a fully qualified name. + + Assume that the name is defined. This happens in the global namespace -- the local + module namespace is ignored. + """ + assert '.' in name + parts = name.split('.') + n = self.modules[parts[0]] + for i in range(1, len(parts) - 1): + next_sym = n.names.get(parts[i]) + if not next_sym: + return None + n = cast(MypyFile, next_sym.node) + return n.names.get(parts[-1]) + + def qualified_name(self, n: str) -> str: + return self.cur_mod_id + '.' + n + + def enter(self) -> None: + self.locals.append(SymbolTable()) + self.global_decls.append(set()) + self.nonlocal_decls.append(set()) + # -1 since entering block will increment this to 0. + self.block_depth.append(-1) + + def leave(self) -> None: + self.locals.pop() + self.global_decls.pop() + self.nonlocal_decls.pop() + self.block_depth.pop() + + def is_func_scope(self) -> bool: + return self.locals[-1] is not None + + def is_class_scope(self) -> bool: + return self.type is not None and not self.is_func_scope() + + def add_symbol(self, name: str, node: SymbolTableNode, + context: Context) -> None: + if self.is_func_scope(): + if name in self.locals[-1]: + # Flag redefinition unless this is a reimport of a module. + if not (node.kind == MODULE_REF and + self.locals[-1][name].node == node.node): + self.name_already_defined(name, context) + self.locals[-1][name] = node + elif self.type: + self.type.names[name] = node + else: + existing = self.globals.get(name) + if existing and (not isinstance(node.node, MypyFile) or + existing.node != node.node) and existing.kind != UNBOUND_IMPORTED: + # Modules can be imported multiple times to support import + # of multiple submodules of a package (e.g. a.x and a.y). + ok = False + # Only report an error if the symbol collision provides a different type. + if existing.type and node.type and is_same_type(existing.type, node.type): + ok = True + if not ok: + self.name_already_defined(name, context) + self.globals[name] = node + + def add_var(self, v: Var, ctx: Context) -> None: + if self.is_func_scope(): + self.add_local(v, ctx) + else: + self.globals[v.name()] = SymbolTableNode(GDEF, v, self.cur_mod_id) + v._fullname = self.qualified_name(v.name()) + + def add_local(self, node: Union[Var, FuncBase], ctx: Context) -> None: + name = node.name() + if name in self.locals[-1]: + self.name_already_defined(name, ctx) + node._fullname = name + self.locals[-1][name] = SymbolTableNode(LDEF, node) + + def add_exports(self, *exps: Node) -> None: + for exp in exps: + if isinstance(exp, StrExpr): + self.all_exports.add(exp.value) + + def check_no_global(self, n: str, ctx: Context, + is_overloaded_func: bool = False) -> None: + if n in self.globals: + prev_is_overloaded = isinstance(self.globals[n], OverloadedFuncDef) + if is_overloaded_func and prev_is_overloaded: + self.fail("Nonconsecutive overload {} found".format(n), ctx) + elif prev_is_overloaded: + self.fail("Definition of '{}' missing 'overload'".format(n), ctx) + else: + self.name_already_defined(n, ctx) + + def name_not_defined(self, name: str, ctx: Context) -> None: + message = "Name '{}' is not defined".format(name) + extra = self.undefined_name_extra_info(name) + if extra: + message += ' {}'.format(extra) + self.fail(message, ctx) + + def name_already_defined(self, name: str, ctx: Context) -> None: + self.fail("Name '{}' already defined".format(name), ctx) + + def fail(self, msg: str, ctx: Context, serious: bool = False, *, + blocker: bool = False) -> None: + if (not serious and + not self.options.check_untyped_defs and + self.function_stack and + self.function_stack[-1].is_dynamic()): + return + self.errors.report(ctx.get_line(), msg, blocker=blocker) + + def note(self, msg: str, ctx: Context) -> None: + if (not self.options.check_untyped_defs and + self.function_stack and + self.function_stack[-1].is_dynamic()): + return + self.errors.report(ctx.get_line(), msg, severity='note') + + def undefined_name_extra_info(self, fullname: str) -> Optional[str]: + if fullname in obsolete_name_mapping: + return "(it's now called '{}')".format(obsolete_name_mapping[fullname]) + else: + return None + + def accept(self, node: Node) -> None: + try: + node.accept(self) + except Exception as err: + report_internal_error(err, self.errors.file, node.line) + + +class FirstPass(NodeVisitor): + """First phase of semantic analysis. + + See docstring of 'analyze()' below for a description of what this does. + """ + + def __init__(self, sem: SemanticAnalyzer) -> None: + self.sem = sem + self.pyversion = sem.options.python_version + + def analyze(self, file: MypyFile, fnam: str, mod_id: str) -> None: + """Perform the first analysis pass. + + Populate module global table. Resolve the full names of + definitions not nested within functions and construct type + info structures, but do not resolve inter-definition + references such as base classes. + + Also add implicit definitions such as __name__. + + In this phase we don't resolve imports. For 'from ... import', + we generate dummy symbol table nodes for the imported names, + and these will get resolved in later phases of semantic + analysis. + """ + sem = self.sem + sem.cur_mod_id = mod_id + sem.errors.set_file(fnam) + sem.globals = SymbolTable() + sem.global_decls = [set()] + sem.nonlocal_decls = [set()] + sem.block_depth = [0] + + defs = file.defs + + # Add implicit definitions of module '__name__' etc. + for name, t in implicit_module_attrs.items(): + v = Var(name, UnboundType(t)) + v._fullname = self.sem.qualified_name(name) + self.sem.globals[name] = SymbolTableNode(GDEF, v, self.sem.cur_mod_id) + + for d in defs: + d.accept(self) + + # Add implicit definition of literals/keywords to builtins, as we + # cannot define a variable with them explicitly. + if mod_id == 'builtins': + literal_types = [ + ('None', NoneTyp()), + # reveal_type is a mypy-only function that gives an error with the type of its arg + ('reveal_type', AnyType()), + ] # type: List[Tuple[str, Type]] + + # TODO(ddfisher): This guard is only needed because mypy defines + # fake builtins for its tests which often don't define bool. If + # mypy is fast enough that we no longer need those, this + # conditional check should be removed. + if 'bool' in self.sem.globals: + bool_type = self.sem.named_type('bool') + literal_types.extend([ + ('True', bool_type), + ('False', bool_type), + ('__debug__', bool_type), + ]) + + for name, typ in literal_types: + v = Var(name, typ) + v._fullname = self.sem.qualified_name(name) + self.sem.globals[name] = SymbolTableNode(GDEF, v, self.sem.cur_mod_id) + + def visit_block(self, b: Block) -> None: + if b.is_unreachable: + return + self.sem.block_depth[-1] += 1 + for node in b.body: + node.accept(self) + self.sem.block_depth[-1] -= 1 + + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + for lval in s.lvalues: + self.analyze_lvalue(lval, explicit_type=s.type is not None) + + def visit_func_def(self, func: FuncDef) -> None: + sem = self.sem + func.is_conditional = sem.block_depth[-1] > 0 + func._fullname = sem.qualified_name(func.name()) + if func.name() in sem.globals: + # Already defined in this module. + original_sym = sem.globals[func.name()] + if original_sym.kind == UNBOUND_IMPORTED: + # Ah this is an imported name. We can't resolve them now, so we'll postpone + # this until the main phase of semantic analysis. + return + original_def = original_sym.node + if sem.is_conditional_func(original_def, func): + # Conditional function definition -- multiple defs are ok. + func.original_def = cast(FuncDef, original_def) + else: + # Report error. + sem.check_no_global(func.name(), func) + else: + sem.globals[func.name()] = SymbolTableNode(GDEF, func, sem.cur_mod_id) + + def visit_overloaded_func_def(self, func: OverloadedFuncDef) -> None: + self.sem.check_no_global(func.name(), func, True) + func._fullname = self.sem.qualified_name(func.name()) + self.sem.globals[func.name()] = SymbolTableNode(GDEF, func, + self.sem.cur_mod_id) + + def visit_class_def(self, cdef: ClassDef) -> None: + self.sem.check_no_global(cdef.name, cdef) + cdef.fullname = self.sem.qualified_name(cdef.name) + info = TypeInfo(SymbolTable(), cdef) + info.set_line(cdef.line) + cdef.info = info + self.sem.globals[cdef.name] = SymbolTableNode(GDEF, info, + self.sem.cur_mod_id) + self.process_nested_classes(cdef) + + def process_nested_classes(self, outer_def: ClassDef) -> None: + for node in outer_def.defs.body: + if isinstance(node, ClassDef): + node.info = TypeInfo(SymbolTable(), node) + if outer_def.fullname: + node.info._fullname = outer_def.fullname + '.' + node.info.name() + else: + node.info._fullname = node.info.name() + symbol = SymbolTableNode(MDEF, node.info) + outer_def.info.names[node.name] = symbol + self.process_nested_classes(node) + + def visit_import_from(self, node: ImportFrom) -> None: + # We can't bind module names during the first pass, as the target module might be + # unprocessed. However, we add dummy unbound imported names to the symbol table so + # that we at least know that the name refers to a module. + node.is_top_level = True + for name, as_name in node.names: + imported_name = as_name or name + if imported_name not in self.sem.globals: + self.sem.add_symbol(imported_name, SymbolTableNode(UNBOUND_IMPORTED, None), node) + + def visit_import(self, node: Import) -> None: + node.is_top_level = True + # This is similar to visit_import_from -- see the comment there. + for id, as_id in node.ids: + imported_id = as_id or id + if imported_id not in self.sem.globals: + self.sem.add_symbol(imported_id, SymbolTableNode(UNBOUND_IMPORTED, None), node) + else: + # If the previous symbol is a variable, this should take precedence. + self.sem.globals[imported_id] = SymbolTableNode(UNBOUND_IMPORTED, None) + + def visit_import_all(self, node: ImportAll) -> None: + node.is_top_level = True + + def visit_while_stmt(self, s: WhileStmt) -> None: + s.body.accept(self) + if s.else_body: + s.else_body.accept(self) + + def visit_for_stmt(self, s: ForStmt) -> None: + self.analyze_lvalue(s.index) + s.body.accept(self) + if s.else_body: + s.else_body.accept(self) + + def visit_with_stmt(self, s: WithStmt) -> None: + for n in s.target: + if n: + self.analyze_lvalue(n) + s.body.accept(self) + + def visit_decorator(self, d: Decorator) -> None: + d.var._fullname = self.sem.qualified_name(d.var.name()) + self.sem.add_symbol(d.var.name(), SymbolTableNode(GDEF, d.var), d) + + def visit_if_stmt(self, s: IfStmt) -> None: + infer_reachability_of_if_statement(s, pyversion=self.pyversion) + for node in s.body: + node.accept(self) + if s.else_body: + s.else_body.accept(self) + + def visit_try_stmt(self, s: TryStmt) -> None: + self.sem.analyze_try_stmt(s, self, add_global=True) + + def analyze_lvalue(self, lvalue: Node, explicit_type: bool = False) -> None: + self.sem.analyze_lvalue(lvalue, add_global=True, explicit_type=explicit_type) + + +class ThirdPass(TraverserVisitor): + """The third and final pass of semantic analysis. + + Check type argument counts and values of generic types, and perform some + straightforward type inference. + """ + + def __init__(self, modules: Dict[str, MypyFile], errors: Errors) -> None: + self.modules = modules + self.errors = errors + + def visit_file(self, file_node: MypyFile, fnam: str) -> None: + self.errors.set_file(fnam) + self.accept(file_node) + + def accept(self, node: Node) -> None: + try: + node.accept(self) + except Exception as err: + report_internal_error(err, self.errors.file, node.line) + + def visit_block(self, b: Block) -> None: + if b.is_unreachable: + return + super().visit_block(b) + + def visit_func_def(self, fdef: FuncDef) -> None: + self.errors.push_function(fdef.name()) + self.analyze(fdef.type) + super().visit_func_def(fdef) + self.errors.pop_function() + + def visit_class_def(self, tdef: ClassDef) -> None: + for type in tdef.info.bases: + self.analyze(type) + # Recompute MRO now that we have analyzed all modules, to pick + # up superclasses of bases imported from other modules in an + # import loop. (Only do so if we succeeded the first time.) + if tdef.info.mro: + tdef.info.mro = [] # Force recomputation + calculate_class_mro(tdef, self.fail) + super().visit_class_def(tdef) + + def visit_decorator(self, dec: Decorator) -> None: + """Try to infer the type of the decorated function. + + This lets us resolve references to decorated functions during + type checking when there are cyclic imports, as otherwise the + type might not be available when we need it. + + This basically uses a simple special-purpose type inference + engine just for decorators. + """ + super().visit_decorator(dec) + if dec.var.is_property: + # Decorators are expected to have a callable type (it's a little odd). + if dec.func.type is None: + dec.var.type = CallableType( + [AnyType()], + [ARG_POS], + [None], + AnyType(), + self.builtin_type('function'), + name=dec.var.name()) + elif isinstance(dec.func.type, CallableType): + dec.var.type = dec.func.type + return + decorator_preserves_type = True + for expr in dec.decorators: + preserve_type = False + if isinstance(expr, RefExpr) and isinstance(expr.node, FuncDef): + if is_identity_signature(expr.node.type): + preserve_type = True + if not preserve_type: + decorator_preserves_type = False + break + if decorator_preserves_type: + # No non-identity decorators left. We can trivially infer the type + # of the function here. + dec.var.type = function_type(dec.func, self.builtin_type('function')) + if dec.decorators: + if returns_any_if_called(dec.decorators[0]): + # The outermost decorator will return Any so we know the type of the + # decorated function. + dec.var.type = AnyType() + sig = find_fixed_callable_return(dec.decorators[0]) + if sig: + # The outermost decorator always returns the same kind of function, + # so we know that this is the type of the decoratored function. + orig_sig = function_type(dec.func, self.builtin_type('function')) + sig.name = orig_sig.items()[0].name + dec.var.type = sig + + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + self.analyze(s.type) + super().visit_assignment_stmt(s) + + def visit_cast_expr(self, e: CastExpr) -> None: + self.analyze(e.type) + super().visit_cast_expr(e) + + def visit_reveal_type_expr(self, e: RevealTypeExpr) -> None: + super().visit_reveal_type_expr(e) + + def visit_type_application(self, e: TypeApplication) -> None: + for type in e.types: + self.analyze(type) + super().visit_type_application(e) + + # Helpers + + def analyze(self, type: Type) -> None: + if type: + analyzer = TypeAnalyserPass3(self.fail) + type.accept(analyzer) + + def fail(self, msg: str, ctx: Context) -> None: + self.errors.report(ctx.get_line(), msg) + + def builtin_type(self, name: str, args: List[Type] = None) -> Instance: + names = self.modules['builtins'] + sym = names.names[name] + assert isinstance(sym.node, TypeInfo) + return Instance(sym.node, args or []) + + +def self_type(typ: TypeInfo) -> Union[Instance, TupleType]: + """For a non-generic type, return instance type representing the type. + For a generic G type with parameters T1, .., Tn, return G[T1, ..., Tn]. + """ + tv = [] # type: List[Type] + for i in range(len(typ.type_vars)): + tv.append(TypeVarType(typ.defn.type_vars[i])) + return Instance(typ, tv) + + +def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: + if isinstance(sig, CallableType): + return replace_leading_arg_type(sig, new) + else: + sig = cast(Overloaded, sig) + return Overloaded([cast(CallableType, replace_implicit_first_type(i, new)) + for i in sig.items()]) + + +def set_callable_name(sig: Type, fdef: FuncDef) -> Type: + if isinstance(sig, FunctionLike): + if fdef.info: + return sig.with_name( + '"{}" of "{}"'.format(fdef.name(), fdef.info.name())) + else: + return sig.with_name('"{}"'.format(fdef.name())) + else: + return sig + + +def refers_to_fullname(node: Node, fullname: str) -> bool: + """Is node a name or member expression with the given full name?""" + return isinstance(node, RefExpr) and node.fullname == fullname + + +def refers_to_class_or_function(node: Node) -> bool: + """Does semantically analyzed node refer to a class?""" + return (isinstance(node, RefExpr) and + isinstance(node.node, (TypeInfo, FuncDef, OverloadedFuncDef))) + + +def calculate_class_mro(defn: ClassDef, fail: Callable[[str, Context], None]) -> None: + try: + defn.info.calculate_mro() + except MroError: + fail("Cannot determine consistent method resolution order " + '(MRO) for "%s"' % defn.name, defn) + defn.info.mro = [] + # The property of falling back to Any is inherited. + defn.info.fallback_to_any = any(baseinfo.fallback_to_any for baseinfo in defn.info.mro) + + +def find_duplicate(list: List[T]) -> T: + """If the list has duplicates, return one of the duplicates. + + Otherwise, return None. + """ + for i in range(1, len(list)): + if list[i] in list[:i]: + return list[i] + return None + + +def disable_typevars(nodes: List[SymbolTableNode]) -> None: + for node in nodes: + assert node.kind in (BOUND_TVAR, UNBOUND_TVAR) + node.kind = UNBOUND_TVAR + + +def enable_typevars(nodes: List[SymbolTableNode]) -> None: + for node in nodes: + assert node.kind in (BOUND_TVAR, UNBOUND_TVAR) + node.kind = BOUND_TVAR + + +def remove_imported_names_from_symtable(names: SymbolTable, + module: str) -> None: + """Remove all imported names from the symbol table of a module.""" + removed = [] # type: List[str] + for name, node in names.items(): + if node.node is None: + continue + fullname = node.node.fullname() + prefix = fullname[:fullname.rfind('.')] + if prefix != module: + removed.append(name) + for name in removed: + del names[name] + + +def infer_reachability_of_if_statement(s: IfStmt, + pyversion: Tuple[int, int]) -> None: + for i in range(len(s.expr)): + result = infer_if_condition_value(s.expr[i], pyversion) + if result == ALWAYS_FALSE: + # The condition is always false, so we skip the if/elif body. + mark_block_unreachable(s.body[i]) + elif result == ALWAYS_TRUE: + # This condition is always true, so all of the remaining + # elif/else bodies will never be executed. + for body in s.body[i + 1:]: + mark_block_unreachable(body) + if s.else_body: + mark_block_unreachable(s.else_body) + break + + +def infer_if_condition_value(expr: Node, pyversion: Tuple[int, int]) -> int: + """Infer whether if condition is always true/false. + + Return ALWAYS_TRUE if always true, ALWAYS_FALSE if always false, + and TRUTH_VALUE_UNKNOWN otherwise. + """ + name = '' + negated = False + alias = expr + if isinstance(alias, UnaryExpr): + if alias.op == 'not': + expr = alias.expr + negated = True + if isinstance(expr, NameExpr): + name = expr.name + elif isinstance(expr, MemberExpr): + name = expr.name + result = TRUTH_VALUE_UNKNOWN + if name == 'PY2': + result = ALWAYS_TRUE if pyversion[0] == 2 else ALWAYS_FALSE + elif name == 'PY3': + result = ALWAYS_TRUE if pyversion[0] == 3 else ALWAYS_FALSE + elif name == 'MYPY': + result = ALWAYS_TRUE + if negated: + if result == ALWAYS_TRUE: + result = ALWAYS_FALSE + elif result == ALWAYS_FALSE: + result = ALWAYS_TRUE + return result + + +def mark_block_unreachable(block: Block) -> None: + block.is_unreachable = True + block.accept(MarkImportsUnreachableVisitor()) + + +class MarkImportsUnreachableVisitor(TraverserVisitor): + """Visitor that flags all imports nested within a node as unreachable.""" + + def visit_import(self, node: Import) -> None: + node.is_unreachable = True + + def visit_import_from(self, node: ImportFrom) -> None: + node.is_unreachable = True + + def visit_import_all(self, node: ImportAll) -> None: + node.is_unreachable = True + + +def is_identity_signature(sig: Type) -> bool: + """Is type a callable of form T -> T (where T is a type variable)?""" + if isinstance(sig, CallableType) and sig.arg_kinds == [ARG_POS]: + if isinstance(sig.arg_types[0], TypeVarType) and isinstance(sig.ret_type, TypeVarType): + return sig.arg_types[0].id == sig.ret_type.id + return False + + +def returns_any_if_called(expr: Node) -> bool: + """Return True if we can predict that expr will return Any if called. + + This only uses information available during semantic analysis so this + will sometimes return False because of insufficient information (as + type inference hasn't run yet). + """ + if isinstance(expr, RefExpr): + if isinstance(expr.node, FuncDef): + typ = expr.node.type + if typ is None: + # No signature -> default to Any. + return True + # Explicit Any return? + return isinstance(typ, CallableType) and isinstance(typ.ret_type, AnyType) + elif isinstance(expr.node, Var): + typ = expr.node.type + return typ is None or isinstance(typ, AnyType) + elif isinstance(expr, CallExpr): + return returns_any_if_called(expr.callee) + return False + + +def find_fixed_callable_return(expr: Node) -> Optional[CallableType]: + if isinstance(expr, RefExpr): + if isinstance(expr.node, FuncDef): + typ = expr.node.type + if typ: + if isinstance(typ, CallableType) and has_no_typevars(typ.ret_type): + if isinstance(typ.ret_type, CallableType): + return typ.ret_type + elif isinstance(expr, CallExpr): + t = find_fixed_callable_return(expr.callee) + if t: + if isinstance(t.ret_type, CallableType): + return t.ret_type + return None + + +def has_no_typevars(typ: Type) -> bool: + return is_same_type(typ, erase_typevars(typ)) diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 9c2d6c4a86d8f..cd23b4fea08bd 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -1,209 +1,210 @@ -[case testNamedTupleUsedAsTuple] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -a, b = x -b = x[0] -a = x[1] -a, b, c = x # E: Need more than 2 values to unpack (3 expected) -x[2] # E: Tuple index out of range - -[case testAccessingNamedTupleAttributes] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -x.x -x.y -x.z # E: "X" has no attribute "z" - -[case testCreateNamedTupleWithPositionalArguments] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = X(1, 'x') -x.x -x.z # E: "X" has no attribute "z" -x = X(1) # E: Too few arguments for "X" -x = X(1, 2, 3) # E: Too many arguments for "X" - -[case testCreateNamedTupleWithKeywordArguments] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = X(x=1, y='x') -x = X(1, y='x') -x = X(x=1, z=1) # E: Unexpected keyword argument "z" for "X" -x = X(y=1) # E: Missing positional argument "x" in call to "X" - -[case testCreateNamedTupleAndUseAsTuple] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = X(1, 'x') -a, b = x -a, b, c = x # E: Need more than 2 values to unpack (3 expected) - -[case testNamedTupleWithItemTypes] -from typing import NamedTuple -N = NamedTuple('N', [('a', int), - ('b', str)]) -n = N(1, 'x') -s = n.a # type: str # E: Incompatible types in assignment (expression has type "int", \ - variable has type "str") -i = n.b # type: int # E: Incompatible types in assignment (expression has type "str", \ - variable has type "int") -x, y = n -x = y # E: Incompatible types in assignment (expression has type "str", variable has type "int") - -[case testNamedTupleConstructorArgumentTypes] -from typing import NamedTuple -N = NamedTuple('N', [('a', int), - ('b', str)]) -n = N('x', 'x') # E: Argument 1 to "N" has incompatible type "str"; expected "int" -n = N(1, b=2) # E: Argument 2 to "N" has incompatible type "int"; expected "str" -N(1, 'x') -N(b='x', a=1) - -[case testNamedTupleAsBaseClass] -from typing import NamedTuple -N = NamedTuple('N', [('a', int), - ('b', str)]) -class X(N): - pass -x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" -s = '' -i = 0 -s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") -i, s = x -s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") - -[case testNamedTupleAsBaseClass2] -from typing import NamedTuple -class X(NamedTuple('N', [('a', int), - ('b', str)])): - pass -x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" -s = '' -i = 0 -s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") -i, s = x -s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") - -[case testTwoNamedTuplesAsBaseClasses] -from typing import NamedTuple -A = NamedTuple('A', [('a', int)]) -B = NamedTuple('B', [('a', int)]) -class X(A, B): # E: Class has two incompatible bases derived from tuple - pass - -[case testTwoNamedTuplesAsBaseClasses2] -from typing import NamedTuple -A = NamedTuple('A', [('a', int)]) -class X(A, NamedTuple('B', [('a', int)])): # E: Class has two incompatible bases derived from tuple - pass - -[case testSelfTypeWithNamedTupleAsBase] -from typing import NamedTuple -A = NamedTuple('A', [('a', int), ('b', str)]) -class B(A): - def f(self, x: int) -> None: - self.f(self.a) - self.f(self.b) # E: Argument 1 to "f" of "B" has incompatible type "str"; expected "int" - i = 0 - s = '' - i, s = self - i, i = self # E: Incompatible types in assignment (expression has type "str", \ - variable has type "int") -[out] -main: note: In member "f" of class "B": - -[case testTypeReferenceToClassDerivedFromNamedTuple] -from typing import NamedTuple -A = NamedTuple('A', [('a', int), ('b', str)]) -class B(A): - def f(self, x: 'B') -> None: - i = 0 - s = '' - self = x - i, s = x - i, s = x.a, x.b - i, s = x.a, x.a # E: Incompatible types in assignment (expression has type "int", \ - variable has type "str") - i, i = self # E: Incompatible types in assignment (expression has type "str", \ - variable has type "int") -[out] -main: note: In member "f" of class "B": - -[case testNamedTupleSubtyping] -from typing import NamedTuple, Tuple -A = NamedTuple('A', [('a', int), ('b', str)]) -class B(A): pass -a = A(1, '') -b = B(1, '') -t = None # type: Tuple[int, str] -b = a # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "A") -b = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "B") -t = a -t = (1, '') -t = b -a = b - -[case testNamedTupleSimpleTypeInference] -from typing import NamedTuple, Tuple -A = NamedTuple('A', [('a', int)]) -l = [A(1), A(2)] -a = A(1) -a = l[0] -(i,) = l[0] -i, i = l[0] # E: Need more than 1 value to unpack (2 expected) -l = [A(1)] -a = (1,) # E: Incompatible types in assignment (expression has type "Tuple[int]", \ - variable has type "A") -[builtins fixtures/list.py] - -[case testNamedTupleMissingClassAttribute] -import collections -MyNamedTuple = collections.namedtuple('MyNamedTuple', ['spam', 'eggs']) -MyNamedTuple.x # E: "MyNamedTuple" has no attribute "x" - -[case testNamedTupleEmptyItems] -from typing import NamedTuple -A = NamedTuple('A', []) -[builtins fixtures/list.py] - -[case testNamedTupleProperty] -from typing import NamedTuple -A = NamedTuple('A', [('a', int)]) -class B(A): - @property - def b(self) -> int: - return self.a -class C(B): pass -B(1).b -C(2).b -[builtins fixtures/property.py] - -[case testNamedTupleAsDict] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -a = x._asdict() - -[case testNamedTupleReplace] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -x = x._replace(y=5) - -[case testNamedTupleReplaceTyped] -from typing import NamedTuple - -X = NamedTuple('X', [('x', int), ('y', str)]) -x = None # type: X -x = x._replace(x=5) -x = x._replace(y=5) # E: Argument 1 to X has incompatible type "int"; expected "str" +[case testNamedTupleUsedAsTuple] +import typing +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = X(1, 2) # type: X # None does not seem to work for TYPE_ALIAS +a, b = x +b = x[0] +a = x[1] +a, b, c = x # E: Need more than 2 values to unpack (3 expected) +x[2] # E: Tuple index out of range + +[case testAccessingNamedTupleAttributes] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +x.x +x.y +x.z # E: "X" has no attribute "z" + +[case testCreateNamedTupleWithPositionalArguments] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = X(1, 'x') +x.x +x.z # E: "X" has no attribute "z" +x = X(1) # E: Too few arguments for "X" +x = X(1, 2, 3) # E: Too many arguments for "X" + +[case testCreateNamedTupleWithKeywordArguments] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = X(x=1, y='x') +x = X(1, y='x') +x = X(x=1, z=1) # E: Unexpected keyword argument "z" for "X" +x = X(y=1) # E: Missing positional argument "x" in call to "X" + +[case testCreateNamedTupleAndUseAsTuple] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = X(1, 'x') +a, b = x +a, b, c = x # E: Need more than 2 values to unpack (3 expected) + +[case testNamedTupleWithItemTypes] +from typing import NamedTuple +N = NamedTuple('N', [('a', int), + ('b', str)]) +n = N(1, 'x') +s = n.a # type: str # E: Incompatible types in assignment (expression has type "int", \ + variable has type "str") +i = n.b # type: int # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +x, y = n +x = y # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testNamedTupleConstructorArgumentTypes] +from typing import NamedTuple +N = NamedTuple('N', [('a', int), + ('b', str)]) +n = N('x', 'x') # E: Argument 1 to "N" has incompatible type "str"; expected "int" +n = N(1, b=2) # E: Argument 2 to "N" has incompatible type "int"; expected "str" +N(1, 'x') +N(b='x', a=1) + +[case testNamedTupleAsBaseClass] +from typing import NamedTuple +N = NamedTuple('N', [('a', int), + ('b', str)]) +class X(N): + pass +x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" +s = '' +i = 0 +s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") +i, s = x +s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testNamedTupleAsBaseClass2] +from typing import NamedTuple +class X(NamedTuple('N', [('a', int), + ('b', str)])): + pass +x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" +s = '' +i = 0 +s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") +i, s = x +s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testTwoNamedTuplesAsBaseClasses] +from typing import NamedTuple +A = NamedTuple('A', [('a', int)]) +B = NamedTuple('B', [('a', int)]) +class X(A, B): # E: Class has two incompatible bases derived from tuple + pass + +[case testTwoNamedTuplesAsBaseClasses2] +from typing import NamedTuple +A = NamedTuple('A', [('a', int)]) +class X(A, NamedTuple('B', [('a', int)])): # E: Class has two incompatible bases derived from tuple + pass + +[case testSelfTypeWithNamedTupleAsBase] +from typing import NamedTuple +A = NamedTuple('A', [('a', int), ('b', str)]) +class B(A): + def f(self, x: int) -> None: + self.f(self.a) + self.f(self.b) # E: Argument 1 to "f" of "B" has incompatible type "str"; expected "int" + i = 0 + s = '' + i, s = self + i, i = self # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +[out] +main: note: In member "f" of class "B": + +[case testTypeReferenceToClassDerivedFromNamedTuple] +from typing import NamedTuple +A = NamedTuple('A', [('a', int), ('b', str)]) +class B(A): + def f(self, x: 'B') -> None: + i = 0 + s = '' + self = x + i, s = x + i, s = x.a, x.b + i, s = x.a, x.a # E: Incompatible types in assignment (expression has type "int", \ + variable has type "str") + i, i = self # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +[out] +main: note: In member "f" of class "B": + +[case testNamedTupleSubtyping] +from typing import NamedTuple, Tuple +A = NamedTuple('A', [('a', int), ('b', str)]) +class B(A): pass +a = A(1, '') +b = B(1, '') +t = None # type: Tuple[int, str] +b = a # E: Incompatible types in assignment (expression has type "A", variable has type "B") +a = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "A") +b = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "B") +t = a +t = (1, '') +t = b +a = b + +[case testNamedTupleSimpleTypeInference] +from typing import NamedTuple, Tuple +A = NamedTuple('A', [('a', int)]) +l = [A(1), A(2)] +a = A(1) +a = l[0] +(i,) = l[0] +i, i = l[0] # E: Need more than 1 value to unpack (2 expected) +l = [A(1)] +a = (1,) # E: Incompatible types in assignment (expression has type "Tuple[int]", \ + variable has type "A") +[builtins fixtures/list.py] + +[case testNamedTupleMissingClassAttribute] +import collections +MyNamedTuple = collections.namedtuple('MyNamedTuple', ['spam', 'eggs']) +MyNamedTuple.x # E: "MyNamedTuple" has no attribute "x" + +[case testNamedTupleEmptyItems] +from typing import NamedTuple +A = NamedTuple('A', []) +[builtins fixtures/list.py] + +[case testNamedTupleProperty] +from typing import NamedTuple +A = NamedTuple('A', [('a', int)]) +class B(A): + @property + def b(self) -> int: + return self.a +class C(B): pass +B(1).b +C(2).b +[builtins fixtures/property.py] + +[case testNamedTupleAsDict] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = X(1, 2) # type: X +a = x._asdict() + +[case testNamedTupleReplace] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = X(1, 2) # type: X +x = x._replace(y=5) + +[case testNamedTupleReplaceTyped] +from typing import NamedTuple + +X = NamedTuple('X', [('x', int), ('y', str)]) +x = X(1, 'a') # type: X +x = x._replace(x=5) +x = x._replace(y=5) # E: Argument 1 to X has incompatible type "int"; expected "str" From 763e0c0126aefae54f580f1401e1313095d5cf06 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Sun, 10 Jul 2016 10:33:21 +0300 Subject: [PATCH 12/74] run both Replace and AsTuple --- mypy/nodes.py | 2 +- mypy/semanal.py | 15 +++++++++++---- mypy/types.py | 8 ++++++++ 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index f4c26634ba25b..645c54dd015e7 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1977,7 +1977,7 @@ def deserialize(cls, data: JsonDict) -> 'TypeInfo': class NamedTupleTypeInfo(TypeInfo): - is_named_tuple = False + is_named_tuple = True def __init__(self, tup, *args): super().__init__(*args) tup.fullname = lambda : '__builtins__.tuple' diff --git a/mypy/semanal.py b/mypy/semanal.py index 50bd040f91536..e89d3560fcc6f 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -72,7 +72,8 @@ from mypy.types import ( NoneTyp, CallableType, Overloaded, Instance, Type, TypeVarType, AnyType, FunctionLike, UnboundType, TypeList, ErrorType, TypeVarDef, - replace_leading_arg_type, TupleType, UnionType, StarType, EllipsisType + replace_leading_arg_type, TupleType, UnionType, StarType, EllipsisType, + NamedTupleType ) from mypy.nodes import function_type, implicit_module_attrs from mypy.typeanal import TypeAnalyser, TypeAnalyserPass3, analyze_type_alias @@ -1524,7 +1525,7 @@ def fail_namedtuple_arg(self, message: str, def build_namedtuple_typeinfo(self, name: str, items: List[str], types: List[Type]) -> TypeInfo: symbols = SymbolTable() - tup = TupleType(types, self.named_type('__builtins__.tuple', [AnyType()])) + tup = NamedTupleType(items, types, self.named_type('__builtins__.tuple', [AnyType()])) class_def = ClassDef(name, Block([])) class_def.fullname = self.qualified_name(name) info = NamedTupleTypeInfo(tup, symbols, class_def) @@ -1537,7 +1538,7 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], # Add __init__, _replace, _asdict methods and _fields static field self.add_namedtuple_method('__init__', symbols, info, NoneTyp(), self.make_factory_args(vars, ARG_POS)) - self.add_namedtuple_method('_replace', symbols, info, Instance(info, []), + self.add_namedtuple_method('_replace', symbols, info, self_type(info), self.make_factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) # TODO: refine to OrderedDict[Union[types]] self.add_namedtuple_method('_asdict', symbols, info, AnyType()) @@ -2630,7 +2631,13 @@ def self_type(typ: TypeInfo) -> Union[Instance, TupleType]: tv = [] # type: List[Type] for i in range(len(typ.type_vars)): tv.append(TypeVarType(typ.defn.type_vars[i])) - return Instance(typ, tv) + inst = Instance(typ, tv) + if typ.tuple_type is None: + return inst + if not typ.is_named_tuple: + inst = TupleType(typ.tuple_type.items, inst) + return inst + return NamedTupleType(typ.tuple_type.names, typ.tuple_type.items, inst) def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: diff --git a/mypy/types.py b/mypy/types.py index 8f0f938d09d1c..26dc26bf6c12a 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -761,6 +761,14 @@ def deserialize(cls, data: JsonDict) -> 'TupleType': implicit=data['implicit']) +class NamedTupleType(TupleType): + names = None # type: List[str] + + def __init__(self, names: List[str], *args) -> None: + self.names = names + super().__init__(*args) + + class StarType(Type): """The star type *type_parameter. From 4b10a35c1d3c3692d6e347d3ceb96a52966bdcea Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Mon, 11 Jul 2016 00:09:04 +0300 Subject: [PATCH 13/74] fix most of the tests --- mypy/nodes.py | 8 ++++---- mypy/semanal.py | 11 ++++------- mypy/types.py | 12 +++++++++--- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index 645c54dd015e7..0f512d0b6d718 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1978,12 +1978,12 @@ def deserialize(cls, data: JsonDict) -> 'TypeInfo': class NamedTupleTypeInfo(TypeInfo): is_named_tuple = True - def __init__(self, tup, *args): + bases = None # type: List[mypy.types.Instance] + def __init__(self, tup: 'mypy.types.NamedTupleType', *args) -> None: super().__init__(*args) - tup.fullname = lambda : '__builtins__.tuple' self.tuple_type = tup - self.bases = [tup] - self.mro = [self, tup] + self.bases = [] + self.mro = [self] # self._promote = tup diff --git a/mypy/semanal.py b/mypy/semanal.py index e89d3560fcc6f..232f918f23a48 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1418,7 +1418,7 @@ def process_namedtuple_definition(self, s: AssignmentStmt) -> None: return # Yes, it's a valid namedtuple definition. Add it to the symbol table. node = self.lookup(name, s) - node.kind = TYPE_ALIAS # TODO locally defined namedtuple + node.kind = GDEF # TODO locally defined namedtuple # TODO call.analyzed node.node = named_tuple @@ -1525,7 +1525,7 @@ def fail_namedtuple_arg(self, message: str, def build_namedtuple_typeinfo(self, name: str, items: List[str], types: List[Type]) -> TypeInfo: symbols = SymbolTable() - tup = NamedTupleType(items, types, self.named_type('__builtins__.tuple', [AnyType()])) + tup = NamedTupleType(items, types, self.named_type('__builtins__.tuple', types)) class_def = ClassDef(name, Block([])) class_def.fullname = self.qualified_name(name) info = NamedTupleTypeInfo(tup, symbols, class_def) @@ -1534,7 +1534,7 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], # TODO: Make them read-only. for var in vars: var.info = info - symbols[var.name] = SymbolTableNode(MDEF, var) + symbols[var.name()] = SymbolTableNode(MDEF, var) # Add __init__, _replace, _asdict methods and _fields static field self.add_namedtuple_method('__init__', symbols, info, NoneTyp(), self.make_factory_args(vars, ARG_POS)) @@ -2634,10 +2634,7 @@ def self_type(typ: TypeInfo) -> Union[Instance, TupleType]: inst = Instance(typ, tv) if typ.tuple_type is None: return inst - if not typ.is_named_tuple: - inst = TupleType(typ.tuple_type.items, inst) - return inst - return NamedTupleType(typ.tuple_type.names, typ.tuple_type.items, inst) + return typ.tuple_type.self_type(inst) def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: diff --git a/mypy/types.py b/mypy/types.py index 26dc26bf6c12a..bd31bf25d49f3 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -760,13 +760,19 @@ def deserialize(cls, data: JsonDict) -> 'TupleType': Instance.deserialize(data['fallback']), implicit=data['implicit']) + def self_type(self, inst: Instance) -> 'TupleType': + return TupleType(self.items, inst) + class NamedTupleType(TupleType): - names = None # type: List[str] + attrs = None # type: List[str] - def __init__(self, names: List[str], *args) -> None: - self.names = names + def __init__(self, attrs: List[str], *args) -> None: + self.attrs = attrs super().__init__(*args) + + def self_type(self, inst: Instance) -> 'NamedTupleType': + return NamedTupleType(self.attrs, self.items, inst) class StarType(Type): From ee047f8a5175f8049babced9f84e7cb86970426e Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Mon, 11 Jul 2016 01:32:18 +0300 Subject: [PATCH 14/74] complete support for _replace --- mypy/checker.py | 4 +++- mypy/nodes.py | 6 ++---- mypy/semanal.py | 22 ++++++++++++---------- test-data/unit/check-namedtuple.test | 9 ++++----- 4 files changed, 21 insertions(+), 20 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index e2aef9334ea7e..7063758096dfa 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -801,7 +801,9 @@ def visit_class_def(self, defn: ClassDef) -> Type: with self.binder.frame_context(): self.accept(defn.defs) self.binder = old_binder - self.check_multiple_inheritance(typ) + if not hasattr(defn, 'incompatible_baseclass'): + # Otherwise we've already found errors; more errors are not useful + self.check_multiple_inheritance(typ) self.leave_partial_types() self.errors.pop_type() diff --git a/mypy/nodes.py b/mypy/nodes.py index 0f512d0b6d718..b027c6223e528 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1982,10 +1982,8 @@ class NamedTupleTypeInfo(TypeInfo): def __init__(self, tup: 'mypy.types.NamedTupleType', *args) -> None: super().__init__(*args) self.tuple_type = tup - self.bases = [] - self.mro = [self] - # self._promote = tup - + self.bases = [tup.fallback] + self.mro = [cast(TypeInfo, self)] + tup.fallback.type.mro class SymbolTableNode: diff --git a/mypy/semanal.py b/mypy/semanal.py index 232f918f23a48..a19287787a9f0 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -744,48 +744,50 @@ def analyze_base_classes(self, defn: ClassDef) -> None: """ base_types = [] # type: List[Instance] + info = defn.info for base_expr in defn.base_type_exprs: try: base = self.expr_to_analyzed_type(base_expr) except TypeTranslationError: self.fail('Invalid base class', base_expr) - defn.info.fallback_to_any = True + info.fallback_to_any = True continue if isinstance(base, TupleType): - if defn.info.tuple_type: + if info.tuple_type: self.fail("Class has two incompatible bases derived from tuple", defn) + defn.incompatible_baseclass = True if (not self.is_stub_file - and not defn.info.is_named_tuple + and not info.is_named_tuple and base.fallback.type.fullname() == 'builtins.tuple'): self.fail("Tuple[...] not supported as a base class outside a stub file", defn) - defn.info.tuple_type = base + info.tuple_type = base base_types.append(base.fallback) elif isinstance(base, Instance): base_types.append(base) elif isinstance(base, AnyType): - defn.info.fallback_to_any = True + info.fallback_to_any = True else: self.fail('Invalid base class', base_expr) - defn.info.fallback_to_any = True + info.fallback_to_any = True # Add 'object' as implicit base if there is no other base class. if (not base_types and defn.fullname != 'builtins.object'): base_types.append(self.object_type()) - defn.info.bases = base_types + info.bases = base_types # Calculate the MRO. It might be incomplete at this point if # the bases of defn include classes imported from other # modules in an import loop. We'll recompute it in ThirdPass. if not self.verify_base_classes(defn): - defn.info.mro = [] + info.mro = [] return calculate_class_mro(defn, self.fail) # If there are cyclic imports, we may be missing 'object' in # the MRO. Fix MRO if needed. - if defn.info.mro and defn.info.mro[-1].fullname() != 'builtins.object': - defn.info.mro.append(self.object_type().type) + if info.mro and info.mro[-1].fullname() != 'builtins.object': + info.mro.append(self.object_type().type) def expr_to_analyzed_type(self, expr: Node) -> Type: if isinstance(expr, CallExpr): diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index cd23b4fea08bd..bc37c40208198 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -1,9 +1,8 @@ [case testNamedTupleUsedAsTuple] -import typing from collections import namedtuple X = namedtuple('X', ['x', 'y']) -x = X(1, 2) # type: X # None does not seem to work for TYPE_ALIAS +x = None # type: X a, b = x b = x[0] a = x[1] @@ -191,20 +190,20 @@ C(2).b from collections import namedtuple X = namedtuple('X', ['x', 'y']) -x = X(1, 2) # type: X +x = None # type: X a = x._asdict() [case testNamedTupleReplace] from collections import namedtuple X = namedtuple('X', ['x', 'y']) -x = X(1, 2) # type: X +x = None # type: X x = x._replace(y=5) [case testNamedTupleReplaceTyped] from typing import NamedTuple X = NamedTuple('X', [('x', int), ('y', str)]) -x = X(1, 'a') # type: X +x = None # type: X x = x._replace(x=5) x = x._replace(y=5) # E: Argument 1 to X has incompatible type "int"; expected "str" From 725d53c7983917e12da2d18b8e0c65627ae5187d Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Mon, 11 Jul 2016 01:38:41 +0300 Subject: [PATCH 15/74] flake8-ing --- mypy/nodes.py | 1 + mypy/semanal.py | 5674 +++++++++++++------------- mypy/types.py | 2 +- test-data/unit/check-namedtuple.test | 418 +- 4 files changed, 3048 insertions(+), 3047 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index b027c6223e528..9211d65485106 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1979,6 +1979,7 @@ def deserialize(cls, data: JsonDict) -> 'TypeInfo': class NamedTupleTypeInfo(TypeInfo): is_named_tuple = True bases = None # type: List[mypy.types.Instance] + def __init__(self, tup: 'mypy.types.NamedTupleType', *args) -> None: super().__init__(*args) self.tuple_type = tup diff --git a/mypy/semanal.py b/mypy/semanal.py index a19287787a9f0..fed7475ae557a 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1,2837 +1,2837 @@ -"""The semantic analyzer. - -Bind names to definitions and do various other simple consistency -checks. For example, consider this program: - - x = 1 - y = x - -Here semantic analysis would detect that the assignment 'x = 1' -defines a new variable, the type of which is to be inferred (in a -later pass; type inference or type checking is not part of semantic -analysis). Also, it would bind both references to 'x' to the same -module-level variable node. The second assignment would also be -analyzed, and the type of 'y' marked as being inferred. - -Semantic analysis is the first analysis pass after parsing, and it is -subdivided into three passes: - - * FirstPass looks up externally visible names defined in a module but - ignores imports and local definitions. It helps enable (some) - cyclic references between modules, such as module 'a' that imports - module 'b' and used names defined in b *and* vice versa. The first - pass can be performed before dependent modules have been processed. - - * SemanticAnalyzer is the second pass. It does the bulk of the work. - It assumes that dependent modules have been semantically analyzed, - up to the second pass, unless there is a import cycle. - - * ThirdPass checks that type argument counts are valid; for example, - it will reject Dict[int]. We don't do this in the second pass, - since we infer the type argument counts of classes during this - pass, and it is possible to refer to classes defined later in a - file, which would not have the type argument count set yet. This - pass also recomputes the method resolution order of each class, in - case one of its bases belongs to a module involved in an import - loop. - -Semantic analysis of types is implemented in module mypy.typeanal. - -TODO: Check if the third pass slows down type checking significantly. - We could probably get rid of it -- for example, we could collect all - analyzed types in a collection and check them without having to - traverse the entire AST. -""" - -from typing import ( - List, Dict, Set, Tuple, cast, Any, overload, TypeVar, Union, Optional, Callable -) - -from mypy.nodes import ( - MypyFile, TypeInfo, Node, AssignmentStmt, FuncDef, OverloadedFuncDef, - ClassDef, Var, GDEF, MODULE_REF, FuncItem, Import, - ImportFrom, ImportAll, Block, LDEF, NameExpr, MemberExpr, - IndexExpr, TupleExpr, ListExpr, ExpressionStmt, ReturnStmt, - RaiseStmt, AssertStmt, OperatorAssignmentStmt, WhileStmt, - ForStmt, BreakStmt, ContinueStmt, IfStmt, TryStmt, WithStmt, DelStmt, - GlobalDecl, SuperExpr, DictExpr, CallExpr, RefExpr, OpExpr, UnaryExpr, - SliceExpr, CastExpr, RevealTypeExpr, TypeApplication, Context, SymbolTable, - SymbolTableNode, BOUND_TVAR, UNBOUND_TVAR, ListComprehension, GeneratorExpr, - FuncExpr, MDEF, FuncBase, Decorator, SetExpr, TypeVarExpr, - StrExpr, BytesExpr, PrintStmt, ConditionalExpr, PromoteExpr, - ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, MroError, type_aliases, - YieldFromExpr, NamedTupleExpr, NonlocalDecl, - SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr, - YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, COVARIANT, CONTRAVARIANT, - IntExpr, FloatExpr, UnicodeExpr, - INVARIANT, UNBOUND_IMPORTED, Expression, EllipsisExpr, NamedTupleTypeInfo -) -from mypy.visitor import NodeVisitor -from mypy.traverser import TraverserVisitor -from mypy.errors import Errors, report_internal_error -from mypy.types import ( - NoneTyp, CallableType, Overloaded, Instance, Type, TypeVarType, AnyType, - FunctionLike, UnboundType, TypeList, ErrorType, TypeVarDef, - replace_leading_arg_type, TupleType, UnionType, StarType, EllipsisType, - NamedTupleType -) -from mypy.nodes import function_type, implicit_module_attrs -from mypy.typeanal import TypeAnalyser, TypeAnalyserPass3, analyze_type_alias -from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError -from mypy.lex import lex -from mypy.parsetype import parse_type -from mypy.sametypes import is_same_type -from mypy.erasetype import erase_typevars -from mypy import defaults -from mypy.options import Options - - -T = TypeVar('T') - - -# Inferred value of an expression. -ALWAYS_TRUE = 0 -ALWAYS_FALSE = 1 -TRUTH_VALUE_UNKNOWN = 2 - -# Map from obsolete name to the current spelling. -obsolete_name_mapping = { - 'typing.Function': 'typing.Callable', - 'typing.typevar': 'typing.TypeVar', -} - -# Hard coded type promotions (shared between all Python versions). -# These add extra ad-hoc edges to the subtyping relation. For example, -# int is considered a subtype of float, even though there is no -# subclass relationship. -TYPE_PROMOTIONS = { - 'builtins.int': 'builtins.float', - 'builtins.float': 'builtins.complex', -} - -# Hard coded type promotions for Python 3. -# -# Note that the bytearray -> bytes promotion is a little unsafe -# as some functions only accept bytes objects. Here convenience -# trumps safety. -TYPE_PROMOTIONS_PYTHON3 = TYPE_PROMOTIONS.copy() -TYPE_PROMOTIONS_PYTHON3.update({ - 'builtins.bytearray': 'builtins.bytes', -}) - -# Hard coded type promotions for Python 2. -# -# These promotions are unsafe, but we are doing them anyway -# for convenience and also for Python 3 compatibility -# (bytearray -> str). -TYPE_PROMOTIONS_PYTHON2 = TYPE_PROMOTIONS.copy() -TYPE_PROMOTIONS_PYTHON2.update({ - 'builtins.str': 'builtins.unicode', - 'builtins.bytearray': 'builtins.str', -}) - -# When analyzing a function, should we analyze the whole function in one go, or -# should we only perform one phase of the analysis? The latter is used for -# nested functions. In the first phase we add the function to the symbol table -# but don't process body. In the second phase we process function body. This -# way we can have mutually recursive nested functions. -FUNCTION_BOTH_PHASES = 0 # Everthing in one go -FUNCTION_FIRST_PHASE_POSTPONE_SECOND = 1 # Add to symbol table but postpone body -FUNCTION_SECOND_PHASE = 2 # Only analyze body - - -class SemanticAnalyzer(NodeVisitor): - """Semantically analyze parsed mypy files. - - The analyzer binds names and does various consistency checks for a - parse tree. Note that type checking is performed as a separate - pass. - - This is the second phase of semantic analysis. - """ - - # Library search paths - lib_path = None # type: List[str] - # Module name space - modules = None # type: Dict[str, MypyFile] - # Global name space for current module - globals = None # type: SymbolTable - # Names declared using "global" (separate set for each scope) - global_decls = None # type: List[Set[str]] - # Names declated using "nonlocal" (separate set for each scope) - nonlocal_decls = None # type: List[Set[str]] - # Local names of function scopes; None for non-function scopes. - locals = None # type: List[SymbolTable] - # Nested block depths of scopes - block_depth = None # type: List[int] - # TypeInfo of directly enclosing class (or None) - type = None # type: TypeInfo - # Stack of outer classes (the second tuple item contains tvars). - type_stack = None # type: List[TypeInfo] - # Type variables that are bound by the directly enclosing class - bound_tvars = None # type: List[SymbolTableNode] - # Stack of type variables that were bound by outer classess - tvar_stack = None # type: List[List[SymbolTableNode]] - # Do weak type checking in this file - weak_opts = set() # type: Set[str] - - # Stack of functions being analyzed - function_stack = None # type: List[FuncItem] - # Stack of next available function type variable ids - next_function_tvar_id_stack = None # type: List[int] - - # Status of postponing analysis of nested function bodies. By using this we - # can have mutually recursive nested functions. Values are FUNCTION_x - # constants. Note that separate phasea are not used for methods. - postpone_nested_functions_stack = None # type: List[int] - # Postponed functions collected if - # postpone_nested_functions_stack[-1] == FUNCTION_FIRST_PHASE_POSTPONE_SECOND. - postponed_functions_stack = None # type: List[List[Node]] - - loop_depth = 0 # Depth of breakable loops - cur_mod_id = '' # Current module id (or None) (phase 2) - is_stub_file = False # Are we analyzing a stub file? - imports = None # type: Set[str] # Imported modules (during phase 2 analysis) - errors = None # type: Errors # Keeps track of generated errors - - def __init__(self, - lib_path: List[str], - errors: Errors, - options: Options) -> None: - """Construct semantic analyzer. - - Use lib_path to search for modules, and report analysis errors - using the Errors instance. - """ - self.locals = [None] - self.imports = set() - self.type = None - self.type_stack = [] - self.bound_tvars = None - self.tvar_stack = [] - self.function_stack = [] - self.next_function_tvar_id_stack = [-1] - self.block_depth = [0] - self.loop_depth = 0 - self.lib_path = lib_path - self.errors = errors - self.modules = {} - self.options = options - self.postpone_nested_functions_stack = [FUNCTION_BOTH_PHASES] - self.postponed_functions_stack = [] - self.all_exports = set() # type: Set[str] - - def visit_file(self, file_node: MypyFile, fnam: str) -> None: - self.errors.set_file(fnam) - self.cur_mod_node = file_node - self.cur_mod_id = file_node.fullname() - self.is_stub_file = fnam.lower().endswith('.pyi') - self.globals = file_node.names - self.weak_opts = file_node.weak_opts - - if 'builtins' in self.modules: - self.globals['__builtins__'] = SymbolTableNode( - MODULE_REF, self.modules['builtins'], self.cur_mod_id) - - for name in implicit_module_attrs: - v = self.globals[name].node - if isinstance(v, Var): - v.type = self.anal_type(v.type) - v.is_ready = True - - defs = file_node.defs - for d in defs: - self.accept(d) - - if self.cur_mod_id == 'builtins': - remove_imported_names_from_symtable(self.globals, 'builtins') - - if '__all__' in self.globals: - for name, g in self.globals.items(): - if name not in self.all_exports: - g.module_public = False - - def visit_func_def(self, defn: FuncDef) -> None: - phase_info = self.postpone_nested_functions_stack[-1] - if phase_info != FUNCTION_SECOND_PHASE: - self.function_stack.append(defn) - # First phase of analysis for function. - self.errors.push_function(defn.name()) - self.update_function_type_variables(defn) - self.errors.pop_function() - self.function_stack.pop() - - defn.is_conditional = self.block_depth[-1] > 0 - - # TODO(jukka): Figure out how to share the various cases. It doesn't - # make sense to have (almost) duplicate code (here and elsewhere) for - # 3 cases: module-level, class-level and local names. Maybe implement - # a common stack of namespaces. As the 3 kinds of namespaces have - # different semantics, this wouldn't always work, but it might still - # be a win. - if self.is_class_scope(): - # Method definition - defn.info = self.type - if not defn.is_decorated and not defn.is_overload: - if defn.name() in self.type.names: - # Redefinition. Conditional redefinition is okay. - n = self.type.names[defn.name()].node - if self.is_conditional_func(n, defn): - defn.original_def = cast(FuncDef, n) - else: - self.name_already_defined(defn.name(), defn) - self.type.names[defn.name()] = SymbolTableNode(MDEF, defn) - self.prepare_method_signature(defn) - elif self.is_func_scope(): - # Nested function - if not defn.is_decorated and not defn.is_overload: - if defn.name() in self.locals[-1]: - # Redefinition. Conditional redefinition is okay. - n = self.locals[-1][defn.name()].node - if self.is_conditional_func(n, defn): - defn.original_def = cast(FuncDef, n) - else: - self.name_already_defined(defn.name(), defn) - else: - self.add_local(defn, defn) - else: - # Top-level function - if not defn.is_decorated and not defn.is_overload: - symbol = self.globals.get(defn.name()) - if isinstance(symbol.node, FuncDef) and symbol.node != defn: - # This is redefinition. Conditional redefinition is okay. - original_def = symbol.node - if self.is_conditional_func(original_def, defn): - # Conditional function definition -- multiple defs are ok. - defn.original_def = original_def - else: - # Report error. - self.check_no_global(defn.name(), defn, True) - if phase_info == FUNCTION_FIRST_PHASE_POSTPONE_SECOND: - # Postpone this function (for the second phase). - self.postponed_functions_stack[-1].append(defn) - return - if phase_info != FUNCTION_FIRST_PHASE_POSTPONE_SECOND: - # Second phase of analysis for function. - self.errors.push_function(defn.name()) - self.analyze_function(defn) - self.errors.pop_function() - - def prepare_method_signature(self, func: FuncDef) -> None: - """Check basic signature validity and tweak annotation of self/cls argument.""" - # Only non-static methods are special. - if not func.is_static: - if not func.arguments: - self.fail('Method must have at least one argument', func) - elif func.type: - sig = cast(FunctionLike, func.type) - if func.is_class: - leading_type = self.class_type(self.type) - else: - leading_type = self_type(self.type) - func.type = replace_implicit_first_type(sig, leading_type) - - def is_conditional_func(self, previous: Node, new: FuncDef) -> bool: - """Does 'new' conditionally redefine 'previous'? - - We reject straight redefinitions of functions, as they are usually - a programming error. For example: - - . def f(): ... - . def f(): ... # Error: 'f' redefined - """ - return isinstance(previous, (FuncDef, Var)) and new.is_conditional - - def update_function_type_variables(self, defn: FuncDef) -> None: - """Make any type variables in the signature of defn explicit. - - Update the signature of defn to contain type variable definitions - if defn is generic. - """ - if defn.type: - functype = cast(CallableType, defn.type) - typevars = self.infer_type_variables(functype) - # Do not define a new type variable if already defined in scope. - typevars = [(name, tvar) for name, tvar in typevars - if not self.is_defined_type_var(name, defn)] - if typevars: - next_tvar_id = self.next_function_tvar_id() - defs = [TypeVarDef(tvar[0], next_tvar_id - i, - tvar[1].values, tvar[1].upper_bound, - tvar[1].variance) - for i, tvar in enumerate(typevars)] - functype.variables = defs - - def infer_type_variables(self, - type: CallableType) -> List[Tuple[str, TypeVarExpr]]: - """Return list of unique type variables referred to in a callable.""" - names = [] # type: List[str] - tvars = [] # type: List[TypeVarExpr] - for arg in type.arg_types + [type.ret_type]: - for name, tvar_expr in self.find_type_variables_in_type(arg): - if name not in names: - names.append(name) - tvars.append(tvar_expr) - return list(zip(names, tvars)) - - def find_type_variables_in_type( - self, type: Type) -> List[Tuple[str, TypeVarExpr]]: - """Return a list of all unique type variable references in type. - - This effectively does partial name binding, results of which are mostly thrown away. - """ - result = [] # type: List[Tuple[str, TypeVarExpr]] - if isinstance(type, UnboundType): - name = type.name - node = self.lookup_qualified(name, type) - if node and node.kind == UNBOUND_TVAR: - result.append((name, cast(TypeVarExpr, node.node))) - for arg in type.args: - result.extend(self.find_type_variables_in_type(arg)) - elif isinstance(type, TypeList): - for item in type.items: - result.extend(self.find_type_variables_in_type(item)) - elif isinstance(type, UnionType): - for item in type.items: - result.extend(self.find_type_variables_in_type(item)) - elif isinstance(type, AnyType): - pass - elif isinstance(type, EllipsisType) or isinstance(type, TupleType): - pass - else: - assert False, 'Unsupported type %s' % type - return result - - def is_defined_type_var(self, tvar: str, context: Node) -> bool: - return self.lookup_qualified(tvar, context).kind == BOUND_TVAR - - def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: - t = [] # type: List[CallableType] - for i, item in enumerate(defn.items): - # TODO support decorated overloaded functions properly - item.is_overload = True - item.func.is_overload = True - item.accept(self) - t.append(cast(CallableType, function_type(item.func, - self.builtin_type('builtins.function')))) - if item.func.is_property and i == 0: - # This defines a property, probably with a setter and/or deleter. - self.analyze_property_with_multi_part_definition(defn) - break - if not [dec for dec in item.decorators - if refers_to_fullname(dec, 'typing.overload')]: - self.fail("'overload' decorator expected", item) - - defn.type = Overloaded(t) - defn.type.line = defn.line - - if self.is_class_scope(): - self.type.names[defn.name()] = SymbolTableNode(MDEF, defn, - typ=defn.type) - defn.info = self.type - elif self.is_func_scope(): - self.add_local(defn, defn) - - def analyze_property_with_multi_part_definition(self, defn: OverloadedFuncDef) -> None: - """Analyze a property defined using multiple methods (e.g., using @x.setter). - - Assume that the first method (@property) has already been analyzed. - """ - defn.is_property = True - items = defn.items - for item in items[1:]: - if len(item.decorators) == 1: - node = item.decorators[0] - if isinstance(node, MemberExpr): - if node.name == 'setter': - # The first item represents the entire property. - defn.items[0].var.is_settable_property = True - # Get abstractness from the original definition. - item.func.is_abstract = items[0].func.is_abstract - else: - self.fail("Decorated property not supported", item) - item.func.accept(self) - - def next_function_tvar_id(self) -> int: - return self.next_function_tvar_id_stack[-1] - - def analyze_function(self, defn: FuncItem) -> None: - is_method = self.is_class_scope() - - tvarnodes = self.add_func_type_variables_to_symbol_table(defn) - next_function_tvar_id = min([self.next_function_tvar_id()] + - [n.tvar_def.id.raw_id - 1 for n in tvarnodes]) - self.next_function_tvar_id_stack.append(next_function_tvar_id) - - if defn.type: - # Signature must be analyzed in the surrounding scope so that - # class-level imported names and type variables are in scope. - defn.type = self.anal_type(defn.type) - self.check_function_signature(defn) - if isinstance(defn, FuncDef): - defn.type = set_callable_name(defn.type, defn) - for arg in defn.arguments: - if arg.initializer: - arg.initializer.accept(self) - self.function_stack.append(defn) - self.enter() - for arg in defn.arguments: - self.add_local(arg.variable, defn) - for arg in defn.arguments: - if arg.initialization_statement: - lvalue = arg.initialization_statement.lvalues[0] - lvalue.accept(self) - - # The first argument of a non-static, non-class method is like 'self' - # (though the name could be different), having the enclosing class's - # instance type. - if is_method and not defn.is_static and not defn.is_class and defn.arguments: - defn.arguments[0].variable.is_self = True - - # First analyze body of the function but ignore nested functions. - self.postpone_nested_functions_stack.append(FUNCTION_FIRST_PHASE_POSTPONE_SECOND) - self.postponed_functions_stack.append([]) - defn.body.accept(self) - - # Analyze nested functions (if any) as a second phase. - self.postpone_nested_functions_stack[-1] = FUNCTION_SECOND_PHASE - for postponed in self.postponed_functions_stack[-1]: - postponed.accept(self) - self.postpone_nested_functions_stack.pop() - self.postponed_functions_stack.pop() - - self.next_function_tvar_id_stack.pop() - disable_typevars(tvarnodes) - - self.leave() - self.function_stack.pop() - - def add_func_type_variables_to_symbol_table( - self, defn: FuncItem) -> List[SymbolTableNode]: - nodes = [] # type: List[SymbolTableNode] - if defn.type: - tt = defn.type - names = self.type_var_names() - items = cast(CallableType, tt).variables - for item in items: - name = item.name - if name in names: - self.name_already_defined(name, defn) - node = self.bind_type_var(name, item, defn) - nodes.append(node) - names.add(name) - return nodes - - def type_var_names(self) -> Set[str]: - if not self.type: - return set() - else: - return set(self.type.type_vars) - - def bind_type_var(self, fullname: str, tvar_def: TypeVarDef, - context: Context) -> SymbolTableNode: - node = self.lookup_qualified(fullname, context) - node.kind = BOUND_TVAR - node.tvar_def = tvar_def - return node - - def check_function_signature(self, fdef: FuncItem) -> None: - sig = cast(CallableType, fdef.type) - if len(sig.arg_types) < len(fdef.arguments): - self.fail('Type signature has too few arguments', fdef) - # Add dummy Any arguments to prevent crashes later. - extra_anys = [AnyType()] * (len(fdef.arguments) - len(sig.arg_types)) - sig.arg_types.extend(extra_anys) - elif len(sig.arg_types) > len(fdef.arguments): - self.fail('Type signature has too many arguments', fdef) - - def visit_class_def(self, defn: ClassDef) -> None: - self.clean_up_bases_and_infer_type_variables(defn) - self.setup_class_def_analysis(defn) - - self.bind_class_type_vars(defn) - - self.analyze_base_classes(defn) - self.analyze_metaclass(defn) - - for decorator in defn.decorators: - self.analyze_class_decorator(defn, decorator) - - self.enter_class(defn) - - self.setup_is_builtinclass(defn) - - # Analyze class body. - defn.defs.accept(self) - - self.calculate_abstract_status(defn.info) - self.setup_type_promotion(defn) - - self.leave_class() - self.unbind_class_type_vars() - - def enter_class(self, defn: ClassDef) -> None: - # Remember previous active class - self.type_stack.append(self.type) - self.locals.append(None) # Add class scope - self.block_depth.append(-1) # The class body increments this to 0 - self.postpone_nested_functions_stack.append(FUNCTION_BOTH_PHASES) - self.type = defn.info - - def leave_class(self) -> None: - """ Restore analyzer state. """ - self.postpone_nested_functions_stack.pop() - self.block_depth.pop() - self.locals.pop() - self.type = self.type_stack.pop() - - def bind_class_type_vars(self, defn: ClassDef) -> None: - """ Unbind type variables of previously active class and bind - the type variables for the active class. - """ - if self.bound_tvars: - disable_typevars(self.bound_tvars) - self.tvar_stack.append(self.bound_tvars) - self.bound_tvars = self.bind_class_type_variables_in_symbol_table(defn.info) - - def unbind_class_type_vars(self) -> None: - """ Unbind the active class' type vars and rebind the - type vars of the previously active class. - """ - disable_typevars(self.bound_tvars) - self.bound_tvars = self.tvar_stack.pop() - if self.bound_tvars: - enable_typevars(self.bound_tvars) - - def analyze_class_decorator(self, defn: ClassDef, decorator: Node) -> None: - decorator.accept(self) - - def setup_is_builtinclass(self, defn: ClassDef): - for decorator in defn.decorators: - if refers_to_fullname(decorator, 'typing.builtinclass'): - defn.is_builtinclass = True - if defn.fullname == 'builtins.object': - # Only 'object' is marked as a built-in class, as otherwise things elsewhere - # would break. We need a better way of dealing with built-in classes. - defn.is_builtinclass = True - - def calculate_abstract_status(self, typ: TypeInfo) -> None: - """Calculate abstract status of a class. - - Set is_abstract of the type to True if the type has an unimplemented - abstract attribute. Also compute a list of abstract attributes. - """ - concrete = set() # type: Set[str] - abstract = [] # type: List[str] - for base in typ.mro: - for name, symnode in base.names.items(): - node = symnode.node - if isinstance(node, OverloadedFuncDef): - # Unwrap an overloaded function definition. We can just - # check arbitrarily the first overload item. If the - # different items have a different abstract status, there - # should be an error reported elsewhere. - func = node.items[0] # type: Node - else: - func = node - if isinstance(func, Decorator): - fdef = func.func - if fdef.is_abstract and name not in concrete: - typ.is_abstract = True - abstract.append(name) - concrete.add(name) - typ.abstract_attributes = sorted(abstract) - - def setup_type_promotion(self, defn: ClassDef) -> None: - """Setup extra, ad-hoc subtyping relationships between classes (promotion). - - This includes things like 'int' being compatible with 'float'. - """ - promote_target = None # type: Type - for decorator in defn.decorators: - if isinstance(decorator, CallExpr): - analyzed = decorator.analyzed - if isinstance(analyzed, PromoteExpr): - # _promote class decorator (undocumented faeture). - promote_target = analyzed.type - if not promote_target: - promotions = (TYPE_PROMOTIONS_PYTHON3 if self.options.python_version[0] >= 3 - else TYPE_PROMOTIONS_PYTHON2) - if defn.fullname in promotions: - promote_target = self.named_type_or_none(promotions[defn.fullname]) - defn.info._promote = promote_target - - def clean_up_bases_and_infer_type_variables(self, defn: ClassDef) -> None: - """Remove extra base classes such as Generic and infer type vars. - - For example, consider this class: - - . class Foo(Bar, Generic[T]): ... - - Now we will remove Generic[T] from bases of Foo and infer that the - type variable 'T' is a type argument of Foo. - - Note that this is performed *before* semantic analysis. - """ - removed = [] # type: List[int] - type_vars = [] # type: List[TypeVarDef] - for i, base_expr in enumerate(defn.base_type_exprs): - try: - base = expr_to_unanalyzed_type(base_expr) - except TypeTranslationError: - # This error will be caught later. - continue - tvars = self.analyze_typevar_declaration(base) - if tvars is not None: - if type_vars: - self.fail('Duplicate Generic in bases', defn) - removed.append(i) - for j, (name, tvar_expr) in enumerate(tvars): - type_vars.append(TypeVarDef(name, j + 1, tvar_expr.values, - tvar_expr.upper_bound, tvar_expr.variance)) - if type_vars: - defn.type_vars = type_vars - if defn.info: - defn.info.type_vars = [tv.name for tv in type_vars] - for i in reversed(removed): - del defn.base_type_exprs[i] - - def analyze_typevar_declaration(self, t: Type) -> Optional[List[Tuple[str, TypeVarExpr]]]: - if not isinstance(t, UnboundType): - return None - unbound = t - sym = self.lookup_qualified(unbound.name, unbound) - if sym is None or sym.node is None: - return None - if sym.node.fullname() == 'typing.Generic': - tvars = [] # type: List[Tuple[str, TypeVarExpr]] - for arg in unbound.args: - tvar = self.analyze_unbound_tvar(arg) - if tvar: - tvars.append(tvar) - else: - self.fail('Free type variable expected in %s[...]' % - sym.node.name(), t) - return tvars - return None - - def analyze_unbound_tvar(self, t: Type) -> Tuple[str, TypeVarExpr]: - if not isinstance(t, UnboundType): - return None - unbound = t - sym = self.lookup_qualified(unbound.name, unbound) - if sym is not None and sym.kind == UNBOUND_TVAR: - return unbound.name, cast(TypeVarExpr, sym.node) - return None - - def setup_class_def_analysis(self, defn: ClassDef) -> None: - """Prepare for the analysis of a class definition.""" - if not defn.info: - defn.info = TypeInfo(SymbolTable(), defn) - defn.info._fullname = defn.info.name() - if self.is_func_scope() or self.type: - kind = MDEF - if self.is_func_scope(): - kind = LDEF - self.add_symbol(defn.name, SymbolTableNode(kind, defn.info), defn) - - def analyze_base_classes(self, defn: ClassDef) -> None: - """Analyze and set up base classes. - - This computes several attributes on the corresponding TypeInfo defn.info - related to the base classes: defn.info.bases, defn.info.mro, and - miscellaneous others (at least tuple_type, fallback_to_any, and is_enum.) - """ - - base_types = [] # type: List[Instance] - info = defn.info - for base_expr in defn.base_type_exprs: - try: - base = self.expr_to_analyzed_type(base_expr) - except TypeTranslationError: - self.fail('Invalid base class', base_expr) - info.fallback_to_any = True - continue - - if isinstance(base, TupleType): - if info.tuple_type: - self.fail("Class has two incompatible bases derived from tuple", defn) - defn.incompatible_baseclass = True - if (not self.is_stub_file - and not info.is_named_tuple - and base.fallback.type.fullname() == 'builtins.tuple'): - self.fail("Tuple[...] not supported as a base class outside a stub file", defn) - info.tuple_type = base - base_types.append(base.fallback) - elif isinstance(base, Instance): - base_types.append(base) - elif isinstance(base, AnyType): - info.fallback_to_any = True - else: - self.fail('Invalid base class', base_expr) - info.fallback_to_any = True - - # Add 'object' as implicit base if there is no other base class. - if (not base_types and defn.fullname != 'builtins.object'): - base_types.append(self.object_type()) - - info.bases = base_types - - # Calculate the MRO. It might be incomplete at this point if - # the bases of defn include classes imported from other - # modules in an import loop. We'll recompute it in ThirdPass. - if not self.verify_base_classes(defn): - info.mro = [] - return - calculate_class_mro(defn, self.fail) - # If there are cyclic imports, we may be missing 'object' in - # the MRO. Fix MRO if needed. - if info.mro and info.mro[-1].fullname() != 'builtins.object': - info.mro.append(self.object_type().type) - - def expr_to_analyzed_type(self, expr: Node) -> Type: - if isinstance(expr, CallExpr): - expr.accept(self) - info = self.check_namedtuple(expr) - if info is None: - # Some form of namedtuple is the only valid type that looks like a call - # expression. This isn't a valid type. - raise TypeTranslationError() - fallback = Instance(info, []) - return TupleType(info.tuple_type.items, fallback=fallback) - typ = expr_to_unanalyzed_type(expr) - return self.anal_type(typ) - - def verify_base_classes(self, defn: ClassDef) -> bool: - info = defn.info - for base in info.bases: - baseinfo = base.type - if self.is_base_class(info, baseinfo): - self.fail('Cycle in inheritance hierarchy', defn) - # Clear bases to forcefully get rid of the cycle. - info.bases = [] - if baseinfo.fullname() == 'builtins.bool': - self.fail("'%s' is not a valid base class" % - baseinfo.name(), defn) - return False - dup = find_duplicate(info.direct_base_classes()) - if dup: - self.fail('Duplicate base class "%s"' % dup.name(), defn) - return False - return True - - def is_base_class(self, t: TypeInfo, s: TypeInfo) -> bool: - """Determine if t is a base class of s (but do not use mro).""" - # Search the base class graph for t, starting from s. - worklist = [s] - visited = {s} - while worklist: - nxt = worklist.pop() - if nxt == t: - return True - for base in nxt.bases: - if base.type not in visited: - worklist.append(base.type) - visited.add(base.type) - return False - - def analyze_metaclass(self, defn: ClassDef) -> None: - if defn.metaclass: - sym = self.lookup_qualified(defn.metaclass, defn) - if sym is not None and not isinstance(sym.node, TypeInfo): - self.fail("Invalid metaclass '%s'" % defn.metaclass, defn) - - def object_type(self) -> Instance: - return self.named_type('__builtins__.object') - - def class_type(self, info: TypeInfo) -> Type: - # Construct a function type whose fallback is cls. - from mypy import checkmember # To avoid import cycle. - leading_type = checkmember.type_object_type(info, self.builtin_type) - if isinstance(leading_type, Overloaded): - # Overloaded __init__ is too complex to handle. Plus it's stubs only. - return AnyType() - else: - return leading_type - - def named_type(self, qualified_name: str, args: List[Type] = None) -> Instance: - sym = self.lookup_qualified(qualified_name, None) - return Instance(cast(TypeInfo, sym.node), args or []) - - def named_type_or_none(self, qualified_name: str) -> Instance: - sym = self.lookup_fully_qualified_or_none(qualified_name) - if not sym: - return None - return Instance(cast(TypeInfo, sym.node), []) - - def is_instance_type(self, t: Type) -> bool: - return isinstance(t, Instance) - - def bind_class_type_variables_in_symbol_table( - self, info: TypeInfo) -> List[SymbolTableNode]: - nodes = [] # type: List[SymbolTableNode] - for var, binder in zip(info.type_vars, info.defn.type_vars): - node = self.bind_type_var(var, binder, info) - nodes.append(node) - return nodes - - def visit_import(self, i: Import) -> None: - for id, as_id in i.ids: - if as_id is not None: - self.add_module_symbol(id, as_id, module_public=True, context=i) - else: - # Modules imported in a stub file without using 'as x' won't get exported when - # doing 'from m import *'. - module_public = not self.is_stub_file - base = id.split('.')[0] - self.add_module_symbol(base, base, module_public=module_public, - context=i) - - def add_module_symbol(self, id: str, as_id: str, module_public: bool, - context: Context) -> None: - if id in self.modules: - m = self.modules[id] - self.add_symbol(as_id, SymbolTableNode(MODULE_REF, m, self.cur_mod_id, - module_public=module_public), context) - else: - self.add_unknown_symbol(as_id, context) - - def visit_import_from(self, imp: ImportFrom) -> None: - import_id = self.correct_relative_import(imp) - if import_id in self.modules: - module = self.modules[import_id] - for id, as_id in imp.names: - node = module.names.get(id) - if node and node.kind != UNBOUND_IMPORTED: - node = self.normalize_type_alias(node, imp) - if not node: - return - imported_id = as_id or id - existing_symbol = self.globals.get(imported_id) - if existing_symbol: - # Import can redefine a variable. They get special treatment. - if self.process_import_over_existing_name( - imported_id, existing_symbol, node, imp): - continue - # 'from m import x as x' exports x in a stub file. - module_public = not self.is_stub_file or as_id is not None - symbol = SymbolTableNode(node.kind, node.node, - self.cur_mod_id, - node.type_override, - module_public=module_public) - self.add_symbol(imported_id, symbol, imp) - else: - message = "Module has no attribute '{}'".format(id) - extra = self.undefined_name_extra_info('{}.{}'.format(import_id, id)) - if extra: - message += " {}".format(extra) - self.fail(message, imp) - else: - # Missing module. - for id, as_id in imp.names: - self.add_unknown_symbol(as_id or id, imp) - - def process_import_over_existing_name(self, - imported_id: str, existing_symbol: SymbolTableNode, - module_symbol: SymbolTableNode, - import_node: ImportBase) -> bool: - if (existing_symbol.kind in (LDEF, GDEF, MDEF) and - isinstance(existing_symbol.node, (Var, FuncDef))): - # This is a valid import over an existing definition in the file. Construct a dummy - # assignment that we'll use to type check the import. - lvalue = NameExpr(imported_id) - lvalue.kind = existing_symbol.kind - lvalue.node = existing_symbol.node - rvalue = NameExpr(imported_id) - rvalue.kind = module_symbol.kind - rvalue.node = module_symbol.node - assignment = AssignmentStmt([lvalue], rvalue) - for node in assignment, lvalue, rvalue: - node.set_line(import_node) - import_node.assignments.append(assignment) - return True - return False - - def normalize_type_alias(self, node: SymbolTableNode, - ctx: Context) -> SymbolTableNode: - if node.fullname in type_aliases: - # Node refers to an aliased type such as typing.List; normalize. - node = self.lookup_qualified(type_aliases[node.fullname], ctx) - if node.fullname == 'typing.DefaultDict': - self.add_module_symbol('collections', '__mypy_collections__', False, ctx) - node = self.lookup_qualified('__mypy_collections__.defaultdict', ctx) - return node - - def correct_relative_import(self, node: Union[ImportFrom, ImportAll]) -> str: - if node.relative == 0: - return node.id - - parts = self.cur_mod_id.split(".") - cur_mod_id = self.cur_mod_id - - rel = node.relative - if self.cur_mod_node.is_package_init_file(): - rel -= 1 - if len(parts) < rel: - self.fail("Relative import climbs too many namespaces", node) - if rel != 0: - cur_mod_id = ".".join(parts[:-rel]) - - return cur_mod_id + (("." + node.id) if node.id else "") - - def visit_import_all(self, i: ImportAll) -> None: - i_id = self.correct_relative_import(i) - if i_id in self.modules: - m = self.modules[i_id] - for name, node in m.names.items(): - node = self.normalize_type_alias(node, i) - if not name.startswith('_') and node.module_public: - existing_symbol = self.globals.get(name) - if existing_symbol: - # Import can redefine a variable. They get special treatment. - if self.process_import_over_existing_name( - name, existing_symbol, node, i): - continue - self.add_symbol(name, SymbolTableNode(node.kind, node.node, - self.cur_mod_id), i) - else: - # Don't add any dummy symbols for 'from x import *' if 'x' is unknown. - pass - - def add_unknown_symbol(self, name: str, context: Context) -> None: - var = Var(name) - var._fullname = self.qualified_name(name) - var.is_ready = True - var.type = AnyType() - self.add_symbol(name, SymbolTableNode(GDEF, var, self.cur_mod_id), context) - - # - # Statements - # - - def visit_block(self, b: Block) -> None: - if b.is_unreachable: - return - self.block_depth[-1] += 1 - for s in b.body: - self.accept(s) - self.block_depth[-1] -= 1 - - def visit_block_maybe(self, b: Block) -> None: - if b: - self.visit_block(b) - - def anal_type(self, t: Type, allow_tuple_literal: bool = False) -> Type: - if t: - if allow_tuple_literal: - # Types such as (t1, t2, ...) only allowed in assignment statements. They'll - # generate errors elsewhere, and Tuple[t1, t2, ...] must be used instead. - if isinstance(t, TupleType): - # Unlike TypeAnalyser, also allow implicit tuple types (without Tuple[...]). - star_count = sum(1 for item in t.items if isinstance(item, StarType)) - if star_count > 1: - self.fail('At most one star type allowed in a tuple', t) - return TupleType([AnyType() for _ in t.items], - self.builtin_type('builtins.tuple'), t.line) - items = [self.anal_type(item, True) - for item in t.items] - return TupleType(items, self.builtin_type('builtins.tuple'), t.line) - a = TypeAnalyser(self.lookup_qualified, - self.lookup_fully_qualified, - self.fail) - return t.accept(a) - else: - return None - - def visit_assignment_stmt(self, s: AssignmentStmt) -> None: - for lval in s.lvalues: - self.analyze_lvalue(lval, explicit_type=s.type is not None) - s.rvalue.accept(self) - if s.type: - allow_tuple_literal = isinstance(s.lvalues[-1], (TupleExpr, ListExpr)) - s.type = self.anal_type(s.type, allow_tuple_literal) - else: - # For simple assignments, allow binding type aliases. - # Also set the type if the rvalue is a simple literal. - if (s.type is None and len(s.lvalues) == 1 and - isinstance(s.lvalues[0], NameExpr)): - if s.lvalues[0].is_def: - s.type = self.analyze_simple_literal_type(s.rvalue) - res = analyze_type_alias(s.rvalue, - self.lookup_qualified, - self.lookup_fully_qualified, - self.fail) - if res and (not isinstance(res, Instance) or cast(Instance, res).args): - # TODO: What if this gets reassigned? - name = s.lvalues[0] - node = self.lookup(name.name, name) - node.kind = TYPE_ALIAS - node.type_override = res - if isinstance(s.rvalue, IndexExpr): - s.rvalue.analyzed = TypeAliasExpr(res) - if s.type: - # Store type into nodes. - for lvalue in s.lvalues: - self.store_declared_types(lvalue, s.type) - self.check_and_set_up_type_alias(s) - self.process_typevar_declaration(s) - self.process_namedtuple_definition(s) - - if (len(s.lvalues) == 1 and isinstance(s.lvalues[0], NameExpr) and - s.lvalues[0].name == '__all__' and s.lvalues[0].kind == GDEF and - isinstance(s.rvalue, (ListExpr, TupleExpr))): - self.add_exports(*s.rvalue.items) - - def analyze_simple_literal_type(self, rvalue: Node) -> Optional[Type]: - """Return builtins.int if rvalue is an int literal, etc.""" - if self.weak_opts or self.options.semantic_analysis_only or self.function_stack: - # Skip this if any weak options are set. - # Also skip if we're only doing the semantic analysis pass. - # This is mostly to avoid breaking unit tests. - # Also skip inside a function; this is to avoid confusing - # the code that handles dead code due to isinstance() - # inside type variables with value restrictions (like - # AnyStr). - return None - if isinstance(rvalue, IntExpr): - return self.named_type_or_none('builtins.int') - if isinstance(rvalue, FloatExpr): - return self.named_type_or_none('builtins.float') - if isinstance(rvalue, StrExpr): - return self.named_type_or_none('builtins.str') - if isinstance(rvalue, BytesExpr): - return self.named_type_or_none('builtins.bytes') - if isinstance(rvalue, UnicodeExpr): - return self.named_type_or_none('builtins.unicode') - return None - - def check_and_set_up_type_alias(self, s: AssignmentStmt) -> None: - """Check if assignment creates a type alias and set it up as needed.""" - # For now, type aliases only work at the top level of a module. - if (len(s.lvalues) == 1 and not self.is_func_scope() and not self.type - and not s.type): - lvalue = s.lvalues[0] - if isinstance(lvalue, NameExpr): - if not lvalue.is_def: - # Only a definition can create a type alias, not regular assignment. - return - rvalue = s.rvalue - if isinstance(rvalue, RefExpr): - node = rvalue.node - if isinstance(node, TypeInfo): - # TODO: We should record the fact that this is a variable - # that refers to a type, rather than making this - # just an alias for the type. - self.globals[lvalue.name].node = node - - def analyze_lvalue(self, lval: Node, nested: bool = False, - add_global: bool = False, - explicit_type: bool = False) -> None: - """Analyze an lvalue or assignment target. - - Only if add_global is True, add name to globals table. If nested - is true, the lvalue is within a tuple or list lvalue expression. - """ - - if isinstance(lval, NameExpr): - # Top-level definitions within some statements (at least while) are - # not handled in the first pass, so they have to be added now. - nested_global = (not self.is_func_scope() and - self.block_depth[-1] > 0 and - not self.type) - if (add_global or nested_global) and lval.name not in self.globals: - # Define new global name. - v = Var(lval.name) - v._fullname = self.qualified_name(lval.name) - v.is_ready = False # Type not inferred yet - lval.node = v - lval.is_def = True - lval.kind = GDEF - lval.fullname = v._fullname - self.globals[lval.name] = SymbolTableNode(GDEF, v, - self.cur_mod_id) - elif isinstance(lval.node, Var) and lval.is_def: - # Since the is_def flag is set, this must have been analyzed - # already in the first pass and added to the symbol table. - assert lval.node.name() in self.globals - elif (self.is_func_scope() and lval.name not in self.locals[-1] and - lval.name not in self.global_decls[-1] and - lval.name not in self.nonlocal_decls[-1]): - # Define new local name. - v = Var(lval.name) - lval.node = v - lval.is_def = True - lval.kind = LDEF - lval.fullname = lval.name - self.add_local(v, lval) - elif not self.is_func_scope() and (self.type and - lval.name not in self.type.names): - # Define a new attribute within class body. - v = Var(lval.name) - v.info = self.type - v.is_initialized_in_class = True - v.set_line(lval) - lval.node = v - lval.is_def = True - lval.kind = MDEF - lval.fullname = lval.name - self.type.names[lval.name] = SymbolTableNode(MDEF, v) - else: - # Bind to an existing name. - if explicit_type: - self.name_already_defined(lval.name, lval) - lval.accept(self) - self.check_lvalue_validity(lval.node, lval) - elif isinstance(lval, MemberExpr): - if not add_global: - self.analyze_member_lvalue(lval) - if explicit_type and not self.is_self_member_ref(lval): - self.fail('Type cannot be declared in assignment to non-self ' - 'attribute', lval) - elif isinstance(lval, IndexExpr): - if explicit_type: - self.fail('Unexpected type declaration', lval) - if not add_global: - lval.accept(self) - elif (isinstance(lval, TupleExpr) or - isinstance(lval, ListExpr)): - items = cast(Any, lval).items - if len(items) == 0 and isinstance(lval, TupleExpr): - self.fail("Can't assign to ()", lval) - self.analyze_tuple_or_list_lvalue(cast(Union[ListExpr, TupleExpr], lval), - add_global, explicit_type) - elif isinstance(lval, StarExpr): - if nested: - self.analyze_lvalue(lval.expr, nested, add_global, explicit_type) - else: - self.fail('Starred assignment target must be in a list or tuple', lval) - else: - self.fail('Invalid assignment target', lval) - - def analyze_tuple_or_list_lvalue(self, lval: Union[ListExpr, TupleExpr], - add_global: bool = False, - explicit_type: bool = False) -> None: - """Analyze an lvalue or assignment target that is a list or tuple.""" - items = lval.items - star_exprs = [cast(StarExpr, item) - for item in items - if isinstance(item, StarExpr)] - - if len(star_exprs) > 1: - self.fail('Two starred expressions in assignment', lval) - else: - if len(star_exprs) == 1: - star_exprs[0].valid = True - for i in items: - self.analyze_lvalue(i, nested=True, add_global=add_global, - explicit_type = explicit_type) - - def analyze_member_lvalue(self, lval: MemberExpr) -> None: - lval.accept(self) - if (self.is_self_member_ref(lval) and - self.type.get(lval.name) is None): - # Implicit attribute definition in __init__. - lval.is_def = True - v = Var(lval.name) - v.info = self.type - v.is_ready = False - lval.def_var = v - lval.node = v - self.type.names[lval.name] = SymbolTableNode(MDEF, v) - self.check_lvalue_validity(lval.node, lval) - - def is_self_member_ref(self, memberexpr: MemberExpr) -> bool: - """Does memberexpr to refer to an attribute of self?""" - if not isinstance(memberexpr.expr, NameExpr): - return False - node = memberexpr.expr.node - return isinstance(node, Var) and node.is_self - - def check_lvalue_validity(self, node: Node, ctx: Context) -> None: - if isinstance(node, (TypeInfo, TypeVarExpr)): - self.fail('Invalid assignment target', ctx) - - def store_declared_types(self, lvalue: Node, typ: Type) -> None: - if isinstance(typ, StarType) and not isinstance(lvalue, StarExpr): - self.fail('Star type only allowed for starred expressions', lvalue) - if isinstance(lvalue, RefExpr): - lvalue.is_def = False - if isinstance(lvalue.node, Var): - var = lvalue.node - var.type = typ - var.is_ready = True - # If node is not a variable, we'll catch it elsewhere. - elif isinstance(lvalue, TupleExpr): - if isinstance(typ, TupleType): - if len(lvalue.items) != len(typ.items): - self.fail('Incompatible number of tuple items', lvalue) - return - for item, itemtype in zip(lvalue.items, typ.items): - self.store_declared_types(item, itemtype) - else: - self.fail('Tuple type expected for multiple variables', - lvalue) - elif isinstance(lvalue, StarExpr): - if isinstance(typ, StarType): - self.store_declared_types(lvalue.expr, typ.type) - else: - self.fail('Star type expected for starred expression', lvalue) - else: - # This has been flagged elsewhere as an error, so just ignore here. - pass - - def process_typevar_declaration(self, s: AssignmentStmt) -> None: - """Check if s declares a TypeVar; it yes, store it in symbol table.""" - call = self.get_typevar_declaration(s) - if not call: - return - - lvalue = cast(NameExpr, s.lvalues[0]) - name = lvalue.name - if not lvalue.is_def: - if s.type: - self.fail("Cannot declare the type of a type variable", s) - else: - self.fail("Cannot redefine '%s' as a type variable" % name, s) - return - - if not self.check_typevar_name(call, name, s): - return - - # Constraining types - n_values = call.arg_kinds[1:].count(ARG_POS) - values = self.analyze_types(call.args[1:1 + n_values]) - - res = self.process_typevar_parameters(call.args[1 + n_values:], - call.arg_names[1 + n_values:], - call.arg_kinds[1 + n_values:], - bool(values), - s) - if res is None: - return - variance, upper_bound = res - - # Yes, it's a valid type variable definition! Add it to the symbol table. - node = self.lookup(name, s) - node.kind = UNBOUND_TVAR - TypeVar = TypeVarExpr(name, node.fullname, values, upper_bound, variance) - TypeVar.line = call.line - call.analyzed = TypeVar - node.node = TypeVar - - def check_typevar_name(self, call: CallExpr, name: str, context: Context) -> bool: - if len(call.args) < 1: - self.fail("Too few arguments for TypeVar()", context) - return False - if not isinstance(call.args[0], (StrExpr, BytesExpr)) or not call.arg_kinds[0] == ARG_POS: - self.fail("TypeVar() expects a string literal as first argument", context) - return False - if cast(StrExpr, call.args[0]).value != name: - self.fail("Unexpected TypeVar() argument value", context) - return False - return True - - def get_typevar_declaration(self, s: AssignmentStmt) -> Optional[CallExpr]: - """Returns the TypeVar() call expression if `s` is a type var declaration - or None otherwise. - """ - if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): - return None - if not isinstance(s.rvalue, CallExpr): - return None - call = s.rvalue - if not isinstance(call.callee, RefExpr): - return None - callee = call.callee - if callee.fullname != 'typing.TypeVar': - return None - return call - - def process_typevar_parameters(self, args: List[Node], - names: List[Optional[str]], - kinds: List[int], - has_values: bool, - context: Context) -> Optional[Tuple[int, Type]]: - covariant = False - contravariant = False - upper_bound = self.object_type() # type: Type - for param_value, param_name, param_kind in zip(args, names, kinds): - if not param_kind == ARG_NAMED: - self.fail("Unexpected argument to TypeVar()", context) - return None - if param_name == 'covariant': - if isinstance(param_value, NameExpr): - if param_value.name == 'True': - covariant = True - else: - self.fail("TypeVar 'covariant' may only be 'True'", context) - return None - else: - self.fail("TypeVar 'covariant' may only be 'True'", context) - return None - elif param_name == 'contravariant': - if isinstance(param_value, NameExpr): - if param_value.name == 'True': - contravariant = True - else: - self.fail("TypeVar 'contravariant' may only be 'True'", context) - return None - else: - self.fail("TypeVar 'contravariant' may only be 'True'", context) - return None - elif param_name == 'bound': - if has_values: - self.fail("TypeVar cannot have both values and an upper bound", context) - return None - try: - upper_bound = self.expr_to_analyzed_type(param_value) - except TypeTranslationError: - self.fail("TypeVar 'bound' must be a type", param_value) - return None - elif param_name == 'values': - # Probably using obsolete syntax with values=(...). Explain the current syntax. - self.fail("TypeVar 'values' argument not supported", context) - self.fail("Use TypeVar('T', t, ...) instead of TypeVar('T', values=(t, ...))", - context) - return None - else: - self.fail("Unexpected argument to TypeVar(): {}".format(param_name), context) - return None - - if covariant and contravariant: - self.fail("TypeVar cannot be both covariant and contravariant", context) - return None - elif covariant: - variance = COVARIANT - elif contravariant: - variance = CONTRAVARIANT - else: - variance = INVARIANT - return (variance, upper_bound) - - def process_namedtuple_definition(self, s: AssignmentStmt) -> None: - """Check if s defines a namedtuple; if yes, store the definition in symbol table.""" - if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): - return - lvalue = cast(NameExpr, s.lvalues[0]) - name = lvalue.name - named_tuple = self.check_namedtuple(s.rvalue, name) - if named_tuple is None: - return - # Yes, it's a valid namedtuple definition. Add it to the symbol table. - node = self.lookup(name, s) - node.kind = GDEF # TODO locally defined namedtuple - # TODO call.analyzed - node.node = named_tuple - - def check_namedtuple(self, node: Node, var_name: str = None) -> TypeInfo: - """Check if a call defines a namedtuple. - - The optional var_name argument is the name of the variable to - which this is assigned, if any. - - If it does, return the corresponding TypeInfo. Return None otherwise. - - If the definition is invalid but looks like a namedtuple, - report errors but return (some) TypeInfo. - """ - if not isinstance(node, CallExpr): - return None - call = node - if not isinstance(call.callee, RefExpr): - return None - callee = call.callee - fullname = callee.fullname - if fullname not in ('collections.namedtuple', 'typing.NamedTuple'): - return None - items, types, ok = self.parse_namedtuple_args(call, fullname) - if not ok: - # Error. Construct dummy return value. - return self.build_namedtuple_typeinfo('namedtuple', [], []) - else: - # Give it a unique name derived from the line number. - name = cast(StrExpr, call.args[0]).value - if name != var_name: - name += '@' + str(call.line) - info = self.build_namedtuple_typeinfo(name, items, types) - # Store it as a global just in case it would remain anonymous. - self.globals[name] = SymbolTableNode(GDEF, info, self.cur_mod_id) - call.analyzed = NamedTupleExpr(info).set_line(call.line) - return info - - def parse_namedtuple_args(self, call: CallExpr, - fullname: str) -> Tuple[List[str], List[Type], bool]: - # TODO Share code with check_argument_count in checkexpr.py? - args = call.args - if len(args) < 2: - return self.fail_namedtuple_arg("Too few arguments for namedtuple()", call) - if len(args) > 2: - return self.fail_namedtuple_arg("Too many arguments for namedtuple()", call) - if call.arg_kinds != [ARG_POS, ARG_POS]: - return self.fail_namedtuple_arg("Unexpected arguments to namedtuple()", call) - if not isinstance(args[0], (StrExpr, BytesExpr)): - return self.fail_namedtuple_arg( - "namedtuple() expects a string literal as the first argument", call) - types = [] # type: List[Type] - ok = True - if not isinstance(args[1], ListExpr): - if fullname == 'collections.namedtuple' and isinstance(args[1], (StrExpr, BytesExpr)): - str_expr = cast(StrExpr, args[1]) - items = str_expr.value.split() - else: - return self.fail_namedtuple_arg( - "List literal expected as the second argument to namedtuple()", call) - else: - listexpr = args[1] - if fullname == 'collections.namedtuple': - # The fields argument contains just names, with implicit Any types. - if any(not isinstance(item, (StrExpr, BytesExpr)) for item in listexpr.items): - return self.fail_namedtuple_arg("String literal expected as namedtuple() item", - call) - items = [cast(StrExpr, item).value for item in listexpr.items] - else: - # The fields argument contains (name, type) tuples. - items, types, ok = self.parse_namedtuple_fields_with_types(listexpr.items, call) - if not types: - types = [AnyType() for _ in items] - return items, types, ok - - def parse_namedtuple_fields_with_types(self, nodes: List[Node], - context: Context) -> Tuple[List[str], List[Type], bool]: - items = [] # type: List[str] - types = [] # type: List[Type] - for item in nodes: - if isinstance(item, TupleExpr): - if len(item.items) != 2: - return self.fail_namedtuple_arg("Invalid NamedTuple field definition", - item) - name, type_node = item.items - if isinstance(name, (StrExpr, BytesExpr)): - items.append(name.value) - else: - return self.fail_namedtuple_arg("Invalid NamedTuple() field name", item) - try: - type = expr_to_unanalyzed_type(type_node) - except TypeTranslationError: - return self.fail_namedtuple_arg('Invalid field type', type_node) - types.append(self.anal_type(type)) - else: - return self.fail_namedtuple_arg("Tuple expected as NamedTuple() field", item) - return items, types, True - - def fail_namedtuple_arg(self, message: str, - context: Context) -> Tuple[List[str], List[Type], bool]: - self.fail(message, context) - return [], [], False - - def build_namedtuple_typeinfo(self, name: str, items: List[str], - types: List[Type]) -> TypeInfo: - symbols = SymbolTable() - tup = NamedTupleType(items, types, self.named_type('__builtins__.tuple', types)) - class_def = ClassDef(name, Block([])) - class_def.fullname = self.qualified_name(name) - info = NamedTupleTypeInfo(tup, symbols, class_def) - vars = [Var(item, typ) for item, typ in zip(items, types)] - # Add named tuple items as attributes. - # TODO: Make them read-only. - for var in vars: - var.info = info - symbols[var.name()] = SymbolTableNode(MDEF, var) - # Add __init__, _replace, _asdict methods and _fields static field - self.add_namedtuple_method('__init__', symbols, info, NoneTyp(), - self.make_factory_args(vars, ARG_POS)) - self.add_namedtuple_method('_replace', symbols, info, self_type(info), - self.make_factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) - # TODO: refine to OrderedDict[Union[types]] - self.add_namedtuple_method('_asdict', symbols, info, AnyType()) - return info - - def make_factory_args(self, vars: List[Var], kind: int, - initializer: Expression = None) -> List[Argument]: - return [Argument(var, var.type, initializer, kind) for var in vars] - - def add_namedtuple_method(self, funcname: str, symbols: SymbolTable, info: TypeInfo, - ret: Type, args: List[Argument] = []) -> None: - types = [cast(Type, None)] + [arg.type_annotation for arg in args] - args = [Argument(Var('__self'), NoneTyp(), None, ARG_POS)] + args - items = [arg.variable.name() for arg in args] - arg_kinds = [arg.kind for arg in args] - # TODO: Make sure that the self argument name is not visible? - signature = CallableType(types, arg_kinds, items, ret, - self.named_type('__builtins__.function'), - name=info.name()) - func = FuncDef(funcname, - args, - Block([]), - typ=signature) - func.info = info - symbols[funcname] = SymbolTableNode(MDEF, func) - - def analyze_types(self, items: List[Node]) -> List[Type]: - result = [] # type: List[Type] - for node in items: - try: - result.append(self.anal_type(expr_to_unanalyzed_type(node))) - except TypeTranslationError: - self.fail('Type expected', node) - result.append(AnyType()) - return result - - def visit_decorator(self, dec: Decorator) -> None: - for d in dec.decorators: - d.accept(self) - removed = [] # type: List[int] - no_type_check = False - for i, d in enumerate(dec.decorators): - # A bunch of decorators are special cased here. - if refers_to_fullname(d, 'abc.abstractmethod'): - removed.append(i) - dec.func.is_abstract = True - self.check_decorated_function_is_method('abstractmethod', dec) - elif refers_to_fullname(d, 'asyncio.tasks.coroutine'): - removed.append(i) - elif refers_to_fullname(d, 'builtins.staticmethod'): - removed.append(i) - dec.func.is_static = True - dec.var.is_staticmethod = True - self.check_decorated_function_is_method('staticmethod', dec) - elif refers_to_fullname(d, 'builtins.classmethod'): - removed.append(i) - dec.func.is_class = True - dec.var.is_classmethod = True - self.check_decorated_function_is_method('classmethod', dec) - elif (refers_to_fullname(d, 'builtins.property') or - refers_to_fullname(d, 'abc.abstractproperty')): - removed.append(i) - dec.func.is_property = True - dec.var.is_property = True - if refers_to_fullname(d, 'abc.abstractproperty'): - dec.func.is_abstract = True - self.check_decorated_function_is_method('property', dec) - if len(dec.func.arguments) > 1: - self.fail('Too many arguments', dec.func) - elif refers_to_fullname(d, 'typing.no_type_check'): - dec.var.type = AnyType() - no_type_check = True - for i in reversed(removed): - del dec.decorators[i] - if not dec.is_overload or dec.var.is_property: - if self.is_func_scope(): - self.add_symbol(dec.var.name(), SymbolTableNode(LDEF, dec), - dec) - elif self.type: - dec.var.info = self.type - dec.var.is_initialized_in_class = True - self.add_symbol(dec.var.name(), SymbolTableNode(MDEF, dec), - dec) - if not no_type_check: - dec.func.accept(self) - if dec.decorators and dec.var.is_property: - self.fail('Decorated property not supported', dec) - - def check_decorated_function_is_method(self, decorator: str, - context: Context) -> None: - if not self.type or self.is_func_scope(): - self.fail("'%s' used with a non-method" % decorator, context) - - def visit_expression_stmt(self, s: ExpressionStmt) -> None: - s.expr.accept(self) - - def visit_return_stmt(self, s: ReturnStmt) -> None: - if not self.is_func_scope(): - self.fail("'return' outside function", s) - if s.expr: - s.expr.accept(self) - - def visit_raise_stmt(self, s: RaiseStmt) -> None: - if s.expr: - s.expr.accept(self) - if s.from_expr: - s.from_expr.accept(self) - - def visit_assert_stmt(self, s: AssertStmt) -> None: - if s.expr: - s.expr.accept(self) - - def visit_operator_assignment_stmt(self, - s: OperatorAssignmentStmt) -> None: - s.lvalue.accept(self) - s.rvalue.accept(self) - if (isinstance(s.lvalue, NameExpr) and s.lvalue.name == '__all__' and - s.lvalue.kind == GDEF and isinstance(s.rvalue, (ListExpr, TupleExpr))): - self.add_exports(*s.rvalue.items) - - def visit_while_stmt(self, s: WhileStmt) -> None: - s.expr.accept(self) - self.loop_depth += 1 - s.body.accept(self) - self.loop_depth -= 1 - self.visit_block_maybe(s.else_body) - - def visit_for_stmt(self, s: ForStmt) -> None: - s.expr.accept(self) - - # Bind index variables and check if they define new names. - self.analyze_lvalue(s.index) - - self.loop_depth += 1 - self.visit_block(s.body) - self.loop_depth -= 1 - - self.visit_block_maybe(s.else_body) - - def visit_break_stmt(self, s: BreakStmt) -> None: - if self.loop_depth == 0: - self.fail("'break' outside loop", s, True, blocker=True) - - def visit_continue_stmt(self, s: ContinueStmt) -> None: - if self.loop_depth == 0: - self.fail("'continue' outside loop", s, True, blocker=True) - - def visit_if_stmt(self, s: IfStmt) -> None: - infer_reachability_of_if_statement(s, pyversion=self.options.python_version) - for i in range(len(s.expr)): - s.expr[i].accept(self) - self.visit_block(s.body[i]) - self.visit_block_maybe(s.else_body) - - def visit_try_stmt(self, s: TryStmt) -> None: - self.analyze_try_stmt(s, self) - - def analyze_try_stmt(self, s: TryStmt, visitor: NodeVisitor, - add_global: bool = False) -> None: - s.body.accept(visitor) - for type, var, handler in zip(s.types, s.vars, s.handlers): - if type: - type.accept(visitor) - if var: - self.analyze_lvalue(var, add_global=add_global) - handler.accept(visitor) - if s.else_body: - s.else_body.accept(visitor) - if s.finally_body: - s.finally_body.accept(visitor) - - def visit_with_stmt(self, s: WithStmt) -> None: - for e, n in zip(s.expr, s.target): - e.accept(self) - if n: - self.analyze_lvalue(n) - self.visit_block(s.body) - - def visit_del_stmt(self, s: DelStmt) -> None: - s.expr.accept(self) - if not self.is_valid_del_target(s.expr): - self.fail('Invalid delete target', s) - - def is_valid_del_target(self, s: Node) -> bool: - if isinstance(s, (IndexExpr, NameExpr, MemberExpr)): - return True - elif isinstance(s, TupleExpr): - return all(self.is_valid_del_target(item) for item in s.items) - - def visit_global_decl(self, g: GlobalDecl) -> None: - for name in g.names: - if name in self.nonlocal_decls[-1]: - self.fail("Name '{}' is nonlocal and global".format(name), g) - self.global_decls[-1].add(name) - - def visit_nonlocal_decl(self, d: NonlocalDecl) -> None: - if not self.is_func_scope(): - self.fail("nonlocal declaration not allowed at module level", d) - else: - for name in d.names: - for table in reversed(self.locals[:-1]): - if table is not None and name in table: - break - else: - self.fail("No binding for nonlocal '{}' found".format(name), d) - - if self.locals[-1] is not None and name in self.locals[-1]: - self.fail("Name '{}' is already defined in local " - "scope before nonlocal declaration".format(name), d) - - if name in self.global_decls[-1]: - self.fail("Name '{}' is nonlocal and global".format(name), d) - self.nonlocal_decls[-1].add(name) - - def visit_print_stmt(self, s: PrintStmt) -> None: - for arg in s.args: - arg.accept(self) - if s.target: - s.target.accept(self) - - def visit_exec_stmt(self, s: ExecStmt) -> None: - s.expr.accept(self) - if s.variables1: - s.variables1.accept(self) - if s.variables2: - s.variables2.accept(self) - - # - # Expressions - # - - def visit_name_expr(self, expr: NameExpr) -> None: - n = self.lookup(expr.name, expr) - if n: - if n.kind == BOUND_TVAR: - self.fail("'{}' is a type variable and only valid in type " - "context".format(expr.name), expr) - else: - expr.kind = n.kind - expr.node = n.node - expr.fullname = n.fullname - - def visit_super_expr(self, expr: SuperExpr) -> None: - if not self.type: - self.fail('"super" used outside class', expr) - return - expr.info = self.type - - def visit_tuple_expr(self, expr: TupleExpr) -> None: - for item in expr.items: - item.accept(self) - - def visit_list_expr(self, expr: ListExpr) -> None: - for item in expr.items: - item.accept(self) - - def visit_set_expr(self, expr: SetExpr) -> None: - for item in expr.items: - item.accept(self) - - def visit_dict_expr(self, expr: DictExpr) -> None: - for key, value in expr.items: - key.accept(self) - value.accept(self) - - def visit_star_expr(self, expr: StarExpr) -> None: - if not expr.valid: - self.fail('Can use starred expression only as assignment target', expr) - else: - expr.expr.accept(self) - - def visit_yield_from_expr(self, e: YieldFromExpr) -> None: - if not self.is_func_scope(): # not sure - self.fail("'yield from' outside function", e, True, blocker=True) - else: - self.function_stack[-1].is_generator = True - if e.expr: - e.expr.accept(self) - - def visit_call_expr(self, expr: CallExpr) -> None: - """Analyze a call expression. - - Some call expressions are recognized as special forms, including - cast(...) and Any(...). - """ - expr.callee.accept(self) - if refers_to_fullname(expr.callee, 'typing.cast'): - # Special form cast(...). - if not self.check_fixed_args(expr, 2, 'cast'): - return - # Translate first argument to an unanalyzed type. - try: - target = expr_to_unanalyzed_type(expr.args[0]) - except TypeTranslationError: - self.fail('Cast target is not a type', expr) - return - # Piggyback CastExpr object to the CallExpr object; it takes - # precedence over the CallExpr semantics. - expr.analyzed = CastExpr(expr.args[1], target) - expr.analyzed.line = expr.line - expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'builtins.reveal_type'): - if not self.check_fixed_args(expr, 1, 'reveal_type'): - return - expr.analyzed = RevealTypeExpr(expr.args[0]) - expr.analyzed.line = expr.line - expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'typing.Any'): - # Special form Any(...). - if not self.check_fixed_args(expr, 1, 'Any'): - return - expr.analyzed = CastExpr(expr.args[0], AnyType()) - expr.analyzed.line = expr.line - expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'typing._promote'): - # Special form _promote(...). - if not self.check_fixed_args(expr, 1, '_promote'): - return - # Translate first argument to an unanalyzed type. - try: - target = expr_to_unanalyzed_type(expr.args[0]) - except TypeTranslationError: - self.fail('Argument 1 to _promote is not a type', expr) - return - expr.analyzed = PromoteExpr(target) - expr.analyzed.line = expr.line - expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'builtins.dict'): - expr.analyzed = self.translate_dict_call(expr) - else: - # Normal call expression. - for a in expr.args: - a.accept(self) - - if (isinstance(expr.callee, MemberExpr) and - isinstance(expr.callee.expr, NameExpr) and - expr.callee.expr.name == '__all__' and - expr.callee.expr.kind == GDEF and - expr.callee.name in ('append', 'extend')): - if expr.callee.name == 'append' and expr.args: - self.add_exports(expr.args[0]) - elif (expr.callee.name == 'extend' and expr.args and - isinstance(expr.args[0], (ListExpr, TupleExpr))): - self.add_exports(*expr.args[0].items) - - def translate_dict_call(self, call: CallExpr) -> Optional[DictExpr]: - """Translate 'dict(x=y, ...)' to {'x': y, ...}. - - For other variants of dict(...), return None. - """ - if not call.args: - return None - if not all(kind == ARG_NAMED for kind in call.arg_kinds): - # Must still accept those args. - for a in call.args: - a.accept(self) - return None - expr = DictExpr([(StrExpr(key), value) - for key, value in zip(call.arg_names, call.args)]) - expr.set_line(call) - expr.accept(self) - return expr - - def check_fixed_args(self, expr: CallExpr, numargs: int, - name: str) -> bool: - """Verify that expr has specified number of positional args. - - Return True if the arguments are valid. - """ - s = 's' - if numargs == 1: - s = '' - if len(expr.args) != numargs: - self.fail("'%s' expects %d argument%s" % (name, numargs, s), - expr) - return False - if expr.arg_kinds != [ARG_POS] * numargs: - self.fail("'%s' must be called with %s positional argument%s" % - (name, numargs, s), expr) - return False - return True - - def visit_member_expr(self, expr: MemberExpr) -> None: - base = expr.expr - base.accept(self) - # Bind references to module attributes. - if isinstance(base, RefExpr) and base.kind == MODULE_REF: - file = cast(MypyFile, base.node) - n = file.names.get(expr.name, None) if file is not None else None - if n: - n = self.normalize_type_alias(n, expr) - if not n: - return - expr.kind = n.kind - expr.fullname = n.fullname - expr.node = n.node - else: - # We only catch some errors here; the rest will be - # catched during type checking. - # - # This way we can report a larger number of errors in - # one type checker run. If we reported errors here, - # the build would terminate after semantic analysis - # and we wouldn't be able to report any type errors. - full_name = '%s.%s' % (file.fullname() if file is not None else None, expr.name) - if full_name in obsolete_name_mapping: - self.fail("Module has no attribute %r (it's now called %r)" % ( - expr.name, obsolete_name_mapping[full_name]), expr) - - def visit_op_expr(self, expr: OpExpr) -> None: - expr.left.accept(self) - expr.right.accept(self) - - def visit_comparison_expr(self, expr: ComparisonExpr) -> None: - for operand in expr.operands: - operand.accept(self) - - def visit_unary_expr(self, expr: UnaryExpr) -> None: - expr.expr.accept(self) - - def visit_index_expr(self, expr: IndexExpr) -> None: - expr.base.accept(self) - if refers_to_class_or_function(expr.base): - # Special form -- type application. - # Translate index to an unanalyzed type. - types = [] # type: List[Type] - if isinstance(expr.index, TupleExpr): - items = expr.index.items - else: - items = [expr.index] - for item in items: - try: - typearg = expr_to_unanalyzed_type(item) - except TypeTranslationError: - self.fail('Type expected within [...]', expr) - return - typearg = self.anal_type(typearg) - types.append(typearg) - expr.analyzed = TypeApplication(expr.base, types) - expr.analyzed.line = expr.line - else: - expr.index.accept(self) - - def visit_slice_expr(self, expr: SliceExpr) -> None: - if expr.begin_index: - expr.begin_index.accept(self) - if expr.end_index: - expr.end_index.accept(self) - if expr.stride: - expr.stride.accept(self) - - def visit_cast_expr(self, expr: CastExpr) -> None: - expr.expr.accept(self) - expr.type = self.anal_type(expr.type) - - def visit_reveal_type_expr(self, expr: RevealTypeExpr) -> None: - expr.expr.accept(self) - - def visit_type_application(self, expr: TypeApplication) -> None: - expr.expr.accept(self) - for i in range(len(expr.types)): - expr.types[i] = self.anal_type(expr.types[i]) - - def visit_list_comprehension(self, expr: ListComprehension) -> None: - expr.generator.accept(self) - - def visit_set_comprehension(self, expr: SetComprehension) -> None: - expr.generator.accept(self) - - def visit_dictionary_comprehension(self, expr: DictionaryComprehension) -> None: - self.enter() - self.analyze_comp_for(expr) - expr.key.accept(self) - expr.value.accept(self) - self.leave() - self.analyze_comp_for_2(expr) - - def visit_generator_expr(self, expr: GeneratorExpr) -> None: - self.enter() - self.analyze_comp_for(expr) - expr.left_expr.accept(self) - self.leave() - self.analyze_comp_for_2(expr) - - def analyze_comp_for(self, expr: Union[GeneratorExpr, - DictionaryComprehension]) -> None: - """Analyses the 'comp_for' part of comprehensions (part 1). - - That is the part after 'for' in (x for x in l if p). This analyzes - variables and conditions which are analyzed in a local scope. - """ - for i, (index, sequence, conditions) in enumerate(zip(expr.indices, - expr.sequences, - expr.condlists)): - if i > 0: - sequence.accept(self) - # Bind index variables. - self.analyze_lvalue(index) - for cond in conditions: - cond.accept(self) - - def analyze_comp_for_2(self, expr: Union[GeneratorExpr, - DictionaryComprehension]) -> None: - """Analyses the 'comp_for' part of comprehensions (part 2). - - That is the part after 'for' in (x for x in l if p). This analyzes - the 'l' part which is analyzed in the surrounding scope. - """ - expr.sequences[0].accept(self) - - def visit_func_expr(self, expr: FuncExpr) -> None: - self.analyze_function(expr) - - def visit_conditional_expr(self, expr: ConditionalExpr) -> None: - expr.if_expr.accept(self) - expr.cond.accept(self) - expr.else_expr.accept(self) - - def visit_backquote_expr(self, expr: BackquoteExpr) -> None: - expr.expr.accept(self) - - def visit__promote_expr(self, expr: PromoteExpr) -> None: - expr.type = self.anal_type(expr.type) - - def visit_yield_expr(self, expr: YieldExpr) -> None: - if not self.is_func_scope(): - self.fail("'yield' outside function", expr, True, blocker=True) - else: - self.function_stack[-1].is_generator = True - if expr.expr: - expr.expr.accept(self) - - # - # Helpers - # - - def lookup(self, name: str, ctx: Context) -> SymbolTableNode: - """Look up an unqualified name in all active namespaces.""" - # 1a. Name declared using 'global x' takes precedence - if name in self.global_decls[-1]: - if name in self.globals: - return self.globals[name] - else: - self.name_not_defined(name, ctx) - return None - # 1b. Name declared using 'nonlocal x' takes precedence - if name in self.nonlocal_decls[-1]: - for table in reversed(self.locals[:-1]): - if table is not None and name in table: - return table[name] - else: - self.name_not_defined(name, ctx) - return None - # 2. Class attributes (if within class definition) - if self.is_class_scope() and name in self.type.names: - return self.type[name] - # 3. Local (function) scopes - for table in reversed(self.locals): - if table is not None and name in table: - return table[name] - # 4. Current file global scope - if name in self.globals: - return self.globals[name] - # 5. Builtins - b = self.globals.get('__builtins__', None) - if b: - table = cast(MypyFile, b.node).names - if name in table: - if name[0] == "_" and name[1] != "_": - self.name_not_defined(name, ctx) - return None - node = table[name] - return node - # Give up. - self.name_not_defined(name, ctx) - self.check_for_obsolete_short_name(name, ctx) - return None - - def check_for_obsolete_short_name(self, name: str, ctx: Context) -> None: - matches = [obsolete_name - for obsolete_name in obsolete_name_mapping - if obsolete_name.rsplit('.', 1)[-1] == name] - if len(matches) == 1: - self.note("(Did you mean '{}'?)".format(obsolete_name_mapping[matches[0]]), ctx) - - def lookup_qualified(self, name: str, ctx: Context) -> SymbolTableNode: - if '.' not in name: - return self.lookup(name, ctx) - else: - parts = name.split('.') - n = self.lookup(parts[0], ctx) # type: SymbolTableNode - if n: - for i in range(1, len(parts)): - if isinstance(n.node, TypeInfo): - if n.node.mro is None: - # We haven't yet analyzed the class `n.node`. Fall back to direct - # lookup in the names declared directly under it, without its base - # classes. This can happen when we have a forward reference to a - # nested class, and the reference is bound before the outer class - # has been fully semantically analyzed. - # - # A better approach would be to introduce a new analysis pass or - # to move things around between passes, but this unblocks a common - # use case even though this is a little limited in case there is - # inheritance involved. - result = n.node.names.get(parts[i]) - else: - result = n.node.get(parts[i]) - n = result - elif isinstance(n.node, MypyFile): - n = n.node.names.get(parts[i], None) - # TODO: What if node is Var or FuncDef? - if not n: - self.name_not_defined(name, ctx) - break - if n: - n = self.normalize_type_alias(n, ctx) - return n - - def builtin_type(self, fully_qualified_name: str) -> Instance: - node = self.lookup_fully_qualified(fully_qualified_name) - info = cast(TypeInfo, node.node) - return Instance(info, []) - - def lookup_fully_qualified(self, name: str) -> SymbolTableNode: - """Lookup a fully qualified name. - - Assume that the name is defined. This happens in the global namespace -- the local - module namespace is ignored. - """ - assert '.' in name - parts = name.split('.') - n = self.modules[parts[0]] - for i in range(1, len(parts) - 1): - n = cast(MypyFile, n.names[parts[i]].node) - return n.names[parts[-1]] - - def lookup_fully_qualified_or_none(self, name: str) -> SymbolTableNode: - """Lookup a fully qualified name. - - Assume that the name is defined. This happens in the global namespace -- the local - module namespace is ignored. - """ - assert '.' in name - parts = name.split('.') - n = self.modules[parts[0]] - for i in range(1, len(parts) - 1): - next_sym = n.names.get(parts[i]) - if not next_sym: - return None - n = cast(MypyFile, next_sym.node) - return n.names.get(parts[-1]) - - def qualified_name(self, n: str) -> str: - return self.cur_mod_id + '.' + n - - def enter(self) -> None: - self.locals.append(SymbolTable()) - self.global_decls.append(set()) - self.nonlocal_decls.append(set()) - # -1 since entering block will increment this to 0. - self.block_depth.append(-1) - - def leave(self) -> None: - self.locals.pop() - self.global_decls.pop() - self.nonlocal_decls.pop() - self.block_depth.pop() - - def is_func_scope(self) -> bool: - return self.locals[-1] is not None - - def is_class_scope(self) -> bool: - return self.type is not None and not self.is_func_scope() - - def add_symbol(self, name: str, node: SymbolTableNode, - context: Context) -> None: - if self.is_func_scope(): - if name in self.locals[-1]: - # Flag redefinition unless this is a reimport of a module. - if not (node.kind == MODULE_REF and - self.locals[-1][name].node == node.node): - self.name_already_defined(name, context) - self.locals[-1][name] = node - elif self.type: - self.type.names[name] = node - else: - existing = self.globals.get(name) - if existing and (not isinstance(node.node, MypyFile) or - existing.node != node.node) and existing.kind != UNBOUND_IMPORTED: - # Modules can be imported multiple times to support import - # of multiple submodules of a package (e.g. a.x and a.y). - ok = False - # Only report an error if the symbol collision provides a different type. - if existing.type and node.type and is_same_type(existing.type, node.type): - ok = True - if not ok: - self.name_already_defined(name, context) - self.globals[name] = node - - def add_var(self, v: Var, ctx: Context) -> None: - if self.is_func_scope(): - self.add_local(v, ctx) - else: - self.globals[v.name()] = SymbolTableNode(GDEF, v, self.cur_mod_id) - v._fullname = self.qualified_name(v.name()) - - def add_local(self, node: Union[Var, FuncBase], ctx: Context) -> None: - name = node.name() - if name in self.locals[-1]: - self.name_already_defined(name, ctx) - node._fullname = name - self.locals[-1][name] = SymbolTableNode(LDEF, node) - - def add_exports(self, *exps: Node) -> None: - for exp in exps: - if isinstance(exp, StrExpr): - self.all_exports.add(exp.value) - - def check_no_global(self, n: str, ctx: Context, - is_overloaded_func: bool = False) -> None: - if n in self.globals: - prev_is_overloaded = isinstance(self.globals[n], OverloadedFuncDef) - if is_overloaded_func and prev_is_overloaded: - self.fail("Nonconsecutive overload {} found".format(n), ctx) - elif prev_is_overloaded: - self.fail("Definition of '{}' missing 'overload'".format(n), ctx) - else: - self.name_already_defined(n, ctx) - - def name_not_defined(self, name: str, ctx: Context) -> None: - message = "Name '{}' is not defined".format(name) - extra = self.undefined_name_extra_info(name) - if extra: - message += ' {}'.format(extra) - self.fail(message, ctx) - - def name_already_defined(self, name: str, ctx: Context) -> None: - self.fail("Name '{}' already defined".format(name), ctx) - - def fail(self, msg: str, ctx: Context, serious: bool = False, *, - blocker: bool = False) -> None: - if (not serious and - not self.options.check_untyped_defs and - self.function_stack and - self.function_stack[-1].is_dynamic()): - return - self.errors.report(ctx.get_line(), msg, blocker=blocker) - - def note(self, msg: str, ctx: Context) -> None: - if (not self.options.check_untyped_defs and - self.function_stack and - self.function_stack[-1].is_dynamic()): - return - self.errors.report(ctx.get_line(), msg, severity='note') - - def undefined_name_extra_info(self, fullname: str) -> Optional[str]: - if fullname in obsolete_name_mapping: - return "(it's now called '{}')".format(obsolete_name_mapping[fullname]) - else: - return None - - def accept(self, node: Node) -> None: - try: - node.accept(self) - except Exception as err: - report_internal_error(err, self.errors.file, node.line) - - -class FirstPass(NodeVisitor): - """First phase of semantic analysis. - - See docstring of 'analyze()' below for a description of what this does. - """ - - def __init__(self, sem: SemanticAnalyzer) -> None: - self.sem = sem - self.pyversion = sem.options.python_version - - def analyze(self, file: MypyFile, fnam: str, mod_id: str) -> None: - """Perform the first analysis pass. - - Populate module global table. Resolve the full names of - definitions not nested within functions and construct type - info structures, but do not resolve inter-definition - references such as base classes. - - Also add implicit definitions such as __name__. - - In this phase we don't resolve imports. For 'from ... import', - we generate dummy symbol table nodes for the imported names, - and these will get resolved in later phases of semantic - analysis. - """ - sem = self.sem - sem.cur_mod_id = mod_id - sem.errors.set_file(fnam) - sem.globals = SymbolTable() - sem.global_decls = [set()] - sem.nonlocal_decls = [set()] - sem.block_depth = [0] - - defs = file.defs - - # Add implicit definitions of module '__name__' etc. - for name, t in implicit_module_attrs.items(): - v = Var(name, UnboundType(t)) - v._fullname = self.sem.qualified_name(name) - self.sem.globals[name] = SymbolTableNode(GDEF, v, self.sem.cur_mod_id) - - for d in defs: - d.accept(self) - - # Add implicit definition of literals/keywords to builtins, as we - # cannot define a variable with them explicitly. - if mod_id == 'builtins': - literal_types = [ - ('None', NoneTyp()), - # reveal_type is a mypy-only function that gives an error with the type of its arg - ('reveal_type', AnyType()), - ] # type: List[Tuple[str, Type]] - - # TODO(ddfisher): This guard is only needed because mypy defines - # fake builtins for its tests which often don't define bool. If - # mypy is fast enough that we no longer need those, this - # conditional check should be removed. - if 'bool' in self.sem.globals: - bool_type = self.sem.named_type('bool') - literal_types.extend([ - ('True', bool_type), - ('False', bool_type), - ('__debug__', bool_type), - ]) - - for name, typ in literal_types: - v = Var(name, typ) - v._fullname = self.sem.qualified_name(name) - self.sem.globals[name] = SymbolTableNode(GDEF, v, self.sem.cur_mod_id) - - def visit_block(self, b: Block) -> None: - if b.is_unreachable: - return - self.sem.block_depth[-1] += 1 - for node in b.body: - node.accept(self) - self.sem.block_depth[-1] -= 1 - - def visit_assignment_stmt(self, s: AssignmentStmt) -> None: - for lval in s.lvalues: - self.analyze_lvalue(lval, explicit_type=s.type is not None) - - def visit_func_def(self, func: FuncDef) -> None: - sem = self.sem - func.is_conditional = sem.block_depth[-1] > 0 - func._fullname = sem.qualified_name(func.name()) - if func.name() in sem.globals: - # Already defined in this module. - original_sym = sem.globals[func.name()] - if original_sym.kind == UNBOUND_IMPORTED: - # Ah this is an imported name. We can't resolve them now, so we'll postpone - # this until the main phase of semantic analysis. - return - original_def = original_sym.node - if sem.is_conditional_func(original_def, func): - # Conditional function definition -- multiple defs are ok. - func.original_def = cast(FuncDef, original_def) - else: - # Report error. - sem.check_no_global(func.name(), func) - else: - sem.globals[func.name()] = SymbolTableNode(GDEF, func, sem.cur_mod_id) - - def visit_overloaded_func_def(self, func: OverloadedFuncDef) -> None: - self.sem.check_no_global(func.name(), func, True) - func._fullname = self.sem.qualified_name(func.name()) - self.sem.globals[func.name()] = SymbolTableNode(GDEF, func, - self.sem.cur_mod_id) - - def visit_class_def(self, cdef: ClassDef) -> None: - self.sem.check_no_global(cdef.name, cdef) - cdef.fullname = self.sem.qualified_name(cdef.name) - info = TypeInfo(SymbolTable(), cdef) - info.set_line(cdef.line) - cdef.info = info - self.sem.globals[cdef.name] = SymbolTableNode(GDEF, info, - self.sem.cur_mod_id) - self.process_nested_classes(cdef) - - def process_nested_classes(self, outer_def: ClassDef) -> None: - for node in outer_def.defs.body: - if isinstance(node, ClassDef): - node.info = TypeInfo(SymbolTable(), node) - if outer_def.fullname: - node.info._fullname = outer_def.fullname + '.' + node.info.name() - else: - node.info._fullname = node.info.name() - symbol = SymbolTableNode(MDEF, node.info) - outer_def.info.names[node.name] = symbol - self.process_nested_classes(node) - - def visit_import_from(self, node: ImportFrom) -> None: - # We can't bind module names during the first pass, as the target module might be - # unprocessed. However, we add dummy unbound imported names to the symbol table so - # that we at least know that the name refers to a module. - node.is_top_level = True - for name, as_name in node.names: - imported_name = as_name or name - if imported_name not in self.sem.globals: - self.sem.add_symbol(imported_name, SymbolTableNode(UNBOUND_IMPORTED, None), node) - - def visit_import(self, node: Import) -> None: - node.is_top_level = True - # This is similar to visit_import_from -- see the comment there. - for id, as_id in node.ids: - imported_id = as_id or id - if imported_id not in self.sem.globals: - self.sem.add_symbol(imported_id, SymbolTableNode(UNBOUND_IMPORTED, None), node) - else: - # If the previous symbol is a variable, this should take precedence. - self.sem.globals[imported_id] = SymbolTableNode(UNBOUND_IMPORTED, None) - - def visit_import_all(self, node: ImportAll) -> None: - node.is_top_level = True - - def visit_while_stmt(self, s: WhileStmt) -> None: - s.body.accept(self) - if s.else_body: - s.else_body.accept(self) - - def visit_for_stmt(self, s: ForStmt) -> None: - self.analyze_lvalue(s.index) - s.body.accept(self) - if s.else_body: - s.else_body.accept(self) - - def visit_with_stmt(self, s: WithStmt) -> None: - for n in s.target: - if n: - self.analyze_lvalue(n) - s.body.accept(self) - - def visit_decorator(self, d: Decorator) -> None: - d.var._fullname = self.sem.qualified_name(d.var.name()) - self.sem.add_symbol(d.var.name(), SymbolTableNode(GDEF, d.var), d) - - def visit_if_stmt(self, s: IfStmt) -> None: - infer_reachability_of_if_statement(s, pyversion=self.pyversion) - for node in s.body: - node.accept(self) - if s.else_body: - s.else_body.accept(self) - - def visit_try_stmt(self, s: TryStmt) -> None: - self.sem.analyze_try_stmt(s, self, add_global=True) - - def analyze_lvalue(self, lvalue: Node, explicit_type: bool = False) -> None: - self.sem.analyze_lvalue(lvalue, add_global=True, explicit_type=explicit_type) - - -class ThirdPass(TraverserVisitor): - """The third and final pass of semantic analysis. - - Check type argument counts and values of generic types, and perform some - straightforward type inference. - """ - - def __init__(self, modules: Dict[str, MypyFile], errors: Errors) -> None: - self.modules = modules - self.errors = errors - - def visit_file(self, file_node: MypyFile, fnam: str) -> None: - self.errors.set_file(fnam) - self.accept(file_node) - - def accept(self, node: Node) -> None: - try: - node.accept(self) - except Exception as err: - report_internal_error(err, self.errors.file, node.line) - - def visit_block(self, b: Block) -> None: - if b.is_unreachable: - return - super().visit_block(b) - - def visit_func_def(self, fdef: FuncDef) -> None: - self.errors.push_function(fdef.name()) - self.analyze(fdef.type) - super().visit_func_def(fdef) - self.errors.pop_function() - - def visit_class_def(self, tdef: ClassDef) -> None: - for type in tdef.info.bases: - self.analyze(type) - # Recompute MRO now that we have analyzed all modules, to pick - # up superclasses of bases imported from other modules in an - # import loop. (Only do so if we succeeded the first time.) - if tdef.info.mro: - tdef.info.mro = [] # Force recomputation - calculate_class_mro(tdef, self.fail) - super().visit_class_def(tdef) - - def visit_decorator(self, dec: Decorator) -> None: - """Try to infer the type of the decorated function. - - This lets us resolve references to decorated functions during - type checking when there are cyclic imports, as otherwise the - type might not be available when we need it. - - This basically uses a simple special-purpose type inference - engine just for decorators. - """ - super().visit_decorator(dec) - if dec.var.is_property: - # Decorators are expected to have a callable type (it's a little odd). - if dec.func.type is None: - dec.var.type = CallableType( - [AnyType()], - [ARG_POS], - [None], - AnyType(), - self.builtin_type('function'), - name=dec.var.name()) - elif isinstance(dec.func.type, CallableType): - dec.var.type = dec.func.type - return - decorator_preserves_type = True - for expr in dec.decorators: - preserve_type = False - if isinstance(expr, RefExpr) and isinstance(expr.node, FuncDef): - if is_identity_signature(expr.node.type): - preserve_type = True - if not preserve_type: - decorator_preserves_type = False - break - if decorator_preserves_type: - # No non-identity decorators left. We can trivially infer the type - # of the function here. - dec.var.type = function_type(dec.func, self.builtin_type('function')) - if dec.decorators: - if returns_any_if_called(dec.decorators[0]): - # The outermost decorator will return Any so we know the type of the - # decorated function. - dec.var.type = AnyType() - sig = find_fixed_callable_return(dec.decorators[0]) - if sig: - # The outermost decorator always returns the same kind of function, - # so we know that this is the type of the decoratored function. - orig_sig = function_type(dec.func, self.builtin_type('function')) - sig.name = orig_sig.items()[0].name - dec.var.type = sig - - def visit_assignment_stmt(self, s: AssignmentStmt) -> None: - self.analyze(s.type) - super().visit_assignment_stmt(s) - - def visit_cast_expr(self, e: CastExpr) -> None: - self.analyze(e.type) - super().visit_cast_expr(e) - - def visit_reveal_type_expr(self, e: RevealTypeExpr) -> None: - super().visit_reveal_type_expr(e) - - def visit_type_application(self, e: TypeApplication) -> None: - for type in e.types: - self.analyze(type) - super().visit_type_application(e) - - # Helpers - - def analyze(self, type: Type) -> None: - if type: - analyzer = TypeAnalyserPass3(self.fail) - type.accept(analyzer) - - def fail(self, msg: str, ctx: Context) -> None: - self.errors.report(ctx.get_line(), msg) - - def builtin_type(self, name: str, args: List[Type] = None) -> Instance: - names = self.modules['builtins'] - sym = names.names[name] - assert isinstance(sym.node, TypeInfo) - return Instance(sym.node, args or []) - - -def self_type(typ: TypeInfo) -> Union[Instance, TupleType]: - """For a non-generic type, return instance type representing the type. - For a generic G type with parameters T1, .., Tn, return G[T1, ..., Tn]. - """ - tv = [] # type: List[Type] - for i in range(len(typ.type_vars)): - tv.append(TypeVarType(typ.defn.type_vars[i])) - inst = Instance(typ, tv) - if typ.tuple_type is None: - return inst - return typ.tuple_type.self_type(inst) - - -def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: - if isinstance(sig, CallableType): - return replace_leading_arg_type(sig, new) - else: - sig = cast(Overloaded, sig) - return Overloaded([cast(CallableType, replace_implicit_first_type(i, new)) - for i in sig.items()]) - - -def set_callable_name(sig: Type, fdef: FuncDef) -> Type: - if isinstance(sig, FunctionLike): - if fdef.info: - return sig.with_name( - '"{}" of "{}"'.format(fdef.name(), fdef.info.name())) - else: - return sig.with_name('"{}"'.format(fdef.name())) - else: - return sig - - -def refers_to_fullname(node: Node, fullname: str) -> bool: - """Is node a name or member expression with the given full name?""" - return isinstance(node, RefExpr) and node.fullname == fullname - - -def refers_to_class_or_function(node: Node) -> bool: - """Does semantically analyzed node refer to a class?""" - return (isinstance(node, RefExpr) and - isinstance(node.node, (TypeInfo, FuncDef, OverloadedFuncDef))) - - -def calculate_class_mro(defn: ClassDef, fail: Callable[[str, Context], None]) -> None: - try: - defn.info.calculate_mro() - except MroError: - fail("Cannot determine consistent method resolution order " - '(MRO) for "%s"' % defn.name, defn) - defn.info.mro = [] - # The property of falling back to Any is inherited. - defn.info.fallback_to_any = any(baseinfo.fallback_to_any for baseinfo in defn.info.mro) - - -def find_duplicate(list: List[T]) -> T: - """If the list has duplicates, return one of the duplicates. - - Otherwise, return None. - """ - for i in range(1, len(list)): - if list[i] in list[:i]: - return list[i] - return None - - -def disable_typevars(nodes: List[SymbolTableNode]) -> None: - for node in nodes: - assert node.kind in (BOUND_TVAR, UNBOUND_TVAR) - node.kind = UNBOUND_TVAR - - -def enable_typevars(nodes: List[SymbolTableNode]) -> None: - for node in nodes: - assert node.kind in (BOUND_TVAR, UNBOUND_TVAR) - node.kind = BOUND_TVAR - - -def remove_imported_names_from_symtable(names: SymbolTable, - module: str) -> None: - """Remove all imported names from the symbol table of a module.""" - removed = [] # type: List[str] - for name, node in names.items(): - if node.node is None: - continue - fullname = node.node.fullname() - prefix = fullname[:fullname.rfind('.')] - if prefix != module: - removed.append(name) - for name in removed: - del names[name] - - -def infer_reachability_of_if_statement(s: IfStmt, - pyversion: Tuple[int, int]) -> None: - for i in range(len(s.expr)): - result = infer_if_condition_value(s.expr[i], pyversion) - if result == ALWAYS_FALSE: - # The condition is always false, so we skip the if/elif body. - mark_block_unreachable(s.body[i]) - elif result == ALWAYS_TRUE: - # This condition is always true, so all of the remaining - # elif/else bodies will never be executed. - for body in s.body[i + 1:]: - mark_block_unreachable(body) - if s.else_body: - mark_block_unreachable(s.else_body) - break - - -def infer_if_condition_value(expr: Node, pyversion: Tuple[int, int]) -> int: - """Infer whether if condition is always true/false. - - Return ALWAYS_TRUE if always true, ALWAYS_FALSE if always false, - and TRUTH_VALUE_UNKNOWN otherwise. - """ - name = '' - negated = False - alias = expr - if isinstance(alias, UnaryExpr): - if alias.op == 'not': - expr = alias.expr - negated = True - if isinstance(expr, NameExpr): - name = expr.name - elif isinstance(expr, MemberExpr): - name = expr.name - result = TRUTH_VALUE_UNKNOWN - if name == 'PY2': - result = ALWAYS_TRUE if pyversion[0] == 2 else ALWAYS_FALSE - elif name == 'PY3': - result = ALWAYS_TRUE if pyversion[0] == 3 else ALWAYS_FALSE - elif name == 'MYPY': - result = ALWAYS_TRUE - if negated: - if result == ALWAYS_TRUE: - result = ALWAYS_FALSE - elif result == ALWAYS_FALSE: - result = ALWAYS_TRUE - return result - - -def mark_block_unreachable(block: Block) -> None: - block.is_unreachable = True - block.accept(MarkImportsUnreachableVisitor()) - - -class MarkImportsUnreachableVisitor(TraverserVisitor): - """Visitor that flags all imports nested within a node as unreachable.""" - - def visit_import(self, node: Import) -> None: - node.is_unreachable = True - - def visit_import_from(self, node: ImportFrom) -> None: - node.is_unreachable = True - - def visit_import_all(self, node: ImportAll) -> None: - node.is_unreachable = True - - -def is_identity_signature(sig: Type) -> bool: - """Is type a callable of form T -> T (where T is a type variable)?""" - if isinstance(sig, CallableType) and sig.arg_kinds == [ARG_POS]: - if isinstance(sig.arg_types[0], TypeVarType) and isinstance(sig.ret_type, TypeVarType): - return sig.arg_types[0].id == sig.ret_type.id - return False - - -def returns_any_if_called(expr: Node) -> bool: - """Return True if we can predict that expr will return Any if called. - - This only uses information available during semantic analysis so this - will sometimes return False because of insufficient information (as - type inference hasn't run yet). - """ - if isinstance(expr, RefExpr): - if isinstance(expr.node, FuncDef): - typ = expr.node.type - if typ is None: - # No signature -> default to Any. - return True - # Explicit Any return? - return isinstance(typ, CallableType) and isinstance(typ.ret_type, AnyType) - elif isinstance(expr.node, Var): - typ = expr.node.type - return typ is None or isinstance(typ, AnyType) - elif isinstance(expr, CallExpr): - return returns_any_if_called(expr.callee) - return False - - -def find_fixed_callable_return(expr: Node) -> Optional[CallableType]: - if isinstance(expr, RefExpr): - if isinstance(expr.node, FuncDef): - typ = expr.node.type - if typ: - if isinstance(typ, CallableType) and has_no_typevars(typ.ret_type): - if isinstance(typ.ret_type, CallableType): - return typ.ret_type - elif isinstance(expr, CallExpr): - t = find_fixed_callable_return(expr.callee) - if t: - if isinstance(t.ret_type, CallableType): - return t.ret_type - return None - - -def has_no_typevars(typ: Type) -> bool: - return is_same_type(typ, erase_typevars(typ)) +"""The semantic analyzer. + +Bind names to definitions and do various other simple consistency +checks. For example, consider this program: + + x = 1 + y = x + +Here semantic analysis would detect that the assignment 'x = 1' +defines a new variable, the type of which is to be inferred (in a +later pass; type inference or type checking is not part of semantic +analysis). Also, it would bind both references to 'x' to the same +module-level variable node. The second assignment would also be +analyzed, and the type of 'y' marked as being inferred. + +Semantic analysis is the first analysis pass after parsing, and it is +subdivided into three passes: + + * FirstPass looks up externally visible names defined in a module but + ignores imports and local definitions. It helps enable (some) + cyclic references between modules, such as module 'a' that imports + module 'b' and used names defined in b *and* vice versa. The first + pass can be performed before dependent modules have been processed. + + * SemanticAnalyzer is the second pass. It does the bulk of the work. + It assumes that dependent modules have been semantically analyzed, + up to the second pass, unless there is a import cycle. + + * ThirdPass checks that type argument counts are valid; for example, + it will reject Dict[int]. We don't do this in the second pass, + since we infer the type argument counts of classes during this + pass, and it is possible to refer to classes defined later in a + file, which would not have the type argument count set yet. This + pass also recomputes the method resolution order of each class, in + case one of its bases belongs to a module involved in an import + loop. + +Semantic analysis of types is implemented in module mypy.typeanal. + +TODO: Check if the third pass slows down type checking significantly. + We could probably get rid of it -- for example, we could collect all + analyzed types in a collection and check them without having to + traverse the entire AST. +""" + +from typing import ( + List, Dict, Set, Tuple, cast, Any, overload, TypeVar, Union, Optional, Callable +) + +from mypy.nodes import ( + MypyFile, TypeInfo, Node, AssignmentStmt, FuncDef, OverloadedFuncDef, + ClassDef, Var, GDEF, MODULE_REF, FuncItem, Import, + ImportFrom, ImportAll, Block, LDEF, NameExpr, MemberExpr, + IndexExpr, TupleExpr, ListExpr, ExpressionStmt, ReturnStmt, + RaiseStmt, AssertStmt, OperatorAssignmentStmt, WhileStmt, + ForStmt, BreakStmt, ContinueStmt, IfStmt, TryStmt, WithStmt, DelStmt, + GlobalDecl, SuperExpr, DictExpr, CallExpr, RefExpr, OpExpr, UnaryExpr, + SliceExpr, CastExpr, RevealTypeExpr, TypeApplication, Context, SymbolTable, + SymbolTableNode, BOUND_TVAR, UNBOUND_TVAR, ListComprehension, GeneratorExpr, + FuncExpr, MDEF, FuncBase, Decorator, SetExpr, TypeVarExpr, + StrExpr, BytesExpr, PrintStmt, ConditionalExpr, PromoteExpr, + ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, MroError, type_aliases, + YieldFromExpr, NamedTupleExpr, NonlocalDecl, + SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr, + YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, COVARIANT, CONTRAVARIANT, + IntExpr, FloatExpr, UnicodeExpr, + INVARIANT, UNBOUND_IMPORTED, Expression, EllipsisExpr, NamedTupleTypeInfo +) +from mypy.visitor import NodeVisitor +from mypy.traverser import TraverserVisitor +from mypy.errors import Errors, report_internal_error +from mypy.types import ( + NoneTyp, CallableType, Overloaded, Instance, Type, TypeVarType, AnyType, + FunctionLike, UnboundType, TypeList, ErrorType, TypeVarDef, + replace_leading_arg_type, TupleType, UnionType, StarType, EllipsisType, + NamedTupleType +) +from mypy.nodes import function_type, implicit_module_attrs +from mypy.typeanal import TypeAnalyser, TypeAnalyserPass3, analyze_type_alias +from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError +from mypy.lex import lex +from mypy.parsetype import parse_type +from mypy.sametypes import is_same_type +from mypy.erasetype import erase_typevars +from mypy import defaults +from mypy.options import Options + + +T = TypeVar('T') + + +# Inferred value of an expression. +ALWAYS_TRUE = 0 +ALWAYS_FALSE = 1 +TRUTH_VALUE_UNKNOWN = 2 + +# Map from obsolete name to the current spelling. +obsolete_name_mapping = { + 'typing.Function': 'typing.Callable', + 'typing.typevar': 'typing.TypeVar', +} + +# Hard coded type promotions (shared between all Python versions). +# These add extra ad-hoc edges to the subtyping relation. For example, +# int is considered a subtype of float, even though there is no +# subclass relationship. +TYPE_PROMOTIONS = { + 'builtins.int': 'builtins.float', + 'builtins.float': 'builtins.complex', +} + +# Hard coded type promotions for Python 3. +# +# Note that the bytearray -> bytes promotion is a little unsafe +# as some functions only accept bytes objects. Here convenience +# trumps safety. +TYPE_PROMOTIONS_PYTHON3 = TYPE_PROMOTIONS.copy() +TYPE_PROMOTIONS_PYTHON3.update({ + 'builtins.bytearray': 'builtins.bytes', +}) + +# Hard coded type promotions for Python 2. +# +# These promotions are unsafe, but we are doing them anyway +# for convenience and also for Python 3 compatibility +# (bytearray -> str). +TYPE_PROMOTIONS_PYTHON2 = TYPE_PROMOTIONS.copy() +TYPE_PROMOTIONS_PYTHON2.update({ + 'builtins.str': 'builtins.unicode', + 'builtins.bytearray': 'builtins.str', +}) + +# When analyzing a function, should we analyze the whole function in one go, or +# should we only perform one phase of the analysis? The latter is used for +# nested functions. In the first phase we add the function to the symbol table +# but don't process body. In the second phase we process function body. This +# way we can have mutually recursive nested functions. +FUNCTION_BOTH_PHASES = 0 # Everthing in one go +FUNCTION_FIRST_PHASE_POSTPONE_SECOND = 1 # Add to symbol table but postpone body +FUNCTION_SECOND_PHASE = 2 # Only analyze body + + +class SemanticAnalyzer(NodeVisitor): + """Semantically analyze parsed mypy files. + + The analyzer binds names and does various consistency checks for a + parse tree. Note that type checking is performed as a separate + pass. + + This is the second phase of semantic analysis. + """ + + # Library search paths + lib_path = None # type: List[str] + # Module name space + modules = None # type: Dict[str, MypyFile] + # Global name space for current module + globals = None # type: SymbolTable + # Names declared using "global" (separate set for each scope) + global_decls = None # type: List[Set[str]] + # Names declated using "nonlocal" (separate set for each scope) + nonlocal_decls = None # type: List[Set[str]] + # Local names of function scopes; None for non-function scopes. + locals = None # type: List[SymbolTable] + # Nested block depths of scopes + block_depth = None # type: List[int] + # TypeInfo of directly enclosing class (or None) + type = None # type: TypeInfo + # Stack of outer classes (the second tuple item contains tvars). + type_stack = None # type: List[TypeInfo] + # Type variables that are bound by the directly enclosing class + bound_tvars = None # type: List[SymbolTableNode] + # Stack of type variables that were bound by outer classess + tvar_stack = None # type: List[List[SymbolTableNode]] + # Do weak type checking in this file + weak_opts = set() # type: Set[str] + + # Stack of functions being analyzed + function_stack = None # type: List[FuncItem] + # Stack of next available function type variable ids + next_function_tvar_id_stack = None # type: List[int] + + # Status of postponing analysis of nested function bodies. By using this we + # can have mutually recursive nested functions. Values are FUNCTION_x + # constants. Note that separate phasea are not used for methods. + postpone_nested_functions_stack = None # type: List[int] + # Postponed functions collected if + # postpone_nested_functions_stack[-1] == FUNCTION_FIRST_PHASE_POSTPONE_SECOND. + postponed_functions_stack = None # type: List[List[Node]] + + loop_depth = 0 # Depth of breakable loops + cur_mod_id = '' # Current module id (or None) (phase 2) + is_stub_file = False # Are we analyzing a stub file? + imports = None # type: Set[str] # Imported modules (during phase 2 analysis) + errors = None # type: Errors # Keeps track of generated errors + + def __init__(self, + lib_path: List[str], + errors: Errors, + options: Options) -> None: + """Construct semantic analyzer. + + Use lib_path to search for modules, and report analysis errors + using the Errors instance. + """ + self.locals = [None] + self.imports = set() + self.type = None + self.type_stack = [] + self.bound_tvars = None + self.tvar_stack = [] + self.function_stack = [] + self.next_function_tvar_id_stack = [-1] + self.block_depth = [0] + self.loop_depth = 0 + self.lib_path = lib_path + self.errors = errors + self.modules = {} + self.options = options + self.postpone_nested_functions_stack = [FUNCTION_BOTH_PHASES] + self.postponed_functions_stack = [] + self.all_exports = set() # type: Set[str] + + def visit_file(self, file_node: MypyFile, fnam: str) -> None: + self.errors.set_file(fnam) + self.cur_mod_node = file_node + self.cur_mod_id = file_node.fullname() + self.is_stub_file = fnam.lower().endswith('.pyi') + self.globals = file_node.names + self.weak_opts = file_node.weak_opts + + if 'builtins' in self.modules: + self.globals['__builtins__'] = SymbolTableNode( + MODULE_REF, self.modules['builtins'], self.cur_mod_id) + + for name in implicit_module_attrs: + v = self.globals[name].node + if isinstance(v, Var): + v.type = self.anal_type(v.type) + v.is_ready = True + + defs = file_node.defs + for d in defs: + self.accept(d) + + if self.cur_mod_id == 'builtins': + remove_imported_names_from_symtable(self.globals, 'builtins') + + if '__all__' in self.globals: + for name, g in self.globals.items(): + if name not in self.all_exports: + g.module_public = False + + def visit_func_def(self, defn: FuncDef) -> None: + phase_info = self.postpone_nested_functions_stack[-1] + if phase_info != FUNCTION_SECOND_PHASE: + self.function_stack.append(defn) + # First phase of analysis for function. + self.errors.push_function(defn.name()) + self.update_function_type_variables(defn) + self.errors.pop_function() + self.function_stack.pop() + + defn.is_conditional = self.block_depth[-1] > 0 + + # TODO(jukka): Figure out how to share the various cases. It doesn't + # make sense to have (almost) duplicate code (here and elsewhere) for + # 3 cases: module-level, class-level and local names. Maybe implement + # a common stack of namespaces. As the 3 kinds of namespaces have + # different semantics, this wouldn't always work, but it might still + # be a win. + if self.is_class_scope(): + # Method definition + defn.info = self.type + if not defn.is_decorated and not defn.is_overload: + if defn.name() in self.type.names: + # Redefinition. Conditional redefinition is okay. + n = self.type.names[defn.name()].node + if self.is_conditional_func(n, defn): + defn.original_def = cast(FuncDef, n) + else: + self.name_already_defined(defn.name(), defn) + self.type.names[defn.name()] = SymbolTableNode(MDEF, defn) + self.prepare_method_signature(defn) + elif self.is_func_scope(): + # Nested function + if not defn.is_decorated and not defn.is_overload: + if defn.name() in self.locals[-1]: + # Redefinition. Conditional redefinition is okay. + n = self.locals[-1][defn.name()].node + if self.is_conditional_func(n, defn): + defn.original_def = cast(FuncDef, n) + else: + self.name_already_defined(defn.name(), defn) + else: + self.add_local(defn, defn) + else: + # Top-level function + if not defn.is_decorated and not defn.is_overload: + symbol = self.globals.get(defn.name()) + if isinstance(symbol.node, FuncDef) and symbol.node != defn: + # This is redefinition. Conditional redefinition is okay. + original_def = symbol.node + if self.is_conditional_func(original_def, defn): + # Conditional function definition -- multiple defs are ok. + defn.original_def = original_def + else: + # Report error. + self.check_no_global(defn.name(), defn, True) + if phase_info == FUNCTION_FIRST_PHASE_POSTPONE_SECOND: + # Postpone this function (for the second phase). + self.postponed_functions_stack[-1].append(defn) + return + if phase_info != FUNCTION_FIRST_PHASE_POSTPONE_SECOND: + # Second phase of analysis for function. + self.errors.push_function(defn.name()) + self.analyze_function(defn) + self.errors.pop_function() + + def prepare_method_signature(self, func: FuncDef) -> None: + """Check basic signature validity and tweak annotation of self/cls argument.""" + # Only non-static methods are special. + if not func.is_static: + if not func.arguments: + self.fail('Method must have at least one argument', func) + elif func.type: + sig = cast(FunctionLike, func.type) + if func.is_class: + leading_type = self.class_type(self.type) + else: + leading_type = self_type(self.type) + func.type = replace_implicit_first_type(sig, leading_type) + + def is_conditional_func(self, previous: Node, new: FuncDef) -> bool: + """Does 'new' conditionally redefine 'previous'? + + We reject straight redefinitions of functions, as they are usually + a programming error. For example: + + . def f(): ... + . def f(): ... # Error: 'f' redefined + """ + return isinstance(previous, (FuncDef, Var)) and new.is_conditional + + def update_function_type_variables(self, defn: FuncDef) -> None: + """Make any type variables in the signature of defn explicit. + + Update the signature of defn to contain type variable definitions + if defn is generic. + """ + if defn.type: + functype = cast(CallableType, defn.type) + typevars = self.infer_type_variables(functype) + # Do not define a new type variable if already defined in scope. + typevars = [(name, tvar) for name, tvar in typevars + if not self.is_defined_type_var(name, defn)] + if typevars: + next_tvar_id = self.next_function_tvar_id() + defs = [TypeVarDef(tvar[0], next_tvar_id - i, + tvar[1].values, tvar[1].upper_bound, + tvar[1].variance) + for i, tvar in enumerate(typevars)] + functype.variables = defs + + def infer_type_variables(self, + type: CallableType) -> List[Tuple[str, TypeVarExpr]]: + """Return list of unique type variables referred to in a callable.""" + names = [] # type: List[str] + tvars = [] # type: List[TypeVarExpr] + for arg in type.arg_types + [type.ret_type]: + for name, tvar_expr in self.find_type_variables_in_type(arg): + if name not in names: + names.append(name) + tvars.append(tvar_expr) + return list(zip(names, tvars)) + + def find_type_variables_in_type( + self, type: Type) -> List[Tuple[str, TypeVarExpr]]: + """Return a list of all unique type variable references in type. + + This effectively does partial name binding, results of which are mostly thrown away. + """ + result = [] # type: List[Tuple[str, TypeVarExpr]] + if isinstance(type, UnboundType): + name = type.name + node = self.lookup_qualified(name, type) + if node and node.kind == UNBOUND_TVAR: + result.append((name, cast(TypeVarExpr, node.node))) + for arg in type.args: + result.extend(self.find_type_variables_in_type(arg)) + elif isinstance(type, TypeList): + for item in type.items: + result.extend(self.find_type_variables_in_type(item)) + elif isinstance(type, UnionType): + for item in type.items: + result.extend(self.find_type_variables_in_type(item)) + elif isinstance(type, AnyType): + pass + elif isinstance(type, EllipsisType) or isinstance(type, TupleType): + pass + else: + assert False, 'Unsupported type %s' % type + return result + + def is_defined_type_var(self, tvar: str, context: Node) -> bool: + return self.lookup_qualified(tvar, context).kind == BOUND_TVAR + + def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: + t = [] # type: List[CallableType] + for i, item in enumerate(defn.items): + # TODO support decorated overloaded functions properly + item.is_overload = True + item.func.is_overload = True + item.accept(self) + t.append(cast(CallableType, function_type(item.func, + self.builtin_type('builtins.function')))) + if item.func.is_property and i == 0: + # This defines a property, probably with a setter and/or deleter. + self.analyze_property_with_multi_part_definition(defn) + break + if not [dec for dec in item.decorators + if refers_to_fullname(dec, 'typing.overload')]: + self.fail("'overload' decorator expected", item) + + defn.type = Overloaded(t) + defn.type.line = defn.line + + if self.is_class_scope(): + self.type.names[defn.name()] = SymbolTableNode(MDEF, defn, + typ=defn.type) + defn.info = self.type + elif self.is_func_scope(): + self.add_local(defn, defn) + + def analyze_property_with_multi_part_definition(self, defn: OverloadedFuncDef) -> None: + """Analyze a property defined using multiple methods (e.g., using @x.setter). + + Assume that the first method (@property) has already been analyzed. + """ + defn.is_property = True + items = defn.items + for item in items[1:]: + if len(item.decorators) == 1: + node = item.decorators[0] + if isinstance(node, MemberExpr): + if node.name == 'setter': + # The first item represents the entire property. + defn.items[0].var.is_settable_property = True + # Get abstractness from the original definition. + item.func.is_abstract = items[0].func.is_abstract + else: + self.fail("Decorated property not supported", item) + item.func.accept(self) + + def next_function_tvar_id(self) -> int: + return self.next_function_tvar_id_stack[-1] + + def analyze_function(self, defn: FuncItem) -> None: + is_method = self.is_class_scope() + + tvarnodes = self.add_func_type_variables_to_symbol_table(defn) + next_function_tvar_id = min([self.next_function_tvar_id()] + + [n.tvar_def.id.raw_id - 1 for n in tvarnodes]) + self.next_function_tvar_id_stack.append(next_function_tvar_id) + + if defn.type: + # Signature must be analyzed in the surrounding scope so that + # class-level imported names and type variables are in scope. + defn.type = self.anal_type(defn.type) + self.check_function_signature(defn) + if isinstance(defn, FuncDef): + defn.type = set_callable_name(defn.type, defn) + for arg in defn.arguments: + if arg.initializer: + arg.initializer.accept(self) + self.function_stack.append(defn) + self.enter() + for arg in defn.arguments: + self.add_local(arg.variable, defn) + for arg in defn.arguments: + if arg.initialization_statement: + lvalue = arg.initialization_statement.lvalues[0] + lvalue.accept(self) + + # The first argument of a non-static, non-class method is like 'self' + # (though the name could be different), having the enclosing class's + # instance type. + if is_method and not defn.is_static and not defn.is_class and defn.arguments: + defn.arguments[0].variable.is_self = True + + # First analyze body of the function but ignore nested functions. + self.postpone_nested_functions_stack.append(FUNCTION_FIRST_PHASE_POSTPONE_SECOND) + self.postponed_functions_stack.append([]) + defn.body.accept(self) + + # Analyze nested functions (if any) as a second phase. + self.postpone_nested_functions_stack[-1] = FUNCTION_SECOND_PHASE + for postponed in self.postponed_functions_stack[-1]: + postponed.accept(self) + self.postpone_nested_functions_stack.pop() + self.postponed_functions_stack.pop() + + self.next_function_tvar_id_stack.pop() + disable_typevars(tvarnodes) + + self.leave() + self.function_stack.pop() + + def add_func_type_variables_to_symbol_table( + self, defn: FuncItem) -> List[SymbolTableNode]: + nodes = [] # type: List[SymbolTableNode] + if defn.type: + tt = defn.type + names = self.type_var_names() + items = cast(CallableType, tt).variables + for item in items: + name = item.name + if name in names: + self.name_already_defined(name, defn) + node = self.bind_type_var(name, item, defn) + nodes.append(node) + names.add(name) + return nodes + + def type_var_names(self) -> Set[str]: + if not self.type: + return set() + else: + return set(self.type.type_vars) + + def bind_type_var(self, fullname: str, tvar_def: TypeVarDef, + context: Context) -> SymbolTableNode: + node = self.lookup_qualified(fullname, context) + node.kind = BOUND_TVAR + node.tvar_def = tvar_def + return node + + def check_function_signature(self, fdef: FuncItem) -> None: + sig = cast(CallableType, fdef.type) + if len(sig.arg_types) < len(fdef.arguments): + self.fail('Type signature has too few arguments', fdef) + # Add dummy Any arguments to prevent crashes later. + extra_anys = [AnyType()] * (len(fdef.arguments) - len(sig.arg_types)) + sig.arg_types.extend(extra_anys) + elif len(sig.arg_types) > len(fdef.arguments): + self.fail('Type signature has too many arguments', fdef) + + def visit_class_def(self, defn: ClassDef) -> None: + self.clean_up_bases_and_infer_type_variables(defn) + self.setup_class_def_analysis(defn) + + self.bind_class_type_vars(defn) + + self.analyze_base_classes(defn) + self.analyze_metaclass(defn) + + for decorator in defn.decorators: + self.analyze_class_decorator(defn, decorator) + + self.enter_class(defn) + + self.setup_is_builtinclass(defn) + + # Analyze class body. + defn.defs.accept(self) + + self.calculate_abstract_status(defn.info) + self.setup_type_promotion(defn) + + self.leave_class() + self.unbind_class_type_vars() + + def enter_class(self, defn: ClassDef) -> None: + # Remember previous active class + self.type_stack.append(self.type) + self.locals.append(None) # Add class scope + self.block_depth.append(-1) # The class body increments this to 0 + self.postpone_nested_functions_stack.append(FUNCTION_BOTH_PHASES) + self.type = defn.info + + def leave_class(self) -> None: + """ Restore analyzer state. """ + self.postpone_nested_functions_stack.pop() + self.block_depth.pop() + self.locals.pop() + self.type = self.type_stack.pop() + + def bind_class_type_vars(self, defn: ClassDef) -> None: + """ Unbind type variables of previously active class and bind + the type variables for the active class. + """ + if self.bound_tvars: + disable_typevars(self.bound_tvars) + self.tvar_stack.append(self.bound_tvars) + self.bound_tvars = self.bind_class_type_variables_in_symbol_table(defn.info) + + def unbind_class_type_vars(self) -> None: + """ Unbind the active class' type vars and rebind the + type vars of the previously active class. + """ + disable_typevars(self.bound_tvars) + self.bound_tvars = self.tvar_stack.pop() + if self.bound_tvars: + enable_typevars(self.bound_tvars) + + def analyze_class_decorator(self, defn: ClassDef, decorator: Node) -> None: + decorator.accept(self) + + def setup_is_builtinclass(self, defn: ClassDef): + for decorator in defn.decorators: + if refers_to_fullname(decorator, 'typing.builtinclass'): + defn.is_builtinclass = True + if defn.fullname == 'builtins.object': + # Only 'object' is marked as a built-in class, as otherwise things elsewhere + # would break. We need a better way of dealing with built-in classes. + defn.is_builtinclass = True + + def calculate_abstract_status(self, typ: TypeInfo) -> None: + """Calculate abstract status of a class. + + Set is_abstract of the type to True if the type has an unimplemented + abstract attribute. Also compute a list of abstract attributes. + """ + concrete = set() # type: Set[str] + abstract = [] # type: List[str] + for base in typ.mro: + for name, symnode in base.names.items(): + node = symnode.node + if isinstance(node, OverloadedFuncDef): + # Unwrap an overloaded function definition. We can just + # check arbitrarily the first overload item. If the + # different items have a different abstract status, there + # should be an error reported elsewhere. + func = node.items[0] # type: Node + else: + func = node + if isinstance(func, Decorator): + fdef = func.func + if fdef.is_abstract and name not in concrete: + typ.is_abstract = True + abstract.append(name) + concrete.add(name) + typ.abstract_attributes = sorted(abstract) + + def setup_type_promotion(self, defn: ClassDef) -> None: + """Setup extra, ad-hoc subtyping relationships between classes (promotion). + + This includes things like 'int' being compatible with 'float'. + """ + promote_target = None # type: Type + for decorator in defn.decorators: + if isinstance(decorator, CallExpr): + analyzed = decorator.analyzed + if isinstance(analyzed, PromoteExpr): + # _promote class decorator (undocumented faeture). + promote_target = analyzed.type + if not promote_target: + promotions = (TYPE_PROMOTIONS_PYTHON3 if self.options.python_version[0] >= 3 + else TYPE_PROMOTIONS_PYTHON2) + if defn.fullname in promotions: + promote_target = self.named_type_or_none(promotions[defn.fullname]) + defn.info._promote = promote_target + + def clean_up_bases_and_infer_type_variables(self, defn: ClassDef) -> None: + """Remove extra base classes such as Generic and infer type vars. + + For example, consider this class: + + . class Foo(Bar, Generic[T]): ... + + Now we will remove Generic[T] from bases of Foo and infer that the + type variable 'T' is a type argument of Foo. + + Note that this is performed *before* semantic analysis. + """ + removed = [] # type: List[int] + type_vars = [] # type: List[TypeVarDef] + for i, base_expr in enumerate(defn.base_type_exprs): + try: + base = expr_to_unanalyzed_type(base_expr) + except TypeTranslationError: + # This error will be caught later. + continue + tvars = self.analyze_typevar_declaration(base) + if tvars is not None: + if type_vars: + self.fail('Duplicate Generic in bases', defn) + removed.append(i) + for j, (name, tvar_expr) in enumerate(tvars): + type_vars.append(TypeVarDef(name, j + 1, tvar_expr.values, + tvar_expr.upper_bound, tvar_expr.variance)) + if type_vars: + defn.type_vars = type_vars + if defn.info: + defn.info.type_vars = [tv.name for tv in type_vars] + for i in reversed(removed): + del defn.base_type_exprs[i] + + def analyze_typevar_declaration(self, t: Type) -> Optional[List[Tuple[str, TypeVarExpr]]]: + if not isinstance(t, UnboundType): + return None + unbound = t + sym = self.lookup_qualified(unbound.name, unbound) + if sym is None or sym.node is None: + return None + if sym.node.fullname() == 'typing.Generic': + tvars = [] # type: List[Tuple[str, TypeVarExpr]] + for arg in unbound.args: + tvar = self.analyze_unbound_tvar(arg) + if tvar: + tvars.append(tvar) + else: + self.fail('Free type variable expected in %s[...]' % + sym.node.name(), t) + return tvars + return None + + def analyze_unbound_tvar(self, t: Type) -> Tuple[str, TypeVarExpr]: + if not isinstance(t, UnboundType): + return None + unbound = t + sym = self.lookup_qualified(unbound.name, unbound) + if sym is not None and sym.kind == UNBOUND_TVAR: + return unbound.name, cast(TypeVarExpr, sym.node) + return None + + def setup_class_def_analysis(self, defn: ClassDef) -> None: + """Prepare for the analysis of a class definition.""" + if not defn.info: + defn.info = TypeInfo(SymbolTable(), defn) + defn.info._fullname = defn.info.name() + if self.is_func_scope() or self.type: + kind = MDEF + if self.is_func_scope(): + kind = LDEF + self.add_symbol(defn.name, SymbolTableNode(kind, defn.info), defn) + + def analyze_base_classes(self, defn: ClassDef) -> None: + """Analyze and set up base classes. + + This computes several attributes on the corresponding TypeInfo defn.info + related to the base classes: defn.info.bases, defn.info.mro, and + miscellaneous others (at least tuple_type, fallback_to_any, and is_enum.) + """ + + base_types = [] # type: List[Instance] + info = defn.info + for base_expr in defn.base_type_exprs: + try: + base = self.expr_to_analyzed_type(base_expr) + except TypeTranslationError: + self.fail('Invalid base class', base_expr) + info.fallback_to_any = True + continue + + if isinstance(base, TupleType): + if info.tuple_type: + self.fail("Class has two incompatible bases derived from tuple", defn) + defn.incompatible_baseclass = True + if (not self.is_stub_file + and not info.is_named_tuple + and base.fallback.type.fullname() == 'builtins.tuple'): + self.fail("Tuple[...] not supported as a base class outside a stub file", defn) + info.tuple_type = base + base_types.append(base.fallback) + elif isinstance(base, Instance): + base_types.append(base) + elif isinstance(base, AnyType): + info.fallback_to_any = True + else: + self.fail('Invalid base class', base_expr) + info.fallback_to_any = True + + # Add 'object' as implicit base if there is no other base class. + if (not base_types and defn.fullname != 'builtins.object'): + base_types.append(self.object_type()) + + info.bases = base_types + + # Calculate the MRO. It might be incomplete at this point if + # the bases of defn include classes imported from other + # modules in an import loop. We'll recompute it in ThirdPass. + if not self.verify_base_classes(defn): + info.mro = [] + return + calculate_class_mro(defn, self.fail) + # If there are cyclic imports, we may be missing 'object' in + # the MRO. Fix MRO if needed. + if info.mro and info.mro[-1].fullname() != 'builtins.object': + info.mro.append(self.object_type().type) + + def expr_to_analyzed_type(self, expr: Node) -> Type: + if isinstance(expr, CallExpr): + expr.accept(self) + info = self.check_namedtuple(expr) + if info is None: + # Some form of namedtuple is the only valid type that looks like a call + # expression. This isn't a valid type. + raise TypeTranslationError() + fallback = Instance(info, []) + return TupleType(info.tuple_type.items, fallback=fallback) + typ = expr_to_unanalyzed_type(expr) + return self.anal_type(typ) + + def verify_base_classes(self, defn: ClassDef) -> bool: + info = defn.info + for base in info.bases: + baseinfo = base.type + if self.is_base_class(info, baseinfo): + self.fail('Cycle in inheritance hierarchy', defn) + # Clear bases to forcefully get rid of the cycle. + info.bases = [] + if baseinfo.fullname() == 'builtins.bool': + self.fail("'%s' is not a valid base class" % + baseinfo.name(), defn) + return False + dup = find_duplicate(info.direct_base_classes()) + if dup: + self.fail('Duplicate base class "%s"' % dup.name(), defn) + return False + return True + + def is_base_class(self, t: TypeInfo, s: TypeInfo) -> bool: + """Determine if t is a base class of s (but do not use mro).""" + # Search the base class graph for t, starting from s. + worklist = [s] + visited = {s} + while worklist: + nxt = worklist.pop() + if nxt == t: + return True + for base in nxt.bases: + if base.type not in visited: + worklist.append(base.type) + visited.add(base.type) + return False + + def analyze_metaclass(self, defn: ClassDef) -> None: + if defn.metaclass: + sym = self.lookup_qualified(defn.metaclass, defn) + if sym is not None and not isinstance(sym.node, TypeInfo): + self.fail("Invalid metaclass '%s'" % defn.metaclass, defn) + + def object_type(self) -> Instance: + return self.named_type('__builtins__.object') + + def class_type(self, info: TypeInfo) -> Type: + # Construct a function type whose fallback is cls. + from mypy import checkmember # To avoid import cycle. + leading_type = checkmember.type_object_type(info, self.builtin_type) + if isinstance(leading_type, Overloaded): + # Overloaded __init__ is too complex to handle. Plus it's stubs only. + return AnyType() + else: + return leading_type + + def named_type(self, qualified_name: str, args: List[Type] = None) -> Instance: + sym = self.lookup_qualified(qualified_name, None) + return Instance(cast(TypeInfo, sym.node), args or []) + + def named_type_or_none(self, qualified_name: str) -> Instance: + sym = self.lookup_fully_qualified_or_none(qualified_name) + if not sym: + return None + return Instance(cast(TypeInfo, sym.node), []) + + def is_instance_type(self, t: Type) -> bool: + return isinstance(t, Instance) + + def bind_class_type_variables_in_symbol_table( + self, info: TypeInfo) -> List[SymbolTableNode]: + nodes = [] # type: List[SymbolTableNode] + for var, binder in zip(info.type_vars, info.defn.type_vars): + node = self.bind_type_var(var, binder, info) + nodes.append(node) + return nodes + + def visit_import(self, i: Import) -> None: + for id, as_id in i.ids: + if as_id is not None: + self.add_module_symbol(id, as_id, module_public=True, context=i) + else: + # Modules imported in a stub file without using 'as x' won't get exported when + # doing 'from m import *'. + module_public = not self.is_stub_file + base = id.split('.')[0] + self.add_module_symbol(base, base, module_public=module_public, + context=i) + + def add_module_symbol(self, id: str, as_id: str, module_public: bool, + context: Context) -> None: + if id in self.modules: + m = self.modules[id] + self.add_symbol(as_id, SymbolTableNode(MODULE_REF, m, self.cur_mod_id, + module_public=module_public), context) + else: + self.add_unknown_symbol(as_id, context) + + def visit_import_from(self, imp: ImportFrom) -> None: + import_id = self.correct_relative_import(imp) + if import_id in self.modules: + module = self.modules[import_id] + for id, as_id in imp.names: + node = module.names.get(id) + if node and node.kind != UNBOUND_IMPORTED: + node = self.normalize_type_alias(node, imp) + if not node: + return + imported_id = as_id or id + existing_symbol = self.globals.get(imported_id) + if existing_symbol: + # Import can redefine a variable. They get special treatment. + if self.process_import_over_existing_name( + imported_id, existing_symbol, node, imp): + continue + # 'from m import x as x' exports x in a stub file. + module_public = not self.is_stub_file or as_id is not None + symbol = SymbolTableNode(node.kind, node.node, + self.cur_mod_id, + node.type_override, + module_public=module_public) + self.add_symbol(imported_id, symbol, imp) + else: + message = "Module has no attribute '{}'".format(id) + extra = self.undefined_name_extra_info('{}.{}'.format(import_id, id)) + if extra: + message += " {}".format(extra) + self.fail(message, imp) + else: + # Missing module. + for id, as_id in imp.names: + self.add_unknown_symbol(as_id or id, imp) + + def process_import_over_existing_name(self, + imported_id: str, existing_symbol: SymbolTableNode, + module_symbol: SymbolTableNode, + import_node: ImportBase) -> bool: + if (existing_symbol.kind in (LDEF, GDEF, MDEF) and + isinstance(existing_symbol.node, (Var, FuncDef))): + # This is a valid import over an existing definition in the file. Construct a dummy + # assignment that we'll use to type check the import. + lvalue = NameExpr(imported_id) + lvalue.kind = existing_symbol.kind + lvalue.node = existing_symbol.node + rvalue = NameExpr(imported_id) + rvalue.kind = module_symbol.kind + rvalue.node = module_symbol.node + assignment = AssignmentStmt([lvalue], rvalue) + for node in assignment, lvalue, rvalue: + node.set_line(import_node) + import_node.assignments.append(assignment) + return True + return False + + def normalize_type_alias(self, node: SymbolTableNode, + ctx: Context) -> SymbolTableNode: + if node.fullname in type_aliases: + # Node refers to an aliased type such as typing.List; normalize. + node = self.lookup_qualified(type_aliases[node.fullname], ctx) + if node.fullname == 'typing.DefaultDict': + self.add_module_symbol('collections', '__mypy_collections__', False, ctx) + node = self.lookup_qualified('__mypy_collections__.defaultdict', ctx) + return node + + def correct_relative_import(self, node: Union[ImportFrom, ImportAll]) -> str: + if node.relative == 0: + return node.id + + parts = self.cur_mod_id.split(".") + cur_mod_id = self.cur_mod_id + + rel = node.relative + if self.cur_mod_node.is_package_init_file(): + rel -= 1 + if len(parts) < rel: + self.fail("Relative import climbs too many namespaces", node) + if rel != 0: + cur_mod_id = ".".join(parts[:-rel]) + + return cur_mod_id + (("." + node.id) if node.id else "") + + def visit_import_all(self, i: ImportAll) -> None: + i_id = self.correct_relative_import(i) + if i_id in self.modules: + m = self.modules[i_id] + for name, node in m.names.items(): + node = self.normalize_type_alias(node, i) + if not name.startswith('_') and node.module_public: + existing_symbol = self.globals.get(name) + if existing_symbol: + # Import can redefine a variable. They get special treatment. + if self.process_import_over_existing_name( + name, existing_symbol, node, i): + continue + self.add_symbol(name, SymbolTableNode(node.kind, node.node, + self.cur_mod_id), i) + else: + # Don't add any dummy symbols for 'from x import *' if 'x' is unknown. + pass + + def add_unknown_symbol(self, name: str, context: Context) -> None: + var = Var(name) + var._fullname = self.qualified_name(name) + var.is_ready = True + var.type = AnyType() + self.add_symbol(name, SymbolTableNode(GDEF, var, self.cur_mod_id), context) + + # + # Statements + # + + def visit_block(self, b: Block) -> None: + if b.is_unreachable: + return + self.block_depth[-1] += 1 + for s in b.body: + self.accept(s) + self.block_depth[-1] -= 1 + + def visit_block_maybe(self, b: Block) -> None: + if b: + self.visit_block(b) + + def anal_type(self, t: Type, allow_tuple_literal: bool = False) -> Type: + if t: + if allow_tuple_literal: + # Types such as (t1, t2, ...) only allowed in assignment statements. They'll + # generate errors elsewhere, and Tuple[t1, t2, ...] must be used instead. + if isinstance(t, TupleType): + # Unlike TypeAnalyser, also allow implicit tuple types (without Tuple[...]). + star_count = sum(1 for item in t.items if isinstance(item, StarType)) + if star_count > 1: + self.fail('At most one star type allowed in a tuple', t) + return TupleType([AnyType() for _ in t.items], + self.builtin_type('builtins.tuple'), t.line) + items = [self.anal_type(item, True) + for item in t.items] + return TupleType(items, self.builtin_type('builtins.tuple'), t.line) + a = TypeAnalyser(self.lookup_qualified, + self.lookup_fully_qualified, + self.fail) + return t.accept(a) + else: + return None + + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + for lval in s.lvalues: + self.analyze_lvalue(lval, explicit_type=s.type is not None) + s.rvalue.accept(self) + if s.type: + allow_tuple_literal = isinstance(s.lvalues[-1], (TupleExpr, ListExpr)) + s.type = self.anal_type(s.type, allow_tuple_literal) + else: + # For simple assignments, allow binding type aliases. + # Also set the type if the rvalue is a simple literal. + if (s.type is None and len(s.lvalues) == 1 and + isinstance(s.lvalues[0], NameExpr)): + if s.lvalues[0].is_def: + s.type = self.analyze_simple_literal_type(s.rvalue) + res = analyze_type_alias(s.rvalue, + self.lookup_qualified, + self.lookup_fully_qualified, + self.fail) + if res and (not isinstance(res, Instance) or cast(Instance, res).args): + # TODO: What if this gets reassigned? + name = s.lvalues[0] + node = self.lookup(name.name, name) + node.kind = TYPE_ALIAS + node.type_override = res + if isinstance(s.rvalue, IndexExpr): + s.rvalue.analyzed = TypeAliasExpr(res) + if s.type: + # Store type into nodes. + for lvalue in s.lvalues: + self.store_declared_types(lvalue, s.type) + self.check_and_set_up_type_alias(s) + self.process_typevar_declaration(s) + self.process_namedtuple_definition(s) + + if (len(s.lvalues) == 1 and isinstance(s.lvalues[0], NameExpr) and + s.lvalues[0].name == '__all__' and s.lvalues[0].kind == GDEF and + isinstance(s.rvalue, (ListExpr, TupleExpr))): + self.add_exports(*s.rvalue.items) + + def analyze_simple_literal_type(self, rvalue: Node) -> Optional[Type]: + """Return builtins.int if rvalue is an int literal, etc.""" + if self.weak_opts or self.options.semantic_analysis_only or self.function_stack: + # Skip this if any weak options are set. + # Also skip if we're only doing the semantic analysis pass. + # This is mostly to avoid breaking unit tests. + # Also skip inside a function; this is to avoid confusing + # the code that handles dead code due to isinstance() + # inside type variables with value restrictions (like + # AnyStr). + return None + if isinstance(rvalue, IntExpr): + return self.named_type_or_none('builtins.int') + if isinstance(rvalue, FloatExpr): + return self.named_type_or_none('builtins.float') + if isinstance(rvalue, StrExpr): + return self.named_type_or_none('builtins.str') + if isinstance(rvalue, BytesExpr): + return self.named_type_or_none('builtins.bytes') + if isinstance(rvalue, UnicodeExpr): + return self.named_type_or_none('builtins.unicode') + return None + + def check_and_set_up_type_alias(self, s: AssignmentStmt) -> None: + """Check if assignment creates a type alias and set it up as needed.""" + # For now, type aliases only work at the top level of a module. + if (len(s.lvalues) == 1 and not self.is_func_scope() and not self.type + and not s.type): + lvalue = s.lvalues[0] + if isinstance(lvalue, NameExpr): + if not lvalue.is_def: + # Only a definition can create a type alias, not regular assignment. + return + rvalue = s.rvalue + if isinstance(rvalue, RefExpr): + node = rvalue.node + if isinstance(node, TypeInfo): + # TODO: We should record the fact that this is a variable + # that refers to a type, rather than making this + # just an alias for the type. + self.globals[lvalue.name].node = node + + def analyze_lvalue(self, lval: Node, nested: bool = False, + add_global: bool = False, + explicit_type: bool = False) -> None: + """Analyze an lvalue or assignment target. + + Only if add_global is True, add name to globals table. If nested + is true, the lvalue is within a tuple or list lvalue expression. + """ + + if isinstance(lval, NameExpr): + # Top-level definitions within some statements (at least while) are + # not handled in the first pass, so they have to be added now. + nested_global = (not self.is_func_scope() and + self.block_depth[-1] > 0 and + not self.type) + if (add_global or nested_global) and lval.name not in self.globals: + # Define new global name. + v = Var(lval.name) + v._fullname = self.qualified_name(lval.name) + v.is_ready = False # Type not inferred yet + lval.node = v + lval.is_def = True + lval.kind = GDEF + lval.fullname = v._fullname + self.globals[lval.name] = SymbolTableNode(GDEF, v, + self.cur_mod_id) + elif isinstance(lval.node, Var) and lval.is_def: + # Since the is_def flag is set, this must have been analyzed + # already in the first pass and added to the symbol table. + assert lval.node.name() in self.globals + elif (self.is_func_scope() and lval.name not in self.locals[-1] and + lval.name not in self.global_decls[-1] and + lval.name not in self.nonlocal_decls[-1]): + # Define new local name. + v = Var(lval.name) + lval.node = v + lval.is_def = True + lval.kind = LDEF + lval.fullname = lval.name + self.add_local(v, lval) + elif not self.is_func_scope() and (self.type and + lval.name not in self.type.names): + # Define a new attribute within class body. + v = Var(lval.name) + v.info = self.type + v.is_initialized_in_class = True + v.set_line(lval) + lval.node = v + lval.is_def = True + lval.kind = MDEF + lval.fullname = lval.name + self.type.names[lval.name] = SymbolTableNode(MDEF, v) + else: + # Bind to an existing name. + if explicit_type: + self.name_already_defined(lval.name, lval) + lval.accept(self) + self.check_lvalue_validity(lval.node, lval) + elif isinstance(lval, MemberExpr): + if not add_global: + self.analyze_member_lvalue(lval) + if explicit_type and not self.is_self_member_ref(lval): + self.fail('Type cannot be declared in assignment to non-self ' + 'attribute', lval) + elif isinstance(lval, IndexExpr): + if explicit_type: + self.fail('Unexpected type declaration', lval) + if not add_global: + lval.accept(self) + elif (isinstance(lval, TupleExpr) or + isinstance(lval, ListExpr)): + items = cast(Any, lval).items + if len(items) == 0 and isinstance(lval, TupleExpr): + self.fail("Can't assign to ()", lval) + self.analyze_tuple_or_list_lvalue(cast(Union[ListExpr, TupleExpr], lval), + add_global, explicit_type) + elif isinstance(lval, StarExpr): + if nested: + self.analyze_lvalue(lval.expr, nested, add_global, explicit_type) + else: + self.fail('Starred assignment target must be in a list or tuple', lval) + else: + self.fail('Invalid assignment target', lval) + + def analyze_tuple_or_list_lvalue(self, lval: Union[ListExpr, TupleExpr], + add_global: bool = False, + explicit_type: bool = False) -> None: + """Analyze an lvalue or assignment target that is a list or tuple.""" + items = lval.items + star_exprs = [cast(StarExpr, item) + for item in items + if isinstance(item, StarExpr)] + + if len(star_exprs) > 1: + self.fail('Two starred expressions in assignment', lval) + else: + if len(star_exprs) == 1: + star_exprs[0].valid = True + for i in items: + self.analyze_lvalue(i, nested=True, add_global=add_global, + explicit_type = explicit_type) + + def analyze_member_lvalue(self, lval: MemberExpr) -> None: + lval.accept(self) + if (self.is_self_member_ref(lval) and + self.type.get(lval.name) is None): + # Implicit attribute definition in __init__. + lval.is_def = True + v = Var(lval.name) + v.info = self.type + v.is_ready = False + lval.def_var = v + lval.node = v + self.type.names[lval.name] = SymbolTableNode(MDEF, v) + self.check_lvalue_validity(lval.node, lval) + + def is_self_member_ref(self, memberexpr: MemberExpr) -> bool: + """Does memberexpr to refer to an attribute of self?""" + if not isinstance(memberexpr.expr, NameExpr): + return False + node = memberexpr.expr.node + return isinstance(node, Var) and node.is_self + + def check_lvalue_validity(self, node: Node, ctx: Context) -> None: + if isinstance(node, (TypeInfo, TypeVarExpr)): + self.fail('Invalid assignment target', ctx) + + def store_declared_types(self, lvalue: Node, typ: Type) -> None: + if isinstance(typ, StarType) and not isinstance(lvalue, StarExpr): + self.fail('Star type only allowed for starred expressions', lvalue) + if isinstance(lvalue, RefExpr): + lvalue.is_def = False + if isinstance(lvalue.node, Var): + var = lvalue.node + var.type = typ + var.is_ready = True + # If node is not a variable, we'll catch it elsewhere. + elif isinstance(lvalue, TupleExpr): + if isinstance(typ, TupleType): + if len(lvalue.items) != len(typ.items): + self.fail('Incompatible number of tuple items', lvalue) + return + for item, itemtype in zip(lvalue.items, typ.items): + self.store_declared_types(item, itemtype) + else: + self.fail('Tuple type expected for multiple variables', + lvalue) + elif isinstance(lvalue, StarExpr): + if isinstance(typ, StarType): + self.store_declared_types(lvalue.expr, typ.type) + else: + self.fail('Star type expected for starred expression', lvalue) + else: + # This has been flagged elsewhere as an error, so just ignore here. + pass + + def process_typevar_declaration(self, s: AssignmentStmt) -> None: + """Check if s declares a TypeVar; it yes, store it in symbol table.""" + call = self.get_typevar_declaration(s) + if not call: + return + + lvalue = cast(NameExpr, s.lvalues[0]) + name = lvalue.name + if not lvalue.is_def: + if s.type: + self.fail("Cannot declare the type of a type variable", s) + else: + self.fail("Cannot redefine '%s' as a type variable" % name, s) + return + + if not self.check_typevar_name(call, name, s): + return + + # Constraining types + n_values = call.arg_kinds[1:].count(ARG_POS) + values = self.analyze_types(call.args[1:1 + n_values]) + + res = self.process_typevar_parameters(call.args[1 + n_values:], + call.arg_names[1 + n_values:], + call.arg_kinds[1 + n_values:], + bool(values), + s) + if res is None: + return + variance, upper_bound = res + + # Yes, it's a valid type variable definition! Add it to the symbol table. + node = self.lookup(name, s) + node.kind = UNBOUND_TVAR + TypeVar = TypeVarExpr(name, node.fullname, values, upper_bound, variance) + TypeVar.line = call.line + call.analyzed = TypeVar + node.node = TypeVar + + def check_typevar_name(self, call: CallExpr, name: str, context: Context) -> bool: + if len(call.args) < 1: + self.fail("Too few arguments for TypeVar()", context) + return False + if not isinstance(call.args[0], (StrExpr, BytesExpr)) or not call.arg_kinds[0] == ARG_POS: + self.fail("TypeVar() expects a string literal as first argument", context) + return False + if cast(StrExpr, call.args[0]).value != name: + self.fail("Unexpected TypeVar() argument value", context) + return False + return True + + def get_typevar_declaration(self, s: AssignmentStmt) -> Optional[CallExpr]: + """Returns the TypeVar() call expression if `s` is a type var declaration + or None otherwise. + """ + if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): + return None + if not isinstance(s.rvalue, CallExpr): + return None + call = s.rvalue + if not isinstance(call.callee, RefExpr): + return None + callee = call.callee + if callee.fullname != 'typing.TypeVar': + return None + return call + + def process_typevar_parameters(self, args: List[Node], + names: List[Optional[str]], + kinds: List[int], + has_values: bool, + context: Context) -> Optional[Tuple[int, Type]]: + covariant = False + contravariant = False + upper_bound = self.object_type() # type: Type + for param_value, param_name, param_kind in zip(args, names, kinds): + if not param_kind == ARG_NAMED: + self.fail("Unexpected argument to TypeVar()", context) + return None + if param_name == 'covariant': + if isinstance(param_value, NameExpr): + if param_value.name == 'True': + covariant = True + else: + self.fail("TypeVar 'covariant' may only be 'True'", context) + return None + else: + self.fail("TypeVar 'covariant' may only be 'True'", context) + return None + elif param_name == 'contravariant': + if isinstance(param_value, NameExpr): + if param_value.name == 'True': + contravariant = True + else: + self.fail("TypeVar 'contravariant' may only be 'True'", context) + return None + else: + self.fail("TypeVar 'contravariant' may only be 'True'", context) + return None + elif param_name == 'bound': + if has_values: + self.fail("TypeVar cannot have both values and an upper bound", context) + return None + try: + upper_bound = self.expr_to_analyzed_type(param_value) + except TypeTranslationError: + self.fail("TypeVar 'bound' must be a type", param_value) + return None + elif param_name == 'values': + # Probably using obsolete syntax with values=(...). Explain the current syntax. + self.fail("TypeVar 'values' argument not supported", context) + self.fail("Use TypeVar('T', t, ...) instead of TypeVar('T', values=(t, ...))", + context) + return None + else: + self.fail("Unexpected argument to TypeVar(): {}".format(param_name), context) + return None + + if covariant and contravariant: + self.fail("TypeVar cannot be both covariant and contravariant", context) + return None + elif covariant: + variance = COVARIANT + elif contravariant: + variance = CONTRAVARIANT + else: + variance = INVARIANT + return (variance, upper_bound) + + def process_namedtuple_definition(self, s: AssignmentStmt) -> None: + """Check if s defines a namedtuple; if yes, store the definition in symbol table.""" + if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): + return + lvalue = cast(NameExpr, s.lvalues[0]) + name = lvalue.name + named_tuple = self.check_namedtuple(s.rvalue, name) + if named_tuple is None: + return + # Yes, it's a valid namedtuple definition. Add it to the symbol table. + node = self.lookup(name, s) + node.kind = GDEF # TODO locally defined namedtuple + # TODO call.analyzed + node.node = named_tuple + + def check_namedtuple(self, node: Node, var_name: str = None) -> TypeInfo: + """Check if a call defines a namedtuple. + + The optional var_name argument is the name of the variable to + which this is assigned, if any. + + If it does, return the corresponding TypeInfo. Return None otherwise. + + If the definition is invalid but looks like a namedtuple, + report errors but return (some) TypeInfo. + """ + if not isinstance(node, CallExpr): + return None + call = node + if not isinstance(call.callee, RefExpr): + return None + callee = call.callee + fullname = callee.fullname + if fullname not in ('collections.namedtuple', 'typing.NamedTuple'): + return None + items, types, ok = self.parse_namedtuple_args(call, fullname) + if not ok: + # Error. Construct dummy return value. + return self.build_namedtuple_typeinfo('namedtuple', [], []) + else: + # Give it a unique name derived from the line number. + name = cast(StrExpr, call.args[0]).value + if name != var_name: + name += '@' + str(call.line) + info = self.build_namedtuple_typeinfo(name, items, types) + # Store it as a global just in case it would remain anonymous. + self.globals[name] = SymbolTableNode(GDEF, info, self.cur_mod_id) + call.analyzed = NamedTupleExpr(info).set_line(call.line) + return info + + def parse_namedtuple_args(self, call: CallExpr, + fullname: str) -> Tuple[List[str], List[Type], bool]: + # TODO Share code with check_argument_count in checkexpr.py? + args = call.args + if len(args) < 2: + return self.fail_namedtuple_arg("Too few arguments for namedtuple()", call) + if len(args) > 2: + return self.fail_namedtuple_arg("Too many arguments for namedtuple()", call) + if call.arg_kinds != [ARG_POS, ARG_POS]: + return self.fail_namedtuple_arg("Unexpected arguments to namedtuple()", call) + if not isinstance(args[0], (StrExpr, BytesExpr)): + return self.fail_namedtuple_arg( + "namedtuple() expects a string literal as the first argument", call) + types = [] # type: List[Type] + ok = True + if not isinstance(args[1], ListExpr): + if fullname == 'collections.namedtuple' and isinstance(args[1], (StrExpr, BytesExpr)): + str_expr = cast(StrExpr, args[1]) + items = str_expr.value.split() + else: + return self.fail_namedtuple_arg( + "List literal expected as the second argument to namedtuple()", call) + else: + listexpr = args[1] + if fullname == 'collections.namedtuple': + # The fields argument contains just names, with implicit Any types. + if any(not isinstance(item, (StrExpr, BytesExpr)) for item in listexpr.items): + return self.fail_namedtuple_arg("String literal expected as namedtuple() item", + call) + items = [cast(StrExpr, item).value for item in listexpr.items] + else: + # The fields argument contains (name, type) tuples. + items, types, ok = self.parse_namedtuple_fields_with_types(listexpr.items, call) + if not types: + types = [AnyType() for _ in items] + return items, types, ok + + def parse_namedtuple_fields_with_types(self, nodes: List[Node], + context: Context) -> Tuple[List[str], List[Type], bool]: + items = [] # type: List[str] + types = [] # type: List[Type] + for item in nodes: + if isinstance(item, TupleExpr): + if len(item.items) != 2: + return self.fail_namedtuple_arg("Invalid NamedTuple field definition", + item) + name, type_node = item.items + if isinstance(name, (StrExpr, BytesExpr)): + items.append(name.value) + else: + return self.fail_namedtuple_arg("Invalid NamedTuple() field name", item) + try: + type = expr_to_unanalyzed_type(type_node) + except TypeTranslationError: + return self.fail_namedtuple_arg('Invalid field type', type_node) + types.append(self.anal_type(type)) + else: + return self.fail_namedtuple_arg("Tuple expected as NamedTuple() field", item) + return items, types, True + + def fail_namedtuple_arg(self, message: str, + context: Context) -> Tuple[List[str], List[Type], bool]: + self.fail(message, context) + return [], [], False + + def build_namedtuple_typeinfo(self, name: str, items: List[str], + types: List[Type]) -> TypeInfo: + symbols = SymbolTable() + tup = NamedTupleType(items, types, self.named_type('__builtins__.tuple', types)) + class_def = ClassDef(name, Block([])) + class_def.fullname = self.qualified_name(name) + info = NamedTupleTypeInfo(tup, symbols, class_def) + vars = [Var(item, typ) for item, typ in zip(items, types)] + # Add named tuple items as attributes. + # TODO: Make them read-only. + for var in vars: + var.info = info + symbols[var.name()] = SymbolTableNode(MDEF, var) + # Add __init__, _replace, _asdict methods and _fields static field + self.add_namedtuple_method('__init__', symbols, info, NoneTyp(), + self.make_factory_args(vars, ARG_POS)) + self.add_namedtuple_method('_replace', symbols, info, self_type(info), + self.make_factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) + # TODO: refine to OrderedDict[Union[types]] + self.add_namedtuple_method('_asdict', symbols, info, AnyType()) + return info + + def make_factory_args(self, vars: List[Var], kind: int, + initializer: Expression = None) -> List[Argument]: + return [Argument(var, var.type, initializer, kind) for var in vars] + + def add_namedtuple_method(self, funcname: str, symbols: SymbolTable, info: TypeInfo, + ret: Type, args: List[Argument] = []) -> None: + types = [cast(Type, None)] + [arg.type_annotation for arg in args] + args = [Argument(Var('__self'), NoneTyp(), None, ARG_POS)] + args + items = [arg.variable.name() for arg in args] + arg_kinds = [arg.kind for arg in args] + # TODO: Make sure that the self argument name is not visible? + signature = CallableType(types, arg_kinds, items, ret, + self.named_type('__builtins__.function'), + name=info.name()) + func = FuncDef(funcname, + args, + Block([]), + typ=signature) + func.info = info + symbols[funcname] = SymbolTableNode(MDEF, func) + + def analyze_types(self, items: List[Node]) -> List[Type]: + result = [] # type: List[Type] + for node in items: + try: + result.append(self.anal_type(expr_to_unanalyzed_type(node))) + except TypeTranslationError: + self.fail('Type expected', node) + result.append(AnyType()) + return result + + def visit_decorator(self, dec: Decorator) -> None: + for d in dec.decorators: + d.accept(self) + removed = [] # type: List[int] + no_type_check = False + for i, d in enumerate(dec.decorators): + # A bunch of decorators are special cased here. + if refers_to_fullname(d, 'abc.abstractmethod'): + removed.append(i) + dec.func.is_abstract = True + self.check_decorated_function_is_method('abstractmethod', dec) + elif refers_to_fullname(d, 'asyncio.tasks.coroutine'): + removed.append(i) + elif refers_to_fullname(d, 'builtins.staticmethod'): + removed.append(i) + dec.func.is_static = True + dec.var.is_staticmethod = True + self.check_decorated_function_is_method('staticmethod', dec) + elif refers_to_fullname(d, 'builtins.classmethod'): + removed.append(i) + dec.func.is_class = True + dec.var.is_classmethod = True + self.check_decorated_function_is_method('classmethod', dec) + elif (refers_to_fullname(d, 'builtins.property') or + refers_to_fullname(d, 'abc.abstractproperty')): + removed.append(i) + dec.func.is_property = True + dec.var.is_property = True + if refers_to_fullname(d, 'abc.abstractproperty'): + dec.func.is_abstract = True + self.check_decorated_function_is_method('property', dec) + if len(dec.func.arguments) > 1: + self.fail('Too many arguments', dec.func) + elif refers_to_fullname(d, 'typing.no_type_check'): + dec.var.type = AnyType() + no_type_check = True + for i in reversed(removed): + del dec.decorators[i] + if not dec.is_overload or dec.var.is_property: + if self.is_func_scope(): + self.add_symbol(dec.var.name(), SymbolTableNode(LDEF, dec), + dec) + elif self.type: + dec.var.info = self.type + dec.var.is_initialized_in_class = True + self.add_symbol(dec.var.name(), SymbolTableNode(MDEF, dec), + dec) + if not no_type_check: + dec.func.accept(self) + if dec.decorators and dec.var.is_property: + self.fail('Decorated property not supported', dec) + + def check_decorated_function_is_method(self, decorator: str, + context: Context) -> None: + if not self.type or self.is_func_scope(): + self.fail("'%s' used with a non-method" % decorator, context) + + def visit_expression_stmt(self, s: ExpressionStmt) -> None: + s.expr.accept(self) + + def visit_return_stmt(self, s: ReturnStmt) -> None: + if not self.is_func_scope(): + self.fail("'return' outside function", s) + if s.expr: + s.expr.accept(self) + + def visit_raise_stmt(self, s: RaiseStmt) -> None: + if s.expr: + s.expr.accept(self) + if s.from_expr: + s.from_expr.accept(self) + + def visit_assert_stmt(self, s: AssertStmt) -> None: + if s.expr: + s.expr.accept(self) + + def visit_operator_assignment_stmt(self, + s: OperatorAssignmentStmt) -> None: + s.lvalue.accept(self) + s.rvalue.accept(self) + if (isinstance(s.lvalue, NameExpr) and s.lvalue.name == '__all__' and + s.lvalue.kind == GDEF and isinstance(s.rvalue, (ListExpr, TupleExpr))): + self.add_exports(*s.rvalue.items) + + def visit_while_stmt(self, s: WhileStmt) -> None: + s.expr.accept(self) + self.loop_depth += 1 + s.body.accept(self) + self.loop_depth -= 1 + self.visit_block_maybe(s.else_body) + + def visit_for_stmt(self, s: ForStmt) -> None: + s.expr.accept(self) + + # Bind index variables and check if they define new names. + self.analyze_lvalue(s.index) + + self.loop_depth += 1 + self.visit_block(s.body) + self.loop_depth -= 1 + + self.visit_block_maybe(s.else_body) + + def visit_break_stmt(self, s: BreakStmt) -> None: + if self.loop_depth == 0: + self.fail("'break' outside loop", s, True, blocker=True) + + def visit_continue_stmt(self, s: ContinueStmt) -> None: + if self.loop_depth == 0: + self.fail("'continue' outside loop", s, True, blocker=True) + + def visit_if_stmt(self, s: IfStmt) -> None: + infer_reachability_of_if_statement(s, pyversion=self.options.python_version) + for i in range(len(s.expr)): + s.expr[i].accept(self) + self.visit_block(s.body[i]) + self.visit_block_maybe(s.else_body) + + def visit_try_stmt(self, s: TryStmt) -> None: + self.analyze_try_stmt(s, self) + + def analyze_try_stmt(self, s: TryStmt, visitor: NodeVisitor, + add_global: bool = False) -> None: + s.body.accept(visitor) + for type, var, handler in zip(s.types, s.vars, s.handlers): + if type: + type.accept(visitor) + if var: + self.analyze_lvalue(var, add_global=add_global) + handler.accept(visitor) + if s.else_body: + s.else_body.accept(visitor) + if s.finally_body: + s.finally_body.accept(visitor) + + def visit_with_stmt(self, s: WithStmt) -> None: + for e, n in zip(s.expr, s.target): + e.accept(self) + if n: + self.analyze_lvalue(n) + self.visit_block(s.body) + + def visit_del_stmt(self, s: DelStmt) -> None: + s.expr.accept(self) + if not self.is_valid_del_target(s.expr): + self.fail('Invalid delete target', s) + + def is_valid_del_target(self, s: Node) -> bool: + if isinstance(s, (IndexExpr, NameExpr, MemberExpr)): + return True + elif isinstance(s, TupleExpr): + return all(self.is_valid_del_target(item) for item in s.items) + + def visit_global_decl(self, g: GlobalDecl) -> None: + for name in g.names: + if name in self.nonlocal_decls[-1]: + self.fail("Name '{}' is nonlocal and global".format(name), g) + self.global_decls[-1].add(name) + + def visit_nonlocal_decl(self, d: NonlocalDecl) -> None: + if not self.is_func_scope(): + self.fail("nonlocal declaration not allowed at module level", d) + else: + for name in d.names: + for table in reversed(self.locals[:-1]): + if table is not None and name in table: + break + else: + self.fail("No binding for nonlocal '{}' found".format(name), d) + + if self.locals[-1] is not None and name in self.locals[-1]: + self.fail("Name '{}' is already defined in local " + "scope before nonlocal declaration".format(name), d) + + if name in self.global_decls[-1]: + self.fail("Name '{}' is nonlocal and global".format(name), d) + self.nonlocal_decls[-1].add(name) + + def visit_print_stmt(self, s: PrintStmt) -> None: + for arg in s.args: + arg.accept(self) + if s.target: + s.target.accept(self) + + def visit_exec_stmt(self, s: ExecStmt) -> None: + s.expr.accept(self) + if s.variables1: + s.variables1.accept(self) + if s.variables2: + s.variables2.accept(self) + + # + # Expressions + # + + def visit_name_expr(self, expr: NameExpr) -> None: + n = self.lookup(expr.name, expr) + if n: + if n.kind == BOUND_TVAR: + self.fail("'{}' is a type variable and only valid in type " + "context".format(expr.name), expr) + else: + expr.kind = n.kind + expr.node = n.node + expr.fullname = n.fullname + + def visit_super_expr(self, expr: SuperExpr) -> None: + if not self.type: + self.fail('"super" used outside class', expr) + return + expr.info = self.type + + def visit_tuple_expr(self, expr: TupleExpr) -> None: + for item in expr.items: + item.accept(self) + + def visit_list_expr(self, expr: ListExpr) -> None: + for item in expr.items: + item.accept(self) + + def visit_set_expr(self, expr: SetExpr) -> None: + for item in expr.items: + item.accept(self) + + def visit_dict_expr(self, expr: DictExpr) -> None: + for key, value in expr.items: + key.accept(self) + value.accept(self) + + def visit_star_expr(self, expr: StarExpr) -> None: + if not expr.valid: + self.fail('Can use starred expression only as assignment target', expr) + else: + expr.expr.accept(self) + + def visit_yield_from_expr(self, e: YieldFromExpr) -> None: + if not self.is_func_scope(): # not sure + self.fail("'yield from' outside function", e, True, blocker=True) + else: + self.function_stack[-1].is_generator = True + if e.expr: + e.expr.accept(self) + + def visit_call_expr(self, expr: CallExpr) -> None: + """Analyze a call expression. + + Some call expressions are recognized as special forms, including + cast(...) and Any(...). + """ + expr.callee.accept(self) + if refers_to_fullname(expr.callee, 'typing.cast'): + # Special form cast(...). + if not self.check_fixed_args(expr, 2, 'cast'): + return + # Translate first argument to an unanalyzed type. + try: + target = expr_to_unanalyzed_type(expr.args[0]) + except TypeTranslationError: + self.fail('Cast target is not a type', expr) + return + # Piggyback CastExpr object to the CallExpr object; it takes + # precedence over the CallExpr semantics. + expr.analyzed = CastExpr(expr.args[1], target) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, 'builtins.reveal_type'): + if not self.check_fixed_args(expr, 1, 'reveal_type'): + return + expr.analyzed = RevealTypeExpr(expr.args[0]) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, 'typing.Any'): + # Special form Any(...). + if not self.check_fixed_args(expr, 1, 'Any'): + return + expr.analyzed = CastExpr(expr.args[0], AnyType()) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, 'typing._promote'): + # Special form _promote(...). + if not self.check_fixed_args(expr, 1, '_promote'): + return + # Translate first argument to an unanalyzed type. + try: + target = expr_to_unanalyzed_type(expr.args[0]) + except TypeTranslationError: + self.fail('Argument 1 to _promote is not a type', expr) + return + expr.analyzed = PromoteExpr(target) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, 'builtins.dict'): + expr.analyzed = self.translate_dict_call(expr) + else: + # Normal call expression. + for a in expr.args: + a.accept(self) + + if (isinstance(expr.callee, MemberExpr) and + isinstance(expr.callee.expr, NameExpr) and + expr.callee.expr.name == '__all__' and + expr.callee.expr.kind == GDEF and + expr.callee.name in ('append', 'extend')): + if expr.callee.name == 'append' and expr.args: + self.add_exports(expr.args[0]) + elif (expr.callee.name == 'extend' and expr.args and + isinstance(expr.args[0], (ListExpr, TupleExpr))): + self.add_exports(*expr.args[0].items) + + def translate_dict_call(self, call: CallExpr) -> Optional[DictExpr]: + """Translate 'dict(x=y, ...)' to {'x': y, ...}. + + For other variants of dict(...), return None. + """ + if not call.args: + return None + if not all(kind == ARG_NAMED for kind in call.arg_kinds): + # Must still accept those args. + for a in call.args: + a.accept(self) + return None + expr = DictExpr([(StrExpr(key), value) + for key, value in zip(call.arg_names, call.args)]) + expr.set_line(call) + expr.accept(self) + return expr + + def check_fixed_args(self, expr: CallExpr, numargs: int, + name: str) -> bool: + """Verify that expr has specified number of positional args. + + Return True if the arguments are valid. + """ + s = 's' + if numargs == 1: + s = '' + if len(expr.args) != numargs: + self.fail("'%s' expects %d argument%s" % (name, numargs, s), + expr) + return False + if expr.arg_kinds != [ARG_POS] * numargs: + self.fail("'%s' must be called with %s positional argument%s" % + (name, numargs, s), expr) + return False + return True + + def visit_member_expr(self, expr: MemberExpr) -> None: + base = expr.expr + base.accept(self) + # Bind references to module attributes. + if isinstance(base, RefExpr) and base.kind == MODULE_REF: + file = cast(MypyFile, base.node) + n = file.names.get(expr.name, None) if file is not None else None + if n: + n = self.normalize_type_alias(n, expr) + if not n: + return + expr.kind = n.kind + expr.fullname = n.fullname + expr.node = n.node + else: + # We only catch some errors here; the rest will be + # catched during type checking. + # + # This way we can report a larger number of errors in + # one type checker run. If we reported errors here, + # the build would terminate after semantic analysis + # and we wouldn't be able to report any type errors. + full_name = '%s.%s' % (file.fullname() if file is not None else None, expr.name) + if full_name in obsolete_name_mapping: + self.fail("Module has no attribute %r (it's now called %r)" % ( + expr.name, obsolete_name_mapping[full_name]), expr) + + def visit_op_expr(self, expr: OpExpr) -> None: + expr.left.accept(self) + expr.right.accept(self) + + def visit_comparison_expr(self, expr: ComparisonExpr) -> None: + for operand in expr.operands: + operand.accept(self) + + def visit_unary_expr(self, expr: UnaryExpr) -> None: + expr.expr.accept(self) + + def visit_index_expr(self, expr: IndexExpr) -> None: + expr.base.accept(self) + if refers_to_class_or_function(expr.base): + # Special form -- type application. + # Translate index to an unanalyzed type. + types = [] # type: List[Type] + if isinstance(expr.index, TupleExpr): + items = expr.index.items + else: + items = [expr.index] + for item in items: + try: + typearg = expr_to_unanalyzed_type(item) + except TypeTranslationError: + self.fail('Type expected within [...]', expr) + return + typearg = self.anal_type(typearg) + types.append(typearg) + expr.analyzed = TypeApplication(expr.base, types) + expr.analyzed.line = expr.line + else: + expr.index.accept(self) + + def visit_slice_expr(self, expr: SliceExpr) -> None: + if expr.begin_index: + expr.begin_index.accept(self) + if expr.end_index: + expr.end_index.accept(self) + if expr.stride: + expr.stride.accept(self) + + def visit_cast_expr(self, expr: CastExpr) -> None: + expr.expr.accept(self) + expr.type = self.anal_type(expr.type) + + def visit_reveal_type_expr(self, expr: RevealTypeExpr) -> None: + expr.expr.accept(self) + + def visit_type_application(self, expr: TypeApplication) -> None: + expr.expr.accept(self) + for i in range(len(expr.types)): + expr.types[i] = self.anal_type(expr.types[i]) + + def visit_list_comprehension(self, expr: ListComprehension) -> None: + expr.generator.accept(self) + + def visit_set_comprehension(self, expr: SetComprehension) -> None: + expr.generator.accept(self) + + def visit_dictionary_comprehension(self, expr: DictionaryComprehension) -> None: + self.enter() + self.analyze_comp_for(expr) + expr.key.accept(self) + expr.value.accept(self) + self.leave() + self.analyze_comp_for_2(expr) + + def visit_generator_expr(self, expr: GeneratorExpr) -> None: + self.enter() + self.analyze_comp_for(expr) + expr.left_expr.accept(self) + self.leave() + self.analyze_comp_for_2(expr) + + def analyze_comp_for(self, expr: Union[GeneratorExpr, + DictionaryComprehension]) -> None: + """Analyses the 'comp_for' part of comprehensions (part 1). + + That is the part after 'for' in (x for x in l if p). This analyzes + variables and conditions which are analyzed in a local scope. + """ + for i, (index, sequence, conditions) in enumerate(zip(expr.indices, + expr.sequences, + expr.condlists)): + if i > 0: + sequence.accept(self) + # Bind index variables. + self.analyze_lvalue(index) + for cond in conditions: + cond.accept(self) + + def analyze_comp_for_2(self, expr: Union[GeneratorExpr, + DictionaryComprehension]) -> None: + """Analyses the 'comp_for' part of comprehensions (part 2). + + That is the part after 'for' in (x for x in l if p). This analyzes + the 'l' part which is analyzed in the surrounding scope. + """ + expr.sequences[0].accept(self) + + def visit_func_expr(self, expr: FuncExpr) -> None: + self.analyze_function(expr) + + def visit_conditional_expr(self, expr: ConditionalExpr) -> None: + expr.if_expr.accept(self) + expr.cond.accept(self) + expr.else_expr.accept(self) + + def visit_backquote_expr(self, expr: BackquoteExpr) -> None: + expr.expr.accept(self) + + def visit__promote_expr(self, expr: PromoteExpr) -> None: + expr.type = self.anal_type(expr.type) + + def visit_yield_expr(self, expr: YieldExpr) -> None: + if not self.is_func_scope(): + self.fail("'yield' outside function", expr, True, blocker=True) + else: + self.function_stack[-1].is_generator = True + if expr.expr: + expr.expr.accept(self) + + # + # Helpers + # + + def lookup(self, name: str, ctx: Context) -> SymbolTableNode: + """Look up an unqualified name in all active namespaces.""" + # 1a. Name declared using 'global x' takes precedence + if name in self.global_decls[-1]: + if name in self.globals: + return self.globals[name] + else: + self.name_not_defined(name, ctx) + return None + # 1b. Name declared using 'nonlocal x' takes precedence + if name in self.nonlocal_decls[-1]: + for table in reversed(self.locals[:-1]): + if table is not None and name in table: + return table[name] + else: + self.name_not_defined(name, ctx) + return None + # 2. Class attributes (if within class definition) + if self.is_class_scope() and name in self.type.names: + return self.type[name] + # 3. Local (function) scopes + for table in reversed(self.locals): + if table is not None and name in table: + return table[name] + # 4. Current file global scope + if name in self.globals: + return self.globals[name] + # 5. Builtins + b = self.globals.get('__builtins__', None) + if b: + table = cast(MypyFile, b.node).names + if name in table: + if name[0] == "_" and name[1] != "_": + self.name_not_defined(name, ctx) + return None + node = table[name] + return node + # Give up. + self.name_not_defined(name, ctx) + self.check_for_obsolete_short_name(name, ctx) + return None + + def check_for_obsolete_short_name(self, name: str, ctx: Context) -> None: + matches = [obsolete_name + for obsolete_name in obsolete_name_mapping + if obsolete_name.rsplit('.', 1)[-1] == name] + if len(matches) == 1: + self.note("(Did you mean '{}'?)".format(obsolete_name_mapping[matches[0]]), ctx) + + def lookup_qualified(self, name: str, ctx: Context) -> SymbolTableNode: + if '.' not in name: + return self.lookup(name, ctx) + else: + parts = name.split('.') + n = self.lookup(parts[0], ctx) # type: SymbolTableNode + if n: + for i in range(1, len(parts)): + if isinstance(n.node, TypeInfo): + if n.node.mro is None: + # We haven't yet analyzed the class `n.node`. Fall back to direct + # lookup in the names declared directly under it, without its base + # classes. This can happen when we have a forward reference to a + # nested class, and the reference is bound before the outer class + # has been fully semantically analyzed. + # + # A better approach would be to introduce a new analysis pass or + # to move things around between passes, but this unblocks a common + # use case even though this is a little limited in case there is + # inheritance involved. + result = n.node.names.get(parts[i]) + else: + result = n.node.get(parts[i]) + n = result + elif isinstance(n.node, MypyFile): + n = n.node.names.get(parts[i], None) + # TODO: What if node is Var or FuncDef? + if not n: + self.name_not_defined(name, ctx) + break + if n: + n = self.normalize_type_alias(n, ctx) + return n + + def builtin_type(self, fully_qualified_name: str) -> Instance: + node = self.lookup_fully_qualified(fully_qualified_name) + info = cast(TypeInfo, node.node) + return Instance(info, []) + + def lookup_fully_qualified(self, name: str) -> SymbolTableNode: + """Lookup a fully qualified name. + + Assume that the name is defined. This happens in the global namespace -- the local + module namespace is ignored. + """ + assert '.' in name + parts = name.split('.') + n = self.modules[parts[0]] + for i in range(1, len(parts) - 1): + n = cast(MypyFile, n.names[parts[i]].node) + return n.names[parts[-1]] + + def lookup_fully_qualified_or_none(self, name: str) -> SymbolTableNode: + """Lookup a fully qualified name. + + Assume that the name is defined. This happens in the global namespace -- the local + module namespace is ignored. + """ + assert '.' in name + parts = name.split('.') + n = self.modules[parts[0]] + for i in range(1, len(parts) - 1): + next_sym = n.names.get(parts[i]) + if not next_sym: + return None + n = cast(MypyFile, next_sym.node) + return n.names.get(parts[-1]) + + def qualified_name(self, n: str) -> str: + return self.cur_mod_id + '.' + n + + def enter(self) -> None: + self.locals.append(SymbolTable()) + self.global_decls.append(set()) + self.nonlocal_decls.append(set()) + # -1 since entering block will increment this to 0. + self.block_depth.append(-1) + + def leave(self) -> None: + self.locals.pop() + self.global_decls.pop() + self.nonlocal_decls.pop() + self.block_depth.pop() + + def is_func_scope(self) -> bool: + return self.locals[-1] is not None + + def is_class_scope(self) -> bool: + return self.type is not None and not self.is_func_scope() + + def add_symbol(self, name: str, node: SymbolTableNode, + context: Context) -> None: + if self.is_func_scope(): + if name in self.locals[-1]: + # Flag redefinition unless this is a reimport of a module. + if not (node.kind == MODULE_REF and + self.locals[-1][name].node == node.node): + self.name_already_defined(name, context) + self.locals[-1][name] = node + elif self.type: + self.type.names[name] = node + else: + existing = self.globals.get(name) + if existing and (not isinstance(node.node, MypyFile) or + existing.node != node.node) and existing.kind != UNBOUND_IMPORTED: + # Modules can be imported multiple times to support import + # of multiple submodules of a package (e.g. a.x and a.y). + ok = False + # Only report an error if the symbol collision provides a different type. + if existing.type and node.type and is_same_type(existing.type, node.type): + ok = True + if not ok: + self.name_already_defined(name, context) + self.globals[name] = node + + def add_var(self, v: Var, ctx: Context) -> None: + if self.is_func_scope(): + self.add_local(v, ctx) + else: + self.globals[v.name()] = SymbolTableNode(GDEF, v, self.cur_mod_id) + v._fullname = self.qualified_name(v.name()) + + def add_local(self, node: Union[Var, FuncBase], ctx: Context) -> None: + name = node.name() + if name in self.locals[-1]: + self.name_already_defined(name, ctx) + node._fullname = name + self.locals[-1][name] = SymbolTableNode(LDEF, node) + + def add_exports(self, *exps: Node) -> None: + for exp in exps: + if isinstance(exp, StrExpr): + self.all_exports.add(exp.value) + + def check_no_global(self, n: str, ctx: Context, + is_overloaded_func: bool = False) -> None: + if n in self.globals: + prev_is_overloaded = isinstance(self.globals[n], OverloadedFuncDef) + if is_overloaded_func and prev_is_overloaded: + self.fail("Nonconsecutive overload {} found".format(n), ctx) + elif prev_is_overloaded: + self.fail("Definition of '{}' missing 'overload'".format(n), ctx) + else: + self.name_already_defined(n, ctx) + + def name_not_defined(self, name: str, ctx: Context) -> None: + message = "Name '{}' is not defined".format(name) + extra = self.undefined_name_extra_info(name) + if extra: + message += ' {}'.format(extra) + self.fail(message, ctx) + + def name_already_defined(self, name: str, ctx: Context) -> None: + self.fail("Name '{}' already defined".format(name), ctx) + + def fail(self, msg: str, ctx: Context, serious: bool = False, *, + blocker: bool = False) -> None: + if (not serious and + not self.options.check_untyped_defs and + self.function_stack and + self.function_stack[-1].is_dynamic()): + return + self.errors.report(ctx.get_line(), msg, blocker=blocker) + + def note(self, msg: str, ctx: Context) -> None: + if (not self.options.check_untyped_defs and + self.function_stack and + self.function_stack[-1].is_dynamic()): + return + self.errors.report(ctx.get_line(), msg, severity='note') + + def undefined_name_extra_info(self, fullname: str) -> Optional[str]: + if fullname in obsolete_name_mapping: + return "(it's now called '{}')".format(obsolete_name_mapping[fullname]) + else: + return None + + def accept(self, node: Node) -> None: + try: + node.accept(self) + except Exception as err: + report_internal_error(err, self.errors.file, node.line) + + +class FirstPass(NodeVisitor): + """First phase of semantic analysis. + + See docstring of 'analyze()' below for a description of what this does. + """ + + def __init__(self, sem: SemanticAnalyzer) -> None: + self.sem = sem + self.pyversion = sem.options.python_version + + def analyze(self, file: MypyFile, fnam: str, mod_id: str) -> None: + """Perform the first analysis pass. + + Populate module global table. Resolve the full names of + definitions not nested within functions and construct type + info structures, but do not resolve inter-definition + references such as base classes. + + Also add implicit definitions such as __name__. + + In this phase we don't resolve imports. For 'from ... import', + we generate dummy symbol table nodes for the imported names, + and these will get resolved in later phases of semantic + analysis. + """ + sem = self.sem + sem.cur_mod_id = mod_id + sem.errors.set_file(fnam) + sem.globals = SymbolTable() + sem.global_decls = [set()] + sem.nonlocal_decls = [set()] + sem.block_depth = [0] + + defs = file.defs + + # Add implicit definitions of module '__name__' etc. + for name, t in implicit_module_attrs.items(): + v = Var(name, UnboundType(t)) + v._fullname = self.sem.qualified_name(name) + self.sem.globals[name] = SymbolTableNode(GDEF, v, self.sem.cur_mod_id) + + for d in defs: + d.accept(self) + + # Add implicit definition of literals/keywords to builtins, as we + # cannot define a variable with them explicitly. + if mod_id == 'builtins': + literal_types = [ + ('None', NoneTyp()), + # reveal_type is a mypy-only function that gives an error with the type of its arg + ('reveal_type', AnyType()), + ] # type: List[Tuple[str, Type]] + + # TODO(ddfisher): This guard is only needed because mypy defines + # fake builtins for its tests which often don't define bool. If + # mypy is fast enough that we no longer need those, this + # conditional check should be removed. + if 'bool' in self.sem.globals: + bool_type = self.sem.named_type('bool') + literal_types.extend([ + ('True', bool_type), + ('False', bool_type), + ('__debug__', bool_type), + ]) + + for name, typ in literal_types: + v = Var(name, typ) + v._fullname = self.sem.qualified_name(name) + self.sem.globals[name] = SymbolTableNode(GDEF, v, self.sem.cur_mod_id) + + def visit_block(self, b: Block) -> None: + if b.is_unreachable: + return + self.sem.block_depth[-1] += 1 + for node in b.body: + node.accept(self) + self.sem.block_depth[-1] -= 1 + + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + for lval in s.lvalues: + self.analyze_lvalue(lval, explicit_type=s.type is not None) + + def visit_func_def(self, func: FuncDef) -> None: + sem = self.sem + func.is_conditional = sem.block_depth[-1] > 0 + func._fullname = sem.qualified_name(func.name()) + if func.name() in sem.globals: + # Already defined in this module. + original_sym = sem.globals[func.name()] + if original_sym.kind == UNBOUND_IMPORTED: + # Ah this is an imported name. We can't resolve them now, so we'll postpone + # this until the main phase of semantic analysis. + return + original_def = original_sym.node + if sem.is_conditional_func(original_def, func): + # Conditional function definition -- multiple defs are ok. + func.original_def = cast(FuncDef, original_def) + else: + # Report error. + sem.check_no_global(func.name(), func) + else: + sem.globals[func.name()] = SymbolTableNode(GDEF, func, sem.cur_mod_id) + + def visit_overloaded_func_def(self, func: OverloadedFuncDef) -> None: + self.sem.check_no_global(func.name(), func, True) + func._fullname = self.sem.qualified_name(func.name()) + self.sem.globals[func.name()] = SymbolTableNode(GDEF, func, + self.sem.cur_mod_id) + + def visit_class_def(self, cdef: ClassDef) -> None: + self.sem.check_no_global(cdef.name, cdef) + cdef.fullname = self.sem.qualified_name(cdef.name) + info = TypeInfo(SymbolTable(), cdef) + info.set_line(cdef.line) + cdef.info = info + self.sem.globals[cdef.name] = SymbolTableNode(GDEF, info, + self.sem.cur_mod_id) + self.process_nested_classes(cdef) + + def process_nested_classes(self, outer_def: ClassDef) -> None: + for node in outer_def.defs.body: + if isinstance(node, ClassDef): + node.info = TypeInfo(SymbolTable(), node) + if outer_def.fullname: + node.info._fullname = outer_def.fullname + '.' + node.info.name() + else: + node.info._fullname = node.info.name() + symbol = SymbolTableNode(MDEF, node.info) + outer_def.info.names[node.name] = symbol + self.process_nested_classes(node) + + def visit_import_from(self, node: ImportFrom) -> None: + # We can't bind module names during the first pass, as the target module might be + # unprocessed. However, we add dummy unbound imported names to the symbol table so + # that we at least know that the name refers to a module. + node.is_top_level = True + for name, as_name in node.names: + imported_name = as_name or name + if imported_name not in self.sem.globals: + self.sem.add_symbol(imported_name, SymbolTableNode(UNBOUND_IMPORTED, None), node) + + def visit_import(self, node: Import) -> None: + node.is_top_level = True + # This is similar to visit_import_from -- see the comment there. + for id, as_id in node.ids: + imported_id = as_id or id + if imported_id not in self.sem.globals: + self.sem.add_symbol(imported_id, SymbolTableNode(UNBOUND_IMPORTED, None), node) + else: + # If the previous symbol is a variable, this should take precedence. + self.sem.globals[imported_id] = SymbolTableNode(UNBOUND_IMPORTED, None) + + def visit_import_all(self, node: ImportAll) -> None: + node.is_top_level = True + + def visit_while_stmt(self, s: WhileStmt) -> None: + s.body.accept(self) + if s.else_body: + s.else_body.accept(self) + + def visit_for_stmt(self, s: ForStmt) -> None: + self.analyze_lvalue(s.index) + s.body.accept(self) + if s.else_body: + s.else_body.accept(self) + + def visit_with_stmt(self, s: WithStmt) -> None: + for n in s.target: + if n: + self.analyze_lvalue(n) + s.body.accept(self) + + def visit_decorator(self, d: Decorator) -> None: + d.var._fullname = self.sem.qualified_name(d.var.name()) + self.sem.add_symbol(d.var.name(), SymbolTableNode(GDEF, d.var), d) + + def visit_if_stmt(self, s: IfStmt) -> None: + infer_reachability_of_if_statement(s, pyversion=self.pyversion) + for node in s.body: + node.accept(self) + if s.else_body: + s.else_body.accept(self) + + def visit_try_stmt(self, s: TryStmt) -> None: + self.sem.analyze_try_stmt(s, self, add_global=True) + + def analyze_lvalue(self, lvalue: Node, explicit_type: bool = False) -> None: + self.sem.analyze_lvalue(lvalue, add_global=True, explicit_type=explicit_type) + + +class ThirdPass(TraverserVisitor): + """The third and final pass of semantic analysis. + + Check type argument counts and values of generic types, and perform some + straightforward type inference. + """ + + def __init__(self, modules: Dict[str, MypyFile], errors: Errors) -> None: + self.modules = modules + self.errors = errors + + def visit_file(self, file_node: MypyFile, fnam: str) -> None: + self.errors.set_file(fnam) + self.accept(file_node) + + def accept(self, node: Node) -> None: + try: + node.accept(self) + except Exception as err: + report_internal_error(err, self.errors.file, node.line) + + def visit_block(self, b: Block) -> None: + if b.is_unreachable: + return + super().visit_block(b) + + def visit_func_def(self, fdef: FuncDef) -> None: + self.errors.push_function(fdef.name()) + self.analyze(fdef.type) + super().visit_func_def(fdef) + self.errors.pop_function() + + def visit_class_def(self, tdef: ClassDef) -> None: + for type in tdef.info.bases: + self.analyze(type) + # Recompute MRO now that we have analyzed all modules, to pick + # up superclasses of bases imported from other modules in an + # import loop. (Only do so if we succeeded the first time.) + if tdef.info.mro: + tdef.info.mro = [] # Force recomputation + calculate_class_mro(tdef, self.fail) + super().visit_class_def(tdef) + + def visit_decorator(self, dec: Decorator) -> None: + """Try to infer the type of the decorated function. + + This lets us resolve references to decorated functions during + type checking when there are cyclic imports, as otherwise the + type might not be available when we need it. + + This basically uses a simple special-purpose type inference + engine just for decorators. + """ + super().visit_decorator(dec) + if dec.var.is_property: + # Decorators are expected to have a callable type (it's a little odd). + if dec.func.type is None: + dec.var.type = CallableType( + [AnyType()], + [ARG_POS], + [None], + AnyType(), + self.builtin_type('function'), + name=dec.var.name()) + elif isinstance(dec.func.type, CallableType): + dec.var.type = dec.func.type + return + decorator_preserves_type = True + for expr in dec.decorators: + preserve_type = False + if isinstance(expr, RefExpr) and isinstance(expr.node, FuncDef): + if is_identity_signature(expr.node.type): + preserve_type = True + if not preserve_type: + decorator_preserves_type = False + break + if decorator_preserves_type: + # No non-identity decorators left. We can trivially infer the type + # of the function here. + dec.var.type = function_type(dec.func, self.builtin_type('function')) + if dec.decorators: + if returns_any_if_called(dec.decorators[0]): + # The outermost decorator will return Any so we know the type of the + # decorated function. + dec.var.type = AnyType() + sig = find_fixed_callable_return(dec.decorators[0]) + if sig: + # The outermost decorator always returns the same kind of function, + # so we know that this is the type of the decoratored function. + orig_sig = function_type(dec.func, self.builtin_type('function')) + sig.name = orig_sig.items()[0].name + dec.var.type = sig + + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + self.analyze(s.type) + super().visit_assignment_stmt(s) + + def visit_cast_expr(self, e: CastExpr) -> None: + self.analyze(e.type) + super().visit_cast_expr(e) + + def visit_reveal_type_expr(self, e: RevealTypeExpr) -> None: + super().visit_reveal_type_expr(e) + + def visit_type_application(self, e: TypeApplication) -> None: + for type in e.types: + self.analyze(type) + super().visit_type_application(e) + + # Helpers + + def analyze(self, type: Type) -> None: + if type: + analyzer = TypeAnalyserPass3(self.fail) + type.accept(analyzer) + + def fail(self, msg: str, ctx: Context) -> None: + self.errors.report(ctx.get_line(), msg) + + def builtin_type(self, name: str, args: List[Type] = None) -> Instance: + names = self.modules['builtins'] + sym = names.names[name] + assert isinstance(sym.node, TypeInfo) + return Instance(sym.node, args or []) + + +def self_type(typ: TypeInfo) -> Union[Instance, TupleType]: + """For a non-generic type, return instance type representing the type. + For a generic G type with parameters T1, .., Tn, return G[T1, ..., Tn]. + """ + tv = [] # type: List[Type] + for i in range(len(typ.type_vars)): + tv.append(TypeVarType(typ.defn.type_vars[i])) + inst = Instance(typ, tv) + if typ.tuple_type is None: + return inst + return typ.tuple_type.self_type(inst) + + +def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: + if isinstance(sig, CallableType): + return replace_leading_arg_type(sig, new) + else: + sig = cast(Overloaded, sig) + return Overloaded([cast(CallableType, replace_implicit_first_type(i, new)) + for i in sig.items()]) + + +def set_callable_name(sig: Type, fdef: FuncDef) -> Type: + if isinstance(sig, FunctionLike): + if fdef.info: + return sig.with_name( + '"{}" of "{}"'.format(fdef.name(), fdef.info.name())) + else: + return sig.with_name('"{}"'.format(fdef.name())) + else: + return sig + + +def refers_to_fullname(node: Node, fullname: str) -> bool: + """Is node a name or member expression with the given full name?""" + return isinstance(node, RefExpr) and node.fullname == fullname + + +def refers_to_class_or_function(node: Node) -> bool: + """Does semantically analyzed node refer to a class?""" + return (isinstance(node, RefExpr) and + isinstance(node.node, (TypeInfo, FuncDef, OverloadedFuncDef))) + + +def calculate_class_mro(defn: ClassDef, fail: Callable[[str, Context], None]) -> None: + try: + defn.info.calculate_mro() + except MroError: + fail("Cannot determine consistent method resolution order " + '(MRO) for "%s"' % defn.name, defn) + defn.info.mro = [] + # The property of falling back to Any is inherited. + defn.info.fallback_to_any = any(baseinfo.fallback_to_any for baseinfo in defn.info.mro) + + +def find_duplicate(list: List[T]) -> T: + """If the list has duplicates, return one of the duplicates. + + Otherwise, return None. + """ + for i in range(1, len(list)): + if list[i] in list[:i]: + return list[i] + return None + + +def disable_typevars(nodes: List[SymbolTableNode]) -> None: + for node in nodes: + assert node.kind in (BOUND_TVAR, UNBOUND_TVAR) + node.kind = UNBOUND_TVAR + + +def enable_typevars(nodes: List[SymbolTableNode]) -> None: + for node in nodes: + assert node.kind in (BOUND_TVAR, UNBOUND_TVAR) + node.kind = BOUND_TVAR + + +def remove_imported_names_from_symtable(names: SymbolTable, + module: str) -> None: + """Remove all imported names from the symbol table of a module.""" + removed = [] # type: List[str] + for name, node in names.items(): + if node.node is None: + continue + fullname = node.node.fullname() + prefix = fullname[:fullname.rfind('.')] + if prefix != module: + removed.append(name) + for name in removed: + del names[name] + + +def infer_reachability_of_if_statement(s: IfStmt, + pyversion: Tuple[int, int]) -> None: + for i in range(len(s.expr)): + result = infer_if_condition_value(s.expr[i], pyversion) + if result == ALWAYS_FALSE: + # The condition is always false, so we skip the if/elif body. + mark_block_unreachable(s.body[i]) + elif result == ALWAYS_TRUE: + # This condition is always true, so all of the remaining + # elif/else bodies will never be executed. + for body in s.body[i + 1:]: + mark_block_unreachable(body) + if s.else_body: + mark_block_unreachable(s.else_body) + break + + +def infer_if_condition_value(expr: Node, pyversion: Tuple[int, int]) -> int: + """Infer whether if condition is always true/false. + + Return ALWAYS_TRUE if always true, ALWAYS_FALSE if always false, + and TRUTH_VALUE_UNKNOWN otherwise. + """ + name = '' + negated = False + alias = expr + if isinstance(alias, UnaryExpr): + if alias.op == 'not': + expr = alias.expr + negated = True + if isinstance(expr, NameExpr): + name = expr.name + elif isinstance(expr, MemberExpr): + name = expr.name + result = TRUTH_VALUE_UNKNOWN + if name == 'PY2': + result = ALWAYS_TRUE if pyversion[0] == 2 else ALWAYS_FALSE + elif name == 'PY3': + result = ALWAYS_TRUE if pyversion[0] == 3 else ALWAYS_FALSE + elif name == 'MYPY': + result = ALWAYS_TRUE + if negated: + if result == ALWAYS_TRUE: + result = ALWAYS_FALSE + elif result == ALWAYS_FALSE: + result = ALWAYS_TRUE + return result + + +def mark_block_unreachable(block: Block) -> None: + block.is_unreachable = True + block.accept(MarkImportsUnreachableVisitor()) + + +class MarkImportsUnreachableVisitor(TraverserVisitor): + """Visitor that flags all imports nested within a node as unreachable.""" + + def visit_import(self, node: Import) -> None: + node.is_unreachable = True + + def visit_import_from(self, node: ImportFrom) -> None: + node.is_unreachable = True + + def visit_import_all(self, node: ImportAll) -> None: + node.is_unreachable = True + + +def is_identity_signature(sig: Type) -> bool: + """Is type a callable of form T -> T (where T is a type variable)?""" + if isinstance(sig, CallableType) and sig.arg_kinds == [ARG_POS]: + if isinstance(sig.arg_types[0], TypeVarType) and isinstance(sig.ret_type, TypeVarType): + return sig.arg_types[0].id == sig.ret_type.id + return False + + +def returns_any_if_called(expr: Node) -> bool: + """Return True if we can predict that expr will return Any if called. + + This only uses information available during semantic analysis so this + will sometimes return False because of insufficient information (as + type inference hasn't run yet). + """ + if isinstance(expr, RefExpr): + if isinstance(expr.node, FuncDef): + typ = expr.node.type + if typ is None: + # No signature -> default to Any. + return True + # Explicit Any return? + return isinstance(typ, CallableType) and isinstance(typ.ret_type, AnyType) + elif isinstance(expr.node, Var): + typ = expr.node.type + return typ is None or isinstance(typ, AnyType) + elif isinstance(expr, CallExpr): + return returns_any_if_called(expr.callee) + return False + + +def find_fixed_callable_return(expr: Node) -> Optional[CallableType]: + if isinstance(expr, RefExpr): + if isinstance(expr.node, FuncDef): + typ = expr.node.type + if typ: + if isinstance(typ, CallableType) and has_no_typevars(typ.ret_type): + if isinstance(typ.ret_type, CallableType): + return typ.ret_type + elif isinstance(expr, CallExpr): + t = find_fixed_callable_return(expr.callee) + if t: + if isinstance(t.ret_type, CallableType): + return t.ret_type + return None + + +def has_no_typevars(typ: Type) -> bool: + return is_same_type(typ, erase_typevars(typ)) diff --git a/mypy/types.py b/mypy/types.py index bd31bf25d49f3..84a0eb12b607b 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -770,7 +770,7 @@ class NamedTupleType(TupleType): def __init__(self, attrs: List[str], *args) -> None: self.attrs = attrs super().__init__(*args) - + def self_type(self, inst: Instance) -> 'NamedTupleType': return NamedTupleType(self.attrs, self.items, inst) diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index bc37c40208198..9c2d6c4a86d8f 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -1,209 +1,209 @@ -[case testNamedTupleUsedAsTuple] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -a, b = x -b = x[0] -a = x[1] -a, b, c = x # E: Need more than 2 values to unpack (3 expected) -x[2] # E: Tuple index out of range - -[case testAccessingNamedTupleAttributes] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -x.x -x.y -x.z # E: "X" has no attribute "z" - -[case testCreateNamedTupleWithPositionalArguments] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = X(1, 'x') -x.x -x.z # E: "X" has no attribute "z" -x = X(1) # E: Too few arguments for "X" -x = X(1, 2, 3) # E: Too many arguments for "X" - -[case testCreateNamedTupleWithKeywordArguments] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = X(x=1, y='x') -x = X(1, y='x') -x = X(x=1, z=1) # E: Unexpected keyword argument "z" for "X" -x = X(y=1) # E: Missing positional argument "x" in call to "X" - -[case testCreateNamedTupleAndUseAsTuple] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = X(1, 'x') -a, b = x -a, b, c = x # E: Need more than 2 values to unpack (3 expected) - -[case testNamedTupleWithItemTypes] -from typing import NamedTuple -N = NamedTuple('N', [('a', int), - ('b', str)]) -n = N(1, 'x') -s = n.a # type: str # E: Incompatible types in assignment (expression has type "int", \ - variable has type "str") -i = n.b # type: int # E: Incompatible types in assignment (expression has type "str", \ - variable has type "int") -x, y = n -x = y # E: Incompatible types in assignment (expression has type "str", variable has type "int") - -[case testNamedTupleConstructorArgumentTypes] -from typing import NamedTuple -N = NamedTuple('N', [('a', int), - ('b', str)]) -n = N('x', 'x') # E: Argument 1 to "N" has incompatible type "str"; expected "int" -n = N(1, b=2) # E: Argument 2 to "N" has incompatible type "int"; expected "str" -N(1, 'x') -N(b='x', a=1) - -[case testNamedTupleAsBaseClass] -from typing import NamedTuple -N = NamedTuple('N', [('a', int), - ('b', str)]) -class X(N): - pass -x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" -s = '' -i = 0 -s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") -i, s = x -s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") - -[case testNamedTupleAsBaseClass2] -from typing import NamedTuple -class X(NamedTuple('N', [('a', int), - ('b', str)])): - pass -x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" -s = '' -i = 0 -s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") -i, s = x -s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") - -[case testTwoNamedTuplesAsBaseClasses] -from typing import NamedTuple -A = NamedTuple('A', [('a', int)]) -B = NamedTuple('B', [('a', int)]) -class X(A, B): # E: Class has two incompatible bases derived from tuple - pass - -[case testTwoNamedTuplesAsBaseClasses2] -from typing import NamedTuple -A = NamedTuple('A', [('a', int)]) -class X(A, NamedTuple('B', [('a', int)])): # E: Class has two incompatible bases derived from tuple - pass - -[case testSelfTypeWithNamedTupleAsBase] -from typing import NamedTuple -A = NamedTuple('A', [('a', int), ('b', str)]) -class B(A): - def f(self, x: int) -> None: - self.f(self.a) - self.f(self.b) # E: Argument 1 to "f" of "B" has incompatible type "str"; expected "int" - i = 0 - s = '' - i, s = self - i, i = self # E: Incompatible types in assignment (expression has type "str", \ - variable has type "int") -[out] -main: note: In member "f" of class "B": - -[case testTypeReferenceToClassDerivedFromNamedTuple] -from typing import NamedTuple -A = NamedTuple('A', [('a', int), ('b', str)]) -class B(A): - def f(self, x: 'B') -> None: - i = 0 - s = '' - self = x - i, s = x - i, s = x.a, x.b - i, s = x.a, x.a # E: Incompatible types in assignment (expression has type "int", \ - variable has type "str") - i, i = self # E: Incompatible types in assignment (expression has type "str", \ - variable has type "int") -[out] -main: note: In member "f" of class "B": - -[case testNamedTupleSubtyping] -from typing import NamedTuple, Tuple -A = NamedTuple('A', [('a', int), ('b', str)]) -class B(A): pass -a = A(1, '') -b = B(1, '') -t = None # type: Tuple[int, str] -b = a # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "A") -b = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "B") -t = a -t = (1, '') -t = b -a = b - -[case testNamedTupleSimpleTypeInference] -from typing import NamedTuple, Tuple -A = NamedTuple('A', [('a', int)]) -l = [A(1), A(2)] -a = A(1) -a = l[0] -(i,) = l[0] -i, i = l[0] # E: Need more than 1 value to unpack (2 expected) -l = [A(1)] -a = (1,) # E: Incompatible types in assignment (expression has type "Tuple[int]", \ - variable has type "A") -[builtins fixtures/list.py] - -[case testNamedTupleMissingClassAttribute] -import collections -MyNamedTuple = collections.namedtuple('MyNamedTuple', ['spam', 'eggs']) -MyNamedTuple.x # E: "MyNamedTuple" has no attribute "x" - -[case testNamedTupleEmptyItems] -from typing import NamedTuple -A = NamedTuple('A', []) -[builtins fixtures/list.py] - -[case testNamedTupleProperty] -from typing import NamedTuple -A = NamedTuple('A', [('a', int)]) -class B(A): - @property - def b(self) -> int: - return self.a -class C(B): pass -B(1).b -C(2).b -[builtins fixtures/property.py] - -[case testNamedTupleAsDict] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -a = x._asdict() - -[case testNamedTupleReplace] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -x = x._replace(y=5) - -[case testNamedTupleReplaceTyped] -from typing import NamedTuple - -X = NamedTuple('X', [('x', int), ('y', str)]) -x = None # type: X -x = x._replace(x=5) -x = x._replace(y=5) # E: Argument 1 to X has incompatible type "int"; expected "str" +[case testNamedTupleUsedAsTuple] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +a, b = x +b = x[0] +a = x[1] +a, b, c = x # E: Need more than 2 values to unpack (3 expected) +x[2] # E: Tuple index out of range + +[case testAccessingNamedTupleAttributes] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +x.x +x.y +x.z # E: "X" has no attribute "z" + +[case testCreateNamedTupleWithPositionalArguments] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = X(1, 'x') +x.x +x.z # E: "X" has no attribute "z" +x = X(1) # E: Too few arguments for "X" +x = X(1, 2, 3) # E: Too many arguments for "X" + +[case testCreateNamedTupleWithKeywordArguments] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = X(x=1, y='x') +x = X(1, y='x') +x = X(x=1, z=1) # E: Unexpected keyword argument "z" for "X" +x = X(y=1) # E: Missing positional argument "x" in call to "X" + +[case testCreateNamedTupleAndUseAsTuple] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = X(1, 'x') +a, b = x +a, b, c = x # E: Need more than 2 values to unpack (3 expected) + +[case testNamedTupleWithItemTypes] +from typing import NamedTuple +N = NamedTuple('N', [('a', int), + ('b', str)]) +n = N(1, 'x') +s = n.a # type: str # E: Incompatible types in assignment (expression has type "int", \ + variable has type "str") +i = n.b # type: int # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +x, y = n +x = y # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testNamedTupleConstructorArgumentTypes] +from typing import NamedTuple +N = NamedTuple('N', [('a', int), + ('b', str)]) +n = N('x', 'x') # E: Argument 1 to "N" has incompatible type "str"; expected "int" +n = N(1, b=2) # E: Argument 2 to "N" has incompatible type "int"; expected "str" +N(1, 'x') +N(b='x', a=1) + +[case testNamedTupleAsBaseClass] +from typing import NamedTuple +N = NamedTuple('N', [('a', int), + ('b', str)]) +class X(N): + pass +x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" +s = '' +i = 0 +s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") +i, s = x +s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testNamedTupleAsBaseClass2] +from typing import NamedTuple +class X(NamedTuple('N', [('a', int), + ('b', str)])): + pass +x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" +s = '' +i = 0 +s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") +i, s = x +s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testTwoNamedTuplesAsBaseClasses] +from typing import NamedTuple +A = NamedTuple('A', [('a', int)]) +B = NamedTuple('B', [('a', int)]) +class X(A, B): # E: Class has two incompatible bases derived from tuple + pass + +[case testTwoNamedTuplesAsBaseClasses2] +from typing import NamedTuple +A = NamedTuple('A', [('a', int)]) +class X(A, NamedTuple('B', [('a', int)])): # E: Class has two incompatible bases derived from tuple + pass + +[case testSelfTypeWithNamedTupleAsBase] +from typing import NamedTuple +A = NamedTuple('A', [('a', int), ('b', str)]) +class B(A): + def f(self, x: int) -> None: + self.f(self.a) + self.f(self.b) # E: Argument 1 to "f" of "B" has incompatible type "str"; expected "int" + i = 0 + s = '' + i, s = self + i, i = self # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +[out] +main: note: In member "f" of class "B": + +[case testTypeReferenceToClassDerivedFromNamedTuple] +from typing import NamedTuple +A = NamedTuple('A', [('a', int), ('b', str)]) +class B(A): + def f(self, x: 'B') -> None: + i = 0 + s = '' + self = x + i, s = x + i, s = x.a, x.b + i, s = x.a, x.a # E: Incompatible types in assignment (expression has type "int", \ + variable has type "str") + i, i = self # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +[out] +main: note: In member "f" of class "B": + +[case testNamedTupleSubtyping] +from typing import NamedTuple, Tuple +A = NamedTuple('A', [('a', int), ('b', str)]) +class B(A): pass +a = A(1, '') +b = B(1, '') +t = None # type: Tuple[int, str] +b = a # E: Incompatible types in assignment (expression has type "A", variable has type "B") +a = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "A") +b = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "B") +t = a +t = (1, '') +t = b +a = b + +[case testNamedTupleSimpleTypeInference] +from typing import NamedTuple, Tuple +A = NamedTuple('A', [('a', int)]) +l = [A(1), A(2)] +a = A(1) +a = l[0] +(i,) = l[0] +i, i = l[0] # E: Need more than 1 value to unpack (2 expected) +l = [A(1)] +a = (1,) # E: Incompatible types in assignment (expression has type "Tuple[int]", \ + variable has type "A") +[builtins fixtures/list.py] + +[case testNamedTupleMissingClassAttribute] +import collections +MyNamedTuple = collections.namedtuple('MyNamedTuple', ['spam', 'eggs']) +MyNamedTuple.x # E: "MyNamedTuple" has no attribute "x" + +[case testNamedTupleEmptyItems] +from typing import NamedTuple +A = NamedTuple('A', []) +[builtins fixtures/list.py] + +[case testNamedTupleProperty] +from typing import NamedTuple +A = NamedTuple('A', [('a', int)]) +class B(A): + @property + def b(self) -> int: + return self.a +class C(B): pass +B(1).b +C(2).b +[builtins fixtures/property.py] + +[case testNamedTupleAsDict] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +a = x._asdict() + +[case testNamedTupleReplace] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +x = x._replace(y=5) + +[case testNamedTupleReplaceTyped] +from typing import NamedTuple + +X = NamedTuple('X', [('x', int), ('y', str)]) +x = None # type: X +x = x._replace(x=5) +x = x._replace(y=5) # E: Argument 1 to X has incompatible type "int"; expected "str" From 044e640f5fcbc28090625b4bc2986b476e77676e Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Tue, 12 Jul 2016 14:16:32 +0300 Subject: [PATCH 16/74] Add partial support for calling '_replace' statically --- mypy/messages.py | 1 + mypy/nodes.py | 18 ++++++++---------- mypy/semanal.py | 20 +++++++++++--------- mypy/subtypes.py | 16 +++++++++++++--- mypy/typeanal.py | 14 +++++++++++++- mypy/types.py | 19 ++++++++++++++++--- test-data/unit/check-namedtuple.test | 10 ++++++++++ 7 files changed, 72 insertions(+), 26 deletions(-) diff --git a/mypy/messages.py b/mypy/messages.py index 209010764eca3..dd21dc4e2d7f7 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -486,6 +486,7 @@ def incompatible_argument(self, n: int, m: int, callee: CallableType, arg_type: except IndexError: # Varargs callees expected_type = callee.arg_types[-1] arg_type_str, expected_type_str = self.format_distinctly(arg_type, expected_type) + import pdb; pdb.set_trace() if arg_kind == ARG_STAR: arg_type_str = '*' + arg_type_str elif arg_kind == ARG_STAR2: diff --git a/mypy/nodes.py b/mypy/nodes.py index 9211d65485106..6e1d3bd353271 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1179,6 +1179,7 @@ def __init__(self, callee: Expression, args: List[Expression], arg_kinds: List[i arg_names: List[str] = None, analyzed: Expression = None) -> None: if not arg_names: arg_names = [None] * len(args) + self.callee = callee self.args = args self.arg_kinds = arg_kinds @@ -1976,16 +1977,13 @@ def deserialize(cls, data: JsonDict) -> 'TypeInfo': return ti -class NamedTupleTypeInfo(TypeInfo): - is_named_tuple = True - bases = None # type: List[mypy.types.Instance] - - def __init__(self, tup: 'mypy.types.NamedTupleType', *args) -> None: - super().__init__(*args) - self.tuple_type = tup - self.bases = [tup.fallback] - self.mro = [cast(TypeInfo, self)] + tup.fallback.type.mro - +def NamedTupleTypeInfo(tup: 'mypy.types.NamedTupleType', *args) -> None: + info = TypeInfo(*args) + info.tuple_type = tup + info.bases = [tup.fallback] + info.is_named_tuple = True + info.mro = [info] + tup.fallback.type.mro + return info class SymbolTableNode: # Kind of node. Possible values: diff --git a/mypy/semanal.py b/mypy/semanal.py index fed7475ae557a..8f77f908a41f8 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1527,23 +1527,25 @@ def fail_namedtuple_arg(self, message: str, def build_namedtuple_typeinfo(self, name: str, items: List[str], types: List[Type]) -> TypeInfo: symbols = SymbolTable() - tup = NamedTupleType(items, types, self.named_type('__builtins__.tuple', types)) + tup = NamedTupleType(name, items, types, self.named_type('__builtins__.tuple', types)) class_def = ClassDef(name, Block([])) class_def.fullname = self.qualified_name(name) info = NamedTupleTypeInfo(tup, symbols, class_def) + info.attrs = items vars = [Var(item, typ) for item, typ in zip(items, types)] # Add named tuple items as attributes. # TODO: Make them read-only. for var in vars: var.info = info symbols[var.name()] = SymbolTableNode(MDEF, var) + this_type = self_type(info) # Add __init__, _replace, _asdict methods and _fields static field - self.add_namedtuple_method('__init__', symbols, info, NoneTyp(), - self.make_factory_args(vars, ARG_POS)) - self.add_namedtuple_method('_replace', symbols, info, self_type(info), - self.make_factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) + self.add_namedtuple_method('_replace', symbols, info, this_type, this_type, + self.make_factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) + self.add_namedtuple_method('__init__', symbols, info, this_type, NoneTyp(), + self.make_factory_args(vars, ARG_NAMED)) # TODO: refine to OrderedDict[Union[types]] - self.add_namedtuple_method('_asdict', symbols, info, AnyType()) + self.add_namedtuple_method('_asdict', symbols, info, this_type, AnyType(), []) return info def make_factory_args(self, vars: List[Var], kind: int, @@ -1551,9 +1553,9 @@ def make_factory_args(self, vars: List[Var], kind: int, return [Argument(var, var.type, initializer, kind) for var in vars] def add_namedtuple_method(self, funcname: str, symbols: SymbolTable, info: TypeInfo, - ret: Type, args: List[Argument] = []) -> None: - types = [cast(Type, None)] + [arg.type_annotation for arg in args] - args = [Argument(Var('__self'), NoneTyp(), None, ARG_POS)] + args + this_type: Type, ret: Type, args: List[Argument]) -> None: + args = [Argument(Var('self'), this_type, None, ARG_POS)] + args + types = [arg.type_annotation for arg in args] items = [arg.variable.name() for arg in args] arg_kinds = [arg.kind for arg in args] # TODO: Make sure that the self argument name is not visible? diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 8e9017a7b355f..ee08cf452413e 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -2,7 +2,7 @@ from mypy.types import ( Type, AnyType, UnboundType, TypeVisitor, ErrorType, Void, NoneTyp, - Instance, TypeVarType, CallableType, TupleType, UnionType, Overloaded, ErasedType, TypeList, + Instance, TypeVarType, CallableType, TupleType, NamedTupleType, UnionType, Overloaded, ErasedType, TypeList, PartialType, DeletedType, UninhabitedType, TypeType, is_named_instance ) import mypy.applytype @@ -178,11 +178,13 @@ def visit_tuple_type(self, left: TupleType) -> bool: iter_type = right.args[0] return all(is_subtype(li, iter_type) for li in left.items) return False + elif isinstance(right, NamedTupleType) and not isinstance(left, NamedTupleType): + return False elif isinstance(right, TupleType): if len(left.items) != len(right.items): return False - for i in range(len(left.items)): - if not is_subtype(left.items[i], right.items[i], self.check_type_parameter): + for l, r in zip(left.items, right.items): + if not is_subtype(l, r, self.check_type_parameter): return False if not is_subtype(left.fallback, right.fallback, self.check_type_parameter): return False @@ -190,6 +192,14 @@ def visit_tuple_type(self, left: TupleType) -> bool: else: return False + def visit_namedtuple_type(self, left: NamedTupleType) -> bool: + if isinstance(self.right, NamedTupleType): + # left.attrs.issubset(right.attrs) is insufficient + # (a: int, b: str) is not subtype of (b: int, a: str) + if any(l != r for l, r in zip(left.attrs, self.right.attrs)): + return False + return self.visit_tuple_type(left) + def visit_overloaded(self, left: Overloaded) -> bool: right = self.right if isinstance(right, Instance): diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 0493d3a15f6b8..cce852a1be525 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -3,7 +3,7 @@ from typing import Callable, cast, List, Tuple from mypy.types import ( - Type, UnboundType, TypeVarType, TupleType, UnionType, Instance, AnyType, CallableType, + Type, UnboundType, TypeVarType, TupleType, NamedTupleType, UnionType, Instance, AnyType, CallableType, Void, NoneTyp, DeletedType, TypeList, TypeVarDef, TypeVisitor, StarType, PartialType, EllipsisType, UninhabitedType, TypeType ) @@ -169,6 +169,12 @@ def visit_unbound_type(self, t: UnboundType) -> Type: if t.args: self.fail('Generic tuple types not supported', t) return AnyType() + # import pdb; pdb.set_trace() + if info.is_named_tuple: + return NamedTupleType(info.name, info.attrs, + self.anal_array(info.tuple_type.items), + fallback=instance, + line=t.line) return TupleType(self.anal_array(info.tuple_type.items), fallback=instance, line=t.line) @@ -217,6 +223,12 @@ def visit_tuple_type(self, t: TupleType) -> Type: fallback = t.fallback if t.fallback else self.builtin_type('builtins.tuple', [AnyType()]) return TupleType(self.anal_array(t.items), fallback, t.line) + def visit_namedtuple_type(self, t: NamedTupleType) -> Type: + return NamedTupleType(t.name, t.attrs, + self.anal_array(t.items), + fallback=t.fallback, + line=t.line) + def visit_star_type(self, t: StarType) -> Type: return StarType(t.type.accept(self), t.line) diff --git a/mypy/types.py b/mypy/types.py index 84a0eb12b607b..61005b6c6c975 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -735,6 +735,7 @@ class TupleType(Type): def __init__(self, items: List[Type], fallback: Instance, line: int = -1, implicit: bool = False) -> None: + #import pdb; pdb.set_trace() self.items = items self.fallback = fallback self.implicit = implicit @@ -766,13 +767,21 @@ def self_type(self, inst: Instance) -> 'TupleType': class NamedTupleType(TupleType): attrs = None # type: List[str] + name = None # type: str - def __init__(self, attrs: List[str], *args) -> None: + def __init__(self, name: str, attrs: List[str], *args, **kwargs) -> None: self.attrs = attrs - super().__init__(*args) + self.name = name + super().__init__(*args, **kwargs) def self_type(self, inst: Instance) -> 'NamedTupleType': - return NamedTupleType(self.attrs, self.items, inst) + return NamedTupleType(self.name, self.attrs, self.items, inst) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + try: + return visitor.visit_namedtuple_type(self) + except: + return super().accept(visitor) class StarType(Type): @@ -1238,6 +1247,10 @@ def visit_tuple_type(self, t): return 'Tuple[{}, fallback={}]'.format(s, t.fallback.accept(self)) return 'Tuple[{}]'.format(s) + def visit_namedtuple_type(self, t): + s = self.list_str(zip(t.attrs, t.items)) + return '{}[{}]'.format(t.name, s) + def visit_star_type(self, t): s = t.type.accept(self) return '*{}'.format(s) diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 9c2d6c4a86d8f..3268209cc3f3b 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -199,6 +199,16 @@ from collections import namedtuple X = namedtuple('X', ['x', 'y']) x = None # type: X x = x._replace(y=5) +x = x._replace(z=5) # E: Unexpected keyword argument "z" for X +x = x._replace(5) # E: Too many positional arguments for X + +[case testNamedTupleReplaceAsClass] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +X._replace(x, x=1, y=2) +X._replace(x=1, y=2) # E: Missing positional argument "self" in call to X._replace [case testNamedTupleReplaceTyped] from typing import NamedTuple From eb6534e434f41be68989b16af8a47a3bbee5ea64 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Thu, 11 Aug 2016 22:55:22 +0300 Subject: [PATCH 17/74] fix method name --- mypy/checkexpr.py | 3 +-- mypy/expandtype.py | 2 +- mypy/messages.py | 2 +- mypy/nodes.py | 2 +- mypy/semanal.py | 6 +++--- mypy/subtypes.py | 1 + mypy/typeanal.py | 2 +- mypy/types.py | 15 +++++++++++++-- test-data/unit/check-namedtuple.test | 12 ++++++------ test-data/unit/check-tuples.test | 2 +- 10 files changed, 29 insertions(+), 18 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index e29a1b383d716..97a50142901eb 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1221,8 +1221,7 @@ def visit_tuple_slice_helper(self, left_type: TupleType, slic: SliceExpr): slic.stride) return AnyType() - return TupleType(left_type.items[begin:end:stride], left_type.fallback, - left_type.line, left_type.implicit) + return left_type.slice(begin, stride, end) def _get_value(self, index: Node) -> Optional[int]: if isinstance(index, IntExpr): diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 87b1641daecc1..ce6d5190fac6a 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -88,7 +88,7 @@ def visit_overloaded(self, t: Overloaded) -> Type: return Overloaded(items) def visit_tuple_type(self, t: TupleType) -> Type: - return TupleType(self.expand_types(t.items), t.fallback, t.line) + return t.with_types(self.expand_types(t.items)) def visit_union_type(self, t: UnionType) -> Type: # After substituting for type variables in t.items, diff --git a/mypy/messages.py b/mypy/messages.py index dd21dc4e2d7f7..698bc7a131b93 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -486,7 +486,7 @@ def incompatible_argument(self, n: int, m: int, callee: CallableType, arg_type: except IndexError: # Varargs callees expected_type = callee.arg_types[-1] arg_type_str, expected_type_str = self.format_distinctly(arg_type, expected_type) - import pdb; pdb.set_trace() + #import pdb; pdb.set_trace() if arg_kind == ARG_STAR: arg_type_str = '*' + arg_type_str elif arg_kind == ARG_STAR2: diff --git a/mypy/nodes.py b/mypy/nodes.py index 6e1d3bd353271..60d7b5d18944f 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1977,7 +1977,7 @@ def deserialize(cls, data: JsonDict) -> 'TypeInfo': return ti -def NamedTupleTypeInfo(tup: 'mypy.types.NamedTupleType', *args) -> None: +def NamedTupleTypeInfo(tup: 'mypy.types.NamedTupleType', *args) -> TypeInfo: info = TypeInfo(*args) info.tuple_type = tup info.bases = [tup.fallback] diff --git a/mypy/semanal.py b/mypy/semanal.py index 8f77f908a41f8..0f4331499d7cc 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1543,7 +1543,7 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], self.add_namedtuple_method('_replace', symbols, info, this_type, this_type, self.make_factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) self.add_namedtuple_method('__init__', symbols, info, this_type, NoneTyp(), - self.make_factory_args(vars, ARG_NAMED)) + self.make_factory_args(vars, ARG_POS), info.name()) # TODO: refine to OrderedDict[Union[types]] self.add_namedtuple_method('_asdict', symbols, info, this_type, AnyType(), []) return info @@ -1553,7 +1553,7 @@ def make_factory_args(self, vars: List[Var], kind: int, return [Argument(var, var.type, initializer, kind) for var in vars] def add_namedtuple_method(self, funcname: str, symbols: SymbolTable, info: TypeInfo, - this_type: Type, ret: Type, args: List[Argument]) -> None: + this_type: Type, ret: Type, args: List[Argument], name=None) -> None: args = [Argument(Var('self'), this_type, None, ARG_POS)] + args types = [arg.type_annotation for arg in args] items = [arg.variable.name() for arg in args] @@ -1561,7 +1561,7 @@ def add_namedtuple_method(self, funcname: str, symbols: SymbolTable, info: TypeI # TODO: Make sure that the self argument name is not visible? signature = CallableType(types, arg_kinds, items, ret, self.named_type('__builtins__.function'), - name=info.name()) + name=name or info.name() + '.' + funcname) func = FuncDef(funcname, args, Block([]), diff --git a/mypy/subtypes.py b/mypy/subtypes.py index ee08cf452413e..2205d014261d6 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -193,6 +193,7 @@ def visit_tuple_type(self, left: TupleType) -> bool: return False def visit_namedtuple_type(self, left: NamedTupleType) -> bool: + #import pdb; pdb.set_trace() if isinstance(self.right, NamedTupleType): # left.attrs.issubset(right.attrs) is insufficient # (a: int, b: str) is not subtype of (b: int, a: str) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index cce852a1be525..1841f283a19e1 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -171,7 +171,7 @@ def visit_unbound_type(self, t: UnboundType) -> Type: return AnyType() # import pdb; pdb.set_trace() if info.is_named_tuple: - return NamedTupleType(info.name, info.attrs, + return NamedTupleType(info.name(), info.attrs, self.anal_array(info.tuple_type.items), fallback=instance, line=t.line) diff --git a/mypy/types.py b/mypy/types.py index 61005b6c6c975..ae08876e8bca0 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -762,7 +762,14 @@ def deserialize(cls, data: JsonDict) -> 'TupleType': implicit=data['implicit']) def self_type(self, inst: Instance) -> 'TupleType': - return TupleType(self.items, inst) + return TupleType(self.items, inst, self.line, self.implicit) + + def with_types(self, items: List[Type]) -> 'TupleType': + return TupleType(items, self.fallback, self.line, self.implicit) + + def slice(self, begin: int, stride: int, end: int) -> 'TupleType': + return TupleType(self.items[begin:end:stride], self.fallback, + self.line, self.implicit) class NamedTupleType(TupleType): @@ -775,7 +782,11 @@ def __init__(self, name: str, attrs: List[str], *args, **kwargs) -> None: super().__init__(*args, **kwargs) def self_type(self, inst: Instance) -> 'NamedTupleType': - return NamedTupleType(self.name, self.attrs, self.items, inst) + return NamedTupleType(self.name, self.attrs, self.items, inst, self.line) + + def with_types(self, items: List[Type]) -> 'NamedTupleType': + return NamedTupleType(self.name, self.attrs, items, + self.fallback, self.line, self.implicit) def accept(self, visitor: 'TypeVisitor[T]') -> T: try: diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 3268209cc3f3b..a14cdd9ab3c77 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -143,9 +143,9 @@ class B(A): pass a = A(1, '') b = B(1, '') t = None # type: Tuple[int, str] -b = a # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "A") -b = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "B") +b = a # E: Incompatible types in assignment (expression has type "A", variable has type "B") +a = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "A") +b = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "B") t = a t = (1, '') t = b @@ -199,8 +199,8 @@ from collections import namedtuple X = namedtuple('X', ['x', 'y']) x = None # type: X x = x._replace(y=5) -x = x._replace(z=5) # E: Unexpected keyword argument "z" for X -x = x._replace(5) # E: Too many positional arguments for X +x = x._replace(z=5) # E: Unexpected keyword argument "z" for X._replace +x = x._replace(5) # E: Too many positional arguments for X._replace [case testNamedTupleReplaceAsClass] from collections import namedtuple @@ -216,4 +216,4 @@ from typing import NamedTuple X = NamedTuple('X', [('x', int), ('y', str)]) x = None # type: X x = x._replace(x=5) -x = x._replace(y=5) # E: Argument 1 to X has incompatible type "int"; expected "str" +x = x._replace(y=5) # E: Argument 1 to X._replace has incompatible type "int"; expected "str" diff --git a/test-data/unit/check-tuples.test b/test-data/unit/check-tuples.test index 7c03014fd5c81..50d1b3e64598d 100644 --- a/test-data/unit/check-tuples.test +++ b/test-data/unit/check-tuples.test @@ -2,7 +2,7 @@ -- ------------------------------- -[case testAssignmentWithTupleTypes] +[case testTupleAssignmentWithTupleTypes] from typing import Tuple t1 = None # type: Tuple[A] t2 = None # type: Tuple[B] From aa0d5dafc34e576cc4843c994dea9a150ef685e3 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Sun, 14 Aug 2016 18:11:37 +0300 Subject: [PATCH 18/74] namedtuple tests pass. list has problems --- mypy/join.py | 9 +++++++-- mypy/messages.py | 1 + mypy/subtypes.py | 3 --- mypy/types.py | 21 +++++++++++++++++++++ test-data/unit/check-lists.test | 2 ++ test-data/unit/check-namedtuple.test | 6 +++++- 6 files changed, 36 insertions(+), 6 deletions(-) diff --git a/mypy/join.py b/mypy/join.py index c28216b5de69d..684d62cfeaebd 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -1,12 +1,12 @@ """Calculation of the least upper bound types (joins).""" -from typing import cast, List +from typing import List from mypy.types import ( Type, AnyType, NoneTyp, Void, TypeVisitor, Instance, UnboundType, ErrorType, TypeVarType, CallableType, TupleType, ErasedType, TypeList, UnionType, FunctionLike, Overloaded, PartialType, DeletedType, - UninhabitedType, TypeType + UninhabitedType, TypeType, NamedTupleType ) from mypy.maptype import map_instance_to_supertype from mypy.subtypes import is_subtype, is_equivalent, is_subtype_ignoring_tvars @@ -210,10 +210,15 @@ def visit_overloaded(self, t: Overloaded) -> Type: return join_types(t.fallback, s) def visit_tuple_type(self, t: TupleType) -> Type: + if self.s == t: + return self.s if isinstance(self.s, TupleType) and self.s.length() == t.length(): items = [] # type: List[Type] for i in range(t.length()): items.append(self.join(t.items[i], self.s.items[i])) + if (not (isinstance(t, NamedTupleType) and isinstance(self.s, NamedTupleType)) + or self.s.attrs == t.attrs): + return self.s.with_types(items) # TODO: What if the fallback types are different? return TupleType(items, t.fallback) else: diff --git a/mypy/messages.py b/mypy/messages.py index 698bc7a131b93..5d82c4845782a 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -146,6 +146,7 @@ def report(self, msg: str, context: Context, severity: str, file: str = None) -> def fail(self, msg: str, context: Context, file: str = None) -> None: """Report an error message (unless disabled).""" + # import pdb; pdb.set_trace() self.report(msg, context, 'error', file=file) def note(self, msg: str, context: Context, file: str = None) -> None: diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 2205d014261d6..6872e297c3a64 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -178,8 +178,6 @@ def visit_tuple_type(self, left: TupleType) -> bool: iter_type = right.args[0] return all(is_subtype(li, iter_type) for li in left.items) return False - elif isinstance(right, NamedTupleType) and not isinstance(left, NamedTupleType): - return False elif isinstance(right, TupleType): if len(left.items) != len(right.items): return False @@ -193,7 +191,6 @@ def visit_tuple_type(self, left: TupleType) -> bool: return False def visit_namedtuple_type(self, left: NamedTupleType) -> bool: - #import pdb; pdb.set_trace() if isinstance(self.right, NamedTupleType): # left.attrs.issubset(right.attrs) is insufficient # (a: int, b: str) is not subtype of (b: int, a: str) diff --git a/mypy/types.py b/mypy/types.py index ae08876e8bca0..7d8dcb1c1a603 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -788,6 +788,27 @@ def with_types(self, items: List[Type]) -> 'NamedTupleType': return NamedTupleType(self.name, self.attrs, items, self.fallback, self.line, self.implicit) + def __eq__(self, other) -> bool: + return self.serialize() == other.serialize() + + def serialize(self) -> JsonDict: + d = super().serialize() + d.update({'.class': 'NamedTupleType', + 'name': self.name, + 'attrs': self.attrs}) + return d + + @classmethod + def deserialize(cls, data: JsonDict) -> 'NamedTupleType': + assert data['.class'] == 'NamedTupleType' + data['.class'] = 'TupleType' + tup = TupleType.deserialize(data) + return NamedTupleType(data['name'], + data['attrs'], + tup.items, + tup.fallback, + implicit=tup.implicit) + def accept(self, visitor: 'TypeVisitor[T]') -> T: try: return visitor.visit_namedtuple_type(self) diff --git a/test-data/unit/check-lists.test b/test-data/unit/check-lists.test index 3bafa99245d18..032e02314f5c4 100644 --- a/test-data/unit/check-lists.test +++ b/test-data/unit/check-lists.test @@ -1,5 +1,7 @@ -- Nested list assignment -- ----------------------------- +[case testListSimplest] +l = [1, 1] [case testNestedListAssignment] diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index a14cdd9ab3c77..adb07ec522705 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -1,3 +1,8 @@ +[case testNamedTupleDeclaration] +from typing import NamedTuple + +A = NamedTuple('A', [('x', str), ('y', int)]) + [case testNamedTupleUsedAsTuple] from collections import namedtuple @@ -159,7 +164,6 @@ a = A(1) a = l[0] (i,) = l[0] i, i = l[0] # E: Need more than 1 value to unpack (2 expected) -l = [A(1)] a = (1,) # E: Incompatible types in assignment (expression has type "Tuple[int]", \ variable has type "A") [builtins fixtures/list.py] From e12bd0100aeb3ec2e8e1cb85b773eb7cd8e92495 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Sun, 14 Aug 2016 18:50:35 +0300 Subject: [PATCH 19/74] tests pass. OrderedDict unsupported. cleanup still required --- mypy/build.py | 3356 +++++++-------- mypy/checker.py | 4918 +++++++++++----------- mypy/checkexpr.py | 3644 ++++++++--------- mypy/expandtype.py | 224 +- mypy/join.py | 734 ++-- mypy/meet.py | 570 +-- mypy/messages.py | 1840 ++++----- mypy/nodes.py | 4484 ++++++++++---------- mypy/parse.py | 4068 +++++++++--------- mypy/semanal.py | 5678 +++++++++++++------------- mypy/subtypes.py | 818 ++-- mypy/typeanal.py | 860 ++-- mypy/types.py | 2902 ++++++------- test-data/unit/check-lists.test | 129 +- test-data/unit/check-namedtuple.test | 446 +- test-data/unit/check-tuples.test | 1540 +++---- 16 files changed, 18104 insertions(+), 18107 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index e8a84c97c7c76..a930614989f90 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -1,1678 +1,1678 @@ -"""Facilities to analyze entire programs, including imported modules. - -Parse and analyze the source files of a program in the correct order -(based on file dependencies), and collect the results. - -This module only directs a build, which is performed in multiple passes per -file. The individual passes are implemented in separate modules. - -The function build() is the main interface to this module. -""" -# TODO: More consistent terminology, e.g. path/fnam, module/id, state/file - -import binascii -import collections -import contextlib -import json -import os -import os.path -import sys -import time -from os.path import dirname, basename - -from typing import (AbstractSet, Dict, Iterable, Iterator, List, - NamedTuple, Optional, Set, Tuple, Union, Mapping) - -from mypy.types import Type -from mypy.nodes import (MypyFile, Node, Import, ImportFrom, ImportAll, - SymbolTableNode, MODULE_REF) -from mypy.semanal import FirstPass, SemanticAnalyzer, ThirdPass -from mypy.checker import TypeChecker -from mypy.errors import Errors, CompileError, DecodeError, report_internal_error -from mypy import fixup -from mypy.report import Reports -from mypy import defaults -from mypy import moduleinfo -from mypy import util -from mypy.fixup import fixup_module_pass_one, fixup_module_pass_two -from mypy.options import Options -from mypy.parse import parse -from mypy.stats import dump_type_stats - - -# We need to know the location of this file to load data, but -# until Python 3.4, __file__ is relative. -__file__ = os.path.realpath(__file__) - -PYTHON_EXTENSIONS = ['.pyi', '.py'] - - -class BuildResult: - """The result of a successful build. - - Attributes: - manager: The build manager. - files: Dictionary from module name to related AST node. - types: Dictionary from parse tree node to its inferred type. - errors: List of error messages. - """ - - def __init__(self, manager: 'BuildManager') -> None: - self.manager = manager - self.files = manager.modules - self.types = manager.type_checker.type_map - self.errors = manager.errors.messages() - - -class BuildSource: - def __init__(self, path: Optional[str], module: Optional[str], - text: Optional[str]) -> None: - self.path = path - self.module = module or '__main__' - self.text = text - - @property - def effective_path(self) -> str: - """Return the effective path (ie, if its from in memory)""" - return self.path or '' - - -class BuildSourceSet: - """Efficiently test a file's membership in the set of build sources.""" - - def __init__(self, sources: List[BuildSource]) -> None: - self.source_text_present = False - self.source_modules = set() # type: Set[str] - self.source_paths = set() # type: Set[str] - - for source in sources: - if source.text is not None: - self.source_text_present = True - elif source.path: - self.source_paths.add(source.path) - else: - self.source_modules.add(source.module) - - def is_source(self, file: MypyFile) -> bool: - if file.path and file.path in self.source_paths: - return True - elif file._fullname in self.source_modules: - return True - elif file.path is None and self.source_text_present: - return True - else: - return False - - -def build(sources: List[BuildSource], - options: Options, - alt_lib_path: str = None, - bin_dir: str = None) -> BuildResult: - """Analyze a program. - - A single call to build performs parsing, semantic analysis and optionally - type checking for the program *and* all imported modules, recursively. - - Return BuildResult if successful or only non-blocking errors were found; - otherwise raise CompileError. - - Args: - sources: list of sources to build - options: build options - alt_lib_path: an additional directory for looking up library modules - (takes precedence over other directories) - bin_dir: directory containing the mypy script, used for finding data - directories; if omitted, use '.' as the data directory - """ - - data_dir = default_data_dir(bin_dir) - - find_module_clear_caches() - - # Determine the default module search path. - lib_path = default_lib_path(data_dir, options.python_version) - - if options.use_builtins_fixtures: - # Use stub builtins (to speed up test cases and to make them easier to - # debug). This is a test-only feature, so assume our files are laid out - # as in the source tree. - root_dir = os.path.dirname(os.path.dirname(__file__)) - lib_path.insert(0, os.path.join(root_dir, 'test-data', 'unit', 'lib-stub')) - else: - for source in sources: - if source.path: - # Include directory of the program file in the module search path. - dir = remove_cwd_prefix_from_path(dirname(source.path)) - if dir not in lib_path: - lib_path.insert(0, dir) - - # Do this even if running as a file, for sanity (mainly because with - # multiple builds, there could be a mix of files/modules, so its easier - # to just define the semantics that we always add the current director - # to the lib_path - lib_path.insert(0, os.getcwd()) - - # Add MYPYPATH environment variable to front of library path, if defined. - lib_path[:0] = mypy_path() - - # If provided, insert the caller-supplied extra module path to the - # beginning (highest priority) of the search path. - if alt_lib_path: - lib_path.insert(0, alt_lib_path) - - reports = Reports(data_dir, options.report_dirs) - - source_set = BuildSourceSet(sources) - - # Construct a build manager object to hold state during the build. - # - # Ignore current directory prefix in error messages. - manager = BuildManager(data_dir, lib_path, - ignore_prefix=os.getcwd(), - source_set=source_set, - reports=reports, - options=options) - - try: - dispatch(sources, manager) - return BuildResult(manager) - finally: - manager.log("Build finished with %d modules, %d types, and %d errors" % - (len(manager.modules), - len(manager.type_checker.type_map), - manager.errors.num_messages())) - # Finish the HTML or XML reports even if CompileError was raised. - reports.finish() - - -def default_data_dir(bin_dir: Optional[str]) -> str: - """Returns directory containing typeshed directory - - Args: - bin_dir: directory containing the mypy script - """ - if not bin_dir: - mypy_package = os.path.dirname(__file__) - parent = os.path.dirname(mypy_package) - if (os.path.basename(parent) == 'site-packages' or - os.path.basename(parent) == 'dist-packages'): - # Installed in site-packages or dist-packages, but invoked with python3 -m mypy; - # __file__ is .../blah/lib/python3.N/site-packages/mypy/build.py - # or .../blah/lib/python3.N/dist-packages/mypy/build.py (Debian) - # or .../blah/lib/site-packages/mypy/build.py (Windows) - # blah may be a virtualenv or /usr/local. We want .../blah/lib/mypy. - lib = parent - for i in range(2): - lib = os.path.dirname(lib) - if os.path.basename(lib) == 'lib': - return os.path.join(lib, 'mypy') - subdir = os.path.join(parent, 'lib', 'mypy') - if os.path.isdir(subdir): - # If installed via buildout, the __file__ is - # somewhere/mypy/__init__.py and what we want is - # somewhere/lib/mypy. - return subdir - # Default to directory containing this file's parent. - return parent - base = os.path.basename(bin_dir) - dir = os.path.dirname(bin_dir) - if (sys.platform == 'win32' and base.lower() == 'scripts' - and not os.path.isdir(os.path.join(dir, 'typeshed'))): - # Installed, on Windows. - return os.path.join(dir, 'Lib', 'mypy') - elif base == 'scripts': - # Assume that we have a repo check out or unpacked source tarball. - return dir - elif base == 'bin': - # Installed to somewhere (can be under /usr/local or anywhere). - return os.path.join(dir, 'lib', 'mypy') - elif base == 'python3': - # Assume we installed python3 with brew on os x - return os.path.join(os.path.dirname(dir), 'lib', 'mypy') - elif dir.endswith('python-exec'): - # Gentoo uses a python wrapper in /usr/lib to which mypy is a symlink. - return os.path.join(os.path.dirname(dir), 'mypy') - else: - # Don't know where to find the data files! - raise RuntimeError("Broken installation: can't determine base dir") - - -def mypy_path() -> List[str]: - path_env = os.getenv('MYPYPATH') - if not path_env: - return [] - return path_env.split(os.pathsep) - - -def default_lib_path(data_dir: str, pyversion: Tuple[int, int]) -> List[str]: - """Return default standard library search paths.""" - # IDEA: Make this more portable. - path = [] # type: List[str] - - auto = os.path.join(data_dir, 'stubs-auto') - if os.path.isdir(auto): - data_dir = auto - - # We allow a module for e.g. version 3.5 to be in 3.4/. The assumption - # is that a module added with 3.4 will still be present in Python 3.5. - versions = ["%d.%d" % (pyversion[0], minor) - for minor in reversed(range(pyversion[1] + 1))] - # E.g. for Python 3.2, try 3.2/, 3.1/, 3.0/, 3/, 2and3/. - # (Note that 3.1 and 3.0 aren't really supported, but we don't care.) - for v in versions + [str(pyversion[0]), '2and3']: - for lib_type in ['stdlib', 'third_party']: - stubdir = os.path.join(data_dir, 'typeshed', lib_type, v) - if os.path.isdir(stubdir): - path.append(stubdir) - - # Add fallback path that can be used if we have a broken installation. - if sys.platform != 'win32': - path.append('/usr/local/lib/mypy') - - return path - - -CacheMeta = NamedTuple('CacheMeta', - [('id', str), - ('path', str), - ('mtime', float), - ('size', int), - ('dependencies', List[str]), # names of imported modules - ('data_mtime', float), # mtime of data_json - ('data_json', str), # path of .data.json - ('suppressed', List[str]), # dependencies that weren't imported - ('options', Optional[Dict[str, bool]]), # build options - ('dep_prios', List[int]), - ]) -# NOTE: dependencies + suppressed == all reachable imports; -# suppressed contains those reachable imports that were prevented by -# --silent-imports or simply not found. - - -# Priorities used for imports. (Here, top-level includes inside a class.) -# These are used to determine a more predictable order in which the -# nodes in an import cycle are processed. -PRI_HIGH = 5 # top-level "from X import blah" -PRI_MED = 10 # top-level "import X" -PRI_LOW = 20 # either form inside a function -PRI_ALL = 99 # include all priorities - - -class BuildManager: - """This class holds shared state for building a mypy program. - - It is used to coordinate parsing, import processing, semantic - analysis and type checking. The actual build steps are carried - out by dispatch(). - - Attributes: - data_dir: Mypy data directory (contains stubs) - lib_path: Library path for looking up modules - modules: Mapping of module ID to MypyFile (shared by the passes) - semantic_analyzer: - Semantic analyzer, pass 2 - semantic_analyzer_pass3: - Semantic analyzer, pass 3 - type_checker: Type checker - errors: Used for reporting all errors - options: Build options - missing_modules: Set of modules that could not be imported encountered so far - """ - - def __init__(self, data_dir: str, - lib_path: List[str], - ignore_prefix: str, - source_set: BuildSourceSet, - reports: Reports, - options: Options) -> None: - self.start_time = time.time() - self.data_dir = data_dir - self.errors = Errors() - self.errors.set_ignore_prefix(ignore_prefix) - self.lib_path = tuple(lib_path) - self.source_set = source_set - self.reports = reports - self.options = options - self.semantic_analyzer = SemanticAnalyzer(lib_path, self.errors, options=options) - self.modules = self.semantic_analyzer.modules - self.semantic_analyzer_pass3 = ThirdPass(self.modules, self.errors) - self.type_checker = TypeChecker(self.errors, self.modules, options=options) - self.missing_modules = set() # type: Set[str] - - def all_imported_modules_in_file(self, - file: MypyFile) -> List[Tuple[int, str, int]]: - """Find all reachable import statements in a file. - - Return list of tuples (priority, module id, import line number) - for all modules imported in file; lower numbers == higher priority. - """ - - def correct_rel_imp(imp: Union[ImportFrom, ImportAll]) -> str: - """Function to correct for relative imports.""" - file_id = file.fullname() - rel = imp.relative - if rel == 0: - return imp.id - if os.path.basename(file.path).startswith('__init__.'): - rel -= 1 - if rel != 0: - file_id = ".".join(file_id.split(".")[:-rel]) - new_id = file_id + "." + imp.id if imp.id else file_id - - return new_id - - res = [] # type: List[Tuple[int, str, int]] - for imp in file.imports: - if not imp.is_unreachable: - if isinstance(imp, Import): - pri = PRI_MED if imp.is_top_level else PRI_LOW - for id, _ in imp.ids: - res.append((pri, id, imp.line)) - elif isinstance(imp, ImportFrom): - cur_id = correct_rel_imp(imp) - pos = len(res) - all_are_submodules = True - # Also add any imported names that are submodules. - pri = PRI_MED if imp.is_top_level else PRI_LOW - for name, __ in imp.names: - sub_id = cur_id + '.' + name - if self.is_module(sub_id): - res.append((pri, sub_id, imp.line)) - else: - all_are_submodules = False - # If all imported names are submodules, don't add - # cur_id as a dependency. Otherwise (i.e., if at - # least one imported name isn't a submodule) - # cur_id is also a dependency, and we should - # insert it *before* any submodules. - if not all_are_submodules: - pri = PRI_HIGH if imp.is_top_level else PRI_LOW - res.insert(pos, ((pri, cur_id, imp.line))) - elif isinstance(imp, ImportAll): - pri = PRI_HIGH if imp.is_top_level else PRI_LOW - res.append((pri, correct_rel_imp(imp), imp.line)) - - return res - - def is_module(self, id: str) -> bool: - """Is there a file in the file system corresponding to module id?""" - return find_module(id, self.lib_path) is not None - - def parse_file(self, id: str, path: str, source: str) -> MypyFile: - """Parse the source of a file with the given name. - - Raise CompileError if there is a parse error. - """ - num_errs = self.errors.num_messages() - tree = parse(source, path, self.errors, options=self.options) - tree._fullname = id - - # We don't want to warn about 'type: ignore' comments on - # imports, but we're about to modify tree.imports, so grab - # these first. - import_lines = set(node.line for node in tree.imports) - - # Skip imports that have been ignored (so that we can ignore a C extension module without - # stub, for example), except for 'from x import *', because we wouldn't be able to - # determine which names should be defined unless we process the module. We can still - # ignore errors such as redefinitions when using the latter form. - imports = [node for node in tree.imports - if node.line not in tree.ignored_lines or isinstance(node, ImportAll)] - tree.imports = imports - - if self.errors.num_messages() != num_errs: - self.log("Bailing due to parse errors") - self.errors.raise_error() - - self.errors.set_file_ignored_lines(path, tree.ignored_lines) - self.errors.mark_file_ignored_lines_used(path, import_lines) - return tree - - def module_not_found(self, path: str, line: int, id: str) -> None: - self.errors.set_file(path) - stub_msg = "(Stub files are from https://github.com/python/typeshed)" - if ((self.options.python_version[0] == 2 and moduleinfo.is_py2_std_lib_module(id)) or - (self.options.python_version[0] >= 3 and moduleinfo.is_py3_std_lib_module(id))): - self.errors.report( - line, "No library stub file for standard library module '{}'".format(id)) - self.errors.report(line, stub_msg, severity='note', only_once=True) - elif moduleinfo.is_third_party_module(id): - self.errors.report(line, "No library stub file for module '{}'".format(id)) - self.errors.report(line, stub_msg, severity='note', only_once=True) - else: - self.errors.report(line, "Cannot find module named '{}'".format(id)) - self.errors.report(line, '(Perhaps setting MYPYPATH ' - 'or using the "--silent-imports" flag would help)', - severity='note', only_once=True) - - def report_file(self, file: MypyFile) -> None: - if self.source_set.is_source(file): - self.reports.file(file, type_map=self.type_checker.type_map) - - def log(self, *message: str) -> None: - if self.options.verbosity >= 1: - print('%.3f:LOG: ' % (time.time() - self.start_time), *message, file=sys.stderr) - sys.stderr.flush() - - def trace(self, *message: str) -> None: - if self.options.verbosity >= 2: - print('%.3f:TRACE:' % (time.time() - self.start_time), *message, file=sys.stderr) - sys.stderr.flush() - - -def remove_cwd_prefix_from_path(p: str) -> str: - """Remove current working directory prefix from p, if present. - - Also crawl up until a directory without __init__.py is found. - - If the result would be empty, return '.' instead. - """ - cur = os.getcwd() - # Add separator to the end of the path, unless one is already present. - if basename(cur) != '': - cur += os.sep - # Compute root path. - while (p and - (os.path.isfile(os.path.join(p, '__init__.py')) or - os.path.isfile(os.path.join(p, '__init__.pyi')))): - dir, base = os.path.split(p) - if not base: - break - p = dir - # Remove current directory prefix from the path, if present. - if p.startswith(cur): - p = p[len(cur):] - # Avoid returning an empty path; replace that with '.'. - if p == '': - p = '.' - return p - - -# Cache find_module: (id, lib_path) -> result. -find_module_cache = {} # type: Dict[Tuple[str, Tuple[str, ...]], str] - -# Cache some repeated work within distinct find_module calls: finding which -# elements of lib_path have even the subdirectory they'd need for the module -# to exist. This is shared among different module ids when they differ only -# in the last component. -find_module_dir_cache = {} # type: Dict[Tuple[str, Tuple[str, ...]], List[str]] - -# Cache directory listings. We assume that while one os.listdir() -# call may be more expensive than one os.stat() call, a small number -# of os.stat() calls is quickly more expensive than caching the -# os.listdir() outcome, and the advantage of the latter is that it -# gives us the case-correct filename on Windows and Mac. -find_module_listdir_cache = {} # type: Dict[str, Optional[List[str]]] - - -def find_module_clear_caches(): - find_module_cache.clear() - find_module_dir_cache.clear() - find_module_listdir_cache.clear() - - -def list_dir(path: str) -> Optional[List[str]]: - """Return a cached directory listing. - - Returns None if the path doesn't exist or isn't a directory. - """ - if path in find_module_listdir_cache: - return find_module_listdir_cache[path] - try: - res = os.listdir(path) - except OSError: - res = None - find_module_listdir_cache[path] = res - return res - - -def is_file(path: str) -> bool: - """Return whether path exists and is a file. - - On case-insensitive filesystems (like Mac or Windows) this returns - False if the case of the path's last component does not exactly - match the case found in the filesystem. - """ - head, tail = os.path.split(path) - if not tail: - return False - names = list_dir(head) - if not names: - return False - if tail not in names: - return False - return os.path.isfile(path) - - -def find_module(id: str, lib_path: Iterable[str]) -> str: - """Return the path of the module source file, or None if not found.""" - if not isinstance(lib_path, tuple): - lib_path = tuple(lib_path) - - def find(): - # If we're looking for a module like 'foo.bar.baz', it's likely that most of the - # many elements of lib_path don't even have a subdirectory 'foo/bar'. Discover - # that only once and cache it for when we look for modules like 'foo.bar.blah' - # that will require the same subdirectory. - components = id.split('.') - dir_chain = os.sep.join(components[:-1]) # e.g., 'foo/bar' - if (dir_chain, lib_path) not in find_module_dir_cache: - dirs = [] - for pathitem in lib_path: - # e.g., '/usr/lib/python3.4/foo/bar' - dir = os.path.normpath(os.path.join(pathitem, dir_chain)) - if os.path.isdir(dir): - dirs.append(dir) - find_module_dir_cache[dir_chain, lib_path] = dirs - candidate_base_dirs = find_module_dir_cache[dir_chain, lib_path] - - # If we're looking for a module like 'foo.bar.baz', then candidate_base_dirs now - # contains just the subdirectories 'foo/bar' that actually exist under the - # elements of lib_path. This is probably much shorter than lib_path itself. - # Now just look for 'baz.pyi', 'baz/__init__.py', etc., inside those directories. - seplast = os.sep + components[-1] # so e.g. '/baz' - sepinit = os.sep + '__init__' - for base_dir in candidate_base_dirs: - base_path = base_dir + seplast # so e.g. '/usr/lib/python3.4/foo/bar/baz' - # Prefer package over module, i.e. baz/__init__.py* over baz.py*. - for extension in PYTHON_EXTENSIONS: - path = base_path + sepinit + extension - if is_file(path) and verify_module(id, path): - return path - # No package, look for module. - for extension in PYTHON_EXTENSIONS: - path = base_path + extension - if is_file(path) and verify_module(id, path): - return path - return None - - key = (id, lib_path) - if key not in find_module_cache: - find_module_cache[key] = find() - return find_module_cache[key] - - -def find_modules_recursive(module: str, lib_path: List[str]) -> List[BuildSource]: - module_path = find_module(module, lib_path) - if not module_path: - return [] - result = [BuildSource(module_path, module, None)] - if module_path.endswith(('__init__.py', '__init__.pyi')): - # Subtle: this code prefers the .pyi over the .py if both - # exists, and also prefers packages over modules if both x/ - # and x.py* exist. How? We sort the directory items, so x - # comes before x.py and x.pyi. But the preference for .pyi - # over .py is encoded in find_module(); even though we see - # x.py before x.pyi, find_module() will find x.pyi first. We - # use hits to avoid adding it a second time when we see x.pyi. - # This also avoids both x.py and x.pyi when x/ was seen first. - hits = set() # type: Set[str] - for item in sorted(os.listdir(os.path.dirname(module_path))): - abs_path = os.path.join(os.path.dirname(module_path), item) - if os.path.isdir(abs_path) and \ - (os.path.isfile(os.path.join(abs_path, '__init__.py')) or - os.path.isfile(os.path.join(abs_path, '__init__.pyi'))): - hits.add(item) - result += find_modules_recursive(module + '.' + item, lib_path) - elif item != '__init__.py' and item != '__init__.pyi' and \ - item.endswith(('.py', '.pyi')): - mod = item.split('.')[0] - if mod not in hits: - hits.add(mod) - result += find_modules_recursive( - module + '.' + mod, lib_path) - return result - - -def verify_module(id: str, path: str) -> bool: - """Check that all packages containing id have a __init__ file.""" - if path.endswith(('__init__.py', '__init__.pyi')): - path = dirname(path) - for i in range(id.count('.')): - path = dirname(path) - if not any(os.path.isfile(os.path.join(path, '__init__{}'.format(extension))) - for extension in PYTHON_EXTENSIONS): - return False - return True - - -def read_with_python_encoding(path: str, pyversion: Tuple[int, int]) -> str: - """Read the Python file with while obeying PEP-263 encoding detection""" - source_bytearray = bytearray() - encoding = 'utf8' if pyversion[0] >= 3 else 'ascii' - - with open(path, 'rb') as f: - # read first two lines and check if PEP-263 coding is present - source_bytearray.extend(f.readline()) - source_bytearray.extend(f.readline()) - - # check for BOM UTF-8 encoding and strip it out if present - if source_bytearray.startswith(b'\xef\xbb\xbf'): - encoding = 'utf8' - source_bytearray = source_bytearray[3:] - else: - _encoding, _ = util.find_python_encoding(source_bytearray, pyversion) - # check that the coding isn't mypy. We skip it since - # registering may not have happened yet - if _encoding != 'mypy': - encoding = _encoding - - source_bytearray.extend(f.read()) - try: - source_bytearray.decode(encoding) - except LookupError as lookuperr: - raise DecodeError(str(lookuperr)) - return source_bytearray.decode(encoding) - - -MYPY_CACHE = '.mypy_cache' - - -def get_cache_names(id: str, path: str, pyversion: Tuple[int, int]) -> Tuple[str, str]: - """Return the file names for the cache files. - - Args: - id: module ID - path: module path (used to recognize packages) - pyversion: Python version (major, minor) - - Returns: - A tuple with the file names to be used for the meta JSON and the - data JSON, respectively. - """ - prefix = os.path.join(MYPY_CACHE, '%d.%d' % pyversion, *id.split('.')) - is_package = os.path.basename(path).startswith('__init__.py') - if is_package: - prefix = os.path.join(prefix, '__init__') - return (prefix + '.meta.json', prefix + '.data.json') - - -def find_cache_meta(id: str, path: str, manager: BuildManager) -> Optional[CacheMeta]: - """Find cache data for a module. - - Args: - id: module ID - path: module path - manager: the build manager (for pyversion, log/trace, and build options) - - Returns: - A CacheMeta instance if the cache data was found and appears - valid; otherwise None. - """ - # TODO: May need to take more build options into account - meta_json, data_json = get_cache_names(id, path, manager.options.python_version) - manager.trace('Looking for {} {}'.format(id, data_json)) - if not os.path.exists(meta_json): - return None - with open(meta_json, 'r') as f: - meta_str = f.read() - manager.trace('Meta {} {}'.format(id, meta_str.rstrip())) - meta = json.loads(meta_str) # TODO: Errors - if not isinstance(meta, dict): - return None - path = os.path.abspath(path) - m = CacheMeta( - meta.get('id'), - meta.get('path'), - meta.get('mtime'), - meta.get('size'), - meta.get('dependencies', []), - meta.get('data_mtime'), - data_json, - meta.get('suppressed', []), - meta.get('options'), - meta.get('dep_prios', []), - ) - if (m.id != id or m.path != path or - m.mtime is None or m.size is None or - m.dependencies is None or m.data_mtime is None): - return None - - # Ignore cache if generated by an older mypy version. - if m.options is None or len(m.dependencies) != len(m.dep_prios): - return None - - # Ignore cache if (relevant) options aren't the same. - cached_options = m.options - current_options = select_options_affecting_cache(manager.options) - if cached_options != current_options: - return None - - # TODO: Share stat() outcome with find_module() - st = os.stat(path) # TODO: Errors - if st.st_mtime != m.mtime or st.st_size != m.size: - manager.log('Metadata abandoned because of modified file {}'.format(path)) - return None - # It's a match on (id, path, mtime, size). - # Check data_json; assume if its mtime matches it's good. - # TODO: stat() errors - if os.path.getmtime(data_json) != m.data_mtime: - return None - manager.log('Found {} {}'.format(id, meta_json)) - return m - - -def select_options_affecting_cache(options: Options) -> Mapping[str, bool]: - return {opt: getattr(options, opt) for opt in OPTIONS_AFFECTING_CACHE} - - -OPTIONS_AFFECTING_CACHE = [ - "silent_imports", - "almost_silent", - "disallow_untyped_calls", - "disallow_untyped_defs", - "check_untyped_defs", -] - - -def random_string(): - return binascii.hexlify(os.urandom(8)).decode('ascii') - - -def write_cache(id: str, path: str, tree: MypyFile, - dependencies: List[str], suppressed: List[str], - dep_prios: List[int], - manager: BuildManager) -> None: - """Write cache files for a module. - - Args: - id: module ID - path: module path - tree: the fully checked module data - dependencies: module IDs on which this module depends - suppressed: module IDs which were suppressed as dependencies - dep_prios: priorities (parallel array to dependencies) - manager: the build manager (for pyversion, log/trace) - """ - path = os.path.abspath(path) - manager.trace('Dumping {} {}'.format(id, path)) - st = os.stat(path) # TODO: Errors - mtime = st.st_mtime - size = st.st_size - meta_json, data_json = get_cache_names(id, path, manager.options.python_version) - manager.log('Writing {} {} {}'.format(id, meta_json, data_json)) - data = tree.serialize() - parent = os.path.dirname(data_json) - if not os.path.isdir(parent): - os.makedirs(parent) - assert os.path.dirname(meta_json) == parent - nonce = '.' + random_string() - data_json_tmp = data_json + nonce - meta_json_tmp = meta_json + nonce - with open(data_json_tmp, 'w') as f: - json.dump(data, f, indent=2, sort_keys=True) - f.write('\n') - data_mtime = os.path.getmtime(data_json_tmp) - meta = {'id': id, - 'path': path, - 'mtime': mtime, - 'size': size, - 'data_mtime': data_mtime, - 'dependencies': dependencies, - 'suppressed': suppressed, - 'options': select_options_affecting_cache(manager.options), - 'dep_prios': dep_prios, - } - with open(meta_json_tmp, 'w') as f: - json.dump(meta, f, sort_keys=True) - f.write('\n') - os.replace(data_json_tmp, data_json) - os.replace(meta_json_tmp, meta_json) - - -"""Dependency manager. - -Design -====== - -Ideally -------- - -A. Collapse cycles (each SCC -- strongly connected component -- - becomes one "supernode"). - -B. Topologically sort nodes based on dependencies. - -C. Process from leaves towards roots. - -Wrinkles --------- - -a. Need to parse source modules to determine dependencies. - -b. Processing order for modules within an SCC. - -c. Must order mtimes of files to decide whether to re-process; depends - on clock never resetting. - -d. from P import M; checks filesystem whether module P.M exists in - filesystem. - -e. Race conditions, where somebody modifies a file while we're - processing. I propose not to modify the algorithm to handle this, - but to detect when this could lead to inconsistencies. (For - example, when we decide on the dependencies based on cache - metadata, and then we decide to re-parse a file because of a stale - dependency, if the re-parsing leads to a different list of - dependencies we should warn the user or start over.) - -Steps ------ - -1. For each explicitly given module find the source file location. - -2. For each such module load and check the cache metadata, and decide - whether it's valid. - -3. Now recursively (or iteratively) find dependencies and add those to - the graph: - - - for cached nodes use the list of dependencies from the cache - metadata (this will be valid even if we later end up re-parsing - the same source); - - - for uncached nodes parse the file and process all imports found, - taking care of (a) above. - -Step 3 should also address (d) above. - -Once step 3 terminates we have the entire dependency graph, and for -each module we've either loaded the cache metadata or parsed the -source code. (However, we may still need to parse those modules for -which we have cache metadata but that depend, directly or indirectly, -on at least one module for which the cache metadata is stale.) - -Now we can execute steps A-C from the first section. Finding SCCs for -step A shouldn't be hard; there's a recipe here: -http://code.activestate.com/recipes/578507/. There's also a plethora -of topsort recipes, e.g. http://code.activestate.com/recipes/577413/. - -For single nodes, processing is simple. If the node was cached, we -deserialize the cache data and fix up cross-references. Otherwise, we -do semantic analysis followed by type checking. We also handle (c) -above; if a module has valid cache data *but* any of its -dependencies was processed from source, then the module should be -processed from source. - -A relatively simple optimization (outside SCCs) we might do in the -future is as follows: if a node's cache data is valid, but one or more -of its dependencies are out of date so we have to re-parse the node -from source, once we have fully type-checked the node, we can decide -whether its symbol table actually changed compared to the cache data -(by reading the cache data and comparing it to the data we would be -writing). If there is no change we can declare the node up to date, -and any node that depends (and for which we have cached data, and -whose other dependencies are up to date) on it won't need to be -re-parsed from source. - -Import cycles -------------- - -Finally we have to decide how to handle (c), import cycles. Here -we'll need a modified version of the original state machine -(build.py), but we only need to do this per SCC, and we won't have to -deal with changes to the list of nodes while we're processing it. - -If all nodes in the SCC have valid cache metadata and all dependencies -outside the SCC are still valid, we can proceed as follows: - - 1. Load cache data for all nodes in the SCC. - - 2. Fix up cross-references for all nodes in the SCC. - -Otherwise, the simplest (but potentially slow) way to proceed is to -invalidate all cache data in the SCC and re-parse all nodes in the SCC -from source. We can do this as follows: - - 1. Parse source for all nodes in the SCC. - - 2. Semantic analysis for all nodes in the SCC. - - 3. Type check all nodes in the SCC. - -(If there are more passes the process is the same -- each pass should -be done for all nodes before starting the next pass for any nodes in -the SCC.) - -We could process the nodes in the SCC in any order. For sentimental -reasons, I've decided to process them in the reverse order in which we -encountered them when originally constructing the graph. That's how -the old build.py deals with cycles, and at least this reproduces the -previous implementation more accurately. - -Can we do better than re-parsing all nodes in the SCC when any of its -dependencies are out of date? It's doubtful. The optimization -mentioned at the end of the previous section would require re-parsing -and type-checking a node and then comparing its symbol table to the -cached data; but because the node is part of a cycle we can't -technically type-check it until the semantic analysis of all other -nodes in the cycle has completed. (This is an important issue because -Dropbox has a very large cycle in production code. But I'd like to -deal with it later.) - -Additional wrinkles -------------------- - -During implementation more wrinkles were found. - -- When a submodule of a package (e.g. x.y) is encountered, the parent - package (e.g. x) must also be loaded, but it is not strictly a - dependency. See State.add_ancestors() below. -""" - - -class ModuleNotFound(Exception): - """Control flow exception to signal that a module was not found.""" - - -class State: - """The state for a module. - - The source is only used for the -c command line option; in that - case path is None. Otherwise source is None and path isn't. - """ - - manager = None # type: BuildManager - order_counter = 0 # Class variable - order = None # type: int # Order in which modules were encountered - id = None # type: str # Fully qualified module name - path = None # type: Optional[str] # Path to module source - xpath = None # type: str # Path or '' - source = None # type: Optional[str] # Module source code - meta = None # type: Optional[CacheMeta] - data = None # type: Optional[str] - tree = None # type: Optional[MypyFile] - dependencies = None # type: List[str] - suppressed = None # type: List[str] # Suppressed/missing dependencies - priorities = None # type: Dict[str, int] - - # Map each dependency to the line number where it is first imported - dep_line_map = None # type: Dict[str, int] - - # Parent package, its parent, etc. - ancestors = None # type: Optional[List[str]] - - # List of (path, line number) tuples giving context for import - import_context = None # type: List[Tuple[str, int]] - - # The State from which this module was imported, if any - caller_state = None # type: Optional[State] - - # If caller_state is set, the line number in the caller where the import occurred - caller_line = 0 - - def __init__(self, - id: Optional[str], - path: Optional[str], - source: Optional[str], - manager: BuildManager, - caller_state: 'State' = None, - caller_line: int = 0, - ancestor_for: 'State' = None, - ) -> None: - assert id or path or source is not None, "Neither id, path nor source given" - self.manager = manager - State.order_counter += 1 - self.order = State.order_counter - self.caller_state = caller_state - self.caller_line = caller_line - if caller_state: - self.import_context = caller_state.import_context[:] - self.import_context.append((caller_state.xpath, caller_line)) - else: - self.import_context = [] - self.id = id or '__main__' - if not path and source is None: - file_id = id - if id == 'builtins' and manager.options.python_version[0] == 2: - # The __builtin__ module is called internally by mypy - # 'builtins' in Python 2 mode (similar to Python 3), - # but the stub file is __builtin__.pyi. The reason is - # that a lot of code hard-codes 'builtins.x' and it's - # easier to work it around like this. It also means - # that the implementation can mostly ignore the - # difference and just assume 'builtins' everywhere, - # which simplifies code. - file_id = '__builtin__' - path = find_module(file_id, manager.lib_path) - if path: - # In silent mode, don't import .py files, except from stubs. - if (manager.options.silent_imports and - path.endswith('.py') and (caller_state or ancestor_for)): - # (Never silence builtins, even if it's a .py file; - # this can happen in tests!) - if (id != 'builtins' and - not ((caller_state and - caller_state.tree and - caller_state.tree.is_stub))): - if manager.options.almost_silent: - if ancestor_for: - self.skipping_ancestor(id, path, ancestor_for) - else: - self.skipping_module(id, path) - path = None - manager.missing_modules.add(id) - raise ModuleNotFound - else: - # Could not find a module. Typically the reason is a - # misspelled module name, missing stub, module not in - # search path or the module has not been installed. - if caller_state: - suppress_message = ((manager.options.silent_imports and - not manager.options.almost_silent) or - (caller_state.tree is not None and - 'import' in caller_state.tree.weak_opts)) - if not suppress_message: - save_import_context = manager.errors.import_context() - manager.errors.set_import_context(caller_state.import_context) - manager.module_not_found(caller_state.xpath, caller_line, id) - manager.errors.set_import_context(save_import_context) - manager.missing_modules.add(id) - raise ModuleNotFound - else: - # If we can't find a root source it's always fatal. - # TODO: This might hide non-fatal errors from - # root sources processed earlier. - raise CompileError(["mypy: can't find module '%s'" % id]) - self.path = path - self.xpath = path or '' - self.source = source - if path and source is None and manager.options.incremental: - self.meta = find_cache_meta(self.id, self.path, manager) - # TODO: Get mtime if not cached. - self.add_ancestors() - if self.meta: - # Make copies, since we may modify these and want to - # compare them to the originals later. - self.dependencies = list(self.meta.dependencies) - self.suppressed = list(self.meta.suppressed) - assert len(self.meta.dependencies) == len(self.meta.dep_prios) - self.priorities = {id: pri - for id, pri in zip(self.meta.dependencies, self.meta.dep_prios)} - self.dep_line_map = {} - else: - # Parse the file (and then some) to get the dependencies. - self.parse_file() - self.suppressed = [] - - def skipping_ancestor(self, id: str, path: str, ancestor_for: 'State') -> None: - # TODO: Read the path (the __init__.py file) and return - # immediately if it's empty or only contains comments. - # But beware, some package may be the ancestor of many modules, - # so we'd need to cache the decision. - manager = self.manager - manager.errors.set_import_context([]) - manager.errors.set_file(ancestor_for.xpath) - manager.errors.report(-1, "Ancestor package '%s' silently ignored" % (id,), - severity='note', only_once=True) - manager.errors.report(-1, "(Using --silent-imports, submodule passed on command line)", - severity='note', only_once=True) - manager.errors.report(-1, "(This note brought to you by --almost-silent)", - severity='note', only_once=True) - - def skipping_module(self, id: str, path: str) -> None: - assert self.caller_state, (id, path) - manager = self.manager - save_import_context = manager.errors.import_context() - manager.errors.set_import_context(self.caller_state.import_context) - manager.errors.set_file(self.caller_state.xpath) - line = self.caller_line - manager.errors.report(line, "Import of '%s' silently ignored" % (id,), - severity='note') - manager.errors.report(line, "(Using --silent-imports, module not passed on command line)", - severity='note', only_once=True) - manager.errors.report(line, "(This note courtesy of --almost-silent)", - severity='note', only_once=True) - manager.errors.set_import_context(save_import_context) - - def add_ancestors(self) -> None: - # All parent packages are new ancestors. - ancestors = [] - parent = self.id - while '.' in parent: - parent, _ = parent.rsplit('.', 1) - ancestors.append(parent) - self.ancestors = ancestors - - def is_fresh(self) -> bool: - """Return whether the cache data for this file is fresh.""" - # NOTE: self.dependencies may differ from - # self.meta.dependencies when a dependency is dropped due to - # suppression by --silent-imports. However when a suppressed - # dependency is added back we find out later in the process. - return self.meta is not None and self.dependencies == self.meta.dependencies - - def mark_stale(self) -> None: - """Throw away the cache data for this file, marking it as stale.""" - self.meta = None - - def check_blockers(self) -> None: - """Raise CompileError if a blocking error is detected.""" - if self.manager.errors.is_blockers(): - self.manager.log("Bailing due to blocking errors") - self.manager.errors.raise_error() - - @contextlib.contextmanager - def wrap_context(self) -> Iterator[None]: - save_import_context = self.manager.errors.import_context() - self.manager.errors.set_import_context(self.import_context) - try: - yield - except CompileError: - raise - except Exception as err: - report_internal_error(err, self.path, 0) - self.manager.errors.set_import_context(save_import_context) - self.check_blockers() - - # Methods for processing cached modules. - - def load_tree(self) -> None: - with open(self.meta.data_json) as f: - data = json.load(f) - # TODO: Assert data file wasn't changed. - self.tree = MypyFile.deserialize(data) - self.manager.modules[self.id] = self.tree - - def fix_cross_refs(self) -> None: - fixup_module_pass_one(self.tree, self.manager.modules) - - def calculate_mros(self) -> None: - fixup_module_pass_two(self.tree, self.manager.modules) - - # Methods for processing modules from source code. - - def parse_file(self) -> None: - if self.tree is not None: - # The file was already parsed (in __init__()). - return - - manager = self.manager - modules = manager.modules - manager.log("Parsing %s (%s)" % (self.xpath, self.id)) - - with self.wrap_context(): - source = self.source - self.source = None # We won't need it again. - if self.path and source is None: - try: - source = read_with_python_encoding(self.path, manager.options.python_version) - except IOError as ioerr: - raise CompileError([ - "mypy: can't read file '{}': {}".format(self.path, ioerr.strerror)]) - except (UnicodeDecodeError, DecodeError) as decodeerr: - raise CompileError([ - "mypy: can't decode file '{}': {}".format(self.path, str(decodeerr))]) - self.tree = manager.parse_file(self.id, self.xpath, source) - - modules[self.id] = self.tree - - # Do the first pass of semantic analysis: add top-level - # definitions in the file to the symbol table. We must do - # this before processing imports, since this may mark some - # import statements as unreachable. - first = FirstPass(manager.semantic_analyzer) - first.analyze(self.tree, self.xpath, self.id) - - # Initialize module symbol table, which was populated by the - # semantic analyzer. - # TODO: Why can't FirstPass .analyze() do this? - self.tree.names = manager.semantic_analyzer.globals - - # Compute (direct) dependencies. - # Add all direct imports (this is why we needed the first pass). - # Also keep track of each dependency's source line. - dependencies = [] - suppressed = [] - priorities = {} # type: Dict[str, int] # id -> priority - dep_line_map = {} # type: Dict[str, int] # id -> line - for pri, id, line in manager.all_imported_modules_in_file(self.tree): - priorities[id] = min(pri, priorities.get(id, PRI_ALL)) - if id == self.id: - continue - # Omit missing modules, as otherwise we could not type-check - # programs with missing modules. - if id in manager.missing_modules: - if id not in dep_line_map: - suppressed.append(id) - dep_line_map[id] = line - continue - if id == '': - # Must be from a relative import. - manager.errors.set_file(self.xpath) - manager.errors.report(line, "No parent module -- cannot perform relative import", - blocker=True) - continue - if id not in dep_line_map: - dependencies.append(id) - dep_line_map[id] = line - # Every module implicitly depends on builtins. - if self.id != 'builtins' and 'builtins' not in dep_line_map: - dependencies.append('builtins') - - # If self.dependencies is already set, it was read from the - # cache, but for some reason we're re-parsing the file. - # NOTE: What to do about race conditions (like editing the - # file while mypy runs)? A previous version of this code - # explicitly checked for this, but ran afoul of other reasons - # for differences (e.g. --silent-imports). - self.dependencies = dependencies - self.suppressed = suppressed - self.priorities = priorities - self.dep_line_map = dep_line_map - self.check_blockers() - - def patch_parent(self) -> None: - # Include module in the symbol table of the enclosing package. - if '.' not in self.id: - return - manager = self.manager - modules = manager.modules - parent, child = self.id.rsplit('.', 1) - if parent in modules: - manager.trace("Added %s.%s" % (parent, child)) - modules[parent].names[child] = SymbolTableNode(MODULE_REF, self.tree, parent) - else: - manager.log("Hm... couldn't add %s.%s" % (parent, child)) - - def semantic_analysis(self) -> None: - with self.wrap_context(): - self.manager.semantic_analyzer.visit_file(self.tree, self.xpath) - - def semantic_analysis_pass_three(self) -> None: - with self.wrap_context(): - self.manager.semantic_analyzer_pass3.visit_file(self.tree, self.xpath) - if self.manager.options.dump_type_stats: - dump_type_stats(self.tree, self.xpath) - - def type_check(self) -> None: - manager = self.manager - if manager.options.semantic_analysis_only: - return - with self.wrap_context(): - manager.type_checker.visit_file(self.tree, self.xpath) - if manager.options.dump_inference_stats: - dump_type_stats(self.tree, self.xpath, inferred=True, - typemap=manager.type_checker.type_map) - manager.report_file(self.tree) - - def write_cache(self) -> None: - if self.path and self.manager.options.incremental and not self.manager.errors.is_errors(): - dep_prios = [self.priorities.get(dep, PRI_HIGH) for dep in self.dependencies] - write_cache(self.id, self.path, self.tree, - list(self.dependencies), list(self.suppressed), - dep_prios, - self.manager) - - -Graph = Dict[str, State] - - -def dispatch(sources: List[BuildSource], manager: BuildManager) -> None: - manager.log("Using new dependency manager") - graph = load_graph(sources, manager) - manager.log("Loaded graph with %d nodes" % len(graph)) - process_graph(graph, manager) - if manager.options.warn_unused_ignores: - manager.errors.generate_unused_ignore_notes() - - -def load_graph(sources: List[BuildSource], manager: BuildManager) -> Graph: - """Given some source files, load the full dependency graph.""" - graph = {} # type: Graph - # The deque is used to implement breadth-first traversal. - # TODO: Consider whether to go depth-first instead. This may - # affect the order in which we process files within import cycles. - new = collections.deque() # type: collections.deque[State] - # Seed the graph with the initial root sources. - for bs in sources: - try: - st = State(id=bs.module, path=bs.path, source=bs.text, manager=manager) - except ModuleNotFound: - continue - if st.id in graph: - manager.errors.set_file(st.xpath) - manager.errors.report(1, "Duplicate module named '%s'" % st.id) - manager.errors.raise_error() - graph[st.id] = st - new.append(st) - # Collect dependencies. We go breadth-first. - while new: - st = new.popleft() - for dep in st.ancestors + st.dependencies: - if dep not in graph: - try: - if dep in st.ancestors: - # TODO: Why not 'if dep not in st.dependencies' ? - # Ancestors don't have import context. - newst = State(id=dep, path=None, source=None, manager=manager, - ancestor_for=st) - else: - newst = State(id=dep, path=None, source=None, manager=manager, - caller_state=st, caller_line=st.dep_line_map.get(dep, 1)) - except ModuleNotFound: - if dep in st.dependencies: - st.dependencies.remove(dep) - st.suppressed.append(dep) - else: - assert newst.id not in graph, newst.id - graph[newst.id] = newst - new.append(newst) - return graph - - -def process_graph(graph: Graph, manager: BuildManager) -> None: - """Process everything in dependency order.""" - sccs = sorted_components(graph) - manager.log("Found %d SCCs; largest has %d nodes" % - (len(sccs), max(len(scc) for scc in sccs))) - # We're processing SCCs from leaves (those without further - # dependencies) to roots (those from which everything else can be - # reached). - for ascc in sccs: - # Order the SCC's nodes using a heuristic. - # Note that ascc is a set, and scc is a list. - scc = order_ascc(graph, ascc) - # If builtins is in the list, move it last. (This is a bit of - # a hack, but it's necessary because the builtins module is - # part of a small cycle involving at least {builtins, abc, - # typing}. Of these, builtins must be processed last or else - # some builtin objects will be incompletely processed.) - if 'builtins' in ascc: - scc.remove('builtins') - scc.append('builtins') - if manager.options.verbosity >= 2: - for id in scc: - manager.trace("Priorities for %s:" % id, - " ".join("%s:%d" % (x, graph[id].priorities[x]) - for x in graph[id].dependencies - if x in ascc and x in graph[id].priorities)) - # Because the SCCs are presented in topological sort order, we - # don't need to look at dependencies recursively for staleness - # -- the immediate dependencies are sufficient. - stale_scc = {id for id in scc if not graph[id].is_fresh()} - fresh = not stale_scc - deps = set() - for id in scc: - deps.update(graph[id].dependencies) - deps -= ascc - stale_deps = {id for id in deps if not graph[id].is_fresh()} - fresh = fresh and not stale_deps - undeps = set() - if fresh: - # Check if any dependencies that were suppressed according - # to the cache have heen added back in this run. - # NOTE: Newly suppressed dependencies are handled by is_fresh(). - for id in scc: - undeps.update(graph[id].suppressed) - undeps &= graph.keys() - if undeps: - fresh = False - if fresh: - # All cache files are fresh. Check that no dependency's - # cache file is newer than any scc node's cache file. - oldest_in_scc = min(graph[id].meta.data_mtime for id in scc) - newest_in_deps = 0 if not deps else max(graph[dep].meta.data_mtime for dep in deps) - if manager.options.verbosity >= 3: # Dump all mtimes for extreme debugging. - all_ids = sorted(ascc | deps, key=lambda id: graph[id].meta.data_mtime) - for id in all_ids: - if id in scc: - if graph[id].meta.data_mtime < newest_in_deps: - key = "*id:" - else: - key = "id:" - else: - if graph[id].meta.data_mtime > oldest_in_scc: - key = "+dep:" - else: - key = "dep:" - manager.trace(" %5s %.0f %s" % (key, graph[id].meta.data_mtime, id)) - # If equal, give the benefit of the doubt, due to 1-sec time granularity - # (on some platforms). - if oldest_in_scc < newest_in_deps: - fresh = False - fresh_msg = "out of date by %.0f seconds" % (newest_in_deps - oldest_in_scc) - else: - fresh_msg = "fresh" - elif undeps: - fresh_msg = "stale due to changed suppression (%s)" % " ".join(sorted(undeps)) - elif stale_scc: - fresh_msg = "inherently stale (%s)" % " ".join(sorted(stale_scc)) - if stale_deps: - fresh_msg += " with stale deps (%s)" % " ".join(sorted(stale_deps)) - else: - fresh_msg = "stale due to deps (%s)" % " ".join(sorted(stale_deps)) - if len(scc) == 1: - manager.log("Processing SCC singleton (%s) as %s" % (" ".join(scc), fresh_msg)) - else: - manager.log("Processing SCC of size %d (%s) as %s" % - (len(scc), " ".join(scc), fresh_msg)) - if fresh: - process_fresh_scc(graph, scc) - else: - process_stale_scc(graph, scc) - - -def order_ascc(graph: Graph, ascc: AbstractSet[str], pri_max: int = PRI_ALL) -> List[str]: - """Come up with the ideal processing order within an SCC. - - Using the priorities assigned by all_imported_modules_in_file(), - try to reduce the cycle to a DAG, by omitting arcs representing - dependencies of lower priority. - - In the simplest case, if we have A <--> B where A has a top-level - "import B" (medium priority) but B only has the reverse "import A" - inside a function (low priority), we turn the cycle into a DAG by - dropping the B --> A arc, which leaves only A --> B. - - If all arcs have the same priority, we fall back to sorting by - reverse global order (the order in which modules were first - encountered). - - The algorithm is recursive, as follows: when as arcs of different - priorities are present, drop all arcs of the lowest priority, - identify SCCs in the resulting graph, and apply the algorithm to - each SCC thus found. The recursion is bounded because at each - recursion the spread in priorities is (at least) one less. - - In practice there are only a few priority levels (currently - N=3) and in the worst case we just carry out the same algorithm - for finding SCCs N times. Thus the complexity is no worse than - the complexity of the original SCC-finding algorithm -- see - strongly_connected_components() below for a reference. - """ - if len(ascc) == 1: - return [s for s in ascc] - pri_spread = set() - for id in ascc: - state = graph[id] - for dep in state.dependencies: - if dep in ascc: - pri = state.priorities.get(dep, PRI_HIGH) - if pri < pri_max: - pri_spread.add(pri) - if len(pri_spread) == 1: - # Filtered dependencies are uniform -- order by global order. - return sorted(ascc, key=lambda id: -graph[id].order) - pri_max = max(pri_spread) - sccs = sorted_components(graph, ascc, pri_max) - # The recursion is bounded by the len(pri_spread) check above. - return [s for ss in sccs for s in order_ascc(graph, ss, pri_max)] - - -def process_fresh_scc(graph: Graph, scc: List[str]) -> None: - """Process the modules in one SCC from their cached data.""" - for id in scc: - graph[id].load_tree() - for id in scc: - graph[id].patch_parent() - for id in scc: - graph[id].fix_cross_refs() - for id in scc: - graph[id].calculate_mros() - - -def process_stale_scc(graph: Graph, scc: List[str]) -> None: - """Process the modules in one SCC from source code.""" - for id in scc: - graph[id].mark_stale() - for id in scc: - # We may already have parsed the module, or not. - # If the former, parse_file() is a no-op. - graph[id].parse_file() - for id in scc: - graph[id].patch_parent() - for id in scc: - graph[id].semantic_analysis() - for id in scc: - graph[id].semantic_analysis_pass_three() - for id in scc: - graph[id].type_check() - graph[id].write_cache() - - -def sorted_components(graph: Graph, - vertices: Optional[AbstractSet[str]] = None, - pri_max: int = PRI_ALL) -> List[AbstractSet[str]]: - """Return the graph's SCCs, topologically sorted by dependencies. - - The sort order is from leaves (nodes without dependencies) to - roots (nodes on which no other nodes depend). - - This works for a subset of the full dependency graph too; - dependencies that aren't present in graph.keys() are ignored. - """ - # Compute SCCs. - if vertices is None: - vertices = set(graph) - edges = {id: deps_filtered(graph, vertices, id, pri_max) for id in vertices} - sccs = list(strongly_connected_components(vertices, edges)) - # Topsort. - sccsmap = {id: frozenset(scc) for scc in sccs for id in scc} - data = {} # type: Dict[AbstractSet[str], Set[AbstractSet[str]]] - for scc in sccs: - deps = set() # type: Set[AbstractSet[str]] - for id in scc: - deps.update(sccsmap[x] for x in deps_filtered(graph, vertices, id, pri_max)) - data[frozenset(scc)] = deps - res = [] - for ready in topsort(data): - # Sort the sets in ready by reversed smallest State.order. Examples: - # - # - If ready is [{x}, {y}], x.order == 1, y.order == 2, we get - # [{y}, {x}]. - # - # - If ready is [{a, b}, {c, d}], a.order == 1, b.order == 3, - # c.order == 2, d.order == 4, the sort keys become [1, 2] - # and the result is [{c, d}, {a, b}]. - res.extend(sorted(ready, - key=lambda scc: -min(graph[id].order for id in scc))) - return res - - -def deps_filtered(graph: Graph, vertices: AbstractSet[str], id: str, pri_max: int) -> List[str]: - """Filter dependencies for id with pri < pri_max.""" - if id not in vertices: - return [] - state = graph[id] - return [dep - for dep in state.dependencies - if dep in vertices and state.priorities.get(dep, PRI_HIGH) < pri_max] - - -def strongly_connected_components(vertices: AbstractSet[str], - edges: Dict[str, List[str]]) -> Iterator[Set[str]]: - """Compute Strongly Connected Components of a directed graph. - - Args: - vertices: the labels for the vertices - edges: for each vertex, gives the target vertices of its outgoing edges - - Returns: - An iterator yielding strongly connected components, each - represented as a set of vertices. Each input vertex will occur - exactly once; vertices not part of a SCC are returned as - singleton sets. - - From http://code.activestate.com/recipes/578507/. - """ - identified = set() # type: Set[str] - stack = [] # type: List[str] - index = {} # type: Dict[str, int] - boundaries = [] # type: List[int] - - def dfs(v: str) -> Iterator[Set[str]]: - index[v] = len(stack) - stack.append(v) - boundaries.append(index[v]) - - for w in edges[v]: - if w not in index: - # For Python >= 3.3, replace with "yield from dfs(w)" - for scc in dfs(w): - yield scc - elif w not in identified: - while index[w] < boundaries[-1]: - boundaries.pop() - - if boundaries[-1] == index[v]: - boundaries.pop() - scc = set(stack[index[v]:]) - del stack[index[v]:] - identified.update(scc) - yield scc - - for v in vertices: - if v not in index: - # For Python >= 3.3, replace with "yield from dfs(v)" - for scc in dfs(v): - yield scc - - -def topsort(data: Dict[AbstractSet[str], - Set[AbstractSet[str]]]) -> Iterable[Set[AbstractSet[str]]]: - """Topological sort. - - Args: - data: A map from SCCs (represented as frozen sets of strings) to - sets of SCCs, its dependencies. NOTE: This data structure - is modified in place -- for normalization purposes, - self-dependencies are removed and entries representing - orphans are added. - - Returns: - An iterator yielding sets of SCCs that have an equivalent - ordering. NOTE: The algorithm doesn't care about the internal - structure of SCCs. - - Example: - Suppose the input has the following structure: - - {A: {B, C}, B: {D}, C: {D}} - - This is normalized to: - - {A: {B, C}, B: {D}, C: {D}, D: {}} - - The algorithm will yield the following values: - - {D} - {B, C} - {A} - - From http://code.activestate.com/recipes/577413/. - """ - # TODO: Use a faster algorithm? - for k, v in data.items(): - v.discard(k) # Ignore self dependencies. - for item in set.union(*data.values()) - set(data.keys()): - data[item] = set() - while True: - ready = {item for item, dep in data.items() if not dep} - if not ready: - break - yield ready - data = {item: (dep - ready) - for item, dep in data.items() - if item not in ready} - assert not data, "A cyclic dependency exists amongst %r" % data +"""Facilities to analyze entire programs, including imported modules. + +Parse and analyze the source files of a program in the correct order +(based on file dependencies), and collect the results. + +This module only directs a build, which is performed in multiple passes per +file. The individual passes are implemented in separate modules. + +The function build() is the main interface to this module. +""" +# TODO: More consistent terminology, e.g. path/fnam, module/id, state/file + +import binascii +import collections +import contextlib +import json +import os +import os.path +import sys +import time +from os.path import dirname, basename + +from typing import (AbstractSet, Dict, Iterable, Iterator, List, + NamedTuple, Optional, Set, Tuple, Union, Mapping) + +from mypy.types import Type +from mypy.nodes import (MypyFile, Node, Import, ImportFrom, ImportAll, + SymbolTableNode, MODULE_REF) +from mypy.semanal import FirstPass, SemanticAnalyzer, ThirdPass +from mypy.checker import TypeChecker +from mypy.errors import Errors, CompileError, DecodeError, report_internal_error +from mypy import fixup +from mypy.report import Reports +from mypy import defaults +from mypy import moduleinfo +from mypy import util +from mypy.fixup import fixup_module_pass_one, fixup_module_pass_two +from mypy.options import Options +from mypy.parse import parse +from mypy.stats import dump_type_stats + + +# We need to know the location of this file to load data, but +# until Python 3.4, __file__ is relative. +__file__ = os.path.realpath(__file__) + +PYTHON_EXTENSIONS = ['.pyi', '.py'] + + +class BuildResult: + """The result of a successful build. + + Attributes: + manager: The build manager. + files: Dictionary from module name to related AST node. + types: Dictionary from parse tree node to its inferred type. + errors: List of error messages. + """ + + def __init__(self, manager: 'BuildManager') -> None: + self.manager = manager + self.files = manager.modules + self.types = manager.type_checker.type_map + self.errors = manager.errors.messages() + + +class BuildSource: + def __init__(self, path: Optional[str], module: Optional[str], + text: Optional[str]) -> None: + self.path = path + self.module = module or '__main__' + self.text = text + + @property + def effective_path(self) -> str: + """Return the effective path (ie, if its from in memory)""" + return self.path or '' + + +class BuildSourceSet: + """Efficiently test a file's membership in the set of build sources.""" + + def __init__(self, sources: List[BuildSource]) -> None: + self.source_text_present = False + self.source_modules = set() # type: Set[str] + self.source_paths = set() # type: Set[str] + + for source in sources: + if source.text is not None: + self.source_text_present = True + elif source.path: + self.source_paths.add(source.path) + else: + self.source_modules.add(source.module) + + def is_source(self, file: MypyFile) -> bool: + if file.path and file.path in self.source_paths: + return True + elif file._fullname in self.source_modules: + return True + elif file.path is None and self.source_text_present: + return True + else: + return False + + +def build(sources: List[BuildSource], + options: Options, + alt_lib_path: str = None, + bin_dir: str = None) -> BuildResult: + """Analyze a program. + + A single call to build performs parsing, semantic analysis and optionally + type checking for the program *and* all imported modules, recursively. + + Return BuildResult if successful or only non-blocking errors were found; + otherwise raise CompileError. + + Args: + sources: list of sources to build + options: build options + alt_lib_path: an additional directory for looking up library modules + (takes precedence over other directories) + bin_dir: directory containing the mypy script, used for finding data + directories; if omitted, use '.' as the data directory + """ + + data_dir = default_data_dir(bin_dir) + + find_module_clear_caches() + + # Determine the default module search path. + lib_path = default_lib_path(data_dir, options.python_version) + + if options.use_builtins_fixtures: + # Use stub builtins (to speed up test cases and to make them easier to + # debug). This is a test-only feature, so assume our files are laid out + # as in the source tree. + root_dir = os.path.dirname(os.path.dirname(__file__)) + lib_path.insert(0, os.path.join(root_dir, 'test-data', 'unit', 'lib-stub')) + else: + for source in sources: + if source.path: + # Include directory of the program file in the module search path. + dir = remove_cwd_prefix_from_path(dirname(source.path)) + if dir not in lib_path: + lib_path.insert(0, dir) + + # Do this even if running as a file, for sanity (mainly because with + # multiple builds, there could be a mix of files/modules, so its easier + # to just define the semantics that we always add the current director + # to the lib_path + lib_path.insert(0, os.getcwd()) + + # Add MYPYPATH environment variable to front of library path, if defined. + lib_path[:0] = mypy_path() + + # If provided, insert the caller-supplied extra module path to the + # beginning (highest priority) of the search path. + if alt_lib_path: + lib_path.insert(0, alt_lib_path) + + reports = Reports(data_dir, options.report_dirs) + + source_set = BuildSourceSet(sources) + + # Construct a build manager object to hold state during the build. + # + # Ignore current directory prefix in error messages. + manager = BuildManager(data_dir, lib_path, + ignore_prefix=os.getcwd(), + source_set=source_set, + reports=reports, + options=options) + + try: + dispatch(sources, manager) + return BuildResult(manager) + finally: + manager.log("Build finished with %d modules, %d types, and %d errors" % + (len(manager.modules), + len(manager.type_checker.type_map), + manager.errors.num_messages())) + # Finish the HTML or XML reports even if CompileError was raised. + reports.finish() + + +def default_data_dir(bin_dir: Optional[str]) -> str: + """Returns directory containing typeshed directory + + Args: + bin_dir: directory containing the mypy script + """ + if not bin_dir: + mypy_package = os.path.dirname(__file__) + parent = os.path.dirname(mypy_package) + if (os.path.basename(parent) == 'site-packages' or + os.path.basename(parent) == 'dist-packages'): + # Installed in site-packages or dist-packages, but invoked with python3 -m mypy; + # __file__ is .../blah/lib/python3.N/site-packages/mypy/build.py + # or .../blah/lib/python3.N/dist-packages/mypy/build.py (Debian) + # or .../blah/lib/site-packages/mypy/build.py (Windows) + # blah may be a virtualenv or /usr/local. We want .../blah/lib/mypy. + lib = parent + for i in range(2): + lib = os.path.dirname(lib) + if os.path.basename(lib) == 'lib': + return os.path.join(lib, 'mypy') + subdir = os.path.join(parent, 'lib', 'mypy') + if os.path.isdir(subdir): + # If installed via buildout, the __file__ is + # somewhere/mypy/__init__.py and what we want is + # somewhere/lib/mypy. + return subdir + # Default to directory containing this file's parent. + return parent + base = os.path.basename(bin_dir) + dir = os.path.dirname(bin_dir) + if (sys.platform == 'win32' and base.lower() == 'scripts' + and not os.path.isdir(os.path.join(dir, 'typeshed'))): + # Installed, on Windows. + return os.path.join(dir, 'Lib', 'mypy') + elif base == 'scripts': + # Assume that we have a repo check out or unpacked source tarball. + return dir + elif base == 'bin': + # Installed to somewhere (can be under /usr/local or anywhere). + return os.path.join(dir, 'lib', 'mypy') + elif base == 'python3': + # Assume we installed python3 with brew on os x + return os.path.join(os.path.dirname(dir), 'lib', 'mypy') + elif dir.endswith('python-exec'): + # Gentoo uses a python wrapper in /usr/lib to which mypy is a symlink. + return os.path.join(os.path.dirname(dir), 'mypy') + else: + # Don't know where to find the data files! + raise RuntimeError("Broken installation: can't determine base dir") + + +def mypy_path() -> List[str]: + path_env = os.getenv('MYPYPATH') + if not path_env: + return [] + return path_env.split(os.pathsep) + + +def default_lib_path(data_dir: str, pyversion: Tuple[int, int]) -> List[str]: + """Return default standard library search paths.""" + # IDEA: Make this more portable. + path = [] # type: List[str] + + auto = os.path.join(data_dir, 'stubs-auto') + if os.path.isdir(auto): + data_dir = auto + + # We allow a module for e.g. version 3.5 to be in 3.4/. The assumption + # is that a module added with 3.4 will still be present in Python 3.5. + versions = ["%d.%d" % (pyversion[0], minor) + for minor in reversed(range(pyversion[1] + 1))] + # E.g. for Python 3.2, try 3.2/, 3.1/, 3.0/, 3/, 2and3/. + # (Note that 3.1 and 3.0 aren't really supported, but we don't care.) + for v in versions + [str(pyversion[0]), '2and3']: + for lib_type in ['stdlib', 'third_party']: + stubdir = os.path.join(data_dir, 'typeshed', lib_type, v) + if os.path.isdir(stubdir): + path.append(stubdir) + + # Add fallback path that can be used if we have a broken installation. + if sys.platform != 'win32': + path.append('/usr/local/lib/mypy') + + return path + + +CacheMeta = NamedTuple('CacheMeta', + [('id', str), + ('path', str), + ('mtime', float), + ('size', int), + ('dependencies', List[str]), # names of imported modules + ('data_mtime', float), # mtime of data_json + ('data_json', str), # path of .data.json + ('suppressed', List[str]), # dependencies that weren't imported + ('options', Optional[Dict[str, bool]]), # build options + ('dep_prios', List[int]), + ]) +# NOTE: dependencies + suppressed == all reachable imports; +# suppressed contains those reachable imports that were prevented by +# --silent-imports or simply not found. + + +# Priorities used for imports. (Here, top-level includes inside a class.) +# These are used to determine a more predictable order in which the +# nodes in an import cycle are processed. +PRI_HIGH = 5 # top-level "from X import blah" +PRI_MED = 10 # top-level "import X" +PRI_LOW = 20 # either form inside a function +PRI_ALL = 99 # include all priorities + + +class BuildManager: + """This class holds shared state for building a mypy program. + + It is used to coordinate parsing, import processing, semantic + analysis and type checking. The actual build steps are carried + out by dispatch(). + + Attributes: + data_dir: Mypy data directory (contains stubs) + lib_path: Library path for looking up modules + modules: Mapping of module ID to MypyFile (shared by the passes) + semantic_analyzer: + Semantic analyzer, pass 2 + semantic_analyzer_pass3: + Semantic analyzer, pass 3 + type_checker: Type checker + errors: Used for reporting all errors + options: Build options + missing_modules: Set of modules that could not be imported encountered so far + """ + + def __init__(self, data_dir: str, + lib_path: List[str], + ignore_prefix: str, + source_set: BuildSourceSet, + reports: Reports, + options: Options) -> None: + self.start_time = time.time() + self.data_dir = data_dir + self.errors = Errors() + self.errors.set_ignore_prefix(ignore_prefix) + self.lib_path = tuple(lib_path) + self.source_set = source_set + self.reports = reports + self.options = options + self.semantic_analyzer = SemanticAnalyzer(lib_path, self.errors, options=options) + self.modules = self.semantic_analyzer.modules + self.semantic_analyzer_pass3 = ThirdPass(self.modules, self.errors) + self.type_checker = TypeChecker(self.errors, self.modules, options=options) + self.missing_modules = set() # type: Set[str] + + def all_imported_modules_in_file(self, + file: MypyFile) -> List[Tuple[int, str, int]]: + """Find all reachable import statements in a file. + + Return list of tuples (priority, module id, import line number) + for all modules imported in file; lower numbers == higher priority. + """ + + def correct_rel_imp(imp: Union[ImportFrom, ImportAll]) -> str: + """Function to correct for relative imports.""" + file_id = file.fullname() + rel = imp.relative + if rel == 0: + return imp.id + if os.path.basename(file.path).startswith('__init__.'): + rel -= 1 + if rel != 0: + file_id = ".".join(file_id.split(".")[:-rel]) + new_id = file_id + "." + imp.id if imp.id else file_id + + return new_id + + res = [] # type: List[Tuple[int, str, int]] + for imp in file.imports: + if not imp.is_unreachable: + if isinstance(imp, Import): + pri = PRI_MED if imp.is_top_level else PRI_LOW + for id, _ in imp.ids: + res.append((pri, id, imp.line)) + elif isinstance(imp, ImportFrom): + cur_id = correct_rel_imp(imp) + pos = len(res) + all_are_submodules = True + # Also add any imported names that are submodules. + pri = PRI_MED if imp.is_top_level else PRI_LOW + for name, __ in imp.names: + sub_id = cur_id + '.' + name + if self.is_module(sub_id): + res.append((pri, sub_id, imp.line)) + else: + all_are_submodules = False + # If all imported names are submodules, don't add + # cur_id as a dependency. Otherwise (i.e., if at + # least one imported name isn't a submodule) + # cur_id is also a dependency, and we should + # insert it *before* any submodules. + if not all_are_submodules: + pri = PRI_HIGH if imp.is_top_level else PRI_LOW + res.insert(pos, ((pri, cur_id, imp.line))) + elif isinstance(imp, ImportAll): + pri = PRI_HIGH if imp.is_top_level else PRI_LOW + res.append((pri, correct_rel_imp(imp), imp.line)) + + return res + + def is_module(self, id: str) -> bool: + """Is there a file in the file system corresponding to module id?""" + return find_module(id, self.lib_path) is not None + + def parse_file(self, id: str, path: str, source: str) -> MypyFile: + """Parse the source of a file with the given name. + + Raise CompileError if there is a parse error. + """ + num_errs = self.errors.num_messages() + tree = parse(source, path, self.errors, options=self.options) + tree._fullname = id + + # We don't want to warn about 'type: ignore' comments on + # imports, but we're about to modify tree.imports, so grab + # these first. + import_lines = set(node.line for node in tree.imports) + + # Skip imports that have been ignored (so that we can ignore a C extension module without + # stub, for example), except for 'from x import *', because we wouldn't be able to + # determine which names should be defined unless we process the module. We can still + # ignore errors such as redefinitions when using the latter form. + imports = [node for node in tree.imports + if node.line not in tree.ignored_lines or isinstance(node, ImportAll)] + tree.imports = imports + + if self.errors.num_messages() != num_errs: + self.log("Bailing due to parse errors") + self.errors.raise_error() + + self.errors.set_file_ignored_lines(path, tree.ignored_lines) + self.errors.mark_file_ignored_lines_used(path, import_lines) + return tree + + def module_not_found(self, path: str, line: int, id: str) -> None: + self.errors.set_file(path) + stub_msg = "(Stub files are from https://github.com/python/typeshed)" + if ((self.options.python_version[0] == 2 and moduleinfo.is_py2_std_lib_module(id)) or + (self.options.python_version[0] >= 3 and moduleinfo.is_py3_std_lib_module(id))): + self.errors.report( + line, "No library stub file for standard library module '{}'".format(id)) + self.errors.report(line, stub_msg, severity='note', only_once=True) + elif moduleinfo.is_third_party_module(id): + self.errors.report(line, "No library stub file for module '{}'".format(id)) + self.errors.report(line, stub_msg, severity='note', only_once=True) + else: + self.errors.report(line, "Cannot find module named '{}'".format(id)) + self.errors.report(line, '(Perhaps setting MYPYPATH ' + 'or using the "--silent-imports" flag would help)', + severity='note', only_once=True) + + def report_file(self, file: MypyFile) -> None: + if self.source_set.is_source(file): + self.reports.file(file, type_map=self.type_checker.type_map) + + def log(self, *message: str) -> None: + if self.options.verbosity >= 1: + print('%.3f:LOG: ' % (time.time() - self.start_time), *message, file=sys.stderr) + sys.stderr.flush() + + def trace(self, *message: str) -> None: + if self.options.verbosity >= 2: + print('%.3f:TRACE:' % (time.time() - self.start_time), *message, file=sys.stderr) + sys.stderr.flush() + + +def remove_cwd_prefix_from_path(p: str) -> str: + """Remove current working directory prefix from p, if present. + + Also crawl up until a directory without __init__.py is found. + + If the result would be empty, return '.' instead. + """ + cur = os.getcwd() + # Add separator to the end of the path, unless one is already present. + if basename(cur) != '': + cur += os.sep + # Compute root path. + while (p and + (os.path.isfile(os.path.join(p, '__init__.py')) or + os.path.isfile(os.path.join(p, '__init__.pyi')))): + dir, base = os.path.split(p) + if not base: + break + p = dir + # Remove current directory prefix from the path, if present. + if p.startswith(cur): + p = p[len(cur):] + # Avoid returning an empty path; replace that with '.'. + if p == '': + p = '.' + return p + + +# Cache find_module: (id, lib_path) -> result. +find_module_cache = {} # type: Dict[Tuple[str, Tuple[str, ...]], str] + +# Cache some repeated work within distinct find_module calls: finding which +# elements of lib_path have even the subdirectory they'd need for the module +# to exist. This is shared among different module ids when they differ only +# in the last component. +find_module_dir_cache = {} # type: Dict[Tuple[str, Tuple[str, ...]], List[str]] + +# Cache directory listings. We assume that while one os.listdir() +# call may be more expensive than one os.stat() call, a small number +# of os.stat() calls is quickly more expensive than caching the +# os.listdir() outcome, and the advantage of the latter is that it +# gives us the case-correct filename on Windows and Mac. +find_module_listdir_cache = {} # type: Dict[str, Optional[List[str]]] + + +def find_module_clear_caches(): + find_module_cache.clear() + find_module_dir_cache.clear() + find_module_listdir_cache.clear() + + +def list_dir(path: str) -> Optional[List[str]]: + """Return a cached directory listing. + + Returns None if the path doesn't exist or isn't a directory. + """ + if path in find_module_listdir_cache: + return find_module_listdir_cache[path] + try: + res = os.listdir(path) + except OSError: + res = None + find_module_listdir_cache[path] = res + return res + + +def is_file(path: str) -> bool: + """Return whether path exists and is a file. + + On case-insensitive filesystems (like Mac or Windows) this returns + False if the case of the path's last component does not exactly + match the case found in the filesystem. + """ + head, tail = os.path.split(path) + if not tail: + return False + names = list_dir(head) + if not names: + return False + if tail not in names: + return False + return os.path.isfile(path) + + +def find_module(id: str, lib_path: Iterable[str]) -> str: + """Return the path of the module source file, or None if not found.""" + if not isinstance(lib_path, tuple): + lib_path = tuple(lib_path) + + def find(): + # If we're looking for a module like 'foo.bar.baz', it's likely that most of the + # many elements of lib_path don't even have a subdirectory 'foo/bar'. Discover + # that only once and cache it for when we look for modules like 'foo.bar.blah' + # that will require the same subdirectory. + components = id.split('.') + dir_chain = os.sep.join(components[:-1]) # e.g., 'foo/bar' + if (dir_chain, lib_path) not in find_module_dir_cache: + dirs = [] + for pathitem in lib_path: + # e.g., '/usr/lib/python3.4/foo/bar' + dir = os.path.normpath(os.path.join(pathitem, dir_chain)) + if os.path.isdir(dir): + dirs.append(dir) + find_module_dir_cache[dir_chain, lib_path] = dirs + candidate_base_dirs = find_module_dir_cache[dir_chain, lib_path] + + # If we're looking for a module like 'foo.bar.baz', then candidate_base_dirs now + # contains just the subdirectories 'foo/bar' that actually exist under the + # elements of lib_path. This is probably much shorter than lib_path itself. + # Now just look for 'baz.pyi', 'baz/__init__.py', etc., inside those directories. + seplast = os.sep + components[-1] # so e.g. '/baz' + sepinit = os.sep + '__init__' + for base_dir in candidate_base_dirs: + base_path = base_dir + seplast # so e.g. '/usr/lib/python3.4/foo/bar/baz' + # Prefer package over module, i.e. baz/__init__.py* over baz.py*. + for extension in PYTHON_EXTENSIONS: + path = base_path + sepinit + extension + if is_file(path) and verify_module(id, path): + return path + # No package, look for module. + for extension in PYTHON_EXTENSIONS: + path = base_path + extension + if is_file(path) and verify_module(id, path): + return path + return None + + key = (id, lib_path) + if key not in find_module_cache: + find_module_cache[key] = find() + return find_module_cache[key] + + +def find_modules_recursive(module: str, lib_path: List[str]) -> List[BuildSource]: + module_path = find_module(module, lib_path) + if not module_path: + return [] + result = [BuildSource(module_path, module, None)] + if module_path.endswith(('__init__.py', '__init__.pyi')): + # Subtle: this code prefers the .pyi over the .py if both + # exists, and also prefers packages over modules if both x/ + # and x.py* exist. How? We sort the directory items, so x + # comes before x.py and x.pyi. But the preference for .pyi + # over .py is encoded in find_module(); even though we see + # x.py before x.pyi, find_module() will find x.pyi first. We + # use hits to avoid adding it a second time when we see x.pyi. + # This also avoids both x.py and x.pyi when x/ was seen first. + hits = set() # type: Set[str] + for item in sorted(os.listdir(os.path.dirname(module_path))): + abs_path = os.path.join(os.path.dirname(module_path), item) + if os.path.isdir(abs_path) and \ + (os.path.isfile(os.path.join(abs_path, '__init__.py')) or + os.path.isfile(os.path.join(abs_path, '__init__.pyi'))): + hits.add(item) + result += find_modules_recursive(module + '.' + item, lib_path) + elif item != '__init__.py' and item != '__init__.pyi' and \ + item.endswith(('.py', '.pyi')): + mod = item.split('.')[0] + if mod not in hits: + hits.add(mod) + result += find_modules_recursive( + module + '.' + mod, lib_path) + return result + + +def verify_module(id: str, path: str) -> bool: + """Check that all packages containing id have a __init__ file.""" + if path.endswith(('__init__.py', '__init__.pyi')): + path = dirname(path) + for i in range(id.count('.')): + path = dirname(path) + if not any(os.path.isfile(os.path.join(path, '__init__{}'.format(extension))) + for extension in PYTHON_EXTENSIONS): + return False + return True + + +def read_with_python_encoding(path: str, pyversion: Tuple[int, int]) -> str: + """Read the Python file with while obeying PEP-263 encoding detection""" + source_bytearray = bytearray() + encoding = 'utf8' if pyversion[0] >= 3 else 'ascii' + + with open(path, 'rb') as f: + # read first two lines and check if PEP-263 coding is present + source_bytearray.extend(f.readline()) + source_bytearray.extend(f.readline()) + + # check for BOM UTF-8 encoding and strip it out if present + if source_bytearray.startswith(b'\xef\xbb\xbf'): + encoding = 'utf8' + source_bytearray = source_bytearray[3:] + else: + _encoding, _ = util.find_python_encoding(source_bytearray, pyversion) + # check that the coding isn't mypy. We skip it since + # registering may not have happened yet + if _encoding != 'mypy': + encoding = _encoding + + source_bytearray.extend(f.read()) + try: + source_bytearray.decode(encoding) + except LookupError as lookuperr: + raise DecodeError(str(lookuperr)) + return source_bytearray.decode(encoding) + + +MYPY_CACHE = '.mypy_cache' + + +def get_cache_names(id: str, path: str, pyversion: Tuple[int, int]) -> Tuple[str, str]: + """Return the file names for the cache files. + + Args: + id: module ID + path: module path (used to recognize packages) + pyversion: Python version (major, minor) + + Returns: + A tuple with the file names to be used for the meta JSON and the + data JSON, respectively. + """ + prefix = os.path.join(MYPY_CACHE, '%d.%d' % pyversion, *id.split('.')) + is_package = os.path.basename(path).startswith('__init__.py') + if is_package: + prefix = os.path.join(prefix, '__init__') + return (prefix + '.meta.json', prefix + '.data.json') + + +def find_cache_meta(id: str, path: str, manager: BuildManager) -> Optional[CacheMeta]: + """Find cache data for a module. + + Args: + id: module ID + path: module path + manager: the build manager (for pyversion, log/trace, and build options) + + Returns: + A CacheMeta instance if the cache data was found and appears + valid; otherwise None. + """ + # TODO: May need to take more build options into account + meta_json, data_json = get_cache_names(id, path, manager.options.python_version) + manager.trace('Looking for {} {}'.format(id, data_json)) + if not os.path.exists(meta_json): + return None + with open(meta_json, 'r') as f: + meta_str = f.read() + manager.trace('Meta {} {}'.format(id, meta_str.rstrip())) + meta = json.loads(meta_str) # TODO: Errors + if not isinstance(meta, dict): + return None + path = os.path.abspath(path) + m = CacheMeta( + meta.get('id'), + meta.get('path'), + meta.get('mtime'), + meta.get('size'), + meta.get('dependencies', []), + meta.get('data_mtime'), + data_json, + meta.get('suppressed', []), + meta.get('options'), + meta.get('dep_prios', []), + ) + if (m.id != id or m.path != path or + m.mtime is None or m.size is None or + m.dependencies is None or m.data_mtime is None): + return None + + # Ignore cache if generated by an older mypy version. + if m.options is None or len(m.dependencies) != len(m.dep_prios): + return None + + # Ignore cache if (relevant) options aren't the same. + cached_options = m.options + current_options = select_options_affecting_cache(manager.options) + if cached_options != current_options: + return None + + # TODO: Share stat() outcome with find_module() + st = os.stat(path) # TODO: Errors + if st.st_mtime != m.mtime or st.st_size != m.size: + manager.log('Metadata abandoned because of modified file {}'.format(path)) + return None + # It's a match on (id, path, mtime, size). + # Check data_json; assume if its mtime matches it's good. + # TODO: stat() errors + if os.path.getmtime(data_json) != m.data_mtime: + return None + manager.log('Found {} {}'.format(id, meta_json)) + return m + + +def select_options_affecting_cache(options: Options) -> Mapping[str, bool]: + return {opt: getattr(options, opt) for opt in OPTIONS_AFFECTING_CACHE} + + +OPTIONS_AFFECTING_CACHE = [ + "silent_imports", + "almost_silent", + "disallow_untyped_calls", + "disallow_untyped_defs", + "check_untyped_defs", +] + + +def random_string(): + return binascii.hexlify(os.urandom(8)).decode('ascii') + + +def write_cache(id: str, path: str, tree: MypyFile, + dependencies: List[str], suppressed: List[str], + dep_prios: List[int], + manager: BuildManager) -> None: + """Write cache files for a module. + + Args: + id: module ID + path: module path + tree: the fully checked module data + dependencies: module IDs on which this module depends + suppressed: module IDs which were suppressed as dependencies + dep_prios: priorities (parallel array to dependencies) + manager: the build manager (for pyversion, log/trace) + """ + path = os.path.abspath(path) + manager.trace('Dumping {} {}'.format(id, path)) + st = os.stat(path) # TODO: Errors + mtime = st.st_mtime + size = st.st_size + meta_json, data_json = get_cache_names(id, path, manager.options.python_version) + manager.log('Writing {} {} {}'.format(id, meta_json, data_json)) + data = tree.serialize() + parent = os.path.dirname(data_json) + if not os.path.isdir(parent): + os.makedirs(parent) + assert os.path.dirname(meta_json) == parent + nonce = '.' + random_string() + data_json_tmp = data_json + nonce + meta_json_tmp = meta_json + nonce + with open(data_json_tmp, 'w') as f: + json.dump(data, f, indent=2, sort_keys=True) + f.write('\n') + data_mtime = os.path.getmtime(data_json_tmp) + meta = {'id': id, + 'path': path, + 'mtime': mtime, + 'size': size, + 'data_mtime': data_mtime, + 'dependencies': dependencies, + 'suppressed': suppressed, + 'options': select_options_affecting_cache(manager.options), + 'dep_prios': dep_prios, + } + with open(meta_json_tmp, 'w') as f: + json.dump(meta, f, sort_keys=True) + f.write('\n') + os.replace(data_json_tmp, data_json) + os.replace(meta_json_tmp, meta_json) + + +"""Dependency manager. + +Design +====== + +Ideally +------- + +A. Collapse cycles (each SCC -- strongly connected component -- + becomes one "supernode"). + +B. Topologically sort nodes based on dependencies. + +C. Process from leaves towards roots. + +Wrinkles +-------- + +a. Need to parse source modules to determine dependencies. + +b. Processing order for modules within an SCC. + +c. Must order mtimes of files to decide whether to re-process; depends + on clock never resetting. + +d. from P import M; checks filesystem whether module P.M exists in + filesystem. + +e. Race conditions, where somebody modifies a file while we're + processing. I propose not to modify the algorithm to handle this, + but to detect when this could lead to inconsistencies. (For + example, when we decide on the dependencies based on cache + metadata, and then we decide to re-parse a file because of a stale + dependency, if the re-parsing leads to a different list of + dependencies we should warn the user or start over.) + +Steps +----- + +1. For each explicitly given module find the source file location. + +2. For each such module load and check the cache metadata, and decide + whether it's valid. + +3. Now recursively (or iteratively) find dependencies and add those to + the graph: + + - for cached nodes use the list of dependencies from the cache + metadata (this will be valid even if we later end up re-parsing + the same source); + + - for uncached nodes parse the file and process all imports found, + taking care of (a) above. + +Step 3 should also address (d) above. + +Once step 3 terminates we have the entire dependency graph, and for +each module we've either loaded the cache metadata or parsed the +source code. (However, we may still need to parse those modules for +which we have cache metadata but that depend, directly or indirectly, +on at least one module for which the cache metadata is stale.) + +Now we can execute steps A-C from the first section. Finding SCCs for +step A shouldn't be hard; there's a recipe here: +http://code.activestate.com/recipes/578507/. There's also a plethora +of topsort recipes, e.g. http://code.activestate.com/recipes/577413/. + +For single nodes, processing is simple. If the node was cached, we +deserialize the cache data and fix up cross-references. Otherwise, we +do semantic analysis followed by type checking. We also handle (c) +above; if a module has valid cache data *but* any of its +dependencies was processed from source, then the module should be +processed from source. + +A relatively simple optimization (outside SCCs) we might do in the +future is as follows: if a node's cache data is valid, but one or more +of its dependencies are out of date so we have to re-parse the node +from source, once we have fully type-checked the node, we can decide +whether its symbol table actually changed compared to the cache data +(by reading the cache data and comparing it to the data we would be +writing). If there is no change we can declare the node up to date, +and any node that depends (and for which we have cached data, and +whose other dependencies are up to date) on it won't need to be +re-parsed from source. + +Import cycles +------------- + +Finally we have to decide how to handle (c), import cycles. Here +we'll need a modified version of the original state machine +(build.py), but we only need to do this per SCC, and we won't have to +deal with changes to the list of nodes while we're processing it. + +If all nodes in the SCC have valid cache metadata and all dependencies +outside the SCC are still valid, we can proceed as follows: + + 1. Load cache data for all nodes in the SCC. + + 2. Fix up cross-references for all nodes in the SCC. + +Otherwise, the simplest (but potentially slow) way to proceed is to +invalidate all cache data in the SCC and re-parse all nodes in the SCC +from source. We can do this as follows: + + 1. Parse source for all nodes in the SCC. + + 2. Semantic analysis for all nodes in the SCC. + + 3. Type check all nodes in the SCC. + +(If there are more passes the process is the same -- each pass should +be done for all nodes before starting the next pass for any nodes in +the SCC.) + +We could process the nodes in the SCC in any order. For sentimental +reasons, I've decided to process them in the reverse order in which we +encountered them when originally constructing the graph. That's how +the old build.py deals with cycles, and at least this reproduces the +previous implementation more accurately. + +Can we do better than re-parsing all nodes in the SCC when any of its +dependencies are out of date? It's doubtful. The optimization +mentioned at the end of the previous section would require re-parsing +and type-checking a node and then comparing its symbol table to the +cached data; but because the node is part of a cycle we can't +technically type-check it until the semantic analysis of all other +nodes in the cycle has completed. (This is an important issue because +Dropbox has a very large cycle in production code. But I'd like to +deal with it later.) + +Additional wrinkles +------------------- + +During implementation more wrinkles were found. + +- When a submodule of a package (e.g. x.y) is encountered, the parent + package (e.g. x) must also be loaded, but it is not strictly a + dependency. See State.add_ancestors() below. +""" + + +class ModuleNotFound(Exception): + """Control flow exception to signal that a module was not found.""" + + +class State: + """The state for a module. + + The source is only used for the -c command line option; in that + case path is None. Otherwise source is None and path isn't. + """ + + manager = None # type: BuildManager + order_counter = 0 # Class variable + order = None # type: int # Order in which modules were encountered + id = None # type: str # Fully qualified module name + path = None # type: Optional[str] # Path to module source + xpath = None # type: str # Path or '' + source = None # type: Optional[str] # Module source code + meta = None # type: Optional[CacheMeta] + data = None # type: Optional[str] + tree = None # type: Optional[MypyFile] + dependencies = None # type: List[str] + suppressed = None # type: List[str] # Suppressed/missing dependencies + priorities = None # type: Dict[str, int] + + # Map each dependency to the line number where it is first imported + dep_line_map = None # type: Dict[str, int] + + # Parent package, its parent, etc. + ancestors = None # type: Optional[List[str]] + + # List of (path, line number) tuples giving context for import + import_context = None # type: List[Tuple[str, int]] + + # The State from which this module was imported, if any + caller_state = None # type: Optional[State] + + # If caller_state is set, the line number in the caller where the import occurred + caller_line = 0 + + def __init__(self, + id: Optional[str], + path: Optional[str], + source: Optional[str], + manager: BuildManager, + caller_state: 'State' = None, + caller_line: int = 0, + ancestor_for: 'State' = None, + ) -> None: + assert id or path or source is not None, "Neither id, path nor source given" + self.manager = manager + State.order_counter += 1 + self.order = State.order_counter + self.caller_state = caller_state + self.caller_line = caller_line + if caller_state: + self.import_context = caller_state.import_context[:] + self.import_context.append((caller_state.xpath, caller_line)) + else: + self.import_context = [] + self.id = id or '__main__' + if not path and source is None: + file_id = id + if id == 'builtins' and manager.options.python_version[0] == 2: + # The __builtin__ module is called internally by mypy + # 'builtins' in Python 2 mode (similar to Python 3), + # but the stub file is __builtin__.pyi. The reason is + # that a lot of code hard-codes 'builtins.x' and it's + # easier to work it around like this. It also means + # that the implementation can mostly ignore the + # difference and just assume 'builtins' everywhere, + # which simplifies code. + file_id = '__builtin__' + path = find_module(file_id, manager.lib_path) + if path: + # In silent mode, don't import .py files, except from stubs. + if (manager.options.silent_imports and + path.endswith('.py') and (caller_state or ancestor_for)): + # (Never silence builtins, even if it's a .py file; + # this can happen in tests!) + if (id != 'builtins' and + not ((caller_state and + caller_state.tree and + caller_state.tree.is_stub))): + if manager.options.almost_silent: + if ancestor_for: + self.skipping_ancestor(id, path, ancestor_for) + else: + self.skipping_module(id, path) + path = None + manager.missing_modules.add(id) + raise ModuleNotFound + else: + # Could not find a module. Typically the reason is a + # misspelled module name, missing stub, module not in + # search path or the module has not been installed. + if caller_state: + suppress_message = ((manager.options.silent_imports and + not manager.options.almost_silent) or + (caller_state.tree is not None and + 'import' in caller_state.tree.weak_opts)) + if not suppress_message: + save_import_context = manager.errors.import_context() + manager.errors.set_import_context(caller_state.import_context) + manager.module_not_found(caller_state.xpath, caller_line, id) + manager.errors.set_import_context(save_import_context) + manager.missing_modules.add(id) + raise ModuleNotFound + else: + # If we can't find a root source it's always fatal. + # TODO: This might hide non-fatal errors from + # root sources processed earlier. + raise CompileError(["mypy: can't find module '%s'" % id]) + self.path = path + self.xpath = path or '' + self.source = source + if path and source is None and manager.options.incremental: + self.meta = find_cache_meta(self.id, self.path, manager) + # TODO: Get mtime if not cached. + self.add_ancestors() + if self.meta: + # Make copies, since we may modify these and want to + # compare them to the originals later. + self.dependencies = list(self.meta.dependencies) + self.suppressed = list(self.meta.suppressed) + assert len(self.meta.dependencies) == len(self.meta.dep_prios) + self.priorities = {id: pri + for id, pri in zip(self.meta.dependencies, self.meta.dep_prios)} + self.dep_line_map = {} + else: + # Parse the file (and then some) to get the dependencies. + self.parse_file() + self.suppressed = [] + + def skipping_ancestor(self, id: str, path: str, ancestor_for: 'State') -> None: + # TODO: Read the path (the __init__.py file) and return + # immediately if it's empty or only contains comments. + # But beware, some package may be the ancestor of many modules, + # so we'd need to cache the decision. + manager = self.manager + manager.errors.set_import_context([]) + manager.errors.set_file(ancestor_for.xpath) + manager.errors.report(-1, "Ancestor package '%s' silently ignored" % (id,), + severity='note', only_once=True) + manager.errors.report(-1, "(Using --silent-imports, submodule passed on command line)", + severity='note', only_once=True) + manager.errors.report(-1, "(This note brought to you by --almost-silent)", + severity='note', only_once=True) + + def skipping_module(self, id: str, path: str) -> None: + assert self.caller_state, (id, path) + manager = self.manager + save_import_context = manager.errors.import_context() + manager.errors.set_import_context(self.caller_state.import_context) + manager.errors.set_file(self.caller_state.xpath) + line = self.caller_line + manager.errors.report(line, "Import of '%s' silently ignored" % (id,), + severity='note') + manager.errors.report(line, "(Using --silent-imports, module not passed on command line)", + severity='note', only_once=True) + manager.errors.report(line, "(This note courtesy of --almost-silent)", + severity='note', only_once=True) + manager.errors.set_import_context(save_import_context) + + def add_ancestors(self) -> None: + # All parent packages are new ancestors. + ancestors = [] + parent = self.id + while '.' in parent: + parent, _ = parent.rsplit('.', 1) + ancestors.append(parent) + self.ancestors = ancestors + + def is_fresh(self) -> bool: + """Return whether the cache data for this file is fresh.""" + # NOTE: self.dependencies may differ from + # self.meta.dependencies when a dependency is dropped due to + # suppression by --silent-imports. However when a suppressed + # dependency is added back we find out later in the process. + return self.meta is not None and self.dependencies == self.meta.dependencies + + def mark_stale(self) -> None: + """Throw away the cache data for this file, marking it as stale.""" + self.meta = None + + def check_blockers(self) -> None: + """Raise CompileError if a blocking error is detected.""" + if self.manager.errors.is_blockers(): + self.manager.log("Bailing due to blocking errors") + self.manager.errors.raise_error() + + @contextlib.contextmanager + def wrap_context(self) -> Iterator[None]: + save_import_context = self.manager.errors.import_context() + self.manager.errors.set_import_context(self.import_context) + try: + yield + except CompileError: + raise + except Exception as err: + report_internal_error(err, self.path, 0) + self.manager.errors.set_import_context(save_import_context) + self.check_blockers() + + # Methods for processing cached modules. + + def load_tree(self) -> None: + with open(self.meta.data_json) as f: + data = json.load(f) + # TODO: Assert data file wasn't changed. + self.tree = MypyFile.deserialize(data) + self.manager.modules[self.id] = self.tree + + def fix_cross_refs(self) -> None: + fixup_module_pass_one(self.tree, self.manager.modules) + + def calculate_mros(self) -> None: + fixup_module_pass_two(self.tree, self.manager.modules) + + # Methods for processing modules from source code. + + def parse_file(self) -> None: + if self.tree is not None: + # The file was already parsed (in __init__()). + return + + manager = self.manager + modules = manager.modules + manager.log("Parsing %s (%s)" % (self.xpath, self.id)) + + with self.wrap_context(): + source = self.source + self.source = None # We won't need it again. + if self.path and source is None: + try: + source = read_with_python_encoding(self.path, manager.options.python_version) + except IOError as ioerr: + raise CompileError([ + "mypy: can't read file '{}': {}".format(self.path, ioerr.strerror)]) + except (UnicodeDecodeError, DecodeError) as decodeerr: + raise CompileError([ + "mypy: can't decode file '{}': {}".format(self.path, str(decodeerr))]) + self.tree = manager.parse_file(self.id, self.xpath, source) + + modules[self.id] = self.tree + + # Do the first pass of semantic analysis: add top-level + # definitions in the file to the symbol table. We must do + # this before processing imports, since this may mark some + # import statements as unreachable. + first = FirstPass(manager.semantic_analyzer) + first.analyze(self.tree, self.xpath, self.id) + + # Initialize module symbol table, which was populated by the + # semantic analyzer. + # TODO: Why can't FirstPass .analyze() do this? + self.tree.names = manager.semantic_analyzer.globals + + # Compute (direct) dependencies. + # Add all direct imports (this is why we needed the first pass). + # Also keep track of each dependency's source line. + dependencies = [] + suppressed = [] + priorities = {} # type: Dict[str, int] # id -> priority + dep_line_map = {} # type: Dict[str, int] # id -> line + for pri, id, line in manager.all_imported_modules_in_file(self.tree): + priorities[id] = min(pri, priorities.get(id, PRI_ALL)) + if id == self.id: + continue + # Omit missing modules, as otherwise we could not type-check + # programs with missing modules. + if id in manager.missing_modules: + if id not in dep_line_map: + suppressed.append(id) + dep_line_map[id] = line + continue + if id == '': + # Must be from a relative import. + manager.errors.set_file(self.xpath) + manager.errors.report(line, "No parent module -- cannot perform relative import", + blocker=True) + continue + if id not in dep_line_map: + dependencies.append(id) + dep_line_map[id] = line + # Every module implicitly depends on builtins. + if self.id != 'builtins' and 'builtins' not in dep_line_map: + dependencies.append('builtins') + + # If self.dependencies is already set, it was read from the + # cache, but for some reason we're re-parsing the file. + # NOTE: What to do about race conditions (like editing the + # file while mypy runs)? A previous version of this code + # explicitly checked for this, but ran afoul of other reasons + # for differences (e.g. --silent-imports). + self.dependencies = dependencies + self.suppressed = suppressed + self.priorities = priorities + self.dep_line_map = dep_line_map + self.check_blockers() + + def patch_parent(self) -> None: + # Include module in the symbol table of the enclosing package. + if '.' not in self.id: + return + manager = self.manager + modules = manager.modules + parent, child = self.id.rsplit('.', 1) + if parent in modules: + manager.trace("Added %s.%s" % (parent, child)) + modules[parent].names[child] = SymbolTableNode(MODULE_REF, self.tree, parent) + else: + manager.log("Hm... couldn't add %s.%s" % (parent, child)) + + def semantic_analysis(self) -> None: + with self.wrap_context(): + self.manager.semantic_analyzer.visit_file(self.tree, self.xpath) + + def semantic_analysis_pass_three(self) -> None: + with self.wrap_context(): + self.manager.semantic_analyzer_pass3.visit_file(self.tree, self.xpath) + if self.manager.options.dump_type_stats: + dump_type_stats(self.tree, self.xpath) + + def type_check(self) -> None: + manager = self.manager + if manager.options.semantic_analysis_only: + return + with self.wrap_context(): + manager.type_checker.visit_file(self.tree, self.xpath) + if manager.options.dump_inference_stats: + dump_type_stats(self.tree, self.xpath, inferred=True, + typemap=manager.type_checker.type_map) + manager.report_file(self.tree) + + def write_cache(self) -> None: + if self.path and self.manager.options.incremental and not self.manager.errors.is_errors(): + dep_prios = [self.priorities.get(dep, PRI_HIGH) for dep in self.dependencies] + write_cache(self.id, self.path, self.tree, + list(self.dependencies), list(self.suppressed), + dep_prios, + self.manager) + + +Graph = Dict[str, State] + + +def dispatch(sources: List[BuildSource], manager: BuildManager) -> None: + manager.log("Using new dependency manager") + graph = load_graph(sources, manager) + manager.log("Loaded graph with %d nodes" % len(graph)) + process_graph(graph, manager) + if manager.options.warn_unused_ignores: + manager.errors.generate_unused_ignore_notes() + + +def load_graph(sources: List[BuildSource], manager: BuildManager) -> Graph: + """Given some source files, load the full dependency graph.""" + graph = {} # type: Graph + # The deque is used to implement breadth-first traversal. + # TODO: Consider whether to go depth-first instead. This may + # affect the order in which we process files within import cycles. + new = collections.deque() # type: collections.deque[State] + # Seed the graph with the initial root sources. + for bs in sources: + try: + st = State(id=bs.module, path=bs.path, source=bs.text, manager=manager) + except ModuleNotFound: + continue + if st.id in graph: + manager.errors.set_file(st.xpath) + manager.errors.report(1, "Duplicate module named '%s'" % st.id) + manager.errors.raise_error() + graph[st.id] = st + new.append(st) + # Collect dependencies. We go breadth-first. + while new: + st = new.popleft() + for dep in st.ancestors + st.dependencies: + if dep not in graph: + try: + if dep in st.ancestors: + # TODO: Why not 'if dep not in st.dependencies' ? + # Ancestors don't have import context. + newst = State(id=dep, path=None, source=None, manager=manager, + ancestor_for=st) + else: + newst = State(id=dep, path=None, source=None, manager=manager, + caller_state=st, caller_line=st.dep_line_map.get(dep, 1)) + except ModuleNotFound: + if dep in st.dependencies: + st.dependencies.remove(dep) + st.suppressed.append(dep) + else: + assert newst.id not in graph, newst.id + graph[newst.id] = newst + new.append(newst) + return graph + + +def process_graph(graph: Graph, manager: BuildManager) -> None: + """Process everything in dependency order.""" + sccs = sorted_components(graph) + manager.log("Found %d SCCs; largest has %d nodes" % + (len(sccs), max(len(scc) for scc in sccs))) + # We're processing SCCs from leaves (those without further + # dependencies) to roots (those from which everything else can be + # reached). + for ascc in sccs: + # Order the SCC's nodes using a heuristic. + # Note that ascc is a set, and scc is a list. + scc = order_ascc(graph, ascc) + # If builtins is in the list, move it last. (This is a bit of + # a hack, but it's necessary because the builtins module is + # part of a small cycle involving at least {builtins, abc, + # typing}. Of these, builtins must be processed last or else + # some builtin objects will be incompletely processed.) + if 'builtins' in ascc: + scc.remove('builtins') + scc.append('builtins') + if manager.options.verbosity >= 2: + for id in scc: + manager.trace("Priorities for %s:" % id, + " ".join("%s:%d" % (x, graph[id].priorities[x]) + for x in graph[id].dependencies + if x in ascc and x in graph[id].priorities)) + # Because the SCCs are presented in topological sort order, we + # don't need to look at dependencies recursively for staleness + # -- the immediate dependencies are sufficient. + stale_scc = {id for id in scc if not graph[id].is_fresh()} + fresh = not stale_scc + deps = set() + for id in scc: + deps.update(graph[id].dependencies) + deps -= ascc + stale_deps = {id for id in deps if not graph[id].is_fresh()} + fresh = fresh and not stale_deps + undeps = set() + if fresh: + # Check if any dependencies that were suppressed according + # to the cache have heen added back in this run. + # NOTE: Newly suppressed dependencies are handled by is_fresh(). + for id in scc: + undeps.update(graph[id].suppressed) + undeps &= graph.keys() + if undeps: + fresh = False + if fresh: + # All cache files are fresh. Check that no dependency's + # cache file is newer than any scc node's cache file. + oldest_in_scc = min(graph[id].meta.data_mtime for id in scc) + newest_in_deps = 0 if not deps else max(graph[dep].meta.data_mtime for dep in deps) + if manager.options.verbosity >= 3: # Dump all mtimes for extreme debugging. + all_ids = sorted(ascc | deps, key=lambda id: graph[id].meta.data_mtime) + for id in all_ids: + if id in scc: + if graph[id].meta.data_mtime < newest_in_deps: + key = "*id:" + else: + key = "id:" + else: + if graph[id].meta.data_mtime > oldest_in_scc: + key = "+dep:" + else: + key = "dep:" + manager.trace(" %5s %.0f %s" % (key, graph[id].meta.data_mtime, id)) + # If equal, give the benefit of the doubt, due to 1-sec time granularity + # (on some platforms). + if oldest_in_scc < newest_in_deps: + fresh = False + fresh_msg = "out of date by %.0f seconds" % (newest_in_deps - oldest_in_scc) + else: + fresh_msg = "fresh" + elif undeps: + fresh_msg = "stale due to changed suppression (%s)" % " ".join(sorted(undeps)) + elif stale_scc: + fresh_msg = "inherently stale (%s)" % " ".join(sorted(stale_scc)) + if stale_deps: + fresh_msg += " with stale deps (%s)" % " ".join(sorted(stale_deps)) + else: + fresh_msg = "stale due to deps (%s)" % " ".join(sorted(stale_deps)) + if len(scc) == 1: + manager.log("Processing SCC singleton (%s) as %s" % (" ".join(scc), fresh_msg)) + else: + manager.log("Processing SCC of size %d (%s) as %s" % + (len(scc), " ".join(scc), fresh_msg)) + if fresh: + process_fresh_scc(graph, scc) + else: + process_stale_scc(graph, scc) + + +def order_ascc(graph: Graph, ascc: AbstractSet[str], pri_max: int = PRI_ALL) -> List[str]: + """Come up with the ideal processing order within an SCC. + + Using the priorities assigned by all_imported_modules_in_file(), + try to reduce the cycle to a DAG, by omitting arcs representing + dependencies of lower priority. + + In the simplest case, if we have A <--> B where A has a top-level + "import B" (medium priority) but B only has the reverse "import A" + inside a function (low priority), we turn the cycle into a DAG by + dropping the B --> A arc, which leaves only A --> B. + + If all arcs have the same priority, we fall back to sorting by + reverse global order (the order in which modules were first + encountered). + + The algorithm is recursive, as follows: when as arcs of different + priorities are present, drop all arcs of the lowest priority, + identify SCCs in the resulting graph, and apply the algorithm to + each SCC thus found. The recursion is bounded because at each + recursion the spread in priorities is (at least) one less. + + In practice there are only a few priority levels (currently + N=3) and in the worst case we just carry out the same algorithm + for finding SCCs N times. Thus the complexity is no worse than + the complexity of the original SCC-finding algorithm -- see + strongly_connected_components() below for a reference. + """ + if len(ascc) == 1: + return [s for s in ascc] + pri_spread = set() + for id in ascc: + state = graph[id] + for dep in state.dependencies: + if dep in ascc: + pri = state.priorities.get(dep, PRI_HIGH) + if pri < pri_max: + pri_spread.add(pri) + if len(pri_spread) == 1: + # Filtered dependencies are uniform -- order by global order. + return sorted(ascc, key=lambda id: -graph[id].order) + pri_max = max(pri_spread) + sccs = sorted_components(graph, ascc, pri_max) + # The recursion is bounded by the len(pri_spread) check above. + return [s for ss in sccs for s in order_ascc(graph, ss, pri_max)] + + +def process_fresh_scc(graph: Graph, scc: List[str]) -> None: + """Process the modules in one SCC from their cached data.""" + for id in scc: + graph[id].load_tree() + for id in scc: + graph[id].patch_parent() + for id in scc: + graph[id].fix_cross_refs() + for id in scc: + graph[id].calculate_mros() + + +def process_stale_scc(graph: Graph, scc: List[str]) -> None: + """Process the modules in one SCC from source code.""" + for id in scc: + graph[id].mark_stale() + for id in scc: + # We may already have parsed the module, or not. + # If the former, parse_file() is a no-op. + graph[id].parse_file() + for id in scc: + graph[id].patch_parent() + for id in scc: + graph[id].semantic_analysis() + for id in scc: + graph[id].semantic_analysis_pass_three() + for id in scc: + graph[id].type_check() + graph[id].write_cache() + + +def sorted_components(graph: Graph, + vertices: Optional[AbstractSet[str]] = None, + pri_max: int = PRI_ALL) -> List[AbstractSet[str]]: + """Return the graph's SCCs, topologically sorted by dependencies. + + The sort order is from leaves (nodes without dependencies) to + roots (nodes on which no other nodes depend). + + This works for a subset of the full dependency graph too; + dependencies that aren't present in graph.keys() are ignored. + """ + # Compute SCCs. + if vertices is None: + vertices = set(graph) + edges = {id: deps_filtered(graph, vertices, id, pri_max) for id in vertices} + sccs = list(strongly_connected_components(vertices, edges)) + # Topsort. + sccsmap = {id: frozenset(scc) for scc in sccs for id in scc} + data = {} # type: Dict[AbstractSet[str], Set[AbstractSet[str]]] + for scc in sccs: + deps = set() # type: Set[AbstractSet[str]] + for id in scc: + deps.update(sccsmap[x] for x in deps_filtered(graph, vertices, id, pri_max)) + data[frozenset(scc)] = deps + res = [] + for ready in topsort(data): + # Sort the sets in ready by reversed smallest State.order. Examples: + # + # - If ready is [{x}, {y}], x.order == 1, y.order == 2, we get + # [{y}, {x}]. + # + # - If ready is [{a, b}, {c, d}], a.order == 1, b.order == 3, + # c.order == 2, d.order == 4, the sort keys become [1, 2] + # and the result is [{c, d}, {a, b}]. + res.extend(sorted(ready, + key=lambda scc: -min(graph[id].order for id in scc))) + return res + + +def deps_filtered(graph: Graph, vertices: AbstractSet[str], id: str, pri_max: int) -> List[str]: + """Filter dependencies for id with pri < pri_max.""" + if id not in vertices: + return [] + state = graph[id] + return [dep + for dep in state.dependencies + if dep in vertices and state.priorities.get(dep, PRI_HIGH) < pri_max] + + +def strongly_connected_components(vertices: AbstractSet[str], + edges: Dict[str, List[str]]) -> Iterator[Set[str]]: + """Compute Strongly Connected Components of a directed graph. + + Args: + vertices: the labels for the vertices + edges: for each vertex, gives the target vertices of its outgoing edges + + Returns: + An iterator yielding strongly connected components, each + represented as a set of vertices. Each input vertex will occur + exactly once; vertices not part of a SCC are returned as + singleton sets. + + From http://code.activestate.com/recipes/578507/. + """ + identified = set() # type: Set[str] + stack = [] # type: List[str] + index = {} # type: Dict[str, int] + boundaries = [] # type: List[int] + + def dfs(v: str) -> Iterator[Set[str]]: + index[v] = len(stack) + stack.append(v) + boundaries.append(index[v]) + + for w in edges[v]: + if w not in index: + # For Python >= 3.3, replace with "yield from dfs(w)" + for scc in dfs(w): + yield scc + elif w not in identified: + while index[w] < boundaries[-1]: + boundaries.pop() + + if boundaries[-1] == index[v]: + boundaries.pop() + scc = set(stack[index[v]:]) + del stack[index[v]:] + identified.update(scc) + yield scc + + for v in vertices: + if v not in index: + # For Python >= 3.3, replace with "yield from dfs(v)" + for scc in dfs(v): + yield scc + + +def topsort(data: Dict[AbstractSet[str], + Set[AbstractSet[str]]]) -> Iterable[Set[AbstractSet[str]]]: + """Topological sort. + + Args: + data: A map from SCCs (represented as frozen sets of strings) to + sets of SCCs, its dependencies. NOTE: This data structure + is modified in place -- for normalization purposes, + self-dependencies are removed and entries representing + orphans are added. + + Returns: + An iterator yielding sets of SCCs that have an equivalent + ordering. NOTE: The algorithm doesn't care about the internal + structure of SCCs. + + Example: + Suppose the input has the following structure: + + {A: {B, C}, B: {D}, C: {D}} + + This is normalized to: + + {A: {B, C}, B: {D}, C: {D}, D: {}} + + The algorithm will yield the following values: + + {D} + {B, C} + {A} + + From http://code.activestate.com/recipes/577413/. + """ + # TODO: Use a faster algorithm? + for k, v in data.items(): + v.discard(k) # Ignore self dependencies. + for item in set.union(*data.values()) - set(data.keys()): + data[item] = set() + while True: + ready = {item for item, dep in data.items() if not dep} + if not ready: + break + yield ready + data = {item: (dep - ready) + for item, dep in data.items() + if item not in ready} + assert not data, "A cyclic dependency exists amongst %r" % data diff --git a/mypy/checker.py b/mypy/checker.py index 7063758096dfa..62f4c250ea272 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -1,2459 +1,2459 @@ -"""Mypy type checker.""" - -import itertools -import contextlib -import os -import os.path - -from typing import ( - Any, Dict, Set, List, cast, Tuple, TypeVar, Union, Optional, NamedTuple -) - -from mypy.errors import Errors, report_internal_error -from mypy.nodes import ( - SymbolTable, Node, MypyFile, Var, - OverloadedFuncDef, FuncDef, FuncItem, FuncBase, TypeInfo, - ClassDef, GDEF, Block, AssignmentStmt, NameExpr, MemberExpr, IndexExpr, - TupleExpr, ListExpr, ExpressionStmt, ReturnStmt, IfStmt, - WhileStmt, OperatorAssignmentStmt, WithStmt, AssertStmt, - RaiseStmt, TryStmt, ForStmt, DelStmt, CallExpr, IntExpr, StrExpr, - BytesExpr, UnicodeExpr, FloatExpr, OpExpr, UnaryExpr, CastExpr, RevealTypeExpr, SuperExpr, - TypeApplication, DictExpr, SliceExpr, FuncExpr, TempNode, SymbolTableNode, - Context, ListComprehension, ConditionalExpr, GeneratorExpr, - Decorator, SetExpr, TypeVarExpr, PrintStmt, - LITERAL_TYPE, BreakStmt, ContinueStmt, ComparisonExpr, StarExpr, - YieldFromExpr, NamedTupleExpr, SetComprehension, - DictionaryComprehension, ComplexExpr, EllipsisExpr, TypeAliasExpr, - RefExpr, YieldExpr, BackquoteExpr, ImportFrom, ImportAll, ImportBase, - CONTRAVARIANT, COVARIANT -) -from mypy.nodes import function_type, method_type, method_type_with_fallback -from mypy import nodes -from mypy.types import ( - Type, AnyType, CallableType, Void, FunctionLike, Overloaded, TupleType, - Instance, NoneTyp, ErrorType, strip_type, - UnionType, TypeVarId, TypeVarType, PartialType, DeletedType, UninhabitedType -) -from mypy.sametypes import is_same_type -from mypy.messages import MessageBuilder -import mypy.checkexpr -from mypy.checkmember import map_type_from_supertype -from mypy import defaults -from mypy import messages -from mypy.subtypes import ( - is_subtype, is_equivalent, is_proper_subtype, - is_more_precise, restrict_subtype_away -) -from mypy.maptype import map_instance_to_supertype -from mypy.semanal import self_type, set_callable_name, refers_to_fullname -from mypy.erasetype import erase_typevars -from mypy.expandtype import expand_type_by_instance, expand_type -from mypy.visitor import NodeVisitor -from mypy.join import join_types -from mypy.treetransform import TransformVisitor -from mypy.meet import meet_simple, nearest_builtin_ancestor, is_overlapping_types -from mypy.binder import ConditionalTypeBinder -from mypy.options import Options - -from mypy import experiments - - -T = TypeVar('T') - - -# A node which is postponed to be type checked during the next pass. -DeferredNode = NamedTuple( - 'DeferredNode', - [ - ('node', Node), - ('context_type_name', Optional[str]), # Name of the surrounding class (for error messages) - ]) - - -class TypeChecker(NodeVisitor[Type]): - """Mypy type checker. - - Type check mypy source files that have been semantically analyzed. - """ - - # Are we type checking a stub? - is_stub = False - # Error message reporter - errors = None # type: Errors - # Utility for generating messages - msg = None # type: MessageBuilder - # Types of type checked nodes - type_map = None # type: Dict[Node, Type] - - # Helper for managing conditional types - binder = None # type: ConditionalTypeBinder - # Helper for type checking expressions - expr_checker = None # type: mypy.checkexpr.ExpressionChecker - - # Stack of function return types - return_types = None # type: List[Type] - # Type context for type inference - type_context = None # type: List[Type] - # Flags; true for dynamically typed functions - dynamic_funcs = None # type: List[bool] - # Stack of functions being type checked - function_stack = None # type: List[FuncItem] - # Do weak type checking in this file - weak_opts = set() # type: Set[str] - # Stack of collections of variables with partial types - partial_types = None # type: List[Dict[Var, Context]] - globals = None # type: SymbolTable - modules = None # type: Dict[str, MypyFile] - # Nodes that couldn't be checked because some types weren't available. We'll run - # another pass and try these again. - deferred_nodes = None # type: List[DeferredNode] - # Type checking pass number (0 = first pass) - pass_num = 0 - # Have we deferred the current function? If yes, don't infer additional - # types during this pass within the function. - current_node_deferred = False - is_typeshed_stub = False - options = None # type: Options - - def __init__(self, errors: Errors, modules: Dict[str, MypyFile], options: Options) -> None: - """Construct a type checker. - - Use errors to report type check errors. - """ - self.errors = errors - self.modules = modules - self.options = options - self.msg = MessageBuilder(errors, modules) - self.type_map = {} - self.binder = ConditionalTypeBinder() - self.expr_checker = mypy.checkexpr.ExpressionChecker(self, self.msg) - self.return_types = [] - self.type_context = [] - self.dynamic_funcs = [] - self.function_stack = [] - self.weak_opts = set() # type: Set[str] - self.partial_types = [] - self.deferred_nodes = [] - self.pass_num = 0 - self.current_node_deferred = False - - def visit_file(self, file_node: MypyFile, path: str) -> None: - """Type check a mypy file with the given path.""" - self.pass_num = 0 - self.is_stub = file_node.is_stub - self.errors.set_file(path) - self.globals = file_node.names - self.weak_opts = file_node.weak_opts - self.enter_partial_types() - # gross, but no other clear way to tell - self.is_typeshed_stub = self.is_stub and 'typeshed' in os.path.normpath(path).split(os.sep) - - for d in file_node.defs: - self.accept(d) - - self.leave_partial_types() - - if self.deferred_nodes: - self.check_second_pass() - - self.current_node_deferred = False - - all_ = self.globals.get('__all__') - if all_ is not None and all_.type is not None: - seq_str = self.named_generic_type('typing.Sequence', - [self.named_type('builtins.str')]) - if not is_subtype(all_.type, seq_str): - str_seq_s, all_s = self.msg.format_distinctly(seq_str, all_.type) - self.fail(messages.ALL_MUST_BE_SEQ_STR.format(str_seq_s, all_s), - all_.node) - - def check_second_pass(self): - """Run second pass of type checking which goes through deferred nodes.""" - self.pass_num = 1 - for node, type_name in self.deferred_nodes: - if type_name: - self.errors.push_type(type_name) - self.accept(node) - if type_name: - self.errors.pop_type() - self.deferred_nodes = [] - - def handle_cannot_determine_type(self, name: str, context: Context) -> None: - if self.pass_num == 0 and self.function_stack: - # Don't report an error yet. Just defer. - node = self.function_stack[-1] - if self.errors.type_name: - type_name = self.errors.type_name[-1] - else: - type_name = None - self.deferred_nodes.append(DeferredNode(node, type_name)) - # Set a marker so that we won't infer additional types in this - # function. Any inferred types could be bogus, because there's at - # least one type that we don't know. - self.current_node_deferred = True - else: - self.msg.cannot_determine_type(name, context) - - def accept(self, node: Node, type_context: Type = None) -> Type: - """Type check a node in the given type context.""" - self.type_context.append(type_context) - try: - typ = node.accept(self) - except Exception as err: - report_internal_error(err, self.errors.file, node.line) - self.type_context.pop() - self.store_type(node, typ) - if self.typing_mode_none(): - return AnyType() - else: - return typ - - def accept_loop(self, body: Node, else_body: Node = None) -> Type: - """Repeatedly type check a loop body until the frame doesn't change. - - Then check the else_body. - """ - # The outer frame accumulates the results of all iterations - with self.binder.frame_context(1) as outer_frame: - self.binder.push_loop_frame() - while True: - with self.binder.frame_context(1): - # We may skip each iteration - self.binder.options_on_return[-1].append(outer_frame) - self.accept(body) - if not self.binder.last_pop_changed: - break - self.binder.pop_loop_frame() - if else_body: - self.accept(else_body) - - # - # Definitions - # - - def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> Type: - num_abstract = 0 - if defn.is_property: - # HACK: Infer the type of the property. - self.visit_decorator(defn.items[0]) - for fdef in defn.items: - self.check_func_item(fdef.func, name=fdef.func.name()) - if fdef.func.is_abstract: - num_abstract += 1 - if num_abstract not in (0, len(defn.items)): - self.fail(messages.INCONSISTENT_ABSTRACT_OVERLOAD, defn) - if defn.info: - self.check_method_override(defn) - self.check_inplace_operator_method(defn) - self.check_overlapping_overloads(defn) - - def check_overlapping_overloads(self, defn: OverloadedFuncDef) -> None: - for i, item in enumerate(defn.items): - for j, item2 in enumerate(defn.items[i + 1:]): - # TODO overloads involving decorators - sig1 = self.function_type(item.func) - sig2 = self.function_type(item2.func) - if is_unsafe_overlapping_signatures(sig1, sig2): - self.msg.overloaded_signatures_overlap(i + 1, i + j + 2, - item.func) - - def is_generator_return_type(self, typ: Type) -> bool: - return is_subtype(self.named_generic_type('typing.Generator', - [AnyType(), AnyType(), AnyType()]), - typ) - - def get_generator_yield_type(self, return_type: Type) -> Type: - if isinstance(return_type, AnyType): - return AnyType() - elif not self.is_generator_return_type(return_type): - # If the function doesn't have a proper Generator (or superclass) return type, anything - # is permissible. - return AnyType() - elif not isinstance(return_type, Instance): - # Same as above, but written as a separate branch so the typechecker can understand. - return AnyType() - elif return_type.args: - return return_type.args[0] - else: - # If the function's declared supertype of Generator has no type - # parameters (i.e. is `object`), then the yielded values can't - # be accessed so any type is acceptable. - return AnyType() - - def get_generator_receive_type(self, return_type: Type) -> Type: - if isinstance(return_type, AnyType): - return AnyType() - elif not self.is_generator_return_type(return_type): - # If the function doesn't have a proper Generator (or superclass) return type, anything - # is permissible. - return AnyType() - elif not isinstance(return_type, Instance): - # Same as above, but written as a separate branch so the typechecker can understand. - return AnyType() - elif return_type.type.fullname() == 'typing.Generator': - # Generator is the only type which specifies the type of values it can receive. - return return_type.args[1] - else: - # `return_type` is a supertype of Generator, so callers won't be able to send it - # values. - return Void() - - def get_generator_return_type(self, return_type: Type) -> Type: - if isinstance(return_type, AnyType): - return AnyType() - elif not self.is_generator_return_type(return_type): - # If the function doesn't have a proper Generator (or superclass) return type, anything - # is permissible. - return AnyType() - elif not isinstance(return_type, Instance): - # Same as above, but written as a separate branch so the typechecker can understand. - return AnyType() - elif return_type.type.fullname() == 'typing.Generator': - # Generator is the only type which specifies the type of values it returns into - # `yield from` expressions. - return return_type.args[2] - else: - # `return_type` is supertype of Generator, so callers won't be able to see the return - # type when used in a `yield from` expression. - return AnyType() - - def visit_func_def(self, defn: FuncDef) -> Type: - """Type check a function definition.""" - self.check_func_item(defn, name=defn.name()) - if defn.info: - if not defn.is_dynamic(): - self.check_method_override(defn) - self.check_inplace_operator_method(defn) - if defn.original_def: - # Override previous definition. - new_type = self.function_type(defn) - if isinstance(defn.original_def, FuncDef): - # Function definition overrides function definition. - if not is_same_type(new_type, self.function_type(defn.original_def)): - self.msg.incompatible_conditional_function_def(defn) - else: - # Function definition overrides a variable initialized via assignment. - orig_type = defn.original_def.type - if orig_type is None: - # XXX This can be None, as happens in - # test_testcheck_TypeCheckSuite.testRedefinedFunctionInTryWithElse - self.msg.note("Internal mypy error checking function redefinition.", defn) - return None - if isinstance(orig_type, PartialType): - if orig_type.type is None: - # Ah this is a partial type. Give it the type of the function. - var = defn.original_def - partial_types = self.find_partial_types(var) - if partial_types is not None: - var.type = new_type - del partial_types[var] - else: - # Trying to redefine something like partial empty list as function. - self.fail(messages.INCOMPATIBLE_REDEFINITION, defn) - else: - # TODO: Update conditional type binder. - self.check_subtype(new_type, orig_type, defn, - messages.INCOMPATIBLE_REDEFINITION, - 'redefinition with type', - 'original type') - - def check_func_item(self, defn: FuncItem, - type_override: CallableType = None, - name: str = None) -> Type: - """Type check a function. - - If type_override is provided, use it as the function type. - """ - # We may be checking a function definition or an anonymous function. In - # the first case, set up another reference with the precise type. - fdef = None # type: FuncDef - if isinstance(defn, FuncDef): - fdef = defn - - self.function_stack.append(defn) - self.dynamic_funcs.append(defn.is_dynamic() and not type_override) - - if fdef: - self.errors.push_function(fdef.name()) - - self.enter_partial_types() - - typ = self.function_type(defn) - if type_override: - typ = type_override - if isinstance(typ, CallableType): - self.check_func_def(defn, typ, name) - else: - raise RuntimeError('Not supported') - - self.leave_partial_types() - - if fdef: - self.errors.pop_function() - - self.dynamic_funcs.pop() - self.function_stack.pop() - self.current_node_deferred = False - - def check_func_def(self, defn: FuncItem, typ: CallableType, name: str) -> None: - """Type check a function definition.""" - # Expand type variables with value restrictions to ordinary types. - for item, typ in self.expand_typevars(defn, typ): - old_binder = self.binder - self.binder = ConditionalTypeBinder() - with self.binder.frame_context(): - defn.expanded.append(item) - - # We may be checking a function definition or an anonymous - # function. In the first case, set up another reference with the - # precise type. - if isinstance(item, FuncDef): - fdef = item - else: - fdef = None - - if fdef: - # Check if __init__ has an invalid, non-None return type. - if (fdef.info and fdef.name() == '__init__' and - not isinstance(typ.ret_type, Void) and - not self.dynamic_funcs[-1]): - self.fail(messages.INIT_MUST_HAVE_NONE_RETURN_TYPE, - item.type) - - show_untyped = not self.is_typeshed_stub or self.options.warn_incomplete_stub - if self.options.disallow_untyped_defs and show_untyped: - # Check for functions with unspecified/not fully specified types. - def is_implicit_any(t: Type) -> bool: - return isinstance(t, AnyType) and t.implicit - - if fdef.type is None: - self.fail(messages.FUNCTION_TYPE_EXPECTED, fdef) - elif isinstance(fdef.type, CallableType): - if is_implicit_any(fdef.type.ret_type): - self.fail(messages.RETURN_TYPE_EXPECTED, fdef) - if any(is_implicit_any(t) for t in fdef.type.arg_types): - self.fail(messages.ARGUMENT_TYPE_EXPECTED, fdef) - - if name in nodes.reverse_op_method_set: - self.check_reverse_op_method(item, typ, name) - elif name == '__getattr__': - self.check_getattr_method(typ, defn) - - # Refuse contravariant return type variable - if isinstance(typ.ret_type, TypeVarType): - if typ.ret_type.variance == CONTRAVARIANT: - self.fail(messages.RETURN_TYPE_CANNOT_BE_CONTRAVARIANT, - typ.ret_type) - - # Check that Generator functions have the appropriate return type. - if defn.is_generator: - if not self.is_generator_return_type(typ.ret_type): - self.fail(messages.INVALID_RETURN_TYPE_FOR_GENERATOR, typ) - - # Python 2 generators aren't allowed to return values. - if (self.options.python_version[0] == 2 and - isinstance(typ.ret_type, Instance) and - typ.ret_type.type.fullname() == 'typing.Generator'): - if not (isinstance(typ.ret_type.args[2], Void) - or isinstance(typ.ret_type.args[2], AnyType)): - self.fail(messages.INVALID_GENERATOR_RETURN_ITEM_TYPE, typ) - - # Push return type. - self.return_types.append(typ.ret_type) - - # Store argument types. - for i in range(len(typ.arg_types)): - arg_type = typ.arg_types[i] - - # Refuse covariant parameter type variables - if isinstance(arg_type, TypeVarType): - if arg_type.variance == COVARIANT: - self.fail(messages.FUNCTION_PARAMETER_CANNOT_BE_COVARIANT, - arg_type) - - if typ.arg_kinds[i] == nodes.ARG_STAR: - # builtins.tuple[T] is typing.Tuple[T, ...] - arg_type = self.named_generic_type('builtins.tuple', - [arg_type]) - elif typ.arg_kinds[i] == nodes.ARG_STAR2: - arg_type = self.named_generic_type('builtins.dict', - [self.str_type(), - arg_type]) - item.arguments[i].variable.type = arg_type - - # Type check initialization expressions. - for arg in item.arguments: - init = arg.initialization_statement - if init: - self.accept(init) - - # Type check body in a new scope. - with self.binder.frame_context(): - self.accept(item.body) - - self.return_types.pop() - - self.binder = old_binder - - def check_reverse_op_method(self, defn: FuncItem, typ: CallableType, - method: str) -> None: - """Check a reverse operator method such as __radd__.""" - - # This used to check for some very obscure scenario. It now - # just decides whether it's worth calling - # check_overlapping_op_methods(). - - if method in ('__eq__', '__ne__'): - # These are defined for all objects => can't cause trouble. - return - - # With 'Any' or 'object' return type we are happy, since any possible - # return value is valid. - ret_type = typ.ret_type - if isinstance(ret_type, AnyType): - return - if isinstance(ret_type, Instance): - if ret_type.type.fullname() == 'builtins.object': - return - # Plausibly the method could have too few arguments, which would result - # in an error elsewhere. - if len(typ.arg_types) <= 2: - # TODO check self argument kind - - # Check for the issue described above. - arg_type = typ.arg_types[1] - other_method = nodes.normal_from_reverse_op[method] - if isinstance(arg_type, Instance): - if not arg_type.type.has_readable_member(other_method): - return - elif isinstance(arg_type, AnyType): - return - elif isinstance(arg_type, UnionType): - if not arg_type.has_readable_member(other_method): - return - else: - return - - typ2 = self.expr_checker.analyze_external_member_access( - other_method, arg_type, defn) - self.check_overlapping_op_methods( - typ, method, defn.info, - typ2, other_method, cast(Instance, arg_type), - defn) - - def check_overlapping_op_methods(self, - reverse_type: CallableType, - reverse_name: str, - reverse_class: TypeInfo, - forward_type: Type, - forward_name: str, - forward_base: Instance, - context: Context) -> None: - """Check for overlapping method and reverse method signatures. - - Assume reverse method has valid argument count and kinds. - """ - - # Reverse operator method that overlaps unsafely with the - # forward operator method can result in type unsafety. This is - # similar to overlapping overload variants. - # - # This example illustrates the issue: - # - # class X: pass - # class A: - # def __add__(self, x: X) -> int: - # if isinstance(x, X): - # return 1 - # return NotImplemented - # class B: - # def __radd__(self, x: A) -> str: return 'x' - # class C(X, B): pass - # def f(b: B) -> None: - # A() + b # Result is 1, even though static type seems to be str! - # f(C()) - # - # The reason for the problem is that B and X are overlapping - # types, and the return types are different. Also, if the type - # of x in __radd__ would not be A, the methods could be - # non-overlapping. - - if isinstance(forward_type, CallableType): - # TODO check argument kinds - if len(forward_type.arg_types) < 1: - # Not a valid operator method -- can't succeed anyway. - return - - # Construct normalized function signatures corresponding to the - # operator methods. The first argument is the left operand and the - # second operand is the right argument -- we switch the order of - # the arguments of the reverse method. - forward_tweaked = CallableType( - [forward_base, forward_type.arg_types[0]], - [nodes.ARG_POS] * 2, - [None] * 2, - forward_type.ret_type, - forward_type.fallback, - name=forward_type.name) - reverse_args = reverse_type.arg_types - reverse_tweaked = CallableType( - [reverse_args[1], reverse_args[0]], - [nodes.ARG_POS] * 2, - [None] * 2, - reverse_type.ret_type, - fallback=self.named_type('builtins.function'), - name=reverse_type.name) - - if is_unsafe_overlapping_signatures(forward_tweaked, - reverse_tweaked): - self.msg.operator_method_signatures_overlap( - reverse_class.name(), reverse_name, - forward_base.type.name(), forward_name, context) - elif isinstance(forward_type, Overloaded): - for item in forward_type.items(): - self.check_overlapping_op_methods( - reverse_type, reverse_name, reverse_class, - item, forward_name, forward_base, context) - elif not isinstance(forward_type, AnyType): - self.msg.forward_operator_not_callable(forward_name, context) - - def check_inplace_operator_method(self, defn: FuncBase) -> None: - """Check an inplace operator method such as __iadd__. - - They cannot arbitrarily overlap with __add__. - """ - method = defn.name() - if method not in nodes.inplace_operator_methods: - return - typ = self.method_type(defn) - cls = defn.info - other_method = '__' + method[3:] - if cls.has_readable_member(other_method): - instance = self_type(cls) - typ2 = self.expr_checker.analyze_external_member_access( - other_method, instance, defn) - fail = False - if isinstance(typ2, FunctionLike): - if not is_more_general_arg_prefix(typ, typ2): - fail = True - else: - # TODO overloads - fail = True - if fail: - self.msg.signatures_incompatible(method, other_method, defn) - - def check_getattr_method(self, typ: CallableType, context: Context) -> None: - method_type = CallableType([AnyType(), self.named_type('builtins.str')], - [nodes.ARG_POS, nodes.ARG_POS], - [None], - AnyType(), - self.named_type('builtins.function')) - if not is_subtype(typ, method_type): - self.msg.invalid_signature(typ, context) - - def expand_typevars(self, defn: FuncItem, - typ: CallableType) -> List[Tuple[FuncItem, CallableType]]: - # TODO use generator - subst = [] # type: List[List[Tuple[TypeVarId, Type]]] - tvars = typ.variables or [] - tvars = tvars[:] - if defn.info: - # Class type variables - tvars += defn.info.defn.type_vars or [] - for tvar in tvars: - if tvar.values: - subst.append([(tvar.id, value) - for value in tvar.values]) - if subst: - result = [] # type: List[Tuple[FuncItem, CallableType]] - for substitutions in itertools.product(*subst): - mapping = dict(substitutions) - expanded = cast(CallableType, expand_type(typ, mapping)) - result.append((expand_func(defn, mapping), expanded)) - return result - else: - return [(defn, typ)] - - def check_method_override(self, defn: FuncBase) -> None: - """Check if function definition is compatible with base classes.""" - # Check against definitions in base classes. - for base in defn.info.mro[1:]: - self.check_method_or_accessor_override_for_base(defn, base) - - def check_method_or_accessor_override_for_base(self, defn: FuncBase, - base: TypeInfo) -> None: - """Check if method definition is compatible with a base class.""" - if base: - name = defn.name() - if name not in ('__init__', '__new__'): - # Check method override (__init__ and __new__ are special). - self.check_method_override_for_base_with_name(defn, name, base) - if name in nodes.inplace_operator_methods: - # Figure out the name of the corresponding operator method. - method = '__' + name[3:] - # An inplace operator method such as __iadd__ might not be - # always introduced safely if a base class defined __add__. - # TODO can't come up with an example where this is - # necessary; now it's "just in case" - self.check_method_override_for_base_with_name(defn, method, - base) - - def check_method_override_for_base_with_name( - self, defn: FuncBase, name: str, base: TypeInfo) -> None: - base_attr = base.names.get(name) - if base_attr: - # The name of the method is defined in the base class. - - # Construct the type of the overriding method. - typ = self.method_type(defn) - # Map the overridden method type to subtype context so that - # it can be checked for compatibility. - original_type = base_attr.type - if original_type is None and isinstance(base_attr.node, - FuncDef): - original_type = self.function_type(base_attr.node) - if isinstance(original_type, FunctionLike): - original = map_type_from_supertype( - method_type(original_type), - defn.info, base) - # Check that the types are compatible. - # TODO overloaded signatures - self.check_override(typ, - cast(FunctionLike, original), - defn.name(), - name, - base.name(), - defn) - else: - assert original_type is not None - self.msg.signature_incompatible_with_supertype( - defn.name(), name, base.name(), defn) - - def check_override(self, override: FunctionLike, original: FunctionLike, - name: str, name_in_super: str, supertype: str, - node: Context) -> None: - """Check a method override with given signatures. - - Arguments: - override: The signature of the overriding method. - original: The signature of the original supertype method. - name: The name of the subtype. This and the next argument are - only used for generating error messages. - supertype: The name of the supertype. - """ - # Use boolean variable to clarify code. - fail = False - if not is_subtype(override, original): - fail = True - elif (not isinstance(original, Overloaded) and - isinstance(override, Overloaded) and - name in nodes.reverse_op_methods.keys()): - # Operator method overrides cannot introduce overloading, as - # this could be unsafe with reverse operator methods. - fail = True - - if fail: - emitted_msg = False - if (isinstance(override, CallableType) and - isinstance(original, CallableType) and - len(override.arg_types) == len(original.arg_types) and - override.min_args == original.min_args): - # Give more detailed messages for the common case of both - # signatures having the same number of arguments and no - # overloads. - - # override might have its own generic function type - # variables. If an argument or return type of override - # does not have the correct subtyping relationship - # with the original type even after these variables - # are erased, then it is definitely an incompatiblity. - - override_ids = override.type_var_ids() - - def erase_override(t: Type) -> Type: - return erase_typevars(t, ids_to_erase=override_ids) - - for i in range(len(override.arg_types)): - if not is_subtype(original.arg_types[i], - erase_override(override.arg_types[i])): - self.msg.argument_incompatible_with_supertype( - i + 1, name, name_in_super, supertype, node) - emitted_msg = True - - if not is_subtype(erase_override(override.ret_type), - original.ret_type): - self.msg.return_type_incompatible_with_supertype( - name, name_in_super, supertype, node) - emitted_msg = True - - if not emitted_msg: - # Fall back to generic incompatibility message. - self.msg.signature_incompatible_with_supertype( - name, name_in_super, supertype, node) - - def visit_class_def(self, defn: ClassDef) -> Type: - """Type check a class definition.""" - typ = defn.info - self.errors.push_type(defn.name) - self.enter_partial_types() - old_binder = self.binder - self.binder = ConditionalTypeBinder() - with self.binder.frame_context(): - self.accept(defn.defs) - self.binder = old_binder - if not hasattr(defn, 'incompatible_baseclass'): - # Otherwise we've already found errors; more errors are not useful - self.check_multiple_inheritance(typ) - self.leave_partial_types() - self.errors.pop_type() - - def check_multiple_inheritance(self, typ: TypeInfo) -> None: - """Check for multiple inheritance related errors.""" - - if len(typ.bases) <= 1: - # No multiple inheritance. - return - # Verify that inherited attributes are compatible. - mro = typ.mro[1:] - for i, base in enumerate(mro): - for name in base.names: - for base2 in mro[i + 1:]: - # We only need to check compatibility of attributes from classes not - # in a subclass relationship. For subclasses, normal (single inheritance) - # checks suffice (these are implemented elsewhere). - if name in base2.names and base2 not in base.mro: - self.check_compatibility(name, base, base2, typ) - # Verify that base class layouts are compatible. - builtin_bases = [nearest_builtin_ancestor(base.type) - for base in typ.bases] - for base1 in builtin_bases: - for base2 in builtin_bases: - if not (base1 in base2.mro or base2 in base1.mro): - self.fail(messages.INSTANCE_LAYOUT_CONFLICT, typ) - - def check_compatibility(self, name: str, base1: TypeInfo, - base2: TypeInfo, ctx: Context) -> None: - """Check if attribute name in base1 is compatible with base2 in multiple inheritance. - - Assume base1 comes before base2 in the MRO, and that base1 and base2 don't have - a direct subclass relationship (i.e., the compatibility requirement only derives from - multiple inheritance). - """ - if name == '__init__': - # __init__ can be incompatible -- it's a special case. - return - first = base1[name] - second = base2[name] - first_type = first.type - if first_type is None and isinstance(first.node, FuncDef): - first_type = self.function_type(first.node) - second_type = second.type - if second_type is None and isinstance(second.node, FuncDef): - second_type = self.function_type(second.node) - # TODO: What if some classes are generic? - if (isinstance(first_type, FunctionLike) and - isinstance(second_type, FunctionLike)): - # Method override - first_sig = method_type(first_type) - second_sig = method_type(second_type) - ok = is_subtype(first_sig, second_sig) - elif first_type and second_type: - ok = is_equivalent(first_type, second_type) - else: - if first_type is None: - self.msg.cannot_determine_type_in_base(name, base1.name(), ctx) - if second_type is None: - self.msg.cannot_determine_type_in_base(name, base2.name(), ctx) - ok = True - if not ok: - self.msg.base_class_definitions_incompatible(name, base1, base2, - ctx) - - def visit_import_from(self, node: ImportFrom) -> Type: - self.check_import(node) - - def visit_import_all(self, node: ImportAll) -> Type: - self.check_import(node) - - def check_import(self, node: ImportBase) -> Type: - for assign in node.assignments: - lvalue = assign.lvalues[0] - lvalue_type, _, __ = self.check_lvalue(lvalue) - if lvalue_type is None: - # TODO: This is broken. - lvalue_type = AnyType() - message = '{} "{}"'.format(messages.INCOMPATIBLE_IMPORT_OF, - cast(NameExpr, assign.rvalue).name) - self.check_simple_assignment(lvalue_type, assign.rvalue, node, - msg=message, lvalue_name='local name', - rvalue_name='imported name') - - # - # Statements - # - - def visit_block(self, b: Block) -> Type: - if b.is_unreachable: - return None - for s in b.body: - self.accept(s) - if self.binder.breaking_out: - break - - def visit_assignment_stmt(self, s: AssignmentStmt) -> Type: - """Type check an assignment statement. - - Handle all kinds of assignment statements (simple, indexed, multiple). - """ - self.check_assignment(s.lvalues[-1], s.rvalue, s.type is None) - - if len(s.lvalues) > 1: - # Chained assignment (e.g. x = y = ...). - # Make sure that rvalue type will not be reinferred. - if s.rvalue not in self.type_map: - self.accept(s.rvalue) - rvalue = self.temp_node(self.type_map[s.rvalue], s) - for lv in s.lvalues[:-1]: - self.check_assignment(lv, rvalue, s.type is None) - - def check_assignment(self, lvalue: Node, rvalue: Node, infer_lvalue_type: bool = True) -> None: - """Type check a single assignment: lvalue = rvalue.""" - if isinstance(lvalue, TupleExpr) or isinstance(lvalue, ListExpr): - ltuple = cast(Union[TupleExpr, ListExpr], lvalue) - - self.check_assignment_to_multiple_lvalues(ltuple.items, rvalue, lvalue, - infer_lvalue_type) - else: - lvalue_type, index_lvalue, inferred = self.check_lvalue(lvalue) - if lvalue_type: - if isinstance(lvalue_type, PartialType) and lvalue_type.type is None: - # Try to infer a proper type for a variable with a partial None type. - rvalue_type = self.accept(rvalue) - if isinstance(rvalue_type, NoneTyp): - # This doesn't actually provide any additional information -- multiple - # None initializers preserve the partial None type. - return - if is_valid_inferred_type(rvalue_type): - var = lvalue_type.var - partial_types = self.find_partial_types(var) - if partial_types is not None: - if not self.current_node_deferred: - if experiments.STRICT_OPTIONAL: - var.type = UnionType.make_simplified_union( - [rvalue_type, NoneTyp()]) - else: - var.type = rvalue_type - else: - var.type = None - del partial_types[var] - # Try to infer a partial type. No need to check the return value, as - # an error will be reported elsewhere. - self.infer_partial_type(lvalue_type.var, lvalue, rvalue_type) - return - if (is_literal_none(rvalue) and - isinstance(lvalue, NameExpr) and - isinstance(lvalue.node, Var) and - lvalue.node.is_initialized_in_class): - # Allow None's to be assigned to class variables with non-Optional types. - rvalue_type = lvalue_type - else: - rvalue_type = self.check_simple_assignment(lvalue_type, rvalue, lvalue) - - if rvalue_type and infer_lvalue_type: - self.binder.assign_type(lvalue, - rvalue_type, - lvalue_type, - self.typing_mode_weak()) - elif index_lvalue: - self.check_indexed_assignment(index_lvalue, rvalue, rvalue) - - if inferred: - self.infer_variable_type(inferred, lvalue, self.accept(rvalue), - rvalue) - - def check_assignment_to_multiple_lvalues(self, lvalues: List[Node], rvalue: Node, - context: Context, - infer_lvalue_type: bool = True) -> None: - if isinstance(rvalue, TupleExpr) or isinstance(rvalue, ListExpr): - # Recursively go into Tuple or List expression rhs instead of - # using the type of rhs, because this allowed more fine grained - # control in cases like: a, b = [int, str] where rhs would get - # type List[object] - - rvalues = cast(Union[TupleExpr, ListExpr], rvalue).items - - if self.check_rvalue_count_in_assignment(lvalues, len(rvalues), context): - star_index = next((i for i, lv in enumerate(lvalues) if - isinstance(lv, StarExpr)), len(lvalues)) - - left_lvs = lvalues[:star_index] - star_lv = cast(StarExpr, - lvalues[star_index]) if star_index != len(lvalues) else None - right_lvs = lvalues[star_index + 1:] - - left_rvs, star_rvs, right_rvs = self.split_around_star( - rvalues, star_index, len(lvalues)) - - lr_pairs = list(zip(left_lvs, left_rvs)) - if star_lv: - rv_list = ListExpr(star_rvs) - rv_list.set_line(rvalue.get_line()) - lr_pairs.append((star_lv.expr, rv_list)) - lr_pairs.extend(zip(right_lvs, right_rvs)) - - for lv, rv in lr_pairs: - self.check_assignment(lv, rv, infer_lvalue_type) - else: - self.check_multi_assignment(lvalues, rvalue, context, infer_lvalue_type) - - def check_rvalue_count_in_assignment(self, lvalues: List[Node], rvalue_count: int, - context: Context) -> bool: - if any(isinstance(lvalue, StarExpr) for lvalue in lvalues): - if len(lvalues) - 1 > rvalue_count: - self.msg.wrong_number_values_to_unpack(rvalue_count, - len(lvalues) - 1, context) - return False - elif rvalue_count != len(lvalues): - self.msg.wrong_number_values_to_unpack(rvalue_count, - len(lvalues), context) - return False - return True - - def check_multi_assignment(self, lvalues: List[Node], - rvalue: Node, - context: Context, - infer_lvalue_type: bool = True, - msg: str = None) -> None: - """Check the assignment of one rvalue to a number of lvalues.""" - - # Infer the type of an ordinary rvalue expression. - rvalue_type = self.accept(rvalue) # TODO maybe elsewhere; redundant - undefined_rvalue = False - - if isinstance(rvalue_type, AnyType): - for lv in lvalues: - if isinstance(lv, StarExpr): - lv = lv.expr - self.check_assignment(lv, self.temp_node(AnyType(), context), infer_lvalue_type) - elif isinstance(rvalue_type, TupleType): - self.check_multi_assignment_from_tuple(lvalues, rvalue, rvalue_type, - context, undefined_rvalue, infer_lvalue_type) - else: - self.check_multi_assignment_from_iterable(lvalues, rvalue_type, - context, infer_lvalue_type) - - def check_multi_assignment_from_tuple(self, lvalues: List[Node], rvalue: Node, - rvalue_type: TupleType, context: Context, - undefined_rvalue: bool, - infer_lvalue_type: bool = True) -> None: - if self.check_rvalue_count_in_assignment(lvalues, len(rvalue_type.items), context): - star_index = next((i for i, lv in enumerate(lvalues) - if isinstance(lv, StarExpr)), len(lvalues)) - - left_lvs = lvalues[:star_index] - star_lv = cast(StarExpr, lvalues[star_index]) if star_index != len(lvalues) else None - right_lvs = lvalues[star_index + 1:] - - if not undefined_rvalue: - # Infer rvalue again, now in the correct type context. - lvalue_type = self.lvalue_type_for_inference(lvalues, rvalue_type) - rvalue_type = cast(TupleType, self.accept(rvalue, lvalue_type)) - - left_rv_types, star_rv_types, right_rv_types = self.split_around_star( - rvalue_type.items, star_index, len(lvalues)) - - for lv, rv_type in zip(left_lvs, left_rv_types): - self.check_assignment(lv, self.temp_node(rv_type, context), infer_lvalue_type) - if star_lv: - nodes = [self.temp_node(rv_type, context) for rv_type in star_rv_types] - list_expr = ListExpr(nodes) - list_expr.set_line(context.get_line()) - self.check_assignment(star_lv.expr, list_expr, infer_lvalue_type) - for lv, rv_type in zip(right_lvs, right_rv_types): - self.check_assignment(lv, self.temp_node(rv_type, context), infer_lvalue_type) - - def lvalue_type_for_inference(self, lvalues: List[Node], rvalue_type: TupleType) -> Type: - star_index = next((i for i, lv in enumerate(lvalues) - if isinstance(lv, StarExpr)), len(lvalues)) - left_lvs = lvalues[:star_index] - star_lv = cast(StarExpr, lvalues[star_index]) if star_index != len(lvalues) else None - right_lvs = lvalues[star_index + 1:] - left_rv_types, star_rv_types, right_rv_types = self.split_around_star( - rvalue_type.items, star_index, len(lvalues)) - - type_parameters = [] # type: List[Type] - - def append_types_for_inference(lvs: List[Node], rv_types: List[Type]) -> None: - for lv, rv_type in zip(lvs, rv_types): - sub_lvalue_type, index_expr, inferred = self.check_lvalue(lv) - if sub_lvalue_type: - type_parameters.append(sub_lvalue_type) - else: # index lvalue - # TODO Figure out more precise type context, probably - # based on the type signature of the _set method. - type_parameters.append(rv_type) - - append_types_for_inference(left_lvs, left_rv_types) - - if star_lv: - sub_lvalue_type, index_expr, inferred = self.check_lvalue(star_lv.expr) - if sub_lvalue_type: - type_parameters.extend([sub_lvalue_type] * len(star_rv_types)) - else: # index lvalue - # TODO Figure out more precise type context, probably - # based on the type signature of the _set method. - type_parameters.extend(star_rv_types) - - append_types_for_inference(right_lvs, right_rv_types) - - return TupleType(type_parameters, self.named_type('builtins.tuple')) - - def split_around_star(self, items: List[T], star_index: int, - length: int) -> Tuple[List[T], List[T], List[T]]: - """Splits a list of items in three to match another list of length 'length' - that contains a starred expression at 'star_index' in the following way: - - star_index = 2, length = 5 (i.e., [a,b,*,c,d]), items = [1,2,3,4,5,6,7] - returns in: ([1,2], [3,4,5], [6,7]) - """ - nr_right_of_star = length - star_index - 1 - right_index = nr_right_of_star if -nr_right_of_star != 0 else len(items) - left = items[:star_index] - star = items[star_index:right_index] - right = items[right_index:] - return (left, star, right) - - def type_is_iterable(self, type: Type) -> bool: - return (is_subtype(type, self.named_generic_type('typing.Iterable', - [AnyType()])) and - isinstance(type, Instance)) - - def check_multi_assignment_from_iterable(self, lvalues: List[Node], rvalue_type: Type, - context: Context, - infer_lvalue_type: bool = True) -> None: - if self.type_is_iterable(rvalue_type): - item_type = self.iterable_item_type(cast(Instance, rvalue_type)) - for lv in lvalues: - if isinstance(lv, StarExpr): - self.check_assignment(lv.expr, self.temp_node(rvalue_type, context), - infer_lvalue_type) - else: - self.check_assignment(lv, self.temp_node(item_type, context), - infer_lvalue_type) - else: - self.msg.type_not_iterable(rvalue_type, context) - - def check_lvalue(self, lvalue: Node) -> Tuple[Type, IndexExpr, Var]: - lvalue_type = None # type: Type - index_lvalue = None # type: IndexExpr - inferred = None # type: Var - - if self.is_definition(lvalue): - if isinstance(lvalue, NameExpr): - inferred = cast(Var, lvalue.node) - assert isinstance(inferred, Var) - else: - m = cast(MemberExpr, lvalue) - self.accept(m.expr) - inferred = m.def_var - elif isinstance(lvalue, IndexExpr): - index_lvalue = lvalue - elif isinstance(lvalue, MemberExpr): - lvalue_type = self.expr_checker.analyze_ordinary_member_access(lvalue, - True) - self.store_type(lvalue, lvalue_type) - elif isinstance(lvalue, NameExpr): - lvalue_type = self.expr_checker.analyze_ref_expr(lvalue, lvalue=True) - self.store_type(lvalue, lvalue_type) - elif isinstance(lvalue, TupleExpr) or isinstance(lvalue, ListExpr): - lv = cast(Union[TupleExpr, ListExpr], lvalue) - types = [self.check_lvalue(sub_expr)[0] for sub_expr in lv.items] - lvalue_type = TupleType(types, self.named_type('builtins.tuple')) - else: - lvalue_type = self.accept(lvalue) - - return lvalue_type, index_lvalue, inferred - - def is_definition(self, s: Node) -> bool: - if isinstance(s, NameExpr): - if s.is_def: - return True - # If the node type is not defined, this must the first assignment - # that we process => this is a definition, even though the semantic - # analyzer did not recognize this as such. This can arise in code - # that uses isinstance checks, if type checking of the primary - # definition is skipped due to an always False type check. - node = s.node - if isinstance(node, Var): - return node.type is None - elif isinstance(s, MemberExpr): - return s.is_def - return False - - def infer_variable_type(self, name: Var, lvalue: Node, - init_type: Type, context: Context) -> None: - """Infer the type of initialized variables from initializer type.""" - if self.typing_mode_weak(): - self.set_inferred_type(name, lvalue, AnyType()) - self.binder.assign_type(lvalue, init_type, self.binder.get_declaration(lvalue), True) - elif isinstance(init_type, Void): - self.check_not_void(init_type, context) - self.set_inference_error_fallback_type(name, lvalue, init_type, context) - elif isinstance(init_type, DeletedType): - self.msg.deleted_as_rvalue(init_type, context) - elif not is_valid_inferred_type(init_type): - # We cannot use the type of the initialization expression for full type - # inference (it's not specific enough), but we might be able to give - # partial type which will be made more specific later. A partial type - # gets generated in assignment like 'x = []' where item type is not known. - if not self.infer_partial_type(name, lvalue, init_type): - self.fail(messages.NEED_ANNOTATION_FOR_VAR, context) - self.set_inference_error_fallback_type(name, lvalue, init_type, context) - else: - # Infer type of the target. - - # Make the type more general (strip away function names etc.). - init_type = strip_type(init_type) - - self.set_inferred_type(name, lvalue, init_type) - - def infer_partial_type(self, name: Var, lvalue: Node, init_type: Type) -> bool: - if isinstance(init_type, (NoneTyp, UninhabitedType)): - partial_type = PartialType(None, name, [init_type]) - elif isinstance(init_type, Instance): - fullname = init_type.type.fullname() - if (isinstance(lvalue, NameExpr) and - (fullname == 'builtins.list' or - fullname == 'builtins.set' or - fullname == 'builtins.dict') and - all(isinstance(t, (NoneTyp, UninhabitedType)) for t in init_type.args)): - partial_type = PartialType(init_type.type, name, init_type.args) - else: - return False - else: - return False - self.set_inferred_type(name, lvalue, partial_type) - self.partial_types[-1][name] = lvalue - return True - - def set_inferred_type(self, var: Var, lvalue: Node, type: Type) -> None: - """Store inferred variable type. - - Store the type to both the variable node and the expression node that - refers to the variable (lvalue). If var is None, do nothing. - """ - if var and not self.current_node_deferred: - var.type = type - self.store_type(lvalue, type) - - def set_inference_error_fallback_type(self, var: Var, lvalue: Node, type: Type, - context: Context) -> None: - """If errors on context line are ignored, store dummy type for variable. - - If a program ignores error on type inference error, the variable should get some - inferred type so that if can used later on in the program. Example: - - x = [] # type: ignore - x.append(1) # Should be ok! - - We implement this here by giving x a valid type (Any). - """ - if context.get_line() in self.errors.ignored_lines[self.errors.file]: - self.set_inferred_type(var, lvalue, AnyType()) - - def narrow_type_from_binder(self, expr: Node, known_type: Type) -> Type: - if expr.literal >= LITERAL_TYPE: - restriction = self.binder.get(expr) - if restriction: - ans = meet_simple(known_type, restriction) - return ans - return known_type - - def check_simple_assignment(self, lvalue_type: Type, rvalue: Node, - context: Node, - msg: str = messages.INCOMPATIBLE_TYPES_IN_ASSIGNMENT, - lvalue_name: str = 'variable', - rvalue_name: str = 'expression') -> Type: - if self.is_stub and isinstance(rvalue, EllipsisExpr): - # '...' is always a valid initializer in a stub. - return AnyType() - else: - rvalue_type = self.accept(rvalue, lvalue_type) - if isinstance(rvalue_type, DeletedType): - self.msg.deleted_as_rvalue(rvalue_type, context) - if self.typing_mode_weak(): - return rvalue_type - if isinstance(lvalue_type, DeletedType): - self.msg.deleted_as_lvalue(lvalue_type, context) - else: - self.check_subtype(rvalue_type, lvalue_type, context, msg, - '{} has type'.format(rvalue_name), - '{} has type'.format(lvalue_name)) - return rvalue_type - - def check_indexed_assignment(self, lvalue: IndexExpr, - rvalue: Node, context: Context) -> None: - """Type check indexed assignment base[index] = rvalue. - - The lvalue argument is the base[index] expression. - """ - self.try_infer_partial_type_from_indexed_assignment(lvalue, rvalue) - basetype = self.accept(lvalue.base) - method_type = self.expr_checker.analyze_external_member_access( - '__setitem__', basetype, context) - lvalue.method_type = method_type - self.expr_checker.check_call(method_type, [lvalue.index, rvalue], - [nodes.ARG_POS, nodes.ARG_POS], - context) - - def try_infer_partial_type_from_indexed_assignment( - self, lvalue: IndexExpr, rvalue: Node) -> None: - # TODO: Should we share some of this with try_infer_partial_type? - if isinstance(lvalue.base, RefExpr) and isinstance(lvalue.base.node, Var): - var = lvalue.base.node - if isinstance(var.type, PartialType): - type_type = var.type.type - if type_type is None: - return # The partial type is None. - partial_types = self.find_partial_types(var) - if partial_types is None: - return - typename = type_type.fullname() - if typename == 'builtins.dict': - # TODO: Don't infer things twice. - key_type = self.accept(lvalue.index) - value_type = self.accept(rvalue) - full_key_type = UnionType.make_simplified_union( - [key_type, var.type.inner_types[0]]) - full_value_type = UnionType.make_simplified_union( - [value_type, var.type.inner_types[1]]) - if (is_valid_inferred_type(full_key_type) and - is_valid_inferred_type(full_value_type)): - if not self.current_node_deferred: - var.type = self.named_generic_type('builtins.dict', - [full_key_type, full_value_type]) - del partial_types[var] - - def visit_expression_stmt(self, s: ExpressionStmt) -> Type: - self.accept(s.expr) - - def visit_return_stmt(self, s: ReturnStmt) -> Type: - """Type check a return statement.""" - self.binder.breaking_out = True - if self.is_within_function(): - if self.function_stack[-1].is_generator: - return_type = self.get_generator_return_type(self.return_types[-1]) - else: - return_type = self.return_types[-1] - - if s.expr: - # Return with a value. - typ = self.accept(s.expr, return_type) - # Returning a value of type Any is always fine. - if isinstance(typ, AnyType): - return None - - if isinstance(return_type, Void): - # Lambdas are allowed to have a Void return. - # Functions returning a value of type None are allowed to have a Void return. - if isinstance(self.function_stack[-1], FuncExpr) or isinstance(typ, NoneTyp): - return None - self.fail(messages.NO_RETURN_VALUE_EXPECTED, s) - else: - self.check_subtype( - subtype_label='got', - subtype=typ, - supertype_label='expected', - supertype=return_type, - context=s, - msg=messages.INCOMPATIBLE_RETURN_VALUE_TYPE) - else: - # Empty returns are valid in Generators with Any typed returns. - if (self.function_stack[-1].is_generator and isinstance(return_type, AnyType)): - return None - - if isinstance(return_type, Void): - return None - - if isinstance(return_type, AnyType): - return None - - if self.typing_mode_full(): - self.fail(messages.RETURN_VALUE_EXPECTED, s) - - def wrap_generic_type(self, typ: Instance, rtyp: Instance, check_type: - str, context: Context) -> Type: - n_diff = self.count_nested_types(rtyp, check_type) - self.count_nested_types(typ, - check_type) - if n_diff == 1: - return self.named_generic_type(check_type, [typ]) - elif n_diff == 0 or n_diff > 1: - self.fail(messages.INCOMPATIBLE_RETURN_VALUE_TYPE - + ": expected {}, got {}".format(rtyp, typ), context) - return typ - return typ - - def count_nested_types(self, typ: Instance, check_type: str) -> int: - c = 0 - while is_subtype(typ, self.named_type(check_type)): - c += 1 - typ = map_instance_to_supertype(self.named_generic_type(check_type, typ.args), - self.lookup_typeinfo(check_type)) - if typ.args: - typ = cast(Instance, typ.args[0]) - else: - return c - return c - - def visit_if_stmt(self, s: IfStmt) -> Type: - """Type check an if statement.""" - breaking_out = True - # This frame records the knowledge from previous if/elif clauses not being taken. - with self.binder.frame_context(): - for e, b in zip(s.expr, s.body): - t = self.accept(e) - self.check_not_void(t, e) - if_map, else_map = find_isinstance_check( - e, self.type_map, - self.typing_mode_weak() - ) - if if_map is None: - # The condition is always false - # XXX should issue a warning? - pass - else: - # Only type check body if the if condition can be true. - with self.binder.frame_context(2): - if if_map: - for var, type in if_map.items(): - self.binder.push(var, type) - - self.accept(b) - breaking_out = breaking_out and self.binder.last_pop_breaking_out - - if else_map: - for var, type in else_map.items(): - self.binder.push(var, type) - if else_map is None: - # The condition is always true => remaining elif/else blocks - # can never be reached. - - # Might also want to issue a warning - # print("Warning: isinstance always true") - break - else: # Didn't break => can't prove one of the conditions is always true - with self.binder.frame_context(2): - if s.else_body: - self.accept(s.else_body) - breaking_out = breaking_out and self.binder.last_pop_breaking_out - if breaking_out: - self.binder.breaking_out = True - return None - - def visit_while_stmt(self, s: WhileStmt) -> Type: - """Type check a while statement.""" - self.accept_loop(IfStmt([s.expr], [s.body], None), s.else_body) - - def visit_operator_assignment_stmt(self, - s: OperatorAssignmentStmt) -> Type: - """Type check an operator assignment statement, e.g. x += 1.""" - lvalue_type = self.accept(s.lvalue) - method = infer_operator_assignment_method(lvalue_type, s.op) - rvalue_type, method_type = self.expr_checker.check_op( - method, lvalue_type, s.rvalue, s) - - if isinstance(s.lvalue, IndexExpr): - self.check_indexed_assignment(s.lvalue, s.rvalue, s.rvalue) - else: - if not is_subtype(rvalue_type, lvalue_type): - self.msg.incompatible_operator_assignment(s.op, s) - - def visit_assert_stmt(self, s: AssertStmt) -> Type: - self.accept(s.expr) - - # If this is asserting some isinstance check, bind that type in the following code - true_map, _ = find_isinstance_check( - s.expr, self.type_map, - self.typing_mode_weak() - ) - - if true_map: - for var, type in true_map.items(): - self.binder.push(var, type) - - def visit_raise_stmt(self, s: RaiseStmt) -> Type: - """Type check a raise statement.""" - self.binder.breaking_out = True - if s.expr: - self.type_check_raise(s.expr, s) - if s.from_expr: - self.type_check_raise(s.from_expr, s) - - def type_check_raise(self, e: Node, s: RaiseStmt) -> None: - typ = self.accept(e) - if isinstance(typ, FunctionLike): - if typ.is_type_obj(): - # Cases like "raise/from ExceptionClass". - typeinfo = typ.type_object() - base = self.lookup_typeinfo('builtins.BaseException') - if base in typeinfo.mro or typeinfo.fallback_to_any: - # Good! - return - # Else fall back to the checks below (which will fail). - if isinstance(typ, TupleType) and self.options.python_version[0] == 2: - # allow `raise type, value, traceback` - # https://docs.python.org/2/reference/simple_stmts.html#the-raise-statement - # TODO: Also check tuple item types. - if len(typ.items) in (2, 3): - return - if isinstance(typ, Instance) and typ.type.fallback_to_any: - # OK! - return - self.check_subtype(typ, - self.named_type('builtins.BaseException'), s, - messages.INVALID_EXCEPTION) - - def visit_try_stmt(self, s: TryStmt) -> Type: - """Type check a try statement.""" - # Our enclosing frame will get the result if the try/except falls through. - # This one gets all possible intermediate states - with self.binder.frame_context(): - if s.finally_body: - self.binder.try_frames.add(len(self.binder.frames) - 1) - breaking_out = self.visit_try_without_finally(s) - self.binder.try_frames.remove(len(self.binder.frames) - 1) - # First we check finally_body is type safe for all intermediate frames - self.accept(s.finally_body) - breaking_out = breaking_out or self.binder.breaking_out - else: - breaking_out = self.visit_try_without_finally(s) - - if not breaking_out and s.finally_body: - # Then we try again for the more restricted set of options that can fall through - self.accept(s.finally_body) - self.binder.breaking_out = breaking_out - return None - - def visit_try_without_finally(self, s: TryStmt) -> bool: - """Type check a try statement, ignoring the finally block. - - Return whether we are guaranteed to be breaking out. - Otherwise, it will place the results possible frames of - that don't break out into self.binder.frames[-2]. - """ - breaking_out = True - # This frame records the possible states that exceptions can leave variables in - # during the try: block - with self.binder.frame_context(): - with self.binder.frame_context(3): - self.binder.try_frames.add(len(self.binder.frames) - 2) - self.accept(s.body) - self.binder.try_frames.remove(len(self.binder.frames) - 2) - if s.else_body: - self.accept(s.else_body) - breaking_out = breaking_out and self.binder.last_pop_breaking_out - for i in range(len(s.handlers)): - with self.binder.frame_context(3): - if s.types[i]: - t = self.visit_except_handler_test(s.types[i]) - if s.vars[i]: - # To support local variables, we make this a definition line, - # causing assignment to set the variable's type. - s.vars[i].is_def = True - self.check_assignment(s.vars[i], self.temp_node(t, s.vars[i])) - self.accept(s.handlers[i]) - if s.vars[i]: - # Exception variables are deleted in python 3 but not python 2. - # But, since it's bad form in python 2 and the type checking - # wouldn't work very well, we delete it anyway. - - # Unfortunately, this doesn't let us detect usage before the - # try/except block. - if self.options.python_version[0] >= 3: - source = s.vars[i].name - else: - source = ('(exception variable "{}", which we do not accept outside' - 'except: blocks even in python 2)'.format(s.vars[i].name)) - var = cast(Var, s.vars[i].node) - var.type = DeletedType(source=source) - self.binder.cleanse(s.vars[i]) - breaking_out = breaking_out and self.binder.last_pop_breaking_out - return breaking_out - - def visit_except_handler_test(self, n: Node) -> Type: - """Type check an exception handler test clause.""" - type = self.accept(n) - - all_types = [] # type: List[Type] - test_types = type.items if isinstance(type, TupleType) else [type] - - for ttype in test_types: - if isinstance(ttype, AnyType): - all_types.append(ttype) - continue - - if not isinstance(ttype, FunctionLike): - self.fail(messages.INVALID_EXCEPTION_TYPE, n) - return AnyType() - - item = ttype.items()[0] - ret_type = item.ret_type - if not (is_subtype(ret_type, self.named_type('builtins.BaseException')) - and item.is_type_obj()): - self.fail(messages.INVALID_EXCEPTION_TYPE, n) - return AnyType() - - all_types.append(ret_type) - - return UnionType.make_simplified_union(all_types) - - def visit_for_stmt(self, s: ForStmt) -> Type: - """Type check a for statement.""" - item_type = self.analyze_iterable_item_type(s.expr) - self.analyze_index_variables(s.index, item_type, s) - self.accept_loop(s.body, s.else_body) - - def analyze_iterable_item_type(self, expr: Node) -> Type: - """Analyse iterable expression and return iterator item type.""" - iterable = self.accept(expr) - - self.check_not_void(iterable, expr) - if isinstance(iterable, TupleType): - if experiments.STRICT_OPTIONAL: - joined = UninhabitedType() # type: Type - else: - joined = NoneTyp() - for item in iterable.items: - joined = join_types(joined, item) - if isinstance(joined, ErrorType): - self.fail(messages.CANNOT_INFER_ITEM_TYPE, expr) - return AnyType() - return joined - else: - # Non-tuple iterable. - self.check_subtype(iterable, - self.named_generic_type('typing.Iterable', - [AnyType()]), - expr, messages.ITERABLE_EXPECTED) - - echk = self.expr_checker - method = echk.analyze_external_member_access('__iter__', iterable, - expr) - iterator = echk.check_call(method, [], [], expr)[0] - if self.options.python_version[0] >= 3: - nextmethod = '__next__' - else: - nextmethod = 'next' - method = echk.analyze_external_member_access(nextmethod, iterator, - expr) - return echk.check_call(method, [], [], expr)[0] - - def analyze_index_variables(self, index: Node, item_type: Type, - context: Context) -> None: - """Type check or infer for loop or list comprehension index vars.""" - self.check_assignment(index, self.temp_node(item_type, context)) - - def visit_del_stmt(self, s: DelStmt) -> Type: - if isinstance(s.expr, IndexExpr): - e = s.expr - m = MemberExpr(e.base, '__delitem__') - m.line = s.line - c = CallExpr(m, [e.index], [nodes.ARG_POS], [None]) - c.line = s.line - return c.accept(self) - else: - def flatten(t: Node) -> List[Node]: - """Flatten a nested sequence of tuples/lists into one list of nodes.""" - if isinstance(t, TupleExpr) or isinstance(t, ListExpr): - t = cast(Union[TupleExpr, ListExpr], t) - return [b for a in t.items for b in flatten(a)] - else: - return [t] - - s.expr.accept(self) - for elt in flatten(s.expr): - if isinstance(elt, NameExpr): - self.binder.assign_type(elt, - DeletedType(source=elt.name), - self.binder.get_declaration(elt), - self.typing_mode_weak()) - return None - - def visit_decorator(self, e: Decorator) -> Type: - for d in e.decorators: - if isinstance(d, RefExpr): - if d.fullname == 'typing.no_type_check': - e.var.type = AnyType() - e.var.is_ready = True - return NoneTyp() - - e.func.accept(self) - sig = self.function_type(e.func) # type: Type - # Process decorators from the inside out. - for i in range(len(e.decorators)): - n = len(e.decorators) - 1 - i - d = e.decorators[n] - if isinstance(d, NameExpr) and d.fullname == 'typing.overload': - self.fail('Single overload definition, multiple required', e) - continue - dec = self.accept(d) - temp = self.temp_node(sig) - sig, t2 = self.expr_checker.check_call(dec, [temp], - [nodes.ARG_POS], e) - sig = cast(FunctionLike, sig) - sig = set_callable_name(sig, e.func) - e.var.type = sig - e.var.is_ready = True - if e.func.is_property: - self.check_incompatible_property_override(e) - - def check_incompatible_property_override(self, e: Decorator) -> None: - if not e.var.is_settable_property and e.func.info is not None: - name = e.func.name() - for base in e.func.info.mro[1:]: - base_attr = base.names.get(name) - if not base_attr: - continue - if (isinstance(base_attr.node, OverloadedFuncDef) and - base_attr.node.is_property and - base_attr.node.items[0].var.is_settable_property): - self.fail(messages.READ_ONLY_PROPERTY_OVERRIDES_READ_WRITE, e) - - def visit_with_stmt(self, s: WithStmt) -> Type: - echk = self.expr_checker - for expr, target in zip(s.expr, s.target): - ctx = self.accept(expr) - enter = echk.analyze_external_member_access('__enter__', ctx, expr) - obj = echk.check_call(enter, [], [], expr)[0] - if target: - self.check_assignment(target, self.temp_node(obj, expr)) - exit = echk.analyze_external_member_access('__exit__', ctx, expr) - arg = self.temp_node(AnyType(), expr) - echk.check_call(exit, [arg] * 3, [nodes.ARG_POS] * 3, expr) - self.accept(s.body) - - def visit_print_stmt(self, s: PrintStmt) -> Type: - for arg in s.args: - self.accept(arg) - if s.target: - target_type = self.accept(s.target) - if not isinstance(target_type, NoneTyp): - # TODO: Also verify the type of 'write'. - self.expr_checker.analyze_external_member_access('write', target_type, s.target) - - # - # Expressions - # - - def visit_name_expr(self, e: NameExpr) -> Type: - return self.expr_checker.visit_name_expr(e) - - def visit_call_expr(self, e: CallExpr) -> Type: - return self.expr_checker.visit_call_expr(e) - - def visit_yield_from_expr(self, e: YieldFromExpr) -> Type: - return_type = self.return_types[-1] - subexpr_type = self.accept(e.expr, return_type) - iter_type = None # type: Type - - # Check that the expr is an instance of Iterable and get the type of the iterator produced - # by __iter__. - if isinstance(subexpr_type, AnyType): - iter_type = AnyType() - elif (isinstance(subexpr_type, Instance) and - is_subtype(subexpr_type, self.named_type('typing.Iterable'))): - iter_method_type = self.expr_checker.analyze_external_member_access( - '__iter__', - subexpr_type, - AnyType()) - - generic_generator_type = self.named_generic_type('typing.Generator', - [AnyType(), AnyType(), AnyType()]) - iter_type, _ = self.expr_checker.check_call(iter_method_type, [], [], - context=generic_generator_type) - else: - self.msg.yield_from_invalid_operand_type(subexpr_type, e) - iter_type = AnyType() - - # Check that the iterator's item type matches the type yielded by the Generator function - # containing this `yield from` expression. - expected_item_type = self.get_generator_yield_type(return_type) - actual_item_type = self.get_generator_yield_type(iter_type) - - self.check_subtype(actual_item_type, expected_item_type, e, - messages.INCOMPATIBLE_TYPES_IN_YIELD_FROM, - 'actual type', 'expected type') - - # Determine the type of the entire yield from expression. - if (isinstance(iter_type, Instance) and - iter_type.type.fullname() == 'typing.Generator'): - return self.get_generator_return_type(iter_type) - else: - # Non-Generators don't return anything from `yield from` expressions. - return Void() - - def visit_member_expr(self, e: MemberExpr) -> Type: - return self.expr_checker.visit_member_expr(e) - - def visit_break_stmt(self, s: BreakStmt) -> Type: - self.binder.breaking_out = True - self.binder.allow_jump(self.binder.loop_frames[-1] - 1) - return None - - def visit_continue_stmt(self, s: ContinueStmt) -> Type: - self.binder.breaking_out = True - self.binder.allow_jump(self.binder.loop_frames[-1]) - return None - - def visit_int_expr(self, e: IntExpr) -> Type: - return self.expr_checker.visit_int_expr(e) - - def visit_str_expr(self, e: StrExpr) -> Type: - return self.expr_checker.visit_str_expr(e) - - def visit_bytes_expr(self, e: BytesExpr) -> Type: - return self.expr_checker.visit_bytes_expr(e) - - def visit_unicode_expr(self, e: UnicodeExpr) -> Type: - return self.expr_checker.visit_unicode_expr(e) - - def visit_float_expr(self, e: FloatExpr) -> Type: - return self.expr_checker.visit_float_expr(e) - - def visit_complex_expr(self, e: ComplexExpr) -> Type: - return self.expr_checker.visit_complex_expr(e) - - def visit_ellipsis(self, e: EllipsisExpr) -> Type: - return self.expr_checker.visit_ellipsis(e) - - def visit_op_expr(self, e: OpExpr) -> Type: - return self.expr_checker.visit_op_expr(e) - - def visit_comparison_expr(self, e: ComparisonExpr) -> Type: - return self.expr_checker.visit_comparison_expr(e) - - def visit_unary_expr(self, e: UnaryExpr) -> Type: - return self.expr_checker.visit_unary_expr(e) - - def visit_index_expr(self, e: IndexExpr) -> Type: - return self.expr_checker.visit_index_expr(e) - - def visit_cast_expr(self, e: CastExpr) -> Type: - return self.expr_checker.visit_cast_expr(e) - - def visit_reveal_type_expr(self, e: RevealTypeExpr) -> Type: - return self.expr_checker.visit_reveal_type_expr(e) - - def visit_super_expr(self, e: SuperExpr) -> Type: - return self.expr_checker.visit_super_expr(e) - - def visit_type_application(self, e: TypeApplication) -> Type: - return self.expr_checker.visit_type_application(e) - - def visit_type_alias_expr(self, e: TypeAliasExpr) -> Type: - return self.expr_checker.visit_type_alias_expr(e) - - def visit_type_var_expr(self, e: TypeVarExpr) -> Type: - # TODO: Perhaps return a special type used for type variables only? - return AnyType() - - def visit_namedtuple_expr(self, e: NamedTupleExpr) -> Type: - # TODO: Perhaps return a type object type? - return AnyType() - - def visit_list_expr(self, e: ListExpr) -> Type: - return self.expr_checker.visit_list_expr(e) - - def visit_set_expr(self, e: SetExpr) -> Type: - return self.expr_checker.visit_set_expr(e) - - def visit_tuple_expr(self, e: TupleExpr) -> Type: - return self.expr_checker.visit_tuple_expr(e) - - def visit_dict_expr(self, e: DictExpr) -> Type: - return self.expr_checker.visit_dict_expr(e) - - def visit_slice_expr(self, e: SliceExpr) -> Type: - return self.expr_checker.visit_slice_expr(e) - - def visit_func_expr(self, e: FuncExpr) -> Type: - return self.expr_checker.visit_func_expr(e) - - def visit_list_comprehension(self, e: ListComprehension) -> Type: - return self.expr_checker.visit_list_comprehension(e) - - def visit_set_comprehension(self, e: SetComprehension) -> Type: - return self.expr_checker.visit_set_comprehension(e) - - def visit_generator_expr(self, e: GeneratorExpr) -> Type: - return self.expr_checker.visit_generator_expr(e) - - def visit_dictionary_comprehension(self, e: DictionaryComprehension) -> Type: - return self.expr_checker.visit_dictionary_comprehension(e) - - def visit_temp_node(self, e: TempNode) -> Type: - return e.type - - def visit_conditional_expr(self, e: ConditionalExpr) -> Type: - return self.expr_checker.visit_conditional_expr(e) - - def visit_backquote_expr(self, e: BackquoteExpr) -> Type: - return self.expr_checker.visit_backquote_expr(e) - - def visit_yield_expr(self, e: YieldExpr) -> Type: - return_type = self.return_types[-1] - expected_item_type = self.get_generator_yield_type(return_type) - if e.expr is None: - if (not (isinstance(expected_item_type, Void) or - isinstance(expected_item_type, AnyType)) - and self.typing_mode_full()): - self.fail(messages.YIELD_VALUE_EXPECTED, e) - else: - actual_item_type = self.accept(e.expr, expected_item_type) - self.check_subtype(actual_item_type, expected_item_type, e, - messages.INCOMPATIBLE_TYPES_IN_YIELD, - 'actual type', 'expected type') - return self.get_generator_receive_type(return_type) - - # - # Helpers - # - - def check_subtype(self, subtype: Type, supertype: Type, context: Context, - msg: str = messages.INCOMPATIBLE_TYPES, - subtype_label: str = None, - supertype_label: str = None) -> None: - """Generate an error if the subtype is not compatible with - supertype.""" - if not is_subtype(subtype, supertype): - if isinstance(subtype, Void): - self.msg.does_not_return_value(subtype, context) - else: - extra_info = [] # type: List[str] - if subtype_label is not None or supertype_label is not None: - subtype_str, supertype_str = self.msg.format_distinctly(subtype, supertype) - if subtype_label is not None: - extra_info.append(subtype_label + ' ' + subtype_str) - if supertype_label is not None: - extra_info.append(supertype_label + ' ' + supertype_str) - if extra_info: - msg += ' (' + ', '.join(extra_info) + ')' - self.fail(msg, context) - - def named_type(self, name: str) -> Instance: - """Return an instance type with type given by the name and no - type arguments. For example, named_type('builtins.object') - produces the object type. - """ - # Assume that the name refers to a type. - sym = self.lookup_qualified(name) - return Instance(cast(TypeInfo, sym.node), []) - - def named_generic_type(self, name: str, args: List[Type]) -> Instance: - """Return an instance with the given name and type arguments. - - Assume that the number of arguments is correct. Assume that - the name refers to a compatible generic type. - """ - return Instance(self.lookup_typeinfo(name), args) - - def lookup_typeinfo(self, fullname: str) -> TypeInfo: - # Assume that the name refers to a class. - sym = self.lookup_qualified(fullname) - return cast(TypeInfo, sym.node) - - def type_type(self) -> Instance: - """Return instance type 'type'.""" - return self.named_type('builtins.type') - - def object_type(self) -> Instance: - """Return instance type 'object'.""" - return self.named_type('builtins.object') - - def bool_type(self) -> Instance: - """Return instance type 'bool'.""" - return self.named_type('builtins.bool') - - def str_type(self) -> Instance: - """Return instance type 'str'.""" - return self.named_type('builtins.str') - - def check_type_equivalency(self, t1: Type, t2: Type, node: Context, - msg: str = messages.INCOMPATIBLE_TYPES) -> None: - """Generate an error if the types are not equivalent. The - dynamic type is equivalent with all types. - """ - if not is_equivalent(t1, t2): - self.fail(msg, node) - - def store_type(self, node: Node, typ: Type) -> None: - """Store the type of a node in the type map.""" - self.type_map[node] = typ - - def typing_mode_none(self) -> bool: - if self.is_dynamic_function() and not self.options.check_untyped_defs: - return not self.weak_opts - elif self.function_stack: - return False - else: - return False - - def typing_mode_weak(self) -> bool: - if self.is_dynamic_function() and not self.options.check_untyped_defs: - return bool(self.weak_opts) - elif self.function_stack: - return False - else: - return 'global' in self.weak_opts - - def typing_mode_full(self) -> bool: - if self.is_dynamic_function() and not self.options.check_untyped_defs: - return False - elif self.function_stack: - return True - else: - return 'global' not in self.weak_opts - - def is_dynamic_function(self) -> bool: - return len(self.dynamic_funcs) > 0 and self.dynamic_funcs[-1] - - def lookup(self, name: str, kind: int) -> SymbolTableNode: - """Look up a definition from the symbol table with the given name. - TODO remove kind argument - """ - if name in self.globals: - return self.globals[name] - else: - b = self.globals.get('__builtins__', None) - if b: - table = cast(MypyFile, b.node).names - if name in table: - return table[name] - raise KeyError('Failed lookup: {}'.format(name)) - - def lookup_qualified(self, name: str) -> SymbolTableNode: - if '.' not in name: - return self.lookup(name, GDEF) # FIX kind - else: - parts = name.split('.') - n = self.modules[parts[0]] - for i in range(1, len(parts) - 1): - n = cast(MypyFile, n.names.get(parts[i], None).node) - return n.names[parts[-1]] - - def enter_partial_types(self) -> None: - """Push a new scope for collecting partial types.""" - self.partial_types.append({}) - - def leave_partial_types(self) -> None: - """Pop partial type scope. - - Also report errors for variables which still have partial - types, i.e. we couldn't infer a complete type. - """ - partial_types = self.partial_types.pop() - if not self.current_node_deferred: - for var, context in partial_types.items(): - if experiments.STRICT_OPTIONAL and cast(PartialType, var.type).type is None: - # None partial type: assume variable is intended to have type None - var.type = NoneTyp() - else: - self.msg.fail(messages.NEED_ANNOTATION_FOR_VAR, context) - var.type = AnyType() - - def find_partial_types(self, var: Var) -> Optional[Dict[Var, Context]]: - for partial_types in reversed(self.partial_types): - if var in partial_types: - return partial_types - return None - - def is_within_function(self) -> bool: - """Are we currently type checking within a function? - - I.e. not at class body or at the top level. - """ - return self.return_types != [] - - def check_not_void(self, typ: Type, context: Context) -> None: - """Generate an error if the type is Void.""" - if isinstance(typ, Void): - self.msg.does_not_return_value(typ, context) - - def temp_node(self, t: Type, context: Context = None) -> Node: - """Create a temporary node with the given, fixed type.""" - temp = TempNode(t) - if context: - temp.set_line(context.get_line()) - return temp - - def fail(self, msg: str, context: Context) -> None: - """Produce an error message.""" - self.msg.fail(msg, context) - - def iterable_item_type(self, instance: Instance) -> Type: - iterable = map_instance_to_supertype( - instance, - self.lookup_typeinfo('typing.Iterable')) - return iterable.args[0] - - def function_type(self, func: FuncBase) -> FunctionLike: - return function_type(func, self.named_type('builtins.function')) - - def method_type(self, func: FuncBase) -> FunctionLike: - return method_type_with_fallback(func, self.named_type('builtins.function')) - - -# Data structure returned by find_isinstance_check representing -# information learned from the truth or falsehood of a condition. The -# dict maps nodes representing expressions like 'a[0].x' to their -# refined types under the assumption that the condition has a -# particular truth value. A value of None means that the condition can -# never have that truth value. - -# NB: The keys of this dict are nodes in the original source program, -# which are compared by reference equality--effectively, being *the -# same* expression of the program, not just two identical expressions -# (such as two references to the same variable). TODO: it would -# probably be better to have the dict keyed by the nodes' literal_hash -# field instead. - -# NB: This should be `TypeMap = Optional[Dict[Node, Type]]`! -# But see https://github.com/python/mypy/issues/1637 -TypeMap = Dict[Node, Type] - - -def conditional_type_map(expr: Node, - current_type: Optional[Type], - proposed_type: Optional[Type], - *, - weak: bool = False - ) -> Tuple[TypeMap, TypeMap]: - """Takes in an expression, the current type of the expression, and a - proposed type of that expression. - - Returns a 2-tuple: The first element is a map from the expression to - the proposed type, if the expression can be the proposed type. The - second element is a map from the expression to the type it would hold - if it was not the proposed type, if any.""" - if proposed_type: - if current_type: - if is_proper_subtype(current_type, proposed_type): - return {expr: proposed_type}, None - elif not is_overlapping_types(current_type, proposed_type): - return None, {expr: current_type} - else: - remaining_type = restrict_subtype_away(current_type, proposed_type) - return {expr: proposed_type}, {expr: remaining_type} - else: - return {expr: proposed_type}, {} - else: - # An isinstance check, but we don't understand the type - if weak: - return {expr: AnyType()}, {expr: current_type} - else: - return {}, {} - - -def is_literal_none(n: Node) -> bool: - return isinstance(n, NameExpr) and n.fullname == 'builtins.None' - - -def and_conditional_maps(m1: TypeMap, m2: TypeMap) -> TypeMap: - """Calculate what information we can learn from the truth of (e1 and e2) - in terms of the information that we can learn from the truth of e1 and - the truth of e2. - """ - - if m1 is None or m2 is None: - # One of the conditions can never be true. - return None - # Both conditions can be true; combine the information. Anything - # we learn from either conditions's truth is valid. If the same - # expression's type is refined by both conditions, we somewhat - # arbitrarily give precedence to m2. (In the future, we could use - # an intersection type.) - result = m2.copy() - m2_keys = set(n2.literal_hash for n2 in m2) - for n1 in m1: - if n1.literal_hash not in m2_keys: - result[n1] = m1[n1] - return result - - -def or_conditional_maps(m1: TypeMap, m2: TypeMap) -> TypeMap: - """Calculate what information we can learn from the truth of (e1 or e2) - in terms of the information that we can learn from the truth of e1 and - the truth of e2. - """ - - if m1 is None: - return m2 - if m2 is None: - return m1 - # Both conditions can be true. Combine information about - # expressions whose type is refined by both conditions. (We do not - # learn anything about expressions whose type is refined by only - # one condition.) - result = {} - for n1 in m1: - for n2 in m2: - if n1.literal_hash == n2.literal_hash: - result[n1] = UnionType.make_simplified_union([m1[n1], m2[n2]]) - return result - - -def find_isinstance_check(node: Node, - type_map: Dict[Node, Type], - weak: bool=False - ) -> Tuple[TypeMap, TypeMap]: - """Find any isinstance checks (within a chain of ands). Includes - implicit and explicit checks for None. - - Return value is a map of variables to their types if the condition - is true and a map of variables to their types if the condition is false. - - If either of the values in the tuple is None, then that particular - branch can never occur. - - Guaranteed to not return None, None. (But may return {}, {}) - """ - if isinstance(node, CallExpr): - if refers_to_fullname(node.callee, 'builtins.isinstance'): - expr = node.args[0] - if expr.literal == LITERAL_TYPE: - vartype = type_map[expr] - type = get_isinstance_type(node.args[1], type_map) - return conditional_type_map(expr, vartype, type, weak=weak) - elif (isinstance(node, ComparisonExpr) and any(is_literal_none(n) for n in node.operands) and - experiments.STRICT_OPTIONAL): - # Check for `x is None` and `x is not None`. - is_not = node.operators == ['is not'] - if is_not or node.operators == ['is']: - if_vars = {} # type: Dict[Node, Type] - else_vars = {} # type: Dict[Node, Type] - for expr in node.operands: - if expr.literal == LITERAL_TYPE and not is_literal_none(expr) and expr in type_map: - # This should only be true at most once: there should be - # two elements in node.operands, and at least one of them - # should represent a None. - vartype = type_map[expr] - if_vars, else_vars = conditional_type_map(expr, vartype, NoneTyp(), weak=weak) - break - - if is_not: - if_vars, else_vars = else_vars, if_vars - return if_vars, else_vars - elif isinstance(node, RefExpr) and experiments.STRICT_OPTIONAL: - # The type could be falsy, so we can't deduce anything new about the else branch - vartype = type_map[node] - _, if_vars = conditional_type_map(node, vartype, NoneTyp(), weak=weak) - return if_vars, {} - elif isinstance(node, OpExpr) and node.op == 'and': - left_if_vars, left_else_vars = find_isinstance_check( - node.left, - type_map, - weak, - ) - - right_if_vars, right_else_vars = find_isinstance_check( - node.right, - type_map, - weak, - ) - - # (e1 and e2) is true if both e1 and e2 are true, - # and false if at least one of e1 and e2 is false. - return (and_conditional_maps(left_if_vars, right_if_vars), - or_conditional_maps(left_else_vars, right_else_vars)) - elif isinstance(node, OpExpr) and node.op == 'or': - left_if_vars, left_else_vars = find_isinstance_check( - node.left, - type_map, - weak, - ) - - right_if_vars, right_else_vars = find_isinstance_check( - node.right, - type_map, - weak, - ) - - # (e1 or e2) is true if at least one of e1 or e2 is true, - # and false if both e1 and e2 are false. - return (or_conditional_maps(left_if_vars, right_if_vars), - and_conditional_maps(left_else_vars, right_else_vars)) - elif isinstance(node, UnaryExpr) and node.op == 'not': - left, right = find_isinstance_check(node.expr, type_map, weak) - return right, left - - # Not a supported isinstance check - return {}, {} - - -def get_isinstance_type(node: Node, type_map: Dict[Node, Type]) -> Type: - type = type_map[node] - - if isinstance(type, TupleType): - all_types = type.items - else: - all_types = [type] - - types = [] # type: List[Type] - - for type in all_types: - if isinstance(type, FunctionLike): - if type.is_type_obj(): - # Type variables may be present -- erase them, which is the best - # we can do (outside disallowing them here). - type = erase_typevars(type.items()[0].ret_type) - - types.append(type) - - if len(types) == 0: - return None - elif len(types) == 1: - return types[0] - else: - return UnionType(types) - - -def expand_node(defn: Node, map: Dict[TypeVarId, Type]) -> Node: - visitor = TypeTransformVisitor(map) - return defn.accept(visitor) - - -def expand_func(defn: FuncItem, map: Dict[TypeVarId, Type]) -> FuncItem: - return cast(FuncItem, expand_node(defn, map)) - - -class TypeTransformVisitor(TransformVisitor): - def __init__(self, map: Dict[TypeVarId, Type]) -> None: - super().__init__() - self.map = map - - def type(self, type: Type) -> Type: - return expand_type(type, self.map) - - -def is_unsafe_overlapping_signatures(signature: Type, other: Type) -> bool: - """Check if two signatures may be unsafely overlapping. - - Two signatures s and t are overlapping if both can be valid for the same - statically typed values and the return types are incompatible. - - Assume calls are first checked against 'signature', then against 'other'. - Thus if 'signature' is more general than 'other', there is no unsafe - overlapping. - - TODO If argument types vary covariantly, the return type may vary - covariantly as well. - """ - if isinstance(signature, CallableType): - if isinstance(other, CallableType): - # TODO varargs - # TODO keyword args - # TODO erasure - # TODO allow to vary covariantly - # Check if the argument counts are overlapping. - min_args = max(signature.min_args, other.min_args) - max_args = min(len(signature.arg_types), len(other.arg_types)) - if min_args > max_args: - # Argument counts are not overlapping. - return False - # Signatures are overlapping iff if they are overlapping for the - # smallest common argument count. - for i in range(min_args): - t1 = signature.arg_types[i] - t2 = other.arg_types[i] - if not is_overlapping_types(t1, t2): - return False - # All arguments types for the smallest common argument count are - # overlapping => the signature is overlapping. The overlapping is - # safe if the return types are identical. - if is_same_type(signature.ret_type, other.ret_type): - return False - # If the first signature has more general argument types, the - # latter will never be called - if is_more_general_arg_prefix(signature, other): - return False - return not is_more_precise_signature(signature, other) - return True - - -def is_more_general_arg_prefix(t: FunctionLike, s: FunctionLike) -> bool: - """Does t have wider arguments than s?""" - # TODO should an overload with additional items be allowed to be more - # general than one with fewer items (or just one item)? - # TODO check argument kinds - if isinstance(t, CallableType): - if isinstance(s, CallableType): - return all(is_proper_subtype(args, argt) - for argt, args in zip(t.arg_types, s.arg_types)) - elif isinstance(t, FunctionLike): - if isinstance(s, FunctionLike): - if len(t.items()) == len(s.items()): - return all(is_same_arg_prefix(items, itemt) - for items, itemt in zip(t.items(), s.items())) - return False - - -def is_same_arg_prefix(t: CallableType, s: CallableType) -> bool: - # TODO check argument kinds - return all(is_same_type(argt, args) - for argt, args in zip(t.arg_types, s.arg_types)) - - -def is_more_precise_signature(t: CallableType, s: CallableType) -> bool: - """Is t more precise than s? - - A signature t is more precise than s if all argument types and the return - type of t are more precise than the corresponding types in s. - - Assume that the argument kinds and names are compatible, and that the - argument counts are overlapping. - """ - # TODO generic function types - # Only consider the common prefix of argument types. - for argt, args in zip(t.arg_types, s.arg_types): - if not is_more_precise(argt, args): - return False - return is_more_precise(t.ret_type, s.ret_type) - - -def infer_operator_assignment_method(type: Type, operator: str) -> str: - """Return the method used for operator assignment for given value type. - - For example, if operator is '+', return '__iadd__' or '__add__' depending - on which method is supported by the type. - """ - method = nodes.op_methods[operator] - if isinstance(type, Instance): - if operator in nodes.ops_with_inplace_method: - inplace = '__i' + method[2:] - if type.type.has_readable_member(inplace): - method = inplace - return method - - -def is_valid_inferred_type(typ: Type) -> bool: - """Is an inferred type valid? - - Examples of invalid types include the None type or a type with a None component. - """ - if is_same_type(typ, NoneTyp()): - # With strict Optional checking, we *may* eventually infer NoneTyp, but - # we only do that if we can't infer a specific Optional type. This - # resolution happens in leave_partial_types when we pop a partial types - # scope. - return False - if is_same_type(typ, UninhabitedType()): - return False - elif isinstance(typ, Instance): - for arg in typ.args: - if not is_valid_inferred_type(arg): - return False - elif isinstance(typ, TupleType): - for item in typ.items: - if not is_valid_inferred_type(item): - return False - return True +"""Mypy type checker.""" + +import itertools +import contextlib +import os +import os.path + +from typing import ( + Any, Dict, Set, List, cast, Tuple, TypeVar, Union, Optional, NamedTuple +) + +from mypy.errors import Errors, report_internal_error +from mypy.nodes import ( + SymbolTable, Node, MypyFile, Var, + OverloadedFuncDef, FuncDef, FuncItem, FuncBase, TypeInfo, + ClassDef, GDEF, Block, AssignmentStmt, NameExpr, MemberExpr, IndexExpr, + TupleExpr, ListExpr, ExpressionStmt, ReturnStmt, IfStmt, + WhileStmt, OperatorAssignmentStmt, WithStmt, AssertStmt, + RaiseStmt, TryStmt, ForStmt, DelStmt, CallExpr, IntExpr, StrExpr, + BytesExpr, UnicodeExpr, FloatExpr, OpExpr, UnaryExpr, CastExpr, RevealTypeExpr, SuperExpr, + TypeApplication, DictExpr, SliceExpr, FuncExpr, TempNode, SymbolTableNode, + Context, ListComprehension, ConditionalExpr, GeneratorExpr, + Decorator, SetExpr, TypeVarExpr, PrintStmt, + LITERAL_TYPE, BreakStmt, ContinueStmt, ComparisonExpr, StarExpr, + YieldFromExpr, NamedTupleExpr, SetComprehension, + DictionaryComprehension, ComplexExpr, EllipsisExpr, TypeAliasExpr, + RefExpr, YieldExpr, BackquoteExpr, ImportFrom, ImportAll, ImportBase, + CONTRAVARIANT, COVARIANT +) +from mypy.nodes import function_type, method_type, method_type_with_fallback +from mypy import nodes +from mypy.types import ( + Type, AnyType, CallableType, Void, FunctionLike, Overloaded, TupleType, + Instance, NoneTyp, ErrorType, strip_type, + UnionType, TypeVarId, TypeVarType, PartialType, DeletedType, UninhabitedType +) +from mypy.sametypes import is_same_type +from mypy.messages import MessageBuilder +import mypy.checkexpr +from mypy.checkmember import map_type_from_supertype +from mypy import defaults +from mypy import messages +from mypy.subtypes import ( + is_subtype, is_equivalent, is_proper_subtype, + is_more_precise, restrict_subtype_away +) +from mypy.maptype import map_instance_to_supertype +from mypy.semanal import self_type, set_callable_name, refers_to_fullname +from mypy.erasetype import erase_typevars +from mypy.expandtype import expand_type_by_instance, expand_type +from mypy.visitor import NodeVisitor +from mypy.join import join_types +from mypy.treetransform import TransformVisitor +from mypy.meet import meet_simple, nearest_builtin_ancestor, is_overlapping_types +from mypy.binder import ConditionalTypeBinder +from mypy.options import Options + +from mypy import experiments + + +T = TypeVar('T') + + +# A node which is postponed to be type checked during the next pass. +DeferredNode = NamedTuple( + 'DeferredNode', + [ + ('node', Node), + ('context_type_name', Optional[str]), # Name of the surrounding class (for error messages) + ]) + + +class TypeChecker(NodeVisitor[Type]): + """Mypy type checker. + + Type check mypy source files that have been semantically analyzed. + """ + + # Are we type checking a stub? + is_stub = False + # Error message reporter + errors = None # type: Errors + # Utility for generating messages + msg = None # type: MessageBuilder + # Types of type checked nodes + type_map = None # type: Dict[Node, Type] + + # Helper for managing conditional types + binder = None # type: ConditionalTypeBinder + # Helper for type checking expressions + expr_checker = None # type: mypy.checkexpr.ExpressionChecker + + # Stack of function return types + return_types = None # type: List[Type] + # Type context for type inference + type_context = None # type: List[Type] + # Flags; true for dynamically typed functions + dynamic_funcs = None # type: List[bool] + # Stack of functions being type checked + function_stack = None # type: List[FuncItem] + # Do weak type checking in this file + weak_opts = set() # type: Set[str] + # Stack of collections of variables with partial types + partial_types = None # type: List[Dict[Var, Context]] + globals = None # type: SymbolTable + modules = None # type: Dict[str, MypyFile] + # Nodes that couldn't be checked because some types weren't available. We'll run + # another pass and try these again. + deferred_nodes = None # type: List[DeferredNode] + # Type checking pass number (0 = first pass) + pass_num = 0 + # Have we deferred the current function? If yes, don't infer additional + # types during this pass within the function. + current_node_deferred = False + is_typeshed_stub = False + options = None # type: Options + + def __init__(self, errors: Errors, modules: Dict[str, MypyFile], options: Options) -> None: + """Construct a type checker. + + Use errors to report type check errors. + """ + self.errors = errors + self.modules = modules + self.options = options + self.msg = MessageBuilder(errors, modules) + self.type_map = {} + self.binder = ConditionalTypeBinder() + self.expr_checker = mypy.checkexpr.ExpressionChecker(self, self.msg) + self.return_types = [] + self.type_context = [] + self.dynamic_funcs = [] + self.function_stack = [] + self.weak_opts = set() # type: Set[str] + self.partial_types = [] + self.deferred_nodes = [] + self.pass_num = 0 + self.current_node_deferred = False + + def visit_file(self, file_node: MypyFile, path: str) -> None: + """Type check a mypy file with the given path.""" + self.pass_num = 0 + self.is_stub = file_node.is_stub + self.errors.set_file(path) + self.globals = file_node.names + self.weak_opts = file_node.weak_opts + self.enter_partial_types() + # gross, but no other clear way to tell + self.is_typeshed_stub = self.is_stub and 'typeshed' in os.path.normpath(path).split(os.sep) + + for d in file_node.defs: + self.accept(d) + + self.leave_partial_types() + + if self.deferred_nodes: + self.check_second_pass() + + self.current_node_deferred = False + + all_ = self.globals.get('__all__') + if all_ is not None and all_.type is not None: + seq_str = self.named_generic_type('typing.Sequence', + [self.named_type('builtins.str')]) + if not is_subtype(all_.type, seq_str): + str_seq_s, all_s = self.msg.format_distinctly(seq_str, all_.type) + self.fail(messages.ALL_MUST_BE_SEQ_STR.format(str_seq_s, all_s), + all_.node) + + def check_second_pass(self): + """Run second pass of type checking which goes through deferred nodes.""" + self.pass_num = 1 + for node, type_name in self.deferred_nodes: + if type_name: + self.errors.push_type(type_name) + self.accept(node) + if type_name: + self.errors.pop_type() + self.deferred_nodes = [] + + def handle_cannot_determine_type(self, name: str, context: Context) -> None: + if self.pass_num == 0 and self.function_stack: + # Don't report an error yet. Just defer. + node = self.function_stack[-1] + if self.errors.type_name: + type_name = self.errors.type_name[-1] + else: + type_name = None + self.deferred_nodes.append(DeferredNode(node, type_name)) + # Set a marker so that we won't infer additional types in this + # function. Any inferred types could be bogus, because there's at + # least one type that we don't know. + self.current_node_deferred = True + else: + self.msg.cannot_determine_type(name, context) + + def accept(self, node: Node, type_context: Type = None) -> Type: + """Type check a node in the given type context.""" + self.type_context.append(type_context) + try: + typ = node.accept(self) + except Exception as err: + report_internal_error(err, self.errors.file, node.line) + self.type_context.pop() + self.store_type(node, typ) + if self.typing_mode_none(): + return AnyType() + else: + return typ + + def accept_loop(self, body: Node, else_body: Node = None) -> Type: + """Repeatedly type check a loop body until the frame doesn't change. + + Then check the else_body. + """ + # The outer frame accumulates the results of all iterations + with self.binder.frame_context(1) as outer_frame: + self.binder.push_loop_frame() + while True: + with self.binder.frame_context(1): + # We may skip each iteration + self.binder.options_on_return[-1].append(outer_frame) + self.accept(body) + if not self.binder.last_pop_changed: + break + self.binder.pop_loop_frame() + if else_body: + self.accept(else_body) + + # + # Definitions + # + + def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> Type: + num_abstract = 0 + if defn.is_property: + # HACK: Infer the type of the property. + self.visit_decorator(defn.items[0]) + for fdef in defn.items: + self.check_func_item(fdef.func, name=fdef.func.name()) + if fdef.func.is_abstract: + num_abstract += 1 + if num_abstract not in (0, len(defn.items)): + self.fail(messages.INCONSISTENT_ABSTRACT_OVERLOAD, defn) + if defn.info: + self.check_method_override(defn) + self.check_inplace_operator_method(defn) + self.check_overlapping_overloads(defn) + + def check_overlapping_overloads(self, defn: OverloadedFuncDef) -> None: + for i, item in enumerate(defn.items): + for j, item2 in enumerate(defn.items[i + 1:]): + # TODO overloads involving decorators + sig1 = self.function_type(item.func) + sig2 = self.function_type(item2.func) + if is_unsafe_overlapping_signatures(sig1, sig2): + self.msg.overloaded_signatures_overlap(i + 1, i + j + 2, + item.func) + + def is_generator_return_type(self, typ: Type) -> bool: + return is_subtype(self.named_generic_type('typing.Generator', + [AnyType(), AnyType(), AnyType()]), + typ) + + def get_generator_yield_type(self, return_type: Type) -> Type: + if isinstance(return_type, AnyType): + return AnyType() + elif not self.is_generator_return_type(return_type): + # If the function doesn't have a proper Generator (or superclass) return type, anything + # is permissible. + return AnyType() + elif not isinstance(return_type, Instance): + # Same as above, but written as a separate branch so the typechecker can understand. + return AnyType() + elif return_type.args: + return return_type.args[0] + else: + # If the function's declared supertype of Generator has no type + # parameters (i.e. is `object`), then the yielded values can't + # be accessed so any type is acceptable. + return AnyType() + + def get_generator_receive_type(self, return_type: Type) -> Type: + if isinstance(return_type, AnyType): + return AnyType() + elif not self.is_generator_return_type(return_type): + # If the function doesn't have a proper Generator (or superclass) return type, anything + # is permissible. + return AnyType() + elif not isinstance(return_type, Instance): + # Same as above, but written as a separate branch so the typechecker can understand. + return AnyType() + elif return_type.type.fullname() == 'typing.Generator': + # Generator is the only type which specifies the type of values it can receive. + return return_type.args[1] + else: + # `return_type` is a supertype of Generator, so callers won't be able to send it + # values. + return Void() + + def get_generator_return_type(self, return_type: Type) -> Type: + if isinstance(return_type, AnyType): + return AnyType() + elif not self.is_generator_return_type(return_type): + # If the function doesn't have a proper Generator (or superclass) return type, anything + # is permissible. + return AnyType() + elif not isinstance(return_type, Instance): + # Same as above, but written as a separate branch so the typechecker can understand. + return AnyType() + elif return_type.type.fullname() == 'typing.Generator': + # Generator is the only type which specifies the type of values it returns into + # `yield from` expressions. + return return_type.args[2] + else: + # `return_type` is supertype of Generator, so callers won't be able to see the return + # type when used in a `yield from` expression. + return AnyType() + + def visit_func_def(self, defn: FuncDef) -> Type: + """Type check a function definition.""" + self.check_func_item(defn, name=defn.name()) + if defn.info: + if not defn.is_dynamic(): + self.check_method_override(defn) + self.check_inplace_operator_method(defn) + if defn.original_def: + # Override previous definition. + new_type = self.function_type(defn) + if isinstance(defn.original_def, FuncDef): + # Function definition overrides function definition. + if not is_same_type(new_type, self.function_type(defn.original_def)): + self.msg.incompatible_conditional_function_def(defn) + else: + # Function definition overrides a variable initialized via assignment. + orig_type = defn.original_def.type + if orig_type is None: + # XXX This can be None, as happens in + # test_testcheck_TypeCheckSuite.testRedefinedFunctionInTryWithElse + self.msg.note("Internal mypy error checking function redefinition.", defn) + return None + if isinstance(orig_type, PartialType): + if orig_type.type is None: + # Ah this is a partial type. Give it the type of the function. + var = defn.original_def + partial_types = self.find_partial_types(var) + if partial_types is not None: + var.type = new_type + del partial_types[var] + else: + # Trying to redefine something like partial empty list as function. + self.fail(messages.INCOMPATIBLE_REDEFINITION, defn) + else: + # TODO: Update conditional type binder. + self.check_subtype(new_type, orig_type, defn, + messages.INCOMPATIBLE_REDEFINITION, + 'redefinition with type', + 'original type') + + def check_func_item(self, defn: FuncItem, + type_override: CallableType = None, + name: str = None) -> Type: + """Type check a function. + + If type_override is provided, use it as the function type. + """ + # We may be checking a function definition or an anonymous function. In + # the first case, set up another reference with the precise type. + fdef = None # type: FuncDef + if isinstance(defn, FuncDef): + fdef = defn + + self.function_stack.append(defn) + self.dynamic_funcs.append(defn.is_dynamic() and not type_override) + + if fdef: + self.errors.push_function(fdef.name()) + + self.enter_partial_types() + + typ = self.function_type(defn) + if type_override: + typ = type_override + if isinstance(typ, CallableType): + self.check_func_def(defn, typ, name) + else: + raise RuntimeError('Not supported') + + self.leave_partial_types() + + if fdef: + self.errors.pop_function() + + self.dynamic_funcs.pop() + self.function_stack.pop() + self.current_node_deferred = False + + def check_func_def(self, defn: FuncItem, typ: CallableType, name: str) -> None: + """Type check a function definition.""" + # Expand type variables with value restrictions to ordinary types. + for item, typ in self.expand_typevars(defn, typ): + old_binder = self.binder + self.binder = ConditionalTypeBinder() + with self.binder.frame_context(): + defn.expanded.append(item) + + # We may be checking a function definition or an anonymous + # function. In the first case, set up another reference with the + # precise type. + if isinstance(item, FuncDef): + fdef = item + else: + fdef = None + + if fdef: + # Check if __init__ has an invalid, non-None return type. + if (fdef.info and fdef.name() == '__init__' and + not isinstance(typ.ret_type, Void) and + not self.dynamic_funcs[-1]): + self.fail(messages.INIT_MUST_HAVE_NONE_RETURN_TYPE, + item.type) + + show_untyped = not self.is_typeshed_stub or self.options.warn_incomplete_stub + if self.options.disallow_untyped_defs and show_untyped: + # Check for functions with unspecified/not fully specified types. + def is_implicit_any(t: Type) -> bool: + return isinstance(t, AnyType) and t.implicit + + if fdef.type is None: + self.fail(messages.FUNCTION_TYPE_EXPECTED, fdef) + elif isinstance(fdef.type, CallableType): + if is_implicit_any(fdef.type.ret_type): + self.fail(messages.RETURN_TYPE_EXPECTED, fdef) + if any(is_implicit_any(t) for t in fdef.type.arg_types): + self.fail(messages.ARGUMENT_TYPE_EXPECTED, fdef) + + if name in nodes.reverse_op_method_set: + self.check_reverse_op_method(item, typ, name) + elif name == '__getattr__': + self.check_getattr_method(typ, defn) + + # Refuse contravariant return type variable + if isinstance(typ.ret_type, TypeVarType): + if typ.ret_type.variance == CONTRAVARIANT: + self.fail(messages.RETURN_TYPE_CANNOT_BE_CONTRAVARIANT, + typ.ret_type) + + # Check that Generator functions have the appropriate return type. + if defn.is_generator: + if not self.is_generator_return_type(typ.ret_type): + self.fail(messages.INVALID_RETURN_TYPE_FOR_GENERATOR, typ) + + # Python 2 generators aren't allowed to return values. + if (self.options.python_version[0] == 2 and + isinstance(typ.ret_type, Instance) and + typ.ret_type.type.fullname() == 'typing.Generator'): + if not (isinstance(typ.ret_type.args[2], Void) + or isinstance(typ.ret_type.args[2], AnyType)): + self.fail(messages.INVALID_GENERATOR_RETURN_ITEM_TYPE, typ) + + # Push return type. + self.return_types.append(typ.ret_type) + + # Store argument types. + for i in range(len(typ.arg_types)): + arg_type = typ.arg_types[i] + + # Refuse covariant parameter type variables + if isinstance(arg_type, TypeVarType): + if arg_type.variance == COVARIANT: + self.fail(messages.FUNCTION_PARAMETER_CANNOT_BE_COVARIANT, + arg_type) + + if typ.arg_kinds[i] == nodes.ARG_STAR: + # builtins.tuple[T] is typing.Tuple[T, ...] + arg_type = self.named_generic_type('builtins.tuple', + [arg_type]) + elif typ.arg_kinds[i] == nodes.ARG_STAR2: + arg_type = self.named_generic_type('builtins.dict', + [self.str_type(), + arg_type]) + item.arguments[i].variable.type = arg_type + + # Type check initialization expressions. + for arg in item.arguments: + init = arg.initialization_statement + if init: + self.accept(init) + + # Type check body in a new scope. + with self.binder.frame_context(): + self.accept(item.body) + + self.return_types.pop() + + self.binder = old_binder + + def check_reverse_op_method(self, defn: FuncItem, typ: CallableType, + method: str) -> None: + """Check a reverse operator method such as __radd__.""" + + # This used to check for some very obscure scenario. It now + # just decides whether it's worth calling + # check_overlapping_op_methods(). + + if method in ('__eq__', '__ne__'): + # These are defined for all objects => can't cause trouble. + return + + # With 'Any' or 'object' return type we are happy, since any possible + # return value is valid. + ret_type = typ.ret_type + if isinstance(ret_type, AnyType): + return + if isinstance(ret_type, Instance): + if ret_type.type.fullname() == 'builtins.object': + return + # Plausibly the method could have too few arguments, which would result + # in an error elsewhere. + if len(typ.arg_types) <= 2: + # TODO check self argument kind + + # Check for the issue described above. + arg_type = typ.arg_types[1] + other_method = nodes.normal_from_reverse_op[method] + if isinstance(arg_type, Instance): + if not arg_type.type.has_readable_member(other_method): + return + elif isinstance(arg_type, AnyType): + return + elif isinstance(arg_type, UnionType): + if not arg_type.has_readable_member(other_method): + return + else: + return + + typ2 = self.expr_checker.analyze_external_member_access( + other_method, arg_type, defn) + self.check_overlapping_op_methods( + typ, method, defn.info, + typ2, other_method, cast(Instance, arg_type), + defn) + + def check_overlapping_op_methods(self, + reverse_type: CallableType, + reverse_name: str, + reverse_class: TypeInfo, + forward_type: Type, + forward_name: str, + forward_base: Instance, + context: Context) -> None: + """Check for overlapping method and reverse method signatures. + + Assume reverse method has valid argument count and kinds. + """ + + # Reverse operator method that overlaps unsafely with the + # forward operator method can result in type unsafety. This is + # similar to overlapping overload variants. + # + # This example illustrates the issue: + # + # class X: pass + # class A: + # def __add__(self, x: X) -> int: + # if isinstance(x, X): + # return 1 + # return NotImplemented + # class B: + # def __radd__(self, x: A) -> str: return 'x' + # class C(X, B): pass + # def f(b: B) -> None: + # A() + b # Result is 1, even though static type seems to be str! + # f(C()) + # + # The reason for the problem is that B and X are overlapping + # types, and the return types are different. Also, if the type + # of x in __radd__ would not be A, the methods could be + # non-overlapping. + + if isinstance(forward_type, CallableType): + # TODO check argument kinds + if len(forward_type.arg_types) < 1: + # Not a valid operator method -- can't succeed anyway. + return + + # Construct normalized function signatures corresponding to the + # operator methods. The first argument is the left operand and the + # second operand is the right argument -- we switch the order of + # the arguments of the reverse method. + forward_tweaked = CallableType( + [forward_base, forward_type.arg_types[0]], + [nodes.ARG_POS] * 2, + [None] * 2, + forward_type.ret_type, + forward_type.fallback, + name=forward_type.name) + reverse_args = reverse_type.arg_types + reverse_tweaked = CallableType( + [reverse_args[1], reverse_args[0]], + [nodes.ARG_POS] * 2, + [None] * 2, + reverse_type.ret_type, + fallback=self.named_type('builtins.function'), + name=reverse_type.name) + + if is_unsafe_overlapping_signatures(forward_tweaked, + reverse_tweaked): + self.msg.operator_method_signatures_overlap( + reverse_class.name(), reverse_name, + forward_base.type.name(), forward_name, context) + elif isinstance(forward_type, Overloaded): + for item in forward_type.items(): + self.check_overlapping_op_methods( + reverse_type, reverse_name, reverse_class, + item, forward_name, forward_base, context) + elif not isinstance(forward_type, AnyType): + self.msg.forward_operator_not_callable(forward_name, context) + + def check_inplace_operator_method(self, defn: FuncBase) -> None: + """Check an inplace operator method such as __iadd__. + + They cannot arbitrarily overlap with __add__. + """ + method = defn.name() + if method not in nodes.inplace_operator_methods: + return + typ = self.method_type(defn) + cls = defn.info + other_method = '__' + method[3:] + if cls.has_readable_member(other_method): + instance = self_type(cls) + typ2 = self.expr_checker.analyze_external_member_access( + other_method, instance, defn) + fail = False + if isinstance(typ2, FunctionLike): + if not is_more_general_arg_prefix(typ, typ2): + fail = True + else: + # TODO overloads + fail = True + if fail: + self.msg.signatures_incompatible(method, other_method, defn) + + def check_getattr_method(self, typ: CallableType, context: Context) -> None: + method_type = CallableType([AnyType(), self.named_type('builtins.str')], + [nodes.ARG_POS, nodes.ARG_POS], + [None], + AnyType(), + self.named_type('builtins.function')) + if not is_subtype(typ, method_type): + self.msg.invalid_signature(typ, context) + + def expand_typevars(self, defn: FuncItem, + typ: CallableType) -> List[Tuple[FuncItem, CallableType]]: + # TODO use generator + subst = [] # type: List[List[Tuple[TypeVarId, Type]]] + tvars = typ.variables or [] + tvars = tvars[:] + if defn.info: + # Class type variables + tvars += defn.info.defn.type_vars or [] + for tvar in tvars: + if tvar.values: + subst.append([(tvar.id, value) + for value in tvar.values]) + if subst: + result = [] # type: List[Tuple[FuncItem, CallableType]] + for substitutions in itertools.product(*subst): + mapping = dict(substitutions) + expanded = cast(CallableType, expand_type(typ, mapping)) + result.append((expand_func(defn, mapping), expanded)) + return result + else: + return [(defn, typ)] + + def check_method_override(self, defn: FuncBase) -> None: + """Check if function definition is compatible with base classes.""" + # Check against definitions in base classes. + for base in defn.info.mro[1:]: + self.check_method_or_accessor_override_for_base(defn, base) + + def check_method_or_accessor_override_for_base(self, defn: FuncBase, + base: TypeInfo) -> None: + """Check if method definition is compatible with a base class.""" + if base: + name = defn.name() + if name not in ('__init__', '__new__'): + # Check method override (__init__ and __new__ are special). + self.check_method_override_for_base_with_name(defn, name, base) + if name in nodes.inplace_operator_methods: + # Figure out the name of the corresponding operator method. + method = '__' + name[3:] + # An inplace operator method such as __iadd__ might not be + # always introduced safely if a base class defined __add__. + # TODO can't come up with an example where this is + # necessary; now it's "just in case" + self.check_method_override_for_base_with_name(defn, method, + base) + + def check_method_override_for_base_with_name( + self, defn: FuncBase, name: str, base: TypeInfo) -> None: + base_attr = base.names.get(name) + if base_attr: + # The name of the method is defined in the base class. + + # Construct the type of the overriding method. + typ = self.method_type(defn) + # Map the overridden method type to subtype context so that + # it can be checked for compatibility. + original_type = base_attr.type + if original_type is None and isinstance(base_attr.node, + FuncDef): + original_type = self.function_type(base_attr.node) + if isinstance(original_type, FunctionLike): + original = map_type_from_supertype( + method_type(original_type), + defn.info, base) + # Check that the types are compatible. + # TODO overloaded signatures + self.check_override(typ, + cast(FunctionLike, original), + defn.name(), + name, + base.name(), + defn) + else: + assert original_type is not None + self.msg.signature_incompatible_with_supertype( + defn.name(), name, base.name(), defn) + + def check_override(self, override: FunctionLike, original: FunctionLike, + name: str, name_in_super: str, supertype: str, + node: Context) -> None: + """Check a method override with given signatures. + + Arguments: + override: The signature of the overriding method. + original: The signature of the original supertype method. + name: The name of the subtype. This and the next argument are + only used for generating error messages. + supertype: The name of the supertype. + """ + # Use boolean variable to clarify code. + fail = False + if not is_subtype(override, original): + fail = True + elif (not isinstance(original, Overloaded) and + isinstance(override, Overloaded) and + name in nodes.reverse_op_methods.keys()): + # Operator method overrides cannot introduce overloading, as + # this could be unsafe with reverse operator methods. + fail = True + + if fail: + emitted_msg = False + if (isinstance(override, CallableType) and + isinstance(original, CallableType) and + len(override.arg_types) == len(original.arg_types) and + override.min_args == original.min_args): + # Give more detailed messages for the common case of both + # signatures having the same number of arguments and no + # overloads. + + # override might have its own generic function type + # variables. If an argument or return type of override + # does not have the correct subtyping relationship + # with the original type even after these variables + # are erased, then it is definitely an incompatiblity. + + override_ids = override.type_var_ids() + + def erase_override(t: Type) -> Type: + return erase_typevars(t, ids_to_erase=override_ids) + + for i in range(len(override.arg_types)): + if not is_subtype(original.arg_types[i], + erase_override(override.arg_types[i])): + self.msg.argument_incompatible_with_supertype( + i + 1, name, name_in_super, supertype, node) + emitted_msg = True + + if not is_subtype(erase_override(override.ret_type), + original.ret_type): + self.msg.return_type_incompatible_with_supertype( + name, name_in_super, supertype, node) + emitted_msg = True + + if not emitted_msg: + # Fall back to generic incompatibility message. + self.msg.signature_incompatible_with_supertype( + name, name_in_super, supertype, node) + + def visit_class_def(self, defn: ClassDef) -> Type: + """Type check a class definition.""" + typ = defn.info + self.errors.push_type(defn.name) + self.enter_partial_types() + old_binder = self.binder + self.binder = ConditionalTypeBinder() + with self.binder.frame_context(): + self.accept(defn.defs) + self.binder = old_binder + if not hasattr(defn, 'incompatible_baseclass'): + # Otherwise we've already found errors; more errors are not useful + self.check_multiple_inheritance(typ) + self.leave_partial_types() + self.errors.pop_type() + + def check_multiple_inheritance(self, typ: TypeInfo) -> None: + """Check for multiple inheritance related errors.""" + + if len(typ.bases) <= 1: + # No multiple inheritance. + return + # Verify that inherited attributes are compatible. + mro = typ.mro[1:] + for i, base in enumerate(mro): + for name in base.names: + for base2 in mro[i + 1:]: + # We only need to check compatibility of attributes from classes not + # in a subclass relationship. For subclasses, normal (single inheritance) + # checks suffice (these are implemented elsewhere). + if name in base2.names and base2 not in base.mro: + self.check_compatibility(name, base, base2, typ) + # Verify that base class layouts are compatible. + builtin_bases = [nearest_builtin_ancestor(base.type) + for base in typ.bases] + for base1 in builtin_bases: + for base2 in builtin_bases: + if not (base1 in base2.mro or base2 in base1.mro): + self.fail(messages.INSTANCE_LAYOUT_CONFLICT, typ) + + def check_compatibility(self, name: str, base1: TypeInfo, + base2: TypeInfo, ctx: Context) -> None: + """Check if attribute name in base1 is compatible with base2 in multiple inheritance. + + Assume base1 comes before base2 in the MRO, and that base1 and base2 don't have + a direct subclass relationship (i.e., the compatibility requirement only derives from + multiple inheritance). + """ + if name == '__init__': + # __init__ can be incompatible -- it's a special case. + return + first = base1[name] + second = base2[name] + first_type = first.type + if first_type is None and isinstance(first.node, FuncDef): + first_type = self.function_type(first.node) + second_type = second.type + if second_type is None and isinstance(second.node, FuncDef): + second_type = self.function_type(second.node) + # TODO: What if some classes are generic? + if (isinstance(first_type, FunctionLike) and + isinstance(second_type, FunctionLike)): + # Method override + first_sig = method_type(first_type) + second_sig = method_type(second_type) + ok = is_subtype(first_sig, second_sig) + elif first_type and second_type: + ok = is_equivalent(first_type, second_type) + else: + if first_type is None: + self.msg.cannot_determine_type_in_base(name, base1.name(), ctx) + if second_type is None: + self.msg.cannot_determine_type_in_base(name, base2.name(), ctx) + ok = True + if not ok: + self.msg.base_class_definitions_incompatible(name, base1, base2, + ctx) + + def visit_import_from(self, node: ImportFrom) -> Type: + self.check_import(node) + + def visit_import_all(self, node: ImportAll) -> Type: + self.check_import(node) + + def check_import(self, node: ImportBase) -> Type: + for assign in node.assignments: + lvalue = assign.lvalues[0] + lvalue_type, _, __ = self.check_lvalue(lvalue) + if lvalue_type is None: + # TODO: This is broken. + lvalue_type = AnyType() + message = '{} "{}"'.format(messages.INCOMPATIBLE_IMPORT_OF, + cast(NameExpr, assign.rvalue).name) + self.check_simple_assignment(lvalue_type, assign.rvalue, node, + msg=message, lvalue_name='local name', + rvalue_name='imported name') + + # + # Statements + # + + def visit_block(self, b: Block) -> Type: + if b.is_unreachable: + return None + for s in b.body: + self.accept(s) + if self.binder.breaking_out: + break + + def visit_assignment_stmt(self, s: AssignmentStmt) -> Type: + """Type check an assignment statement. + + Handle all kinds of assignment statements (simple, indexed, multiple). + """ + self.check_assignment(s.lvalues[-1], s.rvalue, s.type is None) + + if len(s.lvalues) > 1: + # Chained assignment (e.g. x = y = ...). + # Make sure that rvalue type will not be reinferred. + if s.rvalue not in self.type_map: + self.accept(s.rvalue) + rvalue = self.temp_node(self.type_map[s.rvalue], s) + for lv in s.lvalues[:-1]: + self.check_assignment(lv, rvalue, s.type is None) + + def check_assignment(self, lvalue: Node, rvalue: Node, infer_lvalue_type: bool = True) -> None: + """Type check a single assignment: lvalue = rvalue.""" + if isinstance(lvalue, TupleExpr) or isinstance(lvalue, ListExpr): + ltuple = cast(Union[TupleExpr, ListExpr], lvalue) + + self.check_assignment_to_multiple_lvalues(ltuple.items, rvalue, lvalue, + infer_lvalue_type) + else: + lvalue_type, index_lvalue, inferred = self.check_lvalue(lvalue) + if lvalue_type: + if isinstance(lvalue_type, PartialType) and lvalue_type.type is None: + # Try to infer a proper type for a variable with a partial None type. + rvalue_type = self.accept(rvalue) + if isinstance(rvalue_type, NoneTyp): + # This doesn't actually provide any additional information -- multiple + # None initializers preserve the partial None type. + return + if is_valid_inferred_type(rvalue_type): + var = lvalue_type.var + partial_types = self.find_partial_types(var) + if partial_types is not None: + if not self.current_node_deferred: + if experiments.STRICT_OPTIONAL: + var.type = UnionType.make_simplified_union( + [rvalue_type, NoneTyp()]) + else: + var.type = rvalue_type + else: + var.type = None + del partial_types[var] + # Try to infer a partial type. No need to check the return value, as + # an error will be reported elsewhere. + self.infer_partial_type(lvalue_type.var, lvalue, rvalue_type) + return + if (is_literal_none(rvalue) and + isinstance(lvalue, NameExpr) and + isinstance(lvalue.node, Var) and + lvalue.node.is_initialized_in_class): + # Allow None's to be assigned to class variables with non-Optional types. + rvalue_type = lvalue_type + else: + rvalue_type = self.check_simple_assignment(lvalue_type, rvalue, lvalue) + + if rvalue_type and infer_lvalue_type: + self.binder.assign_type(lvalue, + rvalue_type, + lvalue_type, + self.typing_mode_weak()) + elif index_lvalue: + self.check_indexed_assignment(index_lvalue, rvalue, rvalue) + + if inferred: + self.infer_variable_type(inferred, lvalue, self.accept(rvalue), + rvalue) + + def check_assignment_to_multiple_lvalues(self, lvalues: List[Node], rvalue: Node, + context: Context, + infer_lvalue_type: bool = True) -> None: + if isinstance(rvalue, TupleExpr) or isinstance(rvalue, ListExpr): + # Recursively go into Tuple or List expression rhs instead of + # using the type of rhs, because this allowed more fine grained + # control in cases like: a, b = [int, str] where rhs would get + # type List[object] + + rvalues = cast(Union[TupleExpr, ListExpr], rvalue).items + + if self.check_rvalue_count_in_assignment(lvalues, len(rvalues), context): + star_index = next((i for i, lv in enumerate(lvalues) if + isinstance(lv, StarExpr)), len(lvalues)) + + left_lvs = lvalues[:star_index] + star_lv = cast(StarExpr, + lvalues[star_index]) if star_index != len(lvalues) else None + right_lvs = lvalues[star_index + 1:] + + left_rvs, star_rvs, right_rvs = self.split_around_star( + rvalues, star_index, len(lvalues)) + + lr_pairs = list(zip(left_lvs, left_rvs)) + if star_lv: + rv_list = ListExpr(star_rvs) + rv_list.set_line(rvalue.get_line()) + lr_pairs.append((star_lv.expr, rv_list)) + lr_pairs.extend(zip(right_lvs, right_rvs)) + + for lv, rv in lr_pairs: + self.check_assignment(lv, rv, infer_lvalue_type) + else: + self.check_multi_assignment(lvalues, rvalue, context, infer_lvalue_type) + + def check_rvalue_count_in_assignment(self, lvalues: List[Node], rvalue_count: int, + context: Context) -> bool: + if any(isinstance(lvalue, StarExpr) for lvalue in lvalues): + if len(lvalues) - 1 > rvalue_count: + self.msg.wrong_number_values_to_unpack(rvalue_count, + len(lvalues) - 1, context) + return False + elif rvalue_count != len(lvalues): + self.msg.wrong_number_values_to_unpack(rvalue_count, + len(lvalues), context) + return False + return True + + def check_multi_assignment(self, lvalues: List[Node], + rvalue: Node, + context: Context, + infer_lvalue_type: bool = True, + msg: str = None) -> None: + """Check the assignment of one rvalue to a number of lvalues.""" + + # Infer the type of an ordinary rvalue expression. + rvalue_type = self.accept(rvalue) # TODO maybe elsewhere; redundant + undefined_rvalue = False + + if isinstance(rvalue_type, AnyType): + for lv in lvalues: + if isinstance(lv, StarExpr): + lv = lv.expr + self.check_assignment(lv, self.temp_node(AnyType(), context), infer_lvalue_type) + elif isinstance(rvalue_type, TupleType): + self.check_multi_assignment_from_tuple(lvalues, rvalue, rvalue_type, + context, undefined_rvalue, infer_lvalue_type) + else: + self.check_multi_assignment_from_iterable(lvalues, rvalue_type, + context, infer_lvalue_type) + + def check_multi_assignment_from_tuple(self, lvalues: List[Node], rvalue: Node, + rvalue_type: TupleType, context: Context, + undefined_rvalue: bool, + infer_lvalue_type: bool = True) -> None: + if self.check_rvalue_count_in_assignment(lvalues, len(rvalue_type.items), context): + star_index = next((i for i, lv in enumerate(lvalues) + if isinstance(lv, StarExpr)), len(lvalues)) + + left_lvs = lvalues[:star_index] + star_lv = cast(StarExpr, lvalues[star_index]) if star_index != len(lvalues) else None + right_lvs = lvalues[star_index + 1:] + + if not undefined_rvalue: + # Infer rvalue again, now in the correct type context. + lvalue_type = self.lvalue_type_for_inference(lvalues, rvalue_type) + rvalue_type = cast(TupleType, self.accept(rvalue, lvalue_type)) + + left_rv_types, star_rv_types, right_rv_types = self.split_around_star( + rvalue_type.items, star_index, len(lvalues)) + + for lv, rv_type in zip(left_lvs, left_rv_types): + self.check_assignment(lv, self.temp_node(rv_type, context), infer_lvalue_type) + if star_lv: + nodes = [self.temp_node(rv_type, context) for rv_type in star_rv_types] + list_expr = ListExpr(nodes) + list_expr.set_line(context.get_line()) + self.check_assignment(star_lv.expr, list_expr, infer_lvalue_type) + for lv, rv_type in zip(right_lvs, right_rv_types): + self.check_assignment(lv, self.temp_node(rv_type, context), infer_lvalue_type) + + def lvalue_type_for_inference(self, lvalues: List[Node], rvalue_type: TupleType) -> Type: + star_index = next((i for i, lv in enumerate(lvalues) + if isinstance(lv, StarExpr)), len(lvalues)) + left_lvs = lvalues[:star_index] + star_lv = cast(StarExpr, lvalues[star_index]) if star_index != len(lvalues) else None + right_lvs = lvalues[star_index + 1:] + left_rv_types, star_rv_types, right_rv_types = self.split_around_star( + rvalue_type.items, star_index, len(lvalues)) + + type_parameters = [] # type: List[Type] + + def append_types_for_inference(lvs: List[Node], rv_types: List[Type]) -> None: + for lv, rv_type in zip(lvs, rv_types): + sub_lvalue_type, index_expr, inferred = self.check_lvalue(lv) + if sub_lvalue_type: + type_parameters.append(sub_lvalue_type) + else: # index lvalue + # TODO Figure out more precise type context, probably + # based on the type signature of the _set method. + type_parameters.append(rv_type) + + append_types_for_inference(left_lvs, left_rv_types) + + if star_lv: + sub_lvalue_type, index_expr, inferred = self.check_lvalue(star_lv.expr) + if sub_lvalue_type: + type_parameters.extend([sub_lvalue_type] * len(star_rv_types)) + else: # index lvalue + # TODO Figure out more precise type context, probably + # based on the type signature of the _set method. + type_parameters.extend(star_rv_types) + + append_types_for_inference(right_lvs, right_rv_types) + + return TupleType(type_parameters, self.named_type('builtins.tuple')) + + def split_around_star(self, items: List[T], star_index: int, + length: int) -> Tuple[List[T], List[T], List[T]]: + """Splits a list of items in three to match another list of length 'length' + that contains a starred expression at 'star_index' in the following way: + + star_index = 2, length = 5 (i.e., [a,b,*,c,d]), items = [1,2,3,4,5,6,7] + returns in: ([1,2], [3,4,5], [6,7]) + """ + nr_right_of_star = length - star_index - 1 + right_index = nr_right_of_star if -nr_right_of_star != 0 else len(items) + left = items[:star_index] + star = items[star_index:right_index] + right = items[right_index:] + return (left, star, right) + + def type_is_iterable(self, type: Type) -> bool: + return (is_subtype(type, self.named_generic_type('typing.Iterable', + [AnyType()])) and + isinstance(type, Instance)) + + def check_multi_assignment_from_iterable(self, lvalues: List[Node], rvalue_type: Type, + context: Context, + infer_lvalue_type: bool = True) -> None: + if self.type_is_iterable(rvalue_type): + item_type = self.iterable_item_type(cast(Instance, rvalue_type)) + for lv in lvalues: + if isinstance(lv, StarExpr): + self.check_assignment(lv.expr, self.temp_node(rvalue_type, context), + infer_lvalue_type) + else: + self.check_assignment(lv, self.temp_node(item_type, context), + infer_lvalue_type) + else: + self.msg.type_not_iterable(rvalue_type, context) + + def check_lvalue(self, lvalue: Node) -> Tuple[Type, IndexExpr, Var]: + lvalue_type = None # type: Type + index_lvalue = None # type: IndexExpr + inferred = None # type: Var + + if self.is_definition(lvalue): + if isinstance(lvalue, NameExpr): + inferred = cast(Var, lvalue.node) + assert isinstance(inferred, Var) + else: + m = cast(MemberExpr, lvalue) + self.accept(m.expr) + inferred = m.def_var + elif isinstance(lvalue, IndexExpr): + index_lvalue = lvalue + elif isinstance(lvalue, MemberExpr): + lvalue_type = self.expr_checker.analyze_ordinary_member_access(lvalue, + True) + self.store_type(lvalue, lvalue_type) + elif isinstance(lvalue, NameExpr): + lvalue_type = self.expr_checker.analyze_ref_expr(lvalue, lvalue=True) + self.store_type(lvalue, lvalue_type) + elif isinstance(lvalue, TupleExpr) or isinstance(lvalue, ListExpr): + lv = cast(Union[TupleExpr, ListExpr], lvalue) + types = [self.check_lvalue(sub_expr)[0] for sub_expr in lv.items] + lvalue_type = TupleType(types, self.named_type('builtins.tuple')) + else: + lvalue_type = self.accept(lvalue) + + return lvalue_type, index_lvalue, inferred + + def is_definition(self, s: Node) -> bool: + if isinstance(s, NameExpr): + if s.is_def: + return True + # If the node type is not defined, this must the first assignment + # that we process => this is a definition, even though the semantic + # analyzer did not recognize this as such. This can arise in code + # that uses isinstance checks, if type checking of the primary + # definition is skipped due to an always False type check. + node = s.node + if isinstance(node, Var): + return node.type is None + elif isinstance(s, MemberExpr): + return s.is_def + return False + + def infer_variable_type(self, name: Var, lvalue: Node, + init_type: Type, context: Context) -> None: + """Infer the type of initialized variables from initializer type.""" + if self.typing_mode_weak(): + self.set_inferred_type(name, lvalue, AnyType()) + self.binder.assign_type(lvalue, init_type, self.binder.get_declaration(lvalue), True) + elif isinstance(init_type, Void): + self.check_not_void(init_type, context) + self.set_inference_error_fallback_type(name, lvalue, init_type, context) + elif isinstance(init_type, DeletedType): + self.msg.deleted_as_rvalue(init_type, context) + elif not is_valid_inferred_type(init_type): + # We cannot use the type of the initialization expression for full type + # inference (it's not specific enough), but we might be able to give + # partial type which will be made more specific later. A partial type + # gets generated in assignment like 'x = []' where item type is not known. + if not self.infer_partial_type(name, lvalue, init_type): + self.fail(messages.NEED_ANNOTATION_FOR_VAR, context) + self.set_inference_error_fallback_type(name, lvalue, init_type, context) + else: + # Infer type of the target. + + # Make the type more general (strip away function names etc.). + init_type = strip_type(init_type) + + self.set_inferred_type(name, lvalue, init_type) + + def infer_partial_type(self, name: Var, lvalue: Node, init_type: Type) -> bool: + if isinstance(init_type, (NoneTyp, UninhabitedType)): + partial_type = PartialType(None, name, [init_type]) + elif isinstance(init_type, Instance): + fullname = init_type.type.fullname() + if (isinstance(lvalue, NameExpr) and + (fullname == 'builtins.list' or + fullname == 'builtins.set' or + fullname == 'builtins.dict') and + all(isinstance(t, (NoneTyp, UninhabitedType)) for t in init_type.args)): + partial_type = PartialType(init_type.type, name, init_type.args) + else: + return False + else: + return False + self.set_inferred_type(name, lvalue, partial_type) + self.partial_types[-1][name] = lvalue + return True + + def set_inferred_type(self, var: Var, lvalue: Node, type: Type) -> None: + """Store inferred variable type. + + Store the type to both the variable node and the expression node that + refers to the variable (lvalue). If var is None, do nothing. + """ + if var and not self.current_node_deferred: + var.type = type + self.store_type(lvalue, type) + + def set_inference_error_fallback_type(self, var: Var, lvalue: Node, type: Type, + context: Context) -> None: + """If errors on context line are ignored, store dummy type for variable. + + If a program ignores error on type inference error, the variable should get some + inferred type so that if can used later on in the program. Example: + + x = [] # type: ignore + x.append(1) # Should be ok! + + We implement this here by giving x a valid type (Any). + """ + if context.get_line() in self.errors.ignored_lines[self.errors.file]: + self.set_inferred_type(var, lvalue, AnyType()) + + def narrow_type_from_binder(self, expr: Node, known_type: Type) -> Type: + if expr.literal >= LITERAL_TYPE: + restriction = self.binder.get(expr) + if restriction: + ans = meet_simple(known_type, restriction) + return ans + return known_type + + def check_simple_assignment(self, lvalue_type: Type, rvalue: Node, + context: Node, + msg: str = messages.INCOMPATIBLE_TYPES_IN_ASSIGNMENT, + lvalue_name: str = 'variable', + rvalue_name: str = 'expression') -> Type: + if self.is_stub and isinstance(rvalue, EllipsisExpr): + # '...' is always a valid initializer in a stub. + return AnyType() + else: + rvalue_type = self.accept(rvalue, lvalue_type) + if isinstance(rvalue_type, DeletedType): + self.msg.deleted_as_rvalue(rvalue_type, context) + if self.typing_mode_weak(): + return rvalue_type + if isinstance(lvalue_type, DeletedType): + self.msg.deleted_as_lvalue(lvalue_type, context) + else: + self.check_subtype(rvalue_type, lvalue_type, context, msg, + '{} has type'.format(rvalue_name), + '{} has type'.format(lvalue_name)) + return rvalue_type + + def check_indexed_assignment(self, lvalue: IndexExpr, + rvalue: Node, context: Context) -> None: + """Type check indexed assignment base[index] = rvalue. + + The lvalue argument is the base[index] expression. + """ + self.try_infer_partial_type_from_indexed_assignment(lvalue, rvalue) + basetype = self.accept(lvalue.base) + method_type = self.expr_checker.analyze_external_member_access( + '__setitem__', basetype, context) + lvalue.method_type = method_type + self.expr_checker.check_call(method_type, [lvalue.index, rvalue], + [nodes.ARG_POS, nodes.ARG_POS], + context) + + def try_infer_partial_type_from_indexed_assignment( + self, lvalue: IndexExpr, rvalue: Node) -> None: + # TODO: Should we share some of this with try_infer_partial_type? + if isinstance(lvalue.base, RefExpr) and isinstance(lvalue.base.node, Var): + var = lvalue.base.node + if isinstance(var.type, PartialType): + type_type = var.type.type + if type_type is None: + return # The partial type is None. + partial_types = self.find_partial_types(var) + if partial_types is None: + return + typename = type_type.fullname() + if typename == 'builtins.dict': + # TODO: Don't infer things twice. + key_type = self.accept(lvalue.index) + value_type = self.accept(rvalue) + full_key_type = UnionType.make_simplified_union( + [key_type, var.type.inner_types[0]]) + full_value_type = UnionType.make_simplified_union( + [value_type, var.type.inner_types[1]]) + if (is_valid_inferred_type(full_key_type) and + is_valid_inferred_type(full_value_type)): + if not self.current_node_deferred: + var.type = self.named_generic_type('builtins.dict', + [full_key_type, full_value_type]) + del partial_types[var] + + def visit_expression_stmt(self, s: ExpressionStmt) -> Type: + self.accept(s.expr) + + def visit_return_stmt(self, s: ReturnStmt) -> Type: + """Type check a return statement.""" + self.binder.breaking_out = True + if self.is_within_function(): + if self.function_stack[-1].is_generator: + return_type = self.get_generator_return_type(self.return_types[-1]) + else: + return_type = self.return_types[-1] + + if s.expr: + # Return with a value. + typ = self.accept(s.expr, return_type) + # Returning a value of type Any is always fine. + if isinstance(typ, AnyType): + return None + + if isinstance(return_type, Void): + # Lambdas are allowed to have a Void return. + # Functions returning a value of type None are allowed to have a Void return. + if isinstance(self.function_stack[-1], FuncExpr) or isinstance(typ, NoneTyp): + return None + self.fail(messages.NO_RETURN_VALUE_EXPECTED, s) + else: + self.check_subtype( + subtype_label='got', + subtype=typ, + supertype_label='expected', + supertype=return_type, + context=s, + msg=messages.INCOMPATIBLE_RETURN_VALUE_TYPE) + else: + # Empty returns are valid in Generators with Any typed returns. + if (self.function_stack[-1].is_generator and isinstance(return_type, AnyType)): + return None + + if isinstance(return_type, Void): + return None + + if isinstance(return_type, AnyType): + return None + + if self.typing_mode_full(): + self.fail(messages.RETURN_VALUE_EXPECTED, s) + + def wrap_generic_type(self, typ: Instance, rtyp: Instance, check_type: + str, context: Context) -> Type: + n_diff = self.count_nested_types(rtyp, check_type) - self.count_nested_types(typ, + check_type) + if n_diff == 1: + return self.named_generic_type(check_type, [typ]) + elif n_diff == 0 or n_diff > 1: + self.fail(messages.INCOMPATIBLE_RETURN_VALUE_TYPE + + ": expected {}, got {}".format(rtyp, typ), context) + return typ + return typ + + def count_nested_types(self, typ: Instance, check_type: str) -> int: + c = 0 + while is_subtype(typ, self.named_type(check_type)): + c += 1 + typ = map_instance_to_supertype(self.named_generic_type(check_type, typ.args), + self.lookup_typeinfo(check_type)) + if typ.args: + typ = cast(Instance, typ.args[0]) + else: + return c + return c + + def visit_if_stmt(self, s: IfStmt) -> Type: + """Type check an if statement.""" + breaking_out = True + # This frame records the knowledge from previous if/elif clauses not being taken. + with self.binder.frame_context(): + for e, b in zip(s.expr, s.body): + t = self.accept(e) + self.check_not_void(t, e) + if_map, else_map = find_isinstance_check( + e, self.type_map, + self.typing_mode_weak() + ) + if if_map is None: + # The condition is always false + # XXX should issue a warning? + pass + else: + # Only type check body if the if condition can be true. + with self.binder.frame_context(2): + if if_map: + for var, type in if_map.items(): + self.binder.push(var, type) + + self.accept(b) + breaking_out = breaking_out and self.binder.last_pop_breaking_out + + if else_map: + for var, type in else_map.items(): + self.binder.push(var, type) + if else_map is None: + # The condition is always true => remaining elif/else blocks + # can never be reached. + + # Might also want to issue a warning + # print("Warning: isinstance always true") + break + else: # Didn't break => can't prove one of the conditions is always true + with self.binder.frame_context(2): + if s.else_body: + self.accept(s.else_body) + breaking_out = breaking_out and self.binder.last_pop_breaking_out + if breaking_out: + self.binder.breaking_out = True + return None + + def visit_while_stmt(self, s: WhileStmt) -> Type: + """Type check a while statement.""" + self.accept_loop(IfStmt([s.expr], [s.body], None), s.else_body) + + def visit_operator_assignment_stmt(self, + s: OperatorAssignmentStmt) -> Type: + """Type check an operator assignment statement, e.g. x += 1.""" + lvalue_type = self.accept(s.lvalue) + method = infer_operator_assignment_method(lvalue_type, s.op) + rvalue_type, method_type = self.expr_checker.check_op( + method, lvalue_type, s.rvalue, s) + + if isinstance(s.lvalue, IndexExpr): + self.check_indexed_assignment(s.lvalue, s.rvalue, s.rvalue) + else: + if not is_subtype(rvalue_type, lvalue_type): + self.msg.incompatible_operator_assignment(s.op, s) + + def visit_assert_stmt(self, s: AssertStmt) -> Type: + self.accept(s.expr) + + # If this is asserting some isinstance check, bind that type in the following code + true_map, _ = find_isinstance_check( + s.expr, self.type_map, + self.typing_mode_weak() + ) + + if true_map: + for var, type in true_map.items(): + self.binder.push(var, type) + + def visit_raise_stmt(self, s: RaiseStmt) -> Type: + """Type check a raise statement.""" + self.binder.breaking_out = True + if s.expr: + self.type_check_raise(s.expr, s) + if s.from_expr: + self.type_check_raise(s.from_expr, s) + + def type_check_raise(self, e: Node, s: RaiseStmt) -> None: + typ = self.accept(e) + if isinstance(typ, FunctionLike): + if typ.is_type_obj(): + # Cases like "raise/from ExceptionClass". + typeinfo = typ.type_object() + base = self.lookup_typeinfo('builtins.BaseException') + if base in typeinfo.mro or typeinfo.fallback_to_any: + # Good! + return + # Else fall back to the checks below (which will fail). + if isinstance(typ, TupleType) and self.options.python_version[0] == 2: + # allow `raise type, value, traceback` + # https://docs.python.org/2/reference/simple_stmts.html#the-raise-statement + # TODO: Also check tuple item types. + if len(typ.items) in (2, 3): + return + if isinstance(typ, Instance) and typ.type.fallback_to_any: + # OK! + return + self.check_subtype(typ, + self.named_type('builtins.BaseException'), s, + messages.INVALID_EXCEPTION) + + def visit_try_stmt(self, s: TryStmt) -> Type: + """Type check a try statement.""" + # Our enclosing frame will get the result if the try/except falls through. + # This one gets all possible intermediate states + with self.binder.frame_context(): + if s.finally_body: + self.binder.try_frames.add(len(self.binder.frames) - 1) + breaking_out = self.visit_try_without_finally(s) + self.binder.try_frames.remove(len(self.binder.frames) - 1) + # First we check finally_body is type safe for all intermediate frames + self.accept(s.finally_body) + breaking_out = breaking_out or self.binder.breaking_out + else: + breaking_out = self.visit_try_without_finally(s) + + if not breaking_out and s.finally_body: + # Then we try again for the more restricted set of options that can fall through + self.accept(s.finally_body) + self.binder.breaking_out = breaking_out + return None + + def visit_try_without_finally(self, s: TryStmt) -> bool: + """Type check a try statement, ignoring the finally block. + + Return whether we are guaranteed to be breaking out. + Otherwise, it will place the results possible frames of + that don't break out into self.binder.frames[-2]. + """ + breaking_out = True + # This frame records the possible states that exceptions can leave variables in + # during the try: block + with self.binder.frame_context(): + with self.binder.frame_context(3): + self.binder.try_frames.add(len(self.binder.frames) - 2) + self.accept(s.body) + self.binder.try_frames.remove(len(self.binder.frames) - 2) + if s.else_body: + self.accept(s.else_body) + breaking_out = breaking_out and self.binder.last_pop_breaking_out + for i in range(len(s.handlers)): + with self.binder.frame_context(3): + if s.types[i]: + t = self.visit_except_handler_test(s.types[i]) + if s.vars[i]: + # To support local variables, we make this a definition line, + # causing assignment to set the variable's type. + s.vars[i].is_def = True + self.check_assignment(s.vars[i], self.temp_node(t, s.vars[i])) + self.accept(s.handlers[i]) + if s.vars[i]: + # Exception variables are deleted in python 3 but not python 2. + # But, since it's bad form in python 2 and the type checking + # wouldn't work very well, we delete it anyway. + + # Unfortunately, this doesn't let us detect usage before the + # try/except block. + if self.options.python_version[0] >= 3: + source = s.vars[i].name + else: + source = ('(exception variable "{}", which we do not accept outside' + 'except: blocks even in python 2)'.format(s.vars[i].name)) + var = cast(Var, s.vars[i].node) + var.type = DeletedType(source=source) + self.binder.cleanse(s.vars[i]) + breaking_out = breaking_out and self.binder.last_pop_breaking_out + return breaking_out + + def visit_except_handler_test(self, n: Node) -> Type: + """Type check an exception handler test clause.""" + type = self.accept(n) + + all_types = [] # type: List[Type] + test_types = type.items if isinstance(type, TupleType) else [type] + + for ttype in test_types: + if isinstance(ttype, AnyType): + all_types.append(ttype) + continue + + if not isinstance(ttype, FunctionLike): + self.fail(messages.INVALID_EXCEPTION_TYPE, n) + return AnyType() + + item = ttype.items()[0] + ret_type = item.ret_type + if not (is_subtype(ret_type, self.named_type('builtins.BaseException')) + and item.is_type_obj()): + self.fail(messages.INVALID_EXCEPTION_TYPE, n) + return AnyType() + + all_types.append(ret_type) + + return UnionType.make_simplified_union(all_types) + + def visit_for_stmt(self, s: ForStmt) -> Type: + """Type check a for statement.""" + item_type = self.analyze_iterable_item_type(s.expr) + self.analyze_index_variables(s.index, item_type, s) + self.accept_loop(s.body, s.else_body) + + def analyze_iterable_item_type(self, expr: Node) -> Type: + """Analyse iterable expression and return iterator item type.""" + iterable = self.accept(expr) + + self.check_not_void(iterable, expr) + if isinstance(iterable, TupleType): + if experiments.STRICT_OPTIONAL: + joined = UninhabitedType() # type: Type + else: + joined = NoneTyp() + for item in iterable.items: + joined = join_types(joined, item) + if isinstance(joined, ErrorType): + self.fail(messages.CANNOT_INFER_ITEM_TYPE, expr) + return AnyType() + return joined + else: + # Non-tuple iterable. + self.check_subtype(iterable, + self.named_generic_type('typing.Iterable', + [AnyType()]), + expr, messages.ITERABLE_EXPECTED) + + echk = self.expr_checker + method = echk.analyze_external_member_access('__iter__', iterable, + expr) + iterator = echk.check_call(method, [], [], expr)[0] + if self.options.python_version[0] >= 3: + nextmethod = '__next__' + else: + nextmethod = 'next' + method = echk.analyze_external_member_access(nextmethod, iterator, + expr) + return echk.check_call(method, [], [], expr)[0] + + def analyze_index_variables(self, index: Node, item_type: Type, + context: Context) -> None: + """Type check or infer for loop or list comprehension index vars.""" + self.check_assignment(index, self.temp_node(item_type, context)) + + def visit_del_stmt(self, s: DelStmt) -> Type: + if isinstance(s.expr, IndexExpr): + e = s.expr + m = MemberExpr(e.base, '__delitem__') + m.line = s.line + c = CallExpr(m, [e.index], [nodes.ARG_POS], [None]) + c.line = s.line + return c.accept(self) + else: + def flatten(t: Node) -> List[Node]: + """Flatten a nested sequence of tuples/lists into one list of nodes.""" + if isinstance(t, TupleExpr) or isinstance(t, ListExpr): + t = cast(Union[TupleExpr, ListExpr], t) + return [b for a in t.items for b in flatten(a)] + else: + return [t] + + s.expr.accept(self) + for elt in flatten(s.expr): + if isinstance(elt, NameExpr): + self.binder.assign_type(elt, + DeletedType(source=elt.name), + self.binder.get_declaration(elt), + self.typing_mode_weak()) + return None + + def visit_decorator(self, e: Decorator) -> Type: + for d in e.decorators: + if isinstance(d, RefExpr): + if d.fullname == 'typing.no_type_check': + e.var.type = AnyType() + e.var.is_ready = True + return NoneTyp() + + e.func.accept(self) + sig = self.function_type(e.func) # type: Type + # Process decorators from the inside out. + for i in range(len(e.decorators)): + n = len(e.decorators) - 1 - i + d = e.decorators[n] + if isinstance(d, NameExpr) and d.fullname == 'typing.overload': + self.fail('Single overload definition, multiple required', e) + continue + dec = self.accept(d) + temp = self.temp_node(sig) + sig, t2 = self.expr_checker.check_call(dec, [temp], + [nodes.ARG_POS], e) + sig = cast(FunctionLike, sig) + sig = set_callable_name(sig, e.func) + e.var.type = sig + e.var.is_ready = True + if e.func.is_property: + self.check_incompatible_property_override(e) + + def check_incompatible_property_override(self, e: Decorator) -> None: + if not e.var.is_settable_property and e.func.info is not None: + name = e.func.name() + for base in e.func.info.mro[1:]: + base_attr = base.names.get(name) + if not base_attr: + continue + if (isinstance(base_attr.node, OverloadedFuncDef) and + base_attr.node.is_property and + base_attr.node.items[0].var.is_settable_property): + self.fail(messages.READ_ONLY_PROPERTY_OVERRIDES_READ_WRITE, e) + + def visit_with_stmt(self, s: WithStmt) -> Type: + echk = self.expr_checker + for expr, target in zip(s.expr, s.target): + ctx = self.accept(expr) + enter = echk.analyze_external_member_access('__enter__', ctx, expr) + obj = echk.check_call(enter, [], [], expr)[0] + if target: + self.check_assignment(target, self.temp_node(obj, expr)) + exit = echk.analyze_external_member_access('__exit__', ctx, expr) + arg = self.temp_node(AnyType(), expr) + echk.check_call(exit, [arg] * 3, [nodes.ARG_POS] * 3, expr) + self.accept(s.body) + + def visit_print_stmt(self, s: PrintStmt) -> Type: + for arg in s.args: + self.accept(arg) + if s.target: + target_type = self.accept(s.target) + if not isinstance(target_type, NoneTyp): + # TODO: Also verify the type of 'write'. + self.expr_checker.analyze_external_member_access('write', target_type, s.target) + + # + # Expressions + # + + def visit_name_expr(self, e: NameExpr) -> Type: + return self.expr_checker.visit_name_expr(e) + + def visit_call_expr(self, e: CallExpr) -> Type: + return self.expr_checker.visit_call_expr(e) + + def visit_yield_from_expr(self, e: YieldFromExpr) -> Type: + return_type = self.return_types[-1] + subexpr_type = self.accept(e.expr, return_type) + iter_type = None # type: Type + + # Check that the expr is an instance of Iterable and get the type of the iterator produced + # by __iter__. + if isinstance(subexpr_type, AnyType): + iter_type = AnyType() + elif (isinstance(subexpr_type, Instance) and + is_subtype(subexpr_type, self.named_type('typing.Iterable'))): + iter_method_type = self.expr_checker.analyze_external_member_access( + '__iter__', + subexpr_type, + AnyType()) + + generic_generator_type = self.named_generic_type('typing.Generator', + [AnyType(), AnyType(), AnyType()]) + iter_type, _ = self.expr_checker.check_call(iter_method_type, [], [], + context=generic_generator_type) + else: + self.msg.yield_from_invalid_operand_type(subexpr_type, e) + iter_type = AnyType() + + # Check that the iterator's item type matches the type yielded by the Generator function + # containing this `yield from` expression. + expected_item_type = self.get_generator_yield_type(return_type) + actual_item_type = self.get_generator_yield_type(iter_type) + + self.check_subtype(actual_item_type, expected_item_type, e, + messages.INCOMPATIBLE_TYPES_IN_YIELD_FROM, + 'actual type', 'expected type') + + # Determine the type of the entire yield from expression. + if (isinstance(iter_type, Instance) and + iter_type.type.fullname() == 'typing.Generator'): + return self.get_generator_return_type(iter_type) + else: + # Non-Generators don't return anything from `yield from` expressions. + return Void() + + def visit_member_expr(self, e: MemberExpr) -> Type: + return self.expr_checker.visit_member_expr(e) + + def visit_break_stmt(self, s: BreakStmt) -> Type: + self.binder.breaking_out = True + self.binder.allow_jump(self.binder.loop_frames[-1] - 1) + return None + + def visit_continue_stmt(self, s: ContinueStmt) -> Type: + self.binder.breaking_out = True + self.binder.allow_jump(self.binder.loop_frames[-1]) + return None + + def visit_int_expr(self, e: IntExpr) -> Type: + return self.expr_checker.visit_int_expr(e) + + def visit_str_expr(self, e: StrExpr) -> Type: + return self.expr_checker.visit_str_expr(e) + + def visit_bytes_expr(self, e: BytesExpr) -> Type: + return self.expr_checker.visit_bytes_expr(e) + + def visit_unicode_expr(self, e: UnicodeExpr) -> Type: + return self.expr_checker.visit_unicode_expr(e) + + def visit_float_expr(self, e: FloatExpr) -> Type: + return self.expr_checker.visit_float_expr(e) + + def visit_complex_expr(self, e: ComplexExpr) -> Type: + return self.expr_checker.visit_complex_expr(e) + + def visit_ellipsis(self, e: EllipsisExpr) -> Type: + return self.expr_checker.visit_ellipsis(e) + + def visit_op_expr(self, e: OpExpr) -> Type: + return self.expr_checker.visit_op_expr(e) + + def visit_comparison_expr(self, e: ComparisonExpr) -> Type: + return self.expr_checker.visit_comparison_expr(e) + + def visit_unary_expr(self, e: UnaryExpr) -> Type: + return self.expr_checker.visit_unary_expr(e) + + def visit_index_expr(self, e: IndexExpr) -> Type: + return self.expr_checker.visit_index_expr(e) + + def visit_cast_expr(self, e: CastExpr) -> Type: + return self.expr_checker.visit_cast_expr(e) + + def visit_reveal_type_expr(self, e: RevealTypeExpr) -> Type: + return self.expr_checker.visit_reveal_type_expr(e) + + def visit_super_expr(self, e: SuperExpr) -> Type: + return self.expr_checker.visit_super_expr(e) + + def visit_type_application(self, e: TypeApplication) -> Type: + return self.expr_checker.visit_type_application(e) + + def visit_type_alias_expr(self, e: TypeAliasExpr) -> Type: + return self.expr_checker.visit_type_alias_expr(e) + + def visit_type_var_expr(self, e: TypeVarExpr) -> Type: + # TODO: Perhaps return a special type used for type variables only? + return AnyType() + + def visit_namedtuple_expr(self, e: NamedTupleExpr) -> Type: + # TODO: Perhaps return a type object type? + return AnyType() + + def visit_list_expr(self, e: ListExpr) -> Type: + return self.expr_checker.visit_list_expr(e) + + def visit_set_expr(self, e: SetExpr) -> Type: + return self.expr_checker.visit_set_expr(e) + + def visit_tuple_expr(self, e: TupleExpr) -> Type: + return self.expr_checker.visit_tuple_expr(e) + + def visit_dict_expr(self, e: DictExpr) -> Type: + return self.expr_checker.visit_dict_expr(e) + + def visit_slice_expr(self, e: SliceExpr) -> Type: + return self.expr_checker.visit_slice_expr(e) + + def visit_func_expr(self, e: FuncExpr) -> Type: + return self.expr_checker.visit_func_expr(e) + + def visit_list_comprehension(self, e: ListComprehension) -> Type: + return self.expr_checker.visit_list_comprehension(e) + + def visit_set_comprehension(self, e: SetComprehension) -> Type: + return self.expr_checker.visit_set_comprehension(e) + + def visit_generator_expr(self, e: GeneratorExpr) -> Type: + return self.expr_checker.visit_generator_expr(e) + + def visit_dictionary_comprehension(self, e: DictionaryComprehension) -> Type: + return self.expr_checker.visit_dictionary_comprehension(e) + + def visit_temp_node(self, e: TempNode) -> Type: + return e.type + + def visit_conditional_expr(self, e: ConditionalExpr) -> Type: + return self.expr_checker.visit_conditional_expr(e) + + def visit_backquote_expr(self, e: BackquoteExpr) -> Type: + return self.expr_checker.visit_backquote_expr(e) + + def visit_yield_expr(self, e: YieldExpr) -> Type: + return_type = self.return_types[-1] + expected_item_type = self.get_generator_yield_type(return_type) + if e.expr is None: + if (not (isinstance(expected_item_type, Void) or + isinstance(expected_item_type, AnyType)) + and self.typing_mode_full()): + self.fail(messages.YIELD_VALUE_EXPECTED, e) + else: + actual_item_type = self.accept(e.expr, expected_item_type) + self.check_subtype(actual_item_type, expected_item_type, e, + messages.INCOMPATIBLE_TYPES_IN_YIELD, + 'actual type', 'expected type') + return self.get_generator_receive_type(return_type) + + # + # Helpers + # + + def check_subtype(self, subtype: Type, supertype: Type, context: Context, + msg: str = messages.INCOMPATIBLE_TYPES, + subtype_label: str = None, + supertype_label: str = None) -> None: + """Generate an error if the subtype is not compatible with + supertype.""" + if not is_subtype(subtype, supertype): + if isinstance(subtype, Void): + self.msg.does_not_return_value(subtype, context) + else: + extra_info = [] # type: List[str] + if subtype_label is not None or supertype_label is not None: + subtype_str, supertype_str = self.msg.format_distinctly(subtype, supertype) + if subtype_label is not None: + extra_info.append(subtype_label + ' ' + subtype_str) + if supertype_label is not None: + extra_info.append(supertype_label + ' ' + supertype_str) + if extra_info: + msg += ' (' + ', '.join(extra_info) + ')' + self.fail(msg, context) + + def named_type(self, name: str) -> Instance: + """Return an instance type with type given by the name and no + type arguments. For example, named_type('builtins.object') + produces the object type. + """ + # Assume that the name refers to a type. + sym = self.lookup_qualified(name) + return Instance(cast(TypeInfo, sym.node), []) + + def named_generic_type(self, name: str, args: List[Type]) -> Instance: + """Return an instance with the given name and type arguments. + + Assume that the number of arguments is correct. Assume that + the name refers to a compatible generic type. + """ + return Instance(self.lookup_typeinfo(name), args) + + def lookup_typeinfo(self, fullname: str) -> TypeInfo: + # Assume that the name refers to a class. + sym = self.lookup_qualified(fullname) + return cast(TypeInfo, sym.node) + + def type_type(self) -> Instance: + """Return instance type 'type'.""" + return self.named_type('builtins.type') + + def object_type(self) -> Instance: + """Return instance type 'object'.""" + return self.named_type('builtins.object') + + def bool_type(self) -> Instance: + """Return instance type 'bool'.""" + return self.named_type('builtins.bool') + + def str_type(self) -> Instance: + """Return instance type 'str'.""" + return self.named_type('builtins.str') + + def check_type_equivalency(self, t1: Type, t2: Type, node: Context, + msg: str = messages.INCOMPATIBLE_TYPES) -> None: + """Generate an error if the types are not equivalent. The + dynamic type is equivalent with all types. + """ + if not is_equivalent(t1, t2): + self.fail(msg, node) + + def store_type(self, node: Node, typ: Type) -> None: + """Store the type of a node in the type map.""" + self.type_map[node] = typ + + def typing_mode_none(self) -> bool: + if self.is_dynamic_function() and not self.options.check_untyped_defs: + return not self.weak_opts + elif self.function_stack: + return False + else: + return False + + def typing_mode_weak(self) -> bool: + if self.is_dynamic_function() and not self.options.check_untyped_defs: + return bool(self.weak_opts) + elif self.function_stack: + return False + else: + return 'global' in self.weak_opts + + def typing_mode_full(self) -> bool: + if self.is_dynamic_function() and not self.options.check_untyped_defs: + return False + elif self.function_stack: + return True + else: + return 'global' not in self.weak_opts + + def is_dynamic_function(self) -> bool: + return len(self.dynamic_funcs) > 0 and self.dynamic_funcs[-1] + + def lookup(self, name: str, kind: int) -> SymbolTableNode: + """Look up a definition from the symbol table with the given name. + TODO remove kind argument + """ + if name in self.globals: + return self.globals[name] + else: + b = self.globals.get('__builtins__', None) + if b: + table = cast(MypyFile, b.node).names + if name in table: + return table[name] + raise KeyError('Failed lookup: {}'.format(name)) + + def lookup_qualified(self, name: str) -> SymbolTableNode: + if '.' not in name: + return self.lookup(name, GDEF) # FIX kind + else: + parts = name.split('.') + n = self.modules[parts[0]] + for i in range(1, len(parts) - 1): + n = cast(MypyFile, n.names.get(parts[i], None).node) + return n.names[parts[-1]] + + def enter_partial_types(self) -> None: + """Push a new scope for collecting partial types.""" + self.partial_types.append({}) + + def leave_partial_types(self) -> None: + """Pop partial type scope. + + Also report errors for variables which still have partial + types, i.e. we couldn't infer a complete type. + """ + partial_types = self.partial_types.pop() + if not self.current_node_deferred: + for var, context in partial_types.items(): + if experiments.STRICT_OPTIONAL and cast(PartialType, var.type).type is None: + # None partial type: assume variable is intended to have type None + var.type = NoneTyp() + else: + self.msg.fail(messages.NEED_ANNOTATION_FOR_VAR, context) + var.type = AnyType() + + def find_partial_types(self, var: Var) -> Optional[Dict[Var, Context]]: + for partial_types in reversed(self.partial_types): + if var in partial_types: + return partial_types + return None + + def is_within_function(self) -> bool: + """Are we currently type checking within a function? + + I.e. not at class body or at the top level. + """ + return self.return_types != [] + + def check_not_void(self, typ: Type, context: Context) -> None: + """Generate an error if the type is Void.""" + if isinstance(typ, Void): + self.msg.does_not_return_value(typ, context) + + def temp_node(self, t: Type, context: Context = None) -> Node: + """Create a temporary node with the given, fixed type.""" + temp = TempNode(t) + if context: + temp.set_line(context.get_line()) + return temp + + def fail(self, msg: str, context: Context) -> None: + """Produce an error message.""" + self.msg.fail(msg, context) + + def iterable_item_type(self, instance: Instance) -> Type: + iterable = map_instance_to_supertype( + instance, + self.lookup_typeinfo('typing.Iterable')) + return iterable.args[0] + + def function_type(self, func: FuncBase) -> FunctionLike: + return function_type(func, self.named_type('builtins.function')) + + def method_type(self, func: FuncBase) -> FunctionLike: + return method_type_with_fallback(func, self.named_type('builtins.function')) + + +# Data structure returned by find_isinstance_check representing +# information learned from the truth or falsehood of a condition. The +# dict maps nodes representing expressions like 'a[0].x' to their +# refined types under the assumption that the condition has a +# particular truth value. A value of None means that the condition can +# never have that truth value. + +# NB: The keys of this dict are nodes in the original source program, +# which are compared by reference equality--effectively, being *the +# same* expression of the program, not just two identical expressions +# (such as two references to the same variable). TODO: it would +# probably be better to have the dict keyed by the nodes' literal_hash +# field instead. + +# NB: This should be `TypeMap = Optional[Dict[Node, Type]]`! +# But see https://github.com/python/mypy/issues/1637 +TypeMap = Dict[Node, Type] + + +def conditional_type_map(expr: Node, + current_type: Optional[Type], + proposed_type: Optional[Type], + *, + weak: bool = False + ) -> Tuple[TypeMap, TypeMap]: + """Takes in an expression, the current type of the expression, and a + proposed type of that expression. + + Returns a 2-tuple: The first element is a map from the expression to + the proposed type, if the expression can be the proposed type. The + second element is a map from the expression to the type it would hold + if it was not the proposed type, if any.""" + if proposed_type: + if current_type: + if is_proper_subtype(current_type, proposed_type): + return {expr: proposed_type}, None + elif not is_overlapping_types(current_type, proposed_type): + return None, {expr: current_type} + else: + remaining_type = restrict_subtype_away(current_type, proposed_type) + return {expr: proposed_type}, {expr: remaining_type} + else: + return {expr: proposed_type}, {} + else: + # An isinstance check, but we don't understand the type + if weak: + return {expr: AnyType()}, {expr: current_type} + else: + return {}, {} + + +def is_literal_none(n: Node) -> bool: + return isinstance(n, NameExpr) and n.fullname == 'builtins.None' + + +def and_conditional_maps(m1: TypeMap, m2: TypeMap) -> TypeMap: + """Calculate what information we can learn from the truth of (e1 and e2) + in terms of the information that we can learn from the truth of e1 and + the truth of e2. + """ + + if m1 is None or m2 is None: + # One of the conditions can never be true. + return None + # Both conditions can be true; combine the information. Anything + # we learn from either conditions's truth is valid. If the same + # expression's type is refined by both conditions, we somewhat + # arbitrarily give precedence to m2. (In the future, we could use + # an intersection type.) + result = m2.copy() + m2_keys = set(n2.literal_hash for n2 in m2) + for n1 in m1: + if n1.literal_hash not in m2_keys: + result[n1] = m1[n1] + return result + + +def or_conditional_maps(m1: TypeMap, m2: TypeMap) -> TypeMap: + """Calculate what information we can learn from the truth of (e1 or e2) + in terms of the information that we can learn from the truth of e1 and + the truth of e2. + """ + + if m1 is None: + return m2 + if m2 is None: + return m1 + # Both conditions can be true. Combine information about + # expressions whose type is refined by both conditions. (We do not + # learn anything about expressions whose type is refined by only + # one condition.) + result = {} + for n1 in m1: + for n2 in m2: + if n1.literal_hash == n2.literal_hash: + result[n1] = UnionType.make_simplified_union([m1[n1], m2[n2]]) + return result + + +def find_isinstance_check(node: Node, + type_map: Dict[Node, Type], + weak: bool=False + ) -> Tuple[TypeMap, TypeMap]: + """Find any isinstance checks (within a chain of ands). Includes + implicit and explicit checks for None. + + Return value is a map of variables to their types if the condition + is true and a map of variables to their types if the condition is false. + + If either of the values in the tuple is None, then that particular + branch can never occur. + + Guaranteed to not return None, None. (But may return {}, {}) + """ + if isinstance(node, CallExpr): + if refers_to_fullname(node.callee, 'builtins.isinstance'): + expr = node.args[0] + if expr.literal == LITERAL_TYPE: + vartype = type_map[expr] + type = get_isinstance_type(node.args[1], type_map) + return conditional_type_map(expr, vartype, type, weak=weak) + elif (isinstance(node, ComparisonExpr) and any(is_literal_none(n) for n in node.operands) and + experiments.STRICT_OPTIONAL): + # Check for `x is None` and `x is not None`. + is_not = node.operators == ['is not'] + if is_not or node.operators == ['is']: + if_vars = {} # type: Dict[Node, Type] + else_vars = {} # type: Dict[Node, Type] + for expr in node.operands: + if expr.literal == LITERAL_TYPE and not is_literal_none(expr) and expr in type_map: + # This should only be true at most once: there should be + # two elements in node.operands, and at least one of them + # should represent a None. + vartype = type_map[expr] + if_vars, else_vars = conditional_type_map(expr, vartype, NoneTyp(), weak=weak) + break + + if is_not: + if_vars, else_vars = else_vars, if_vars + return if_vars, else_vars + elif isinstance(node, RefExpr) and experiments.STRICT_OPTIONAL: + # The type could be falsy, so we can't deduce anything new about the else branch + vartype = type_map[node] + _, if_vars = conditional_type_map(node, vartype, NoneTyp(), weak=weak) + return if_vars, {} + elif isinstance(node, OpExpr) and node.op == 'and': + left_if_vars, left_else_vars = find_isinstance_check( + node.left, + type_map, + weak, + ) + + right_if_vars, right_else_vars = find_isinstance_check( + node.right, + type_map, + weak, + ) + + # (e1 and e2) is true if both e1 and e2 are true, + # and false if at least one of e1 and e2 is false. + return (and_conditional_maps(left_if_vars, right_if_vars), + or_conditional_maps(left_else_vars, right_else_vars)) + elif isinstance(node, OpExpr) and node.op == 'or': + left_if_vars, left_else_vars = find_isinstance_check( + node.left, + type_map, + weak, + ) + + right_if_vars, right_else_vars = find_isinstance_check( + node.right, + type_map, + weak, + ) + + # (e1 or e2) is true if at least one of e1 or e2 is true, + # and false if both e1 and e2 are false. + return (or_conditional_maps(left_if_vars, right_if_vars), + and_conditional_maps(left_else_vars, right_else_vars)) + elif isinstance(node, UnaryExpr) and node.op == 'not': + left, right = find_isinstance_check(node.expr, type_map, weak) + return right, left + + # Not a supported isinstance check + return {}, {} + + +def get_isinstance_type(node: Node, type_map: Dict[Node, Type]) -> Type: + type = type_map[node] + + if isinstance(type, TupleType): + all_types = type.items + else: + all_types = [type] + + types = [] # type: List[Type] + + for type in all_types: + if isinstance(type, FunctionLike): + if type.is_type_obj(): + # Type variables may be present -- erase them, which is the best + # we can do (outside disallowing them here). + type = erase_typevars(type.items()[0].ret_type) + + types.append(type) + + if len(types) == 0: + return None + elif len(types) == 1: + return types[0] + else: + return UnionType(types) + + +def expand_node(defn: Node, map: Dict[TypeVarId, Type]) -> Node: + visitor = TypeTransformVisitor(map) + return defn.accept(visitor) + + +def expand_func(defn: FuncItem, map: Dict[TypeVarId, Type]) -> FuncItem: + return cast(FuncItem, expand_node(defn, map)) + + +class TypeTransformVisitor(TransformVisitor): + def __init__(self, map: Dict[TypeVarId, Type]) -> None: + super().__init__() + self.map = map + + def type(self, type: Type) -> Type: + return expand_type(type, self.map) + + +def is_unsafe_overlapping_signatures(signature: Type, other: Type) -> bool: + """Check if two signatures may be unsafely overlapping. + + Two signatures s and t are overlapping if both can be valid for the same + statically typed values and the return types are incompatible. + + Assume calls are first checked against 'signature', then against 'other'. + Thus if 'signature' is more general than 'other', there is no unsafe + overlapping. + + TODO If argument types vary covariantly, the return type may vary + covariantly as well. + """ + if isinstance(signature, CallableType): + if isinstance(other, CallableType): + # TODO varargs + # TODO keyword args + # TODO erasure + # TODO allow to vary covariantly + # Check if the argument counts are overlapping. + min_args = max(signature.min_args, other.min_args) + max_args = min(len(signature.arg_types), len(other.arg_types)) + if min_args > max_args: + # Argument counts are not overlapping. + return False + # Signatures are overlapping iff if they are overlapping for the + # smallest common argument count. + for i in range(min_args): + t1 = signature.arg_types[i] + t2 = other.arg_types[i] + if not is_overlapping_types(t1, t2): + return False + # All arguments types for the smallest common argument count are + # overlapping => the signature is overlapping. The overlapping is + # safe if the return types are identical. + if is_same_type(signature.ret_type, other.ret_type): + return False + # If the first signature has more general argument types, the + # latter will never be called + if is_more_general_arg_prefix(signature, other): + return False + return not is_more_precise_signature(signature, other) + return True + + +def is_more_general_arg_prefix(t: FunctionLike, s: FunctionLike) -> bool: + """Does t have wider arguments than s?""" + # TODO should an overload with additional items be allowed to be more + # general than one with fewer items (or just one item)? + # TODO check argument kinds + if isinstance(t, CallableType): + if isinstance(s, CallableType): + return all(is_proper_subtype(args, argt) + for argt, args in zip(t.arg_types, s.arg_types)) + elif isinstance(t, FunctionLike): + if isinstance(s, FunctionLike): + if len(t.items()) == len(s.items()): + return all(is_same_arg_prefix(items, itemt) + for items, itemt in zip(t.items(), s.items())) + return False + + +def is_same_arg_prefix(t: CallableType, s: CallableType) -> bool: + # TODO check argument kinds + return all(is_same_type(argt, args) + for argt, args in zip(t.arg_types, s.arg_types)) + + +def is_more_precise_signature(t: CallableType, s: CallableType) -> bool: + """Is t more precise than s? + + A signature t is more precise than s if all argument types and the return + type of t are more precise than the corresponding types in s. + + Assume that the argument kinds and names are compatible, and that the + argument counts are overlapping. + """ + # TODO generic function types + # Only consider the common prefix of argument types. + for argt, args in zip(t.arg_types, s.arg_types): + if not is_more_precise(argt, args): + return False + return is_more_precise(t.ret_type, s.ret_type) + + +def infer_operator_assignment_method(type: Type, operator: str) -> str: + """Return the method used for operator assignment for given value type. + + For example, if operator is '+', return '__iadd__' or '__add__' depending + on which method is supported by the type. + """ + method = nodes.op_methods[operator] + if isinstance(type, Instance): + if operator in nodes.ops_with_inplace_method: + inplace = '__i' + method[2:] + if type.type.has_readable_member(inplace): + method = inplace + return method + + +def is_valid_inferred_type(typ: Type) -> bool: + """Is an inferred type valid? + + Examples of invalid types include the None type or a type with a None component. + """ + if is_same_type(typ, NoneTyp()): + # With strict Optional checking, we *may* eventually infer NoneTyp, but + # we only do that if we can't infer a specific Optional type. This + # resolution happens in leave_partial_types when we pop a partial types + # scope. + return False + if is_same_type(typ, UninhabitedType()): + return False + elif isinstance(typ, Instance): + for arg in typ.args: + if not is_valid_inferred_type(arg): + return False + elif isinstance(typ, TupleType): + for item in typ.items: + if not is_valid_inferred_type(item): + return False + return True diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 97a50142901eb..2fee197d9c86d 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1,1822 +1,1822 @@ -"""Expression type checker. This file is conceptually part of TypeChecker.""" - -from typing import cast, Dict, List, Tuple, Callable, Union, Optional - -from mypy.types import ( - Type, AnyType, CallableType, Overloaded, NoneTyp, Void, TypeVarDef, - TupleType, Instance, TypeVarId, TypeVarType, ErasedType, UnionType, - PartialType, DeletedType, UnboundType, UninhabitedType, TypeType -) -from mypy.nodes import ( - NameExpr, RefExpr, Var, FuncDef, OverloadedFuncDef, TypeInfo, CallExpr, - Node, MemberExpr, IntExpr, StrExpr, BytesExpr, UnicodeExpr, FloatExpr, - OpExpr, UnaryExpr, IndexExpr, CastExpr, RevealTypeExpr, TypeApplication, ListExpr, - TupleExpr, DictExpr, FuncExpr, SuperExpr, SliceExpr, Context, - ListComprehension, GeneratorExpr, SetExpr, MypyFile, Decorator, - ConditionalExpr, ComparisonExpr, TempNode, SetComprehension, - DictionaryComprehension, ComplexExpr, EllipsisExpr, - TypeAliasExpr, BackquoteExpr, ARG_POS, ARG_NAMED, ARG_STAR2 -) -from mypy.nodes import function_type -from mypy import nodes -import mypy.checker -from mypy import types -from mypy.sametypes import is_same_type -from mypy.erasetype import replace_meta_vars -from mypy.messages import MessageBuilder -from mypy import messages -from mypy.infer import infer_type_arguments, infer_function_type_arguments -from mypy import join -from mypy.subtypes import is_subtype, is_equivalent -from mypy import applytype -from mypy import erasetype -from mypy.checkmember import analyze_member_access, type_object_type -from mypy.semanal import self_type -from mypy.constraints import get_actual_type -from mypy.checkstrformat import StringFormatterChecker -from mypy.expandtype import expand_type -import mypy.checkexpr - -from mypy import experiments - -# Type of callback user for checking individual function arguments. See -# check_args() below for details. -ArgChecker = Callable[[Type, Type, int, Type, int, int, CallableType, Context, MessageBuilder], - None] - - -class Finished(Exception): - """Raised if we can terminate overload argument check early (no match).""" - - -class ExpressionChecker: - """Expression type checker. - - This class works closely together with checker.TypeChecker. - """ - - # Some services are provided by a TypeChecker instance. - chk = None # type: mypy.checker.TypeChecker - # This is shared with TypeChecker, but stored also here for convenience. - msg = None # type: MessageBuilder - - strfrm_checker = None # type: mypy.checkstrformat.StringFormatterChecker - - def __init__(self, - chk: 'mypy.checker.TypeChecker', - msg: MessageBuilder) -> None: - """Construct an expression type checker.""" - self.chk = chk - self.msg = msg - self.strfrm_checker = mypy.checkexpr.StringFormatterChecker(self, self.chk, self.msg) - - def visit_name_expr(self, e: NameExpr) -> Type: - """Type check a name expression. - - It can be of any kind: local, member or global. - """ - result = self.analyze_ref_expr(e) - return self.chk.narrow_type_from_binder(e, result) - - def analyze_ref_expr(self, e: RefExpr, lvalue: bool = False) -> Type: - result = None # type: Type - node = e.node - if isinstance(node, Var): - # Variable reference. - result = self.analyze_var_ref(node, e) - if isinstance(result, PartialType): - if result.type is None: - # 'None' partial type. It has a well-defined type. In an lvalue context - # we want to preserve the knowledge of it being a partial type. - if not lvalue: - result = NoneTyp() - else: - partial_types = self.chk.find_partial_types(node) - if partial_types is not None and not self.chk.current_node_deferred: - context = partial_types[node] - self.msg.fail(messages.NEED_ANNOTATION_FOR_VAR, context) - result = AnyType() - elif isinstance(node, FuncDef): - # Reference to a global function. - result = function_type(node, self.named_type('builtins.function')) - elif isinstance(node, OverloadedFuncDef): - result = node.type - elif isinstance(node, TypeInfo): - # Reference to a type object. - result = type_object_type(node, self.named_type) - elif isinstance(node, MypyFile): - # Reference to a module object. - result = self.named_type('builtins.module') - elif isinstance(node, Decorator): - result = self.analyze_var_ref(node.var, e) - else: - # Unknown reference; use any type implicitly to avoid - # generating extra type errors. - result = AnyType() - return result - - def analyze_var_ref(self, var: Var, context: Context) -> Type: - if not var.type: - if not var.is_ready and self.chk.typing_mode_full(): - self.chk.handle_cannot_determine_type(var.name(), context) - # Implicit 'Any' type. - return AnyType() - else: - # Look up local type of variable with type (inferred or explicit). - val = self.chk.binder.get(var) - if val is None: - return var.type - else: - return val - - def visit_call_expr(self, e: CallExpr) -> Type: - """Type check a call expression.""" - if e.analyzed: - # It's really a special form that only looks like a call. - return self.accept(e.analyzed, self.chk.type_context[-1]) - self.try_infer_partial_type(e) - callee_type = self.accept(e.callee) - if (self.chk.options.disallow_untyped_calls and - self.chk.typing_mode_full() and - isinstance(callee_type, CallableType) - and callee_type.implicit): - return self.msg.untyped_function_call(callee_type, e) - return self.check_call_expr_with_callee_type(callee_type, e) - - # Types and methods that can be used to infer partial types. - item_args = {'builtins.list': ['append'], - 'builtins.set': ['add', 'discard'], - } - container_args = {'builtins.list': {'extend': ['builtins.list']}, - 'builtins.dict': {'update': ['builtins.dict']}, - 'builtins.set': {'update': ['builtins.set', 'builtins.list']}, - } - - def try_infer_partial_type(self, e: CallExpr) -> None: - if isinstance(e.callee, MemberExpr) and isinstance(e.callee.expr, RefExpr): - var = cast(Var, e.callee.expr.node) - partial_types = self.chk.find_partial_types(var) - if partial_types is not None and not self.chk.current_node_deferred: - partial_type = cast(PartialType, var.type) - if partial_type is None or partial_type.type is None: - # A partial None type -> can't infer anything. - return - typename = partial_type.type.fullname() - methodname = e.callee.name - # Sometimes we can infer a full type for a partial List, Dict or Set type. - # TODO: Don't infer argument expression twice. - if (typename in self.item_args and methodname in self.item_args[typename] - and e.arg_kinds == [ARG_POS]): - item_type = self.accept(e.args[0]) - full_item_type = UnionType.make_simplified_union( - [item_type, partial_type.inner_types[0]]) - if mypy.checker.is_valid_inferred_type(full_item_type): - var.type = self.chk.named_generic_type(typename, [full_item_type]) - del partial_types[var] - elif (typename in self.container_args - and methodname in self.container_args[typename] - and e.arg_kinds == [ARG_POS]): - arg_type = self.accept(e.args[0]) - if isinstance(arg_type, Instance): - arg_typename = arg_type.type.fullname() - if arg_typename in self.container_args[typename][methodname]: - full_item_types = [ - UnionType.make_simplified_union([item_type, prev_type]) - for item_type, prev_type - in zip(arg_type.args, partial_type.inner_types) - ] - if all(mypy.checker.is_valid_inferred_type(item_type) - for item_type in full_item_types): - var.type = self.chk.named_generic_type(typename, - list(full_item_types)) - del partial_types[var] - - def check_call_expr_with_callee_type(self, callee_type: Type, - e: CallExpr) -> Type: - """Type check call expression. - - The given callee type overrides the type of the callee - expression. - """ - return self.check_call(callee_type, e.args, e.arg_kinds, e, - e.arg_names, callable_node=e.callee)[0] - - def check_call(self, callee: Type, args: List[Node], - arg_kinds: List[int], context: Context, - arg_names: List[str] = None, - callable_node: Node = None, - arg_messages: MessageBuilder = None) -> Tuple[Type, Type]: - """Type check a call. - - Also infer type arguments if the callee is a generic function. - - Return (result type, inferred callee type). - - Arguments: - callee: type of the called value - args: actual argument expressions - arg_kinds: contains nodes.ARG_* constant for each argument in args - describing whether the argument is positional, *arg, etc. - arg_names: names of arguments (optional) - callable_node: associate the inferred callable type to this node, - if specified - arg_messages: TODO - """ - arg_messages = arg_messages or self.msg - if isinstance(callee, CallableType): - if callee.is_concrete_type_obj() and callee.type_object().is_abstract: - type = callee.type_object() - self.msg.cannot_instantiate_abstract_class( - callee.type_object().name(), type.abstract_attributes, - context) - - formal_to_actual = map_actuals_to_formals( - arg_kinds, arg_names, - callee.arg_kinds, callee.arg_names, - lambda i: self.accept(args[i])) - - if callee.is_generic(): - callee = freshen_generic_callable(callee) - callee = self.infer_function_type_arguments_using_context( - callee, context) - callee = self.infer_function_type_arguments( - callee, args, arg_kinds, formal_to_actual, context) - - arg_types = self.infer_arg_types_in_context2( - callee, args, arg_kinds, formal_to_actual) - - self.check_argument_count(callee, arg_types, arg_kinds, - arg_names, formal_to_actual, context, self.msg) - - self.check_argument_types(arg_types, arg_kinds, callee, - formal_to_actual, context, - messages=arg_messages) - - if (callee.is_type_obj() and (len(arg_types) == 1) - and is_equivalent(callee.ret_type, self.named_type('builtins.type'))): - callee = callee.copy_modified(ret_type=TypeType(arg_types[0])) - - if callable_node: - # Store the inferred callable type. - self.chk.store_type(callable_node, callee) - return callee.ret_type, callee - elif isinstance(callee, Overloaded): - # Type check arguments in empty context. They will be checked again - # later in a context derived from the signature; these types are - # only used to pick a signature variant. - self.msg.disable_errors() - arg_types = self.infer_arg_types_in_context(None, args) - self.msg.enable_errors() - - target = self.overload_call_target(arg_types, arg_kinds, arg_names, - callee, context, - messages=arg_messages) - return self.check_call(target, args, arg_kinds, context, arg_names, - arg_messages=arg_messages) - elif isinstance(callee, AnyType) or self.chk.typing_mode_none(): - self.infer_arg_types_in_context(None, args) - return AnyType(), AnyType() - elif isinstance(callee, UnionType): - self.msg.disable_type_names += 1 - results = [self.check_call(subtype, args, arg_kinds, context, arg_names, - arg_messages=arg_messages) - for subtype in callee.items] - self.msg.disable_type_names -= 1 - return (UnionType.make_simplified_union([res[0] for res in results]), - callee) - elif isinstance(callee, Instance): - call_function = analyze_member_access('__call__', callee, context, - False, False, self.named_type, self.not_ready_callback, - self.msg) - return self.check_call(call_function, args, arg_kinds, context, arg_names, - callable_node, arg_messages) - elif isinstance(callee, TypeVarType): - return self.check_call(callee.upper_bound, args, arg_kinds, context, arg_names, - callable_node, arg_messages) - elif isinstance(callee, TypeType): - # Pass the original Type[] as context since that's where errors should go. - item = self.analyze_type_type_callee(callee.item, callee) - return self.check_call(item, args, arg_kinds, context, arg_names, - callable_node, arg_messages) - else: - return self.msg.not_callable(callee, context), AnyType() - - def analyze_type_type_callee(self, item: Type, context: Context) -> Type: - """Analyze the callee X in X(...) where X is Type[item]. - - Return a Y that we can pass to check_call(Y, ...). - """ - if isinstance(item, AnyType): - return AnyType() - if isinstance(item, Instance): - return type_object_type(item.type, self.named_type) - if isinstance(item, UnionType): - return UnionType([self.analyze_type_type_callee(item, context) - for item in item.items], item.line) - if isinstance(item, TypeVarType): - # Pretend we're calling the typevar's upper bound, - # i.e. its constructor (a poor approximation for reality, - # but better than AnyType...), but replace the return type - # with typevar. - callee = self.analyze_type_type_callee(item.upper_bound, context) - if isinstance(callee, CallableType): - if callee.is_generic(): - callee = None - else: - callee = callee.copy_modified(ret_type=item) - elif isinstance(callee, Overloaded): - if callee.items()[0].is_generic(): - callee = None - else: - callee = Overloaded([c.copy_modified(ret_type=item) - for c in callee.items()]) - if callee: - return callee - - self.msg.unsupported_type_type(item, context) - return AnyType() - - def infer_arg_types_in_context(self, callee: Optional[CallableType], - args: List[Node]) -> List[Type]: - """Infer argument expression types using a callable type as context. - - For example, if callee argument 2 has type List[int], infer the - argument expression with List[int] type context. - """ - # TODO Always called with callee as None, i.e. empty context. - res = [] # type: List[Type] - - fixed = len(args) - if callee: - fixed = min(fixed, callee.max_fixed_args()) - - arg_type = None # type: Type - ctx = None # type: Type - for i, arg in enumerate(args): - if i < fixed: - if callee and i < len(callee.arg_types): - ctx = callee.arg_types[i] - arg_type = self.accept(arg, ctx) - else: - if callee and callee.is_var_arg: - arg_type = self.accept(arg, callee.arg_types[-1]) - else: - arg_type = self.accept(arg) - if has_erased_component(arg_type): - res.append(NoneTyp()) - else: - res.append(arg_type) - return res - - def infer_arg_types_in_context2( - self, callee: CallableType, args: List[Node], arg_kinds: List[int], - formal_to_actual: List[List[int]]) -> List[Type]: - """Infer argument expression types using a callable type as context. - - For example, if callee argument 2 has type List[int], infer the - argument expression with List[int] type context. - - Returns the inferred types of *actual arguments*. - """ - res = [None] * len(args) # type: List[Type] - - for i, actuals in enumerate(formal_to_actual): - for ai in actuals: - if arg_kinds[ai] not in (nodes.ARG_STAR, nodes.ARG_STAR2): - res[ai] = self.accept(args[ai], callee.arg_types[i]) - - # Fill in the rest of the argument types. - for i, t in enumerate(res): - if not t: - res[i] = self.accept(args[i]) - return res - - def infer_function_type_arguments_using_context( - self, callable: CallableType, error_context: Context) -> CallableType: - """Unify callable return type to type context to infer type vars. - - For example, if the return type is set[t] where 't' is a type variable - of callable, and if the context is set[int], return callable modified - by substituting 't' with 'int'. - """ - ctx = self.chk.type_context[-1] - if not ctx: - return callable - # The return type may have references to type metavariables that - # we are inferring right now. We must consider them as indeterminate - # and they are not potential results; thus we replace them with the - # special ErasedType type. On the other hand, class type variables are - # valid results. - erased_ctx = replace_meta_vars(ctx, ErasedType()) - ret_type = callable.ret_type - if isinstance(ret_type, TypeVarType): - if ret_type.values or (not isinstance(ctx, Instance) or - not cast(Instance, ctx).args): - # The return type is a type variable. If it has values, we can't easily restrict - # type inference to conform to the valid values. If it's unrestricted, we could - # infer a too general type for the type variable if we use context, and this could - # result in confusing and spurious type errors elsewhere. - # - # Give up and just use function arguments for type inference. As an exception, - # if the context is a generic instance type, actually use it as context, as - # this *seems* to usually be the reasonable thing to do. - # - # See also github issues #462 and #360. - ret_type = NoneTyp() - args = infer_type_arguments(callable.type_var_ids(), ret_type, erased_ctx) - # Only substitute non-None and non-erased types. - new_args = [] # type: List[Type] - for arg in args: - if isinstance(arg, (NoneTyp, UninhabitedType)) or has_erased_component(arg): - new_args.append(None) - else: - new_args.append(arg) - return cast(CallableType, self.apply_generic_arguments(callable, new_args, - error_context)) - - def infer_function_type_arguments(self, callee_type: CallableType, - args: List[Node], - arg_kinds: List[int], - formal_to_actual: List[List[int]], - context: Context) -> CallableType: - """Infer the type arguments for a generic callee type. - - Infer based on the types of arguments. - - Return a derived callable type that has the arguments applied. - """ - if not self.chk.typing_mode_none(): - # Disable type errors during type inference. There may be errors - # due to partial available context information at this time, but - # these errors can be safely ignored as the arguments will be - # inferred again later. - self.msg.disable_errors() - - arg_types = self.infer_arg_types_in_context2( - callee_type, args, arg_kinds, formal_to_actual) - - self.msg.enable_errors() - - arg_pass_nums = self.get_arg_infer_passes( - callee_type.arg_types, formal_to_actual, len(args)) - - pass1_args = [] # type: List[Optional[Type]] - for i, arg in enumerate(arg_types): - if arg_pass_nums[i] > 1: - pass1_args.append(None) - else: - pass1_args.append(arg) - - inferred_args = infer_function_type_arguments( - callee_type, pass1_args, arg_kinds, formal_to_actual, - strict=self.chk.typing_mode_full()) # type: List[Type] - - if 2 in arg_pass_nums: - # Second pass of type inference. - (callee_type, - inferred_args) = self.infer_function_type_arguments_pass2( - callee_type, args, arg_kinds, formal_to_actual, - inferred_args, context) - - if callee_type.special_sig == 'dict' and len(inferred_args) == 2 and ( - ARG_NAMED in arg_kinds or ARG_STAR2 in arg_kinds): - # HACK: Infer str key type for dict(...) with keyword args. The type system - # can't represent this so we special case it, as this is a pretty common - # thing. This doesn't quite work with all possible subclasses of dict - # if they shuffle type variables around, as we assume that there is a 1-1 - # correspondence with dict type variables. This is a marginal issue and - # a little tricky to fix so it's left unfixed for now. - if isinstance(inferred_args[0], (NoneTyp, UninhabitedType)): - inferred_args[0] = self.named_type('builtins.str') - elif not is_subtype(self.named_type('builtins.str'), inferred_args[0]): - self.msg.fail(messages.KEYWORD_ARGUMENT_REQUIRES_STR_KEY_TYPE, - context) - else: - # In dynamically typed functions use implicit 'Any' types for - # type variables. - inferred_args = [AnyType()] * len(callee_type.variables) - return self.apply_inferred_arguments(callee_type, inferred_args, - context) - - def infer_function_type_arguments_pass2( - self, callee_type: CallableType, - args: List[Node], - arg_kinds: List[int], - formal_to_actual: List[List[int]], - inferred_args: List[Type], - context: Context) -> Tuple[CallableType, List[Type]]: - """Perform second pass of generic function type argument inference. - - The second pass is needed for arguments with types such as Callable[[T], S], - where both T and S are type variables, when the actual argument is a - lambda with inferred types. The idea is to infer the type variable T - in the first pass (based on the types of other arguments). This lets - us infer the argument and return type of the lambda expression and - thus also the type variable S in this second pass. - - Return (the callee with type vars applied, inferred actual arg types). - """ - # None or erased types in inferred types mean that there was not enough - # information to infer the argument. Replace them with None values so - # that they are not applied yet below. - for i, arg in enumerate(inferred_args): - if isinstance(arg, (NoneTyp, UninhabitedType)) or has_erased_component(arg): - inferred_args[i] = None - - callee_type = cast(CallableType, self.apply_generic_arguments( - callee_type, inferred_args, context)) - arg_types = self.infer_arg_types_in_context2( - callee_type, args, arg_kinds, formal_to_actual) - - inferred_args = infer_function_type_arguments( - callee_type, arg_types, arg_kinds, formal_to_actual) - - return callee_type, inferred_args - - def get_arg_infer_passes(self, arg_types: List[Type], - formal_to_actual: List[List[int]], - num_actuals: int) -> List[int]: - """Return pass numbers for args for two-pass argument type inference. - - For each actual, the pass number is either 1 (first pass) or 2 (second - pass). - - Two-pass argument type inference primarily lets us infer types of - lambdas more effectively. - """ - res = [1] * num_actuals - for i, arg in enumerate(arg_types): - if arg.accept(ArgInferSecondPassQuery()): - for j in formal_to_actual[i]: - res[j] = 2 - return res - - def apply_inferred_arguments(self, callee_type: CallableType, - inferred_args: List[Type], - context: Context) -> CallableType: - """Apply inferred values of type arguments to a generic function. - - Inferred_args contains the values of function type arguments. - """ - # Report error if some of the variables could not be solved. In that - # case assume that all variables have type Any to avoid extra - # bogus error messages. - for i, inferred_type in enumerate(inferred_args): - if not inferred_type or has_erased_component(inferred_type): - # Could not infer a non-trivial type for a type variable. - self.msg.could_not_infer_type_arguments( - callee_type, i + 1, context) - inferred_args = [AnyType()] * len(inferred_args) - # Apply the inferred types to the function type. In this case the - # return type must be CallableType, since we give the right number of type - # arguments. - return cast(CallableType, self.apply_generic_arguments(callee_type, - inferred_args, context)) - - def check_argument_count(self, callee: CallableType, actual_types: List[Type], - actual_kinds: List[int], actual_names: List[str], - formal_to_actual: List[List[int]], - context: Context, - messages: Optional[MessageBuilder]) -> bool: - """Check that there is a value for all required arguments to a function. - - Also check that there are no duplicate values for arguments. Report found errors - using 'messages' if it's not None. - - Return False if there were any errors. Otherwise return True - """ - # TODO(jukka): We could return as soon as we find an error if messages is None. - formal_kinds = callee.arg_kinds - - # Collect list of all actual arguments matched to formal arguments. - all_actuals = [] # type: List[int] - for actuals in formal_to_actual: - all_actuals.extend(actuals) - - is_unexpected_arg_error = False # Keep track of errors to avoid duplicate errors. - ok = True # False if we've found any error. - for i, kind in enumerate(actual_kinds): - if i not in all_actuals and ( - kind != nodes.ARG_STAR or - not is_empty_tuple(actual_types[i])): - # Extra actual: not matched by a formal argument. - ok = False - if kind != nodes.ARG_NAMED: - if messages: - messages.too_many_arguments(callee, context) - else: - if messages: - messages.unexpected_keyword_argument( - callee, actual_names[i], context) - is_unexpected_arg_error = True - elif kind == nodes.ARG_STAR and ( - nodes.ARG_STAR not in formal_kinds): - actual_type = actual_types[i] - if isinstance(actual_type, TupleType): - if all_actuals.count(i) < len(actual_type.items): - # Too many tuple items as some did not match. - if messages: - messages.too_many_arguments(callee, context) - ok = False - # *args can be applied even if the function takes a fixed - # number of positional arguments. This may succeed at runtime. - - for i, kind in enumerate(formal_kinds): - if kind == nodes.ARG_POS and (not formal_to_actual[i] and - not is_unexpected_arg_error): - # No actual for a mandatory positional formal. - if messages: - messages.too_few_arguments(callee, context, actual_names) - ok = False - elif kind in [nodes.ARG_POS, nodes.ARG_OPT, - nodes.ARG_NAMED] and is_duplicate_mapping( - formal_to_actual[i], actual_kinds): - if (self.chk.typing_mode_full() or - isinstance(actual_types[formal_to_actual[i][0]], TupleType)): - if messages: - messages.duplicate_argument_value(callee, i, context) - ok = False - elif (kind == nodes.ARG_NAMED and formal_to_actual[i] and - actual_kinds[formal_to_actual[i][0]] != nodes.ARG_NAMED): - # Positional argument when expecting a keyword argument. - if messages: - messages.too_many_positional_arguments(callee, context) - ok = False - return ok - - def check_argument_types(self, arg_types: List[Type], arg_kinds: List[int], - callee: CallableType, - formal_to_actual: List[List[int]], - context: Context, - messages: MessageBuilder = None, - check_arg: ArgChecker = None) -> None: - """Check argument types against a callable type. - - Report errors if the argument types are not compatible. - """ - messages = messages or self.msg - check_arg = check_arg or self.check_arg - # Keep track of consumed tuple *arg items. - tuple_counter = [0] - for i, actuals in enumerate(formal_to_actual): - for actual in actuals: - arg_type = arg_types[actual] - # Check that a *arg is valid as varargs. - if (arg_kinds[actual] == nodes.ARG_STAR and - not self.is_valid_var_arg(arg_type)): - messages.invalid_var_arg(arg_type, context) - if (arg_kinds[actual] == nodes.ARG_STAR2 and - not self.is_valid_keyword_var_arg(arg_type)): - messages.invalid_keyword_var_arg(arg_type, context) - # Get the type of an individual actual argument (for *args - # and **args this is the item type, not the collection type). - actual_type = get_actual_type(arg_type, arg_kinds[actual], - tuple_counter) - check_arg(actual_type, arg_type, arg_kinds[actual], - callee.arg_types[i], - actual + 1, i + 1, callee, context, messages) - - # There may be some remaining tuple varargs items that haven't - # been checked yet. Handle them. - if (callee.arg_kinds[i] == nodes.ARG_STAR and - arg_kinds[actual] == nodes.ARG_STAR and - isinstance(arg_types[actual], TupleType)): - tuplet = cast(TupleType, arg_types[actual]) - while tuple_counter[0] < len(tuplet.items): - actual_type = get_actual_type(arg_type, - arg_kinds[actual], - tuple_counter) - check_arg(actual_type, arg_type, arg_kinds[actual], - callee.arg_types[i], - actual + 1, i + 1, callee, context, messages) - - def check_arg(self, caller_type: Type, original_caller_type: Type, - caller_kind: int, - callee_type: Type, n: int, m: int, callee: CallableType, - context: Context, messages: MessageBuilder) -> None: - """Check the type of a single argument in a call.""" - if isinstance(caller_type, Void): - messages.does_not_return_value(caller_type, context) - elif isinstance(caller_type, DeletedType): - messages.deleted_as_rvalue(caller_type, context) - elif not is_subtype(caller_type, callee_type): - messages.incompatible_argument(n, m, callee, original_caller_type, - caller_kind, context) - - def overload_call_target(self, arg_types: List[Type], arg_kinds: List[int], - arg_names: List[str], - overload: Overloaded, context: Context, - messages: MessageBuilder = None) -> Type: - """Infer the correct overload item to call with given argument types. - - The return value may be CallableType or AnyType (if an unique item - could not be determined). - """ - messages = messages or self.msg - # TODO: For overlapping signatures we should try to get a more precise - # result than 'Any'. - match = [] # type: List[CallableType] - best_match = 0 - for typ in overload.items(): - similarity = self.erased_signature_similarity(arg_types, arg_kinds, arg_names, - typ, context=context) - if similarity > 0 and similarity >= best_match: - if (match and not is_same_type(match[-1].ret_type, - typ.ret_type) and - not mypy.checker.is_more_precise_signature( - match[-1], typ)): - # Ambiguous return type. Either the function overload is - # overlapping (which we don't handle very well here) or the - # caller has provided some Any argument types; in either - # case we'll fall back to Any. It's okay to use Any types - # in calls. - # - # Overlapping overload items are generally fine if the - # overlapping is only possible when there is multiple - # inheritance, as this is rare. See docstring of - # mypy.meet.is_overlapping_types for more about this. - # - # Note that there is no ambiguity if the items are - # covariant in both argument types and return types with - # respect to type precision. We'll pick the best/closest - # match. - # - # TODO: Consider returning a union type instead if the - # overlapping is NOT due to Any types? - return AnyType() - else: - match.append(typ) - best_match = max(best_match, similarity) - if not match: - messages.no_variant_matches_arguments(overload, arg_types, context) - return AnyType() - else: - if len(match) == 1: - return match[0] - else: - # More than one signature matches. Pick the first *non-erased* - # matching signature, or default to the first one if none - # match. - for m in match: - if self.match_signature_types(arg_types, arg_kinds, arg_names, m, - context=context): - return m - return match[0] - - def erased_signature_similarity(self, arg_types: List[Type], arg_kinds: List[int], - arg_names: List[str], callee: CallableType, - context: Context) -> int: - """Determine whether arguments could match the signature at runtime. - - Return similarity level (0 = no match, 1 = can match, 2 = non-promotion match). See - overload_arg_similarity for a discussion of similarity levels. - """ - formal_to_actual = map_actuals_to_formals(arg_kinds, - arg_names, - callee.arg_kinds, - callee.arg_names, - lambda i: arg_types[i]) - - if not self.check_argument_count(callee, arg_types, arg_kinds, arg_names, - formal_to_actual, None, None): - # Too few or many arguments -> no match. - return 0 - - similarity = 2 - - def check_arg(caller_type: Type, original_caller_type: Type, caller_kind: int, - callee_type: Type, n: int, m: int, callee: CallableType, - context: Context, messages: MessageBuilder) -> None: - nonlocal similarity - similarity = min(similarity, - overload_arg_similarity(caller_type, callee_type)) - if similarity == 0: - # No match -- exit early since none of the remaining work can change - # the result. - raise Finished - - try: - self.check_argument_types(arg_types, arg_kinds, callee, formal_to_actual, - context=context, check_arg=check_arg) - except Finished: - pass - - return similarity - - def match_signature_types(self, arg_types: List[Type], arg_kinds: List[int], - arg_names: List[str], callee: CallableType, - context: Context) -> bool: - """Determine whether arguments types match the signature. - - Assume that argument counts are compatible. - - Return True if arguments match. - """ - formal_to_actual = map_actuals_to_formals(arg_kinds, - arg_names, - callee.arg_kinds, - callee.arg_names, - lambda i: arg_types[i]) - ok = True - - def check_arg(caller_type: Type, original_caller_type: Type, caller_kind: int, - callee_type: Type, n: int, m: int, callee: CallableType, - context: Context, messages: MessageBuilder) -> None: - nonlocal ok - if not is_subtype(caller_type, callee_type): - ok = False - - self.check_argument_types(arg_types, arg_kinds, callee, formal_to_actual, - context=context, check_arg=check_arg) - return ok - - def apply_generic_arguments(self, callable: CallableType, types: List[Type], - context: Context) -> Type: - """Simple wrapper around mypy.applytype.apply_generic_arguments.""" - return applytype.apply_generic_arguments(callable, types, self.msg, context) - - def apply_generic_arguments2(self, overload: Overloaded, types: List[Type], - context: Context) -> Type: - items = [] # type: List[CallableType] - for item in overload.items(): - applied = self.apply_generic_arguments(item, types, context) - if isinstance(applied, CallableType): - items.append(applied) - else: - # There was an error. - return AnyType() - return Overloaded(items) - - def visit_member_expr(self, e: MemberExpr) -> Type: - """Visit member expression (of form e.id).""" - result = self.analyze_ordinary_member_access(e, False) - return self.chk.narrow_type_from_binder(e, result) - - def analyze_ordinary_member_access(self, e: MemberExpr, - is_lvalue: bool) -> Type: - """Analyse member expression or member lvalue.""" - if e.kind is not None: - # This is a reference to a module attribute. - return self.analyze_ref_expr(e) - else: - # This is a reference to a non-module attribute. - return analyze_member_access(e.name, self.accept(e.expr), e, - is_lvalue, False, - self.named_type, self.not_ready_callback, self.msg) - - def analyze_external_member_access(self, member: str, base_type: Type, - context: Context) -> Type: - """Analyse member access that is external, i.e. it cannot - refer to private definitions. Return the result type. - """ - # TODO remove; no private definitions in mypy - return analyze_member_access(member, base_type, context, False, False, - self.named_type, self.not_ready_callback, self.msg) - - def visit_int_expr(self, e: IntExpr) -> Type: - """Type check an integer literal (trivial).""" - return self.named_type('builtins.int') - - def visit_str_expr(self, e: StrExpr) -> Type: - """Type check a string literal (trivial).""" - return self.named_type('builtins.str') - - def visit_bytes_expr(self, e: BytesExpr) -> Type: - """Type check a bytes literal (trivial).""" - return self.named_type('builtins.bytes') - - def visit_unicode_expr(self, e: UnicodeExpr) -> Type: - """Type check a unicode literal (trivial).""" - return self.named_type('builtins.unicode') - - def visit_float_expr(self, e: FloatExpr) -> Type: - """Type check a float literal (trivial).""" - return self.named_type('builtins.float') - - def visit_complex_expr(self, e: ComplexExpr) -> Type: - """Type check a complex literal.""" - return self.named_type('builtins.complex') - - def visit_ellipsis(self, e: EllipsisExpr) -> Type: - """Type check '...'.""" - if self.chk.options.python_version[0] >= 3: - return self.named_type('builtins.ellipsis') - else: - # '...' is not valid in normal Python 2 code, but it can - # be used in stubs. The parser makes sure that we only - # get this far if we are in a stub, and we can safely - # return 'object' as ellipsis is special cased elsewhere. - # The builtins.ellipsis type does not exist in Python 2. - return self.named_type('builtins.object') - - def visit_op_expr(self, e: OpExpr) -> Type: - """Type check a binary operator expression.""" - if e.op == 'and' or e.op == 'or': - return self.check_boolean_op(e, e) - if e.op == '*' and isinstance(e.left, ListExpr): - # Expressions of form [...] * e get special type inference. - return self.check_list_multiply(e) - if e.op == '%' and isinstance(e.left, (StrExpr, BytesExpr)): - return self.strfrm_checker.check_str_interpolation(cast(StrExpr, e.left), e.right) - left_type = self.accept(e.left) - - if e.op in nodes.op_methods: - method = self.get_operator_method(e.op) - result, method_type = self.check_op(method, left_type, e.right, e, - allow_reverse=True) - e.method_type = method_type - return result - else: - raise RuntimeError('Unknown operator {}'.format(e.op)) - - def visit_comparison_expr(self, e: ComparisonExpr) -> Type: - """Type check a comparison expression. - - Comparison expressions are type checked consecutive-pair-wise - That is, 'a < b > c == d' is check as 'a < b and b > c and c == d' - """ - result = None # type: mypy.types.Type - - # Check each consecutive operand pair and their operator - for left, right, operator in zip(e.operands, e.operands[1:], e.operators): - left_type = self.accept(left) - - method_type = None # type: mypy.types.Type - - if operator == 'in' or operator == 'not in': - right_type = self.accept(right) # TODO only evaluate if needed - - # Keep track of whether we get type check errors (these won't be reported, they - # are just to verify whether something is valid typing wise). - local_errors = self.msg.copy() - local_errors.disable_count = 0 - sub_result, method_type = self.check_op_local('__contains__', right_type, - left, e, local_errors) - if isinstance(right_type, PartialType): - # We don't really know if this is an error or not, so just shut up. - pass - elif (local_errors.is_errors() and - # is_valid_var_arg is True for any Iterable - self.is_valid_var_arg(right_type)): - itertype = self.chk.analyze_iterable_item_type(right) - method_type = CallableType( - [left_type], - [nodes.ARG_POS], - [None], - self.chk.bool_type(), - self.named_type('builtins.function')) - sub_result = self.chk.bool_type() - if not is_subtype(left_type, itertype): - self.msg.unsupported_operand_types('in', left_type, right_type, e) - else: - self.msg.add_errors(local_errors) - if operator == 'not in': - sub_result = self.chk.bool_type() - elif operator in nodes.op_methods: - method = self.get_operator_method(operator) - sub_result, method_type = self.check_op(method, left_type, right, e, - allow_reverse=True) - - elif operator == 'is' or operator == 'is not': - sub_result = self.chk.bool_type() - method_type = None - else: - raise RuntimeError('Unknown comparison operator {}'.format(operator)) - - e.method_types.append(method_type) - - # Determine type of boolean-and of result and sub_result - if result is None: - result = sub_result - else: - # TODO: check on void needed? - self.check_not_void(sub_result, e) - result = join.join_types(result, sub_result) - - return result - - def get_operator_method(self, op: str) -> str: - if op == '/' and self.chk.options.python_version[0] == 2: - # TODO also check for "from __future__ import division" - return '__div__' - else: - return nodes.op_methods[op] - - def check_op_local(self, method: str, base_type: Type, arg: Node, - context: Context, local_errors: MessageBuilder) -> Tuple[Type, Type]: - """Type check a binary operation which maps to a method call. - - Return tuple (result type, inferred operator method type). - """ - method_type = analyze_member_access(method, base_type, context, False, False, - self.named_type, self.not_ready_callback, local_errors) - return self.check_call(method_type, [arg], [nodes.ARG_POS], - context, arg_messages=local_errors) - - def check_op(self, method: str, base_type: Type, arg: Node, - context: Context, - allow_reverse: bool = False) -> Tuple[Type, Type]: - """Type check a binary operation which maps to a method call. - - Return tuple (result type, inferred operator method type). - """ - # Use a local error storage for errors related to invalid argument - # type (but NOT other errors). This error may need to be suppressed - # for operators which support __rX methods. - local_errors = self.msg.copy() - local_errors.disable_count = 0 - if not allow_reverse or self.has_member(base_type, method): - result = self.check_op_local(method, base_type, arg, context, - local_errors) - if allow_reverse: - arg_type = self.chk.type_map[arg] - if isinstance(arg_type, AnyType): - # If the right operand has type Any, we can't make any - # conjectures about the type of the result, since the - # operand could have a __r method that returns anything. - - # However, in weak mode, we do make conjectures. - if not self.chk.typing_mode_weak(): - result = AnyType(), result[1] - success = not local_errors.is_errors() - else: - result = AnyType(), AnyType() - success = False - if success or not allow_reverse or isinstance(base_type, AnyType): - # We were able to call the normal variant of the operator method, - # or there was some problem not related to argument type - # validity, or the operator has no __rX method. In any case, we - # don't need to consider the __rX method. - self.msg.add_errors(local_errors) - return result - else: - # Calling the operator method was unsuccessful. Try the __rX - # method of the other operand instead. - rmethod = self.get_reverse_op_method(method) - arg_type = self.accept(arg) - if self.has_member(arg_type, rmethod): - method_type = self.analyze_external_member_access( - rmethod, arg_type, context) - temp = TempNode(base_type) - return self.check_call(method_type, [temp], [nodes.ARG_POS], - context) - else: - # No __rX method either. Do deferred type checking to produce - # error message that we may have missed previously. - # TODO Fix type checking an expression more than once. - return self.check_op_local(method, base_type, arg, context, - self.msg) - - def get_reverse_op_method(self, method: str) -> str: - if method == '__div__' and self.chk.options.python_version[0] == 2: - return '__rdiv__' - else: - return nodes.reverse_op_methods[method] - - def check_boolean_op(self, e: OpExpr, context: Context) -> Type: - """Type check a boolean operation ('and' or 'or').""" - - # A boolean operation can evaluate to either of the operands. - - # We use the current type context to guide the type inference of of - # the left operand. We also use the left operand type to guide the type - # inference of the right operand so that expressions such as - # '[1] or []' are inferred correctly. - ctx = self.chk.type_context[-1] - left_type = self.accept(e.left, ctx) - - if e.op == 'and': - right_map, _ = \ - mypy.checker.find_isinstance_check(e.left, self.chk.type_map, - self.chk.typing_mode_weak()) - elif e.op == 'or': - _, right_map = \ - mypy.checker.find_isinstance_check(e.left, self.chk.type_map, - self.chk.typing_mode_weak()) - else: - right_map = None - - with self.chk.binder.frame_context(): - if right_map: - for var, type in right_map.items(): - self.chk.binder.push(var, type) - - right_type = self.accept(e.right, left_type) - - self.check_not_void(left_type, context) - self.check_not_void(right_type, context) - return UnionType.make_simplified_union([left_type, right_type]) - - def check_list_multiply(self, e: OpExpr) -> Type: - """Type check an expression of form '[...] * e'. - - Type inference is special-cased for this common construct. - """ - right_type = self.accept(e.right) - if is_subtype(right_type, self.named_type('builtins.int')): - # Special case: [...] * . Use the type context of the - # OpExpr, since the multiplication does not affect the type. - left_type = self.accept(e.left, context=self.chk.type_context[-1]) - else: - left_type = self.accept(e.left) - result, method_type = self.check_op('__mul__', left_type, e.right, e) - e.method_type = method_type - return result - - def visit_unary_expr(self, e: UnaryExpr) -> Type: - """Type check an unary operation ('not', '-', '+' or '~').""" - operand_type = self.accept(e.expr) - op = e.op - if op == 'not': - self.check_not_void(operand_type, e) - result = self.chk.bool_type() # type: Type - elif op == '-': - method_type = self.analyze_external_member_access('__neg__', - operand_type, e) - result, method_type = self.check_call(method_type, [], [], e) - e.method_type = method_type - elif op == '+': - method_type = self.analyze_external_member_access('__pos__', - operand_type, e) - result, method_type = self.check_call(method_type, [], [], e) - e.method_type = method_type - else: - assert op == '~', "unhandled unary operator" - method_type = self.analyze_external_member_access('__invert__', - operand_type, e) - result, method_type = self.check_call(method_type, [], [], e) - e.method_type = method_type - return result - - def visit_index_expr(self, e: IndexExpr) -> Type: - """Type check an index expression (base[index]). - - It may also represent type application. - """ - result = self.visit_index_expr_helper(e) - return self.chk.narrow_type_from_binder(e, result) - - def visit_index_expr_helper(self, e: IndexExpr) -> Type: - if e.analyzed: - # It's actually a type application. - return self.accept(e.analyzed) - left_type = self.accept(e.base) - if isinstance(left_type, TupleType) and self.chk.typing_mode_full(): - # Special case for tuples. They support indexing only by integer - # literals. (Except in weak type checking mode.) - index = e.index - if isinstance(index, SliceExpr): - return self.visit_tuple_slice_helper(left_type, index) - - ok = False - if isinstance(index, IntExpr): - n = index.value - ok = True - elif isinstance(index, UnaryExpr): - if index.op == '-': - operand = index.expr - if isinstance(operand, IntExpr): - n = len(left_type.items) - operand.value - ok = True - if ok: - if n >= 0 and n < len(left_type.items): - return left_type.items[n] - else: - self.chk.fail(messages.TUPLE_INDEX_OUT_OF_RANGE, e) - return AnyType() - else: - self.chk.fail(messages.TUPLE_INDEX_MUST_BE_AN_INT_LITERAL, e) - return AnyType() - else: - result, method_type = self.check_op('__getitem__', left_type, e.index, e) - e.method_type = method_type - return result - - def visit_tuple_slice_helper(self, left_type: TupleType, slic: SliceExpr): - begin = None # type: int - end = None # type: int - stride = None # type:int - - if slic.begin_index: - begin = self._get_value(slic.begin_index) - if begin is None: - self.chk.fail( - messages.TUPLE_SLICE_MUST_BE_AN_INT_LITERAL, - slic.begin_index) - return AnyType() - - if slic.end_index: - end = self._get_value(slic.end_index) - if end is None: - self.chk.fail( - messages.TUPLE_SLICE_MUST_BE_AN_INT_LITERAL, - slic.end_index) - return AnyType() - - if slic.stride: - stride = self._get_value(slic.stride) - if stride is None: - self.chk.fail( - messages.TUPLE_SLICE_MUST_BE_AN_INT_LITERAL, - slic.stride) - return AnyType() - - return left_type.slice(begin, stride, end) - - def _get_value(self, index: Node) -> Optional[int]: - if isinstance(index, IntExpr): - return index.value - elif isinstance(index, UnaryExpr): - if index.op == '-': - operand = index.expr - if isinstance(operand, IntExpr): - return -1 * operand.value - return None - - def visit_cast_expr(self, expr: CastExpr) -> Type: - """Type check a cast expression.""" - source_type = self.accept(expr.expr, context=AnyType()) - target_type = expr.type - if self.chk.options.warn_redundant_casts and is_same_type(source_type, target_type): - self.msg.redundant_cast(target_type, expr) - if not self.is_valid_cast(source_type, target_type): - self.msg.invalid_cast(target_type, source_type, expr) - return target_type - - def is_valid_cast(self, source_type: Type, target_type: Type) -> bool: - """Is a cast from source_type to target_type meaningful?""" - return (isinstance(target_type, AnyType) or - (not isinstance(source_type, Void) and - not isinstance(target_type, Void))) - - def visit_reveal_type_expr(self, expr: RevealTypeExpr) -> Type: - """Type check a reveal_type expression.""" - revealed_type = self.accept(expr.expr) - self.msg.reveal_type(revealed_type, expr) - return revealed_type - - def visit_type_application(self, tapp: TypeApplication) -> Type: - """Type check a type application (expr[type, ...]).""" - self.chk.fail(messages.GENERIC_TYPE_NOT_VALID_AS_EXPRESSION, tapp) - return AnyType() - - def visit_type_alias_expr(self, alias: TypeAliasExpr) -> Type: - return AnyType() - - def visit_list_expr(self, e: ListExpr) -> Type: - """Type check a list expression [...].""" - return self.check_list_or_set_expr(e.items, 'builtins.list', '', - e) - - def visit_set_expr(self, e: SetExpr) -> Type: - return self.check_list_or_set_expr(e.items, 'builtins.set', '', e) - - def check_list_or_set_expr(self, items: List[Node], fullname: str, - tag: str, context: Context) -> Type: - # Translate into type checking a generic function call. - tvdef = TypeVarDef('T', -1, [], self.chk.object_type()) - tv = TypeVarType(tvdef) - constructor = CallableType( - [tv], - [nodes.ARG_STAR], - [None], - self.chk.named_generic_type(fullname, [tv]), - self.named_type('builtins.function'), - name=tag, - variables=[tvdef]) - return self.check_call(constructor, - items, - [nodes.ARG_POS] * len(items), context)[0] - - def visit_tuple_expr(self, e: TupleExpr) -> Type: - """Type check a tuple expression.""" - ctx = None # type: TupleType - # Try to determine type context for type inference. - if isinstance(self.chk.type_context[-1], TupleType): - t = self.chk.type_context[-1] - if len(t.items) == len(e.items): - ctx = t - # Infer item types. - items = [] # type: List[Type] - for i in range(len(e.items)): - item = e.items[i] - tt = None # type: Type - if not ctx: - tt = self.accept(item) - else: - tt = self.accept(item, ctx.items[i]) - self.check_not_void(tt, e) - items.append(tt) - fallback_item = join.join_type_list(items) - return TupleType(items, self.chk.named_generic_type('builtins.tuple', [fallback_item])) - - def visit_dict_expr(self, e: DictExpr) -> Type: - # Translate into type checking a generic function call. - ktdef = TypeVarDef('KT', -1, [], self.chk.object_type()) - vtdef = TypeVarDef('VT', -2, [], self.chk.object_type()) - kt = TypeVarType(ktdef) - vt = TypeVarType(vtdef) - # The callable type represents a function like this: - # - # def (*v: Tuple[kt, vt]) -> Dict[kt, vt]: ... - constructor = CallableType( - [TupleType([kt, vt], self.named_type('builtins.tuple'))], - [nodes.ARG_STAR], - [None], - self.chk.named_generic_type('builtins.dict', [kt, vt]), - self.named_type('builtins.function'), - name='', - variables=[ktdef, vtdef]) - # Synthesize function arguments. - args = [] # type: List[Node] - for key, value in e.items: - args.append(TupleExpr([key, value])) - return self.check_call(constructor, - args, - [nodes.ARG_POS] * len(args), e)[0] - - def visit_func_expr(self, e: FuncExpr) -> Type: - """Type check lambda expression.""" - inferred_type = self.infer_lambda_type_using_context(e) - if not inferred_type: - # No useful type context. - ret_type = e.expr().accept(self.chk) - if not e.arguments: - # Form 'lambda: e'; just use the inferred return type. - return CallableType([], [], [], ret_type, self.named_type('builtins.function')) - else: - # TODO: Consider reporting an error. However, this is fine if - # we are just doing the first pass in contextual type - # inference. - return AnyType() - else: - # Type context available. - self.chk.check_func_item(e, type_override=inferred_type) - if e.expr() not in self.chk.type_map: - self.accept(e.expr()) - ret_type = self.chk.type_map[e.expr()] - if isinstance(ret_type, NoneTyp): - # For "lambda ...: None", just use type from the context. - # Important when the context is Callable[..., None] which - # really means Void. See #1425. - return inferred_type - return replace_callable_return_type(inferred_type, ret_type) - - def infer_lambda_type_using_context(self, e: FuncExpr) -> CallableType: - """Try to infer lambda expression type using context. - - Return None if could not infer type. - """ - # TODO also accept 'Any' context - ctx = self.chk.type_context[-1] - if not ctx or not isinstance(ctx, CallableType): - return None - - # The context may have function type variables in it. We replace them - # since these are the type variables we are ultimately trying to infer; - # they must be considered as indeterminate. We use ErasedType since it - # does not affect type inference results (it is for purposes like this - # only). - ctx = replace_meta_vars(ctx, ErasedType()) - - callable_ctx = cast(CallableType, ctx) - - arg_kinds = [arg.kind for arg in e.arguments] - - if callable_ctx.is_ellipsis_args: - # Fill in Any arguments to match the arguments of the lambda. - callable_ctx = callable_ctx.copy_modified( - is_ellipsis_args=False, - arg_types=[AnyType()] * len(arg_kinds), - arg_kinds=arg_kinds - ) - - if callable_ctx.arg_kinds != arg_kinds: - # Incompatible context; cannot use it to infer types. - self.chk.fail(messages.CANNOT_INFER_LAMBDA_TYPE, e) - return None - - return callable_ctx - - def visit_super_expr(self, e: SuperExpr) -> Type: - """Type check a super expression (non-lvalue).""" - t = self.analyze_super(e, False) - return t - - def analyze_super(self, e: SuperExpr, is_lvalue: bool) -> Type: - """Type check a super expression.""" - if e.info and e.info.bases: - # TODO fix multiple inheritance etc - if len(e.info.mro) < 2: - self.chk.fail('Internal error: unexpected mro for {}: {}'.format( - e.info.name(), e.info.mro), e) - return AnyType() - for base in e.info.mro[1:]: - if e.name in base.names or base == e.info.mro[-1]: - if e.info.fallback_to_any and base == e.info.mro[-1]: - # There's an undefined base class, and we're - # at the end of the chain. That's not an error. - return AnyType() - if not self.chk.typing_mode_full(): - return AnyType() - return analyze_member_access(e.name, self_type(e.info), e, - is_lvalue, True, - self.named_type, self.not_ready_callback, - self.msg, base) - else: - # Invalid super. This has been reported by the semantic analyzer. - return AnyType() - - def visit_slice_expr(self, e: SliceExpr) -> Type: - for index in [e.begin_index, e.end_index, e.stride]: - if index: - t = self.accept(index) - self.chk.check_subtype(t, self.named_type('builtins.int'), - index, messages.INVALID_SLICE_INDEX) - return self.named_type('builtins.slice') - - def visit_list_comprehension(self, e: ListComprehension) -> Type: - return self.check_generator_or_comprehension( - e.generator, 'builtins.list', '') - - def visit_set_comprehension(self, e: SetComprehension) -> Type: - return self.check_generator_or_comprehension( - e.generator, 'builtins.set', '') - - def visit_generator_expr(self, e: GeneratorExpr) -> Type: - return self.check_generator_or_comprehension(e, 'typing.Iterator', - '') - - def check_generator_or_comprehension(self, gen: GeneratorExpr, - type_name: str, - id_for_messages: str) -> Type: - """Type check a generator expression or a list comprehension.""" - self.check_for_comp(gen) - - # Infer the type of the list comprehension by using a synthetic generic - # callable type. - tvdef = TypeVarDef('T', -1, [], self.chk.object_type()) - tv = TypeVarType(tvdef) - constructor = CallableType( - [tv], - [nodes.ARG_POS], - [None], - self.chk.named_generic_type(type_name, [tv]), - self.chk.named_type('builtins.function'), - name=id_for_messages, - variables=[tvdef]) - return self.check_call(constructor, - [gen.left_expr], [nodes.ARG_POS], gen)[0] - - def visit_dictionary_comprehension(self, e: DictionaryComprehension): - """Type check a dictionary comprehension.""" - self.check_for_comp(e) - - # Infer the type of the list comprehension by using a synthetic generic - # callable type. - ktdef = TypeVarDef('KT', -1, [], self.chk.object_type()) - vtdef = TypeVarDef('VT', -2, [], self.chk.object_type()) - kt = TypeVarType(ktdef) - vt = TypeVarType(vtdef) - constructor = CallableType( - [kt, vt], - [nodes.ARG_POS, nodes.ARG_POS], - [None, None], - self.chk.named_generic_type('builtins.dict', [kt, vt]), - self.chk.named_type('builtins.function'), - name='', - variables=[ktdef, vtdef]) - return self.check_call(constructor, - [e.key, e.value], [nodes.ARG_POS, nodes.ARG_POS], e)[0] - - def check_for_comp(self, e: Union[GeneratorExpr, DictionaryComprehension]) -> None: - """Check the for_comp part of comprehensions. That is the part from 'for': - ... for x in y if z - """ - with self.chk.binder.frame_context(): - for index, sequence, conditions in zip(e.indices, e.sequences, - e.condlists): - sequence_type = self.chk.analyze_iterable_item_type(sequence) - self.chk.analyze_index_variables(index, sequence_type, e) - for condition in conditions: - self.accept(condition) - - def visit_conditional_expr(self, e: ConditionalExpr) -> Type: - cond_type = self.accept(e.cond) - self.check_not_void(cond_type, e) - ctx = self.chk.type_context[-1] - - # Gain type information from isinstance if it is there - # but only for the current expression - if_map, else_map = mypy.checker.find_isinstance_check( - e.cond, - self.chk.type_map, - self.chk.typing_mode_weak()) - - if_type = self.analyze_cond_branch(if_map, e.if_expr, context=ctx) - - if not mypy.checker.is_valid_inferred_type(if_type): - # Analyze the right branch disregarding the left branch. - else_type = self.analyze_cond_branch(else_map, e.else_expr, context=ctx) - - # If it would make a difference, re-analyze the left - # branch using the right branch's type as context. - if ctx is None or not is_equivalent(else_type, ctx): - # TODO: If it's possible that the previous analysis of - # the left branch produced errors that are avoided - # using this context, suppress those errors. - if_type = self.analyze_cond_branch(if_map, e.if_expr, context=else_type) - - else: - # Analyze the right branch in the context of the left - # branch's type. - else_type = self.analyze_cond_branch(else_map, e.else_expr, context=if_type) - - res = join.join_types(if_type, else_type) - - return res - - def analyze_cond_branch(self, map: Optional[Dict[Node, Type]], - node: Node, context: Optional[Type]) -> Type: - with self.chk.binder.frame_context(): - if map: - for var, type in map.items(): - self.chk.binder.push(var, type) - return self.accept(node, context=context) - - def visit_backquote_expr(self, e: BackquoteExpr) -> Type: - self.accept(e.expr) - return self.named_type('builtins.str') - - # - # Helpers - # - - def accept(self, node: Node, context: Type = None) -> Type: - """Type check a node. Alias for TypeChecker.accept.""" - return self.chk.accept(node, context) - - def check_not_void(self, typ: Type, context: Context) -> None: - """Generate an error if type is Void.""" - self.chk.check_not_void(typ, context) - - def is_boolean(self, typ: Type) -> bool: - """Is type compatible with bool?""" - return is_subtype(typ, self.chk.bool_type()) - - def named_type(self, name: str) -> Instance: - """Return an instance type with type given by the name and no type - arguments. Alias for TypeChecker.named_type. - """ - return self.chk.named_type(name) - - def is_valid_var_arg(self, typ: Type) -> bool: - """Is a type valid as a *args argument?""" - return (isinstance(typ, TupleType) or - is_subtype(typ, self.chk.named_generic_type('typing.Iterable', - [AnyType()])) or - isinstance(typ, AnyType)) - - def is_valid_keyword_var_arg(self, typ: Type) -> bool: - """Is a type valid as a **kwargs argument?""" - return is_subtype(typ, self.chk.named_generic_type( - 'builtins.dict', [self.named_type('builtins.str'), AnyType()])) - - def has_non_method(self, typ: Type, member: str) -> bool: - """Does type have a member variable / property with the given name?""" - if isinstance(typ, Instance): - return (not typ.type.has_method(member) and - typ.type.has_readable_member(member)) - else: - return False - - def has_member(self, typ: Type, member: str) -> bool: - """Does type have member with the given name?""" - # TODO TupleType => also consider tuple attributes - if isinstance(typ, Instance): - return typ.type.has_readable_member(member) - elif isinstance(typ, AnyType): - return True - elif isinstance(typ, UnionType): - result = all(self.has_member(x, member) for x in typ.items) - return result - elif isinstance(typ, TupleType): - return self.has_member(typ.fallback, member) - else: - return False - - def not_ready_callback(self, name: str, context: Context) -> None: - """Called when we can't infer the type of a variable because it's not ready yet. - - Either defer type checking of the enclosing function to the next - pass or report an error. - """ - self.chk.handle_cannot_determine_type(name, context) - - -def map_actuals_to_formals(caller_kinds: List[int], - caller_names: List[str], - callee_kinds: List[int], - callee_names: List[str], - caller_arg_type: Callable[[int], - Type]) -> List[List[int]]: - """Calculate mapping between actual (caller) args and formals. - - The result contains a list of caller argument indexes mapping to each - callee argument index, indexed by callee index. - - The caller_arg_type argument should evaluate to the type of the actual - argument type with the given index. - """ - ncallee = len(callee_kinds) - map = [None] * ncallee # type: List[List[int]] - for i in range(ncallee): - map[i] = [] - j = 0 - for i, kind in enumerate(caller_kinds): - if kind == nodes.ARG_POS: - if j < ncallee: - if callee_kinds[j] in [nodes.ARG_POS, nodes.ARG_OPT, - nodes.ARG_NAMED]: - map[j].append(i) - j += 1 - elif callee_kinds[j] == nodes.ARG_STAR: - map[j].append(i) - elif kind == nodes.ARG_STAR: - # We need to know the actual type to map varargs. - argt = caller_arg_type(i) - if isinstance(argt, TupleType): - # A tuple actual maps to a fixed number of formals. - for _ in range(len(argt.items)): - if j < ncallee: - if callee_kinds[j] != nodes.ARG_STAR2: - map[j].append(i) - else: - raise NotImplementedError() - j += 1 - else: - # Assume that it is an iterable (if it isn't, there will be - # an error later). - while j < ncallee: - if callee_kinds[j] in (nodes.ARG_NAMED, nodes.ARG_STAR2): - break - else: - map[j].append(i) - j += 1 - elif kind == nodes.ARG_NAMED: - name = caller_names[i] - if name in callee_names: - map[callee_names.index(name)].append(i) - elif nodes.ARG_STAR2 in callee_kinds: - map[callee_kinds.index(nodes.ARG_STAR2)].append(i) - else: - assert kind == nodes.ARG_STAR2 - for j in range(ncallee): - # TODO tuple varargs complicate this - no_certain_match = ( - not map[j] or caller_kinds[map[j][0]] == nodes.ARG_STAR) - if ((callee_names[j] and no_certain_match) - or callee_kinds[j] == nodes.ARG_STAR2): - map[j].append(i) - return map - - -def is_empty_tuple(t: Type) -> bool: - return isinstance(t, TupleType) and not t.items - - -def is_duplicate_mapping(mapping: List[int], actual_kinds: List[int]) -> bool: - # Multiple actuals can map to the same formal only if they both come from - # varargs (*args and **kwargs); in this case at runtime it is possible that - # there are no duplicates. We need to allow this, as the convention - # f(..., *args, **kwargs) is common enough. - return len(mapping) > 1 and not ( - len(mapping) == 2 and - actual_kinds[mapping[0]] == nodes.ARG_STAR and - actual_kinds[mapping[1]] == nodes.ARG_STAR2) - - -def replace_callable_return_type(c: CallableType, new_ret_type: Type) -> CallableType: - """Return a copy of a callable type with a different return type.""" - return c.copy_modified(ret_type=new_ret_type) - - -class ArgInferSecondPassQuery(types.TypeQuery): - """Query whether an argument type should be inferred in the second pass. - - The result is True if the type has a type variable in a callable return - type anywhere. For example, the result for Callable[[], T] is True if t is - a type variable. - """ - def __init__(self) -> None: - super().__init__(False, types.ANY_TYPE_STRATEGY) - - def visit_callable_type(self, t: CallableType) -> bool: - return self.query_types(t.arg_types) or t.accept(HasTypeVarQuery()) - - -class HasTypeVarQuery(types.TypeQuery): - """Visitor for querying whether a type has a type variable component.""" - def __init__(self) -> None: - super().__init__(False, types.ANY_TYPE_STRATEGY) - - def visit_type_var(self, t: TypeVarType) -> bool: - return True - - -def has_erased_component(t: Type) -> bool: - return t is not None and t.accept(HasErasedComponentsQuery()) - - -class HasErasedComponentsQuery(types.TypeQuery): - """Visitor for querying whether a type has an erased component.""" - def __init__(self) -> None: - super().__init__(False, types.ANY_TYPE_STRATEGY) - - def visit_erased_type(self, t: ErasedType) -> bool: - return True - - -def overload_arg_similarity(actual: Type, formal: Type) -> int: - """Return if caller argument (actual) is compatible with overloaded signature arg (formal). - - Return a similarity level: - 0: no match - 1: actual is compatible, but only using type promotions (e.g. int vs float) - 2: actual is compatible without type promotions (e.g. int vs object) - - The distinction is important in cases where multiple overload items match. We want - give priority to higher similarity matches. - """ - # Replace type variables with their upper bounds. Overloading - # resolution is based on runtime behavior which erases type - # parameters, so no need to handle type variables occurring within - # a type. - if isinstance(actual, TypeVarType): - actual = actual.erase_to_union_or_bound() - if isinstance(formal, TypeVarType): - formal = formal.erase_to_union_or_bound() - if (isinstance(actual, UninhabitedType) or isinstance(actual, AnyType) or - isinstance(formal, AnyType) or isinstance(formal, CallableType) or - (isinstance(actual, Instance) and actual.type.fallback_to_any)): - # These could match anything at runtime. - return 2 - if not experiments.STRICT_OPTIONAL and isinstance(actual, NoneTyp): - # NoneTyp matches anything if we're not doing strict Optional checking - return 2 - if isinstance(actual, UnionType): - return max(overload_arg_similarity(item, formal) - for item in actual.items) - if isinstance(formal, UnionType): - return max(overload_arg_similarity(actual, item) - for item in formal.items) - if isinstance(formal, TypeType): - if isinstance(actual, TypeType): - # Since Type[T] is covariant, check if actual = Type[A] is - # a subtype of formal = Type[F]. - return overload_arg_similarity(actual.item, formal.item) - elif isinstance(actual, CallableType) and actual.is_type_obj(): - # Check if the actual is a constructor of some sort. - # Note that this is this unsound, since we don't check the __init__ signature. - return overload_arg_similarity(actual.ret_type, formal.item) - else: - return 0 - if isinstance(formal, Instance): - if isinstance(actual, CallableType): - actual = actual.fallback - if isinstance(actual, Overloaded): - actual = actual.items()[0].fallback - if isinstance(actual, TupleType): - actual = actual.fallback - if isinstance(actual, Instance): - # First perform a quick check (as an optimization) and fall back to generic - # subtyping algorithm if type promotions are possible (e.g., int vs. float). - if formal.type in actual.type.mro: - return 2 - elif actual.type._promote and is_subtype(actual, formal): - return 1 - else: - return 0 - elif isinstance(actual, TypeType): - if formal.type.fullname() in {"builtins.object", "builtins.type"}: - return 2 - else: - return 0 - else: - return 0 - if isinstance(actual, UnboundType) or isinstance(formal, UnboundType): - # Either actual or formal is the result of an error; shut up. - return 2 - # Fall back to a conservative equality check for the remaining kinds of type. - return 2 if is_same_type(erasetype.erase_type(actual), erasetype.erase_type(formal)) else 0 - - -def freshen_generic_callable(callee: CallableType) -> CallableType: - tvdefs = [] - tvmap = {} # type: Dict[TypeVarId, Type] - for v in callee.variables: - tvdef = TypeVarDef.new_unification_variable(v) - tvdefs.append(tvdef) - tvmap[v.id] = TypeVarType(tvdef) - - return cast(CallableType, expand_type(callee, tvmap)).copy_modified(variables=tvdefs) +"""Expression type checker. This file is conceptually part of TypeChecker.""" + +from typing import cast, Dict, List, Tuple, Callable, Union, Optional + +from mypy.types import ( + Type, AnyType, CallableType, Overloaded, NoneTyp, Void, TypeVarDef, + TupleType, Instance, TypeVarId, TypeVarType, ErasedType, UnionType, + PartialType, DeletedType, UnboundType, UninhabitedType, TypeType +) +from mypy.nodes import ( + NameExpr, RefExpr, Var, FuncDef, OverloadedFuncDef, TypeInfo, CallExpr, + Node, MemberExpr, IntExpr, StrExpr, BytesExpr, UnicodeExpr, FloatExpr, + OpExpr, UnaryExpr, IndexExpr, CastExpr, RevealTypeExpr, TypeApplication, ListExpr, + TupleExpr, DictExpr, FuncExpr, SuperExpr, SliceExpr, Context, + ListComprehension, GeneratorExpr, SetExpr, MypyFile, Decorator, + ConditionalExpr, ComparisonExpr, TempNode, SetComprehension, + DictionaryComprehension, ComplexExpr, EllipsisExpr, + TypeAliasExpr, BackquoteExpr, ARG_POS, ARG_NAMED, ARG_STAR2 +) +from mypy.nodes import function_type +from mypy import nodes +import mypy.checker +from mypy import types +from mypy.sametypes import is_same_type +from mypy.erasetype import replace_meta_vars +from mypy.messages import MessageBuilder +from mypy import messages +from mypy.infer import infer_type_arguments, infer_function_type_arguments +from mypy import join +from mypy.subtypes import is_subtype, is_equivalent +from mypy import applytype +from mypy import erasetype +from mypy.checkmember import analyze_member_access, type_object_type +from mypy.semanal import self_type +from mypy.constraints import get_actual_type +from mypy.checkstrformat import StringFormatterChecker +from mypy.expandtype import expand_type +import mypy.checkexpr + +from mypy import experiments + +# Type of callback user for checking individual function arguments. See +# check_args() below for details. +ArgChecker = Callable[[Type, Type, int, Type, int, int, CallableType, Context, MessageBuilder], + None] + + +class Finished(Exception): + """Raised if we can terminate overload argument check early (no match).""" + + +class ExpressionChecker: + """Expression type checker. + + This class works closely together with checker.TypeChecker. + """ + + # Some services are provided by a TypeChecker instance. + chk = None # type: mypy.checker.TypeChecker + # This is shared with TypeChecker, but stored also here for convenience. + msg = None # type: MessageBuilder + + strfrm_checker = None # type: mypy.checkstrformat.StringFormatterChecker + + def __init__(self, + chk: 'mypy.checker.TypeChecker', + msg: MessageBuilder) -> None: + """Construct an expression type checker.""" + self.chk = chk + self.msg = msg + self.strfrm_checker = mypy.checkexpr.StringFormatterChecker(self, self.chk, self.msg) + + def visit_name_expr(self, e: NameExpr) -> Type: + """Type check a name expression. + + It can be of any kind: local, member or global. + """ + result = self.analyze_ref_expr(e) + return self.chk.narrow_type_from_binder(e, result) + + def analyze_ref_expr(self, e: RefExpr, lvalue: bool = False) -> Type: + result = None # type: Type + node = e.node + if isinstance(node, Var): + # Variable reference. + result = self.analyze_var_ref(node, e) + if isinstance(result, PartialType): + if result.type is None: + # 'None' partial type. It has a well-defined type. In an lvalue context + # we want to preserve the knowledge of it being a partial type. + if not lvalue: + result = NoneTyp() + else: + partial_types = self.chk.find_partial_types(node) + if partial_types is not None and not self.chk.current_node_deferred: + context = partial_types[node] + self.msg.fail(messages.NEED_ANNOTATION_FOR_VAR, context) + result = AnyType() + elif isinstance(node, FuncDef): + # Reference to a global function. + result = function_type(node, self.named_type('builtins.function')) + elif isinstance(node, OverloadedFuncDef): + result = node.type + elif isinstance(node, TypeInfo): + # Reference to a type object. + result = type_object_type(node, self.named_type) + elif isinstance(node, MypyFile): + # Reference to a module object. + result = self.named_type('builtins.module') + elif isinstance(node, Decorator): + result = self.analyze_var_ref(node.var, e) + else: + # Unknown reference; use any type implicitly to avoid + # generating extra type errors. + result = AnyType() + return result + + def analyze_var_ref(self, var: Var, context: Context) -> Type: + if not var.type: + if not var.is_ready and self.chk.typing_mode_full(): + self.chk.handle_cannot_determine_type(var.name(), context) + # Implicit 'Any' type. + return AnyType() + else: + # Look up local type of variable with type (inferred or explicit). + val = self.chk.binder.get(var) + if val is None: + return var.type + else: + return val + + def visit_call_expr(self, e: CallExpr) -> Type: + """Type check a call expression.""" + if e.analyzed: + # It's really a special form that only looks like a call. + return self.accept(e.analyzed, self.chk.type_context[-1]) + self.try_infer_partial_type(e) + callee_type = self.accept(e.callee) + if (self.chk.options.disallow_untyped_calls and + self.chk.typing_mode_full() and + isinstance(callee_type, CallableType) + and callee_type.implicit): + return self.msg.untyped_function_call(callee_type, e) + return self.check_call_expr_with_callee_type(callee_type, e) + + # Types and methods that can be used to infer partial types. + item_args = {'builtins.list': ['append'], + 'builtins.set': ['add', 'discard'], + } + container_args = {'builtins.list': {'extend': ['builtins.list']}, + 'builtins.dict': {'update': ['builtins.dict']}, + 'builtins.set': {'update': ['builtins.set', 'builtins.list']}, + } + + def try_infer_partial_type(self, e: CallExpr) -> None: + if isinstance(e.callee, MemberExpr) and isinstance(e.callee.expr, RefExpr): + var = cast(Var, e.callee.expr.node) + partial_types = self.chk.find_partial_types(var) + if partial_types is not None and not self.chk.current_node_deferred: + partial_type = cast(PartialType, var.type) + if partial_type is None or partial_type.type is None: + # A partial None type -> can't infer anything. + return + typename = partial_type.type.fullname() + methodname = e.callee.name + # Sometimes we can infer a full type for a partial List, Dict or Set type. + # TODO: Don't infer argument expression twice. + if (typename in self.item_args and methodname in self.item_args[typename] + and e.arg_kinds == [ARG_POS]): + item_type = self.accept(e.args[0]) + full_item_type = UnionType.make_simplified_union( + [item_type, partial_type.inner_types[0]]) + if mypy.checker.is_valid_inferred_type(full_item_type): + var.type = self.chk.named_generic_type(typename, [full_item_type]) + del partial_types[var] + elif (typename in self.container_args + and methodname in self.container_args[typename] + and e.arg_kinds == [ARG_POS]): + arg_type = self.accept(e.args[0]) + if isinstance(arg_type, Instance): + arg_typename = arg_type.type.fullname() + if arg_typename in self.container_args[typename][methodname]: + full_item_types = [ + UnionType.make_simplified_union([item_type, prev_type]) + for item_type, prev_type + in zip(arg_type.args, partial_type.inner_types) + ] + if all(mypy.checker.is_valid_inferred_type(item_type) + for item_type in full_item_types): + var.type = self.chk.named_generic_type(typename, + list(full_item_types)) + del partial_types[var] + + def check_call_expr_with_callee_type(self, callee_type: Type, + e: CallExpr) -> Type: + """Type check call expression. + + The given callee type overrides the type of the callee + expression. + """ + return self.check_call(callee_type, e.args, e.arg_kinds, e, + e.arg_names, callable_node=e.callee)[0] + + def check_call(self, callee: Type, args: List[Node], + arg_kinds: List[int], context: Context, + arg_names: List[str] = None, + callable_node: Node = None, + arg_messages: MessageBuilder = None) -> Tuple[Type, Type]: + """Type check a call. + + Also infer type arguments if the callee is a generic function. + + Return (result type, inferred callee type). + + Arguments: + callee: type of the called value + args: actual argument expressions + arg_kinds: contains nodes.ARG_* constant for each argument in args + describing whether the argument is positional, *arg, etc. + arg_names: names of arguments (optional) + callable_node: associate the inferred callable type to this node, + if specified + arg_messages: TODO + """ + arg_messages = arg_messages or self.msg + if isinstance(callee, CallableType): + if callee.is_concrete_type_obj() and callee.type_object().is_abstract: + type = callee.type_object() + self.msg.cannot_instantiate_abstract_class( + callee.type_object().name(), type.abstract_attributes, + context) + + formal_to_actual = map_actuals_to_formals( + arg_kinds, arg_names, + callee.arg_kinds, callee.arg_names, + lambda i: self.accept(args[i])) + + if callee.is_generic(): + callee = freshen_generic_callable(callee) + callee = self.infer_function_type_arguments_using_context( + callee, context) + callee = self.infer_function_type_arguments( + callee, args, arg_kinds, formal_to_actual, context) + + arg_types = self.infer_arg_types_in_context2( + callee, args, arg_kinds, formal_to_actual) + + self.check_argument_count(callee, arg_types, arg_kinds, + arg_names, formal_to_actual, context, self.msg) + + self.check_argument_types(arg_types, arg_kinds, callee, + formal_to_actual, context, + messages=arg_messages) + + if (callee.is_type_obj() and (len(arg_types) == 1) + and is_equivalent(callee.ret_type, self.named_type('builtins.type'))): + callee = callee.copy_modified(ret_type=TypeType(arg_types[0])) + + if callable_node: + # Store the inferred callable type. + self.chk.store_type(callable_node, callee) + return callee.ret_type, callee + elif isinstance(callee, Overloaded): + # Type check arguments in empty context. They will be checked again + # later in a context derived from the signature; these types are + # only used to pick a signature variant. + self.msg.disable_errors() + arg_types = self.infer_arg_types_in_context(None, args) + self.msg.enable_errors() + + target = self.overload_call_target(arg_types, arg_kinds, arg_names, + callee, context, + messages=arg_messages) + return self.check_call(target, args, arg_kinds, context, arg_names, + arg_messages=arg_messages) + elif isinstance(callee, AnyType) or self.chk.typing_mode_none(): + self.infer_arg_types_in_context(None, args) + return AnyType(), AnyType() + elif isinstance(callee, UnionType): + self.msg.disable_type_names += 1 + results = [self.check_call(subtype, args, arg_kinds, context, arg_names, + arg_messages=arg_messages) + for subtype in callee.items] + self.msg.disable_type_names -= 1 + return (UnionType.make_simplified_union([res[0] for res in results]), + callee) + elif isinstance(callee, Instance): + call_function = analyze_member_access('__call__', callee, context, + False, False, self.named_type, self.not_ready_callback, + self.msg) + return self.check_call(call_function, args, arg_kinds, context, arg_names, + callable_node, arg_messages) + elif isinstance(callee, TypeVarType): + return self.check_call(callee.upper_bound, args, arg_kinds, context, arg_names, + callable_node, arg_messages) + elif isinstance(callee, TypeType): + # Pass the original Type[] as context since that's where errors should go. + item = self.analyze_type_type_callee(callee.item, callee) + return self.check_call(item, args, arg_kinds, context, arg_names, + callable_node, arg_messages) + else: + return self.msg.not_callable(callee, context), AnyType() + + def analyze_type_type_callee(self, item: Type, context: Context) -> Type: + """Analyze the callee X in X(...) where X is Type[item]. + + Return a Y that we can pass to check_call(Y, ...). + """ + if isinstance(item, AnyType): + return AnyType() + if isinstance(item, Instance): + return type_object_type(item.type, self.named_type) + if isinstance(item, UnionType): + return UnionType([self.analyze_type_type_callee(item, context) + for item in item.items], item.line) + if isinstance(item, TypeVarType): + # Pretend we're calling the typevar's upper bound, + # i.e. its constructor (a poor approximation for reality, + # but better than AnyType...), but replace the return type + # with typevar. + callee = self.analyze_type_type_callee(item.upper_bound, context) + if isinstance(callee, CallableType): + if callee.is_generic(): + callee = None + else: + callee = callee.copy_modified(ret_type=item) + elif isinstance(callee, Overloaded): + if callee.items()[0].is_generic(): + callee = None + else: + callee = Overloaded([c.copy_modified(ret_type=item) + for c in callee.items()]) + if callee: + return callee + + self.msg.unsupported_type_type(item, context) + return AnyType() + + def infer_arg_types_in_context(self, callee: Optional[CallableType], + args: List[Node]) -> List[Type]: + """Infer argument expression types using a callable type as context. + + For example, if callee argument 2 has type List[int], infer the + argument expression with List[int] type context. + """ + # TODO Always called with callee as None, i.e. empty context. + res = [] # type: List[Type] + + fixed = len(args) + if callee: + fixed = min(fixed, callee.max_fixed_args()) + + arg_type = None # type: Type + ctx = None # type: Type + for i, arg in enumerate(args): + if i < fixed: + if callee and i < len(callee.arg_types): + ctx = callee.arg_types[i] + arg_type = self.accept(arg, ctx) + else: + if callee and callee.is_var_arg: + arg_type = self.accept(arg, callee.arg_types[-1]) + else: + arg_type = self.accept(arg) + if has_erased_component(arg_type): + res.append(NoneTyp()) + else: + res.append(arg_type) + return res + + def infer_arg_types_in_context2( + self, callee: CallableType, args: List[Node], arg_kinds: List[int], + formal_to_actual: List[List[int]]) -> List[Type]: + """Infer argument expression types using a callable type as context. + + For example, if callee argument 2 has type List[int], infer the + argument expression with List[int] type context. + + Returns the inferred types of *actual arguments*. + """ + res = [None] * len(args) # type: List[Type] + + for i, actuals in enumerate(formal_to_actual): + for ai in actuals: + if arg_kinds[ai] not in (nodes.ARG_STAR, nodes.ARG_STAR2): + res[ai] = self.accept(args[ai], callee.arg_types[i]) + + # Fill in the rest of the argument types. + for i, t in enumerate(res): + if not t: + res[i] = self.accept(args[i]) + return res + + def infer_function_type_arguments_using_context( + self, callable: CallableType, error_context: Context) -> CallableType: + """Unify callable return type to type context to infer type vars. + + For example, if the return type is set[t] where 't' is a type variable + of callable, and if the context is set[int], return callable modified + by substituting 't' with 'int'. + """ + ctx = self.chk.type_context[-1] + if not ctx: + return callable + # The return type may have references to type metavariables that + # we are inferring right now. We must consider them as indeterminate + # and they are not potential results; thus we replace them with the + # special ErasedType type. On the other hand, class type variables are + # valid results. + erased_ctx = replace_meta_vars(ctx, ErasedType()) + ret_type = callable.ret_type + if isinstance(ret_type, TypeVarType): + if ret_type.values or (not isinstance(ctx, Instance) or + not cast(Instance, ctx).args): + # The return type is a type variable. If it has values, we can't easily restrict + # type inference to conform to the valid values. If it's unrestricted, we could + # infer a too general type for the type variable if we use context, and this could + # result in confusing and spurious type errors elsewhere. + # + # Give up and just use function arguments for type inference. As an exception, + # if the context is a generic instance type, actually use it as context, as + # this *seems* to usually be the reasonable thing to do. + # + # See also github issues #462 and #360. + ret_type = NoneTyp() + args = infer_type_arguments(callable.type_var_ids(), ret_type, erased_ctx) + # Only substitute non-None and non-erased types. + new_args = [] # type: List[Type] + for arg in args: + if isinstance(arg, (NoneTyp, UninhabitedType)) or has_erased_component(arg): + new_args.append(None) + else: + new_args.append(arg) + return cast(CallableType, self.apply_generic_arguments(callable, new_args, + error_context)) + + def infer_function_type_arguments(self, callee_type: CallableType, + args: List[Node], + arg_kinds: List[int], + formal_to_actual: List[List[int]], + context: Context) -> CallableType: + """Infer the type arguments for a generic callee type. + + Infer based on the types of arguments. + + Return a derived callable type that has the arguments applied. + """ + if not self.chk.typing_mode_none(): + # Disable type errors during type inference. There may be errors + # due to partial available context information at this time, but + # these errors can be safely ignored as the arguments will be + # inferred again later. + self.msg.disable_errors() + + arg_types = self.infer_arg_types_in_context2( + callee_type, args, arg_kinds, formal_to_actual) + + self.msg.enable_errors() + + arg_pass_nums = self.get_arg_infer_passes( + callee_type.arg_types, formal_to_actual, len(args)) + + pass1_args = [] # type: List[Optional[Type]] + for i, arg in enumerate(arg_types): + if arg_pass_nums[i] > 1: + pass1_args.append(None) + else: + pass1_args.append(arg) + + inferred_args = infer_function_type_arguments( + callee_type, pass1_args, arg_kinds, formal_to_actual, + strict=self.chk.typing_mode_full()) # type: List[Type] + + if 2 in arg_pass_nums: + # Second pass of type inference. + (callee_type, + inferred_args) = self.infer_function_type_arguments_pass2( + callee_type, args, arg_kinds, formal_to_actual, + inferred_args, context) + + if callee_type.special_sig == 'dict' and len(inferred_args) == 2 and ( + ARG_NAMED in arg_kinds or ARG_STAR2 in arg_kinds): + # HACK: Infer str key type for dict(...) with keyword args. The type system + # can't represent this so we special case it, as this is a pretty common + # thing. This doesn't quite work with all possible subclasses of dict + # if they shuffle type variables around, as we assume that there is a 1-1 + # correspondence with dict type variables. This is a marginal issue and + # a little tricky to fix so it's left unfixed for now. + if isinstance(inferred_args[0], (NoneTyp, UninhabitedType)): + inferred_args[0] = self.named_type('builtins.str') + elif not is_subtype(self.named_type('builtins.str'), inferred_args[0]): + self.msg.fail(messages.KEYWORD_ARGUMENT_REQUIRES_STR_KEY_TYPE, + context) + else: + # In dynamically typed functions use implicit 'Any' types for + # type variables. + inferred_args = [AnyType()] * len(callee_type.variables) + return self.apply_inferred_arguments(callee_type, inferred_args, + context) + + def infer_function_type_arguments_pass2( + self, callee_type: CallableType, + args: List[Node], + arg_kinds: List[int], + formal_to_actual: List[List[int]], + inferred_args: List[Type], + context: Context) -> Tuple[CallableType, List[Type]]: + """Perform second pass of generic function type argument inference. + + The second pass is needed for arguments with types such as Callable[[T], S], + where both T and S are type variables, when the actual argument is a + lambda with inferred types. The idea is to infer the type variable T + in the first pass (based on the types of other arguments). This lets + us infer the argument and return type of the lambda expression and + thus also the type variable S in this second pass. + + Return (the callee with type vars applied, inferred actual arg types). + """ + # None or erased types in inferred types mean that there was not enough + # information to infer the argument. Replace them with None values so + # that they are not applied yet below. + for i, arg in enumerate(inferred_args): + if isinstance(arg, (NoneTyp, UninhabitedType)) or has_erased_component(arg): + inferred_args[i] = None + + callee_type = cast(CallableType, self.apply_generic_arguments( + callee_type, inferred_args, context)) + arg_types = self.infer_arg_types_in_context2( + callee_type, args, arg_kinds, formal_to_actual) + + inferred_args = infer_function_type_arguments( + callee_type, arg_types, arg_kinds, formal_to_actual) + + return callee_type, inferred_args + + def get_arg_infer_passes(self, arg_types: List[Type], + formal_to_actual: List[List[int]], + num_actuals: int) -> List[int]: + """Return pass numbers for args for two-pass argument type inference. + + For each actual, the pass number is either 1 (first pass) or 2 (second + pass). + + Two-pass argument type inference primarily lets us infer types of + lambdas more effectively. + """ + res = [1] * num_actuals + for i, arg in enumerate(arg_types): + if arg.accept(ArgInferSecondPassQuery()): + for j in formal_to_actual[i]: + res[j] = 2 + return res + + def apply_inferred_arguments(self, callee_type: CallableType, + inferred_args: List[Type], + context: Context) -> CallableType: + """Apply inferred values of type arguments to a generic function. + + Inferred_args contains the values of function type arguments. + """ + # Report error if some of the variables could not be solved. In that + # case assume that all variables have type Any to avoid extra + # bogus error messages. + for i, inferred_type in enumerate(inferred_args): + if not inferred_type or has_erased_component(inferred_type): + # Could not infer a non-trivial type for a type variable. + self.msg.could_not_infer_type_arguments( + callee_type, i + 1, context) + inferred_args = [AnyType()] * len(inferred_args) + # Apply the inferred types to the function type. In this case the + # return type must be CallableType, since we give the right number of type + # arguments. + return cast(CallableType, self.apply_generic_arguments(callee_type, + inferred_args, context)) + + def check_argument_count(self, callee: CallableType, actual_types: List[Type], + actual_kinds: List[int], actual_names: List[str], + formal_to_actual: List[List[int]], + context: Context, + messages: Optional[MessageBuilder]) -> bool: + """Check that there is a value for all required arguments to a function. + + Also check that there are no duplicate values for arguments. Report found errors + using 'messages' if it's not None. + + Return False if there were any errors. Otherwise return True + """ + # TODO(jukka): We could return as soon as we find an error if messages is None. + formal_kinds = callee.arg_kinds + + # Collect list of all actual arguments matched to formal arguments. + all_actuals = [] # type: List[int] + for actuals in formal_to_actual: + all_actuals.extend(actuals) + + is_unexpected_arg_error = False # Keep track of errors to avoid duplicate errors. + ok = True # False if we've found any error. + for i, kind in enumerate(actual_kinds): + if i not in all_actuals and ( + kind != nodes.ARG_STAR or + not is_empty_tuple(actual_types[i])): + # Extra actual: not matched by a formal argument. + ok = False + if kind != nodes.ARG_NAMED: + if messages: + messages.too_many_arguments(callee, context) + else: + if messages: + messages.unexpected_keyword_argument( + callee, actual_names[i], context) + is_unexpected_arg_error = True + elif kind == nodes.ARG_STAR and ( + nodes.ARG_STAR not in formal_kinds): + actual_type = actual_types[i] + if isinstance(actual_type, TupleType): + if all_actuals.count(i) < len(actual_type.items): + # Too many tuple items as some did not match. + if messages: + messages.too_many_arguments(callee, context) + ok = False + # *args can be applied even if the function takes a fixed + # number of positional arguments. This may succeed at runtime. + + for i, kind in enumerate(formal_kinds): + if kind == nodes.ARG_POS and (not formal_to_actual[i] and + not is_unexpected_arg_error): + # No actual for a mandatory positional formal. + if messages: + messages.too_few_arguments(callee, context, actual_names) + ok = False + elif kind in [nodes.ARG_POS, nodes.ARG_OPT, + nodes.ARG_NAMED] and is_duplicate_mapping( + formal_to_actual[i], actual_kinds): + if (self.chk.typing_mode_full() or + isinstance(actual_types[formal_to_actual[i][0]], TupleType)): + if messages: + messages.duplicate_argument_value(callee, i, context) + ok = False + elif (kind == nodes.ARG_NAMED and formal_to_actual[i] and + actual_kinds[formal_to_actual[i][0]] != nodes.ARG_NAMED): + # Positional argument when expecting a keyword argument. + if messages: + messages.too_many_positional_arguments(callee, context) + ok = False + return ok + + def check_argument_types(self, arg_types: List[Type], arg_kinds: List[int], + callee: CallableType, + formal_to_actual: List[List[int]], + context: Context, + messages: MessageBuilder = None, + check_arg: ArgChecker = None) -> None: + """Check argument types against a callable type. + + Report errors if the argument types are not compatible. + """ + messages = messages or self.msg + check_arg = check_arg or self.check_arg + # Keep track of consumed tuple *arg items. + tuple_counter = [0] + for i, actuals in enumerate(formal_to_actual): + for actual in actuals: + arg_type = arg_types[actual] + # Check that a *arg is valid as varargs. + if (arg_kinds[actual] == nodes.ARG_STAR and + not self.is_valid_var_arg(arg_type)): + messages.invalid_var_arg(arg_type, context) + if (arg_kinds[actual] == nodes.ARG_STAR2 and + not self.is_valid_keyword_var_arg(arg_type)): + messages.invalid_keyword_var_arg(arg_type, context) + # Get the type of an individual actual argument (for *args + # and **args this is the item type, not the collection type). + actual_type = get_actual_type(arg_type, arg_kinds[actual], + tuple_counter) + check_arg(actual_type, arg_type, arg_kinds[actual], + callee.arg_types[i], + actual + 1, i + 1, callee, context, messages) + + # There may be some remaining tuple varargs items that haven't + # been checked yet. Handle them. + if (callee.arg_kinds[i] == nodes.ARG_STAR and + arg_kinds[actual] == nodes.ARG_STAR and + isinstance(arg_types[actual], TupleType)): + tuplet = cast(TupleType, arg_types[actual]) + while tuple_counter[0] < len(tuplet.items): + actual_type = get_actual_type(arg_type, + arg_kinds[actual], + tuple_counter) + check_arg(actual_type, arg_type, arg_kinds[actual], + callee.arg_types[i], + actual + 1, i + 1, callee, context, messages) + + def check_arg(self, caller_type: Type, original_caller_type: Type, + caller_kind: int, + callee_type: Type, n: int, m: int, callee: CallableType, + context: Context, messages: MessageBuilder) -> None: + """Check the type of a single argument in a call.""" + if isinstance(caller_type, Void): + messages.does_not_return_value(caller_type, context) + elif isinstance(caller_type, DeletedType): + messages.deleted_as_rvalue(caller_type, context) + elif not is_subtype(caller_type, callee_type): + messages.incompatible_argument(n, m, callee, original_caller_type, + caller_kind, context) + + def overload_call_target(self, arg_types: List[Type], arg_kinds: List[int], + arg_names: List[str], + overload: Overloaded, context: Context, + messages: MessageBuilder = None) -> Type: + """Infer the correct overload item to call with given argument types. + + The return value may be CallableType or AnyType (if an unique item + could not be determined). + """ + messages = messages or self.msg + # TODO: For overlapping signatures we should try to get a more precise + # result than 'Any'. + match = [] # type: List[CallableType] + best_match = 0 + for typ in overload.items(): + similarity = self.erased_signature_similarity(arg_types, arg_kinds, arg_names, + typ, context=context) + if similarity > 0 and similarity >= best_match: + if (match and not is_same_type(match[-1].ret_type, + typ.ret_type) and + not mypy.checker.is_more_precise_signature( + match[-1], typ)): + # Ambiguous return type. Either the function overload is + # overlapping (which we don't handle very well here) or the + # caller has provided some Any argument types; in either + # case we'll fall back to Any. It's okay to use Any types + # in calls. + # + # Overlapping overload items are generally fine if the + # overlapping is only possible when there is multiple + # inheritance, as this is rare. See docstring of + # mypy.meet.is_overlapping_types for more about this. + # + # Note that there is no ambiguity if the items are + # covariant in both argument types and return types with + # respect to type precision. We'll pick the best/closest + # match. + # + # TODO: Consider returning a union type instead if the + # overlapping is NOT due to Any types? + return AnyType() + else: + match.append(typ) + best_match = max(best_match, similarity) + if not match: + messages.no_variant_matches_arguments(overload, arg_types, context) + return AnyType() + else: + if len(match) == 1: + return match[0] + else: + # More than one signature matches. Pick the first *non-erased* + # matching signature, or default to the first one if none + # match. + for m in match: + if self.match_signature_types(arg_types, arg_kinds, arg_names, m, + context=context): + return m + return match[0] + + def erased_signature_similarity(self, arg_types: List[Type], arg_kinds: List[int], + arg_names: List[str], callee: CallableType, + context: Context) -> int: + """Determine whether arguments could match the signature at runtime. + + Return similarity level (0 = no match, 1 = can match, 2 = non-promotion match). See + overload_arg_similarity for a discussion of similarity levels. + """ + formal_to_actual = map_actuals_to_formals(arg_kinds, + arg_names, + callee.arg_kinds, + callee.arg_names, + lambda i: arg_types[i]) + + if not self.check_argument_count(callee, arg_types, arg_kinds, arg_names, + formal_to_actual, None, None): + # Too few or many arguments -> no match. + return 0 + + similarity = 2 + + def check_arg(caller_type: Type, original_caller_type: Type, caller_kind: int, + callee_type: Type, n: int, m: int, callee: CallableType, + context: Context, messages: MessageBuilder) -> None: + nonlocal similarity + similarity = min(similarity, + overload_arg_similarity(caller_type, callee_type)) + if similarity == 0: + # No match -- exit early since none of the remaining work can change + # the result. + raise Finished + + try: + self.check_argument_types(arg_types, arg_kinds, callee, formal_to_actual, + context=context, check_arg=check_arg) + except Finished: + pass + + return similarity + + def match_signature_types(self, arg_types: List[Type], arg_kinds: List[int], + arg_names: List[str], callee: CallableType, + context: Context) -> bool: + """Determine whether arguments types match the signature. + + Assume that argument counts are compatible. + + Return True if arguments match. + """ + formal_to_actual = map_actuals_to_formals(arg_kinds, + arg_names, + callee.arg_kinds, + callee.arg_names, + lambda i: arg_types[i]) + ok = True + + def check_arg(caller_type: Type, original_caller_type: Type, caller_kind: int, + callee_type: Type, n: int, m: int, callee: CallableType, + context: Context, messages: MessageBuilder) -> None: + nonlocal ok + if not is_subtype(caller_type, callee_type): + ok = False + + self.check_argument_types(arg_types, arg_kinds, callee, formal_to_actual, + context=context, check_arg=check_arg) + return ok + + def apply_generic_arguments(self, callable: CallableType, types: List[Type], + context: Context) -> Type: + """Simple wrapper around mypy.applytype.apply_generic_arguments.""" + return applytype.apply_generic_arguments(callable, types, self.msg, context) + + def apply_generic_arguments2(self, overload: Overloaded, types: List[Type], + context: Context) -> Type: + items = [] # type: List[CallableType] + for item in overload.items(): + applied = self.apply_generic_arguments(item, types, context) + if isinstance(applied, CallableType): + items.append(applied) + else: + # There was an error. + return AnyType() + return Overloaded(items) + + def visit_member_expr(self, e: MemberExpr) -> Type: + """Visit member expression (of form e.id).""" + result = self.analyze_ordinary_member_access(e, False) + return self.chk.narrow_type_from_binder(e, result) + + def analyze_ordinary_member_access(self, e: MemberExpr, + is_lvalue: bool) -> Type: + """Analyse member expression or member lvalue.""" + if e.kind is not None: + # This is a reference to a module attribute. + return self.analyze_ref_expr(e) + else: + # This is a reference to a non-module attribute. + return analyze_member_access(e.name, self.accept(e.expr), e, + is_lvalue, False, + self.named_type, self.not_ready_callback, self.msg) + + def analyze_external_member_access(self, member: str, base_type: Type, + context: Context) -> Type: + """Analyse member access that is external, i.e. it cannot + refer to private definitions. Return the result type. + """ + # TODO remove; no private definitions in mypy + return analyze_member_access(member, base_type, context, False, False, + self.named_type, self.not_ready_callback, self.msg) + + def visit_int_expr(self, e: IntExpr) -> Type: + """Type check an integer literal (trivial).""" + return self.named_type('builtins.int') + + def visit_str_expr(self, e: StrExpr) -> Type: + """Type check a string literal (trivial).""" + return self.named_type('builtins.str') + + def visit_bytes_expr(self, e: BytesExpr) -> Type: + """Type check a bytes literal (trivial).""" + return self.named_type('builtins.bytes') + + def visit_unicode_expr(self, e: UnicodeExpr) -> Type: + """Type check a unicode literal (trivial).""" + return self.named_type('builtins.unicode') + + def visit_float_expr(self, e: FloatExpr) -> Type: + """Type check a float literal (trivial).""" + return self.named_type('builtins.float') + + def visit_complex_expr(self, e: ComplexExpr) -> Type: + """Type check a complex literal.""" + return self.named_type('builtins.complex') + + def visit_ellipsis(self, e: EllipsisExpr) -> Type: + """Type check '...'.""" + if self.chk.options.python_version[0] >= 3: + return self.named_type('builtins.ellipsis') + else: + # '...' is not valid in normal Python 2 code, but it can + # be used in stubs. The parser makes sure that we only + # get this far if we are in a stub, and we can safely + # return 'object' as ellipsis is special cased elsewhere. + # The builtins.ellipsis type does not exist in Python 2. + return self.named_type('builtins.object') + + def visit_op_expr(self, e: OpExpr) -> Type: + """Type check a binary operator expression.""" + if e.op == 'and' or e.op == 'or': + return self.check_boolean_op(e, e) + if e.op == '*' and isinstance(e.left, ListExpr): + # Expressions of form [...] * e get special type inference. + return self.check_list_multiply(e) + if e.op == '%' and isinstance(e.left, (StrExpr, BytesExpr)): + return self.strfrm_checker.check_str_interpolation(cast(StrExpr, e.left), e.right) + left_type = self.accept(e.left) + + if e.op in nodes.op_methods: + method = self.get_operator_method(e.op) + result, method_type = self.check_op(method, left_type, e.right, e, + allow_reverse=True) + e.method_type = method_type + return result + else: + raise RuntimeError('Unknown operator {}'.format(e.op)) + + def visit_comparison_expr(self, e: ComparisonExpr) -> Type: + """Type check a comparison expression. + + Comparison expressions are type checked consecutive-pair-wise + That is, 'a < b > c == d' is check as 'a < b and b > c and c == d' + """ + result = None # type: mypy.types.Type + + # Check each consecutive operand pair and their operator + for left, right, operator in zip(e.operands, e.operands[1:], e.operators): + left_type = self.accept(left) + + method_type = None # type: mypy.types.Type + + if operator == 'in' or operator == 'not in': + right_type = self.accept(right) # TODO only evaluate if needed + + # Keep track of whether we get type check errors (these won't be reported, they + # are just to verify whether something is valid typing wise). + local_errors = self.msg.copy() + local_errors.disable_count = 0 + sub_result, method_type = self.check_op_local('__contains__', right_type, + left, e, local_errors) + if isinstance(right_type, PartialType): + # We don't really know if this is an error or not, so just shut up. + pass + elif (local_errors.is_errors() and + # is_valid_var_arg is True for any Iterable + self.is_valid_var_arg(right_type)): + itertype = self.chk.analyze_iterable_item_type(right) + method_type = CallableType( + [left_type], + [nodes.ARG_POS], + [None], + self.chk.bool_type(), + self.named_type('builtins.function')) + sub_result = self.chk.bool_type() + if not is_subtype(left_type, itertype): + self.msg.unsupported_operand_types('in', left_type, right_type, e) + else: + self.msg.add_errors(local_errors) + if operator == 'not in': + sub_result = self.chk.bool_type() + elif operator in nodes.op_methods: + method = self.get_operator_method(operator) + sub_result, method_type = self.check_op(method, left_type, right, e, + allow_reverse=True) + + elif operator == 'is' or operator == 'is not': + sub_result = self.chk.bool_type() + method_type = None + else: + raise RuntimeError('Unknown comparison operator {}'.format(operator)) + + e.method_types.append(method_type) + + # Determine type of boolean-and of result and sub_result + if result is None: + result = sub_result + else: + # TODO: check on void needed? + self.check_not_void(sub_result, e) + result = join.join_types(result, sub_result) + + return result + + def get_operator_method(self, op: str) -> str: + if op == '/' and self.chk.options.python_version[0] == 2: + # TODO also check for "from __future__ import division" + return '__div__' + else: + return nodes.op_methods[op] + + def check_op_local(self, method: str, base_type: Type, arg: Node, + context: Context, local_errors: MessageBuilder) -> Tuple[Type, Type]: + """Type check a binary operation which maps to a method call. + + Return tuple (result type, inferred operator method type). + """ + method_type = analyze_member_access(method, base_type, context, False, False, + self.named_type, self.not_ready_callback, local_errors) + return self.check_call(method_type, [arg], [nodes.ARG_POS], + context, arg_messages=local_errors) + + def check_op(self, method: str, base_type: Type, arg: Node, + context: Context, + allow_reverse: bool = False) -> Tuple[Type, Type]: + """Type check a binary operation which maps to a method call. + + Return tuple (result type, inferred operator method type). + """ + # Use a local error storage for errors related to invalid argument + # type (but NOT other errors). This error may need to be suppressed + # for operators which support __rX methods. + local_errors = self.msg.copy() + local_errors.disable_count = 0 + if not allow_reverse or self.has_member(base_type, method): + result = self.check_op_local(method, base_type, arg, context, + local_errors) + if allow_reverse: + arg_type = self.chk.type_map[arg] + if isinstance(arg_type, AnyType): + # If the right operand has type Any, we can't make any + # conjectures about the type of the result, since the + # operand could have a __r method that returns anything. + + # However, in weak mode, we do make conjectures. + if not self.chk.typing_mode_weak(): + result = AnyType(), result[1] + success = not local_errors.is_errors() + else: + result = AnyType(), AnyType() + success = False + if success or not allow_reverse or isinstance(base_type, AnyType): + # We were able to call the normal variant of the operator method, + # or there was some problem not related to argument type + # validity, or the operator has no __rX method. In any case, we + # don't need to consider the __rX method. + self.msg.add_errors(local_errors) + return result + else: + # Calling the operator method was unsuccessful. Try the __rX + # method of the other operand instead. + rmethod = self.get_reverse_op_method(method) + arg_type = self.accept(arg) + if self.has_member(arg_type, rmethod): + method_type = self.analyze_external_member_access( + rmethod, arg_type, context) + temp = TempNode(base_type) + return self.check_call(method_type, [temp], [nodes.ARG_POS], + context) + else: + # No __rX method either. Do deferred type checking to produce + # error message that we may have missed previously. + # TODO Fix type checking an expression more than once. + return self.check_op_local(method, base_type, arg, context, + self.msg) + + def get_reverse_op_method(self, method: str) -> str: + if method == '__div__' and self.chk.options.python_version[0] == 2: + return '__rdiv__' + else: + return nodes.reverse_op_methods[method] + + def check_boolean_op(self, e: OpExpr, context: Context) -> Type: + """Type check a boolean operation ('and' or 'or').""" + + # A boolean operation can evaluate to either of the operands. + + # We use the current type context to guide the type inference of of + # the left operand. We also use the left operand type to guide the type + # inference of the right operand so that expressions such as + # '[1] or []' are inferred correctly. + ctx = self.chk.type_context[-1] + left_type = self.accept(e.left, ctx) + + if e.op == 'and': + right_map, _ = \ + mypy.checker.find_isinstance_check(e.left, self.chk.type_map, + self.chk.typing_mode_weak()) + elif e.op == 'or': + _, right_map = \ + mypy.checker.find_isinstance_check(e.left, self.chk.type_map, + self.chk.typing_mode_weak()) + else: + right_map = None + + with self.chk.binder.frame_context(): + if right_map: + for var, type in right_map.items(): + self.chk.binder.push(var, type) + + right_type = self.accept(e.right, left_type) + + self.check_not_void(left_type, context) + self.check_not_void(right_type, context) + return UnionType.make_simplified_union([left_type, right_type]) + + def check_list_multiply(self, e: OpExpr) -> Type: + """Type check an expression of form '[...] * e'. + + Type inference is special-cased for this common construct. + """ + right_type = self.accept(e.right) + if is_subtype(right_type, self.named_type('builtins.int')): + # Special case: [...] * . Use the type context of the + # OpExpr, since the multiplication does not affect the type. + left_type = self.accept(e.left, context=self.chk.type_context[-1]) + else: + left_type = self.accept(e.left) + result, method_type = self.check_op('__mul__', left_type, e.right, e) + e.method_type = method_type + return result + + def visit_unary_expr(self, e: UnaryExpr) -> Type: + """Type check an unary operation ('not', '-', '+' or '~').""" + operand_type = self.accept(e.expr) + op = e.op + if op == 'not': + self.check_not_void(operand_type, e) + result = self.chk.bool_type() # type: Type + elif op == '-': + method_type = self.analyze_external_member_access('__neg__', + operand_type, e) + result, method_type = self.check_call(method_type, [], [], e) + e.method_type = method_type + elif op == '+': + method_type = self.analyze_external_member_access('__pos__', + operand_type, e) + result, method_type = self.check_call(method_type, [], [], e) + e.method_type = method_type + else: + assert op == '~', "unhandled unary operator" + method_type = self.analyze_external_member_access('__invert__', + operand_type, e) + result, method_type = self.check_call(method_type, [], [], e) + e.method_type = method_type + return result + + def visit_index_expr(self, e: IndexExpr) -> Type: + """Type check an index expression (base[index]). + + It may also represent type application. + """ + result = self.visit_index_expr_helper(e) + return self.chk.narrow_type_from_binder(e, result) + + def visit_index_expr_helper(self, e: IndexExpr) -> Type: + if e.analyzed: + # It's actually a type application. + return self.accept(e.analyzed) + left_type = self.accept(e.base) + if isinstance(left_type, TupleType) and self.chk.typing_mode_full(): + # Special case for tuples. They support indexing only by integer + # literals. (Except in weak type checking mode.) + index = e.index + if isinstance(index, SliceExpr): + return self.visit_tuple_slice_helper(left_type, index) + + ok = False + if isinstance(index, IntExpr): + n = index.value + ok = True + elif isinstance(index, UnaryExpr): + if index.op == '-': + operand = index.expr + if isinstance(operand, IntExpr): + n = len(left_type.items) - operand.value + ok = True + if ok: + if n >= 0 and n < len(left_type.items): + return left_type.items[n] + else: + self.chk.fail(messages.TUPLE_INDEX_OUT_OF_RANGE, e) + return AnyType() + else: + self.chk.fail(messages.TUPLE_INDEX_MUST_BE_AN_INT_LITERAL, e) + return AnyType() + else: + result, method_type = self.check_op('__getitem__', left_type, e.index, e) + e.method_type = method_type + return result + + def visit_tuple_slice_helper(self, left_type: TupleType, slic: SliceExpr): + begin = None # type: int + end = None # type: int + stride = None # type:int + + if slic.begin_index: + begin = self._get_value(slic.begin_index) + if begin is None: + self.chk.fail( + messages.TUPLE_SLICE_MUST_BE_AN_INT_LITERAL, + slic.begin_index) + return AnyType() + + if slic.end_index: + end = self._get_value(slic.end_index) + if end is None: + self.chk.fail( + messages.TUPLE_SLICE_MUST_BE_AN_INT_LITERAL, + slic.end_index) + return AnyType() + + if slic.stride: + stride = self._get_value(slic.stride) + if stride is None: + self.chk.fail( + messages.TUPLE_SLICE_MUST_BE_AN_INT_LITERAL, + slic.stride) + return AnyType() + + return left_type.slice(begin, stride, end) + + def _get_value(self, index: Node) -> Optional[int]: + if isinstance(index, IntExpr): + return index.value + elif isinstance(index, UnaryExpr): + if index.op == '-': + operand = index.expr + if isinstance(operand, IntExpr): + return -1 * operand.value + return None + + def visit_cast_expr(self, expr: CastExpr) -> Type: + """Type check a cast expression.""" + source_type = self.accept(expr.expr, context=AnyType()) + target_type = expr.type + if self.chk.options.warn_redundant_casts and is_same_type(source_type, target_type): + self.msg.redundant_cast(target_type, expr) + if not self.is_valid_cast(source_type, target_type): + self.msg.invalid_cast(target_type, source_type, expr) + return target_type + + def is_valid_cast(self, source_type: Type, target_type: Type) -> bool: + """Is a cast from source_type to target_type meaningful?""" + return (isinstance(target_type, AnyType) or + (not isinstance(source_type, Void) and + not isinstance(target_type, Void))) + + def visit_reveal_type_expr(self, expr: RevealTypeExpr) -> Type: + """Type check a reveal_type expression.""" + revealed_type = self.accept(expr.expr) + self.msg.reveal_type(revealed_type, expr) + return revealed_type + + def visit_type_application(self, tapp: TypeApplication) -> Type: + """Type check a type application (expr[type, ...]).""" + self.chk.fail(messages.GENERIC_TYPE_NOT_VALID_AS_EXPRESSION, tapp) + return AnyType() + + def visit_type_alias_expr(self, alias: TypeAliasExpr) -> Type: + return AnyType() + + def visit_list_expr(self, e: ListExpr) -> Type: + """Type check a list expression [...].""" + return self.check_list_or_set_expr(e.items, 'builtins.list', '', + e) + + def visit_set_expr(self, e: SetExpr) -> Type: + return self.check_list_or_set_expr(e.items, 'builtins.set', '', e) + + def check_list_or_set_expr(self, items: List[Node], fullname: str, + tag: str, context: Context) -> Type: + # Translate into type checking a generic function call. + tvdef = TypeVarDef('T', -1, [], self.chk.object_type()) + tv = TypeVarType(tvdef) + constructor = CallableType( + [tv], + [nodes.ARG_STAR], + [None], + self.chk.named_generic_type(fullname, [tv]), + self.named_type('builtins.function'), + name=tag, + variables=[tvdef]) + return self.check_call(constructor, + items, + [nodes.ARG_POS] * len(items), context)[0] + + def visit_tuple_expr(self, e: TupleExpr) -> Type: + """Type check a tuple expression.""" + ctx = None # type: TupleType + # Try to determine type context for type inference. + if isinstance(self.chk.type_context[-1], TupleType): + t = self.chk.type_context[-1] + if len(t.items) == len(e.items): + ctx = t + # Infer item types. + items = [] # type: List[Type] + for i in range(len(e.items)): + item = e.items[i] + tt = None # type: Type + if not ctx: + tt = self.accept(item) + else: + tt = self.accept(item, ctx.items[i]) + self.check_not_void(tt, e) + items.append(tt) + fallback_item = join.join_type_list(items) + return TupleType(items, self.chk.named_generic_type('builtins.tuple', [fallback_item])) + + def visit_dict_expr(self, e: DictExpr) -> Type: + # Translate into type checking a generic function call. + ktdef = TypeVarDef('KT', -1, [], self.chk.object_type()) + vtdef = TypeVarDef('VT', -2, [], self.chk.object_type()) + kt = TypeVarType(ktdef) + vt = TypeVarType(vtdef) + # The callable type represents a function like this: + # + # def (*v: Tuple[kt, vt]) -> Dict[kt, vt]: ... + constructor = CallableType( + [TupleType([kt, vt], self.named_type('builtins.tuple'))], + [nodes.ARG_STAR], + [None], + self.chk.named_generic_type('builtins.dict', [kt, vt]), + self.named_type('builtins.function'), + name='', + variables=[ktdef, vtdef]) + # Synthesize function arguments. + args = [] # type: List[Node] + for key, value in e.items: + args.append(TupleExpr([key, value])) + return self.check_call(constructor, + args, + [nodes.ARG_POS] * len(args), e)[0] + + def visit_func_expr(self, e: FuncExpr) -> Type: + """Type check lambda expression.""" + inferred_type = self.infer_lambda_type_using_context(e) + if not inferred_type: + # No useful type context. + ret_type = e.expr().accept(self.chk) + if not e.arguments: + # Form 'lambda: e'; just use the inferred return type. + return CallableType([], [], [], ret_type, self.named_type('builtins.function')) + else: + # TODO: Consider reporting an error. However, this is fine if + # we are just doing the first pass in contextual type + # inference. + return AnyType() + else: + # Type context available. + self.chk.check_func_item(e, type_override=inferred_type) + if e.expr() not in self.chk.type_map: + self.accept(e.expr()) + ret_type = self.chk.type_map[e.expr()] + if isinstance(ret_type, NoneTyp): + # For "lambda ...: None", just use type from the context. + # Important when the context is Callable[..., None] which + # really means Void. See #1425. + return inferred_type + return replace_callable_return_type(inferred_type, ret_type) + + def infer_lambda_type_using_context(self, e: FuncExpr) -> CallableType: + """Try to infer lambda expression type using context. + + Return None if could not infer type. + """ + # TODO also accept 'Any' context + ctx = self.chk.type_context[-1] + if not ctx or not isinstance(ctx, CallableType): + return None + + # The context may have function type variables in it. We replace them + # since these are the type variables we are ultimately trying to infer; + # they must be considered as indeterminate. We use ErasedType since it + # does not affect type inference results (it is for purposes like this + # only). + ctx = replace_meta_vars(ctx, ErasedType()) + + callable_ctx = cast(CallableType, ctx) + + arg_kinds = [arg.kind for arg in e.arguments] + + if callable_ctx.is_ellipsis_args: + # Fill in Any arguments to match the arguments of the lambda. + callable_ctx = callable_ctx.copy_modified( + is_ellipsis_args=False, + arg_types=[AnyType()] * len(arg_kinds), + arg_kinds=arg_kinds + ) + + if callable_ctx.arg_kinds != arg_kinds: + # Incompatible context; cannot use it to infer types. + self.chk.fail(messages.CANNOT_INFER_LAMBDA_TYPE, e) + return None + + return callable_ctx + + def visit_super_expr(self, e: SuperExpr) -> Type: + """Type check a super expression (non-lvalue).""" + t = self.analyze_super(e, False) + return t + + def analyze_super(self, e: SuperExpr, is_lvalue: bool) -> Type: + """Type check a super expression.""" + if e.info and e.info.bases: + # TODO fix multiple inheritance etc + if len(e.info.mro) < 2: + self.chk.fail('Internal error: unexpected mro for {}: {}'.format( + e.info.name(), e.info.mro), e) + return AnyType() + for base in e.info.mro[1:]: + if e.name in base.names or base == e.info.mro[-1]: + if e.info.fallback_to_any and base == e.info.mro[-1]: + # There's an undefined base class, and we're + # at the end of the chain. That's not an error. + return AnyType() + if not self.chk.typing_mode_full(): + return AnyType() + return analyze_member_access(e.name, self_type(e.info), e, + is_lvalue, True, + self.named_type, self.not_ready_callback, + self.msg, base) + else: + # Invalid super. This has been reported by the semantic analyzer. + return AnyType() + + def visit_slice_expr(self, e: SliceExpr) -> Type: + for index in [e.begin_index, e.end_index, e.stride]: + if index: + t = self.accept(index) + self.chk.check_subtype(t, self.named_type('builtins.int'), + index, messages.INVALID_SLICE_INDEX) + return self.named_type('builtins.slice') + + def visit_list_comprehension(self, e: ListComprehension) -> Type: + return self.check_generator_or_comprehension( + e.generator, 'builtins.list', '') + + def visit_set_comprehension(self, e: SetComprehension) -> Type: + return self.check_generator_or_comprehension( + e.generator, 'builtins.set', '') + + def visit_generator_expr(self, e: GeneratorExpr) -> Type: + return self.check_generator_or_comprehension(e, 'typing.Iterator', + '') + + def check_generator_or_comprehension(self, gen: GeneratorExpr, + type_name: str, + id_for_messages: str) -> Type: + """Type check a generator expression or a list comprehension.""" + self.check_for_comp(gen) + + # Infer the type of the list comprehension by using a synthetic generic + # callable type. + tvdef = TypeVarDef('T', -1, [], self.chk.object_type()) + tv = TypeVarType(tvdef) + constructor = CallableType( + [tv], + [nodes.ARG_POS], + [None], + self.chk.named_generic_type(type_name, [tv]), + self.chk.named_type('builtins.function'), + name=id_for_messages, + variables=[tvdef]) + return self.check_call(constructor, + [gen.left_expr], [nodes.ARG_POS], gen)[0] + + def visit_dictionary_comprehension(self, e: DictionaryComprehension): + """Type check a dictionary comprehension.""" + self.check_for_comp(e) + + # Infer the type of the list comprehension by using a synthetic generic + # callable type. + ktdef = TypeVarDef('KT', -1, [], self.chk.object_type()) + vtdef = TypeVarDef('VT', -2, [], self.chk.object_type()) + kt = TypeVarType(ktdef) + vt = TypeVarType(vtdef) + constructor = CallableType( + [kt, vt], + [nodes.ARG_POS, nodes.ARG_POS], + [None, None], + self.chk.named_generic_type('builtins.dict', [kt, vt]), + self.chk.named_type('builtins.function'), + name='', + variables=[ktdef, vtdef]) + return self.check_call(constructor, + [e.key, e.value], [nodes.ARG_POS, nodes.ARG_POS], e)[0] + + def check_for_comp(self, e: Union[GeneratorExpr, DictionaryComprehension]) -> None: + """Check the for_comp part of comprehensions. That is the part from 'for': + ... for x in y if z + """ + with self.chk.binder.frame_context(): + for index, sequence, conditions in zip(e.indices, e.sequences, + e.condlists): + sequence_type = self.chk.analyze_iterable_item_type(sequence) + self.chk.analyze_index_variables(index, sequence_type, e) + for condition in conditions: + self.accept(condition) + + def visit_conditional_expr(self, e: ConditionalExpr) -> Type: + cond_type = self.accept(e.cond) + self.check_not_void(cond_type, e) + ctx = self.chk.type_context[-1] + + # Gain type information from isinstance if it is there + # but only for the current expression + if_map, else_map = mypy.checker.find_isinstance_check( + e.cond, + self.chk.type_map, + self.chk.typing_mode_weak()) + + if_type = self.analyze_cond_branch(if_map, e.if_expr, context=ctx) + + if not mypy.checker.is_valid_inferred_type(if_type): + # Analyze the right branch disregarding the left branch. + else_type = self.analyze_cond_branch(else_map, e.else_expr, context=ctx) + + # If it would make a difference, re-analyze the left + # branch using the right branch's type as context. + if ctx is None or not is_equivalent(else_type, ctx): + # TODO: If it's possible that the previous analysis of + # the left branch produced errors that are avoided + # using this context, suppress those errors. + if_type = self.analyze_cond_branch(if_map, e.if_expr, context=else_type) + + else: + # Analyze the right branch in the context of the left + # branch's type. + else_type = self.analyze_cond_branch(else_map, e.else_expr, context=if_type) + + res = join.join_types(if_type, else_type) + + return res + + def analyze_cond_branch(self, map: Optional[Dict[Node, Type]], + node: Node, context: Optional[Type]) -> Type: + with self.chk.binder.frame_context(): + if map: + for var, type in map.items(): + self.chk.binder.push(var, type) + return self.accept(node, context=context) + + def visit_backquote_expr(self, e: BackquoteExpr) -> Type: + self.accept(e.expr) + return self.named_type('builtins.str') + + # + # Helpers + # + + def accept(self, node: Node, context: Type = None) -> Type: + """Type check a node. Alias for TypeChecker.accept.""" + return self.chk.accept(node, context) + + def check_not_void(self, typ: Type, context: Context) -> None: + """Generate an error if type is Void.""" + self.chk.check_not_void(typ, context) + + def is_boolean(self, typ: Type) -> bool: + """Is type compatible with bool?""" + return is_subtype(typ, self.chk.bool_type()) + + def named_type(self, name: str) -> Instance: + """Return an instance type with type given by the name and no type + arguments. Alias for TypeChecker.named_type. + """ + return self.chk.named_type(name) + + def is_valid_var_arg(self, typ: Type) -> bool: + """Is a type valid as a *args argument?""" + return (isinstance(typ, TupleType) or + is_subtype(typ, self.chk.named_generic_type('typing.Iterable', + [AnyType()])) or + isinstance(typ, AnyType)) + + def is_valid_keyword_var_arg(self, typ: Type) -> bool: + """Is a type valid as a **kwargs argument?""" + return is_subtype(typ, self.chk.named_generic_type( + 'builtins.dict', [self.named_type('builtins.str'), AnyType()])) + + def has_non_method(self, typ: Type, member: str) -> bool: + """Does type have a member variable / property with the given name?""" + if isinstance(typ, Instance): + return (not typ.type.has_method(member) and + typ.type.has_readable_member(member)) + else: + return False + + def has_member(self, typ: Type, member: str) -> bool: + """Does type have member with the given name?""" + # TODO TupleType => also consider tuple attributes + if isinstance(typ, Instance): + return typ.type.has_readable_member(member) + elif isinstance(typ, AnyType): + return True + elif isinstance(typ, UnionType): + result = all(self.has_member(x, member) for x in typ.items) + return result + elif isinstance(typ, TupleType): + return self.has_member(typ.fallback, member) + else: + return False + + def not_ready_callback(self, name: str, context: Context) -> None: + """Called when we can't infer the type of a variable because it's not ready yet. + + Either defer type checking of the enclosing function to the next + pass or report an error. + """ + self.chk.handle_cannot_determine_type(name, context) + + +def map_actuals_to_formals(caller_kinds: List[int], + caller_names: List[str], + callee_kinds: List[int], + callee_names: List[str], + caller_arg_type: Callable[[int], + Type]) -> List[List[int]]: + """Calculate mapping between actual (caller) args and formals. + + The result contains a list of caller argument indexes mapping to each + callee argument index, indexed by callee index. + + The caller_arg_type argument should evaluate to the type of the actual + argument type with the given index. + """ + ncallee = len(callee_kinds) + map = [None] * ncallee # type: List[List[int]] + for i in range(ncallee): + map[i] = [] + j = 0 + for i, kind in enumerate(caller_kinds): + if kind == nodes.ARG_POS: + if j < ncallee: + if callee_kinds[j] in [nodes.ARG_POS, nodes.ARG_OPT, + nodes.ARG_NAMED]: + map[j].append(i) + j += 1 + elif callee_kinds[j] == nodes.ARG_STAR: + map[j].append(i) + elif kind == nodes.ARG_STAR: + # We need to know the actual type to map varargs. + argt = caller_arg_type(i) + if isinstance(argt, TupleType): + # A tuple actual maps to a fixed number of formals. + for _ in range(len(argt.items)): + if j < ncallee: + if callee_kinds[j] != nodes.ARG_STAR2: + map[j].append(i) + else: + raise NotImplementedError() + j += 1 + else: + # Assume that it is an iterable (if it isn't, there will be + # an error later). + while j < ncallee: + if callee_kinds[j] in (nodes.ARG_NAMED, nodes.ARG_STAR2): + break + else: + map[j].append(i) + j += 1 + elif kind == nodes.ARG_NAMED: + name = caller_names[i] + if name in callee_names: + map[callee_names.index(name)].append(i) + elif nodes.ARG_STAR2 in callee_kinds: + map[callee_kinds.index(nodes.ARG_STAR2)].append(i) + else: + assert kind == nodes.ARG_STAR2 + for j in range(ncallee): + # TODO tuple varargs complicate this + no_certain_match = ( + not map[j] or caller_kinds[map[j][0]] == nodes.ARG_STAR) + if ((callee_names[j] and no_certain_match) + or callee_kinds[j] == nodes.ARG_STAR2): + map[j].append(i) + return map + + +def is_empty_tuple(t: Type) -> bool: + return isinstance(t, TupleType) and not t.items + + +def is_duplicate_mapping(mapping: List[int], actual_kinds: List[int]) -> bool: + # Multiple actuals can map to the same formal only if they both come from + # varargs (*args and **kwargs); in this case at runtime it is possible that + # there are no duplicates. We need to allow this, as the convention + # f(..., *args, **kwargs) is common enough. + return len(mapping) > 1 and not ( + len(mapping) == 2 and + actual_kinds[mapping[0]] == nodes.ARG_STAR and + actual_kinds[mapping[1]] == nodes.ARG_STAR2) + + +def replace_callable_return_type(c: CallableType, new_ret_type: Type) -> CallableType: + """Return a copy of a callable type with a different return type.""" + return c.copy_modified(ret_type=new_ret_type) + + +class ArgInferSecondPassQuery(types.TypeQuery): + """Query whether an argument type should be inferred in the second pass. + + The result is True if the type has a type variable in a callable return + type anywhere. For example, the result for Callable[[], T] is True if t is + a type variable. + """ + def __init__(self) -> None: + super().__init__(False, types.ANY_TYPE_STRATEGY) + + def visit_callable_type(self, t: CallableType) -> bool: + return self.query_types(t.arg_types) or t.accept(HasTypeVarQuery()) + + +class HasTypeVarQuery(types.TypeQuery): + """Visitor for querying whether a type has a type variable component.""" + def __init__(self) -> None: + super().__init__(False, types.ANY_TYPE_STRATEGY) + + def visit_type_var(self, t: TypeVarType) -> bool: + return True + + +def has_erased_component(t: Type) -> bool: + return t is not None and t.accept(HasErasedComponentsQuery()) + + +class HasErasedComponentsQuery(types.TypeQuery): + """Visitor for querying whether a type has an erased component.""" + def __init__(self) -> None: + super().__init__(False, types.ANY_TYPE_STRATEGY) + + def visit_erased_type(self, t: ErasedType) -> bool: + return True + + +def overload_arg_similarity(actual: Type, formal: Type) -> int: + """Return if caller argument (actual) is compatible with overloaded signature arg (formal). + + Return a similarity level: + 0: no match + 1: actual is compatible, but only using type promotions (e.g. int vs float) + 2: actual is compatible without type promotions (e.g. int vs object) + + The distinction is important in cases where multiple overload items match. We want + give priority to higher similarity matches. + """ + # Replace type variables with their upper bounds. Overloading + # resolution is based on runtime behavior which erases type + # parameters, so no need to handle type variables occurring within + # a type. + if isinstance(actual, TypeVarType): + actual = actual.erase_to_union_or_bound() + if isinstance(formal, TypeVarType): + formal = formal.erase_to_union_or_bound() + if (isinstance(actual, UninhabitedType) or isinstance(actual, AnyType) or + isinstance(formal, AnyType) or isinstance(formal, CallableType) or + (isinstance(actual, Instance) and actual.type.fallback_to_any)): + # These could match anything at runtime. + return 2 + if not experiments.STRICT_OPTIONAL and isinstance(actual, NoneTyp): + # NoneTyp matches anything if we're not doing strict Optional checking + return 2 + if isinstance(actual, UnionType): + return max(overload_arg_similarity(item, formal) + for item in actual.items) + if isinstance(formal, UnionType): + return max(overload_arg_similarity(actual, item) + for item in formal.items) + if isinstance(formal, TypeType): + if isinstance(actual, TypeType): + # Since Type[T] is covariant, check if actual = Type[A] is + # a subtype of formal = Type[F]. + return overload_arg_similarity(actual.item, formal.item) + elif isinstance(actual, CallableType) and actual.is_type_obj(): + # Check if the actual is a constructor of some sort. + # Note that this is this unsound, since we don't check the __init__ signature. + return overload_arg_similarity(actual.ret_type, formal.item) + else: + return 0 + if isinstance(formal, Instance): + if isinstance(actual, CallableType): + actual = actual.fallback + if isinstance(actual, Overloaded): + actual = actual.items()[0].fallback + if isinstance(actual, TupleType): + actual = actual.fallback + if isinstance(actual, Instance): + # First perform a quick check (as an optimization) and fall back to generic + # subtyping algorithm if type promotions are possible (e.g., int vs. float). + if formal.type in actual.type.mro: + return 2 + elif actual.type._promote and is_subtype(actual, formal): + return 1 + else: + return 0 + elif isinstance(actual, TypeType): + if formal.type.fullname() in {"builtins.object", "builtins.type"}: + return 2 + else: + return 0 + else: + return 0 + if isinstance(actual, UnboundType) or isinstance(formal, UnboundType): + # Either actual or formal is the result of an error; shut up. + return 2 + # Fall back to a conservative equality check for the remaining kinds of type. + return 2 if is_same_type(erasetype.erase_type(actual), erasetype.erase_type(formal)) else 0 + + +def freshen_generic_callable(callee: CallableType) -> CallableType: + tvdefs = [] + tvmap = {} # type: Dict[TypeVarId, Type] + for v in callee.variables: + tvdef = TypeVarDef.new_unification_variable(v) + tvdefs.append(tvdef) + tvmap[v.id] = TypeVarType(tvdef) + + return cast(CallableType, expand_type(callee, tvmap)).copy_modified(variables=tvdefs) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index ce6d5190fac6a..b74c29f83cd9d 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -1,112 +1,112 @@ -from typing import Dict, Tuple, List, cast - -from mypy.types import ( - Type, Instance, CallableType, TypeVisitor, UnboundType, ErrorType, AnyType, - Void, NoneTyp, TypeVarType, Overloaded, TupleType, UnionType, ErasedType, TypeList, - PartialType, DeletedType, UninhabitedType, TypeType, TypeVarId -) - - -def expand_type(typ: Type, env: Dict[TypeVarId, Type]) -> Type: - """Substitute any type variable references in a type given by a type - environment. - """ - - return typ.accept(ExpandTypeVisitor(env)) - - -def expand_type_by_instance(typ: Type, instance: Instance) -> Type: - """Substitute type variables in type using values from an Instance. - Type variables are considered to be bound by the class declaration.""" - - if instance.args == []: - return typ - else: - variables = {} # type: Dict[TypeVarId, Type] - for binder, arg in zip(instance.type.defn.type_vars, instance.args): - variables[binder.id] = arg - return expand_type(typ, variables) - - -class ExpandTypeVisitor(TypeVisitor[Type]): - """Visitor that substitutes type variables with values.""" - - variables = None # type: Dict[TypeVarId, Type] # TypeVar id -> TypeVar value - - def __init__(self, variables: Dict[TypeVarId, Type]) -> None: - self.variables = variables - - def visit_unbound_type(self, t: UnboundType) -> Type: - return t - - def visit_error_type(self, t: ErrorType) -> Type: - return t - - def visit_type_list(self, t: TypeList) -> Type: - assert False, 'Not supported' - - def visit_any(self, t: AnyType) -> Type: - return t - - def visit_void(self, t: Void) -> Type: - return t - - def visit_none_type(self, t: NoneTyp) -> Type: - return t - - def visit_uninhabited_type(self, t: UninhabitedType) -> Type: - return t - - def visit_deleted_type(self, t: DeletedType) -> Type: - return t - - def visit_erased_type(self, t: ErasedType) -> Type: - # Should not get here. - raise RuntimeError() - - def visit_instance(self, t: Instance) -> Type: - args = self.expand_types(t.args) - return Instance(t.type, args, t.line) - - def visit_type_var(self, t: TypeVarType) -> Type: - repl = self.variables.get(t.id, t) - if isinstance(repl, Instance): - inst = repl - # Return copy of instance with type erasure flag on. - return Instance(inst.type, inst.args, inst.line, True) - else: - return repl - - def visit_callable_type(self, t: CallableType) -> Type: - return t.copy_modified(arg_types=self.expand_types(t.arg_types), - ret_type=t.ret_type.accept(self)) - - def visit_overloaded(self, t: Overloaded) -> Type: - items = [] # type: List[CallableType] - for item in t.items(): - items.append(cast(CallableType, item.accept(self))) - return Overloaded(items) - - def visit_tuple_type(self, t: TupleType) -> Type: - return t.with_types(self.expand_types(t.items)) - - def visit_union_type(self, t: UnionType) -> Type: - # After substituting for type variables in t.items, - # some of the resulting types might be subtypes of others. - return UnionType.make_simplified_union(self.expand_types(t.items), t.line) - - def visit_partial_type(self, t: PartialType) -> Type: - return t - - def visit_type_type(self, t: TypeType) -> Type: - # TODO: Verify that the new item type is valid (instance or - # union of instances or Any). Sadly we can't report errors - # here yet. - item = t.item.accept(self) - return TypeType(item) - - def expand_types(self, types: List[Type]) -> List[Type]: - a = [] # type: List[Type] - for t in types: - a.append(t.accept(self)) - return a +from typing import Dict, Tuple, List, cast + +from mypy.types import ( + Type, Instance, CallableType, TypeVisitor, UnboundType, ErrorType, AnyType, + Void, NoneTyp, TypeVarType, Overloaded, TupleType, UnionType, ErasedType, TypeList, + PartialType, DeletedType, UninhabitedType, TypeType, TypeVarId +) + + +def expand_type(typ: Type, env: Dict[TypeVarId, Type]) -> Type: + """Substitute any type variable references in a type given by a type + environment. + """ + + return typ.accept(ExpandTypeVisitor(env)) + + +def expand_type_by_instance(typ: Type, instance: Instance) -> Type: + """Substitute type variables in type using values from an Instance. + Type variables are considered to be bound by the class declaration.""" + + if instance.args == []: + return typ + else: + variables = {} # type: Dict[TypeVarId, Type] + for binder, arg in zip(instance.type.defn.type_vars, instance.args): + variables[binder.id] = arg + return expand_type(typ, variables) + + +class ExpandTypeVisitor(TypeVisitor[Type]): + """Visitor that substitutes type variables with values.""" + + variables = None # type: Dict[TypeVarId, Type] # TypeVar id -> TypeVar value + + def __init__(self, variables: Dict[TypeVarId, Type]) -> None: + self.variables = variables + + def visit_unbound_type(self, t: UnboundType) -> Type: + return t + + def visit_error_type(self, t: ErrorType) -> Type: + return t + + def visit_type_list(self, t: TypeList) -> Type: + assert False, 'Not supported' + + def visit_any(self, t: AnyType) -> Type: + return t + + def visit_void(self, t: Void) -> Type: + return t + + def visit_none_type(self, t: NoneTyp) -> Type: + return t + + def visit_uninhabited_type(self, t: UninhabitedType) -> Type: + return t + + def visit_deleted_type(self, t: DeletedType) -> Type: + return t + + def visit_erased_type(self, t: ErasedType) -> Type: + # Should not get here. + raise RuntimeError() + + def visit_instance(self, t: Instance) -> Type: + args = self.expand_types(t.args) + return Instance(t.type, args, t.line) + + def visit_type_var(self, t: TypeVarType) -> Type: + repl = self.variables.get(t.id, t) + if isinstance(repl, Instance): + inst = repl + # Return copy of instance with type erasure flag on. + return Instance(inst.type, inst.args, inst.line, True) + else: + return repl + + def visit_callable_type(self, t: CallableType) -> Type: + return t.copy_modified(arg_types=self.expand_types(t.arg_types), + ret_type=t.ret_type.accept(self)) + + def visit_overloaded(self, t: Overloaded) -> Type: + items = [] # type: List[CallableType] + for item in t.items(): + items.append(cast(CallableType, item.accept(self))) + return Overloaded(items) + + def visit_tuple_type(self, t: TupleType) -> Type: + return t.with_types(self.expand_types(t.items)) + + def visit_union_type(self, t: UnionType) -> Type: + # After substituting for type variables in t.items, + # some of the resulting types might be subtypes of others. + return UnionType.make_simplified_union(self.expand_types(t.items), t.line) + + def visit_partial_type(self, t: PartialType) -> Type: + return t + + def visit_type_type(self, t: TypeType) -> Type: + # TODO: Verify that the new item type is valid (instance or + # union of instances or Any). Sadly we can't report errors + # here yet. + item = t.item.accept(self) + return TypeType(item) + + def expand_types(self, types: List[Type]) -> List[Type]: + a = [] # type: List[Type] + for t in types: + a.append(t.accept(self)) + return a diff --git a/mypy/join.py b/mypy/join.py index 684d62cfeaebd..e0709c2717911 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -1,367 +1,367 @@ -"""Calculation of the least upper bound types (joins).""" - -from typing import List - -from mypy.types import ( - Type, AnyType, NoneTyp, Void, TypeVisitor, Instance, UnboundType, - ErrorType, TypeVarType, CallableType, TupleType, ErasedType, TypeList, - UnionType, FunctionLike, Overloaded, PartialType, DeletedType, - UninhabitedType, TypeType, NamedTupleType -) -from mypy.maptype import map_instance_to_supertype -from mypy.subtypes import is_subtype, is_equivalent, is_subtype_ignoring_tvars - -from mypy import experiments - - -def join_simple(declaration: Type, s: Type, t: Type) -> Type: - """Return a simple least upper bound given the declared type.""" - - if isinstance(s, AnyType): - return s - - if isinstance(s, ErasedType): - return t - - if is_subtype(s, t): - return t - - if is_subtype(t, s): - return s - - if isinstance(declaration, UnionType): - return UnionType.make_simplified_union([s, t]) - - if isinstance(s, NoneTyp) and not isinstance(t, NoneTyp): - s, t = t, s - - if isinstance(s, UninhabitedType) and not isinstance(t, UninhabitedType): - s, t = t, s - - value = t.accept(TypeJoinVisitor(s)) - - if value is None: - # XXX this code path probably should be avoided. - # It seems to happen when a line (x = y) is a type error, and - # it's not clear that assuming that x is arbitrary afterward - # is a good idea. - return declaration - - if declaration is None or is_subtype(value, declaration): - return value - - return declaration - - -def join_types(s: Type, t: Type) -> Type: - """Return the least upper bound of s and t. - - For example, the join of 'int' and 'object' is 'object'. - - If the join does not exist, return an ErrorType instance. - """ - if isinstance(s, AnyType): - return s - - if isinstance(s, ErasedType): - return t - - if isinstance(s, NoneTyp) and not isinstance(t, NoneTyp): - s, t = t, s - - # Use a visitor to handle non-trivial cases. - return t.accept(TypeJoinVisitor(s)) - - -class TypeJoinVisitor(TypeVisitor[Type]): - """Implementation of the least upper bound algorithm. - - Attributes: - s: The other (left) type operand. - """ - - def __init__(self, s: Type) -> None: - self.s = s - - def visit_unbound_type(self, t: UnboundType) -> Type: - if isinstance(self.s, Void) or isinstance(self.s, ErrorType): - return ErrorType() - else: - return AnyType() - - def visit_union_type(self, t: UnionType) -> Type: - if is_subtype(self.s, t): - return t - else: - return UnionType.make_simplified_union([self.s, t]) - - def visit_error_type(self, t: ErrorType) -> Type: - return t - - def visit_type_list(self, t: TypeList) -> Type: - assert False, 'Not supported' - - def visit_any(self, t: AnyType) -> Type: - return t - - def visit_void(self, t: Void) -> Type: - if isinstance(self.s, Void): - return t - else: - return ErrorType() - - def visit_none_type(self, t: NoneTyp) -> Type: - if experiments.STRICT_OPTIONAL: - if isinstance(self.s, (NoneTyp, UninhabitedType)): - return t - else: - return self.default(self.s) - else: - if not isinstance(self.s, Void): - return self.s - else: - return self.default(self.s) - - def visit_uninhabited_type(self, t: UninhabitedType) -> Type: - if not isinstance(self.s, Void): - return self.s - else: - return self.default(self.s) - - def visit_deleted_type(self, t: DeletedType) -> Type: - if not isinstance(self.s, Void): - return self.s - else: - return self.default(self.s) - - def visit_erased_type(self, t: ErasedType) -> Type: - return self.s - - def visit_type_var(self, t: TypeVarType) -> Type: - if isinstance(self.s, TypeVarType) and self.s.id == t.id: - return self.s - else: - return self.default(self.s) - - def visit_instance(self, t: Instance) -> Type: - if isinstance(self.s, Instance): - return join_instances(t, self.s) - elif isinstance(self.s, FunctionLike): - return join_types(t, self.s.fallback) - elif isinstance(self.s, TypeType): - return join_types(t, self.s) - else: - return self.default(self.s) - - def visit_callable_type(self, t: CallableType) -> Type: - # TODO: Consider subtyping instead of just similarity. - if isinstance(self.s, CallableType) and is_similar_callables(t, self.s): - return combine_similar_callables(t, self.s) - elif isinstance(self.s, Overloaded): - # Switch the order of arguments to that we'll get to visit_overloaded. - return join_types(t, self.s) - else: - return join_types(t.fallback, self.s) - - def visit_overloaded(self, t: Overloaded) -> Type: - # This is more complex than most other cases. Here are some - # examples that illustrate how this works. - # - # First let's define a concise notation: - # - Cn are callable types (for n in 1, 2, ...) - # - Ov(C1, C2, ...) is an overloaded type with items C1, C2, ... - # - Callable[[T, ...], S] is written as [T, ...] -> S. - # - # We want some basic properties to hold (assume Cn are all - # unrelated via Any-similarity): - # - # join(Ov(C1, C2), C1) == C1 - # join(Ov(C1, C2), Ov(C1, C2)) == Ov(C1, C2) - # join(Ov(C1, C2), Ov(C1, C3)) == C1 - # join(Ov(C2, C2), C3) == join of fallback types - # - # The presence of Any types makes things more interesting. The join is the - # most general type we can get with respect to Any: - # - # join(Ov([int] -> int, [str] -> str), [Any] -> str) == Any -> str - # - # We could use a simplification step that removes redundancies, but that's not - # implemented right now. Consider this example, where we get a redundancy: - # - # join(Ov([int, Any] -> Any, [str, Any] -> Any), [Any, int] -> Any) == - # Ov([Any, int] -> Any, [Any, int] -> Any) - # - # TODO: Use callable subtyping instead of just similarity. - result = [] # type: List[CallableType] - s = self.s - if isinstance(s, FunctionLike): - # The interesting case where both types are function types. - for t_item in t.items(): - for s_item in s.items(): - if is_similar_callables(t_item, s_item): - result.append(combine_similar_callables(t_item, s_item)) - if result: - # TODO: Simplify redundancies from the result. - if len(result) == 1: - return result[0] - else: - return Overloaded(result) - return join_types(t.fallback, s.fallback) - return join_types(t.fallback, s) - - def visit_tuple_type(self, t: TupleType) -> Type: - if self.s == t: - return self.s - if isinstance(self.s, TupleType) and self.s.length() == t.length(): - items = [] # type: List[Type] - for i in range(t.length()): - items.append(self.join(t.items[i], self.s.items[i])) - if (not (isinstance(t, NamedTupleType) and isinstance(self.s, NamedTupleType)) - or self.s.attrs == t.attrs): - return self.s.with_types(items) - # TODO: What if the fallback types are different? - return TupleType(items, t.fallback) - else: - return self.default(self.s) - - def visit_partial_type(self, t: PartialType) -> Type: - # We only have partial information so we can't decide the join result. We should - # never get here. - assert False, "Internal error" - - def visit_type_type(self, t: TypeType) -> Type: - if isinstance(self.s, TypeType): - return TypeType(self.join(t.item, self.s.item), line=t.line) - elif isinstance(self.s, Instance) and self.s.type.fullname() == 'builtins.type': - return self.s - else: - return self.default(self.s) - - def join(self, s: Type, t: Type) -> Type: - return join_types(s, t) - - def default(self, typ: Type) -> Type: - if isinstance(typ, Instance): - return object_from_instance(typ) - elif isinstance(typ, UnboundType): - return AnyType() - elif isinstance(typ, Void) or isinstance(typ, ErrorType): - return ErrorType() - elif isinstance(typ, TupleType): - return self.default(typ.fallback) - elif isinstance(typ, FunctionLike): - return self.default(typ.fallback) - elif isinstance(typ, TypeVarType): - return self.default(typ.upper_bound) - else: - return AnyType() - - -def join_instances(t: Instance, s: Instance) -> Type: - """Calculate the join of two instance types. - - If allow_interfaces is True, also consider interface-type results for - non-interface types. - - Return ErrorType if the result is ambiguous. - """ - if t.type == s.type: - # Simplest case: join two types with the same base type (but - # potentially different arguments). - if is_subtype(t, s) or is_subtype(s, t): - # Compatible; combine type arguments. - args = [] # type: List[Type] - for i in range(len(t.args)): - args.append(join_types(t.args[i], s.args[i])) - return Instance(t.type, args) - else: - # Incompatible; return trivial result object. - return object_from_instance(t) - elif t.type.bases and is_subtype_ignoring_tvars(t, s): - return join_instances_via_supertype(t, s) - else: - # Now t is not a subtype of s, and t != s. Now s could be a subtype - # of t; alternatively, we need to find a common supertype. This works - # in of the both cases. - return join_instances_via_supertype(s, t) - - -def join_instances_via_supertype(t: Instance, s: Instance) -> Type: - # Give preference to joins via duck typing relationship, so that - # join(int, float) == float, for example. - if t.type._promote and is_subtype(t.type._promote, s): - return join_types(t.type._promote, s) - elif s.type._promote and is_subtype(s.type._promote, t): - return join_types(t, s.type._promote) - # Compute the "best" supertype of t when joined with s. - # The definition of "best" may evolve; for now it is the one with - # the longest MRO. Ties are broken by using the earlier base. - best = None # type: Type - for base in t.type.bases: - mapped = map_instance_to_supertype(t, base.type) - res = join_instances(mapped, s) - if best is None or is_better(res, best): - best = res - assert best is not None - return best - - -def is_better(t: Type, s: Type) -> bool: - # Given two possible results from join_instances_via_supertype(), - # indicate whether t is the better one. - if isinstance(t, Instance): - if not isinstance(s, Instance): - return True - # Use len(mro) as a proxy for the better choice. - if len(t.type.mro) > len(s.type.mro): - return True - return False - - -def is_similar_callables(t: CallableType, s: CallableType) -> bool: - """Return True if t and s are equivalent and have identical numbers of - arguments, default arguments and varargs. - """ - - return (len(t.arg_types) == len(s.arg_types) and t.min_args == s.min_args - and t.is_var_arg == s.is_var_arg and is_equivalent(t, s)) - - -def combine_similar_callables(t: CallableType, s: CallableType) -> CallableType: - arg_types = [] # type: List[Type] - for i in range(len(t.arg_types)): - arg_types.append(join_types(t.arg_types[i], s.arg_types[i])) - # TODO kinds and argument names - # The fallback type can be either 'function' or 'type'. The result should have 'type' as - # fallback only if both operands have it as 'type'. - if t.fallback.type.fullname() != 'builtins.type': - fallback = t.fallback - else: - fallback = s.fallback - return t.copy_modified(arg_types=arg_types, - ret_type=join_types(t.ret_type, s.ret_type), - fallback=fallback, - name=None) - - -def object_from_instance(instance: Instance) -> Instance: - """Construct the type 'builtins.object' from an instance type.""" - # Use the fact that 'object' is always the last class in the mro. - res = Instance(instance.type.mro[-1], []) - return res - - -def join_type_list(types: List[Type]) -> Type: - if not types: - # This is a little arbitrary but reasonable. Any empty tuple should be compatible - # with all variable length tuples, and this makes it possible. A better approach - # would be to use a special bottom type, which we do when strict Optional - # checking is enabled. - if experiments.STRICT_OPTIONAL: - return UninhabitedType() - else: - return NoneTyp() - joined = types[0] - for t in types[1:]: - joined = join_types(joined, t) - return joined +"""Calculation of the least upper bound types (joins).""" + +from typing import List + +from mypy.types import ( + Type, AnyType, NoneTyp, Void, TypeVisitor, Instance, UnboundType, + ErrorType, TypeVarType, CallableType, TupleType, ErasedType, TypeList, + UnionType, FunctionLike, Overloaded, PartialType, DeletedType, + UninhabitedType, TypeType, NamedTupleType +) +from mypy.maptype import map_instance_to_supertype +from mypy.subtypes import is_subtype, is_equivalent, is_subtype_ignoring_tvars + +from mypy import experiments + + +def join_simple(declaration: Type, s: Type, t: Type) -> Type: + """Return a simple least upper bound given the declared type.""" + + if isinstance(s, AnyType): + return s + + if isinstance(s, ErasedType): + return t + + if is_subtype(s, t): + return t + + if is_subtype(t, s): + return s + + if isinstance(declaration, UnionType): + return UnionType.make_simplified_union([s, t]) + + if isinstance(s, NoneTyp) and not isinstance(t, NoneTyp): + s, t = t, s + + if isinstance(s, UninhabitedType) and not isinstance(t, UninhabitedType): + s, t = t, s + + value = t.accept(TypeJoinVisitor(s)) + + if value is None: + # XXX this code path probably should be avoided. + # It seems to happen when a line (x = y) is a type error, and + # it's not clear that assuming that x is arbitrary afterward + # is a good idea. + return declaration + + if declaration is None or is_subtype(value, declaration): + return value + + return declaration + + +def join_types(s: Type, t: Type) -> Type: + """Return the least upper bound of s and t. + + For example, the join of 'int' and 'object' is 'object'. + + If the join does not exist, return an ErrorType instance. + """ + if isinstance(s, AnyType): + return s + + if isinstance(s, ErasedType): + return t + + if isinstance(s, NoneTyp) and not isinstance(t, NoneTyp): + s, t = t, s + + # Use a visitor to handle non-trivial cases. + return t.accept(TypeJoinVisitor(s)) + + +class TypeJoinVisitor(TypeVisitor[Type]): + """Implementation of the least upper bound algorithm. + + Attributes: + s: The other (left) type operand. + """ + + def __init__(self, s: Type) -> None: + self.s = s + + def visit_unbound_type(self, t: UnboundType) -> Type: + if isinstance(self.s, Void) or isinstance(self.s, ErrorType): + return ErrorType() + else: + return AnyType() + + def visit_union_type(self, t: UnionType) -> Type: + if is_subtype(self.s, t): + return t + else: + return UnionType.make_simplified_union([self.s, t]) + + def visit_error_type(self, t: ErrorType) -> Type: + return t + + def visit_type_list(self, t: TypeList) -> Type: + assert False, 'Not supported' + + def visit_any(self, t: AnyType) -> Type: + return t + + def visit_void(self, t: Void) -> Type: + if isinstance(self.s, Void): + return t + else: + return ErrorType() + + def visit_none_type(self, t: NoneTyp) -> Type: + if experiments.STRICT_OPTIONAL: + if isinstance(self.s, (NoneTyp, UninhabitedType)): + return t + else: + return self.default(self.s) + else: + if not isinstance(self.s, Void): + return self.s + else: + return self.default(self.s) + + def visit_uninhabited_type(self, t: UninhabitedType) -> Type: + if not isinstance(self.s, Void): + return self.s + else: + return self.default(self.s) + + def visit_deleted_type(self, t: DeletedType) -> Type: + if not isinstance(self.s, Void): + return self.s + else: + return self.default(self.s) + + def visit_erased_type(self, t: ErasedType) -> Type: + return self.s + + def visit_type_var(self, t: TypeVarType) -> Type: + if isinstance(self.s, TypeVarType) and self.s.id == t.id: + return self.s + else: + return self.default(self.s) + + def visit_instance(self, t: Instance) -> Type: + if isinstance(self.s, Instance): + return join_instances(t, self.s) + elif isinstance(self.s, FunctionLike): + return join_types(t, self.s.fallback) + elif isinstance(self.s, TypeType): + return join_types(t, self.s) + else: + return self.default(self.s) + + def visit_callable_type(self, t: CallableType) -> Type: + # TODO: Consider subtyping instead of just similarity. + if isinstance(self.s, CallableType) and is_similar_callables(t, self.s): + return combine_similar_callables(t, self.s) + elif isinstance(self.s, Overloaded): + # Switch the order of arguments to that we'll get to visit_overloaded. + return join_types(t, self.s) + else: + return join_types(t.fallback, self.s) + + def visit_overloaded(self, t: Overloaded) -> Type: + # This is more complex than most other cases. Here are some + # examples that illustrate how this works. + # + # First let's define a concise notation: + # - Cn are callable types (for n in 1, 2, ...) + # - Ov(C1, C2, ...) is an overloaded type with items C1, C2, ... + # - Callable[[T, ...], S] is written as [T, ...] -> S. + # + # We want some basic properties to hold (assume Cn are all + # unrelated via Any-similarity): + # + # join(Ov(C1, C2), C1) == C1 + # join(Ov(C1, C2), Ov(C1, C2)) == Ov(C1, C2) + # join(Ov(C1, C2), Ov(C1, C3)) == C1 + # join(Ov(C2, C2), C3) == join of fallback types + # + # The presence of Any types makes things more interesting. The join is the + # most general type we can get with respect to Any: + # + # join(Ov([int] -> int, [str] -> str), [Any] -> str) == Any -> str + # + # We could use a simplification step that removes redundancies, but that's not + # implemented right now. Consider this example, where we get a redundancy: + # + # join(Ov([int, Any] -> Any, [str, Any] -> Any), [Any, int] -> Any) == + # Ov([Any, int] -> Any, [Any, int] -> Any) + # + # TODO: Use callable subtyping instead of just similarity. + result = [] # type: List[CallableType] + s = self.s + if isinstance(s, FunctionLike): + # The interesting case where both types are function types. + for t_item in t.items(): + for s_item in s.items(): + if is_similar_callables(t_item, s_item): + result.append(combine_similar_callables(t_item, s_item)) + if result: + # TODO: Simplify redundancies from the result. + if len(result) == 1: + return result[0] + else: + return Overloaded(result) + return join_types(t.fallback, s.fallback) + return join_types(t.fallback, s) + + def visit_tuple_type(self, t: TupleType) -> Type: + if self.s == t: + return self.s + if isinstance(self.s, TupleType) and self.s.length() == t.length(): + items = [] # type: List[Type] + for i in range(t.length()): + items.append(self.join(t.items[i], self.s.items[i])) + if (not (isinstance(t, NamedTupleType) and isinstance(self.s, NamedTupleType)) + or self.s.attrs == t.attrs): + return self.s.with_types(items) + # TODO: What if the fallback types are different? + return TupleType(items, t.fallback) + else: + return self.default(self.s) + + def visit_partial_type(self, t: PartialType) -> Type: + # We only have partial information so we can't decide the join result. We should + # never get here. + assert False, "Internal error" + + def visit_type_type(self, t: TypeType) -> Type: + if isinstance(self.s, TypeType): + return TypeType(self.join(t.item, self.s.item), line=t.line) + elif isinstance(self.s, Instance) and self.s.type.fullname() == 'builtins.type': + return self.s + else: + return self.default(self.s) + + def join(self, s: Type, t: Type) -> Type: + return join_types(s, t) + + def default(self, typ: Type) -> Type: + if isinstance(typ, Instance): + return object_from_instance(typ) + elif isinstance(typ, UnboundType): + return AnyType() + elif isinstance(typ, Void) or isinstance(typ, ErrorType): + return ErrorType() + elif isinstance(typ, TupleType): + return self.default(typ.fallback) + elif isinstance(typ, FunctionLike): + return self.default(typ.fallback) + elif isinstance(typ, TypeVarType): + return self.default(typ.upper_bound) + else: + return AnyType() + + +def join_instances(t: Instance, s: Instance) -> Type: + """Calculate the join of two instance types. + + If allow_interfaces is True, also consider interface-type results for + non-interface types. + + Return ErrorType if the result is ambiguous. + """ + if t.type == s.type: + # Simplest case: join two types with the same base type (but + # potentially different arguments). + if is_subtype(t, s) or is_subtype(s, t): + # Compatible; combine type arguments. + args = [] # type: List[Type] + for i in range(len(t.args)): + args.append(join_types(t.args[i], s.args[i])) + return Instance(t.type, args) + else: + # Incompatible; return trivial result object. + return object_from_instance(t) + elif t.type.bases and is_subtype_ignoring_tvars(t, s): + return join_instances_via_supertype(t, s) + else: + # Now t is not a subtype of s, and t != s. Now s could be a subtype + # of t; alternatively, we need to find a common supertype. This works + # in of the both cases. + return join_instances_via_supertype(s, t) + + +def join_instances_via_supertype(t: Instance, s: Instance) -> Type: + # Give preference to joins via duck typing relationship, so that + # join(int, float) == float, for example. + if t.type._promote and is_subtype(t.type._promote, s): + return join_types(t.type._promote, s) + elif s.type._promote and is_subtype(s.type._promote, t): + return join_types(t, s.type._promote) + # Compute the "best" supertype of t when joined with s. + # The definition of "best" may evolve; for now it is the one with + # the longest MRO. Ties are broken by using the earlier base. + best = None # type: Type + for base in t.type.bases: + mapped = map_instance_to_supertype(t, base.type) + res = join_instances(mapped, s) + if best is None or is_better(res, best): + best = res + assert best is not None + return best + + +def is_better(t: Type, s: Type) -> bool: + # Given two possible results from join_instances_via_supertype(), + # indicate whether t is the better one. + if isinstance(t, Instance): + if not isinstance(s, Instance): + return True + # Use len(mro) as a proxy for the better choice. + if len(t.type.mro) > len(s.type.mro): + return True + return False + + +def is_similar_callables(t: CallableType, s: CallableType) -> bool: + """Return True if t and s are equivalent and have identical numbers of + arguments, default arguments and varargs. + """ + + return (len(t.arg_types) == len(s.arg_types) and t.min_args == s.min_args + and t.is_var_arg == s.is_var_arg and is_equivalent(t, s)) + + +def combine_similar_callables(t: CallableType, s: CallableType) -> CallableType: + arg_types = [] # type: List[Type] + for i in range(len(t.arg_types)): + arg_types.append(join_types(t.arg_types[i], s.arg_types[i])) + # TODO kinds and argument names + # The fallback type can be either 'function' or 'type'. The result should have 'type' as + # fallback only if both operands have it as 'type'. + if t.fallback.type.fullname() != 'builtins.type': + fallback = t.fallback + else: + fallback = s.fallback + return t.copy_modified(arg_types=arg_types, + ret_type=join_types(t.ret_type, s.ret_type), + fallback=fallback, + name=None) + + +def object_from_instance(instance: Instance) -> Instance: + """Construct the type 'builtins.object' from an instance type.""" + # Use the fact that 'object' is always the last class in the mro. + res = Instance(instance.type.mro[-1], []) + return res + + +def join_type_list(types: List[Type]) -> Type: + if not types: + # This is a little arbitrary but reasonable. Any empty tuple should be compatible + # with all variable length tuples, and this makes it possible. A better approach + # would be to use a special bottom type, which we do when strict Optional + # checking is enabled. + if experiments.STRICT_OPTIONAL: + return UninhabitedType() + else: + return NoneTyp() + joined = types[0] + for t in types[1:]: + joined = join_types(joined, t) + return joined diff --git a/mypy/meet.py b/mypy/meet.py index 5c98385c7f1f2..c4e3831e6f5f7 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -1,285 +1,285 @@ -from typing import cast, List - -from mypy.join import is_similar_callables, combine_similar_callables -from mypy.types import ( - Type, AnyType, TypeVisitor, UnboundType, Void, ErrorType, NoneTyp, TypeVarType, - Instance, CallableType, TupleType, ErasedType, TypeList, UnionType, PartialType, - DeletedType, UninhabitedType, TypeType -) -from mypy.subtypes import is_subtype -from mypy.nodes import TypeInfo - -from mypy import experiments - -# TODO Describe this module. - - -def meet_types(s: Type, t: Type) -> Type: - """Return the greatest lower bound of two types.""" - if isinstance(s, ErasedType): - return s - if isinstance(s, AnyType): - return t - if isinstance(s, UnionType) and not isinstance(t, UnionType): - s, t = t, s - return t.accept(TypeMeetVisitor(s)) - - -def meet_simple(s: Type, t: Type, default_right: bool = True) -> Type: - if s == t: - return s - if isinstance(s, UnionType): - return UnionType.make_simplified_union([meet_types(x, t) for x in s.items]) - elif not is_overlapping_types(s, t, use_promotions=True): - if experiments.STRICT_OPTIONAL: - return UninhabitedType() - else: - return NoneTyp() - else: - if default_right: - return t - else: - return s - - -def is_overlapping_types(t: Type, s: Type, use_promotions: bool = False) -> bool: - """Can a value of type t be a value of type s, or vice versa? - - Note that this effectively checks against erased types, since type - variables are erased at runtime and the overlapping check is based - on runtime behavior. - - If use_promotions is True, also consider type promotions (int and - float would only be overlapping if it's True). - - This does not consider multiple inheritance. For example, A and B in - the following example are not considered overlapping, even though - via C they can be overlapping: - - class A: ... - class B: ... - class C(A, B): ... - - The rationale is that this case is usually very unlikely as multiple - inheritance is rare. Also, we can't reliably determine whether - multiple inheritance actually occurs somewhere in a program, due to - stub files hiding implementation details, dynamic loading etc. - - TODO: Don't consider tuples always overlapping. - TODO: Don't consider callables always overlapping. - TODO: Don't consider type variables with values always overlapping. - """ - # Since we are effectively working with the erased types, we only - # need to handle occurrences of TypeVarType at the top level. - if isinstance(t, TypeVarType): - t = t.erase_to_union_or_bound() - if isinstance(s, TypeVarType): - s = s.erase_to_union_or_bound() - if isinstance(t, Instance): - if isinstance(s, Instance): - # Consider two classes non-disjoint if one is included in the mro - # of another. - if use_promotions: - # Consider cases like int vs float to be overlapping where - # there is only a type promotion relationship but not proper - # subclassing. - if t.type._promote and is_overlapping_types(t.type._promote, s): - return True - if s.type._promote and is_overlapping_types(s.type._promote, t): - return True - return t.type in s.type.mro or s.type in t.type.mro - if isinstance(t, UnionType): - return any(is_overlapping_types(item, s) - for item in t.items) - if isinstance(s, UnionType): - return any(is_overlapping_types(t, item) - for item in s.items) - if isinstance(t, TypeType) and isinstance(s, TypeType): - # If both types are TypeType, compare their inner types. - return is_overlapping_types(t.item, s.item, use_promotions) - elif isinstance(t, TypeType) or isinstance(s, TypeType): - # If exactly only one of t or s is a TypeType, check if one of them - # is an `object` or a `type` and otherwise assume no overlap. - other = s if isinstance(t, TypeType) else t - if isinstance(other, Instance): - return other.type.fullname() in {'builtins.object', 'builtins.type'} - else: - return False - if experiments.STRICT_OPTIONAL: - if isinstance(t, NoneTyp) != isinstance(s, NoneTyp): - # NoneTyp does not overlap with other non-Union types under strict Optional checking - return False - # We conservatively assume that non-instance, non-union, and non-TypeType types can overlap - # any other types. - return True - - -def nearest_builtin_ancestor(type: TypeInfo) -> TypeInfo: - for base in type.mro: - if base.defn.is_builtinclass: - return base - else: - return None - - -class TypeMeetVisitor(TypeVisitor[Type]): - def __init__(self, s: Type) -> None: - self.s = s - - def visit_unbound_type(self, t: UnboundType) -> Type: - if isinstance(self.s, Void) or isinstance(self.s, ErrorType): - return ErrorType() - elif isinstance(self.s, NoneTyp): - if experiments.STRICT_OPTIONAL: - return AnyType() - else: - return self.s - elif isinstance(self.s, UninhabitedType): - return self.s - else: - return AnyType() - - def visit_error_type(self, t: ErrorType) -> Type: - return t - - def visit_type_list(self, t: TypeList) -> Type: - assert False, 'Not supported' - - def visit_any(self, t: AnyType) -> Type: - return self.s - - def visit_union_type(self, t: UnionType) -> Type: - if isinstance(self.s, UnionType): - meets = [] # type: List[Type] - for x in t.items: - for y in self.s.items: - meets.append(meet_types(x, y)) - else: - meets = [meet_types(x, self.s) - for x in t.items] - return UnionType.make_simplified_union(meets) - - def visit_void(self, t: Void) -> Type: - if isinstance(self.s, Void): - return t - else: - return ErrorType() - - def visit_none_type(self, t: NoneTyp) -> Type: - if experiments.STRICT_OPTIONAL: - if isinstance(self.s, NoneTyp) or (isinstance(self.s, Instance) and - self.s.type.fullname() == 'builtins.object'): - return t - else: - return UninhabitedType() - else: - if not isinstance(self.s, Void) and not isinstance(self.s, ErrorType): - return t - else: - return ErrorType() - - def visit_uninhabited_type(self, t: UninhabitedType) -> Type: - if not isinstance(self.s, Void) and not isinstance(self.s, ErrorType): - return t - else: - return ErrorType() - - def visit_deleted_type(self, t: DeletedType) -> Type: - if not isinstance(self.s, Void) and not isinstance(self.s, ErrorType): - if isinstance(self.s, NoneTyp): - if experiments.STRICT_OPTIONAL: - return t - else: - return self.s - elif isinstance(self.s, UninhabitedType): - return self.s - else: - return t - else: - return ErrorType() - - def visit_erased_type(self, t: ErasedType) -> Type: - return self.s - - def visit_type_var(self, t: TypeVarType) -> Type: - if isinstance(self.s, TypeVarType) and self.s.id == t.id: - return self.s - else: - return self.default(self.s) - - def visit_instance(self, t: Instance) -> Type: - if isinstance(self.s, Instance): - si = self.s - if t.type == si.type: - if is_subtype(t, self.s) or is_subtype(self.s, t): - # Combine type arguments. We could have used join below - # equivalently. - args = [] # type: List[Type] - for i in range(len(t.args)): - args.append(self.meet(t.args[i], si.args[i])) - return Instance(t.type, args) - else: - if experiments.STRICT_OPTIONAL: - return UninhabitedType() - else: - return NoneTyp() - else: - if is_subtype(t, self.s): - return t - elif is_subtype(self.s, t): - # See also above comment. - return self.s - else: - if experiments.STRICT_OPTIONAL: - return UninhabitedType() - else: - return NoneTyp() - elif isinstance(self.s, TypeType): - return meet_types(t, self.s) - else: - return self.default(self.s) - - def visit_callable_type(self, t: CallableType) -> Type: - if isinstance(self.s, CallableType) and is_similar_callables(t, self.s): - return combine_similar_callables(t, self.s) - else: - return self.default(self.s) - - def visit_tuple_type(self, t: TupleType) -> Type: - if isinstance(self.s, TupleType) and self.s.length() == t.length(): - items = [] # type: List[Type] - for i in range(t.length()): - items.append(self.meet(t.items[i], self.s.items[i])) - # TODO: What if the fallbacks are different? - return TupleType(items, t.fallback) - else: - return self.default(self.s) - - def visit_partial_type(self, t: PartialType) -> Type: - # We can't determine the meet of partial types. We should never get here. - assert False, 'Internal error' - - def visit_type_type(self, t: TypeType) -> Type: - if isinstance(self.s, TypeType): - typ = self.meet(t.item, self.s.item) - if not isinstance(typ, NoneTyp): - typ = TypeType(typ, line=t.line) - return typ - elif isinstance(self.s, Instance) and self.s.type.fullname() == 'builtins.type': - return t - else: - return self.default(self.s) - - def meet(self, s, t): - return meet_types(s, t) - - def default(self, typ): - if isinstance(typ, UnboundType): - return AnyType() - elif isinstance(typ, Void) or isinstance(typ, ErrorType): - return ErrorType() - else: - if experiments.STRICT_OPTIONAL: - return UninhabitedType() - else: - return NoneTyp() +from typing import cast, List + +from mypy.join import is_similar_callables, combine_similar_callables +from mypy.types import ( + Type, AnyType, TypeVisitor, UnboundType, Void, ErrorType, NoneTyp, TypeVarType, + Instance, CallableType, TupleType, ErasedType, TypeList, UnionType, PartialType, + DeletedType, UninhabitedType, TypeType +) +from mypy.subtypes import is_subtype +from mypy.nodes import TypeInfo + +from mypy import experiments + +# TODO Describe this module. + + +def meet_types(s: Type, t: Type) -> Type: + """Return the greatest lower bound of two types.""" + if isinstance(s, ErasedType): + return s + if isinstance(s, AnyType): + return t + if isinstance(s, UnionType) and not isinstance(t, UnionType): + s, t = t, s + return t.accept(TypeMeetVisitor(s)) + + +def meet_simple(s: Type, t: Type, default_right: bool = True) -> Type: + if s == t: + return s + if isinstance(s, UnionType): + return UnionType.make_simplified_union([meet_types(x, t) for x in s.items]) + elif not is_overlapping_types(s, t, use_promotions=True): + if experiments.STRICT_OPTIONAL: + return UninhabitedType() + else: + return NoneTyp() + else: + if default_right: + return t + else: + return s + + +def is_overlapping_types(t: Type, s: Type, use_promotions: bool = False) -> bool: + """Can a value of type t be a value of type s, or vice versa? + + Note that this effectively checks against erased types, since type + variables are erased at runtime and the overlapping check is based + on runtime behavior. + + If use_promotions is True, also consider type promotions (int and + float would only be overlapping if it's True). + + This does not consider multiple inheritance. For example, A and B in + the following example are not considered overlapping, even though + via C they can be overlapping: + + class A: ... + class B: ... + class C(A, B): ... + + The rationale is that this case is usually very unlikely as multiple + inheritance is rare. Also, we can't reliably determine whether + multiple inheritance actually occurs somewhere in a program, due to + stub files hiding implementation details, dynamic loading etc. + + TODO: Don't consider tuples always overlapping. + TODO: Don't consider callables always overlapping. + TODO: Don't consider type variables with values always overlapping. + """ + # Since we are effectively working with the erased types, we only + # need to handle occurrences of TypeVarType at the top level. + if isinstance(t, TypeVarType): + t = t.erase_to_union_or_bound() + if isinstance(s, TypeVarType): + s = s.erase_to_union_or_bound() + if isinstance(t, Instance): + if isinstance(s, Instance): + # Consider two classes non-disjoint if one is included in the mro + # of another. + if use_promotions: + # Consider cases like int vs float to be overlapping where + # there is only a type promotion relationship but not proper + # subclassing. + if t.type._promote and is_overlapping_types(t.type._promote, s): + return True + if s.type._promote and is_overlapping_types(s.type._promote, t): + return True + return t.type in s.type.mro or s.type in t.type.mro + if isinstance(t, UnionType): + return any(is_overlapping_types(item, s) + for item in t.items) + if isinstance(s, UnionType): + return any(is_overlapping_types(t, item) + for item in s.items) + if isinstance(t, TypeType) and isinstance(s, TypeType): + # If both types are TypeType, compare their inner types. + return is_overlapping_types(t.item, s.item, use_promotions) + elif isinstance(t, TypeType) or isinstance(s, TypeType): + # If exactly only one of t or s is a TypeType, check if one of them + # is an `object` or a `type` and otherwise assume no overlap. + other = s if isinstance(t, TypeType) else t + if isinstance(other, Instance): + return other.type.fullname() in {'builtins.object', 'builtins.type'} + else: + return False + if experiments.STRICT_OPTIONAL: + if isinstance(t, NoneTyp) != isinstance(s, NoneTyp): + # NoneTyp does not overlap with other non-Union types under strict Optional checking + return False + # We conservatively assume that non-instance, non-union, and non-TypeType types can overlap + # any other types. + return True + + +def nearest_builtin_ancestor(type: TypeInfo) -> TypeInfo: + for base in type.mro: + if base.defn.is_builtinclass: + return base + else: + return None + + +class TypeMeetVisitor(TypeVisitor[Type]): + def __init__(self, s: Type) -> None: + self.s = s + + def visit_unbound_type(self, t: UnboundType) -> Type: + if isinstance(self.s, Void) or isinstance(self.s, ErrorType): + return ErrorType() + elif isinstance(self.s, NoneTyp): + if experiments.STRICT_OPTIONAL: + return AnyType() + else: + return self.s + elif isinstance(self.s, UninhabitedType): + return self.s + else: + return AnyType() + + def visit_error_type(self, t: ErrorType) -> Type: + return t + + def visit_type_list(self, t: TypeList) -> Type: + assert False, 'Not supported' + + def visit_any(self, t: AnyType) -> Type: + return self.s + + def visit_union_type(self, t: UnionType) -> Type: + if isinstance(self.s, UnionType): + meets = [] # type: List[Type] + for x in t.items: + for y in self.s.items: + meets.append(meet_types(x, y)) + else: + meets = [meet_types(x, self.s) + for x in t.items] + return UnionType.make_simplified_union(meets) + + def visit_void(self, t: Void) -> Type: + if isinstance(self.s, Void): + return t + else: + return ErrorType() + + def visit_none_type(self, t: NoneTyp) -> Type: + if experiments.STRICT_OPTIONAL: + if isinstance(self.s, NoneTyp) or (isinstance(self.s, Instance) and + self.s.type.fullname() == 'builtins.object'): + return t + else: + return UninhabitedType() + else: + if not isinstance(self.s, Void) and not isinstance(self.s, ErrorType): + return t + else: + return ErrorType() + + def visit_uninhabited_type(self, t: UninhabitedType) -> Type: + if not isinstance(self.s, Void) and not isinstance(self.s, ErrorType): + return t + else: + return ErrorType() + + def visit_deleted_type(self, t: DeletedType) -> Type: + if not isinstance(self.s, Void) and not isinstance(self.s, ErrorType): + if isinstance(self.s, NoneTyp): + if experiments.STRICT_OPTIONAL: + return t + else: + return self.s + elif isinstance(self.s, UninhabitedType): + return self.s + else: + return t + else: + return ErrorType() + + def visit_erased_type(self, t: ErasedType) -> Type: + return self.s + + def visit_type_var(self, t: TypeVarType) -> Type: + if isinstance(self.s, TypeVarType) and self.s.id == t.id: + return self.s + else: + return self.default(self.s) + + def visit_instance(self, t: Instance) -> Type: + if isinstance(self.s, Instance): + si = self.s + if t.type == si.type: + if is_subtype(t, self.s) or is_subtype(self.s, t): + # Combine type arguments. We could have used join below + # equivalently. + args = [] # type: List[Type] + for i in range(len(t.args)): + args.append(self.meet(t.args[i], si.args[i])) + return Instance(t.type, args) + else: + if experiments.STRICT_OPTIONAL: + return UninhabitedType() + else: + return NoneTyp() + else: + if is_subtype(t, self.s): + return t + elif is_subtype(self.s, t): + # See also above comment. + return self.s + else: + if experiments.STRICT_OPTIONAL: + return UninhabitedType() + else: + return NoneTyp() + elif isinstance(self.s, TypeType): + return meet_types(t, self.s) + else: + return self.default(self.s) + + def visit_callable_type(self, t: CallableType) -> Type: + if isinstance(self.s, CallableType) and is_similar_callables(t, self.s): + return combine_similar_callables(t, self.s) + else: + return self.default(self.s) + + def visit_tuple_type(self, t: TupleType) -> Type: + if isinstance(self.s, TupleType) and self.s.length() == t.length(): + items = [] # type: List[Type] + for i in range(t.length()): + items.append(self.meet(t.items[i], self.s.items[i])) + # TODO: What if the fallbacks are different? + return TupleType(items, t.fallback) + else: + return self.default(self.s) + + def visit_partial_type(self, t: PartialType) -> Type: + # We can't determine the meet of partial types. We should never get here. + assert False, 'Internal error' + + def visit_type_type(self, t: TypeType) -> Type: + if isinstance(self.s, TypeType): + typ = self.meet(t.item, self.s.item) + if not isinstance(typ, NoneTyp): + typ = TypeType(typ, line=t.line) + return typ + elif isinstance(self.s, Instance) and self.s.type.fullname() == 'builtins.type': + return t + else: + return self.default(self.s) + + def meet(self, s, t): + return meet_types(s, t) + + def default(self, typ): + if isinstance(typ, UnboundType): + return AnyType() + elif isinstance(typ, Void) or isinstance(typ, ErrorType): + return ErrorType() + else: + if experiments.STRICT_OPTIONAL: + return UninhabitedType() + else: + return NoneTyp() diff --git a/mypy/messages.py b/mypy/messages.py index 5d82c4845782a..cc9b4761b703a 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -1,920 +1,920 @@ -"""Facilities and constants for generating error messages during type checking. - -The type checker itself does not deal with message string literals to -improve code clarity and to simplify localization (in the future).""" - -import re -import difflib - -from typing import cast, List, Dict, Any, Sequence, Iterable, Tuple - -from mypy.errors import Errors -from mypy.types import ( - Type, CallableType, Instance, TypeVarType, TupleType, UnionType, Void, NoneTyp, AnyType, - Overloaded, FunctionLike, DeletedType, TypeType -) -from mypy.nodes import ( - TypeInfo, Context, MypyFile, op_methods, FuncDef, reverse_type_aliases, - ARG_STAR, ARG_STAR2 -) - - -# Constants that represent simple type checker error message, i.e. messages -# that do not have any parameters. - -NO_RETURN_VALUE_EXPECTED = 'No return value expected' -INCOMPATIBLE_RETURN_VALUE_TYPE = 'Incompatible return value type' -RETURN_VALUE_EXPECTED = 'Return value expected' -BOOLEAN_VALUE_EXPECTED = 'Boolean value expected' -BOOLEAN_EXPECTED_FOR_IF = 'Boolean value expected for if condition' -BOOLEAN_EXPECTED_FOR_WHILE = 'Boolean value expected for while condition' -BOOLEAN_EXPECTED_FOR_UNTIL = 'Boolean value expected for until condition' -BOOLEAN_EXPECTED_FOR_NOT = 'Boolean value expected for not operand' -INVALID_EXCEPTION = 'Exception must be derived from BaseException' -INVALID_EXCEPTION_TYPE = 'Exception type must be derived from BaseException' -INVALID_RETURN_TYPE_FOR_GENERATOR = \ - 'The return type of a generator function should be "Generator" or one of its supertypes' -INVALID_GENERATOR_RETURN_ITEM_TYPE = \ - 'The return type of a generator function must be None in its third type parameter in Python 2' -YIELD_VALUE_EXPECTED = 'Yield value expected' -INCOMPATIBLE_TYPES = 'Incompatible types' -INCOMPATIBLE_TYPES_IN_ASSIGNMENT = 'Incompatible types in assignment' -INCOMPATIBLE_REDEFINITION = 'Incompatible redefinition' -INCOMPATIBLE_TYPES_IN_YIELD = 'Incompatible types in yield' -INCOMPATIBLE_TYPES_IN_YIELD_FROM = 'Incompatible types in "yield from"' -INCOMPATIBLE_TYPES_IN_STR_INTERPOLATION = 'Incompatible types in string interpolation' -INIT_MUST_HAVE_NONE_RETURN_TYPE = 'The return type of "__init__" must be None' -GETTER_TYPE_INCOMPATIBLE_WITH_SETTER = \ - 'Type of getter incompatible with setter' -TUPLE_INDEX_MUST_BE_AN_INT_LITERAL = 'Tuple index must be an integer literal' -TUPLE_SLICE_MUST_BE_AN_INT_LITERAL = 'Tuple slice must be an integer literal' -TUPLE_INDEX_OUT_OF_RANGE = 'Tuple index out of range' -TYPE_CONSTANT_EXPECTED = 'Type "Constant" or initializer expected' -INCOMPATIBLE_PAIR_ITEM_TYPE = 'Incompatible Pair item type' -INVALID_TYPE_APPLICATION_TARGET_TYPE = 'Invalid type application target type' -INCOMPATIBLE_TUPLE_ITEM_TYPE = 'Incompatible tuple item type' -INCOMPATIBLE_KEY_TYPE = 'Incompatible dictionary key type' -INCOMPATIBLE_VALUE_TYPE = 'Incompatible dictionary value type' -NEED_ANNOTATION_FOR_VAR = 'Need type annotation for variable' -ITERABLE_EXPECTED = 'Iterable expected' -INCOMPATIBLE_TYPES_IN_FOR = 'Incompatible types in for statement' -INCOMPATIBLE_ARRAY_VAR_ARGS = 'Incompatible variable arguments in call' -INVALID_SLICE_INDEX = 'Slice index must be an integer or None' -CANNOT_INFER_LAMBDA_TYPE = 'Cannot infer type of lambda' -CANNOT_INFER_ITEM_TYPE = 'Cannot infer iterable item type' -CANNOT_ACCESS_INIT = 'Cannot access "__init__" directly' -CANNOT_ASSIGN_TO_METHOD = 'Cannot assign to a method' -CANNOT_ASSIGN_TO_TYPE = 'Cannot assign to a type' -INCONSISTENT_ABSTRACT_OVERLOAD = \ - 'Overloaded method has both abstract and non-abstract variants' -READ_ONLY_PROPERTY_OVERRIDES_READ_WRITE = \ - 'Read-only property cannot override read-write property' -INSTANCE_LAYOUT_CONFLICT = 'Instance layout conflict in multiple inheritance' -FORMAT_REQUIRES_MAPPING = 'Format requires a mapping' -GENERIC_TYPE_NOT_VALID_AS_EXPRESSION = \ - "Generic type is prohibited as a runtime expression (use a type alias or '# type:' comment)" -RETURN_TYPE_CANNOT_BE_CONTRAVARIANT = "Cannot use a contravariant type variable as return type" -FUNCTION_PARAMETER_CANNOT_BE_COVARIANT = "Cannot use a covariant type variable as a parameter" -INCOMPATIBLE_IMPORT_OF = "Incompatible import of" -FUNCTION_TYPE_EXPECTED = "Function is missing a type annotation" -RETURN_TYPE_EXPECTED = "Function is missing a return type annotation" -ARGUMENT_TYPE_EXPECTED = "Function is missing a type annotation for one or more arguments" -KEYWORD_ARGUMENT_REQUIRES_STR_KEY_TYPE = \ - 'Keyword argument only valid with "str" key type in call to "dict"' -ALL_MUST_BE_SEQ_STR = 'Type of __all__ must be {}, not {}' - - -class MessageBuilder: - """Helper class for reporting type checker error messages with parameters. - - The methods of this class need to be provided with the context within a - file; the errors member manages the wider context. - - IDEA: Support a 'verbose mode' that includes full information about types - in error messages and that may otherwise produce more detailed error - messages. - """ - - # Report errors using this instance. It knows about the current file and - # import context. - errors = None # type: Errors - - modules = None # type: Dict[str, MypyFile] - - # Number of times errors have been disabled. - disable_count = 0 - - # Hack to deduplicate error messages from union types - disable_type_names = 0 - - def __init__(self, errors: Errors, modules: Dict[str, MypyFile]) -> None: - self.errors = errors - self.modules = modules - self.disable_count = 0 - self.disable_type_names = 0 - - # - # Helpers - # - - def copy(self) -> 'MessageBuilder': - new = MessageBuilder(self.errors.copy(), self.modules) - new.disable_count = self.disable_count - new.disable_type_names = self.disable_type_names - return new - - def add_errors(self, messages: 'MessageBuilder') -> None: - """Add errors in messages to this builder.""" - if self.disable_count <= 0: - for info in messages.errors.error_info: - self.errors.add_error_info(info) - - def disable_errors(self) -> None: - self.disable_count += 1 - - def enable_errors(self) -> None: - self.disable_count -= 1 - - def is_errors(self) -> bool: - return self.errors.is_errors() - - def report(self, msg: str, context: Context, severity: str, file: str = None) -> None: - """Report an error or note (unless disabled).""" - if self.disable_count <= 0: - self.errors.report(context.get_line() if context else -1, - msg.strip(), severity=severity, file=file) - - def fail(self, msg: str, context: Context, file: str = None) -> None: - """Report an error message (unless disabled).""" - # import pdb; pdb.set_trace() - self.report(msg, context, 'error', file=file) - - def note(self, msg: str, context: Context, file: str = None) -> None: - """Report an error message (unless disabled).""" - self.report(msg, context, 'note', file=file) - - def format(self, typ: Type, verbosity: int = 0) -> str: - """Convert a type to a relatively short string that is suitable for error messages. - - Mostly behave like format_simple below, but never return an empty string. - """ - s = self.format_simple(typ, verbosity) - if s != '': - # If format_simple returns a non-trivial result, use that. - return s - elif isinstance(typ, FunctionLike): - func = typ - if func.is_type_obj(): - # The type of a type object type can be derived from the - # return type (this always works). - itype = cast(Instance, func.items()[0].ret_type) - result = self.format(itype) - if verbosity >= 1: - # In some contexts we want to be explicit about the distinction - # between type X and the type of type object X. - result += ' (type object)' - return result - elif isinstance(func, CallableType): - return_type = strip_quotes(self.format(func.ret_type)) - if func.is_ellipsis_args: - return 'Callable[..., {}]'.format(return_type) - arg_types = [strip_quotes(self.format(t)) for t in func.arg_types] - return 'Callable[[{}], {}]'.format(", ".join(arg_types), return_type) - else: - # Use a simple representation for function types; proper - # function types may result in long and difficult-to-read - # error messages. - return 'overloaded function' - else: - # Default case; we simply have to return something meaningful here. - return 'object' - - def format_simple(self, typ: Type, verbosity: int = 0) -> str: - """Convert simple types to string that is suitable for error messages. - - Return "" for complex types. Try to keep the length of the result - relatively short to avoid overly long error messages. - - Examples: - builtins.int -> 'int' - Any type -> 'Any' - None -> None - callable type -> "" (empty string) - """ - if isinstance(typ, Instance): - itype = typ - # Get the short name of the type. - if verbosity >= 2: - base_str = itype.type.fullname() - else: - base_str = itype.type.name() - if itype.args == []: - # No type arguments. Place the type name in quotes to avoid - # potential for confusion: otherwise, the type name could be - # interpreted as a normal word. - return '"{}"'.format(base_str) - elif itype.type.fullname() == 'builtins.tuple': - item_type = strip_quotes(self.format(itype.args[0])) - return 'Tuple[{}, ...]'.format(item_type) - elif itype.type.fullname() in reverse_type_aliases: - alias = reverse_type_aliases[itype.type.fullname()] - alias = alias.split('.')[-1] - items = [strip_quotes(self.format(arg)) for arg in itype.args] - return '{}[{}]'.format(alias, ', '.join(items)) - else: - # There are type arguments. Convert the arguments to strings - # (using format() instead of format_simple() to avoid empty - # strings). If the result is too long, replace arguments - # with [...]. - a = [] # type: List[str] - for arg in itype.args: - a.append(strip_quotes(self.format(arg))) - s = ', '.join(a) - if len((base_str + s)) < 25: - return '{}[{}]'.format(base_str, s) - else: - return '{}[...]'.format(base_str) - elif isinstance(typ, TypeVarType): - # This is similar to non-generic instance types. - return '"{}"'.format(typ.name) - elif isinstance(typ, TupleType): - # Prefer the name of the fallback class (if not tuple), as it's more informative. - if typ.fallback.type.fullname() != 'builtins.tuple': - return self.format_simple(typ.fallback) - items = [] - for t in typ.items: - items.append(strip_quotes(self.format(t))) - s = '"Tuple[{}]"'.format(', '.join(items)) - if len(s) < 40: - return s - else: - return 'tuple(length {})'.format(len(items)) - elif isinstance(typ, UnionType): - # Only print Unions as Optionals if the Optional wouldn't have to contain another Union - print_as_optional = (len(typ.items) - - sum(isinstance(t, NoneTyp) for t in typ.items) == 1) - if print_as_optional: - rest = [t for t in typ.items if not isinstance(t, NoneTyp)] - return '"Optional[{}]"'.format(strip_quotes(self.format(rest[0]))) - else: - items = [] - for t in typ.items: - items.append(strip_quotes(self.format(t))) - s = '"Union[{}]"'.format(', '.join(items)) - if len(s) < 40: - return s - else: - return 'union type ({} items)'.format(len(items)) - elif isinstance(typ, Void): - return 'None' - elif isinstance(typ, NoneTyp): - return 'None' - elif isinstance(typ, AnyType): - return '"Any"' - elif isinstance(typ, DeletedType): - return '' - elif isinstance(typ, TypeType): - return 'Type[{}]'.format( - strip_quotes(self.format_simple(typ.item, verbosity))) - elif typ is None: - raise RuntimeError('Type is None') - else: - # No simple representation for this type that would convey very - # useful information. No need to mention the type explicitly in a - # message. - return '' - - def format_distinctly(self, type1: Type, type2: Type) -> Tuple[str, str]: - """Jointly format a pair of types to distinct strings. - - Increase the verbosity of the type strings until they become distinct. - """ - verbosity = 0 - for verbosity in range(3): - str1 = self.format(type1, verbosity=verbosity) - str2 = self.format(type2, verbosity=verbosity) - if str1 != str2: - return (str1, str2) - return (str1, str2) - - # - # Specific operations - # - - # The following operations are for generating specific error messages. They - # get some information as arguments, and they build an error message based - # on them. - - def has_no_attr(self, typ: Type, member: str, context: Context) -> Type: - """Report a missing or non-accessible member. - - The type argument is the base type. If member corresponds to - an operator, use the corresponding operator name in the - messages. Return type Any. - """ - if (isinstance(typ, Instance) and - typ.type.has_readable_member(member)): - self.fail('Member "{}" is not assignable'.format(member), context) - elif isinstance(typ, Void): - self.check_void(typ, context) - elif member == '__contains__': - self.fail('Unsupported right operand type for in ({})'.format( - self.format(typ)), context) - elif member in op_methods.values(): - # Access to a binary operator member (e.g. _add). This case does - # not handle indexing operations. - for op, method in op_methods.items(): - if method == member: - self.unsupported_left_operand(op, typ, context) - break - elif member == '__neg__': - self.fail('Unsupported operand type for unary - ({})'.format( - self.format(typ)), context) - elif member == '__pos__': - self.fail('Unsupported operand type for unary + ({})'.format( - self.format(typ)), context) - elif member == '__invert__': - self.fail('Unsupported operand type for ~ ({})'.format( - self.format(typ)), context) - elif member == '__getitem__': - # Indexed get. - self.fail('Value of type {} is not indexable'.format( - self.format(typ)), context) - elif member == '__setitem__': - # Indexed set. - self.fail('Unsupported target for indexed assignment', context) - elif member == '__call__': - self.fail('{} not callable'.format(self.format(typ)), context) - else: - # The non-special case: a missing ordinary attribute. - if not self.disable_type_names: - failed = False - if isinstance(typ, Instance) and typ.type.names: - alternatives = set(typ.type.names.keys()) - matches = [m for m in COMMON_MISTAKES.get(member, []) if m in alternatives] - matches.extend(best_matches(member, alternatives)[:3]) - if matches: - self.fail('{} has no attribute "{}"; maybe {}?'.format( - self.format(typ), member, pretty_or(matches)), context) - failed = True - if not failed: - self.fail('{} has no attribute "{}"'.format(self.format(typ), - member), context) - else: - self.fail('Some element of union has no attribute "{}"'.format( - member), context) - return AnyType() - - def unsupported_operand_types(self, op: str, left_type: Any, - right_type: Any, context: Context) -> None: - """Report unsupported operand types for a binary operation. - - Types can be Type objects or strings. - """ - if isinstance(left_type, Void) or isinstance(right_type, Void): - self.check_void(left_type, context) - self.check_void(right_type, context) - return - left_str = '' - if isinstance(left_type, str): - left_str = left_type - else: - left_str = self.format(left_type) - - right_str = '' - if isinstance(right_type, str): - right_str = right_type - else: - right_str = self.format(right_type) - - if self.disable_type_names: - msg = 'Unsupported operand types for {} (likely involving Union)'.format(op) - else: - msg = 'Unsupported operand types for {} ({} and {})'.format( - op, left_str, right_str) - self.fail(msg, context) - - def unsupported_left_operand(self, op: str, typ: Type, - context: Context) -> None: - if not self.check_void(typ, context): - if self.disable_type_names: - msg = 'Unsupported left operand type for {} (some union)'.format(op) - else: - msg = 'Unsupported left operand type for {} ({})'.format( - op, self.format(typ)) - self.fail(msg, context) - - def type_expected_as_right_operand_of_is(self, context: Context) -> None: - self.fail('Type expected as right operand of "is"', context) - - def not_callable(self, typ: Type, context: Context) -> Type: - self.fail('{} not callable'.format(self.format(typ)), context) - return AnyType() - - def untyped_function_call(self, callee: CallableType, context: Context) -> Type: - name = callee.name if callee.name is not None else '(unknown)' - self.fail('call to untyped function {} in typed context'.format(name), context) - return AnyType() - - def incompatible_argument(self, n: int, m: int, callee: CallableType, arg_type: Type, - arg_kind: int, context: Context) -> None: - """Report an error about an incompatible argument type. - - The argument type is arg_type, argument number is n and the - callee type is 'callee'. If the callee represents a method - that corresponds to an operator, use the corresponding - operator name in the messages. - """ - target = '' - if callee.name: - name = callee.name - base = extract_type(name) - - for op, method in op_methods.items(): - for variant in method, '__r' + method[2:]: - if name.startswith('"{}" of'.format(variant)): - if op == 'in' or variant != method: - # Reversed order of base/argument. - self.unsupported_operand_types(op, arg_type, base, - context) - else: - self.unsupported_operand_types(op, base, arg_type, - context) - return - - if name.startswith('"__getitem__" of'): - self.invalid_index_type(arg_type, base, context) - return - - if name.startswith('"__setitem__" of'): - if n == 1: - self.invalid_index_type(arg_type, base, context) - else: - msg = '{} (expression has type {}, target has type {})' - arg_type_str, callee_type_str = self.format_distinctly(arg_type, - callee.arg_types[n - 1]) - self.fail(msg.format(INCOMPATIBLE_TYPES_IN_ASSIGNMENT, - arg_type_str, callee_type_str), - context) - return - - target = 'to {} '.format(name) - - msg = '' - if callee.name == '': - name = callee.name[1:-1] - n -= 1 - msg = '{} item {} has incompatible type {}'.format( - name[0].upper() + name[1:], n, self.format_simple(arg_type)) - elif callee.name == '': - msg = 'List comprehension has incompatible type List[{}]'.format( - strip_quotes(self.format(arg_type))) - elif callee.name == '': - msg = 'Set comprehension has incompatible type Set[{}]'.format( - strip_quotes(self.format(arg_type))) - elif callee.name == '': - msg = ('{} expression in dictionary comprehension has incompatible type {}; ' - 'expected type {}').format( - 'Key' if n == 1 else 'Value', - self.format(arg_type), - self.format(callee.arg_types[n - 1])) - elif callee.name == '': - msg = 'Generator has incompatible item type {}'.format( - self.format_simple(arg_type)) - else: - try: - expected_type = callee.arg_types[m - 1] - except IndexError: # Varargs callees - expected_type = callee.arg_types[-1] - arg_type_str, expected_type_str = self.format_distinctly(arg_type, expected_type) - #import pdb; pdb.set_trace() - if arg_kind == ARG_STAR: - arg_type_str = '*' + arg_type_str - elif arg_kind == ARG_STAR2: - arg_type_str = '**' + arg_type_str - msg = 'Argument {} {}has incompatible type {}; expected {}'.format( - n, target, arg_type_str, expected_type_str) - self.fail(msg, context) - - def invalid_index_type(self, index_type: Type, base_str: str, - context: Context) -> None: - self.fail('Invalid index type {} for {}'.format( - self.format(index_type), base_str), context) - - def too_few_arguments(self, callee: CallableType, context: Context, - argument_names: List[str]) -> None: - if (argument_names is not None and not all(k is None for k in argument_names) - and len(argument_names) >= 1): - diff = [k for k in callee.arg_names if k not in argument_names] - if len(diff) == 1: - msg = 'Missing positional argument' - else: - msg = 'Missing positional arguments' - if callee.name and diff: - msg += ' "{}" in call to {}'.format('", "'.join(diff), callee.name) - else: - msg = 'Too few arguments' - if callee.name: - msg += ' for {}'.format(callee.name) - self.fail(msg, context) - - def too_many_arguments(self, callee: CallableType, context: Context) -> None: - msg = 'Too many arguments' - if callee.name: - msg += ' for {}'.format(callee.name) - self.fail(msg, context) - - def too_many_positional_arguments(self, callee: CallableType, - context: Context) -> None: - msg = 'Too many positional arguments' - if callee.name: - msg += ' for {}'.format(callee.name) - self.fail(msg, context) - - def unexpected_keyword_argument(self, callee: CallableType, name: str, - context: Context) -> None: - msg = 'Unexpected keyword argument "{}"'.format(name) - if callee.name: - msg += ' for {}'.format(callee.name) - self.fail(msg, context) - if callee.definition: - fullname = callee.definition.fullname() - if fullname is not None and '.' in fullname: - module_name = fullname.rsplit('.', 1)[0] - path = self.modules[module_name].path - self.note('{} defined here'.format(callee.name), callee.definition, file=path) - - def duplicate_argument_value(self, callee: CallableType, index: int, - context: Context) -> None: - self.fail('{} gets multiple values for keyword argument "{}"'. - format(capitalize(callable_name(callee)), - callee.arg_names[index]), context) - - def does_not_return_value(self, void_type: Type, context: Context) -> None: - """Report an error about a void type in a non-void context. - - The first argument must be a void type. If the void type has a - source in it, report it in the error message. This allows - giving messages such as 'Foo does not return a value'. - """ - if (cast(Void, void_type)).source is None: - self.fail('Function does not return a value', context) - else: - self.fail('{} does not return a value'.format( - capitalize((cast(Void, void_type)).source)), context) - - def deleted_as_rvalue(self, typ: DeletedType, context: Context) -> None: - """Report an error about using an deleted type as an rvalue.""" - if typ.source is None: - s = "" - else: - s = " '{}'".format(typ.source) - self.fail('Trying to read deleted variable{}'.format(s), context) - - def deleted_as_lvalue(self, typ: DeletedType, context: Context) -> None: - """Report an error about using an deleted type as an lvalue. - - Currently, this only occurs when trying to assign to an - exception variable outside the local except: blocks. - """ - if typ.source is None: - s = "" - else: - s = " '{}'".format(typ.source) - self.fail('Assignment to variable{} outside except: block'.format(s), context) - - def no_variant_matches_arguments(self, overload: Overloaded, arg_types: List[Type], - context: Context) -> None: - if overload.name(): - self.fail('No overload variant of {} matches argument types {}' - .format(overload.name(), arg_types), context) - else: - self.fail('No overload variant matches argument types {}'.format(arg_types), context) - - def function_variants_overlap(self, n1: int, n2: int, - context: Context) -> None: - self.fail('Function signature variants {} and {} overlap'.format( - n1 + 1, n2 + 1), context) - - def invalid_cast(self, target_type: Type, source_type: Type, - context: Context) -> None: - if not self.check_void(source_type, context): - self.fail('Cannot cast from {} to {}'.format( - self.format(source_type), self.format(target_type)), context) - - def wrong_number_values_to_unpack(self, provided: int, expected: int, - context: Context) -> None: - if provided < expected: - if provided == 1: - self.fail('Need more than 1 value to unpack ({} expected)'.format(expected), - context) - else: - self.fail('Need more than {} values to unpack ({} expected)'.format( - provided, expected), context) - elif provided > expected: - self.fail('Too many values to unpack ({} expected, {} provided)'.format( - expected, provided), context) - - def type_not_iterable(self, type: Type, context: Context) -> None: - self.fail('\'{}\' object is not iterable'.format(type), context) - - def incompatible_operator_assignment(self, op: str, - context: Context) -> None: - self.fail('Result type of {} incompatible in assignment'.format(op), - context) - - def incompatible_value_count_in_assignment(self, lvalue_count: int, - rvalue_count: int, - context: Context) -> None: - if rvalue_count < lvalue_count: - self.fail('Need {} values to assign'.format(lvalue_count), context) - elif rvalue_count > lvalue_count: - self.fail('Too many values to assign', context) - - def type_incompatible_with_supertype(self, name: str, supertype: TypeInfo, - context: Context) -> None: - self.fail('Type of "{}" incompatible with supertype "{}"'.format( - name, supertype.name), context) - - def signature_incompatible_with_supertype( - self, name: str, name_in_super: str, supertype: str, - context: Context) -> None: - target = self.override_target(name, name_in_super, supertype) - self.fail('Signature of "{}" incompatible with {}'.format( - name, target), context) - - def argument_incompatible_with_supertype( - self, arg_num: int, name: str, name_in_supertype: str, - supertype: str, context: Context) -> None: - target = self.override_target(name, name_in_supertype, supertype) - self.fail('Argument {} of "{}" incompatible with {}' - .format(arg_num, name, target), context) - - def return_type_incompatible_with_supertype( - self, name: str, name_in_supertype: str, supertype: str, - context: Context) -> None: - target = self.override_target(name, name_in_supertype, supertype) - self.fail('Return type of "{}" incompatible with {}' - .format(name, target), context) - - def override_target(self, name: str, name_in_super: str, - supertype: str) -> str: - target = 'supertype "{}"'.format(supertype) - if name_in_super != name: - target = '"{}" of {}'.format(name_in_super, target) - return target - - def boolean_return_value_expected(self, method: str, - context: Context) -> None: - self.fail('Boolean return value expected for method "{}"'.format( - method), context) - - def incompatible_type_application(self, expected_arg_count: int, - actual_arg_count: int, - context: Context) -> None: - if expected_arg_count == 0: - self.fail('Type application targets a non-generic function', - context) - elif actual_arg_count > expected_arg_count: - self.fail('Type application has too many types ({} expected)' - .format(expected_arg_count), context) - else: - self.fail('Type application has too few types ({} expected)' - .format(expected_arg_count), context) - - def incompatible_array_item_type(self, typ: Type, index: int, - context: Context) -> None: - self.fail('Array item {} has incompatible type {}'.format( - index, self.format(typ)), context) - - def could_not_infer_type_arguments(self, callee_type: CallableType, n: int, - context: Context) -> None: - if callee_type.name and n > 0: - self.fail('Cannot infer type argument {} of {}'.format( - n, callee_type.name), context) - else: - self.fail('Cannot infer function type argument', context) - - def invalid_var_arg(self, typ: Type, context: Context) -> None: - self.fail('List or tuple expected as variable arguments', context) - - def invalid_keyword_var_arg(self, typ: Type, context: Context) -> None: - if isinstance(typ, Instance) and (typ.type.fullname() == 'builtins.dict'): - self.fail('Keywords must be strings', context) - else: - self.fail('Argument after ** must be a dictionary', - context) - - def incomplete_type_var_match(self, member: str, context: Context) -> None: - self.fail('"{}" has incomplete match to supertype type variable' - .format(member), context) - - def not_implemented(self, msg: str, context: Context) -> Type: - self.fail('Feature not implemented yet ({})'.format(msg), context) - return AnyType() - - def undefined_in_superclass(self, member: str, context: Context) -> None: - self.fail('"{}" undefined in superclass'.format(member), context) - - def check_void(self, typ: Type, context: Context) -> bool: - """If type is void, report an error such as '.. does not - return a value' and return True. Otherwise, return False. - """ - if isinstance(typ, Void): - self.does_not_return_value(typ, context) - return True - else: - return False - - def too_few_string_formatting_arguments(self, context: Context) -> None: - self.fail('Not enough arguments for format string', context) - - def too_many_string_formatting_arguments(self, context: Context) -> None: - self.fail('Not all arguments converted during string formatting', context) - - def incomplete_conversion_specifier_format(self, context: Context) -> None: - self.fail('Incomplete format', context) - - def unsupported_placeholder(self, placeholder: str, context: Context) -> None: - self.fail('Unsupported format character \'%s\'' % placeholder, context) - - def string_interpolation_with_star_and_key(self, context: Context) -> None: - self.fail('String interpolation contains both stars and mapping keys', context) - - def requires_int_or_char(self, context: Context) -> None: - self.fail('%c requires int or char', context) - - def key_not_in_mapping(self, key: str, context: Context) -> None: - self.fail('Key \'%s\' not found in mapping' % key, context) - - def string_interpolation_mixing_key_and_non_keys(self, context: Context) -> None: - self.fail('String interpolation mixes specifier with and without mapping keys', context) - - def cannot_determine_type(self, name: str, context: Context) -> None: - self.fail("Cannot determine type of '%s'" % name, context) - - def cannot_determine_type_in_base(self, name: str, base: str, context: Context) -> None: - self.fail("Cannot determine type of '%s' in base class '%s'" % (name, base), context) - - def invalid_method_type(self, sig: CallableType, context: Context) -> None: - self.fail('Invalid method type', context) - - def invalid_class_method_type(self, sig: CallableType, context: Context) -> None: - self.fail('Invalid class method type', context) - - def incompatible_conditional_function_def(self, defn: FuncDef) -> None: - self.fail('All conditional function variants must have identical ' - 'signatures', defn) - - def cannot_instantiate_abstract_class(self, class_name: str, - abstract_attributes: List[str], - context: Context) -> None: - attrs = format_string_list("'%s'" % a for a in abstract_attributes[:5]) - self.fail("Cannot instantiate abstract class '%s' with abstract " - "attribute%s %s" % (class_name, plural_s(abstract_attributes), - attrs), - context) - - def base_class_definitions_incompatible(self, name: str, base1: TypeInfo, - base2: TypeInfo, - context: Context) -> None: - self.fail('Definition of "{}" in base class "{}" is incompatible ' - 'with definition in base class "{}"'.format( - name, base1.name(), base2.name()), context) - - def cant_assign_to_method(self, context: Context) -> None: - self.fail(CANNOT_ASSIGN_TO_METHOD, context) - - def read_only_property(self, name: str, type: TypeInfo, - context: Context) -> None: - self.fail('Property "{}" defined in "{}" is read-only'.format( - name, type.name()), context) - - def incompatible_typevar_value(self, callee: CallableType, index: int, - type: Type, context: Context) -> None: - self.fail('Type argument {} of {} has incompatible value {}'.format( - index, callable_name(callee), self.format(type)), context) - - def overloaded_signatures_overlap(self, index1: int, index2: int, - context: Context) -> None: - self.fail('Overloaded function signatures {} and {} overlap with ' - 'incompatible return types'.format(index1, index2), context) - - def invalid_reverse_operator_signature(self, reverse: str, other: str, - context: Context) -> None: - self.fail('"Any" return type expected since argument to {} ' - 'does not support {}'.format(reverse, other), context) - - def reverse_operator_method_with_any_arg_must_return_any( - self, method: str, context: Context) -> None: - self.fail('"Any" return type expected since argument to {} ' - 'has type "Any"'.format(method), context) - - def operator_method_signatures_overlap( - self, reverse_class: str, reverse_method: str, forward_class: str, - forward_method: str, context: Context) -> None: - self.fail('Signatures of "{}" of "{}" and "{}" of "{}" ' - 'are unsafely overlapping'.format( - reverse_method, reverse_class, - forward_method, forward_class), - context) - - def forward_operator_not_callable( - self, forward_method: str, context: Context) -> None: - self.fail('Forward operator "{}" is not callable'.format( - forward_method), context) - - def signatures_incompatible(self, method: str, other_method: str, - context: Context) -> None: - self.fail('Signatures of "{}" and "{}" are incompatible'.format( - method, other_method), context) - - def yield_from_invalid_operand_type(self, expr: Type, context: Context) -> Type: - text = self.format(expr) if self.format(expr) != 'object' else expr - self.fail('"yield from" can\'t be applied to {}'.format(text), context) - return AnyType() - - def invalid_signature(self, func_type: Type, context: Context) -> None: - self.fail('Invalid signature "{}"'.format(func_type), context) - - def reveal_type(self, typ: Type, context: Context) -> None: - self.fail('Revealed type is \'{}\''.format(typ), context) - - def unsupported_type_type(self, item: Type, context: Context) -> None: - self.fail('Unsupported type Type[{}]'.format(self.format(item)), context) - - def redundant_cast(self, typ: Type, context: Context) -> None: - self.note('Redundant cast to {}'.format(self.format(typ)), context) - - -def capitalize(s: str) -> str: - """Capitalize the first character of a string.""" - if s == '': - return '' - else: - return s[0].upper() + s[1:] - - -def extract_type(name: str) -> str: - """If the argument is the name of a method (of form C.m), return - the type portion in quotes (e.g. "y"). Otherwise, return the string - unmodified. - """ - name = re.sub('^"[a-zA-Z0-9_]+" of ', '', name) - return name - - -def strip_quotes(s: str) -> str: - """Strip a double quote at the beginning and end of the string, if any.""" - s = re.sub('^"', '', s) - s = re.sub('"$', '', s) - return s - - -def plural_s(s: Sequence[Any]) -> str: - if len(s) > 1: - return 's' - else: - return '' - - -def format_string_list(s: Iterable[str]) -> str: - l = list(s) - assert len(l) > 0 - if len(l) == 1: - return l[0] - else: - return '%s and %s' % (', '.join(l[:-1]), l[-1]) - - -def callable_name(type: CallableType) -> str: - if type.name: - return type.name - else: - return 'function' - - -def temp_message_builder() -> MessageBuilder: - """Return a message builder usable for throwaway errors (which may not format properly).""" - return MessageBuilder(Errors(), {}) - - -# For hard-coding suggested missing member alternatives. -COMMON_MISTAKES = { - 'add': ('append', 'extend'), -} # type: Dict[str, Sequence[str]] - - -def best_matches(current: str, options: Iterable[str]) -> List[str]: - ratios = {v: difflib.SequenceMatcher(a=current, b=v).ratio() for v in options} - return sorted((o for o in options if ratios[o] > 0.75), - reverse=True, key=lambda v: (ratios[v], v)) - - -def pretty_or(args: List[str]) -> str: - quoted = ['"' + a + '"' for a in args] - if len(quoted) == 1: - return quoted[0] - if len(quoted) == 2: - return "{} or {}".format(quoted[0], quoted[1]) - return ", ".join(quoted[:-1]) + ", or " + quoted[-1] +"""Facilities and constants for generating error messages during type checking. + +The type checker itself does not deal with message string literals to +improve code clarity and to simplify localization (in the future).""" + +import re +import difflib + +from typing import cast, List, Dict, Any, Sequence, Iterable, Tuple + +from mypy.errors import Errors +from mypy.types import ( + Type, CallableType, Instance, TypeVarType, TupleType, UnionType, Void, NoneTyp, AnyType, + Overloaded, FunctionLike, DeletedType, TypeType +) +from mypy.nodes import ( + TypeInfo, Context, MypyFile, op_methods, FuncDef, reverse_type_aliases, + ARG_STAR, ARG_STAR2 +) + + +# Constants that represent simple type checker error message, i.e. messages +# that do not have any parameters. + +NO_RETURN_VALUE_EXPECTED = 'No return value expected' +INCOMPATIBLE_RETURN_VALUE_TYPE = 'Incompatible return value type' +RETURN_VALUE_EXPECTED = 'Return value expected' +BOOLEAN_VALUE_EXPECTED = 'Boolean value expected' +BOOLEAN_EXPECTED_FOR_IF = 'Boolean value expected for if condition' +BOOLEAN_EXPECTED_FOR_WHILE = 'Boolean value expected for while condition' +BOOLEAN_EXPECTED_FOR_UNTIL = 'Boolean value expected for until condition' +BOOLEAN_EXPECTED_FOR_NOT = 'Boolean value expected for not operand' +INVALID_EXCEPTION = 'Exception must be derived from BaseException' +INVALID_EXCEPTION_TYPE = 'Exception type must be derived from BaseException' +INVALID_RETURN_TYPE_FOR_GENERATOR = \ + 'The return type of a generator function should be "Generator" or one of its supertypes' +INVALID_GENERATOR_RETURN_ITEM_TYPE = \ + 'The return type of a generator function must be None in its third type parameter in Python 2' +YIELD_VALUE_EXPECTED = 'Yield value expected' +INCOMPATIBLE_TYPES = 'Incompatible types' +INCOMPATIBLE_TYPES_IN_ASSIGNMENT = 'Incompatible types in assignment' +INCOMPATIBLE_REDEFINITION = 'Incompatible redefinition' +INCOMPATIBLE_TYPES_IN_YIELD = 'Incompatible types in yield' +INCOMPATIBLE_TYPES_IN_YIELD_FROM = 'Incompatible types in "yield from"' +INCOMPATIBLE_TYPES_IN_STR_INTERPOLATION = 'Incompatible types in string interpolation' +INIT_MUST_HAVE_NONE_RETURN_TYPE = 'The return type of "__init__" must be None' +GETTER_TYPE_INCOMPATIBLE_WITH_SETTER = \ + 'Type of getter incompatible with setter' +TUPLE_INDEX_MUST_BE_AN_INT_LITERAL = 'Tuple index must be an integer literal' +TUPLE_SLICE_MUST_BE_AN_INT_LITERAL = 'Tuple slice must be an integer literal' +TUPLE_INDEX_OUT_OF_RANGE = 'Tuple index out of range' +TYPE_CONSTANT_EXPECTED = 'Type "Constant" or initializer expected' +INCOMPATIBLE_PAIR_ITEM_TYPE = 'Incompatible Pair item type' +INVALID_TYPE_APPLICATION_TARGET_TYPE = 'Invalid type application target type' +INCOMPATIBLE_TUPLE_ITEM_TYPE = 'Incompatible tuple item type' +INCOMPATIBLE_KEY_TYPE = 'Incompatible dictionary key type' +INCOMPATIBLE_VALUE_TYPE = 'Incompatible dictionary value type' +NEED_ANNOTATION_FOR_VAR = 'Need type annotation for variable' +ITERABLE_EXPECTED = 'Iterable expected' +INCOMPATIBLE_TYPES_IN_FOR = 'Incompatible types in for statement' +INCOMPATIBLE_ARRAY_VAR_ARGS = 'Incompatible variable arguments in call' +INVALID_SLICE_INDEX = 'Slice index must be an integer or None' +CANNOT_INFER_LAMBDA_TYPE = 'Cannot infer type of lambda' +CANNOT_INFER_ITEM_TYPE = 'Cannot infer iterable item type' +CANNOT_ACCESS_INIT = 'Cannot access "__init__" directly' +CANNOT_ASSIGN_TO_METHOD = 'Cannot assign to a method' +CANNOT_ASSIGN_TO_TYPE = 'Cannot assign to a type' +INCONSISTENT_ABSTRACT_OVERLOAD = \ + 'Overloaded method has both abstract and non-abstract variants' +READ_ONLY_PROPERTY_OVERRIDES_READ_WRITE = \ + 'Read-only property cannot override read-write property' +INSTANCE_LAYOUT_CONFLICT = 'Instance layout conflict in multiple inheritance' +FORMAT_REQUIRES_MAPPING = 'Format requires a mapping' +GENERIC_TYPE_NOT_VALID_AS_EXPRESSION = \ + "Generic type is prohibited as a runtime expression (use a type alias or '# type:' comment)" +RETURN_TYPE_CANNOT_BE_CONTRAVARIANT = "Cannot use a contravariant type variable as return type" +FUNCTION_PARAMETER_CANNOT_BE_COVARIANT = "Cannot use a covariant type variable as a parameter" +INCOMPATIBLE_IMPORT_OF = "Incompatible import of" +FUNCTION_TYPE_EXPECTED = "Function is missing a type annotation" +RETURN_TYPE_EXPECTED = "Function is missing a return type annotation" +ARGUMENT_TYPE_EXPECTED = "Function is missing a type annotation for one or more arguments" +KEYWORD_ARGUMENT_REQUIRES_STR_KEY_TYPE = \ + 'Keyword argument only valid with "str" key type in call to "dict"' +ALL_MUST_BE_SEQ_STR = 'Type of __all__ must be {}, not {}' + + +class MessageBuilder: + """Helper class for reporting type checker error messages with parameters. + + The methods of this class need to be provided with the context within a + file; the errors member manages the wider context. + + IDEA: Support a 'verbose mode' that includes full information about types + in error messages and that may otherwise produce more detailed error + messages. + """ + + # Report errors using this instance. It knows about the current file and + # import context. + errors = None # type: Errors + + modules = None # type: Dict[str, MypyFile] + + # Number of times errors have been disabled. + disable_count = 0 + + # Hack to deduplicate error messages from union types + disable_type_names = 0 + + def __init__(self, errors: Errors, modules: Dict[str, MypyFile]) -> None: + self.errors = errors + self.modules = modules + self.disable_count = 0 + self.disable_type_names = 0 + + # + # Helpers + # + + def copy(self) -> 'MessageBuilder': + new = MessageBuilder(self.errors.copy(), self.modules) + new.disable_count = self.disable_count + new.disable_type_names = self.disable_type_names + return new + + def add_errors(self, messages: 'MessageBuilder') -> None: + """Add errors in messages to this builder.""" + if self.disable_count <= 0: + for info in messages.errors.error_info: + self.errors.add_error_info(info) + + def disable_errors(self) -> None: + self.disable_count += 1 + + def enable_errors(self) -> None: + self.disable_count -= 1 + + def is_errors(self) -> bool: + return self.errors.is_errors() + + def report(self, msg: str, context: Context, severity: str, file: str = None) -> None: + """Report an error or note (unless disabled).""" + if self.disable_count <= 0: + self.errors.report(context.get_line() if context else -1, + msg.strip(), severity=severity, file=file) + + def fail(self, msg: str, context: Context, file: str = None) -> None: + """Report an error message (unless disabled).""" + # import pdb; pdb.set_trace() + self.report(msg, context, 'error', file=file) + + def note(self, msg: str, context: Context, file: str = None) -> None: + """Report an error message (unless disabled).""" + self.report(msg, context, 'note', file=file) + + def format(self, typ: Type, verbosity: int = 0) -> str: + """Convert a type to a relatively short string that is suitable for error messages. + + Mostly behave like format_simple below, but never return an empty string. + """ + s = self.format_simple(typ, verbosity) + if s != '': + # If format_simple returns a non-trivial result, use that. + return s + elif isinstance(typ, FunctionLike): + func = typ + if func.is_type_obj(): + # The type of a type object type can be derived from the + # return type (this always works). + itype = cast(Instance, func.items()[0].ret_type) + result = self.format(itype) + if verbosity >= 1: + # In some contexts we want to be explicit about the distinction + # between type X and the type of type object X. + result += ' (type object)' + return result + elif isinstance(func, CallableType): + return_type = strip_quotes(self.format(func.ret_type)) + if func.is_ellipsis_args: + return 'Callable[..., {}]'.format(return_type) + arg_types = [strip_quotes(self.format(t)) for t in func.arg_types] + return 'Callable[[{}], {}]'.format(", ".join(arg_types), return_type) + else: + # Use a simple representation for function types; proper + # function types may result in long and difficult-to-read + # error messages. + return 'overloaded function' + else: + # Default case; we simply have to return something meaningful here. + return 'object' + + def format_simple(self, typ: Type, verbosity: int = 0) -> str: + """Convert simple types to string that is suitable for error messages. + + Return "" for complex types. Try to keep the length of the result + relatively short to avoid overly long error messages. + + Examples: + builtins.int -> 'int' + Any type -> 'Any' + None -> None + callable type -> "" (empty string) + """ + if isinstance(typ, Instance): + itype = typ + # Get the short name of the type. + if verbosity >= 2: + base_str = itype.type.fullname() + else: + base_str = itype.type.name() + if itype.args == []: + # No type arguments. Place the type name in quotes to avoid + # potential for confusion: otherwise, the type name could be + # interpreted as a normal word. + return '"{}"'.format(base_str) + elif itype.type.fullname() == 'builtins.tuple': + item_type = strip_quotes(self.format(itype.args[0])) + return 'Tuple[{}, ...]'.format(item_type) + elif itype.type.fullname() in reverse_type_aliases: + alias = reverse_type_aliases[itype.type.fullname()] + alias = alias.split('.')[-1] + items = [strip_quotes(self.format(arg)) for arg in itype.args] + return '{}[{}]'.format(alias, ', '.join(items)) + else: + # There are type arguments. Convert the arguments to strings + # (using format() instead of format_simple() to avoid empty + # strings). If the result is too long, replace arguments + # with [...]. + a = [] # type: List[str] + for arg in itype.args: + a.append(strip_quotes(self.format(arg))) + s = ', '.join(a) + if len((base_str + s)) < 25: + return '{}[{}]'.format(base_str, s) + else: + return '{}[...]'.format(base_str) + elif isinstance(typ, TypeVarType): + # This is similar to non-generic instance types. + return '"{}"'.format(typ.name) + elif isinstance(typ, TupleType): + # Prefer the name of the fallback class (if not tuple), as it's more informative. + if typ.fallback.type.fullname() != 'builtins.tuple': + return self.format_simple(typ.fallback) + items = [] + for t in typ.items: + items.append(strip_quotes(self.format(t))) + s = '"Tuple[{}]"'.format(', '.join(items)) + if len(s) < 40: + return s + else: + return 'tuple(length {})'.format(len(items)) + elif isinstance(typ, UnionType): + # Only print Unions as Optionals if the Optional wouldn't have to contain another Union + print_as_optional = (len(typ.items) - + sum(isinstance(t, NoneTyp) for t in typ.items) == 1) + if print_as_optional: + rest = [t for t in typ.items if not isinstance(t, NoneTyp)] + return '"Optional[{}]"'.format(strip_quotes(self.format(rest[0]))) + else: + items = [] + for t in typ.items: + items.append(strip_quotes(self.format(t))) + s = '"Union[{}]"'.format(', '.join(items)) + if len(s) < 40: + return s + else: + return 'union type ({} items)'.format(len(items)) + elif isinstance(typ, Void): + return 'None' + elif isinstance(typ, NoneTyp): + return 'None' + elif isinstance(typ, AnyType): + return '"Any"' + elif isinstance(typ, DeletedType): + return '' + elif isinstance(typ, TypeType): + return 'Type[{}]'.format( + strip_quotes(self.format_simple(typ.item, verbosity))) + elif typ is None: + raise RuntimeError('Type is None') + else: + # No simple representation for this type that would convey very + # useful information. No need to mention the type explicitly in a + # message. + return '' + + def format_distinctly(self, type1: Type, type2: Type) -> Tuple[str, str]: + """Jointly format a pair of types to distinct strings. + + Increase the verbosity of the type strings until they become distinct. + """ + verbosity = 0 + for verbosity in range(3): + str1 = self.format(type1, verbosity=verbosity) + str2 = self.format(type2, verbosity=verbosity) + if str1 != str2: + return (str1, str2) + return (str1, str2) + + # + # Specific operations + # + + # The following operations are for generating specific error messages. They + # get some information as arguments, and they build an error message based + # on them. + + def has_no_attr(self, typ: Type, member: str, context: Context) -> Type: + """Report a missing or non-accessible member. + + The type argument is the base type. If member corresponds to + an operator, use the corresponding operator name in the + messages. Return type Any. + """ + if (isinstance(typ, Instance) and + typ.type.has_readable_member(member)): + self.fail('Member "{}" is not assignable'.format(member), context) + elif isinstance(typ, Void): + self.check_void(typ, context) + elif member == '__contains__': + self.fail('Unsupported right operand type for in ({})'.format( + self.format(typ)), context) + elif member in op_methods.values(): + # Access to a binary operator member (e.g. _add). This case does + # not handle indexing operations. + for op, method in op_methods.items(): + if method == member: + self.unsupported_left_operand(op, typ, context) + break + elif member == '__neg__': + self.fail('Unsupported operand type for unary - ({})'.format( + self.format(typ)), context) + elif member == '__pos__': + self.fail('Unsupported operand type for unary + ({})'.format( + self.format(typ)), context) + elif member == '__invert__': + self.fail('Unsupported operand type for ~ ({})'.format( + self.format(typ)), context) + elif member == '__getitem__': + # Indexed get. + self.fail('Value of type {} is not indexable'.format( + self.format(typ)), context) + elif member == '__setitem__': + # Indexed set. + self.fail('Unsupported target for indexed assignment', context) + elif member == '__call__': + self.fail('{} not callable'.format(self.format(typ)), context) + else: + # The non-special case: a missing ordinary attribute. + if not self.disable_type_names: + failed = False + if isinstance(typ, Instance) and typ.type.names: + alternatives = set(typ.type.names.keys()) + matches = [m for m in COMMON_MISTAKES.get(member, []) if m in alternatives] + matches.extend(best_matches(member, alternatives)[:3]) + if matches: + self.fail('{} has no attribute "{}"; maybe {}?'.format( + self.format(typ), member, pretty_or(matches)), context) + failed = True + if not failed: + self.fail('{} has no attribute "{}"'.format(self.format(typ), + member), context) + else: + self.fail('Some element of union has no attribute "{}"'.format( + member), context) + return AnyType() + + def unsupported_operand_types(self, op: str, left_type: Any, + right_type: Any, context: Context) -> None: + """Report unsupported operand types for a binary operation. + + Types can be Type objects or strings. + """ + if isinstance(left_type, Void) or isinstance(right_type, Void): + self.check_void(left_type, context) + self.check_void(right_type, context) + return + left_str = '' + if isinstance(left_type, str): + left_str = left_type + else: + left_str = self.format(left_type) + + right_str = '' + if isinstance(right_type, str): + right_str = right_type + else: + right_str = self.format(right_type) + + if self.disable_type_names: + msg = 'Unsupported operand types for {} (likely involving Union)'.format(op) + else: + msg = 'Unsupported operand types for {} ({} and {})'.format( + op, left_str, right_str) + self.fail(msg, context) + + def unsupported_left_operand(self, op: str, typ: Type, + context: Context) -> None: + if not self.check_void(typ, context): + if self.disable_type_names: + msg = 'Unsupported left operand type for {} (some union)'.format(op) + else: + msg = 'Unsupported left operand type for {} ({})'.format( + op, self.format(typ)) + self.fail(msg, context) + + def type_expected_as_right_operand_of_is(self, context: Context) -> None: + self.fail('Type expected as right operand of "is"', context) + + def not_callable(self, typ: Type, context: Context) -> Type: + self.fail('{} not callable'.format(self.format(typ)), context) + return AnyType() + + def untyped_function_call(self, callee: CallableType, context: Context) -> Type: + name = callee.name if callee.name is not None else '(unknown)' + self.fail('call to untyped function {} in typed context'.format(name), context) + return AnyType() + + def incompatible_argument(self, n: int, m: int, callee: CallableType, arg_type: Type, + arg_kind: int, context: Context) -> None: + """Report an error about an incompatible argument type. + + The argument type is arg_type, argument number is n and the + callee type is 'callee'. If the callee represents a method + that corresponds to an operator, use the corresponding + operator name in the messages. + """ + target = '' + if callee.name: + name = callee.name + base = extract_type(name) + + for op, method in op_methods.items(): + for variant in method, '__r' + method[2:]: + if name.startswith('"{}" of'.format(variant)): + if op == 'in' or variant != method: + # Reversed order of base/argument. + self.unsupported_operand_types(op, arg_type, base, + context) + else: + self.unsupported_operand_types(op, base, arg_type, + context) + return + + if name.startswith('"__getitem__" of'): + self.invalid_index_type(arg_type, base, context) + return + + if name.startswith('"__setitem__" of'): + if n == 1: + self.invalid_index_type(arg_type, base, context) + else: + msg = '{} (expression has type {}, target has type {})' + arg_type_str, callee_type_str = self.format_distinctly(arg_type, + callee.arg_types[n - 1]) + self.fail(msg.format(INCOMPATIBLE_TYPES_IN_ASSIGNMENT, + arg_type_str, callee_type_str), + context) + return + + target = 'to {} '.format(name) + + msg = '' + if callee.name == '': + name = callee.name[1:-1] + n -= 1 + msg = '{} item {} has incompatible type {}'.format( + name[0].upper() + name[1:], n, self.format_simple(arg_type)) + elif callee.name == '': + msg = 'List comprehension has incompatible type List[{}]'.format( + strip_quotes(self.format(arg_type))) + elif callee.name == '': + msg = 'Set comprehension has incompatible type Set[{}]'.format( + strip_quotes(self.format(arg_type))) + elif callee.name == '': + msg = ('{} expression in dictionary comprehension has incompatible type {}; ' + 'expected type {}').format( + 'Key' if n == 1 else 'Value', + self.format(arg_type), + self.format(callee.arg_types[n - 1])) + elif callee.name == '': + msg = 'Generator has incompatible item type {}'.format( + self.format_simple(arg_type)) + else: + try: + expected_type = callee.arg_types[m - 1] + except IndexError: # Varargs callees + expected_type = callee.arg_types[-1] + arg_type_str, expected_type_str = self.format_distinctly(arg_type, expected_type) + #import pdb; pdb.set_trace() + if arg_kind == ARG_STAR: + arg_type_str = '*' + arg_type_str + elif arg_kind == ARG_STAR2: + arg_type_str = '**' + arg_type_str + msg = 'Argument {} {}has incompatible type {}; expected {}'.format( + n, target, arg_type_str, expected_type_str) + self.fail(msg, context) + + def invalid_index_type(self, index_type: Type, base_str: str, + context: Context) -> None: + self.fail('Invalid index type {} for {}'.format( + self.format(index_type), base_str), context) + + def too_few_arguments(self, callee: CallableType, context: Context, + argument_names: List[str]) -> None: + if (argument_names is not None and not all(k is None for k in argument_names) + and len(argument_names) >= 1): + diff = [k for k in callee.arg_names if k not in argument_names] + if len(diff) == 1: + msg = 'Missing positional argument' + else: + msg = 'Missing positional arguments' + if callee.name and diff: + msg += ' "{}" in call to {}'.format('", "'.join(diff), callee.name) + else: + msg = 'Too few arguments' + if callee.name: + msg += ' for {}'.format(callee.name) + self.fail(msg, context) + + def too_many_arguments(self, callee: CallableType, context: Context) -> None: + msg = 'Too many arguments' + if callee.name: + msg += ' for {}'.format(callee.name) + self.fail(msg, context) + + def too_many_positional_arguments(self, callee: CallableType, + context: Context) -> None: + msg = 'Too many positional arguments' + if callee.name: + msg += ' for {}'.format(callee.name) + self.fail(msg, context) + + def unexpected_keyword_argument(self, callee: CallableType, name: str, + context: Context) -> None: + msg = 'Unexpected keyword argument "{}"'.format(name) + if callee.name: + msg += ' for {}'.format(callee.name) + self.fail(msg, context) + if callee.definition: + fullname = callee.definition.fullname() + if fullname is not None and '.' in fullname: + module_name = fullname.rsplit('.', 1)[0] + path = self.modules[module_name].path + self.note('{} defined here'.format(callee.name), callee.definition, file=path) + + def duplicate_argument_value(self, callee: CallableType, index: int, + context: Context) -> None: + self.fail('{} gets multiple values for keyword argument "{}"'. + format(capitalize(callable_name(callee)), + callee.arg_names[index]), context) + + def does_not_return_value(self, void_type: Type, context: Context) -> None: + """Report an error about a void type in a non-void context. + + The first argument must be a void type. If the void type has a + source in it, report it in the error message. This allows + giving messages such as 'Foo does not return a value'. + """ + if (cast(Void, void_type)).source is None: + self.fail('Function does not return a value', context) + else: + self.fail('{} does not return a value'.format( + capitalize((cast(Void, void_type)).source)), context) + + def deleted_as_rvalue(self, typ: DeletedType, context: Context) -> None: + """Report an error about using an deleted type as an rvalue.""" + if typ.source is None: + s = "" + else: + s = " '{}'".format(typ.source) + self.fail('Trying to read deleted variable{}'.format(s), context) + + def deleted_as_lvalue(self, typ: DeletedType, context: Context) -> None: + """Report an error about using an deleted type as an lvalue. + + Currently, this only occurs when trying to assign to an + exception variable outside the local except: blocks. + """ + if typ.source is None: + s = "" + else: + s = " '{}'".format(typ.source) + self.fail('Assignment to variable{} outside except: block'.format(s), context) + + def no_variant_matches_arguments(self, overload: Overloaded, arg_types: List[Type], + context: Context) -> None: + if overload.name(): + self.fail('No overload variant of {} matches argument types {}' + .format(overload.name(), arg_types), context) + else: + self.fail('No overload variant matches argument types {}'.format(arg_types), context) + + def function_variants_overlap(self, n1: int, n2: int, + context: Context) -> None: + self.fail('Function signature variants {} and {} overlap'.format( + n1 + 1, n2 + 1), context) + + def invalid_cast(self, target_type: Type, source_type: Type, + context: Context) -> None: + if not self.check_void(source_type, context): + self.fail('Cannot cast from {} to {}'.format( + self.format(source_type), self.format(target_type)), context) + + def wrong_number_values_to_unpack(self, provided: int, expected: int, + context: Context) -> None: + if provided < expected: + if provided == 1: + self.fail('Need more than 1 value to unpack ({} expected)'.format(expected), + context) + else: + self.fail('Need more than {} values to unpack ({} expected)'.format( + provided, expected), context) + elif provided > expected: + self.fail('Too many values to unpack ({} expected, {} provided)'.format( + expected, provided), context) + + def type_not_iterable(self, type: Type, context: Context) -> None: + self.fail('\'{}\' object is not iterable'.format(type), context) + + def incompatible_operator_assignment(self, op: str, + context: Context) -> None: + self.fail('Result type of {} incompatible in assignment'.format(op), + context) + + def incompatible_value_count_in_assignment(self, lvalue_count: int, + rvalue_count: int, + context: Context) -> None: + if rvalue_count < lvalue_count: + self.fail('Need {} values to assign'.format(lvalue_count), context) + elif rvalue_count > lvalue_count: + self.fail('Too many values to assign', context) + + def type_incompatible_with_supertype(self, name: str, supertype: TypeInfo, + context: Context) -> None: + self.fail('Type of "{}" incompatible with supertype "{}"'.format( + name, supertype.name), context) + + def signature_incompatible_with_supertype( + self, name: str, name_in_super: str, supertype: str, + context: Context) -> None: + target = self.override_target(name, name_in_super, supertype) + self.fail('Signature of "{}" incompatible with {}'.format( + name, target), context) + + def argument_incompatible_with_supertype( + self, arg_num: int, name: str, name_in_supertype: str, + supertype: str, context: Context) -> None: + target = self.override_target(name, name_in_supertype, supertype) + self.fail('Argument {} of "{}" incompatible with {}' + .format(arg_num, name, target), context) + + def return_type_incompatible_with_supertype( + self, name: str, name_in_supertype: str, supertype: str, + context: Context) -> None: + target = self.override_target(name, name_in_supertype, supertype) + self.fail('Return type of "{}" incompatible with {}' + .format(name, target), context) + + def override_target(self, name: str, name_in_super: str, + supertype: str) -> str: + target = 'supertype "{}"'.format(supertype) + if name_in_super != name: + target = '"{}" of {}'.format(name_in_super, target) + return target + + def boolean_return_value_expected(self, method: str, + context: Context) -> None: + self.fail('Boolean return value expected for method "{}"'.format( + method), context) + + def incompatible_type_application(self, expected_arg_count: int, + actual_arg_count: int, + context: Context) -> None: + if expected_arg_count == 0: + self.fail('Type application targets a non-generic function', + context) + elif actual_arg_count > expected_arg_count: + self.fail('Type application has too many types ({} expected)' + .format(expected_arg_count), context) + else: + self.fail('Type application has too few types ({} expected)' + .format(expected_arg_count), context) + + def incompatible_array_item_type(self, typ: Type, index: int, + context: Context) -> None: + self.fail('Array item {} has incompatible type {}'.format( + index, self.format(typ)), context) + + def could_not_infer_type_arguments(self, callee_type: CallableType, n: int, + context: Context) -> None: + if callee_type.name and n > 0: + self.fail('Cannot infer type argument {} of {}'.format( + n, callee_type.name), context) + else: + self.fail('Cannot infer function type argument', context) + + def invalid_var_arg(self, typ: Type, context: Context) -> None: + self.fail('List or tuple expected as variable arguments', context) + + def invalid_keyword_var_arg(self, typ: Type, context: Context) -> None: + if isinstance(typ, Instance) and (typ.type.fullname() == 'builtins.dict'): + self.fail('Keywords must be strings', context) + else: + self.fail('Argument after ** must be a dictionary', + context) + + def incomplete_type_var_match(self, member: str, context: Context) -> None: + self.fail('"{}" has incomplete match to supertype type variable' + .format(member), context) + + def not_implemented(self, msg: str, context: Context) -> Type: + self.fail('Feature not implemented yet ({})'.format(msg), context) + return AnyType() + + def undefined_in_superclass(self, member: str, context: Context) -> None: + self.fail('"{}" undefined in superclass'.format(member), context) + + def check_void(self, typ: Type, context: Context) -> bool: + """If type is void, report an error such as '.. does not + return a value' and return True. Otherwise, return False. + """ + if isinstance(typ, Void): + self.does_not_return_value(typ, context) + return True + else: + return False + + def too_few_string_formatting_arguments(self, context: Context) -> None: + self.fail('Not enough arguments for format string', context) + + def too_many_string_formatting_arguments(self, context: Context) -> None: + self.fail('Not all arguments converted during string formatting', context) + + def incomplete_conversion_specifier_format(self, context: Context) -> None: + self.fail('Incomplete format', context) + + def unsupported_placeholder(self, placeholder: str, context: Context) -> None: + self.fail('Unsupported format character \'%s\'' % placeholder, context) + + def string_interpolation_with_star_and_key(self, context: Context) -> None: + self.fail('String interpolation contains both stars and mapping keys', context) + + def requires_int_or_char(self, context: Context) -> None: + self.fail('%c requires int or char', context) + + def key_not_in_mapping(self, key: str, context: Context) -> None: + self.fail('Key \'%s\' not found in mapping' % key, context) + + def string_interpolation_mixing_key_and_non_keys(self, context: Context) -> None: + self.fail('String interpolation mixes specifier with and without mapping keys', context) + + def cannot_determine_type(self, name: str, context: Context) -> None: + self.fail("Cannot determine type of '%s'" % name, context) + + def cannot_determine_type_in_base(self, name: str, base: str, context: Context) -> None: + self.fail("Cannot determine type of '%s' in base class '%s'" % (name, base), context) + + def invalid_method_type(self, sig: CallableType, context: Context) -> None: + self.fail('Invalid method type', context) + + def invalid_class_method_type(self, sig: CallableType, context: Context) -> None: + self.fail('Invalid class method type', context) + + def incompatible_conditional_function_def(self, defn: FuncDef) -> None: + self.fail('All conditional function variants must have identical ' + 'signatures', defn) + + def cannot_instantiate_abstract_class(self, class_name: str, + abstract_attributes: List[str], + context: Context) -> None: + attrs = format_string_list("'%s'" % a for a in abstract_attributes[:5]) + self.fail("Cannot instantiate abstract class '%s' with abstract " + "attribute%s %s" % (class_name, plural_s(abstract_attributes), + attrs), + context) + + def base_class_definitions_incompatible(self, name: str, base1: TypeInfo, + base2: TypeInfo, + context: Context) -> None: + self.fail('Definition of "{}" in base class "{}" is incompatible ' + 'with definition in base class "{}"'.format( + name, base1.name(), base2.name()), context) + + def cant_assign_to_method(self, context: Context) -> None: + self.fail(CANNOT_ASSIGN_TO_METHOD, context) + + def read_only_property(self, name: str, type: TypeInfo, + context: Context) -> None: + self.fail('Property "{}" defined in "{}" is read-only'.format( + name, type.name()), context) + + def incompatible_typevar_value(self, callee: CallableType, index: int, + type: Type, context: Context) -> None: + self.fail('Type argument {} of {} has incompatible value {}'.format( + index, callable_name(callee), self.format(type)), context) + + def overloaded_signatures_overlap(self, index1: int, index2: int, + context: Context) -> None: + self.fail('Overloaded function signatures {} and {} overlap with ' + 'incompatible return types'.format(index1, index2), context) + + def invalid_reverse_operator_signature(self, reverse: str, other: str, + context: Context) -> None: + self.fail('"Any" return type expected since argument to {} ' + 'does not support {}'.format(reverse, other), context) + + def reverse_operator_method_with_any_arg_must_return_any( + self, method: str, context: Context) -> None: + self.fail('"Any" return type expected since argument to {} ' + 'has type "Any"'.format(method), context) + + def operator_method_signatures_overlap( + self, reverse_class: str, reverse_method: str, forward_class: str, + forward_method: str, context: Context) -> None: + self.fail('Signatures of "{}" of "{}" and "{}" of "{}" ' + 'are unsafely overlapping'.format( + reverse_method, reverse_class, + forward_method, forward_class), + context) + + def forward_operator_not_callable( + self, forward_method: str, context: Context) -> None: + self.fail('Forward operator "{}" is not callable'.format( + forward_method), context) + + def signatures_incompatible(self, method: str, other_method: str, + context: Context) -> None: + self.fail('Signatures of "{}" and "{}" are incompatible'.format( + method, other_method), context) + + def yield_from_invalid_operand_type(self, expr: Type, context: Context) -> Type: + text = self.format(expr) if self.format(expr) != 'object' else expr + self.fail('"yield from" can\'t be applied to {}'.format(text), context) + return AnyType() + + def invalid_signature(self, func_type: Type, context: Context) -> None: + self.fail('Invalid signature "{}"'.format(func_type), context) + + def reveal_type(self, typ: Type, context: Context) -> None: + self.fail('Revealed type is \'{}\''.format(typ), context) + + def unsupported_type_type(self, item: Type, context: Context) -> None: + self.fail('Unsupported type Type[{}]'.format(self.format(item)), context) + + def redundant_cast(self, typ: Type, context: Context) -> None: + self.note('Redundant cast to {}'.format(self.format(typ)), context) + + +def capitalize(s: str) -> str: + """Capitalize the first character of a string.""" + if s == '': + return '' + else: + return s[0].upper() + s[1:] + + +def extract_type(name: str) -> str: + """If the argument is the name of a method (of form C.m), return + the type portion in quotes (e.g. "y"). Otherwise, return the string + unmodified. + """ + name = re.sub('^"[a-zA-Z0-9_]+" of ', '', name) + return name + + +def strip_quotes(s: str) -> str: + """Strip a double quote at the beginning and end of the string, if any.""" + s = re.sub('^"', '', s) + s = re.sub('"$', '', s) + return s + + +def plural_s(s: Sequence[Any]) -> str: + if len(s) > 1: + return 's' + else: + return '' + + +def format_string_list(s: Iterable[str]) -> str: + l = list(s) + assert len(l) > 0 + if len(l) == 1: + return l[0] + else: + return '%s and %s' % (', '.join(l[:-1]), l[-1]) + + +def callable_name(type: CallableType) -> str: + if type.name: + return type.name + else: + return 'function' + + +def temp_message_builder() -> MessageBuilder: + """Return a message builder usable for throwaway errors (which may not format properly).""" + return MessageBuilder(Errors(), {}) + + +# For hard-coding suggested missing member alternatives. +COMMON_MISTAKES = { + 'add': ('append', 'extend'), +} # type: Dict[str, Sequence[str]] + + +def best_matches(current: str, options: Iterable[str]) -> List[str]: + ratios = {v: difflib.SequenceMatcher(a=current, b=v).ratio() for v in options} + return sorted((o for o in options if ratios[o] > 0.75), + reverse=True, key=lambda v: (ratios[v], v)) + + +def pretty_or(args: List[str]) -> str: + quoted = ['"' + a + '"' for a in args] + if len(quoted) == 1: + return quoted[0] + if len(quoted) == 2: + return "{} or {}".format(quoted[0], quoted[1]) + return ", ".join(quoted[:-1]) + ", or " + quoted[-1] diff --git a/mypy/nodes.py b/mypy/nodes.py index 60d7b5d18944f..02f86b1537b93 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1,2242 +1,2242 @@ -"""Abstract syntax tree node classes (i.e. parse tree).""" - -import os -import re -from abc import abstractmethod, ABCMeta - -from typing import ( - Any, TypeVar, List, Tuple, cast, Set, Dict, Union, Optional -) - -from mypy.lex import Token -import mypy.strconv -from mypy.visitor import NodeVisitor -from mypy.util import dump_tagged, short_type - - -class Context: - """Base type for objects that are valid as error message locations.""" - @abstractmethod - def get_line(self) -> int: pass - - -if False: - # break import cycle only needed for mypy - import mypy.types - - -T = TypeVar('T') - -JsonDict = Dict[str, Any] - - -# Symbol table node kinds -# -# TODO rename to use more descriptive names - -LDEF = 0 # type: int -GDEF = 1 # type: int -MDEF = 2 # type: int -MODULE_REF = 3 # type: int -# Type variable declared using TypeVar(...) has kind UNBOUND_TVAR. It's not -# valid as a type. A type variable is valid as a type (kind BOUND_TVAR) within -# (1) a generic class that uses the type variable as a type argument or -# (2) a generic function that refers to the type variable in its signature. -UNBOUND_TVAR = 4 # type: int -BOUND_TVAR = 5 # type: int -TYPE_ALIAS = 6 # type: int -# Placeholder for a name imported via 'from ... import'. Second phase of -# semantic will replace this the actual imported reference. This is -# needed so that we can detect whether a name has been imported during -# XXX what? -UNBOUND_IMPORTED = 7 # type: int - - -LITERAL_YES = 2 -LITERAL_TYPE = 1 -LITERAL_NO = 0 - -# Hard coded name of Enum baseclass. -ENUM_BASECLASS = "enum.Enum" - -node_kinds = { - LDEF: 'Ldef', - GDEF: 'Gdef', - MDEF: 'Mdef', - MODULE_REF: 'ModuleRef', - UNBOUND_TVAR: 'UnboundTvar', - BOUND_TVAR: 'Tvar', - TYPE_ALIAS: 'TypeAlias', - UNBOUND_IMPORTED: 'UnboundImported', -} -inverse_node_kinds = {_kind: _name for _name, _kind in node_kinds.items()} - - -implicit_module_attrs = {'__name__': '__builtins__.str', - '__doc__': '__builtins__.str', - '__file__': '__builtins__.str', - '__package__': '__builtins__.str'} - - -type_aliases = { - 'typing.List': '__builtins__.list', - 'typing.Dict': '__builtins__.dict', - 'typing.Set': '__builtins__.set', -} - -reverse_type_aliases = dict((name.replace('__builtins__', 'builtins'), alias) - for alias, name in type_aliases.items()) # type: Dict[str, str] - - -class Node(Context): - """Common base class for all non-type parse tree nodes.""" - - line = -1 - - literal = LITERAL_NO - literal_hash = None # type: Any - - def __str__(self) -> str: - ans = self.accept(mypy.strconv.StrConv()) - if ans is None: - return repr(self) - return ans - - def set_line(self, target: Union[Token, 'Node', int]) -> 'Node': - if isinstance(target, int): - self.line = target - else: - self.line = target.line - return self - - def get_line(self) -> int: - # TODO this should be just 'line' - return self.line - - def accept(self, visitor: NodeVisitor[T]) -> T: - raise RuntimeError('Not implemented') - - -# These are placeholders for a future refactoring; see #1783. -# For now they serve as (unchecked) documentation of what various -# fields of Node subtypes are expected to contain. -Statement = Node -Expression = Node - - -class SymbolNode(Node): - # Nodes that can be stored in a symbol table. - - # TODO do not use methods for these - - @abstractmethod - def name(self) -> str: pass - - @abstractmethod - def fullname(self) -> str: pass - - # NOTE: Can't use @abstractmethod, since many subclasses of Node - # don't implement serialize(). - def serialize(self) -> Any: - raise NotImplementedError('Cannot serialize {} instance'.format(self.__class__.__name__)) - - @classmethod - def deserialize(cls, data: JsonDict) -> 'SymbolNode': - classname = data['.class'] - glo = globals() - if classname in glo: - cl = glo[classname] - if issubclass(cl, cls) and 'deserialize' in cl.__dict__: - return cl.deserialize(data) - raise NotImplementedError('unexpected .class {}'.format(classname)) - - -class MypyFile(SymbolNode, Statement): - """The abstract syntax tree of a single source file.""" - - # Module name ('__main__' for initial file) - _name = None # type: str - # Fully qualified module name - _fullname = None # type: str - # Path to the file (None if not known) - path = '' - # Top-level definitions and statements - defs = None # type: List[Statement] - # Is there a UTF-8 BOM at the start? - is_bom = False - names = None # type: SymbolTable - # All import nodes within the file (also ones within functions etc.) - imports = None # type: List[ImportBase] - # Lines to ignore when checking - ignored_lines = None # type: Set[int] - # Is this file represented by a stub file (.pyi)? - is_stub = False - # Do weak typing globally in the file? - weak_opts = None # type: Set[str] - - def __init__(self, - defs: List[Statement], - imports: List['ImportBase'], - is_bom: bool = False, - ignored_lines: Set[int] = None, - weak_opts: Set[str] = None) -> None: - self.defs = defs - self.line = 1 # Dummy line number - self.imports = imports - self.is_bom = is_bom - self.weak_opts = weak_opts - if ignored_lines: - self.ignored_lines = ignored_lines - else: - self.ignored_lines = set() - - def name(self) -> str: - return self._name - - def fullname(self) -> str: - return self._fullname - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_mypy_file(self) - - def is_package_init_file(self) -> bool: - return not (self.path is None) and len(self.path) != 0 \ - and os.path.basename(self.path).startswith('__init__.') - - def serialize(self) -> JsonDict: - return {'.class': 'MypyFile', - '_name': self._name, - '_fullname': self._fullname, - 'names': self.names.serialize(self._fullname), - 'is_stub': self.is_stub, - 'path': self.path, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'MypyFile': - assert data['.class'] == 'MypyFile', data - tree = MypyFile([], []) - tree._name = data['_name'] - tree._fullname = data['_fullname'] - tree.names = SymbolTable.deserialize(data['names']) - tree.is_stub = data['is_stub'] - tree.path = data['path'] - return tree - - -class ImportBase(Statement): - """Base class for all import statements.""" - is_unreachable = False - is_top_level = False # Set by semanal.FirstPass - # If an import replaces existing definitions, we construct dummy assignment - # statements that assign the imported names to the names in the current scope, - # for type checking purposes. Example: - # - # x = 1 - # from m import x <-- add assignment representing "x = m.x" - assignments = None # type: List[AssignmentStmt] - - def __init__(self) -> None: - self.assignments = [] - - -class Import(ImportBase): - """import m [as n]""" - - ids = None # type: List[Tuple[str, Optional[str]]] # (module id, as id) - - def __init__(self, ids: List[Tuple[str, Optional[str]]]) -> None: - super().__init__() - self.ids = ids - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_import(self) - - -class ImportFrom(ImportBase): - """from m import x [as y], ...""" - - names = None # type: List[Tuple[str, Optional[str]]] # Tuples (name, as name) - - def __init__(self, id: str, relative: int, names: List[Tuple[str, Optional[str]]]) -> None: - super().__init__() - self.id = id - self.names = names - self.relative = relative - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_import_from(self) - - -class ImportAll(ImportBase): - """from m import *""" - - def __init__(self, id: str, relative: int) -> None: - super().__init__() - self.id = id - self.relative = relative - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_import_all(self) - - -class FuncBase(SymbolNode): - """Abstract base class for function-like nodes""" - - # Type signature. This is usually CallableType or Overloaded, but it can be something else for - # decorated functions/ - type = None # type: mypy.types.Type - # If method, reference to TypeInfo - info = None # type: TypeInfo - is_property = False - _fullname = None # type: str # Name with module prefix - - @abstractmethod - def name(self) -> str: pass - - def fullname(self) -> str: - return self._fullname - - def is_method(self) -> bool: - return bool(self.info) - - -class OverloadedFuncDef(FuncBase, Statement): - """A logical node representing all the variants of an overloaded function. - - This node has no explicit representation in the source program. - Overloaded variants must be consecutive in the source file. - """ - - items = None # type: List[Decorator] - - def __init__(self, items: List['Decorator']) -> None: - self.items = items - self.set_line(items[0].line) - - def name(self) -> str: - return self.items[0].func.name() - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_overloaded_func_def(self) - - def serialize(self) -> JsonDict: - return {'.class': 'OverloadedFuncDef', - 'items': [i.serialize() for i in self.items], - 'type': None if self.type is None else self.type.serialize(), - 'fullname': self._fullname, - 'is_property': self.is_property, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'OverloadedFuncDef': - assert data['.class'] == 'OverloadedFuncDef' - res = OverloadedFuncDef([Decorator.deserialize(d) for d in data['items']]) - if data.get('type') is not None: - res.type = mypy.types.Type.deserialize(data['type']) - res._fullname = data['fullname'] - res.is_property = data['is_property'] - # NOTE: res.info will be set in the fixup phase. - return res - - -class Argument(Node): - """A single argument in a FuncItem.""" - - def __init__(self, variable: 'Var', type_annotation: 'Optional[mypy.types.Type]', - initializer: Optional[Expression], kind: int, - initialization_statement: Optional['AssignmentStmt'] = None) -> None: - self.variable = variable - - self.type_annotation = type_annotation - self.initializer = initializer - - self.initialization_statement = initialization_statement - if not self.initialization_statement: - self.initialization_statement = self._initialization_statement() - - self.kind = kind - - def _initialization_statement(self) -> Optional['AssignmentStmt']: - """Convert the initializer into an assignment statement. - """ - if not self.initializer: - return None - - rvalue = self.initializer - lvalue = NameExpr(self.variable.name()) - assign = AssignmentStmt([lvalue], rvalue) - return assign - - def set_line(self, target: Union[Token, Node, int]) -> Node: - super().set_line(target) - - if self.initializer: - self.initializer.set_line(self.line) - - self.variable.set_line(self.line) - - if self.initialization_statement: - self.initialization_statement.set_line(self.line) - self.initialization_statement.lvalues[0].set_line(self.line) - - def serialize(self) -> JsonDict: - data = {'.class': 'Argument', - 'kind': self.kind, - 'variable': self.variable.serialize(), - 'type_annotation': (None if self.type_annotation is None - else self.type_annotation.serialize()), - } # type: JsonDict - # TODO: initializer? - return data - - @classmethod - def deserialize(cls, data: JsonDict) -> 'Argument': - assert data['.class'] == 'Argument' - return Argument(Var.deserialize(data['variable']), - (None if data.get('type_annotation') is None - else mypy.types.Type.deserialize(data['type_annotation'])), - None, # TODO: initializer? - kind=data['kind']) - - -class FuncItem(FuncBase): - arguments = [] # type: List[Argument] - # Minimum number of arguments - min_args = 0 - # Maximum number of positional arguments, -1 if no explicit limit (*args not included) - max_pos = 0 - body = None # type: Block - # Is this an overload variant of function with more than one overload variant? - is_overload = False - is_generator = False # Contains a yield statement? - is_static = False # Uses @staticmethod? - is_class = False # Uses @classmethod? - # Variants of function with type variables with values expanded - expanded = None # type: List[FuncItem] - - def __init__(self, arguments: List[Argument], body: 'Block', - typ: 'mypy.types.FunctionLike' = None) -> None: - self.arguments = arguments - arg_kinds = [arg.kind for arg in self.arguments] - self.max_pos = arg_kinds.count(ARG_POS) + arg_kinds.count(ARG_OPT) - self.body = body - self.type = typ - self.expanded = [] - - self.min_args = 0 - for i in range(len(self.arguments)): - if self.arguments[i] is None and i < self.max_fixed_argc(): - self.min_args = i + 1 - - def max_fixed_argc(self) -> int: - return self.max_pos - - def set_line(self, target: Union[Token, Node, int]) -> Node: - super().set_line(target) - for arg in self.arguments: - arg.set_line(self.line) - return self - - def is_dynamic(self): - return self.type is None - - -class FuncDef(FuncItem, Statement): - """Function definition. - - This is a non-lambda function defined using 'def'. - """ - - is_decorated = False - is_conditional = False # Defined conditionally (within block)? - is_abstract = False - is_property = False - original_def = None # type: Union[None, FuncDef, Var] # Original conditional definition - - def __init__(self, - name: str, # Function name - arguments: List[Argument], - body: 'Block', - typ: 'mypy.types.FunctionLike' = None) -> None: - super().__init__(arguments, body, typ) - self._name = name - - def name(self) -> str: - return self._name - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_func_def(self) - - def is_constructor(self) -> bool: - return self.info is not None and self._name == '__init__' - - def serialize(self) -> JsonDict: - return {'.class': 'FuncDef', - 'name': self._name, - 'fullname': self._fullname, - 'arguments': [a.serialize() for a in self.arguments], - 'type': None if self.type is None else self.type.serialize(), - 'is_property': self.is_property, - 'is_overload': self.is_overload, - 'is_generator': self.is_generator, - 'is_static': self.is_static, - 'is_class': self.is_class, - 'is_decorated': self.is_decorated, - 'is_conditional': self.is_conditional, - 'is_abstract': self.is_abstract, - # TODO: Do we need expanded, original_def? - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'FuncDef': - assert data['.class'] == 'FuncDef' - body = Block([]) - ret = FuncDef(data['name'], - [Argument.deserialize(a) for a in data['arguments']], - body, - (None if data['type'] is None - else mypy.types.FunctionLike.deserialize(data['type']))) - ret._fullname = data['fullname'] - ret.is_property = data['is_property'] - ret.is_overload = data['is_overload'] - ret.is_generator = data['is_generator'] - ret.is_static = data['is_static'] - ret.is_class = data['is_class'] - ret.is_decorated = data['is_decorated'] - ret.is_conditional = data['is_conditional'] - ret.is_abstract = data['is_abstract'] - # NOTE: ret.info is set in the fixup phase. - return ret - - -class Decorator(SymbolNode, Statement): - """A decorated function. - - A single Decorator object can include any number of function decorators. - """ - - func = None # type: FuncDef # Decorated function - decorators = None # type: List[Expression] # Decorators, at least one # XXX Not true - var = None # type: Var # Represents the decorated function obj - is_overload = False - - def __init__(self, func: FuncDef, decorators: List[Expression], - var: 'Var') -> None: - self.func = func - self.decorators = decorators - self.var = var - self.is_overload = False - - def name(self) -> str: - return self.func.name() - - def fullname(self) -> str: - return self.func.fullname() - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_decorator(self) - - def serialize(self) -> JsonDict: - return {'.class': 'Decorator', - 'func': self.func.serialize(), - 'var': self.var.serialize(), - 'is_overload': self.is_overload, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'Decorator': - assert data['.class'] == 'Decorator' - dec = Decorator(FuncDef.deserialize(data['func']), - [], - Var.deserialize(data['var'])) - dec.is_overload = data['is_overload'] - return dec - - -class Var(SymbolNode, Statement): - """A variable. - - It can refer to global/local variable or a data attribute. - """ - - _name = None # type: str # Name without module prefix - _fullname = None # type: str # Name with module prefix - info = None # type: TypeInfo # Defining class (for member variables) - type = None # type: mypy.types.Type # Declared or inferred type, or None - # Is this the first argument to an ordinary method (usually "self")? - is_self = False - is_ready = False # If inferred, is the inferred type available? - # Is this initialized explicitly to a non-None value in class body? - is_initialized_in_class = False - is_staticmethod = False - is_classmethod = False - is_property = False - is_settable_property = False - - def __init__(self, name: str, type: 'mypy.types.Type' = None) -> None: - self._name = name - self.type = type - self.is_self = False - self.is_ready = True - self.is_initialized_in_class = False - - def name(self) -> str: - return self._name - - def fullname(self) -> str: - return self._fullname - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_var(self) - - def serialize(self) -> JsonDict: - # TODO: Leave default values out? - # NOTE: Sometimes self.is_ready is False here, but we don't care. - data = {'.class': 'Var', - 'name': self._name, - 'fullname': self._fullname, - 'type': None if self.type is None else self.type.serialize(), - 'is_self': self.is_self, - 'is_initialized_in_class': self.is_initialized_in_class, - 'is_staticmethod': self.is_staticmethod, - 'is_classmethod': self.is_classmethod, - 'is_property': self.is_property, - 'is_settable_property': self.is_settable_property, - } # type: JsonDict - return data - - @classmethod - def deserialize(cls, data: JsonDict) -> 'Var': - assert data['.class'] == 'Var' - name = data['name'] - type = None if data['type'] is None else mypy.types.Type.deserialize(data['type']) - v = Var(name, type) - v._fullname = data['fullname'] - v.is_self = data['is_self'] - v.is_initialized_in_class = data['is_initialized_in_class'] - v.is_staticmethod = data['is_staticmethod'] - v.is_classmethod = data['is_classmethod'] - v.is_property = data['is_property'] - v.is_settable_property = data['is_settable_property'] - return v - - -class ClassDef(Statement): - """Class definition""" - - name = None # type: str # Name of the class without module prefix - fullname = None # type: str # Fully qualified name of the class - defs = None # type: Block - type_vars = None # type: List[mypy.types.TypeVarDef] - # Base class expressions (not semantically analyzed -- can be arbitrary expressions) - base_type_exprs = None # type: List[Expression] - info = None # type: TypeInfo # Related TypeInfo - metaclass = '' - decorators = None # type: List[Expression] - # Built-in/extension class? (single implementation inheritance only) - is_builtinclass = False - - def __init__(self, - name: str, - defs: 'Block', - type_vars: List['mypy.types.TypeVarDef'] = None, - base_type_exprs: List[Expression] = None, - metaclass: str = None) -> None: - self.name = name - self.defs = defs - self.type_vars = type_vars or [] - self.base_type_exprs = base_type_exprs or [] - self.metaclass = metaclass - self.decorators = [] - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_class_def(self) - - def is_generic(self) -> bool: - return self.info.is_generic() - - def serialize(self) -> JsonDict: - # Not serialized: defs, base_type_exprs, decorators - return {'.class': 'ClassDef', - 'name': self.name, - 'fullname': self.fullname, - 'type_vars': [v.serialize() for v in self.type_vars], - 'metaclass': self.metaclass, - 'is_builtinclass': self.is_builtinclass, - } - - @classmethod - def deserialize(self, data: JsonDict) -> 'ClassDef': - assert data['.class'] == 'ClassDef' - res = ClassDef(data['name'], - Block([]), - [mypy.types.TypeVarDef.deserialize(v) for v in data['type_vars']], - metaclass=data['metaclass'], - ) - res.fullname = data['fullname'] - res.is_builtinclass = data['is_builtinclass'] - return res - - -class GlobalDecl(Statement): - """Declaration global x, y, ...""" - - names = None # type: List[str] - - def __init__(self, names: List[str]) -> None: - self.names = names - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_global_decl(self) - - -class NonlocalDecl(Statement): - """Declaration nonlocal x, y, ...""" - - names = None # type: List[str] - - def __init__(self, names: List[str]) -> None: - self.names = names - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_nonlocal_decl(self) - - -class Block(Statement): - body = None # type: List[Statement] - # True if we can determine that this block is not executed. For example, - # this applies to blocks that are protected by something like "if PY3:" - # when using Python 2. - is_unreachable = False - - def __init__(self, body: List[Statement]) -> None: - self.body = body - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_block(self) - - -# Statements - - -class ExpressionStmt(Statement): - """An expression as a statement, such as print(s).""" - expr = None # type: Expression - - def __init__(self, expr: Expression) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_expression_stmt(self) - - -class AssignmentStmt(Statement): - """Assignment statement - The same node class is used for single assignment, multiple assignment - (e.g. x, y = z) and chained assignment (e.g. x = y = z), assignments - that define new names, and assignments with explicit types (# type). - - An lvalue can be NameExpr, TupleExpr, ListExpr, MemberExpr, IndexExpr. - """ - - lvalues = None # type: List[Expression] - rvalue = None # type: Expression - # Declared type in a comment, may be None. - type = None # type: mypy.types.Type - - def __init__(self, lvalues: List[Expression], rvalue: Expression, - type: 'mypy.types.Type' = None) -> None: - self.lvalues = lvalues - self.rvalue = rvalue - self.type = type - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_assignment_stmt(self) - - -class OperatorAssignmentStmt(Statement): - """Operator assignment statement such as x += 1""" - - op = '' - lvalue = None # type: Expression - rvalue = None # type: Expression - - def __init__(self, op: str, lvalue: Expression, rvalue: Expression) -> None: - self.op = op - self.lvalue = lvalue - self.rvalue = rvalue - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_operator_assignment_stmt(self) - - -class WhileStmt(Statement): - expr = None # type: Expression - body = None # type: Block - else_body = None # type: Block - - def __init__(self, expr: Expression, body: Block, else_body: Block) -> None: - self.expr = expr - self.body = body - self.else_body = else_body - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_while_stmt(self) - - -class ForStmt(Statement): - # Index variables - index = None # type: Expression - # Expression to iterate - expr = None # type: Expression - body = None # type: Block - else_body = None # type: Block - - def __init__(self, index: Expression, expr: Expression, body: Block, - else_body: Block) -> None: - self.index = index - self.expr = expr - self.body = body - self.else_body = else_body - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_for_stmt(self) - - -class ReturnStmt(Statement): - expr = None # type: Optional[Expression] - - def __init__(self, expr: Optional[Expression]) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_return_stmt(self) - - -class AssertStmt(Statement): - expr = None # type: Expression - - def __init__(self, expr: Expression) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_assert_stmt(self) - - -class DelStmt(Statement): - expr = None # type: Expression - - def __init__(self, expr: Expression) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_del_stmt(self) - - -class BreakStmt(Statement): - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_break_stmt(self) - - -class ContinueStmt(Statement): - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_continue_stmt(self) - - -class PassStmt(Statement): - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_pass_stmt(self) - - -class IfStmt(Statement): - expr = None # type: List[Expression] - body = None # type: List[Block] - else_body = None # type: Block - - def __init__(self, expr: List[Expression], body: List[Block], - else_body: Block) -> None: - self.expr = expr - self.body = body - self.else_body = else_body - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_if_stmt(self) - - -class RaiseStmt(Statement): - expr = None # type: Expression - from_expr = None # type: Expression - - def __init__(self, expr: Expression, from_expr: Expression = None) -> None: - self.expr = expr - self.from_expr = from_expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_raise_stmt(self) - - -class TryStmt(Statement): - body = None # type: Block # Try body - types = None # type: List[Expression] # Except type expressions - vars = None # type: List[NameExpr] # Except variable names - handlers = None # type: List[Block] # Except bodies - else_body = None # type: Block - finally_body = None # type: Block - - def __init__(self, body: Block, vars: List['NameExpr'], types: List[Expression], - handlers: List[Block], else_body: Block, - finally_body: Block) -> None: - self.body = body - self.vars = vars - self.types = types - self.handlers = handlers - self.else_body = else_body - self.finally_body = finally_body - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_try_stmt(self) - - -class WithStmt(Statement): - expr = None # type: List[Expression] - target = None # type: List[Expression] - body = None # type: Block - - def __init__(self, expr: List[Expression], target: List[Expression], - body: Block) -> None: - self.expr = expr - self.target = target - self.body = body - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_with_stmt(self) - - -class PrintStmt(Statement): - """Python 2 print statement""" - - args = None # type: List[Expression] - newline = False - # The file-like target object (given using >>). - target = None # type: Optional[Expression] - - def __init__(self, args: List[Expression], newline: bool, target: Expression = None) -> None: - self.args = args - self.newline = newline - self.target = target - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_print_stmt(self) - - -class ExecStmt(Statement): - """Python 2 exec statement""" - - expr = None # type: Expression - variables1 = None # type: Optional[Expression] - variables2 = None # type: Optional[Expression] - - def __init__(self, expr: Expression, - variables1: Optional[Expression], - variables2: Optional[Expression]) -> None: - self.expr = expr - self.variables1 = variables1 - self.variables2 = variables2 - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_exec_stmt(self) - - -# Expressions - - -class IntExpr(Expression): - """Integer literal""" - - value = 0 - literal = LITERAL_YES - - def __init__(self, value: int) -> None: - self.value = value - self.literal_hash = value - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_int_expr(self) - - -# How mypy uses StrExpr, BytesExpr, and UnicodeExpr: -# In Python 2 mode: -# b'x', 'x' -> StrExpr -# u'x' -> UnicodeExpr -# BytesExpr is unused -# -# In Python 3 mode: -# b'x' -> BytesExpr -# 'x', u'x' -> StrExpr -# UnicodeExpr is unused - -class StrExpr(Expression): - """String literal""" - - value = '' - literal = LITERAL_YES - - def __init__(self, value: str) -> None: - self.value = value - self.literal_hash = value - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_str_expr(self) - - -class BytesExpr(Expression): - """Bytes literal""" - - value = '' # TODO use bytes - literal = LITERAL_YES - - def __init__(self, value: str) -> None: - self.value = value - self.literal_hash = value - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_bytes_expr(self) - - -class UnicodeExpr(Expression): - """Unicode literal (Python 2.x)""" - - value = '' # TODO use bytes - literal = LITERAL_YES - - def __init__(self, value: str) -> None: - self.value = value - self.literal_hash = value - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_unicode_expr(self) - - -class FloatExpr(Expression): - """Float literal""" - - value = 0.0 - literal = LITERAL_YES - - def __init__(self, value: float) -> None: - self.value = value - self.literal_hash = value - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_float_expr(self) - - -class ComplexExpr(Expression): - """Complex literal""" - - value = 0.0j - literal = LITERAL_YES - - def __init__(self, value: complex) -> None: - self.value = value - self.literal_hash = value - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_complex_expr(self) - - -class EllipsisExpr(Expression): - """Ellipsis (...)""" - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_ellipsis(self) - - -class StarExpr(Expression): - """Star expression""" - - expr = None # type: Expression - - def __init__(self, expr: Expression) -> None: - self.expr = expr - self.literal = self.expr.literal - self.literal_hash = ('Star', expr.literal_hash,) - - # Whether this starred expression is used in a tuple/list and as lvalue - self.valid = False - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_star_expr(self) - - -class RefExpr(Expression): - """Abstract base class for name-like constructs""" - - kind = None # type: int # LDEF/GDEF/MDEF/... (None if not available) - node = None # type: SymbolNode # Var, FuncDef or TypeInfo that describes this - fullname = None # type: str # Fully qualified name (or name if not global) - - # Does this define a new name with inferred type? - # - # For members, after semantic analysis, this does not take base - # classes into consideration at all; the type checker deals with these. - is_def = False - - -class NameExpr(RefExpr): - """Name expression - - This refers to a local name, global name or a module. - """ - - name = None # type: str # Name referred to (may be qualified) - # TypeInfo of class surrounding expression (may be None) - info = None # type: TypeInfo - - literal = LITERAL_TYPE - - def __init__(self, name: str) -> None: - self.name = name - self.literal_hash = ('Var', name,) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_name_expr(self) - - def serialize(self) -> JsonDict: - # TODO: Find out where and why NameExpr is being serialized (if at all). - assert False, "Serializing NameExpr: %s" % (self,) - return {'.class': 'NameExpr', - 'kind': self.kind, - 'node': None if self.node is None else self.node.serialize(), - 'fullname': self.fullname, - 'is_def': self.is_def, - 'name': self.name, - 'literal': self.literal, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'NameExpr': - assert data['.class'] == 'NameExpr' - ret = NameExpr(data['name']) - ret.kind = data['kind'] - ret.node = None if data['node'] is None else SymbolNode.deserialize(data['node']) - ret.fullname = data['fullname'] - ret.is_def = data['is_def'] - ret.literal = data['literal'] - return ret - - -class MemberExpr(RefExpr): - """Member access expression x.y""" - - expr = None # type: Expression - name = None # type: str - # The variable node related to a definition. - def_var = None # type: Var - - def __init__(self, expr: Expression, name: str) -> None: - self.expr = expr - self.name = name - self.literal = self.expr.literal - self.literal_hash = ('Member', expr.literal_hash, name) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_member_expr(self) - - -# Kinds of arguments - -# Positional argument -ARG_POS = 0 # type: int -# Positional, optional argument (functions only, not calls) -ARG_OPT = 1 # type: int -# *arg argument -ARG_STAR = 2 # type: int -# Keyword argument x=y in call, or keyword-only function arg -ARG_NAMED = 3 # type: int -# **arg argument -ARG_STAR2 = 4 # type: int - - -class CallExpr(Expression): - """Call expression. - - This can also represent several special forms that are syntactically calls - such as cast(...) and None # type: .... - """ - - callee = None # type: Expression - args = None # type: List[Expression] - arg_kinds = None # type: List[int] # ARG_ constants - # Each name can be None if not a keyword argument. - arg_names = None # type: List[str] - # If not None, the node that represents the meaning of the CallExpr. For - # cast(...) this is a CastExpr. - analyzed = None # type: Optional[Expression] - - def __init__(self, callee: Expression, args: List[Expression], arg_kinds: List[int], - arg_names: List[str] = None, analyzed: Expression = None) -> None: - if not arg_names: - arg_names = [None] * len(args) - - self.callee = callee - self.args = args - self.arg_kinds = arg_kinds - self.arg_names = arg_names - self.analyzed = analyzed - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_call_expr(self) - - -class YieldFromExpr(Expression): - expr = None # type: Expression - - def __init__(self, expr: Expression) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_yield_from_expr(self) - - -class YieldExpr(Expression): - expr = None # type: Optional[Expression] - - def __init__(self, expr: Optional[Expression]) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_yield_expr(self) - - -class IndexExpr(Expression): - """Index expression x[y]. - - Also wraps type application such as List[int] as a special form. - """ - - base = None # type: Expression - index = None # type: Expression - # Inferred __getitem__ method type - method_type = None # type: mypy.types.Type - # If not None, this is actually semantically a type application - # Class[type, ...] or a type alias initializer. - analyzed = None # type: Union[TypeApplication, TypeAliasExpr] - - def __init__(self, base: Expression, index: Expression) -> None: - self.base = base - self.index = index - self.analyzed = None - if self.index.literal == LITERAL_YES: - self.literal = self.base.literal - self.literal_hash = ('Member', base.literal_hash, - index.literal_hash) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_index_expr(self) - - -class UnaryExpr(Expression): - """Unary operation""" - - op = '' - expr = None # type: Expression - # Inferred operator method type - method_type = None # type: mypy.types.Type - - def __init__(self, op: str, expr: Expression) -> None: - self.op = op - self.expr = expr - self.literal = self.expr.literal - self.literal_hash = ('Unary', op, expr.literal_hash) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_unary_expr(self) - - -# Map from binary operator id to related method name (in Python 3). -op_methods = { - '+': '__add__', - '-': '__sub__', - '*': '__mul__', - '/': '__truediv__', - '%': '__mod__', - '//': '__floordiv__', - '**': '__pow__', - '&': '__and__', - '|': '__or__', - '^': '__xor__', - '<<': '__lshift__', - '>>': '__rshift__', - '==': '__eq__', - '!=': '__ne__', - '<': '__lt__', - '>=': '__ge__', - '>': '__gt__', - '<=': '__le__', - 'in': '__contains__', -} # type: Dict[str, str] - -ops_with_inplace_method = { - '+', '-', '*', '/', '%', '//', '**', '&', '|', '^', '<<', '>>'} - -inplace_operator_methods = set( - '__i' + op_methods[op][2:] for op in ops_with_inplace_method) - -reverse_op_methods = { - '__add__': '__radd__', - '__sub__': '__rsub__', - '__mul__': '__rmul__', - '__truediv__': '__rtruediv__', - '__mod__': '__rmod__', - '__floordiv__': '__rfloordiv__', - '__pow__': '__rpow__', - '__and__': '__rand__', - '__or__': '__ror__', - '__xor__': '__rxor__', - '__lshift__': '__rlshift__', - '__rshift__': '__rrshift__', - '__eq__': '__eq__', - '__ne__': '__ne__', - '__lt__': '__gt__', - '__ge__': '__le__', - '__gt__': '__lt__', - '__le__': '__ge__', -} - -normal_from_reverse_op = dict((m, n) for n, m in reverse_op_methods.items()) -reverse_op_method_set = set(reverse_op_methods.values()) - - -class OpExpr(Expression): - """Binary operation (other than . or [] or comparison operators, - which have specific nodes).""" - - op = '' - left = None # type: Expression - right = None # type: Expression - # Inferred type for the operator method type (when relevant). - method_type = None # type: mypy.types.Type - - def __init__(self, op: str, left: Expression, right: Expression) -> None: - self.op = op - self.left = left - self.right = right - self.literal = min(self.left.literal, self.right.literal) - self.literal_hash = ('Binary', op, left.literal_hash, right.literal_hash) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_op_expr(self) - - -class ComparisonExpr(Expression): - """Comparison expression (e.g. a < b > c < d).""" - - operators = None # type: List[str] - operands = None # type: List[Expression] - # Inferred type for the operator methods (when relevant; None for 'is'). - method_types = None # type: List[mypy.types.Type] - - def __init__(self, operators: List[str], operands: List[Expression]) -> None: - self.operators = operators - self.operands = operands - self.method_types = [] - self.literal = min(o.literal for o in self.operands) - self.literal_hash = (('Comparison',) + tuple(operators) + - tuple(o.literal_hash for o in operands)) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_comparison_expr(self) - - -class SliceExpr(Expression): - """Slice expression (e.g. 'x:y', 'x:', '::2' or ':'). - - This is only valid as index in index expressions. - """ - - begin_index = None # type: Optional[Expression] - end_index = None # type: Optional[Expression] - stride = None # type: Optional[Expression] - - def __init__(self, begin_index: Optional[Expression], - end_index: Optional[Expression], - stride: Optional[Expression]) -> None: - self.begin_index = begin_index - self.end_index = end_index - self.stride = stride - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_slice_expr(self) - - -class CastExpr(Expression): - """Cast expression cast(type, expr).""" - - expr = None # type: Expression - type = None # type: mypy.types.Type - - def __init__(self, expr: Expression, typ: 'mypy.types.Type') -> None: - self.expr = expr - self.type = typ - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_cast_expr(self) - - -class RevealTypeExpr(Expression): - """Reveal type expression reveal_type(expr).""" - - expr = None # type: Expression - - def __init__(self, expr: Expression) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_reveal_type_expr(self) - - -class SuperExpr(Expression): - """Expression super().name""" - - name = '' - info = None # type: TypeInfo # Type that contains this super expression - - def __init__(self, name: str) -> None: - self.name = name - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_super_expr(self) - - -class FuncExpr(FuncItem, Expression): - """Lambda expression""" - - def name(self) -> str: - return '' - - def expr(self) -> Expression: - """Return the expression (the body) of the lambda.""" - ret = cast(ReturnStmt, self.body.body[-1]) - return ret.expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_func_expr(self) - - -class ListExpr(Expression): - """List literal expression [...].""" - - items = None # type: List[Expression] - - def __init__(self, items: List[Expression]) -> None: - self.items = items - if all(x.literal == LITERAL_YES for x in items): - self.literal = LITERAL_YES - self.literal_hash = ('List',) + tuple(x.literal_hash for x in items) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_list_expr(self) - - -class DictExpr(Expression): - """Dictionary literal expression {key: value, ...}.""" - - items = None # type: List[Tuple[Expression, Expression]] - - def __init__(self, items: List[Tuple[Expression, Expression]]) -> None: - self.items = items - if all(x[0].literal == LITERAL_YES and x[1].literal == LITERAL_YES - for x in items): - self.literal = LITERAL_YES - self.literal_hash = ('Dict',) + tuple( - (x[0].literal_hash, x[1].literal_hash) for x in items) # type: ignore - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_dict_expr(self) - - -class TupleExpr(Expression): - """Tuple literal expression (..., ...)""" - - items = None # type: List[Expression] - - def __init__(self, items: List[Expression]) -> None: - self.items = items - if all(x.literal == LITERAL_YES for x in items): - self.literal = LITERAL_YES - self.literal_hash = ('Tuple',) + tuple(x.literal_hash for x in items) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_tuple_expr(self) - - -class SetExpr(Expression): - """Set literal expression {value, ...}.""" - - items = None # type: List[Expression] - - def __init__(self, items: List[Expression]) -> None: - self.items = items - if all(x.literal == LITERAL_YES for x in items): - self.literal = LITERAL_YES - self.literal_hash = ('Set',) + tuple(x.literal_hash for x in items) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_set_expr(self) - - -class GeneratorExpr(Expression): - """Generator expression ... for ... in ... [ for ... in ... ] [ if ... ].""" - - left_expr = None # type: Expression - sequences_expr = None # type: List[Expression] - condlists = None # type: List[List[Expression]] - indices = None # type: List[Expression] - - def __init__(self, left_expr: Expression, indices: List[Expression], - sequences: List[Expression], condlists: List[List[Expression]]) -> None: - self.left_expr = left_expr - self.sequences = sequences - self.condlists = condlists - self.indices = indices - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_generator_expr(self) - - -class ListComprehension(Expression): - """List comprehension (e.g. [x + 1 for x in a])""" - - generator = None # type: GeneratorExpr - - def __init__(self, generator: GeneratorExpr) -> None: - self.generator = generator - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_list_comprehension(self) - - -class SetComprehension(Expression): - """Set comprehension (e.g. {x + 1 for x in a})""" - - generator = None # type: GeneratorExpr - - def __init__(self, generator: GeneratorExpr) -> None: - self.generator = generator - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_set_comprehension(self) - - -class DictionaryComprehension(Expression): - """Dictionary comprehension (e.g. {k: v for k, v in a}""" - - key = None # type: Expression - value = None # type: Expression - sequences_expr = None # type: List[Expression] - condlists = None # type: List[List[Expression]] - indices = None # type: List[Expression] - - def __init__(self, key: Expression, value: Expression, indices: List[Expression], - sequences: List[Expression], condlists: List[List[Expression]]) -> None: - self.key = key - self.value = value - self.sequences = sequences - self.condlists = condlists - self.indices = indices - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_dictionary_comprehension(self) - - -class ConditionalExpr(Expression): - """Conditional expression (e.g. x if y else z)""" - - cond = None # type: Expression - if_expr = None # type: Expression - else_expr = None # type: Expression - - def __init__(self, cond: Expression, if_expr: Expression, else_expr: Expression) -> None: - self.cond = cond - self.if_expr = if_expr - self.else_expr = else_expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_conditional_expr(self) - - -class BackquoteExpr(Expression): - """Python 2 expression `...`.""" - - expr = None # type: Expression - - def __init__(self, expr: Expression) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_backquote_expr(self) - - -class TypeApplication(Expression): - """Type application expr[type, ...]""" - - expr = None # type: Expression - types = None # type: List[mypy.types.Type] - - def __init__(self, expr: Expression, types: List['mypy.types.Type']) -> None: - self.expr = expr - self.types = types - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_type_application(self) - - -# Variance of a type variable. For example, T in the definition of -# List[T] is invariant, so List[int] is not a subtype of List[object], -# and also List[object] is not a subtype of List[int]. -# -# The T in Iterable[T] is covariant, so Iterable[int] is a subtype of -# Iterable[object], but not vice versa. -# -# If T is contravariant in Foo[T], Foo[object] is a subtype of -# Foo[int], but not vice versa. -INVARIANT = 0 # type: int -COVARIANT = 1 # type: int -CONTRAVARIANT = 2 # type: int - - -class TypeVarExpr(SymbolNode, Expression): - """Type variable expression TypeVar(...).""" - - _name = '' - _fullname = '' - # Value restriction: only types in the list are valid as values. If the - # list is empty, there is no restriction. - values = None # type: List[mypy.types.Type] - # Upper bound: only subtypes of upper_bound are valid as values. By default - # this is 'object', meaning no restriction. - upper_bound = None # type: mypy.types.Type - # Variance of the type variable. Invariant is the default. - # TypeVar(..., covariant=True) defines a covariant type variable. - # TypeVar(..., contravariant=True) defines a contravariant type - # variable. - variance = INVARIANT - - def __init__(self, name: str, fullname: str, - values: List['mypy.types.Type'], - upper_bound: 'mypy.types.Type', - variance: int=INVARIANT) -> None: - self._name = name - self._fullname = fullname - self.values = values - self.upper_bound = upper_bound - self.variance = variance - - def name(self) -> str: - return self._name - - def fullname(self) -> str: - return self._fullname - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_type_var_expr(self) - - def serialize(self) -> JsonDict: - return {'.class': 'TypeVarExpr', - 'name': self._name, - 'fullname': self._fullname, - 'values': [t.serialize() for t in self.values], - 'upper_bound': self.upper_bound.serialize(), - 'variance': self.variance, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'TypeVarExpr': - assert data['.class'] == 'TypeVarExpr' - return TypeVarExpr(data['name'], - data['fullname'], - [mypy.types.Type.deserialize(v) for v in data['values']], - mypy.types.Type.deserialize(data['upper_bound']), - data['variance']) - - -class TypeAliasExpr(Expression): - """Type alias expression (rvalue).""" - - type = None # type: mypy.types.Type - - def __init__(self, type: 'mypy.types.Type') -> None: - self.type = type - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_type_alias_expr(self) - - -class NamedTupleExpr(Expression): - """Named tuple expression namedtuple(...).""" - - # The class representation of this named tuple (its tuple_type attribute contains - # the tuple item types) - info = None # type: TypeInfo - - def __init__(self, info: 'TypeInfo') -> None: - self.info = info - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_namedtuple_expr(self) - - -class PromoteExpr(Expression): - """Ducktype class decorator expression _promote(...).""" - - type = None # type: mypy.types.Type - - def __init__(self, type: 'mypy.types.Type') -> None: - self.type = type - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit__promote_expr(self) - - -# Constants - - -class TempNode(Expression): - """Temporary dummy node used during type checking. - - This node is not present in the original program; it is just an artifact - of the type checker implementation. It only represents an opaque node with - some fixed type. - """ - - type = None # type: mypy.types.Type - - def __init__(self, typ: 'mypy.types.Type') -> None: - self.type = typ - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_temp_node(self) - - -class TypeInfo(SymbolNode): - """The type structure of a single class. - - Each TypeInfo corresponds one-to-one to a ClassDef, which - represents the AST of the class. - - In type-theory terms, this is a "type constructor", and if the - class is generic then it will be a type constructor of higher kind. - Where the class is used in an actual type, it's in the form of an - Instance, which amounts to a type application of the tycon to - the appropriate number of arguments. - """ - - _fullname = None # type: str # Fully qualified name - defn = None # type: ClassDef # Corresponding ClassDef - # Method Resolution Order: the order of looking up attributes. The first - # value always to refers to this class. - mro = None # type: List[TypeInfo] - subtypes = None # type: Set[TypeInfo] # Direct subclasses encountered so far - names = None # type: SymbolTable # Names defined directly in this type - is_abstract = False # Does the class have any abstract attributes? - abstract_attributes = None # type: List[str] - # Classes inheriting from Enum shadow their true members with a __getattr__, so we - # have to treat them as a special case. - is_enum = False - # If true, any unknown attributes should have type 'Any' instead - # of generating a type error. This would be true if there is a - # base class with type 'Any', but other use cases may be - # possible. This is similar to having __getattr__ that returns Any - # (and __setattr__), but without the __getattr__ method. - fallback_to_any = False - - # Information related to type annotations. - - # Generic type variable names - type_vars = None # type: List[str] - - # Direct base classes. - bases = None # type: List[mypy.types.Instance] - - # Another type which this type will be treated as a subtype of, - # even though it's not a subclass in Python. The non-standard - # `@_promote` decorator introduces this, and there are also - # several builtin examples, in particular `int` -> `float`. - _promote = None # type: mypy.types.Type - - # Representation of a Tuple[...] base class, if the class has any - # (e.g., for named tuples). If this is not None, the actual Type - # object used for this class is not an Instance but a TupleType; - # the corresponding Instance is set as the fallback type of the - # tuple type. - tuple_type = None # type: mypy.types.TupleType - - # Is this a named tuple type? - is_named_tuple = False - - # Is this a dummy from deserialization? - is_dummy = False - - # Alternative to fullname() for 'anonymous' classes. - alt_fullname = None # type: Optional[str] - - def __init__(self, names: 'SymbolTable', defn: ClassDef) -> None: - """Initialize a TypeInfo.""" - self.names = names - self.defn = defn - self.subtypes = set() - self.type_vars = [] - self.bases = [] - # Leave self.mro uninitialized until we compute it for real, - # so we don't accidentally try to use it prematurely. - self._fullname = defn.fullname - self.is_abstract = False - self.abstract_attributes = [] - if defn.type_vars: - for vd in defn.type_vars: - self.type_vars.append(vd.name) - - def name(self) -> str: - """Short name.""" - return self.defn.name - - def fullname(self) -> str: - return self._fullname - - def is_generic(self) -> bool: - """Is the type generic (i.e. does it have type variables)?""" - return len(self.type_vars) > 0 - - def get(self, name: str) -> 'SymbolTableNode': - for cls in self.mro: - n = cls.names.get(name) - if n: - return n - return None - - def __getitem__(self, name: str) -> 'SymbolTableNode': - n = self.get(name) - if n: - return n - else: - raise KeyError(name) - - def __repr__(self) -> str: - return '' % self.fullname() - - # IDEA: Refactor the has* methods to be more consistent and document - # them. - - def has_readable_member(self, name: str) -> bool: - return self.get(name) is not None - - def has_writable_member(self, name: str) -> bool: - return self.has_var(name) - - def has_var(self, name: str) -> bool: - return self.get_var(name) is not None - - def has_method(self, name: str) -> bool: - return self.get_method(name) is not None - - def get_var(self, name: str) -> Var: - for cls in self.mro: - if name in cls.names: - node = cls.names[name].node - if isinstance(node, Var): - return node - else: - return None - return None - - def get_var_or_getter(self, name: str) -> SymbolNode: - # TODO getter - return self.get_var(name) - - def get_var_or_setter(self, name: str) -> SymbolNode: - # TODO setter - return self.get_var(name) - - def get_method(self, name: str) -> FuncBase: - if self.mro is None: # Might be because of a previous error. - return None - for cls in self.mro: - if name in cls.names: - node = cls.names[name].node - if isinstance(node, FuncBase): - return node - else: - return None - return None - - def calculate_mro(self) -> None: - """Calculate and set mro (method resolution order). - - Raise MroError if cannot determine mro. - """ - mro = linearize_hierarchy(self) - assert mro, "Could not produce a MRO at all for %s" % (self,) - self.mro = mro - self.is_enum = self._calculate_is_enum() - - def _calculate_is_enum(self) -> bool: - """ - If this is "enum.Enum" itself, then yes, it's an enum. - If the flag .is_enum has been set on anything in the MRO, it's an enum. - """ - if self.fullname() == ENUM_BASECLASS: - return True - if self.mro: - return any(type_info.is_enum for type_info in self.mro) - return False - - def has_base(self, fullname: str) -> bool: - """Return True if type has a base type with the specified name. - - This can be either via extension or via implementation. - """ - if self.mro: - for cls in self.mro: - if cls.fullname() == fullname: - return True - return False - - def all_subtypes(self) -> 'Set[TypeInfo]': - """Return TypeInfos of all subtypes, including this type, as a set.""" - subtypes = set([self]) - for subt in self.subtypes: - for t in subt.all_subtypes(): - subtypes.add(t) - return subtypes - - def all_base_classes(self) -> 'List[TypeInfo]': - """Return a list of base classes, including indirect bases.""" - assert False - - def direct_base_classes(self) -> 'List[TypeInfo]': - """Return a direct base classes. - - Omit base classes of other base classes. - """ - return [base.type for base in self.bases] - - def __str__(self) -> str: - """Return a string representation of the type. - - This includes the most important information about the type. - """ - base = None # type: str - if self.bases: - base = 'Bases({})'.format(', '.join(str(base) - for base in self.bases)) - return dump_tagged(['Name({})'.format(self.fullname()), - base, - ('Names', sorted(self.names.keys()))], - 'TypeInfo') - - def serialize(self) -> Union[str, JsonDict]: - # NOTE: This is where all ClassDefs originate, so there shouldn't be duplicates. - data = {'.class': 'TypeInfo', - 'fullname': self.fullname(), - 'alt_fullname': self.alt_fullname, - 'names': self.names.serialize(self.alt_fullname or self.fullname()), - 'defn': self.defn.serialize(), - 'is_abstract': self.is_abstract, - 'abstract_attributes': self.abstract_attributes, - 'is_enum': self.is_enum, - 'fallback_to_any': self.fallback_to_any, - 'type_vars': self.type_vars, - 'bases': [b.serialize() for b in self.bases], - '_promote': None if self._promote is None else self._promote.serialize(), - 'tuple_type': None if self.tuple_type is None else self.tuple_type.serialize(), - 'is_named_tuple': self.is_named_tuple, - } - return data - - @classmethod - def deserialize(cls, data: JsonDict) -> 'TypeInfo': - names = SymbolTable.deserialize(data['names']) - defn = ClassDef.deserialize(data['defn']) - ti = TypeInfo(names, defn) - ti._fullname = data['fullname'] - ti.alt_fullname = data['alt_fullname'] - # TODO: Is there a reason to reconstruct ti.subtypes? - ti.is_abstract = data['is_abstract'] - ti.abstract_attributes = data['abstract_attributes'] - ti.is_enum = data['is_enum'] - ti.fallback_to_any = data['fallback_to_any'] - ti.type_vars = data['type_vars'] - ti.bases = [mypy.types.Instance.deserialize(b) for b in data['bases']] - ti._promote = (None if data['_promote'] is None - else mypy.types.Type.deserialize(data['_promote'])) - ti.tuple_type = (None if data['tuple_type'] is None - else mypy.types.TupleType.deserialize(data['tuple_type'])) - ti.is_named_tuple = data['is_named_tuple'] - return ti - - -def NamedTupleTypeInfo(tup: 'mypy.types.NamedTupleType', *args) -> TypeInfo: - info = TypeInfo(*args) - info.tuple_type = tup - info.bases = [tup.fallback] - info.is_named_tuple = True - info.mro = [info] + tup.fallback.type.mro - return info - -class SymbolTableNode: - # Kind of node. Possible values: - # - LDEF: local definition (of any kind) - # - GDEF: global (module-level) definition - # - MDEF: class member definition - # - UNBOUND_TVAR: TypeVar(...) definition, not bound - # - TVAR: type variable in a bound scope (generic function / generic clas) - # - MODULE_REF: reference to a module - # - TYPE_ALIAS: type alias - # - UNBOUND_IMPORTED: temporary kind for imported names - kind = None # type: int - # AST node of definition (FuncDef/Var/TypeInfo/Decorator/TypeVarExpr, - # or None for a bound type variable). - node = None # type: Optional[SymbolNode] - # Type variable definition (for bound type variables only) - tvar_def = None # type: Optional[mypy.types.TypeVarDef] - # Module id (e.g. "foo.bar") or None - mod_id = '' - # If this not None, override the type of the 'node' attribute. - type_override = None # type: Optional[mypy.types.Type] - # If False, this name won't be imported via 'from import *'. - # This has no effect on names within classes. - module_public = True - # For deserialized MODULE_REF nodes, the referenced module name; - # for other nodes, optionally the name of the referenced object. - cross_ref = None # type: Optional[str] - - def __init__(self, kind: int, node: Optional[SymbolNode], mod_id: str = None, - typ: 'mypy.types.Type' = None, - tvar_def: 'mypy.types.TypeVarDef' = None, - module_public: bool = True) -> None: - self.kind = kind - self.node = node - self.type_override = typ - self.mod_id = mod_id - self.tvar_def = tvar_def - self.module_public = module_public - - @property - def fullname(self) -> str: - if self.node is not None: - return self.node.fullname() - else: - return None - - @property - def type(self) -> 'mypy.types.Type': - # IDEA: Get rid of the Any type. - node = self.node # type: Any - if self.type_override is not None: - return self.type_override - elif ((isinstance(node, Var) or isinstance(node, FuncDef)) - and node.type is not None): - return node.type - elif isinstance(node, Decorator): - return node.var.type - else: - return None - - def __str__(self) -> str: - s = '{}/{}'.format(node_kinds[self.kind], short_type(self.node)) - if self.mod_id is not None: - s += ' ({})'.format(self.mod_id) - # Include declared type of variables and functions. - if self.type is not None: - s += ' : {}'.format(self.type) - return s - - def serialize(self, prefix: str, name: str) -> JsonDict: - """Serialize a SymbolTableNode. - - Args: - prefix: full name of the containing module or class; or None - name: name of this object relative to the containing object - """ - data = {'.class': 'SymbolTableNode', - 'kind': node_kinds[self.kind], - } # type: JsonDict - if self.tvar_def: - data['tvar_def'] = self.tvar_def.serialize() - if not self.module_public: - data['module_public'] = False - if self.kind == MODULE_REF: - assert self.node is not None, "Missing module cross ref in %s for %s" % (prefix, name) - data['cross_ref'] = self.node.fullname() - else: - if self.node is not None: - if prefix is not None: - # Check whether this is an alias for another object. - # If the object's canonical full name differs from - # the full name computed from prefix and name, - # it's an alias, and we serialize it as a cross ref. - if isinstance(self.node, TypeInfo): - fullname = self.node.alt_fullname or self.node.fullname() - else: - fullname = self.node.fullname() - if (fullname is not None and '.' in fullname and - fullname != prefix + '.' + name): - data['cross_ref'] = fullname - return data - data['node'] = self.node.serialize() - if self.type_override is not None: - data['type_override'] = self.type_override.serialize() - return data - - @classmethod - def deserialize(cls, data: JsonDict) -> 'SymbolTableNode': - assert data['.class'] == 'SymbolTableNode' - kind = inverse_node_kinds[data['kind']] - if 'cross_ref' in data: - # This will be fixed up later. - stnode = SymbolTableNode(kind, None) - stnode.cross_ref = data['cross_ref'] - else: - node = None - if 'node' in data: - node = SymbolNode.deserialize(data['node']) - typ = None - if 'type_override' in data: - typ = mypy.types.Type.deserialize(data['type_override']) - stnode = SymbolTableNode(kind, node, typ=typ) - if 'tvar_def' in data: - stnode.tvar_def = mypy.types.TypeVarDef.deserialize(data['tvar_def']) - if 'module_public' in data: - stnode.module_public = data['module_public'] - return stnode - - -class SymbolTable(Dict[str, SymbolTableNode]): - def __str__(self) -> str: - a = [] # type: List[str] - for key, value in self.items(): - # Filter out the implicit import of builtins. - if isinstance(value, SymbolTableNode): - if (value.fullname != 'builtins' and - (value.fullname or '').split('.')[-1] not in - implicit_module_attrs): - a.append(' ' + str(key) + ' : ' + str(value)) - else: - a.append(' ') - a = sorted(a) - a.insert(0, 'SymbolTable(') - a[-1] += ')' - return '\n'.join(a) - - def serialize(self, fullname: str) -> JsonDict: - data = {'.class': 'SymbolTable'} # type: JsonDict - for key, value in self.items(): - # Skip __builtins__: it's a reference to the builtins - # module that gets added to every module by - # SemanticAnalyzer.visit_file(), but it shouldn't be - # accessed by users of the module. - if key == '__builtins__': - continue - data[key] = value.serialize(fullname, key) - return data - - @classmethod - def deserialize(cls, data: JsonDict) -> 'SymbolTable': - assert data['.class'] == 'SymbolTable' - st = SymbolTable() - for key, value in data.items(): - if key != '.class': - st[key] = SymbolTableNode.deserialize(value) - return st - - -def function_type(func: FuncBase, fallback: 'mypy.types.Instance') -> 'mypy.types.FunctionLike': - if func.type: - assert isinstance(func.type, mypy.types.FunctionLike) - return func.type - else: - # Implicit type signature with dynamic types. - # Overloaded functions always have a signature, so func must be an ordinary function. - fdef = cast(FuncDef, func) - name = func.name() - if name: - name = '"{}"'.format(name) - names = [] # type: List[str] - for arg in fdef.arguments: - names.append(arg.variable.name()) - - return mypy.types.CallableType( - [mypy.types.AnyType()] * len(fdef.arguments), - [arg.kind for arg in fdef.arguments], - names, - mypy.types.AnyType(), - fallback, - name, - implicit=True, - ) - - -def method_type_with_fallback(func: FuncBase, - fallback: 'mypy.types.Instance') -> 'mypy.types.FunctionLike': - """Return the signature of a method (omit self).""" - return method_type(function_type(func, fallback)) - - -def method_type(sig: 'mypy.types.FunctionLike') -> 'mypy.types.FunctionLike': - if isinstance(sig, mypy.types.CallableType): - return method_callable(sig) - else: - sig = cast(mypy.types.Overloaded, sig) - items = [] # type: List[mypy.types.CallableType] - for c in sig.items(): - items.append(method_callable(c)) - return mypy.types.Overloaded(items) - - -def method_callable(c: 'mypy.types.CallableType') -> 'mypy.types.CallableType': - if c.arg_kinds and c.arg_kinds[0] == ARG_STAR: - # The signature is of the form 'def foo(*args, ...)'. - # In this case we shouldn't drop the first arg, - # since self will be absorbed by the *args. - return c - return c.copy_modified(arg_types=c.arg_types[1:], - arg_kinds=c.arg_kinds[1:], - arg_names=c.arg_names[1:]) - - -class MroError(Exception): - """Raised if a consistent mro cannot be determined for a class.""" - - -def linearize_hierarchy(info: TypeInfo) -> List[TypeInfo]: - # TODO describe - if info.mro: - return info.mro - bases = info.direct_base_classes() - lin_bases = [] - for base in bases: - assert base is not None, "Cannot linearize bases for %s %s" % (info.fullname(), bases) - lin_bases.append(linearize_hierarchy(base)) - lin_bases.append(bases) - return [info] + merge(lin_bases) - - -def merge(seqs: List[List[TypeInfo]]) -> List[TypeInfo]: - seqs = [s[:] for s in seqs] - result = [] # type: List[TypeInfo] - while True: - seqs = [s for s in seqs if s] - if not seqs: - return result - for seq in seqs: - head = seq[0] - if not [s for s in seqs if head in s[1:]]: - break - else: - raise MroError() - result.append(head) - for s in seqs: - if s[0] is head: - del s[0] +"""Abstract syntax tree node classes (i.e. parse tree).""" + +import os +import re +from abc import abstractmethod, ABCMeta + +from typing import ( + Any, TypeVar, List, Tuple, cast, Set, Dict, Union, Optional +) + +from mypy.lex import Token +import mypy.strconv +from mypy.visitor import NodeVisitor +from mypy.util import dump_tagged, short_type + + +class Context: + """Base type for objects that are valid as error message locations.""" + @abstractmethod + def get_line(self) -> int: pass + + +if False: + # break import cycle only needed for mypy + import mypy.types + + +T = TypeVar('T') + +JsonDict = Dict[str, Any] + + +# Symbol table node kinds +# +# TODO rename to use more descriptive names + +LDEF = 0 # type: int +GDEF = 1 # type: int +MDEF = 2 # type: int +MODULE_REF = 3 # type: int +# Type variable declared using TypeVar(...) has kind UNBOUND_TVAR. It's not +# valid as a type. A type variable is valid as a type (kind BOUND_TVAR) within +# (1) a generic class that uses the type variable as a type argument or +# (2) a generic function that refers to the type variable in its signature. +UNBOUND_TVAR = 4 # type: int +BOUND_TVAR = 5 # type: int +TYPE_ALIAS = 6 # type: int +# Placeholder for a name imported via 'from ... import'. Second phase of +# semantic will replace this the actual imported reference. This is +# needed so that we can detect whether a name has been imported during +# XXX what? +UNBOUND_IMPORTED = 7 # type: int + + +LITERAL_YES = 2 +LITERAL_TYPE = 1 +LITERAL_NO = 0 + +# Hard coded name of Enum baseclass. +ENUM_BASECLASS = "enum.Enum" + +node_kinds = { + LDEF: 'Ldef', + GDEF: 'Gdef', + MDEF: 'Mdef', + MODULE_REF: 'ModuleRef', + UNBOUND_TVAR: 'UnboundTvar', + BOUND_TVAR: 'Tvar', + TYPE_ALIAS: 'TypeAlias', + UNBOUND_IMPORTED: 'UnboundImported', +} +inverse_node_kinds = {_kind: _name for _name, _kind in node_kinds.items()} + + +implicit_module_attrs = {'__name__': '__builtins__.str', + '__doc__': '__builtins__.str', + '__file__': '__builtins__.str', + '__package__': '__builtins__.str'} + + +type_aliases = { + 'typing.List': '__builtins__.list', + 'typing.Dict': '__builtins__.dict', + 'typing.Set': '__builtins__.set', +} + +reverse_type_aliases = dict((name.replace('__builtins__', 'builtins'), alias) + for alias, name in type_aliases.items()) # type: Dict[str, str] + + +class Node(Context): + """Common base class for all non-type parse tree nodes.""" + + line = -1 + + literal = LITERAL_NO + literal_hash = None # type: Any + + def __str__(self) -> str: + ans = self.accept(mypy.strconv.StrConv()) + if ans is None: + return repr(self) + return ans + + def set_line(self, target: Union[Token, 'Node', int]) -> 'Node': + if isinstance(target, int): + self.line = target + else: + self.line = target.line + return self + + def get_line(self) -> int: + # TODO this should be just 'line' + return self.line + + def accept(self, visitor: NodeVisitor[T]) -> T: + raise RuntimeError('Not implemented') + + +# These are placeholders for a future refactoring; see #1783. +# For now they serve as (unchecked) documentation of what various +# fields of Node subtypes are expected to contain. +Statement = Node +Expression = Node + + +class SymbolNode(Node): + # Nodes that can be stored in a symbol table. + + # TODO do not use methods for these + + @abstractmethod + def name(self) -> str: pass + + @abstractmethod + def fullname(self) -> str: pass + + # NOTE: Can't use @abstractmethod, since many subclasses of Node + # don't implement serialize(). + def serialize(self) -> Any: + raise NotImplementedError('Cannot serialize {} instance'.format(self.__class__.__name__)) + + @classmethod + def deserialize(cls, data: JsonDict) -> 'SymbolNode': + classname = data['.class'] + glo = globals() + if classname in glo: + cl = glo[classname] + if issubclass(cl, cls) and 'deserialize' in cl.__dict__: + return cl.deserialize(data) + raise NotImplementedError('unexpected .class {}'.format(classname)) + + +class MypyFile(SymbolNode, Statement): + """The abstract syntax tree of a single source file.""" + + # Module name ('__main__' for initial file) + _name = None # type: str + # Fully qualified module name + _fullname = None # type: str + # Path to the file (None if not known) + path = '' + # Top-level definitions and statements + defs = None # type: List[Statement] + # Is there a UTF-8 BOM at the start? + is_bom = False + names = None # type: SymbolTable + # All import nodes within the file (also ones within functions etc.) + imports = None # type: List[ImportBase] + # Lines to ignore when checking + ignored_lines = None # type: Set[int] + # Is this file represented by a stub file (.pyi)? + is_stub = False + # Do weak typing globally in the file? + weak_opts = None # type: Set[str] + + def __init__(self, + defs: List[Statement], + imports: List['ImportBase'], + is_bom: bool = False, + ignored_lines: Set[int] = None, + weak_opts: Set[str] = None) -> None: + self.defs = defs + self.line = 1 # Dummy line number + self.imports = imports + self.is_bom = is_bom + self.weak_opts = weak_opts + if ignored_lines: + self.ignored_lines = ignored_lines + else: + self.ignored_lines = set() + + def name(self) -> str: + return self._name + + def fullname(self) -> str: + return self._fullname + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_mypy_file(self) + + def is_package_init_file(self) -> bool: + return not (self.path is None) and len(self.path) != 0 \ + and os.path.basename(self.path).startswith('__init__.') + + def serialize(self) -> JsonDict: + return {'.class': 'MypyFile', + '_name': self._name, + '_fullname': self._fullname, + 'names': self.names.serialize(self._fullname), + 'is_stub': self.is_stub, + 'path': self.path, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'MypyFile': + assert data['.class'] == 'MypyFile', data + tree = MypyFile([], []) + tree._name = data['_name'] + tree._fullname = data['_fullname'] + tree.names = SymbolTable.deserialize(data['names']) + tree.is_stub = data['is_stub'] + tree.path = data['path'] + return tree + + +class ImportBase(Statement): + """Base class for all import statements.""" + is_unreachable = False + is_top_level = False # Set by semanal.FirstPass + # If an import replaces existing definitions, we construct dummy assignment + # statements that assign the imported names to the names in the current scope, + # for type checking purposes. Example: + # + # x = 1 + # from m import x <-- add assignment representing "x = m.x" + assignments = None # type: List[AssignmentStmt] + + def __init__(self) -> None: + self.assignments = [] + + +class Import(ImportBase): + """import m [as n]""" + + ids = None # type: List[Tuple[str, Optional[str]]] # (module id, as id) + + def __init__(self, ids: List[Tuple[str, Optional[str]]]) -> None: + super().__init__() + self.ids = ids + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_import(self) + + +class ImportFrom(ImportBase): + """from m import x [as y], ...""" + + names = None # type: List[Tuple[str, Optional[str]]] # Tuples (name, as name) + + def __init__(self, id: str, relative: int, names: List[Tuple[str, Optional[str]]]) -> None: + super().__init__() + self.id = id + self.names = names + self.relative = relative + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_import_from(self) + + +class ImportAll(ImportBase): + """from m import *""" + + def __init__(self, id: str, relative: int) -> None: + super().__init__() + self.id = id + self.relative = relative + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_import_all(self) + + +class FuncBase(SymbolNode): + """Abstract base class for function-like nodes""" + + # Type signature. This is usually CallableType or Overloaded, but it can be something else for + # decorated functions/ + type = None # type: mypy.types.Type + # If method, reference to TypeInfo + info = None # type: TypeInfo + is_property = False + _fullname = None # type: str # Name with module prefix + + @abstractmethod + def name(self) -> str: pass + + def fullname(self) -> str: + return self._fullname + + def is_method(self) -> bool: + return bool(self.info) + + +class OverloadedFuncDef(FuncBase, Statement): + """A logical node representing all the variants of an overloaded function. + + This node has no explicit representation in the source program. + Overloaded variants must be consecutive in the source file. + """ + + items = None # type: List[Decorator] + + def __init__(self, items: List['Decorator']) -> None: + self.items = items + self.set_line(items[0].line) + + def name(self) -> str: + return self.items[0].func.name() + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_overloaded_func_def(self) + + def serialize(self) -> JsonDict: + return {'.class': 'OverloadedFuncDef', + 'items': [i.serialize() for i in self.items], + 'type': None if self.type is None else self.type.serialize(), + 'fullname': self._fullname, + 'is_property': self.is_property, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'OverloadedFuncDef': + assert data['.class'] == 'OverloadedFuncDef' + res = OverloadedFuncDef([Decorator.deserialize(d) for d in data['items']]) + if data.get('type') is not None: + res.type = mypy.types.Type.deserialize(data['type']) + res._fullname = data['fullname'] + res.is_property = data['is_property'] + # NOTE: res.info will be set in the fixup phase. + return res + + +class Argument(Node): + """A single argument in a FuncItem.""" + + def __init__(self, variable: 'Var', type_annotation: 'Optional[mypy.types.Type]', + initializer: Optional[Expression], kind: int, + initialization_statement: Optional['AssignmentStmt'] = None) -> None: + self.variable = variable + + self.type_annotation = type_annotation + self.initializer = initializer + + self.initialization_statement = initialization_statement + if not self.initialization_statement: + self.initialization_statement = self._initialization_statement() + + self.kind = kind + + def _initialization_statement(self) -> Optional['AssignmentStmt']: + """Convert the initializer into an assignment statement. + """ + if not self.initializer: + return None + + rvalue = self.initializer + lvalue = NameExpr(self.variable.name()) + assign = AssignmentStmt([lvalue], rvalue) + return assign + + def set_line(self, target: Union[Token, Node, int]) -> Node: + super().set_line(target) + + if self.initializer: + self.initializer.set_line(self.line) + + self.variable.set_line(self.line) + + if self.initialization_statement: + self.initialization_statement.set_line(self.line) + self.initialization_statement.lvalues[0].set_line(self.line) + + def serialize(self) -> JsonDict: + data = {'.class': 'Argument', + 'kind': self.kind, + 'variable': self.variable.serialize(), + 'type_annotation': (None if self.type_annotation is None + else self.type_annotation.serialize()), + } # type: JsonDict + # TODO: initializer? + return data + + @classmethod + def deserialize(cls, data: JsonDict) -> 'Argument': + assert data['.class'] == 'Argument' + return Argument(Var.deserialize(data['variable']), + (None if data.get('type_annotation') is None + else mypy.types.Type.deserialize(data['type_annotation'])), + None, # TODO: initializer? + kind=data['kind']) + + +class FuncItem(FuncBase): + arguments = [] # type: List[Argument] + # Minimum number of arguments + min_args = 0 + # Maximum number of positional arguments, -1 if no explicit limit (*args not included) + max_pos = 0 + body = None # type: Block + # Is this an overload variant of function with more than one overload variant? + is_overload = False + is_generator = False # Contains a yield statement? + is_static = False # Uses @staticmethod? + is_class = False # Uses @classmethod? + # Variants of function with type variables with values expanded + expanded = None # type: List[FuncItem] + + def __init__(self, arguments: List[Argument], body: 'Block', + typ: 'mypy.types.FunctionLike' = None) -> None: + self.arguments = arguments + arg_kinds = [arg.kind for arg in self.arguments] + self.max_pos = arg_kinds.count(ARG_POS) + arg_kinds.count(ARG_OPT) + self.body = body + self.type = typ + self.expanded = [] + + self.min_args = 0 + for i in range(len(self.arguments)): + if self.arguments[i] is None and i < self.max_fixed_argc(): + self.min_args = i + 1 + + def max_fixed_argc(self) -> int: + return self.max_pos + + def set_line(self, target: Union[Token, Node, int]) -> Node: + super().set_line(target) + for arg in self.arguments: + arg.set_line(self.line) + return self + + def is_dynamic(self): + return self.type is None + + +class FuncDef(FuncItem, Statement): + """Function definition. + + This is a non-lambda function defined using 'def'. + """ + + is_decorated = False + is_conditional = False # Defined conditionally (within block)? + is_abstract = False + is_property = False + original_def = None # type: Union[None, FuncDef, Var] # Original conditional definition + + def __init__(self, + name: str, # Function name + arguments: List[Argument], + body: 'Block', + typ: 'mypy.types.FunctionLike' = None) -> None: + super().__init__(arguments, body, typ) + self._name = name + + def name(self) -> str: + return self._name + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_func_def(self) + + def is_constructor(self) -> bool: + return self.info is not None and self._name == '__init__' + + def serialize(self) -> JsonDict: + return {'.class': 'FuncDef', + 'name': self._name, + 'fullname': self._fullname, + 'arguments': [a.serialize() for a in self.arguments], + 'type': None if self.type is None else self.type.serialize(), + 'is_property': self.is_property, + 'is_overload': self.is_overload, + 'is_generator': self.is_generator, + 'is_static': self.is_static, + 'is_class': self.is_class, + 'is_decorated': self.is_decorated, + 'is_conditional': self.is_conditional, + 'is_abstract': self.is_abstract, + # TODO: Do we need expanded, original_def? + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'FuncDef': + assert data['.class'] == 'FuncDef' + body = Block([]) + ret = FuncDef(data['name'], + [Argument.deserialize(a) for a in data['arguments']], + body, + (None if data['type'] is None + else mypy.types.FunctionLike.deserialize(data['type']))) + ret._fullname = data['fullname'] + ret.is_property = data['is_property'] + ret.is_overload = data['is_overload'] + ret.is_generator = data['is_generator'] + ret.is_static = data['is_static'] + ret.is_class = data['is_class'] + ret.is_decorated = data['is_decorated'] + ret.is_conditional = data['is_conditional'] + ret.is_abstract = data['is_abstract'] + # NOTE: ret.info is set in the fixup phase. + return ret + + +class Decorator(SymbolNode, Statement): + """A decorated function. + + A single Decorator object can include any number of function decorators. + """ + + func = None # type: FuncDef # Decorated function + decorators = None # type: List[Expression] # Decorators, at least one # XXX Not true + var = None # type: Var # Represents the decorated function obj + is_overload = False + + def __init__(self, func: FuncDef, decorators: List[Expression], + var: 'Var') -> None: + self.func = func + self.decorators = decorators + self.var = var + self.is_overload = False + + def name(self) -> str: + return self.func.name() + + def fullname(self) -> str: + return self.func.fullname() + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_decorator(self) + + def serialize(self) -> JsonDict: + return {'.class': 'Decorator', + 'func': self.func.serialize(), + 'var': self.var.serialize(), + 'is_overload': self.is_overload, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'Decorator': + assert data['.class'] == 'Decorator' + dec = Decorator(FuncDef.deserialize(data['func']), + [], + Var.deserialize(data['var'])) + dec.is_overload = data['is_overload'] + return dec + + +class Var(SymbolNode, Statement): + """A variable. + + It can refer to global/local variable or a data attribute. + """ + + _name = None # type: str # Name without module prefix + _fullname = None # type: str # Name with module prefix + info = None # type: TypeInfo # Defining class (for member variables) + type = None # type: mypy.types.Type # Declared or inferred type, or None + # Is this the first argument to an ordinary method (usually "self")? + is_self = False + is_ready = False # If inferred, is the inferred type available? + # Is this initialized explicitly to a non-None value in class body? + is_initialized_in_class = False + is_staticmethod = False + is_classmethod = False + is_property = False + is_settable_property = False + + def __init__(self, name: str, type: 'mypy.types.Type' = None) -> None: + self._name = name + self.type = type + self.is_self = False + self.is_ready = True + self.is_initialized_in_class = False + + def name(self) -> str: + return self._name + + def fullname(self) -> str: + return self._fullname + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_var(self) + + def serialize(self) -> JsonDict: + # TODO: Leave default values out? + # NOTE: Sometimes self.is_ready is False here, but we don't care. + data = {'.class': 'Var', + 'name': self._name, + 'fullname': self._fullname, + 'type': None if self.type is None else self.type.serialize(), + 'is_self': self.is_self, + 'is_initialized_in_class': self.is_initialized_in_class, + 'is_staticmethod': self.is_staticmethod, + 'is_classmethod': self.is_classmethod, + 'is_property': self.is_property, + 'is_settable_property': self.is_settable_property, + } # type: JsonDict + return data + + @classmethod + def deserialize(cls, data: JsonDict) -> 'Var': + assert data['.class'] == 'Var' + name = data['name'] + type = None if data['type'] is None else mypy.types.Type.deserialize(data['type']) + v = Var(name, type) + v._fullname = data['fullname'] + v.is_self = data['is_self'] + v.is_initialized_in_class = data['is_initialized_in_class'] + v.is_staticmethod = data['is_staticmethod'] + v.is_classmethod = data['is_classmethod'] + v.is_property = data['is_property'] + v.is_settable_property = data['is_settable_property'] + return v + + +class ClassDef(Statement): + """Class definition""" + + name = None # type: str # Name of the class without module prefix + fullname = None # type: str # Fully qualified name of the class + defs = None # type: Block + type_vars = None # type: List[mypy.types.TypeVarDef] + # Base class expressions (not semantically analyzed -- can be arbitrary expressions) + base_type_exprs = None # type: List[Expression] + info = None # type: TypeInfo # Related TypeInfo + metaclass = '' + decorators = None # type: List[Expression] + # Built-in/extension class? (single implementation inheritance only) + is_builtinclass = False + + def __init__(self, + name: str, + defs: 'Block', + type_vars: List['mypy.types.TypeVarDef'] = None, + base_type_exprs: List[Expression] = None, + metaclass: str = None) -> None: + self.name = name + self.defs = defs + self.type_vars = type_vars or [] + self.base_type_exprs = base_type_exprs or [] + self.metaclass = metaclass + self.decorators = [] + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_class_def(self) + + def is_generic(self) -> bool: + return self.info.is_generic() + + def serialize(self) -> JsonDict: + # Not serialized: defs, base_type_exprs, decorators + return {'.class': 'ClassDef', + 'name': self.name, + 'fullname': self.fullname, + 'type_vars': [v.serialize() for v in self.type_vars], + 'metaclass': self.metaclass, + 'is_builtinclass': self.is_builtinclass, + } + + @classmethod + def deserialize(self, data: JsonDict) -> 'ClassDef': + assert data['.class'] == 'ClassDef' + res = ClassDef(data['name'], + Block([]), + [mypy.types.TypeVarDef.deserialize(v) for v in data['type_vars']], + metaclass=data['metaclass'], + ) + res.fullname = data['fullname'] + res.is_builtinclass = data['is_builtinclass'] + return res + + +class GlobalDecl(Statement): + """Declaration global x, y, ...""" + + names = None # type: List[str] + + def __init__(self, names: List[str]) -> None: + self.names = names + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_global_decl(self) + + +class NonlocalDecl(Statement): + """Declaration nonlocal x, y, ...""" + + names = None # type: List[str] + + def __init__(self, names: List[str]) -> None: + self.names = names + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_nonlocal_decl(self) + + +class Block(Statement): + body = None # type: List[Statement] + # True if we can determine that this block is not executed. For example, + # this applies to blocks that are protected by something like "if PY3:" + # when using Python 2. + is_unreachable = False + + def __init__(self, body: List[Statement]) -> None: + self.body = body + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_block(self) + + +# Statements + + +class ExpressionStmt(Statement): + """An expression as a statement, such as print(s).""" + expr = None # type: Expression + + def __init__(self, expr: Expression) -> None: + self.expr = expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_expression_stmt(self) + + +class AssignmentStmt(Statement): + """Assignment statement + The same node class is used for single assignment, multiple assignment + (e.g. x, y = z) and chained assignment (e.g. x = y = z), assignments + that define new names, and assignments with explicit types (# type). + + An lvalue can be NameExpr, TupleExpr, ListExpr, MemberExpr, IndexExpr. + """ + + lvalues = None # type: List[Expression] + rvalue = None # type: Expression + # Declared type in a comment, may be None. + type = None # type: mypy.types.Type + + def __init__(self, lvalues: List[Expression], rvalue: Expression, + type: 'mypy.types.Type' = None) -> None: + self.lvalues = lvalues + self.rvalue = rvalue + self.type = type + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_assignment_stmt(self) + + +class OperatorAssignmentStmt(Statement): + """Operator assignment statement such as x += 1""" + + op = '' + lvalue = None # type: Expression + rvalue = None # type: Expression + + def __init__(self, op: str, lvalue: Expression, rvalue: Expression) -> None: + self.op = op + self.lvalue = lvalue + self.rvalue = rvalue + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_operator_assignment_stmt(self) + + +class WhileStmt(Statement): + expr = None # type: Expression + body = None # type: Block + else_body = None # type: Block + + def __init__(self, expr: Expression, body: Block, else_body: Block) -> None: + self.expr = expr + self.body = body + self.else_body = else_body + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_while_stmt(self) + + +class ForStmt(Statement): + # Index variables + index = None # type: Expression + # Expression to iterate + expr = None # type: Expression + body = None # type: Block + else_body = None # type: Block + + def __init__(self, index: Expression, expr: Expression, body: Block, + else_body: Block) -> None: + self.index = index + self.expr = expr + self.body = body + self.else_body = else_body + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_for_stmt(self) + + +class ReturnStmt(Statement): + expr = None # type: Optional[Expression] + + def __init__(self, expr: Optional[Expression]) -> None: + self.expr = expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_return_stmt(self) + + +class AssertStmt(Statement): + expr = None # type: Expression + + def __init__(self, expr: Expression) -> None: + self.expr = expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_assert_stmt(self) + + +class DelStmt(Statement): + expr = None # type: Expression + + def __init__(self, expr: Expression) -> None: + self.expr = expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_del_stmt(self) + + +class BreakStmt(Statement): + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_break_stmt(self) + + +class ContinueStmt(Statement): + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_continue_stmt(self) + + +class PassStmt(Statement): + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_pass_stmt(self) + + +class IfStmt(Statement): + expr = None # type: List[Expression] + body = None # type: List[Block] + else_body = None # type: Block + + def __init__(self, expr: List[Expression], body: List[Block], + else_body: Block) -> None: + self.expr = expr + self.body = body + self.else_body = else_body + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_if_stmt(self) + + +class RaiseStmt(Statement): + expr = None # type: Expression + from_expr = None # type: Expression + + def __init__(self, expr: Expression, from_expr: Expression = None) -> None: + self.expr = expr + self.from_expr = from_expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_raise_stmt(self) + + +class TryStmt(Statement): + body = None # type: Block # Try body + types = None # type: List[Expression] # Except type expressions + vars = None # type: List[NameExpr] # Except variable names + handlers = None # type: List[Block] # Except bodies + else_body = None # type: Block + finally_body = None # type: Block + + def __init__(self, body: Block, vars: List['NameExpr'], types: List[Expression], + handlers: List[Block], else_body: Block, + finally_body: Block) -> None: + self.body = body + self.vars = vars + self.types = types + self.handlers = handlers + self.else_body = else_body + self.finally_body = finally_body + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_try_stmt(self) + + +class WithStmt(Statement): + expr = None # type: List[Expression] + target = None # type: List[Expression] + body = None # type: Block + + def __init__(self, expr: List[Expression], target: List[Expression], + body: Block) -> None: + self.expr = expr + self.target = target + self.body = body + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_with_stmt(self) + + +class PrintStmt(Statement): + """Python 2 print statement""" + + args = None # type: List[Expression] + newline = False + # The file-like target object (given using >>). + target = None # type: Optional[Expression] + + def __init__(self, args: List[Expression], newline: bool, target: Expression = None) -> None: + self.args = args + self.newline = newline + self.target = target + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_print_stmt(self) + + +class ExecStmt(Statement): + """Python 2 exec statement""" + + expr = None # type: Expression + variables1 = None # type: Optional[Expression] + variables2 = None # type: Optional[Expression] + + def __init__(self, expr: Expression, + variables1: Optional[Expression], + variables2: Optional[Expression]) -> None: + self.expr = expr + self.variables1 = variables1 + self.variables2 = variables2 + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_exec_stmt(self) + + +# Expressions + + +class IntExpr(Expression): + """Integer literal""" + + value = 0 + literal = LITERAL_YES + + def __init__(self, value: int) -> None: + self.value = value + self.literal_hash = value + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_int_expr(self) + + +# How mypy uses StrExpr, BytesExpr, and UnicodeExpr: +# In Python 2 mode: +# b'x', 'x' -> StrExpr +# u'x' -> UnicodeExpr +# BytesExpr is unused +# +# In Python 3 mode: +# b'x' -> BytesExpr +# 'x', u'x' -> StrExpr +# UnicodeExpr is unused + +class StrExpr(Expression): + """String literal""" + + value = '' + literal = LITERAL_YES + + def __init__(self, value: str) -> None: + self.value = value + self.literal_hash = value + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_str_expr(self) + + +class BytesExpr(Expression): + """Bytes literal""" + + value = '' # TODO use bytes + literal = LITERAL_YES + + def __init__(self, value: str) -> None: + self.value = value + self.literal_hash = value + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_bytes_expr(self) + + +class UnicodeExpr(Expression): + """Unicode literal (Python 2.x)""" + + value = '' # TODO use bytes + literal = LITERAL_YES + + def __init__(self, value: str) -> None: + self.value = value + self.literal_hash = value + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_unicode_expr(self) + + +class FloatExpr(Expression): + """Float literal""" + + value = 0.0 + literal = LITERAL_YES + + def __init__(self, value: float) -> None: + self.value = value + self.literal_hash = value + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_float_expr(self) + + +class ComplexExpr(Expression): + """Complex literal""" + + value = 0.0j + literal = LITERAL_YES + + def __init__(self, value: complex) -> None: + self.value = value + self.literal_hash = value + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_complex_expr(self) + + +class EllipsisExpr(Expression): + """Ellipsis (...)""" + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_ellipsis(self) + + +class StarExpr(Expression): + """Star expression""" + + expr = None # type: Expression + + def __init__(self, expr: Expression) -> None: + self.expr = expr + self.literal = self.expr.literal + self.literal_hash = ('Star', expr.literal_hash,) + + # Whether this starred expression is used in a tuple/list and as lvalue + self.valid = False + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_star_expr(self) + + +class RefExpr(Expression): + """Abstract base class for name-like constructs""" + + kind = None # type: int # LDEF/GDEF/MDEF/... (None if not available) + node = None # type: SymbolNode # Var, FuncDef or TypeInfo that describes this + fullname = None # type: str # Fully qualified name (or name if not global) + + # Does this define a new name with inferred type? + # + # For members, after semantic analysis, this does not take base + # classes into consideration at all; the type checker deals with these. + is_def = False + + +class NameExpr(RefExpr): + """Name expression + + This refers to a local name, global name or a module. + """ + + name = None # type: str # Name referred to (may be qualified) + # TypeInfo of class surrounding expression (may be None) + info = None # type: TypeInfo + + literal = LITERAL_TYPE + + def __init__(self, name: str) -> None: + self.name = name + self.literal_hash = ('Var', name,) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_name_expr(self) + + def serialize(self) -> JsonDict: + # TODO: Find out where and why NameExpr is being serialized (if at all). + assert False, "Serializing NameExpr: %s" % (self,) + return {'.class': 'NameExpr', + 'kind': self.kind, + 'node': None if self.node is None else self.node.serialize(), + 'fullname': self.fullname, + 'is_def': self.is_def, + 'name': self.name, + 'literal': self.literal, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'NameExpr': + assert data['.class'] == 'NameExpr' + ret = NameExpr(data['name']) + ret.kind = data['kind'] + ret.node = None if data['node'] is None else SymbolNode.deserialize(data['node']) + ret.fullname = data['fullname'] + ret.is_def = data['is_def'] + ret.literal = data['literal'] + return ret + + +class MemberExpr(RefExpr): + """Member access expression x.y""" + + expr = None # type: Expression + name = None # type: str + # The variable node related to a definition. + def_var = None # type: Var + + def __init__(self, expr: Expression, name: str) -> None: + self.expr = expr + self.name = name + self.literal = self.expr.literal + self.literal_hash = ('Member', expr.literal_hash, name) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_member_expr(self) + + +# Kinds of arguments + +# Positional argument +ARG_POS = 0 # type: int +# Positional, optional argument (functions only, not calls) +ARG_OPT = 1 # type: int +# *arg argument +ARG_STAR = 2 # type: int +# Keyword argument x=y in call, or keyword-only function arg +ARG_NAMED = 3 # type: int +# **arg argument +ARG_STAR2 = 4 # type: int + + +class CallExpr(Expression): + """Call expression. + + This can also represent several special forms that are syntactically calls + such as cast(...) and None # type: .... + """ + + callee = None # type: Expression + args = None # type: List[Expression] + arg_kinds = None # type: List[int] # ARG_ constants + # Each name can be None if not a keyword argument. + arg_names = None # type: List[str] + # If not None, the node that represents the meaning of the CallExpr. For + # cast(...) this is a CastExpr. + analyzed = None # type: Optional[Expression] + + def __init__(self, callee: Expression, args: List[Expression], arg_kinds: List[int], + arg_names: List[str] = None, analyzed: Expression = None) -> None: + if not arg_names: + arg_names = [None] * len(args) + + self.callee = callee + self.args = args + self.arg_kinds = arg_kinds + self.arg_names = arg_names + self.analyzed = analyzed + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_call_expr(self) + + +class YieldFromExpr(Expression): + expr = None # type: Expression + + def __init__(self, expr: Expression) -> None: + self.expr = expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_yield_from_expr(self) + + +class YieldExpr(Expression): + expr = None # type: Optional[Expression] + + def __init__(self, expr: Optional[Expression]) -> None: + self.expr = expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_yield_expr(self) + + +class IndexExpr(Expression): + """Index expression x[y]. + + Also wraps type application such as List[int] as a special form. + """ + + base = None # type: Expression + index = None # type: Expression + # Inferred __getitem__ method type + method_type = None # type: mypy.types.Type + # If not None, this is actually semantically a type application + # Class[type, ...] or a type alias initializer. + analyzed = None # type: Union[TypeApplication, TypeAliasExpr] + + def __init__(self, base: Expression, index: Expression) -> None: + self.base = base + self.index = index + self.analyzed = None + if self.index.literal == LITERAL_YES: + self.literal = self.base.literal + self.literal_hash = ('Member', base.literal_hash, + index.literal_hash) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_index_expr(self) + + +class UnaryExpr(Expression): + """Unary operation""" + + op = '' + expr = None # type: Expression + # Inferred operator method type + method_type = None # type: mypy.types.Type + + def __init__(self, op: str, expr: Expression) -> None: + self.op = op + self.expr = expr + self.literal = self.expr.literal + self.literal_hash = ('Unary', op, expr.literal_hash) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_unary_expr(self) + + +# Map from binary operator id to related method name (in Python 3). +op_methods = { + '+': '__add__', + '-': '__sub__', + '*': '__mul__', + '/': '__truediv__', + '%': '__mod__', + '//': '__floordiv__', + '**': '__pow__', + '&': '__and__', + '|': '__or__', + '^': '__xor__', + '<<': '__lshift__', + '>>': '__rshift__', + '==': '__eq__', + '!=': '__ne__', + '<': '__lt__', + '>=': '__ge__', + '>': '__gt__', + '<=': '__le__', + 'in': '__contains__', +} # type: Dict[str, str] + +ops_with_inplace_method = { + '+', '-', '*', '/', '%', '//', '**', '&', '|', '^', '<<', '>>'} + +inplace_operator_methods = set( + '__i' + op_methods[op][2:] for op in ops_with_inplace_method) + +reverse_op_methods = { + '__add__': '__radd__', + '__sub__': '__rsub__', + '__mul__': '__rmul__', + '__truediv__': '__rtruediv__', + '__mod__': '__rmod__', + '__floordiv__': '__rfloordiv__', + '__pow__': '__rpow__', + '__and__': '__rand__', + '__or__': '__ror__', + '__xor__': '__rxor__', + '__lshift__': '__rlshift__', + '__rshift__': '__rrshift__', + '__eq__': '__eq__', + '__ne__': '__ne__', + '__lt__': '__gt__', + '__ge__': '__le__', + '__gt__': '__lt__', + '__le__': '__ge__', +} + +normal_from_reverse_op = dict((m, n) for n, m in reverse_op_methods.items()) +reverse_op_method_set = set(reverse_op_methods.values()) + + +class OpExpr(Expression): + """Binary operation (other than . or [] or comparison operators, + which have specific nodes).""" + + op = '' + left = None # type: Expression + right = None # type: Expression + # Inferred type for the operator method type (when relevant). + method_type = None # type: mypy.types.Type + + def __init__(self, op: str, left: Expression, right: Expression) -> None: + self.op = op + self.left = left + self.right = right + self.literal = min(self.left.literal, self.right.literal) + self.literal_hash = ('Binary', op, left.literal_hash, right.literal_hash) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_op_expr(self) + + +class ComparisonExpr(Expression): + """Comparison expression (e.g. a < b > c < d).""" + + operators = None # type: List[str] + operands = None # type: List[Expression] + # Inferred type for the operator methods (when relevant; None for 'is'). + method_types = None # type: List[mypy.types.Type] + + def __init__(self, operators: List[str], operands: List[Expression]) -> None: + self.operators = operators + self.operands = operands + self.method_types = [] + self.literal = min(o.literal for o in self.operands) + self.literal_hash = (('Comparison',) + tuple(operators) + + tuple(o.literal_hash for o in operands)) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_comparison_expr(self) + + +class SliceExpr(Expression): + """Slice expression (e.g. 'x:y', 'x:', '::2' or ':'). + + This is only valid as index in index expressions. + """ + + begin_index = None # type: Optional[Expression] + end_index = None # type: Optional[Expression] + stride = None # type: Optional[Expression] + + def __init__(self, begin_index: Optional[Expression], + end_index: Optional[Expression], + stride: Optional[Expression]) -> None: + self.begin_index = begin_index + self.end_index = end_index + self.stride = stride + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_slice_expr(self) + + +class CastExpr(Expression): + """Cast expression cast(type, expr).""" + + expr = None # type: Expression + type = None # type: mypy.types.Type + + def __init__(self, expr: Expression, typ: 'mypy.types.Type') -> None: + self.expr = expr + self.type = typ + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_cast_expr(self) + + +class RevealTypeExpr(Expression): + """Reveal type expression reveal_type(expr).""" + + expr = None # type: Expression + + def __init__(self, expr: Expression) -> None: + self.expr = expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_reveal_type_expr(self) + + +class SuperExpr(Expression): + """Expression super().name""" + + name = '' + info = None # type: TypeInfo # Type that contains this super expression + + def __init__(self, name: str) -> None: + self.name = name + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_super_expr(self) + + +class FuncExpr(FuncItem, Expression): + """Lambda expression""" + + def name(self) -> str: + return '' + + def expr(self) -> Expression: + """Return the expression (the body) of the lambda.""" + ret = cast(ReturnStmt, self.body.body[-1]) + return ret.expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_func_expr(self) + + +class ListExpr(Expression): + """List literal expression [...].""" + + items = None # type: List[Expression] + + def __init__(self, items: List[Expression]) -> None: + self.items = items + if all(x.literal == LITERAL_YES for x in items): + self.literal = LITERAL_YES + self.literal_hash = ('List',) + tuple(x.literal_hash for x in items) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_list_expr(self) + + +class DictExpr(Expression): + """Dictionary literal expression {key: value, ...}.""" + + items = None # type: List[Tuple[Expression, Expression]] + + def __init__(self, items: List[Tuple[Expression, Expression]]) -> None: + self.items = items + if all(x[0].literal == LITERAL_YES and x[1].literal == LITERAL_YES + for x in items): + self.literal = LITERAL_YES + self.literal_hash = ('Dict',) + tuple( + (x[0].literal_hash, x[1].literal_hash) for x in items) # type: ignore + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_dict_expr(self) + + +class TupleExpr(Expression): + """Tuple literal expression (..., ...)""" + + items = None # type: List[Expression] + + def __init__(self, items: List[Expression]) -> None: + self.items = items + if all(x.literal == LITERAL_YES for x in items): + self.literal = LITERAL_YES + self.literal_hash = ('Tuple',) + tuple(x.literal_hash for x in items) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_tuple_expr(self) + + +class SetExpr(Expression): + """Set literal expression {value, ...}.""" + + items = None # type: List[Expression] + + def __init__(self, items: List[Expression]) -> None: + self.items = items + if all(x.literal == LITERAL_YES for x in items): + self.literal = LITERAL_YES + self.literal_hash = ('Set',) + tuple(x.literal_hash for x in items) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_set_expr(self) + + +class GeneratorExpr(Expression): + """Generator expression ... for ... in ... [ for ... in ... ] [ if ... ].""" + + left_expr = None # type: Expression + sequences_expr = None # type: List[Expression] + condlists = None # type: List[List[Expression]] + indices = None # type: List[Expression] + + def __init__(self, left_expr: Expression, indices: List[Expression], + sequences: List[Expression], condlists: List[List[Expression]]) -> None: + self.left_expr = left_expr + self.sequences = sequences + self.condlists = condlists + self.indices = indices + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_generator_expr(self) + + +class ListComprehension(Expression): + """List comprehension (e.g. [x + 1 for x in a])""" + + generator = None # type: GeneratorExpr + + def __init__(self, generator: GeneratorExpr) -> None: + self.generator = generator + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_list_comprehension(self) + + +class SetComprehension(Expression): + """Set comprehension (e.g. {x + 1 for x in a})""" + + generator = None # type: GeneratorExpr + + def __init__(self, generator: GeneratorExpr) -> None: + self.generator = generator + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_set_comprehension(self) + + +class DictionaryComprehension(Expression): + """Dictionary comprehension (e.g. {k: v for k, v in a}""" + + key = None # type: Expression + value = None # type: Expression + sequences_expr = None # type: List[Expression] + condlists = None # type: List[List[Expression]] + indices = None # type: List[Expression] + + def __init__(self, key: Expression, value: Expression, indices: List[Expression], + sequences: List[Expression], condlists: List[List[Expression]]) -> None: + self.key = key + self.value = value + self.sequences = sequences + self.condlists = condlists + self.indices = indices + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_dictionary_comprehension(self) + + +class ConditionalExpr(Expression): + """Conditional expression (e.g. x if y else z)""" + + cond = None # type: Expression + if_expr = None # type: Expression + else_expr = None # type: Expression + + def __init__(self, cond: Expression, if_expr: Expression, else_expr: Expression) -> None: + self.cond = cond + self.if_expr = if_expr + self.else_expr = else_expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_conditional_expr(self) + + +class BackquoteExpr(Expression): + """Python 2 expression `...`.""" + + expr = None # type: Expression + + def __init__(self, expr: Expression) -> None: + self.expr = expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_backquote_expr(self) + + +class TypeApplication(Expression): + """Type application expr[type, ...]""" + + expr = None # type: Expression + types = None # type: List[mypy.types.Type] + + def __init__(self, expr: Expression, types: List['mypy.types.Type']) -> None: + self.expr = expr + self.types = types + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_type_application(self) + + +# Variance of a type variable. For example, T in the definition of +# List[T] is invariant, so List[int] is not a subtype of List[object], +# and also List[object] is not a subtype of List[int]. +# +# The T in Iterable[T] is covariant, so Iterable[int] is a subtype of +# Iterable[object], but not vice versa. +# +# If T is contravariant in Foo[T], Foo[object] is a subtype of +# Foo[int], but not vice versa. +INVARIANT = 0 # type: int +COVARIANT = 1 # type: int +CONTRAVARIANT = 2 # type: int + + +class TypeVarExpr(SymbolNode, Expression): + """Type variable expression TypeVar(...).""" + + _name = '' + _fullname = '' + # Value restriction: only types in the list are valid as values. If the + # list is empty, there is no restriction. + values = None # type: List[mypy.types.Type] + # Upper bound: only subtypes of upper_bound are valid as values. By default + # this is 'object', meaning no restriction. + upper_bound = None # type: mypy.types.Type + # Variance of the type variable. Invariant is the default. + # TypeVar(..., covariant=True) defines a covariant type variable. + # TypeVar(..., contravariant=True) defines a contravariant type + # variable. + variance = INVARIANT + + def __init__(self, name: str, fullname: str, + values: List['mypy.types.Type'], + upper_bound: 'mypy.types.Type', + variance: int=INVARIANT) -> None: + self._name = name + self._fullname = fullname + self.values = values + self.upper_bound = upper_bound + self.variance = variance + + def name(self) -> str: + return self._name + + def fullname(self) -> str: + return self._fullname + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_type_var_expr(self) + + def serialize(self) -> JsonDict: + return {'.class': 'TypeVarExpr', + 'name': self._name, + 'fullname': self._fullname, + 'values': [t.serialize() for t in self.values], + 'upper_bound': self.upper_bound.serialize(), + 'variance': self.variance, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'TypeVarExpr': + assert data['.class'] == 'TypeVarExpr' + return TypeVarExpr(data['name'], + data['fullname'], + [mypy.types.Type.deserialize(v) for v in data['values']], + mypy.types.Type.deserialize(data['upper_bound']), + data['variance']) + + +class TypeAliasExpr(Expression): + """Type alias expression (rvalue).""" + + type = None # type: mypy.types.Type + + def __init__(self, type: 'mypy.types.Type') -> None: + self.type = type + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_type_alias_expr(self) + + +class NamedTupleExpr(Expression): + """Named tuple expression namedtuple(...).""" + + # The class representation of this named tuple (its tuple_type attribute contains + # the tuple item types) + info = None # type: TypeInfo + + def __init__(self, info: 'TypeInfo') -> None: + self.info = info + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_namedtuple_expr(self) + + +class PromoteExpr(Expression): + """Ducktype class decorator expression _promote(...).""" + + type = None # type: mypy.types.Type + + def __init__(self, type: 'mypy.types.Type') -> None: + self.type = type + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit__promote_expr(self) + + +# Constants + + +class TempNode(Expression): + """Temporary dummy node used during type checking. + + This node is not present in the original program; it is just an artifact + of the type checker implementation. It only represents an opaque node with + some fixed type. + """ + + type = None # type: mypy.types.Type + + def __init__(self, typ: 'mypy.types.Type') -> None: + self.type = typ + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_temp_node(self) + + +class TypeInfo(SymbolNode): + """The type structure of a single class. + + Each TypeInfo corresponds one-to-one to a ClassDef, which + represents the AST of the class. + + In type-theory terms, this is a "type constructor", and if the + class is generic then it will be a type constructor of higher kind. + Where the class is used in an actual type, it's in the form of an + Instance, which amounts to a type application of the tycon to + the appropriate number of arguments. + """ + + _fullname = None # type: str # Fully qualified name + defn = None # type: ClassDef # Corresponding ClassDef + # Method Resolution Order: the order of looking up attributes. The first + # value always to refers to this class. + mro = None # type: List[TypeInfo] + subtypes = None # type: Set[TypeInfo] # Direct subclasses encountered so far + names = None # type: SymbolTable # Names defined directly in this type + is_abstract = False # Does the class have any abstract attributes? + abstract_attributes = None # type: List[str] + # Classes inheriting from Enum shadow their true members with a __getattr__, so we + # have to treat them as a special case. + is_enum = False + # If true, any unknown attributes should have type 'Any' instead + # of generating a type error. This would be true if there is a + # base class with type 'Any', but other use cases may be + # possible. This is similar to having __getattr__ that returns Any + # (and __setattr__), but without the __getattr__ method. + fallback_to_any = False + + # Information related to type annotations. + + # Generic type variable names + type_vars = None # type: List[str] + + # Direct base classes. + bases = None # type: List[mypy.types.Instance] + + # Another type which this type will be treated as a subtype of, + # even though it's not a subclass in Python. The non-standard + # `@_promote` decorator introduces this, and there are also + # several builtin examples, in particular `int` -> `float`. + _promote = None # type: mypy.types.Type + + # Representation of a Tuple[...] base class, if the class has any + # (e.g., for named tuples). If this is not None, the actual Type + # object used for this class is not an Instance but a TupleType; + # the corresponding Instance is set as the fallback type of the + # tuple type. + tuple_type = None # type: mypy.types.TupleType + + # Is this a named tuple type? + is_named_tuple = False + + # Is this a dummy from deserialization? + is_dummy = False + + # Alternative to fullname() for 'anonymous' classes. + alt_fullname = None # type: Optional[str] + + def __init__(self, names: 'SymbolTable', defn: ClassDef) -> None: + """Initialize a TypeInfo.""" + self.names = names + self.defn = defn + self.subtypes = set() + self.type_vars = [] + self.bases = [] + # Leave self.mro uninitialized until we compute it for real, + # so we don't accidentally try to use it prematurely. + self._fullname = defn.fullname + self.is_abstract = False + self.abstract_attributes = [] + if defn.type_vars: + for vd in defn.type_vars: + self.type_vars.append(vd.name) + + def name(self) -> str: + """Short name.""" + return self.defn.name + + def fullname(self) -> str: + return self._fullname + + def is_generic(self) -> bool: + """Is the type generic (i.e. does it have type variables)?""" + return len(self.type_vars) > 0 + + def get(self, name: str) -> 'SymbolTableNode': + for cls in self.mro: + n = cls.names.get(name) + if n: + return n + return None + + def __getitem__(self, name: str) -> 'SymbolTableNode': + n = self.get(name) + if n: + return n + else: + raise KeyError(name) + + def __repr__(self) -> str: + return '' % self.fullname() + + # IDEA: Refactor the has* methods to be more consistent and document + # them. + + def has_readable_member(self, name: str) -> bool: + return self.get(name) is not None + + def has_writable_member(self, name: str) -> bool: + return self.has_var(name) + + def has_var(self, name: str) -> bool: + return self.get_var(name) is not None + + def has_method(self, name: str) -> bool: + return self.get_method(name) is not None + + def get_var(self, name: str) -> Var: + for cls in self.mro: + if name in cls.names: + node = cls.names[name].node + if isinstance(node, Var): + return node + else: + return None + return None + + def get_var_or_getter(self, name: str) -> SymbolNode: + # TODO getter + return self.get_var(name) + + def get_var_or_setter(self, name: str) -> SymbolNode: + # TODO setter + return self.get_var(name) + + def get_method(self, name: str) -> FuncBase: + if self.mro is None: # Might be because of a previous error. + return None + for cls in self.mro: + if name in cls.names: + node = cls.names[name].node + if isinstance(node, FuncBase): + return node + else: + return None + return None + + def calculate_mro(self) -> None: + """Calculate and set mro (method resolution order). + + Raise MroError if cannot determine mro. + """ + mro = linearize_hierarchy(self) + assert mro, "Could not produce a MRO at all for %s" % (self,) + self.mro = mro + self.is_enum = self._calculate_is_enum() + + def _calculate_is_enum(self) -> bool: + """ + If this is "enum.Enum" itself, then yes, it's an enum. + If the flag .is_enum has been set on anything in the MRO, it's an enum. + """ + if self.fullname() == ENUM_BASECLASS: + return True + if self.mro: + return any(type_info.is_enum for type_info in self.mro) + return False + + def has_base(self, fullname: str) -> bool: + """Return True if type has a base type with the specified name. + + This can be either via extension or via implementation. + """ + if self.mro: + for cls in self.mro: + if cls.fullname() == fullname: + return True + return False + + def all_subtypes(self) -> 'Set[TypeInfo]': + """Return TypeInfos of all subtypes, including this type, as a set.""" + subtypes = set([self]) + for subt in self.subtypes: + for t in subt.all_subtypes(): + subtypes.add(t) + return subtypes + + def all_base_classes(self) -> 'List[TypeInfo]': + """Return a list of base classes, including indirect bases.""" + assert False + + def direct_base_classes(self) -> 'List[TypeInfo]': + """Return a direct base classes. + + Omit base classes of other base classes. + """ + return [base.type for base in self.bases] + + def __str__(self) -> str: + """Return a string representation of the type. + + This includes the most important information about the type. + """ + base = None # type: str + if self.bases: + base = 'Bases({})'.format(', '.join(str(base) + for base in self.bases)) + return dump_tagged(['Name({})'.format(self.fullname()), + base, + ('Names', sorted(self.names.keys()))], + 'TypeInfo') + + def serialize(self) -> Union[str, JsonDict]: + # NOTE: This is where all ClassDefs originate, so there shouldn't be duplicates. + data = {'.class': 'TypeInfo', + 'fullname': self.fullname(), + 'alt_fullname': self.alt_fullname, + 'names': self.names.serialize(self.alt_fullname or self.fullname()), + 'defn': self.defn.serialize(), + 'is_abstract': self.is_abstract, + 'abstract_attributes': self.abstract_attributes, + 'is_enum': self.is_enum, + 'fallback_to_any': self.fallback_to_any, + 'type_vars': self.type_vars, + 'bases': [b.serialize() for b in self.bases], + '_promote': None if self._promote is None else self._promote.serialize(), + 'tuple_type': None if self.tuple_type is None else self.tuple_type.serialize(), + 'is_named_tuple': self.is_named_tuple, + } + return data + + @classmethod + def deserialize(cls, data: JsonDict) -> 'TypeInfo': + names = SymbolTable.deserialize(data['names']) + defn = ClassDef.deserialize(data['defn']) + ti = TypeInfo(names, defn) + ti._fullname = data['fullname'] + ti.alt_fullname = data['alt_fullname'] + # TODO: Is there a reason to reconstruct ti.subtypes? + ti.is_abstract = data['is_abstract'] + ti.abstract_attributes = data['abstract_attributes'] + ti.is_enum = data['is_enum'] + ti.fallback_to_any = data['fallback_to_any'] + ti.type_vars = data['type_vars'] + ti.bases = [mypy.types.Instance.deserialize(b) for b in data['bases']] + ti._promote = (None if data['_promote'] is None + else mypy.types.Type.deserialize(data['_promote'])) + ti.tuple_type = (None if data['tuple_type'] is None + else mypy.types.TupleType.deserialize(data['tuple_type'])) + ti.is_named_tuple = data['is_named_tuple'] + return ti + + +def NamedTupleTypeInfo(tup: 'mypy.types.NamedTupleType', *args) -> TypeInfo: + info = TypeInfo(*args) + info.tuple_type = tup + info.bases = [tup.fallback] + info.is_named_tuple = True + info.mro = [info] + tup.fallback.type.mro + return info + +class SymbolTableNode: + # Kind of node. Possible values: + # - LDEF: local definition (of any kind) + # - GDEF: global (module-level) definition + # - MDEF: class member definition + # - UNBOUND_TVAR: TypeVar(...) definition, not bound + # - TVAR: type variable in a bound scope (generic function / generic clas) + # - MODULE_REF: reference to a module + # - TYPE_ALIAS: type alias + # - UNBOUND_IMPORTED: temporary kind for imported names + kind = None # type: int + # AST node of definition (FuncDef/Var/TypeInfo/Decorator/TypeVarExpr, + # or None for a bound type variable). + node = None # type: Optional[SymbolNode] + # Type variable definition (for bound type variables only) + tvar_def = None # type: Optional[mypy.types.TypeVarDef] + # Module id (e.g. "foo.bar") or None + mod_id = '' + # If this not None, override the type of the 'node' attribute. + type_override = None # type: Optional[mypy.types.Type] + # If False, this name won't be imported via 'from import *'. + # This has no effect on names within classes. + module_public = True + # For deserialized MODULE_REF nodes, the referenced module name; + # for other nodes, optionally the name of the referenced object. + cross_ref = None # type: Optional[str] + + def __init__(self, kind: int, node: Optional[SymbolNode], mod_id: str = None, + typ: 'mypy.types.Type' = None, + tvar_def: 'mypy.types.TypeVarDef' = None, + module_public: bool = True) -> None: + self.kind = kind + self.node = node + self.type_override = typ + self.mod_id = mod_id + self.tvar_def = tvar_def + self.module_public = module_public + + @property + def fullname(self) -> str: + if self.node is not None: + return self.node.fullname() + else: + return None + + @property + def type(self) -> 'mypy.types.Type': + # IDEA: Get rid of the Any type. + node = self.node # type: Any + if self.type_override is not None: + return self.type_override + elif ((isinstance(node, Var) or isinstance(node, FuncDef)) + and node.type is not None): + return node.type + elif isinstance(node, Decorator): + return node.var.type + else: + return None + + def __str__(self) -> str: + s = '{}/{}'.format(node_kinds[self.kind], short_type(self.node)) + if self.mod_id is not None: + s += ' ({})'.format(self.mod_id) + # Include declared type of variables and functions. + if self.type is not None: + s += ' : {}'.format(self.type) + return s + + def serialize(self, prefix: str, name: str) -> JsonDict: + """Serialize a SymbolTableNode. + + Args: + prefix: full name of the containing module or class; or None + name: name of this object relative to the containing object + """ + data = {'.class': 'SymbolTableNode', + 'kind': node_kinds[self.kind], + } # type: JsonDict + if self.tvar_def: + data['tvar_def'] = self.tvar_def.serialize() + if not self.module_public: + data['module_public'] = False + if self.kind == MODULE_REF: + assert self.node is not None, "Missing module cross ref in %s for %s" % (prefix, name) + data['cross_ref'] = self.node.fullname() + else: + if self.node is not None: + if prefix is not None: + # Check whether this is an alias for another object. + # If the object's canonical full name differs from + # the full name computed from prefix and name, + # it's an alias, and we serialize it as a cross ref. + if isinstance(self.node, TypeInfo): + fullname = self.node.alt_fullname or self.node.fullname() + else: + fullname = self.node.fullname() + if (fullname is not None and '.' in fullname and + fullname != prefix + '.' + name): + data['cross_ref'] = fullname + return data + data['node'] = self.node.serialize() + if self.type_override is not None: + data['type_override'] = self.type_override.serialize() + return data + + @classmethod + def deserialize(cls, data: JsonDict) -> 'SymbolTableNode': + assert data['.class'] == 'SymbolTableNode' + kind = inverse_node_kinds[data['kind']] + if 'cross_ref' in data: + # This will be fixed up later. + stnode = SymbolTableNode(kind, None) + stnode.cross_ref = data['cross_ref'] + else: + node = None + if 'node' in data: + node = SymbolNode.deserialize(data['node']) + typ = None + if 'type_override' in data: + typ = mypy.types.Type.deserialize(data['type_override']) + stnode = SymbolTableNode(kind, node, typ=typ) + if 'tvar_def' in data: + stnode.tvar_def = mypy.types.TypeVarDef.deserialize(data['tvar_def']) + if 'module_public' in data: + stnode.module_public = data['module_public'] + return stnode + + +class SymbolTable(Dict[str, SymbolTableNode]): + def __str__(self) -> str: + a = [] # type: List[str] + for key, value in self.items(): + # Filter out the implicit import of builtins. + if isinstance(value, SymbolTableNode): + if (value.fullname != 'builtins' and + (value.fullname or '').split('.')[-1] not in + implicit_module_attrs): + a.append(' ' + str(key) + ' : ' + str(value)) + else: + a.append(' ') + a = sorted(a) + a.insert(0, 'SymbolTable(') + a[-1] += ')' + return '\n'.join(a) + + def serialize(self, fullname: str) -> JsonDict: + data = {'.class': 'SymbolTable'} # type: JsonDict + for key, value in self.items(): + # Skip __builtins__: it's a reference to the builtins + # module that gets added to every module by + # SemanticAnalyzer.visit_file(), but it shouldn't be + # accessed by users of the module. + if key == '__builtins__': + continue + data[key] = value.serialize(fullname, key) + return data + + @classmethod + def deserialize(cls, data: JsonDict) -> 'SymbolTable': + assert data['.class'] == 'SymbolTable' + st = SymbolTable() + for key, value in data.items(): + if key != '.class': + st[key] = SymbolTableNode.deserialize(value) + return st + + +def function_type(func: FuncBase, fallback: 'mypy.types.Instance') -> 'mypy.types.FunctionLike': + if func.type: + assert isinstance(func.type, mypy.types.FunctionLike) + return func.type + else: + # Implicit type signature with dynamic types. + # Overloaded functions always have a signature, so func must be an ordinary function. + fdef = cast(FuncDef, func) + name = func.name() + if name: + name = '"{}"'.format(name) + names = [] # type: List[str] + for arg in fdef.arguments: + names.append(arg.variable.name()) + + return mypy.types.CallableType( + [mypy.types.AnyType()] * len(fdef.arguments), + [arg.kind for arg in fdef.arguments], + names, + mypy.types.AnyType(), + fallback, + name, + implicit=True, + ) + + +def method_type_with_fallback(func: FuncBase, + fallback: 'mypy.types.Instance') -> 'mypy.types.FunctionLike': + """Return the signature of a method (omit self).""" + return method_type(function_type(func, fallback)) + + +def method_type(sig: 'mypy.types.FunctionLike') -> 'mypy.types.FunctionLike': + if isinstance(sig, mypy.types.CallableType): + return method_callable(sig) + else: + sig = cast(mypy.types.Overloaded, sig) + items = [] # type: List[mypy.types.CallableType] + for c in sig.items(): + items.append(method_callable(c)) + return mypy.types.Overloaded(items) + + +def method_callable(c: 'mypy.types.CallableType') -> 'mypy.types.CallableType': + if c.arg_kinds and c.arg_kinds[0] == ARG_STAR: + # The signature is of the form 'def foo(*args, ...)'. + # In this case we shouldn't drop the first arg, + # since self will be absorbed by the *args. + return c + return c.copy_modified(arg_types=c.arg_types[1:], + arg_kinds=c.arg_kinds[1:], + arg_names=c.arg_names[1:]) + + +class MroError(Exception): + """Raised if a consistent mro cannot be determined for a class.""" + + +def linearize_hierarchy(info: TypeInfo) -> List[TypeInfo]: + # TODO describe + if info.mro: + return info.mro + bases = info.direct_base_classes() + lin_bases = [] + for base in bases: + assert base is not None, "Cannot linearize bases for %s %s" % (info.fullname(), bases) + lin_bases.append(linearize_hierarchy(base)) + lin_bases.append(bases) + return [info] + merge(lin_bases) + + +def merge(seqs: List[List[TypeInfo]]) -> List[TypeInfo]: + seqs = [s[:] for s in seqs] + result = [] # type: List[TypeInfo] + while True: + seqs = [s for s in seqs if s] + if not seqs: + return result + for seq in seqs: + head = seq[0] + if not [s for s in seqs if head in s[1:]]: + break + else: + raise MroError() + result.append(head) + for s in seqs: + if s[0] is head: + del s[0] diff --git a/mypy/parse.py b/mypy/parse.py index a496f32bebcd3..579f03c57525f 100644 --- a/mypy/parse.py +++ b/mypy/parse.py @@ -1,2034 +1,2034 @@ -"""Mypy parser. - -Constructs a parse tree (abstract syntax tree) based on a string -representing a source file. Performs only minimal semantic checks. -""" - -import re - -from typing import List, Tuple, Any, Set, cast, Union, Optional - -from mypy import lex, docstring -from mypy.lex import ( - Token, Eof, Bom, Break, Name, Colon, Dedent, IntLit, StrLit, BytesLit, - UnicodeLit, FloatLit, Op, Indent, Keyword, Punct, LexError, ComplexLit, - EllipsisToken -) -import mypy.types -from mypy.nodes import ( - MypyFile, Import, Node, ImportAll, ImportFrom, FuncDef, OverloadedFuncDef, - ClassDef, Decorator, Block, Var, OperatorAssignmentStmt, - ExpressionStmt, AssignmentStmt, ReturnStmt, RaiseStmt, AssertStmt, - DelStmt, BreakStmt, ContinueStmt, PassStmt, GlobalDecl, - WhileStmt, ForStmt, IfStmt, TryStmt, WithStmt, - TupleExpr, GeneratorExpr, ListComprehension, ListExpr, ConditionalExpr, - DictExpr, SetExpr, NameExpr, IntExpr, StrExpr, BytesExpr, UnicodeExpr, - FloatExpr, CallExpr, SuperExpr, MemberExpr, IndexExpr, SliceExpr, OpExpr, - UnaryExpr, FuncExpr, PrintStmt, ImportBase, ComparisonExpr, - StarExpr, YieldFromExpr, NonlocalDecl, DictionaryComprehension, - SetComprehension, ComplexExpr, EllipsisExpr, YieldExpr, ExecStmt, Argument, - BackquoteExpr -) -from mypy import defaults -from mypy import nodes -from mypy.errors import Errors, CompileError -from mypy.types import Type, CallableType, AnyType, UnboundType -from mypy.parsetype import ( - parse_type, parse_types, parse_signature, TypeParseError, parse_str_as_signature -) -from mypy.options import Options - -from mypy import experiments - - -precedence = { - '**': 16, - '-u': 15, '+u': 15, '~': 15, # unary operators (-, + and ~) - '': 14, - '*': 13, '/': 13, '//': 13, '%': 13, - '+': 12, '-': 12, - '>>': 11, '<<': 11, - '&': 10, - '^': 9, - '|': 8, - '==': 7, '!=': 7, '<': 7, '>': 7, '<=': 7, '>=': 7, 'is': 7, 'in': 7, - '*u': 7, # unary * for star expressions - 'not': 6, - 'and': 5, - 'or': 4, - '': 3, # conditional expression - '': 2, # list comprehension - ',': 1} - - -op_assign = set([ - '+=', '-=', '*=', '/=', '//=', '%=', '**=', '|=', '&=', '^=', '>>=', - '<<=']) - -op_comp = set([ - '>', '<', '==', '>=', '<=', '<>', '!=', 'is', 'is', 'in', 'not']) - -none = Token('') # Empty token - - -def parse(source: Union[str, bytes], - fnam: str, - errors: Errors, - options: Options) -> MypyFile: - """Parse a source file, without doing any semantic analysis. - - Return the parse tree. If errors is not provided, raise ParseError - on failure. Otherwise, use the errors object to report parse errors. - - The python_version (major, minor) option determines the Python syntax variant. - """ - if options.fast_parser: - import mypy.fastparse - return mypy.fastparse.parse(source, - fnam=fnam, - errors=errors, - pyversion=options.python_version, - custom_typing_module=options.custom_typing_module) - - is_stub_file = bool(fnam) and fnam.endswith('.pyi') - parser = Parser(fnam, - errors, - options.python_version, - options.custom_typing_module, - is_stub_file=is_stub_file) - tree = parser.parse(source) - tree.path = fnam - tree.is_stub = is_stub_file - return tree - - -class Parser: - """Mypy parser that parses a string into an AST. - - Parses type annotations in addition to basic Python syntax. It supports both Python 2 and 3 - (though Python 2 support is incomplete). - - The AST classes are defined in mypy.nodes and mypy.types. - """ - - tok = None # type: List[Token] - ind = 0 - errors = None # type: Errors - # If True, raise an exception on any parse error. Otherwise, errors are reported via 'errors'. - raise_on_error = False - - # Are we currently parsing the body of a class definition? - is_class_body = False - # All import nodes encountered so far in this parse unit. - imports = None # type: List[ImportBase] - # Names imported from __future__. - future_options = None # type: List[str] - # Lines to ignore (using # type: ignore). - ignored_lines = None # type: Set[int] - - def __init__(self, fnam: str, errors: Errors, pyversion: Tuple[int, int], - custom_typing_module: str = None, is_stub_file: bool = False) -> None: - self.raise_on_error = errors is None - self.pyversion = pyversion - self.custom_typing_module = custom_typing_module - self.is_stub_file = is_stub_file - if errors is not None: - self.errors = errors - else: - self.errors = Errors() - if fnam is not None: - self.errors.set_file(fnam) - else: - self.errors.set_file('') - - def parse(self, s: Union[str, bytes]) -> MypyFile: - self.tok, self.ignored_lines = lex.lex(s, pyversion=self.pyversion, - is_stub_file=self.is_stub_file) - self.ind = 0 - self.imports = [] - self.future_options = [] - file = self.parse_file() - if self.raise_on_error and self.errors.is_errors(): - self.errors.raise_error() - return file - - def weak_opts(self) -> Set[str]: - """Do weak typing if any of the first ten tokens is a comment saying so. - - The comment can be one of: - # mypy: weak=global - # mypy: weak=local - # mypy: weak <- defaults to local - """ - regexp = re.compile(r'^[\s]*# *mypy: *weak(=?)([^\s]*)', re.M) - for t in self.tok[:10]: - for s in [t.string, t.pre]: - m = regexp.search(s) - if m: - opts = set(x for x in m.group(2).split(',') if x) - if not opts: - opts.add('local') - return opts - return set() - - def parse_file(self) -> MypyFile: - """Parse a mypy source file.""" - is_bom = self.parse_bom() - defs = self.parse_defs() - weak_opts = self.weak_opts() - self.expect_type(Eof) - node = MypyFile(defs, self.imports, is_bom, self.ignored_lines, - weak_opts=weak_opts) - return node - - # Parse the initial part - - def parse_bom(self) -> bool: - """Parse the optional byte order mark at the beginning of a file.""" - if isinstance(self.current(), Bom): - self.expect_type(Bom) - if isinstance(self.current(), Break): - self.expect_break() - return True - else: - return False - - def parse_import(self) -> Import: - self.expect('import') - ids = [] - while True: - id = self.parse_qualified_name() - translated = self.translate_module_id(id) - as_id = None - if self.current_str() == 'as': - self.expect('as') - name_tok = self.expect_type(Name) - as_id = name_tok.string - elif translated != id: - as_id = id - ids.append((translated, as_id)) - if self.current_str() != ',': - break - self.expect(',') - node = Import(ids) - self.imports.append(node) - return node - - def translate_module_id(self, id: str) -> str: - """Return the actual, internal module id for a source text id. - - For example, translate '__builtin__' in Python 2 to 'builtins'. - """ - if id == self.custom_typing_module: - return 'typing' - elif id == '__builtin__' and self.pyversion[0] == 2: - # HACK: __builtin__ in Python 2 is aliases to builtins. However, the implementation - # is named __builtin__.py (there is another layer of translation elsewhere). - return 'builtins' - return id - - def parse_import_from(self) -> Node: - self.expect('from') - - # Build the list of beginning relative tokens. - relative = 0 - while self.current_str() in (".", "..."): - relative += len(self.current_str()) - self.skip() - - # Parse qualified name to actually import from. - if self.current_str() == "import": - # Empty/default values. - name = "" - else: - name = self.parse_qualified_name() - - name = self.translate_module_id(name) - - # Parse import list - self.expect('import') - node = None # type: ImportBase - if self.current_str() == '*': - if name == '__future__': - self.parse_error() - # An import all from a module node: - self.skip() - node = ImportAll(name, relative) - else: - is_paren = self.current_str() == '(' - if is_paren: - self.expect('(') - targets = [] # type: List[Tuple[str, str]] - while True: - id, as_id = self.parse_import_name() - if '%s.%s' % (name, id) == self.custom_typing_module: - if targets or self.current_str() == ',': - self.fail('You cannot import any other modules when you ' - 'import a custom typing module', - self.current().line) - node = Import([('typing', as_id)]) - self.skip_until_break() - break - targets.append((id, as_id)) - if self.current_str() != ',': - break - self.expect(',') - if is_paren and self.current_str() == ')': - break - if is_paren: - self.expect(')') - if node is None: - node = ImportFrom(name, relative, targets) - self.imports.append(node) - if name == '__future__': - self.future_options.extend(target[0] for target in targets) - return node - - def parse_import_name(self) -> Tuple[str, Optional[str]]: - tok = self.expect_type(Name) - name = tok.string - if self.current_str() == 'as': - self.skip() - as_name = self.expect_type(Name) - return name, as_name.string - else: - return name, None - - def parse_qualified_name(self) -> str: - """Parse a name with an optional module qualifier. - - Return a tuple with the name as a string and a token array - containing all the components of the name. - """ - tok = self.expect_type(Name) - n = tok.string - while self.current_str() == '.': - self.expect('.') - tok = self.expect_type(Name) - n += '.' + tok.string - return n - - # Parsing global definitions - - def parse_defs(self) -> List[Node]: - defs = [] # type: List[Node] - while not self.eof(): - try: - defn, is_simple = self.parse_statement() - if is_simple: - self.expect_break() - if defn is not None: - if not self.try_combine_overloads(defn, defs): - defs.append(defn) - except ParseError: - pass - return defs - - def parse_class_def(self) -> ClassDef: - old_is_class_body = self.is_class_body - self.is_class_body = True - - self.expect('class') - metaclass = None - - try: - base_types = [] # type: List[Node] - try: - name_tok = self.expect_type(Name) - name = name_tok.string - - self.errors.push_type(name) - - if self.current_str() == '(': - self.skip() - while True: - if self.current_str() == ')': - break - if self.peek().string == '=': - metaclass = self.parse_class_keywords() - break - base_types.append(self.parse_super_type()) - if self.current_str() != ',': - break - self.skip() - self.expect(')') - except ParseError: - pass - - defs, _ = self.parse_block() - - node = ClassDef(name, defs, None, base_types, metaclass=metaclass) - return node - finally: - self.errors.pop_type() - self.is_class_body = old_is_class_body - - def parse_class_keywords(self) -> Optional[str]: - """Parse the class keyword arguments, keeping the metaclass but - ignoring all others. Returns None if the metaclass isn't found. - """ - metaclass = None - while True: - key = self.expect_type(Name) - self.expect('=') - if key.string == 'metaclass': - metaclass = self.parse_qualified_name() - else: - # skip the class value - self.parse_expression(precedence[',']) - if self.current_str() != ',': - break - self.skip() - if self.current_str() == ')': - break - return metaclass - - def parse_super_type(self) -> Node: - return self.parse_expression(precedence[',']) - - def parse_decorated_function_or_class(self) -> Node: - decorators = [] - no_type_checks = False - while self.current_str() == '@': - self.expect('@') - d_exp = self.parse_expression() - if self.is_no_type_check_decorator(d_exp): - no_type_checks = True - decorators.append(d_exp) - self.expect_break() - if self.current_str() != 'class': - func = self.parse_function(no_type_checks) - func.is_decorated = True - var = Var(func.name()) - # Types of decorated functions must always be inferred. - var.is_ready = False - var.set_line(decorators[0].line) - node = Decorator(func, decorators, var) - return node - else: - cls = self.parse_class_def() - cls.decorators = decorators - return cls - - def is_no_type_check_decorator(self, expr: Node) -> bool: - if isinstance(expr, NameExpr): - return expr.name == 'no_type_check' - elif isinstance(expr, MemberExpr): - if isinstance(expr.expr, NameExpr): - return expr.expr.name == 'typing' and expr.name == 'no_type_check' - else: - return False - - def parse_function(self, no_type_checks: bool=False) -> FuncDef: - def_tok = self.expect('def') - is_method = self.is_class_body - self.is_class_body = False - try: - (name, args, typ, is_error, extra_stmts) = self.parse_function_header(no_type_checks) - - arg_kinds = [arg.kind for arg in args] - arg_names = [arg.variable.name() for arg in args] - - body, comment_type = self.parse_block(allow_type=True) - # Potentially insert extra assignment statements to the beginning of the - # body, used to decompose Python 2 tuple arguments. - body.body[:0] = extra_stmts - if comment_type: - # The function has a # type: ... signature. - if typ: - self.errors.report( - def_tok.line, 'Function has duplicate type signatures') - sig = cast(CallableType, comment_type) - if sig.is_ellipsis_args: - # When we encounter an ellipsis, fill in the arg_types with - # a bunch of AnyTypes, emulating Callable[..., T] - arg_types = [AnyType()] * len(arg_kinds) # type: List[Type] - typ = CallableType( - arg_types, - arg_kinds, - arg_names, - sig.ret_type, - None, - line=def_tok.line) - elif is_method and len(sig.arg_kinds) < len(arg_kinds): - self.check_argument_kinds(arg_kinds, - [nodes.ARG_POS] + sig.arg_kinds, - def_tok.line) - # Add implicit 'self' argument to signature. - first_arg = [AnyType()] # type: List[Type] - typ = CallableType( - first_arg + sig.arg_types, - arg_kinds, - arg_names, - sig.ret_type, - None, - line=def_tok.line) - else: - self.check_argument_kinds(arg_kinds, sig.arg_kinds, - def_tok.line) - typ = CallableType( - sig.arg_types, - arg_kinds, - arg_names, - sig.ret_type, - None, - line=def_tok.line) - - # If there was a serious error, we really cannot build a parse tree - # node. - if is_error: - return None - - if typ and isinstance(typ.ret_type, UnboundType): - typ.ret_type.is_ret_type = True - - node = FuncDef(name, args, body, typ) - node.set_line(def_tok) - if typ is not None: - typ.definition = node - return node - finally: - self.errors.pop_function() - self.is_class_body = is_method - - def check_argument_kinds(self, funckinds: List[int], sigkinds: List[int], - line: int) -> None: - """Check that arguments are consistent. - - This verifies that they have the same number and the kinds correspond. - - Arguments: - funckinds: kinds of arguments in function definition - sigkinds: kinds of arguments in signature (after # type:) - """ - if len(funckinds) != len(sigkinds): - if len(funckinds) > len(sigkinds): - self.fail("Type signature has too few arguments", line) - else: - self.fail("Type signature has too many arguments", line) - return - for kind, token in [(nodes.ARG_STAR, '*'), - (nodes.ARG_STAR2, '**')]: - if ((funckinds.count(kind) != sigkinds.count(kind)) or - (kind in funckinds and sigkinds.index(kind) != funckinds.index(kind))): - self.fail( - "Inconsistent use of '{}' in function " - "signature".format(token), line) - - def parse_function_header( - self, no_type_checks: bool=False) -> Tuple[str, - List[Argument], - CallableType, - bool, - List[AssignmentStmt]]: - """Parse function header (a name followed by arguments) - - Return a 5-tuple with the following items: - name - arguments - signature (annotation) - error flag (True if error) - extra statements needed to decompose arguments (usually empty) - - See parse_arg_list for an explanation of the final tuple item. - """ - name = '' - - try: - name_tok = self.expect_type(Name) - name = name_tok.string - - self.errors.push_function(name) - - args, typ, extra_stmts = self.parse_args(no_type_checks) - except ParseError: - if not isinstance(self.current(), Break): - self.ind -= 1 # Kludge: go back to the Break token - # Resynchronise parsing by going back over :, if present. - if isinstance(self.tok[self.ind - 1], Colon): - self.ind -= 1 - return (name, [], None, True, []) - - return (name, args, typ, False, extra_stmts) - - def parse_args(self, no_type_checks: bool=False) -> Tuple[List[Argument], - CallableType, - List[AssignmentStmt]]: - """Parse a function signature (...) [-> t]. - - See parse_arg_list for an explanation of the final tuple item. - """ - lparen = self.expect('(') - - # Parse the argument list (everything within '(' and ')'). - args, extra_stmts = self.parse_arg_list(no_type_checks=no_type_checks) - - self.expect(')') - - if self.current_str() == '->': - self.skip() - if no_type_checks: - self.parse_expression() - ret_type = None - else: - ret_type = self.parse_type() - else: - ret_type = None - - arg_kinds = [arg.kind for arg in args] - self.verify_argument_kinds(arg_kinds, lparen.line) - - annotation = self.build_func_annotation( - ret_type, args, lparen.line) - - return args, annotation, extra_stmts - - def build_func_annotation(self, ret_type: Type, args: List[Argument], - line: int, is_default_ret: bool = False) -> CallableType: - arg_types = [arg.type_annotation for arg in args] - # Are there any type annotations? - if ((ret_type and not is_default_ret) - or arg_types != [None] * len(arg_types)): - # Yes. Construct a type for the function signature. - return self.construct_function_type(args, ret_type, line) - else: - return None - - def parse_arg_list(self, allow_signature: bool = True, - no_type_checks: bool=False) -> Tuple[List[Argument], - List[AssignmentStmt]]: - """Parse function definition argument list. - - This includes everything between '(' and ')' (but not the - parentheses). - - Return tuple (arguments, - extra statements for decomposing arguments). - - The final argument is only used for Python 2 argument lists with - tuples; they contain destructuring assignment statements used to - decompose tuple arguments. For example, consider a header like this: - - . def f((x, y)) - - The actual (sole) argument will be __tuple_arg_1 (a generated - name), whereas the extra statement list will contain a single - assignment statement corresponding to this assignment: - - x, y = __tuple_arg_1 - """ - args = [] # type: List[Argument] - extra_stmts = [] - # This is for checking duplicate argument names. - arg_names = [] # type: List[str] - has_tuple_arg = False - - require_named = False - bare_asterisk_before = -1 - - if self.current_str() != ')' and self.current_str() != ':': - while self.current_str() != ')': - if self.current_str() == '*' and self.peek().string == ',': - self.expect('*') - require_named = True - bare_asterisk_before = len(args) - elif self.current_str() in ['*', '**']: - if bare_asterisk_before == len(args): - # named arguments must follow bare * - self.parse_error() - - arg = self.parse_asterisk_arg( - allow_signature, - no_type_checks, - ) - args.append(arg) - require_named = True - elif self.current_str() == '(': - arg, extra_stmt, names = self.parse_tuple_arg(len(args)) - args.append(arg) - if extra_stmt is not None: - extra_stmts.append(extra_stmt) - has_tuple_arg = True - arg_names.extend(names) - else: - arg, require_named = self.parse_normal_arg( - require_named, - allow_signature, - no_type_checks, - ) - args.append(arg) - arg_names.append(arg.variable.name()) - - if self.current().string != ',': - break - - self.expect(',') - - # Non-tuple argument dupes will be checked elsewhere. Avoid - # generating duplicate errors. - if has_tuple_arg: - self.check_duplicate_argument_names(arg_names) - - return args, extra_stmts - - def check_duplicate_argument_names(self, names: List[str]) -> None: - found = set() # type: Set[str] - for name in names: - if name in found: - self.fail('Duplicate argument name "{}"'.format(name), - self.current().line) - found.add(name) - - def parse_asterisk_arg(self, - allow_signature: bool, - no_type_checks: bool) -> Argument: - asterisk = self.skip() - name = self.expect_type(Name) - variable = Var(name.string) - if asterisk.string == '*': - kind = nodes.ARG_STAR - else: - kind = nodes.ARG_STAR2 - - type = None - if no_type_checks: - self.parse_parameter_annotation() - else: - type = self.parse_arg_type(allow_signature) - - return Argument(variable, type, None, kind) - - def parse_tuple_arg(self, index: int) -> Tuple[Argument, AssignmentStmt, List[str]]: - """Parse a single Python 2 tuple argument. - - Example: def f(x, (y, z)): ... - - The tuple arguments gets transformed into an assignment in the - function body (the second return value). - - Return tuple (argument, decomposing assignment, list of names defined). - - Special case: if the argument is just (x) then it's not a tuple; - we indicate this by returning (argument, None, ['x']). - However, if the argument is (x,) then it *is* a (singleton) tuple. - """ - line = self.current().line - # Generate a new argument name that is very unlikely to clash with anything. - arg_name = '__tuple_arg_{}'.format(index + 1) - if self.pyversion[0] >= 3: - self.fail('Tuples in argument lists only supported in Python 2 mode', line) - paren_arg = self.parse_parentheses() - self.verify_tuple_arg(paren_arg) - if isinstance(paren_arg, NameExpr): - # This isn't a tuple. Revert to a normal argument. - arg_name = paren_arg.name - decompose = None - else: - rvalue = NameExpr(arg_name) - rvalue.set_line(line) - decompose = AssignmentStmt([paren_arg], rvalue) - decompose.set_line(line) - kind = nodes.ARG_POS - initializer = None - if self.current_str() == '=': - self.expect('=') - initializer = self.parse_expression(precedence[',']) - kind = nodes.ARG_OPT - var = Var(arg_name) - arg_names = self.find_tuple_arg_argument_names(paren_arg) - return Argument(var, None, initializer, kind), decompose, arg_names - - def verify_tuple_arg(self, paren_arg: Node) -> None: - if isinstance(paren_arg, TupleExpr): - if not paren_arg.items: - self.fail('Empty tuple not valid as an argument', paren_arg.line) - for item in paren_arg.items: - self.verify_tuple_arg(item) - elif not isinstance(paren_arg, NameExpr): - self.fail('Invalid item in tuple argument', paren_arg.line) - - def find_tuple_arg_argument_names(self, node: Node) -> List[str]: - result = [] # type: List[str] - if isinstance(node, TupleExpr): - for item in node.items: - result.extend(self.find_tuple_arg_argument_names(item)) - elif isinstance(node, NameExpr): - result.append(node.name) - return result - - def parse_normal_arg(self, require_named: bool, - allow_signature: bool, - no_type_checks: bool) -> Tuple[Argument, bool]: - name = self.expect_type(Name) - variable = Var(name.string) - - type = None - if no_type_checks: - self.parse_parameter_annotation() - else: - type = self.parse_arg_type(allow_signature) - - initializer = None # type: Node - if self.current_str() == '=': - self.expect('=') - initializer = self.parse_expression(precedence[',']) - if require_named: - kind = nodes.ARG_NAMED - else: - kind = nodes.ARG_OPT - else: - if require_named: - kind = nodes.ARG_NAMED - else: - kind = nodes.ARG_POS - - self.set_type_optional(type, initializer) - - return Argument(variable, type, initializer, kind), require_named - - def set_type_optional(self, type: Type, initializer: Node) -> None: - if not experiments.STRICT_OPTIONAL: - return - # Indicate that type should be wrapped in an Optional if arg is initialized to None. - optional = isinstance(initializer, NameExpr) and initializer.name == 'None' - if isinstance(type, UnboundType): - type.optional = optional - - def parse_parameter_annotation(self) -> Node: - if self.current_str() == ':': - self.skip() - return self.parse_expression(precedence[',']) - - def parse_arg_type(self, allow_signature: bool) -> Type: - if self.current_str() == ':' and allow_signature: - self.skip() - return self.parse_type() - else: - return None - - def verify_argument_kinds(self, kinds: List[int], line: int) -> None: - found = set() # type: Set[int] - for i, kind in enumerate(kinds): - if kind == nodes.ARG_POS and found & set([nodes.ARG_OPT, - nodes.ARG_STAR, - nodes.ARG_STAR2]): - self.fail('Invalid argument list', line) - elif kind == nodes.ARG_STAR and nodes.ARG_STAR in found: - self.fail('Invalid argument list', line) - elif kind == nodes.ARG_STAR2 and i != len(kinds) - 1: - self.fail('Invalid argument list', line) - found.add(kind) - - def construct_function_type(self, args: List[Argument], ret_type: Type, - line: int) -> CallableType: - # Complete the type annotation by replacing omitted types with 'Any'. - arg_types = [arg.type_annotation for arg in args] - for i in range(len(arg_types)): - if arg_types[i] is None: - arg_types[i] = AnyType(implicit=True) - if ret_type is None: - ret_type = AnyType(implicit=True) - arg_kinds = [arg.kind for arg in args] - arg_names = [arg.variable.name() for arg in args] - return CallableType(arg_types, arg_kinds, arg_names, ret_type, None, name=None, - variables=None, line=line) - - # Parsing statements - - def parse_block(self, allow_type: bool = False) -> Tuple[Block, Type]: - colon = self.expect(':') - if not isinstance(self.current(), Break): - # Block immediately after ':'. - nodes = [] - while True: - ind = self.ind - stmt, is_simple = self.parse_statement() - if not is_simple: - self.parse_error_at(self.tok[ind]) - break - nodes.append(stmt) - brk = self.expect_break() - if brk.string != ';': - break - node = Block(nodes) - node.set_line(colon) - return node, None - else: - # Indented block. - brk = self.expect_break() - type = self.parse_type_comment(brk, signature=True) - self.expect_indent() - stmt_list = [] # type: List[Node] - if allow_type: - cur = self.current() - if type is None and isinstance(cur, StrLit): - ds = docstring.parse_docstring(cur.parsed()) - if ds and False: # TODO: Enable when this is working. - try: - type = parse_str_as_signature(ds.as_type_str(), cur.line) - except TypeParseError: - # We don't require docstrings to be actually correct. - # TODO: Report something here. - type = None - while (not isinstance(self.current(), Dedent) and - not isinstance(self.current(), Eof)): - try: - stmt, is_simple = self.parse_statement() - if is_simple: - self.expect_break() - if stmt is not None: - if not self.try_combine_overloads(stmt, stmt_list): - stmt_list.append(stmt) - except ParseError: - pass - if isinstance(self.current(), Dedent): - self.skip() - node = Block(stmt_list) - node.set_line(colon) - return node, type - - def try_combine_overloads(self, s: Node, stmt: List[Node]) -> bool: - if isinstance(s, Decorator) and stmt: - fdef = s - n = fdef.func.name() - if isinstance(stmt[-1], Decorator) and stmt[-1].func.name() == n: - stmt[-1] = OverloadedFuncDef([stmt[-1], fdef]) - return True - elif isinstance(stmt[-1], OverloadedFuncDef) and stmt[-1].name() == n: - stmt[-1].items.append(fdef) - return True - return False - - def parse_statement(self) -> Tuple[Node, bool]: - stmt = None # type: Node - t = self.current() - ts = self.current_str() - is_simple = True # Is this a non-block statement? - if ts == 'if': - stmt = self.parse_if_stmt() - is_simple = False - elif ts == 'def': - stmt = self.parse_function() - is_simple = False - elif ts == 'while': - stmt = self.parse_while_stmt() - is_simple = False - elif ts == 'return': - stmt = self.parse_return_stmt() - elif ts == 'for': - stmt = self.parse_for_stmt() - is_simple = False - elif ts == 'try': - stmt = self.parse_try_stmt() - is_simple = False - elif ts == 'break': - stmt = self.parse_break_stmt() - elif ts == 'continue': - stmt = self.parse_continue_stmt() - elif ts == 'pass': - stmt = self.parse_pass_stmt() - elif ts == 'raise': - stmt = self.parse_raise_stmt() - elif ts == 'import': - stmt = self.parse_import() - elif ts == 'from': - stmt = self.parse_import_from() - elif ts == 'class': - stmt = self.parse_class_def() - is_simple = False - elif ts == 'global': - stmt = self.parse_global_decl() - elif ts == 'nonlocal' and self.pyversion[0] >= 3: - stmt = self.parse_nonlocal_decl() - elif ts == 'assert': - stmt = self.parse_assert_stmt() - elif ts == 'del': - stmt = self.parse_del_stmt() - elif ts == 'with': - stmt = self.parse_with_stmt() - is_simple = False - elif ts == '@': - stmt = self.parse_decorated_function_or_class() - is_simple = False - elif ts == 'print' and (self.pyversion[0] == 2 and - 'print_function' not in self.future_options): - stmt = self.parse_print_stmt() - elif ts == 'exec' and self.pyversion[0] == 2: - stmt = self.parse_exec_stmt() - else: - stmt = self.parse_expression_or_assignment() - if stmt is not None: - stmt.set_line(t) - return stmt, is_simple - - def parse_expression_or_assignment(self) -> Node: - expr = self.parse_expression(star_expr_allowed=True) - if self.current_str() == '=': - return self.parse_assignment(expr) - elif self.current_str() in op_assign: - # Operator assignment statement. - op = self.current_str()[:-1] - self.skip() - rvalue = self.parse_expression() - return OperatorAssignmentStmt(op, expr, rvalue) - else: - # Expression statement. - return ExpressionStmt(expr) - - def parse_assignment(self, lvalue: Any) -> Node: - """Parse an assignment statement. - - Assume that lvalue has been parsed already, and the current token is '='. - Also parse an optional '# type:' comment. - """ - self.expect('=') - lvalues = [lvalue] - expr = self.parse_expression(star_expr_allowed=True) - while self.current_str() == '=': - self.skip() - lvalues.append(expr) - expr = self.parse_expression(star_expr_allowed=True) - cur = self.current() - if isinstance(cur, Break): - type = self.parse_type_comment(cur, signature=False) - else: - type = None - return AssignmentStmt(lvalues, expr, type) - - def parse_return_stmt(self) -> ReturnStmt: - self.expect('return') - expr = None - current = self.current() - if current.string == 'yield': - self.parse_error() - if not isinstance(current, Break): - expr = self.parse_expression() - node = ReturnStmt(expr) - return node - - def parse_raise_stmt(self) -> RaiseStmt: - self.expect('raise') - expr = None - from_expr = None - if not isinstance(self.current(), Break): - expr = self.parse_expression() - if self.current_str() == 'from': - self.expect('from') - from_expr = self.parse_expression() - node = RaiseStmt(expr, from_expr) - return node - - def parse_assert_stmt(self) -> AssertStmt: - self.expect('assert') - expr = self.parse_expression() - node = AssertStmt(expr) - return node - - def parse_yield_or_yield_from_expr(self) -> Union[YieldFromExpr, YieldExpr]: - self.expect("yield") - expr = None - node = YieldExpr(expr) # type: Union[YieldFromExpr, YieldExpr] - if not isinstance(self.current(), Break): - if self.current_str() == "from": - self.expect("from") - expr = self.parse_expression() # when yield from is assigned to a variable - node = YieldFromExpr(expr) - else: - if self.current_str() == ')': - node = YieldExpr(None) - else: - expr = self.parse_expression() - node = YieldExpr(expr) - return node - - def parse_ellipsis(self) -> EllipsisExpr: - self.expect('...') - node = EllipsisExpr() - return node - - def parse_del_stmt(self) -> DelStmt: - self.expect('del') - expr = self.parse_expression() - node = DelStmt(expr) - return node - - def parse_break_stmt(self) -> BreakStmt: - self.expect('break') - node = BreakStmt() - return node - - def parse_continue_stmt(self) -> ContinueStmt: - self.expect('continue') - node = ContinueStmt() - return node - - def parse_pass_stmt(self) -> PassStmt: - self.expect('pass') - node = PassStmt() - return node - - def parse_global_decl(self) -> GlobalDecl: - self.expect('global') - names = self.parse_identifier_list() - node = GlobalDecl(names) - return node - - def parse_nonlocal_decl(self) -> NonlocalDecl: - self.expect('nonlocal') - names = self.parse_identifier_list() - node = NonlocalDecl(names) - return node - - def parse_identifier_list(self) -> List[str]: - names = [] - while True: - n = self.expect_type(Name) - names.append(n.string) - if self.current_str() != ',': - break - self.skip() - return names - - def parse_while_stmt(self) -> WhileStmt: - is_error = False - self.expect('while') - try: - expr = self.parse_expression() - except ParseError: - is_error = True - body, _ = self.parse_block() - if self.current_str() == 'else': - self.expect('else') - else_body, _ = self.parse_block() - else: - else_body = None - if is_error is not None: - node = WhileStmt(expr, body, else_body) - return node - else: - return None - - def parse_for_stmt(self) -> ForStmt: - self.expect('for') - index = self.parse_for_index_variables() - self.expect('in') - expr = self.parse_expression() - - body, _ = self.parse_block() - - if self.current_str() == 'else': - self.expect('else') - else_body, _ = self.parse_block() - else: - else_body = None - - node = ForStmt(index, expr, body, else_body) - return node - - def parse_for_index_variables(self) -> Node: - # Parse index variables of a 'for' statement. - index_items = [] - force_tuple = False - - while True: - v = self.parse_expression(precedence['in'], - star_expr_allowed=True) # Prevent parsing of for stmt 'in' - index_items.append(v) - if self.current_str() != ',': - break - self.skip() - if self.current_str() == 'in': - force_tuple = True - break - - if len(index_items) == 1 and not force_tuple: - index = index_items[0] - else: - index = TupleExpr(index_items) - index.set_line(index_items[0].get_line()) - - return index - - def parse_if_stmt(self) -> IfStmt: - is_error = False - - self.expect('if') - expr = [] - try: - expr.append(self.parse_expression()) - except ParseError: - is_error = True - - body = [self.parse_block()[0]] - - while self.current_str() == 'elif': - self.expect('elif') - try: - expr.append(self.parse_expression()) - except ParseError: - is_error = True - body.append(self.parse_block()[0]) - - if self.current_str() == 'else': - self.expect('else') - else_body, _ = self.parse_block() - else: - else_body = None - - if not is_error: - node = IfStmt(expr, body, else_body) - return node - else: - return None - - def parse_try_stmt(self) -> Node: - self.expect('try') - body, _ = self.parse_block() - is_error = False - vars = [] # type: List[NameExpr] - types = [] # type: List[Node] - handlers = [] # type: List[Block] - while self.current_str() == 'except': - self.expect('except') - if not isinstance(self.current(), Colon): - try: - t = self.current() - types.append(self.parse_expression(precedence[',']).set_line(t)) - if self.current_str() == 'as': - self.expect('as') - vars.append(self.parse_name_expr()) - elif self.pyversion[0] == 2 and self.current_str() == ',': - self.expect(',') - vars.append(self.parse_name_expr()) - else: - vars.append(None) - except ParseError: - is_error = True - else: - types.append(None) - vars.append(None) - handlers.append(self.parse_block()[0]) - if not is_error: - if self.current_str() == 'else': - self.skip() - else_body, _ = self.parse_block() - else: - else_body = None - if self.current_str() == 'finally': - self.expect('finally') - finally_body, _ = self.parse_block() - else: - finally_body = None - node = TryStmt(body, vars, types, handlers, else_body, - finally_body) - return node - else: - return None - - def parse_with_stmt(self) -> WithStmt: - self.expect('with') - exprs = [] - targets = [] - while True: - expr = self.parse_expression(precedence[',']) - if self.current_str() == 'as': - self.expect('as') - target = self.parse_expression(precedence[',']) - else: - target = None - exprs.append(expr) - targets.append(target) - if self.current_str() != ',': - break - self.expect(',') - body, _ = self.parse_block() - return WithStmt(exprs, targets, body) - - def parse_print_stmt(self) -> PrintStmt: - self.expect('print') - args = [] - target = None - if self.current_str() == '>>': - self.skip() - target = self.parse_expression(precedence[',']) - if self.current_str() == ',': - self.skip() - if isinstance(self.current(), Break): - self.parse_error() - else: - if not isinstance(self.current(), Break): - self.parse_error() - comma = False - while not isinstance(self.current(), Break): - args.append(self.parse_expression(precedence[','])) - if self.current_str() == ',': - comma = True - self.skip() - else: - comma = False - break - return PrintStmt(args, newline=not comma, target=target) - - def parse_exec_stmt(self) -> ExecStmt: - self.expect('exec') - expr = self.parse_expression(precedence['in']) - variables1 = None - variables2 = None - if self.current_str() == 'in': - self.skip() - variables1 = self.parse_expression(precedence[',']) - if self.current_str() == ',': - self.skip() - variables2 = self.parse_expression(precedence[',']) - return ExecStmt(expr, variables1, variables2) - - # Parsing expressions - - def parse_expression(self, prec: int = 0, star_expr_allowed: bool = False) -> Node: - """Parse a subexpression within a specific precedence context.""" - expr = None # type: Node - current = self.current() # Remember token for setting the line number. - - # Parse a "value" expression or unary operator expression and store - # that in expr. - s = self.current_str() - if s == '(': - # Parenthesised expression or cast. - expr = self.parse_parentheses() - elif s == '[': - expr = self.parse_list_expr() - elif s in ['-', '+', 'not', '~']: - # Unary operation. - expr = self.parse_unary_expr() - elif s == 'lambda': - expr = self.parse_lambda_expr() - elif s == '{': - expr = self.parse_dict_or_set_expr() - elif s == '*' and star_expr_allowed: - expr = self.parse_star_expr() - elif s == '`' and self.pyversion[0] == 2: - expr = self.parse_backquote_expr() - else: - if isinstance(current, Name): - # Name expression. - expr = self.parse_name_expr() - elif isinstance(current, IntLit): - expr = self.parse_int_expr() - elif isinstance(current, StrLit): - expr = self.parse_str_expr() - elif isinstance(current, BytesLit): - expr = self.parse_bytes_literal() - elif isinstance(current, UnicodeLit): - expr = self.parse_unicode_literal() - elif isinstance(current, FloatLit): - expr = self.parse_float_expr() - elif isinstance(current, ComplexLit): - expr = self.parse_complex_expr() - elif isinstance(current, Keyword) and s == "yield": - # The expression yield from and yield to assign - expr = self.parse_yield_or_yield_from_expr() - elif isinstance(current, EllipsisToken) and (self.pyversion[0] >= 3 - or self.is_stub_file): - expr = self.parse_ellipsis() - else: - # Invalid expression. - self.parse_error() - - # Set the line of the expression node, if not specified. This - # simplifies recording the line number as not every node type needs to - # deal with it separately. - if expr.line < 0: - expr.set_line(current) - - # Parse operations that require a left argument (stored in expr). - while True: - current = self.current() - s = self.current_str() - if s == '(': - # Call expression. - expr = self.parse_call_expr(expr) - elif s == '.': - # Member access expression. - expr = self.parse_member_expr(expr) - elif s == '[': - # Indexing expression. - expr = self.parse_index_expr(expr) - elif s == ',': - # The comma operator is used to build tuples. Comma also - # separates array items and function arguments, but in this - # case the precedence is too low to build a tuple. - if precedence[','] > prec: - expr = self.parse_tuple_expr(expr) - else: - break - elif s == 'for': - if precedence[''] > prec: - # List comprehension or generator expression. Parse as - # generator expression; it will be converted to list - # comprehension if needed elsewhere. - expr = self.parse_generator_expr(expr) - else: - break - elif s == 'if': - # Conditional expression. - if precedence[''] > prec: - expr = self.parse_conditional_expr(expr) - else: - break - else: - # Binary operation or a special case. - if isinstance(current, Op): - op = self.current_str() - op_prec = precedence[op] - if op == 'not': - # Either "not in" or an error. - op_prec = precedence['in'] - if op_prec > prec: - if op in op_comp: - expr = self.parse_comparison_expr(expr, op_prec) - else: - expr = self.parse_bin_op_expr(expr, op_prec) - else: - # The operation cannot be associated with the - # current left operand due to the precedence - # context; let the caller handle it. - break - else: - # Not an operation that accepts a left argument; let the - # caller handle the rest. - break - - # Set the line of the expression node, if not specified. This - # simplifies recording the line number as not every node type - # needs to deal with it separately. - if expr.line < 0: - expr.set_line(current) - - return expr - - def parse_parentheses(self) -> Node: - self.skip() - if self.current_str() == ')': - # Empty tuple (). - expr = self.parse_empty_tuple_expr() # type: Node - else: - # Parenthesised expression. - expr = self.parse_expression(0, star_expr_allowed=True) - self.expect(')') - return expr - - def parse_star_expr(self) -> Node: - star = self.expect('*') - expr = self.parse_expression(precedence['*u']) - expr = StarExpr(expr) - if expr.line < 0: - expr.set_line(star) - return expr - - def parse_empty_tuple_expr(self) -> TupleExpr: - self.expect(')') - node = TupleExpr([]) - return node - - def parse_list_expr(self) -> Node: - """Parse list literal or list comprehension.""" - items = [] - self.expect('[') - while self.current_str() != ']' and not self.eol(): - items.append(self.parse_expression(precedence[''], star_expr_allowed=True)) - if self.current_str() != ',': - break - self.expect(',') - if self.current_str() == 'for' and len(items) == 1: - items[0] = self.parse_generator_expr(items[0]) - self.expect(']') - if len(items) == 1 and isinstance(items[0], GeneratorExpr): - return ListComprehension(items[0]) - else: - expr = ListExpr(items) - return expr - - def parse_generator_expr(self, left_expr: Node) -> GeneratorExpr: - tok = self.current() - indices, sequences, condlists = self.parse_comp_for() - - gen = GeneratorExpr(left_expr, indices, sequences, condlists) - gen.set_line(tok) - return gen - - def parse_comp_for(self) -> Tuple[List[Node], List[Node], List[List[Node]]]: - indices = [] - sequences = [] - condlists = [] # type: List[List[Node]] - while self.current_str() == 'for': - conds = [] - self.expect('for') - index = self.parse_for_index_variables() - indices.append(index) - self.expect('in') - if self.pyversion[0] >= 3: - sequence = self.parse_expression(precedence['']) - else: - sequence = self.parse_expression_list() - sequences.append(sequence) - while self.current_str() == 'if': - self.skip() - conds.append(self.parse_expression(precedence[''])) - condlists.append(conds) - - return indices, sequences, condlists - - def parse_expression_list(self) -> Node: - prec = precedence[''] - expr = self.parse_expression(prec) - if self.current_str() != ',': - return expr - else: - t = self.current() - return self.parse_tuple_expr(expr, prec).set_line(t) - - def parse_conditional_expr(self, left_expr: Node) -> ConditionalExpr: - self.expect('if') - cond = self.parse_expression(precedence['']) - self.expect('else') - else_expr = self.parse_expression(precedence['']) - return ConditionalExpr(cond, left_expr, else_expr) - - def parse_dict_or_set_expr(self) -> Node: - items = [] # type: List[Tuple[Node, Node]] - self.expect('{') - while self.current_str() != '}' and not self.eol(): - key = self.parse_expression(precedence['']) - if self.current_str() in [',', '}'] and items == []: - return self.parse_set_expr(key) - elif self.current_str() == 'for' and items == []: - return self.parse_set_comprehension(key) - elif self.current_str() != ':': - self.parse_error() - colon = self.expect(':') - value = self.parse_expression(precedence['']) - if self.current_str() == 'for' and items == []: - return self.parse_dict_comprehension(key, value, colon) - items.append((key, value)) - if self.current_str() != ',': - break - self.expect(',') - self.expect('}') - node = DictExpr(items) - return node - - def parse_set_expr(self, first: Node) -> SetExpr: - items = [first] - while self.current_str() != '}' and not self.eol(): - self.expect(',') - if self.current_str() == '}': - break - items.append(self.parse_expression(precedence[','])) - self.expect('}') - expr = SetExpr(items) - return expr - - def parse_set_comprehension(self, expr: Node): - gen = self.parse_generator_expr(expr) - self.expect('}') - set_comp = SetComprehension(gen) - return set_comp - - def parse_dict_comprehension(self, key: Node, value: Node, - colon: Token) -> DictionaryComprehension: - indices, sequences, condlists = self.parse_comp_for() - dic = DictionaryComprehension(key, value, indices, sequences, condlists) - dic.set_line(colon) - self.expect('}') - return dic - - def parse_tuple_expr(self, expr: Node, - prec: int = precedence[',']) -> TupleExpr: - items = [expr] - while True: - self.expect(',') - if (self.current_str() in [')', ']', '=', ':'] or - isinstance(self.current(), Break)): - break - items.append(self.parse_expression(prec, star_expr_allowed=True)) - if self.current_str() != ',': break - node = TupleExpr(items) - return node - - def parse_backquote_expr(self) -> BackquoteExpr: - self.expect('`') - expr = self.parse_expression() - self.expect('`') - return BackquoteExpr(expr) - - def parse_name_expr(self) -> NameExpr: - tok = self.expect_type(Name) - node = NameExpr(tok.string) - node.set_line(tok) - return node - - octal_int = re.compile('0+[1-9]') - - def parse_int_expr(self) -> IntExpr: - tok = self.expect_type(IntLit) - string = tok.string.rstrip('lL') # Strip L prefix (Python 2 long literals) - if self.octal_int.match(string): - value = int(string, 8) - else: - value = int(string, 0) - node = IntExpr(value) - return node - - def parse_str_expr(self) -> Node: - # XXX \uxxxx literals - token = self.expect_type(StrLit) - value = cast(StrLit, token).parsed() - is_unicode = False - while isinstance(self.current(), (StrLit, UnicodeLit)): - token = self.skip() - if isinstance(token, StrLit): - value += token.parsed() - elif isinstance(token, UnicodeLit): - value += token.parsed() - is_unicode = True - if is_unicode or (self.pyversion[0] == 2 and 'unicode_literals' in self.future_options): - node = UnicodeExpr(value) # type: Node - else: - node = StrExpr(value) - return node - - def parse_bytes_literal(self) -> Node: - # XXX \uxxxx literals - tok = [self.expect_type(BytesLit)] - value = (cast(BytesLit, tok[0])).parsed() - while isinstance(self.current(), BytesLit): - t = cast(BytesLit, self.skip()) - value += t.parsed() - if self.pyversion[0] >= 3: - node = BytesExpr(value) # type: Node - else: - node = StrExpr(value) - return node - - def parse_unicode_literal(self) -> Node: - # XXX \uxxxx literals - token = self.expect_type(UnicodeLit) - value = cast(UnicodeLit, token).parsed() - while isinstance(self.current(), (UnicodeLit, StrLit)): - token = cast(Union[UnicodeLit, StrLit], self.skip()) - value += token.parsed() - if self.pyversion[0] >= 3: - # Python 3.3 supports u'...' as an alias of '...'. - node = StrExpr(value) # type: Node - else: - node = UnicodeExpr(value) - return node - - def parse_float_expr(self) -> FloatExpr: - tok = self.expect_type(FloatLit) - node = FloatExpr(float(tok.string)) - return node - - def parse_complex_expr(self) -> ComplexExpr: - tok = self.expect_type(ComplexLit) - node = ComplexExpr(complex(tok.string)) - return node - - def parse_call_expr(self, callee: Any) -> CallExpr: - self.expect('(') - args, kinds, names = self.parse_arg_expr() - self.expect(')') - node = CallExpr(callee, args, kinds, names) - return node - - def parse_arg_expr(self) -> Tuple[List[Node], List[int], List[str]]: - """Parse arguments in a call expression (within '(' and ')'). - - Return a tuple with these items: - argument expressions - argument kinds - argument names (for named arguments; None for ordinary args) - """ - args = [] # type: List[Node] - kinds = [] # type: List[int] - names = [] # type: List[str] - var_arg = False - dict_arg = False - named_args = False - while self.current_str() != ')' and not self.eol() and not dict_arg: - if isinstance(self.current(), Name) and self.peek().string == '=': - # Named argument - name = self.expect_type(Name) - self.expect('=') - kinds.append(nodes.ARG_NAMED) - names.append(name.string) - named_args = True - elif (self.current_str() == '*' and not var_arg and not dict_arg): - # *args - var_arg = True - self.expect('*') - kinds.append(nodes.ARG_STAR) - names.append(None) - elif self.current_str() == '**': - # **kwargs - self.expect('**') - dict_arg = True - kinds.append(nodes.ARG_STAR2) - names.append(None) - elif not var_arg and not named_args: - # Ordinary argument - kinds.append(nodes.ARG_POS) - names.append(None) - else: - self.parse_error() - args.append(self.parse_expression(precedence[','])) - if self.current_str() != ',': - break - self.expect(',') - return args, kinds, names - - def parse_member_expr(self, expr: Any) -> Node: - self.expect('.') - name = self.expect_type(Name) - if (isinstance(expr, CallExpr) and isinstance(expr.callee, NameExpr) - and expr.callee.name == 'super'): - # super() expression - node = SuperExpr(name.string) # type: Node - else: - node = MemberExpr(expr, name.string) - return node - - def parse_index_expr(self, base: Any) -> IndexExpr: - self.expect('[') - index = self.parse_slice_item() - if self.current_str() == ',': - # Extended slicing such as x[1:, :2]. - items = [index] - while self.current_str() == ',': - self.skip() - if self.current_str() == ']' or isinstance(self.current(), Break): - break - items.append(self.parse_slice_item()) - index = TupleExpr(items) - index.set_line(items[0].line) - self.expect(']') - node = IndexExpr(base, index) - return node - - def parse_slice_item(self) -> Node: - if self.current_str() != ':': - if self.current_str() == '...': - # Ellipsis is valid here even in Python 2 (but not elsewhere). - ellipsis = EllipsisExpr() - token = self.skip() - ellipsis.set_line(token) - return ellipsis - else: - item = self.parse_expression(precedence[',']) - else: - item = None - if self.current_str() == ':': - # Slice. - index = item - colon = self.expect(':') - if self.current_str() not in (']', ':', ','): - end_index = self.parse_expression(precedence[',']) - else: - end_index = None - stride = None - if self.current_str() == ':': - self.expect(':') - if self.current_str() not in (']', ','): - stride = self.parse_expression(precedence[',']) - item = SliceExpr(index, end_index, stride).set_line(colon.line) - return item - - def parse_bin_op_expr(self, left: Node, prec: int) -> OpExpr: - op = self.expect_type(Op) - op_str = op.string - if op_str == '~': - self.ind -= 1 - self.parse_error() - right = self.parse_expression(prec) - node = OpExpr(op_str, left, right) - return node - - def parse_comparison_expr(self, left: Node, prec: int) -> ComparisonExpr: - operators_str = [] - operands = [left] - - while True: - op = self.expect_type(Op) - op_str = op.string - if op_str == 'not': - if self.current_str() == 'in': - op_str = 'not in' - self.skip() - else: - self.parse_error() - elif op_str == 'is' and self.current_str() == 'not': - op_str = 'is not' - self.skip() - - operators_str.append(op_str) - operand = self.parse_expression(prec) - operands.append(operand) - - # Continue if next token is a comparison operator - self.current() - s = self.current_str() - if s not in op_comp: - break - - node = ComparisonExpr(operators_str, operands) - return node - - def parse_unary_expr(self) -> UnaryExpr: - op_tok = self.skip() - op = op_tok.string - if op == '-' or op == '+': - prec = precedence['-u'] - else: - prec = precedence[op] - expr = self.parse_expression(prec) - node = UnaryExpr(op, expr) - return node - - def parse_lambda_expr(self) -> FuncExpr: - lambda_tok = self.expect('lambda') - - args, extra_stmts = self.parse_arg_list(allow_signature=False) - - # Use 'object' as the placeholder return type; it will be inferred - # later. We can't use 'Any' since it could make type inference results - # less precise. - ret_type = UnboundType('__builtins__.object') - typ = self.build_func_annotation(ret_type, args, - lambda_tok.line, is_default_ret=True) - - colon = self.expect(':') - - expr = self.parse_expression(precedence[',']) - - nodes = [ReturnStmt(expr).set_line(lambda_tok)] - # Potentially insert extra assignment statements to the beginning of the - # body, used to decompose Python 2 tuple arguments. - nodes[:0] = extra_stmts - body = Block(nodes) - body.set_line(colon) - - return FuncExpr(args, body, typ) - - # Helper methods - - def skip(self) -> Token: - self.ind += 1 - return self.tok[self.ind - 1] - - def expect(self, string: str) -> Token: - if self.current_str() == string: - self.ind += 1 - return self.tok[self.ind - 1] - else: - self.parse_error() - - def expect_indent(self) -> Token: - if isinstance(self.current(), Indent): - return self.expect_type(Indent) - else: - self.fail('Expected an indented block', self.current().line) - return none - - def fail(self, msg: str, line: int) -> None: - self.errors.report(line, msg) - - def expect_type(self, typ: type) -> Token: - current = self.current() - if isinstance(current, typ): - self.ind += 1 - return current - else: - self.parse_error() - - def expect_colon_and_break(self) -> Tuple[Token, Token]: - return self.expect_type(Colon), self.expect_type(Break) - - def expect_break(self) -> Token: - return self.expect_type(Break) - - def current(self) -> Token: - return self.tok[self.ind] - - def current_str(self) -> str: - return self.current().string - - def peek(self) -> Token: - return self.tok[self.ind + 1] - - def parse_error(self) -> None: - self.parse_error_at(self.current()) - raise ParseError() - - def parse_error_at(self, tok: Token, skip: bool = True, reason: Optional[str] = None) -> None: - msg = '' - if isinstance(tok, LexError): - msg = token_repr(tok) - msg = msg[0].upper() + msg[1:] - elif isinstance(tok, Indent) or isinstance(tok, Dedent): - msg = 'Inconsistent indentation' - else: - formatted_reason = ": {}".format(reason) if reason else "" - msg = 'Parse error before {}{}'.format(token_repr(tok), formatted_reason) - - self.errors.report(tok.line, msg) - - if skip: - self.skip_until_next_line() - - def skip_until_break(self) -> None: - n = 0 - while (not isinstance(self.current(), Break) - and not isinstance(self.current(), Eof)): - self.skip() - n += 1 - if isinstance(self.tok[self.ind - 1], Colon) and n > 1: - self.ind -= 1 - - def skip_until_next_line(self) -> None: - self.skip_until_break() - if isinstance(self.current(), Break): - self.skip() - - def eol(self) -> bool: - return isinstance(self.current(), Break) or self.eof() - - def eof(self) -> bool: - return isinstance(self.current(), Eof) - - # Type annotation related functionality - - def parse_type(self) -> Type: - try: - typ, self.ind = parse_type(self.tok, self.ind) - except TypeParseError as e: - self.parse_error_at(e.token, reason=e.message) - raise ParseError() - return typ - - annotation_prefix_re = re.compile(r'#\s*type:') - ignore_prefix_re = re.compile(r'ignore\b') - - def parse_type_comment(self, token: Token, signature: bool) -> Type: - """Parse a '# type: ...' annotation. - - Return None if no annotation found. If signature is True, expect - a type signature of form (...) -> t. - """ - whitespace_or_comments = token.rep().strip() - if self.annotation_prefix_re.match(whitespace_or_comments): - type_as_str = whitespace_or_comments.split(':', 1)[1].strip() - if self.ignore_prefix_re.match(type_as_str): - # Actually a "# type: ignore" annotation -> not a type. - return None - tokens = lex.lex(type_as_str, token.line)[0] - if len(tokens) < 2: - # Empty annotation (only Eof token) - self.errors.report(token.line, 'Empty type annotation') - return None - try: - if not signature: - type, index = parse_types(tokens, 0) - else: - type, index = parse_signature(tokens) - except TypeParseError as e: - self.parse_error_at(e.token, skip=False, reason=e.message) - return None - if index < len(tokens) - 2: - self.parse_error_at(tokens[index], skip=False) - return None - return type - else: - return None - - -class ParseError(Exception): pass - - -def token_repr(tok: Token) -> str: - """Return a representation of a token for use in parse error messages.""" - if isinstance(tok, Break): - return 'end of line' - elif isinstance(tok, Eof): - return 'end of file' - elif isinstance(tok, Keyword) or isinstance(tok, Name): - return '"{}"'.format(tok.string) - elif isinstance(tok, IntLit) or isinstance(tok, FloatLit) or isinstance(tok, ComplexLit): - return 'numeric literal' - elif isinstance(tok, StrLit) or isinstance(tok, UnicodeLit): - return 'string literal' - elif (isinstance(tok, Punct) or isinstance(tok, Op) - or isinstance(tok, Colon)): - return tok.string - elif isinstance(tok, Bom): - return 'byte order mark' - elif isinstance(tok, Indent): - return 'indent' - elif isinstance(tok, Dedent): - return 'dedent' - elif isinstance(tok, EllipsisToken): - return '...' - else: - if isinstance(tok, LexError): - t = tok.type - if t == lex.NUMERIC_LITERAL_ERROR: - return 'invalid numeric literal' - elif t == lex.UNTERMINATED_STRING_LITERAL: - return 'unterminated string literal' - elif t == lex.INVALID_CHARACTER: - msg = 'unrecognized character' - if ord(tok.string) in range(33, 127): - msg += ' ' + tok.string - return msg - elif t == lex.INVALID_DEDENT: - return 'inconsistent indentation' - elif t == lex.DECODE_ERROR: - return tok.message - raise ValueError('Unknown token {}'.format(repr(tok))) - - -if __name__ == '__main__': - # Parse a file and dump the AST (or display errors). - import sys - - def usage(): - print('Usage: parse.py [--py2] [--quiet] FILE [...]', file=sys.stderr) - sys.exit(2) - - args = sys.argv[1:] - pyversion = defaults.PYTHON3_VERSION - quiet = False - while args and args[0].startswith('--'): - if args[0] == '--py2': - pyversion = defaults.PYTHON2_VERSION - elif args[0] == '--quiet': - quiet = True - else: - usage() - args = args[1:] - if len(args) < 1: - usage() - status = 0 - for fnam in args: - s = open(fnam, 'rb').read() - errors = Errors() - try: - options = Options() - options.python_version = pyversion - tree = parse(s, fnam, None, options=options) - if not quiet: - print(tree) - except CompileError as e: - for msg in e.messages: - sys.stderr.write('%s\n' % msg) - status = 1 - sys.exit(status) +"""Mypy parser. + +Constructs a parse tree (abstract syntax tree) based on a string +representing a source file. Performs only minimal semantic checks. +""" + +import re + +from typing import List, Tuple, Any, Set, cast, Union, Optional + +from mypy import lex, docstring +from mypy.lex import ( + Token, Eof, Bom, Break, Name, Colon, Dedent, IntLit, StrLit, BytesLit, + UnicodeLit, FloatLit, Op, Indent, Keyword, Punct, LexError, ComplexLit, + EllipsisToken +) +import mypy.types +from mypy.nodes import ( + MypyFile, Import, Node, ImportAll, ImportFrom, FuncDef, OverloadedFuncDef, + ClassDef, Decorator, Block, Var, OperatorAssignmentStmt, + ExpressionStmt, AssignmentStmt, ReturnStmt, RaiseStmt, AssertStmt, + DelStmt, BreakStmt, ContinueStmt, PassStmt, GlobalDecl, + WhileStmt, ForStmt, IfStmt, TryStmt, WithStmt, + TupleExpr, GeneratorExpr, ListComprehension, ListExpr, ConditionalExpr, + DictExpr, SetExpr, NameExpr, IntExpr, StrExpr, BytesExpr, UnicodeExpr, + FloatExpr, CallExpr, SuperExpr, MemberExpr, IndexExpr, SliceExpr, OpExpr, + UnaryExpr, FuncExpr, PrintStmt, ImportBase, ComparisonExpr, + StarExpr, YieldFromExpr, NonlocalDecl, DictionaryComprehension, + SetComprehension, ComplexExpr, EllipsisExpr, YieldExpr, ExecStmt, Argument, + BackquoteExpr +) +from mypy import defaults +from mypy import nodes +from mypy.errors import Errors, CompileError +from mypy.types import Type, CallableType, AnyType, UnboundType +from mypy.parsetype import ( + parse_type, parse_types, parse_signature, TypeParseError, parse_str_as_signature +) +from mypy.options import Options + +from mypy import experiments + + +precedence = { + '**': 16, + '-u': 15, '+u': 15, '~': 15, # unary operators (-, + and ~) + '': 14, + '*': 13, '/': 13, '//': 13, '%': 13, + '+': 12, '-': 12, + '>>': 11, '<<': 11, + '&': 10, + '^': 9, + '|': 8, + '==': 7, '!=': 7, '<': 7, '>': 7, '<=': 7, '>=': 7, 'is': 7, 'in': 7, + '*u': 7, # unary * for star expressions + 'not': 6, + 'and': 5, + 'or': 4, + '': 3, # conditional expression + '': 2, # list comprehension + ',': 1} + + +op_assign = set([ + '+=', '-=', '*=', '/=', '//=', '%=', '**=', '|=', '&=', '^=', '>>=', + '<<=']) + +op_comp = set([ + '>', '<', '==', '>=', '<=', '<>', '!=', 'is', 'is', 'in', 'not']) + +none = Token('') # Empty token + + +def parse(source: Union[str, bytes], + fnam: str, + errors: Errors, + options: Options) -> MypyFile: + """Parse a source file, without doing any semantic analysis. + + Return the parse tree. If errors is not provided, raise ParseError + on failure. Otherwise, use the errors object to report parse errors. + + The python_version (major, minor) option determines the Python syntax variant. + """ + if options.fast_parser: + import mypy.fastparse + return mypy.fastparse.parse(source, + fnam=fnam, + errors=errors, + pyversion=options.python_version, + custom_typing_module=options.custom_typing_module) + + is_stub_file = bool(fnam) and fnam.endswith('.pyi') + parser = Parser(fnam, + errors, + options.python_version, + options.custom_typing_module, + is_stub_file=is_stub_file) + tree = parser.parse(source) + tree.path = fnam + tree.is_stub = is_stub_file + return tree + + +class Parser: + """Mypy parser that parses a string into an AST. + + Parses type annotations in addition to basic Python syntax. It supports both Python 2 and 3 + (though Python 2 support is incomplete). + + The AST classes are defined in mypy.nodes and mypy.types. + """ + + tok = None # type: List[Token] + ind = 0 + errors = None # type: Errors + # If True, raise an exception on any parse error. Otherwise, errors are reported via 'errors'. + raise_on_error = False + + # Are we currently parsing the body of a class definition? + is_class_body = False + # All import nodes encountered so far in this parse unit. + imports = None # type: List[ImportBase] + # Names imported from __future__. + future_options = None # type: List[str] + # Lines to ignore (using # type: ignore). + ignored_lines = None # type: Set[int] + + def __init__(self, fnam: str, errors: Errors, pyversion: Tuple[int, int], + custom_typing_module: str = None, is_stub_file: bool = False) -> None: + self.raise_on_error = errors is None + self.pyversion = pyversion + self.custom_typing_module = custom_typing_module + self.is_stub_file = is_stub_file + if errors is not None: + self.errors = errors + else: + self.errors = Errors() + if fnam is not None: + self.errors.set_file(fnam) + else: + self.errors.set_file('') + + def parse(self, s: Union[str, bytes]) -> MypyFile: + self.tok, self.ignored_lines = lex.lex(s, pyversion=self.pyversion, + is_stub_file=self.is_stub_file) + self.ind = 0 + self.imports = [] + self.future_options = [] + file = self.parse_file() + if self.raise_on_error and self.errors.is_errors(): + self.errors.raise_error() + return file + + def weak_opts(self) -> Set[str]: + """Do weak typing if any of the first ten tokens is a comment saying so. + + The comment can be one of: + # mypy: weak=global + # mypy: weak=local + # mypy: weak <- defaults to local + """ + regexp = re.compile(r'^[\s]*# *mypy: *weak(=?)([^\s]*)', re.M) + for t in self.tok[:10]: + for s in [t.string, t.pre]: + m = regexp.search(s) + if m: + opts = set(x for x in m.group(2).split(',') if x) + if not opts: + opts.add('local') + return opts + return set() + + def parse_file(self) -> MypyFile: + """Parse a mypy source file.""" + is_bom = self.parse_bom() + defs = self.parse_defs() + weak_opts = self.weak_opts() + self.expect_type(Eof) + node = MypyFile(defs, self.imports, is_bom, self.ignored_lines, + weak_opts=weak_opts) + return node + + # Parse the initial part + + def parse_bom(self) -> bool: + """Parse the optional byte order mark at the beginning of a file.""" + if isinstance(self.current(), Bom): + self.expect_type(Bom) + if isinstance(self.current(), Break): + self.expect_break() + return True + else: + return False + + def parse_import(self) -> Import: + self.expect('import') + ids = [] + while True: + id = self.parse_qualified_name() + translated = self.translate_module_id(id) + as_id = None + if self.current_str() == 'as': + self.expect('as') + name_tok = self.expect_type(Name) + as_id = name_tok.string + elif translated != id: + as_id = id + ids.append((translated, as_id)) + if self.current_str() != ',': + break + self.expect(',') + node = Import(ids) + self.imports.append(node) + return node + + def translate_module_id(self, id: str) -> str: + """Return the actual, internal module id for a source text id. + + For example, translate '__builtin__' in Python 2 to 'builtins'. + """ + if id == self.custom_typing_module: + return 'typing' + elif id == '__builtin__' and self.pyversion[0] == 2: + # HACK: __builtin__ in Python 2 is aliases to builtins. However, the implementation + # is named __builtin__.py (there is another layer of translation elsewhere). + return 'builtins' + return id + + def parse_import_from(self) -> Node: + self.expect('from') + + # Build the list of beginning relative tokens. + relative = 0 + while self.current_str() in (".", "..."): + relative += len(self.current_str()) + self.skip() + + # Parse qualified name to actually import from. + if self.current_str() == "import": + # Empty/default values. + name = "" + else: + name = self.parse_qualified_name() + + name = self.translate_module_id(name) + + # Parse import list + self.expect('import') + node = None # type: ImportBase + if self.current_str() == '*': + if name == '__future__': + self.parse_error() + # An import all from a module node: + self.skip() + node = ImportAll(name, relative) + else: + is_paren = self.current_str() == '(' + if is_paren: + self.expect('(') + targets = [] # type: List[Tuple[str, str]] + while True: + id, as_id = self.parse_import_name() + if '%s.%s' % (name, id) == self.custom_typing_module: + if targets or self.current_str() == ',': + self.fail('You cannot import any other modules when you ' + 'import a custom typing module', + self.current().line) + node = Import([('typing', as_id)]) + self.skip_until_break() + break + targets.append((id, as_id)) + if self.current_str() != ',': + break + self.expect(',') + if is_paren and self.current_str() == ')': + break + if is_paren: + self.expect(')') + if node is None: + node = ImportFrom(name, relative, targets) + self.imports.append(node) + if name == '__future__': + self.future_options.extend(target[0] for target in targets) + return node + + def parse_import_name(self) -> Tuple[str, Optional[str]]: + tok = self.expect_type(Name) + name = tok.string + if self.current_str() == 'as': + self.skip() + as_name = self.expect_type(Name) + return name, as_name.string + else: + return name, None + + def parse_qualified_name(self) -> str: + """Parse a name with an optional module qualifier. + + Return a tuple with the name as a string and a token array + containing all the components of the name. + """ + tok = self.expect_type(Name) + n = tok.string + while self.current_str() == '.': + self.expect('.') + tok = self.expect_type(Name) + n += '.' + tok.string + return n + + # Parsing global definitions + + def parse_defs(self) -> List[Node]: + defs = [] # type: List[Node] + while not self.eof(): + try: + defn, is_simple = self.parse_statement() + if is_simple: + self.expect_break() + if defn is not None: + if not self.try_combine_overloads(defn, defs): + defs.append(defn) + except ParseError: + pass + return defs + + def parse_class_def(self) -> ClassDef: + old_is_class_body = self.is_class_body + self.is_class_body = True + + self.expect('class') + metaclass = None + + try: + base_types = [] # type: List[Node] + try: + name_tok = self.expect_type(Name) + name = name_tok.string + + self.errors.push_type(name) + + if self.current_str() == '(': + self.skip() + while True: + if self.current_str() == ')': + break + if self.peek().string == '=': + metaclass = self.parse_class_keywords() + break + base_types.append(self.parse_super_type()) + if self.current_str() != ',': + break + self.skip() + self.expect(')') + except ParseError: + pass + + defs, _ = self.parse_block() + + node = ClassDef(name, defs, None, base_types, metaclass=metaclass) + return node + finally: + self.errors.pop_type() + self.is_class_body = old_is_class_body + + def parse_class_keywords(self) -> Optional[str]: + """Parse the class keyword arguments, keeping the metaclass but + ignoring all others. Returns None if the metaclass isn't found. + """ + metaclass = None + while True: + key = self.expect_type(Name) + self.expect('=') + if key.string == 'metaclass': + metaclass = self.parse_qualified_name() + else: + # skip the class value + self.parse_expression(precedence[',']) + if self.current_str() != ',': + break + self.skip() + if self.current_str() == ')': + break + return metaclass + + def parse_super_type(self) -> Node: + return self.parse_expression(precedence[',']) + + def parse_decorated_function_or_class(self) -> Node: + decorators = [] + no_type_checks = False + while self.current_str() == '@': + self.expect('@') + d_exp = self.parse_expression() + if self.is_no_type_check_decorator(d_exp): + no_type_checks = True + decorators.append(d_exp) + self.expect_break() + if self.current_str() != 'class': + func = self.parse_function(no_type_checks) + func.is_decorated = True + var = Var(func.name()) + # Types of decorated functions must always be inferred. + var.is_ready = False + var.set_line(decorators[0].line) + node = Decorator(func, decorators, var) + return node + else: + cls = self.parse_class_def() + cls.decorators = decorators + return cls + + def is_no_type_check_decorator(self, expr: Node) -> bool: + if isinstance(expr, NameExpr): + return expr.name == 'no_type_check' + elif isinstance(expr, MemberExpr): + if isinstance(expr.expr, NameExpr): + return expr.expr.name == 'typing' and expr.name == 'no_type_check' + else: + return False + + def parse_function(self, no_type_checks: bool=False) -> FuncDef: + def_tok = self.expect('def') + is_method = self.is_class_body + self.is_class_body = False + try: + (name, args, typ, is_error, extra_stmts) = self.parse_function_header(no_type_checks) + + arg_kinds = [arg.kind for arg in args] + arg_names = [arg.variable.name() for arg in args] + + body, comment_type = self.parse_block(allow_type=True) + # Potentially insert extra assignment statements to the beginning of the + # body, used to decompose Python 2 tuple arguments. + body.body[:0] = extra_stmts + if comment_type: + # The function has a # type: ... signature. + if typ: + self.errors.report( + def_tok.line, 'Function has duplicate type signatures') + sig = cast(CallableType, comment_type) + if sig.is_ellipsis_args: + # When we encounter an ellipsis, fill in the arg_types with + # a bunch of AnyTypes, emulating Callable[..., T] + arg_types = [AnyType()] * len(arg_kinds) # type: List[Type] + typ = CallableType( + arg_types, + arg_kinds, + arg_names, + sig.ret_type, + None, + line=def_tok.line) + elif is_method and len(sig.arg_kinds) < len(arg_kinds): + self.check_argument_kinds(arg_kinds, + [nodes.ARG_POS] + sig.arg_kinds, + def_tok.line) + # Add implicit 'self' argument to signature. + first_arg = [AnyType()] # type: List[Type] + typ = CallableType( + first_arg + sig.arg_types, + arg_kinds, + arg_names, + sig.ret_type, + None, + line=def_tok.line) + else: + self.check_argument_kinds(arg_kinds, sig.arg_kinds, + def_tok.line) + typ = CallableType( + sig.arg_types, + arg_kinds, + arg_names, + sig.ret_type, + None, + line=def_tok.line) + + # If there was a serious error, we really cannot build a parse tree + # node. + if is_error: + return None + + if typ and isinstance(typ.ret_type, UnboundType): + typ.ret_type.is_ret_type = True + + node = FuncDef(name, args, body, typ) + node.set_line(def_tok) + if typ is not None: + typ.definition = node + return node + finally: + self.errors.pop_function() + self.is_class_body = is_method + + def check_argument_kinds(self, funckinds: List[int], sigkinds: List[int], + line: int) -> None: + """Check that arguments are consistent. + + This verifies that they have the same number and the kinds correspond. + + Arguments: + funckinds: kinds of arguments in function definition + sigkinds: kinds of arguments in signature (after # type:) + """ + if len(funckinds) != len(sigkinds): + if len(funckinds) > len(sigkinds): + self.fail("Type signature has too few arguments", line) + else: + self.fail("Type signature has too many arguments", line) + return + for kind, token in [(nodes.ARG_STAR, '*'), + (nodes.ARG_STAR2, '**')]: + if ((funckinds.count(kind) != sigkinds.count(kind)) or + (kind in funckinds and sigkinds.index(kind) != funckinds.index(kind))): + self.fail( + "Inconsistent use of '{}' in function " + "signature".format(token), line) + + def parse_function_header( + self, no_type_checks: bool=False) -> Tuple[str, + List[Argument], + CallableType, + bool, + List[AssignmentStmt]]: + """Parse function header (a name followed by arguments) + + Return a 5-tuple with the following items: + name + arguments + signature (annotation) + error flag (True if error) + extra statements needed to decompose arguments (usually empty) + + See parse_arg_list for an explanation of the final tuple item. + """ + name = '' + + try: + name_tok = self.expect_type(Name) + name = name_tok.string + + self.errors.push_function(name) + + args, typ, extra_stmts = self.parse_args(no_type_checks) + except ParseError: + if not isinstance(self.current(), Break): + self.ind -= 1 # Kludge: go back to the Break token + # Resynchronise parsing by going back over :, if present. + if isinstance(self.tok[self.ind - 1], Colon): + self.ind -= 1 + return (name, [], None, True, []) + + return (name, args, typ, False, extra_stmts) + + def parse_args(self, no_type_checks: bool=False) -> Tuple[List[Argument], + CallableType, + List[AssignmentStmt]]: + """Parse a function signature (...) [-> t]. + + See parse_arg_list for an explanation of the final tuple item. + """ + lparen = self.expect('(') + + # Parse the argument list (everything within '(' and ')'). + args, extra_stmts = self.parse_arg_list(no_type_checks=no_type_checks) + + self.expect(')') + + if self.current_str() == '->': + self.skip() + if no_type_checks: + self.parse_expression() + ret_type = None + else: + ret_type = self.parse_type() + else: + ret_type = None + + arg_kinds = [arg.kind for arg in args] + self.verify_argument_kinds(arg_kinds, lparen.line) + + annotation = self.build_func_annotation( + ret_type, args, lparen.line) + + return args, annotation, extra_stmts + + def build_func_annotation(self, ret_type: Type, args: List[Argument], + line: int, is_default_ret: bool = False) -> CallableType: + arg_types = [arg.type_annotation for arg in args] + # Are there any type annotations? + if ((ret_type and not is_default_ret) + or arg_types != [None] * len(arg_types)): + # Yes. Construct a type for the function signature. + return self.construct_function_type(args, ret_type, line) + else: + return None + + def parse_arg_list(self, allow_signature: bool = True, + no_type_checks: bool=False) -> Tuple[List[Argument], + List[AssignmentStmt]]: + """Parse function definition argument list. + + This includes everything between '(' and ')' (but not the + parentheses). + + Return tuple (arguments, + extra statements for decomposing arguments). + + The final argument is only used for Python 2 argument lists with + tuples; they contain destructuring assignment statements used to + decompose tuple arguments. For example, consider a header like this: + + . def f((x, y)) + + The actual (sole) argument will be __tuple_arg_1 (a generated + name), whereas the extra statement list will contain a single + assignment statement corresponding to this assignment: + + x, y = __tuple_arg_1 + """ + args = [] # type: List[Argument] + extra_stmts = [] + # This is for checking duplicate argument names. + arg_names = [] # type: List[str] + has_tuple_arg = False + + require_named = False + bare_asterisk_before = -1 + + if self.current_str() != ')' and self.current_str() != ':': + while self.current_str() != ')': + if self.current_str() == '*' and self.peek().string == ',': + self.expect('*') + require_named = True + bare_asterisk_before = len(args) + elif self.current_str() in ['*', '**']: + if bare_asterisk_before == len(args): + # named arguments must follow bare * + self.parse_error() + + arg = self.parse_asterisk_arg( + allow_signature, + no_type_checks, + ) + args.append(arg) + require_named = True + elif self.current_str() == '(': + arg, extra_stmt, names = self.parse_tuple_arg(len(args)) + args.append(arg) + if extra_stmt is not None: + extra_stmts.append(extra_stmt) + has_tuple_arg = True + arg_names.extend(names) + else: + arg, require_named = self.parse_normal_arg( + require_named, + allow_signature, + no_type_checks, + ) + args.append(arg) + arg_names.append(arg.variable.name()) + + if self.current().string != ',': + break + + self.expect(',') + + # Non-tuple argument dupes will be checked elsewhere. Avoid + # generating duplicate errors. + if has_tuple_arg: + self.check_duplicate_argument_names(arg_names) + + return args, extra_stmts + + def check_duplicate_argument_names(self, names: List[str]) -> None: + found = set() # type: Set[str] + for name in names: + if name in found: + self.fail('Duplicate argument name "{}"'.format(name), + self.current().line) + found.add(name) + + def parse_asterisk_arg(self, + allow_signature: bool, + no_type_checks: bool) -> Argument: + asterisk = self.skip() + name = self.expect_type(Name) + variable = Var(name.string) + if asterisk.string == '*': + kind = nodes.ARG_STAR + else: + kind = nodes.ARG_STAR2 + + type = None + if no_type_checks: + self.parse_parameter_annotation() + else: + type = self.parse_arg_type(allow_signature) + + return Argument(variable, type, None, kind) + + def parse_tuple_arg(self, index: int) -> Tuple[Argument, AssignmentStmt, List[str]]: + """Parse a single Python 2 tuple argument. + + Example: def f(x, (y, z)): ... + + The tuple arguments gets transformed into an assignment in the + function body (the second return value). + + Return tuple (argument, decomposing assignment, list of names defined). + + Special case: if the argument is just (x) then it's not a tuple; + we indicate this by returning (argument, None, ['x']). + However, if the argument is (x,) then it *is* a (singleton) tuple. + """ + line = self.current().line + # Generate a new argument name that is very unlikely to clash with anything. + arg_name = '__tuple_arg_{}'.format(index + 1) + if self.pyversion[0] >= 3: + self.fail('Tuples in argument lists only supported in Python 2 mode', line) + paren_arg = self.parse_parentheses() + self.verify_tuple_arg(paren_arg) + if isinstance(paren_arg, NameExpr): + # This isn't a tuple. Revert to a normal argument. + arg_name = paren_arg.name + decompose = None + else: + rvalue = NameExpr(arg_name) + rvalue.set_line(line) + decompose = AssignmentStmt([paren_arg], rvalue) + decompose.set_line(line) + kind = nodes.ARG_POS + initializer = None + if self.current_str() == '=': + self.expect('=') + initializer = self.parse_expression(precedence[',']) + kind = nodes.ARG_OPT + var = Var(arg_name) + arg_names = self.find_tuple_arg_argument_names(paren_arg) + return Argument(var, None, initializer, kind), decompose, arg_names + + def verify_tuple_arg(self, paren_arg: Node) -> None: + if isinstance(paren_arg, TupleExpr): + if not paren_arg.items: + self.fail('Empty tuple not valid as an argument', paren_arg.line) + for item in paren_arg.items: + self.verify_tuple_arg(item) + elif not isinstance(paren_arg, NameExpr): + self.fail('Invalid item in tuple argument', paren_arg.line) + + def find_tuple_arg_argument_names(self, node: Node) -> List[str]: + result = [] # type: List[str] + if isinstance(node, TupleExpr): + for item in node.items: + result.extend(self.find_tuple_arg_argument_names(item)) + elif isinstance(node, NameExpr): + result.append(node.name) + return result + + def parse_normal_arg(self, require_named: bool, + allow_signature: bool, + no_type_checks: bool) -> Tuple[Argument, bool]: + name = self.expect_type(Name) + variable = Var(name.string) + + type = None + if no_type_checks: + self.parse_parameter_annotation() + else: + type = self.parse_arg_type(allow_signature) + + initializer = None # type: Node + if self.current_str() == '=': + self.expect('=') + initializer = self.parse_expression(precedence[',']) + if require_named: + kind = nodes.ARG_NAMED + else: + kind = nodes.ARG_OPT + else: + if require_named: + kind = nodes.ARG_NAMED + else: + kind = nodes.ARG_POS + + self.set_type_optional(type, initializer) + + return Argument(variable, type, initializer, kind), require_named + + def set_type_optional(self, type: Type, initializer: Node) -> None: + if not experiments.STRICT_OPTIONAL: + return + # Indicate that type should be wrapped in an Optional if arg is initialized to None. + optional = isinstance(initializer, NameExpr) and initializer.name == 'None' + if isinstance(type, UnboundType): + type.optional = optional + + def parse_parameter_annotation(self) -> Node: + if self.current_str() == ':': + self.skip() + return self.parse_expression(precedence[',']) + + def parse_arg_type(self, allow_signature: bool) -> Type: + if self.current_str() == ':' and allow_signature: + self.skip() + return self.parse_type() + else: + return None + + def verify_argument_kinds(self, kinds: List[int], line: int) -> None: + found = set() # type: Set[int] + for i, kind in enumerate(kinds): + if kind == nodes.ARG_POS and found & set([nodes.ARG_OPT, + nodes.ARG_STAR, + nodes.ARG_STAR2]): + self.fail('Invalid argument list', line) + elif kind == nodes.ARG_STAR and nodes.ARG_STAR in found: + self.fail('Invalid argument list', line) + elif kind == nodes.ARG_STAR2 and i != len(kinds) - 1: + self.fail('Invalid argument list', line) + found.add(kind) + + def construct_function_type(self, args: List[Argument], ret_type: Type, + line: int) -> CallableType: + # Complete the type annotation by replacing omitted types with 'Any'. + arg_types = [arg.type_annotation for arg in args] + for i in range(len(arg_types)): + if arg_types[i] is None: + arg_types[i] = AnyType(implicit=True) + if ret_type is None: + ret_type = AnyType(implicit=True) + arg_kinds = [arg.kind for arg in args] + arg_names = [arg.variable.name() for arg in args] + return CallableType(arg_types, arg_kinds, arg_names, ret_type, None, name=None, + variables=None, line=line) + + # Parsing statements + + def parse_block(self, allow_type: bool = False) -> Tuple[Block, Type]: + colon = self.expect(':') + if not isinstance(self.current(), Break): + # Block immediately after ':'. + nodes = [] + while True: + ind = self.ind + stmt, is_simple = self.parse_statement() + if not is_simple: + self.parse_error_at(self.tok[ind]) + break + nodes.append(stmt) + brk = self.expect_break() + if brk.string != ';': + break + node = Block(nodes) + node.set_line(colon) + return node, None + else: + # Indented block. + brk = self.expect_break() + type = self.parse_type_comment(brk, signature=True) + self.expect_indent() + stmt_list = [] # type: List[Node] + if allow_type: + cur = self.current() + if type is None and isinstance(cur, StrLit): + ds = docstring.parse_docstring(cur.parsed()) + if ds and False: # TODO: Enable when this is working. + try: + type = parse_str_as_signature(ds.as_type_str(), cur.line) + except TypeParseError: + # We don't require docstrings to be actually correct. + # TODO: Report something here. + type = None + while (not isinstance(self.current(), Dedent) and + not isinstance(self.current(), Eof)): + try: + stmt, is_simple = self.parse_statement() + if is_simple: + self.expect_break() + if stmt is not None: + if not self.try_combine_overloads(stmt, stmt_list): + stmt_list.append(stmt) + except ParseError: + pass + if isinstance(self.current(), Dedent): + self.skip() + node = Block(stmt_list) + node.set_line(colon) + return node, type + + def try_combine_overloads(self, s: Node, stmt: List[Node]) -> bool: + if isinstance(s, Decorator) and stmt: + fdef = s + n = fdef.func.name() + if isinstance(stmt[-1], Decorator) and stmt[-1].func.name() == n: + stmt[-1] = OverloadedFuncDef([stmt[-1], fdef]) + return True + elif isinstance(stmt[-1], OverloadedFuncDef) and stmt[-1].name() == n: + stmt[-1].items.append(fdef) + return True + return False + + def parse_statement(self) -> Tuple[Node, bool]: + stmt = None # type: Node + t = self.current() + ts = self.current_str() + is_simple = True # Is this a non-block statement? + if ts == 'if': + stmt = self.parse_if_stmt() + is_simple = False + elif ts == 'def': + stmt = self.parse_function() + is_simple = False + elif ts == 'while': + stmt = self.parse_while_stmt() + is_simple = False + elif ts == 'return': + stmt = self.parse_return_stmt() + elif ts == 'for': + stmt = self.parse_for_stmt() + is_simple = False + elif ts == 'try': + stmt = self.parse_try_stmt() + is_simple = False + elif ts == 'break': + stmt = self.parse_break_stmt() + elif ts == 'continue': + stmt = self.parse_continue_stmt() + elif ts == 'pass': + stmt = self.parse_pass_stmt() + elif ts == 'raise': + stmt = self.parse_raise_stmt() + elif ts == 'import': + stmt = self.parse_import() + elif ts == 'from': + stmt = self.parse_import_from() + elif ts == 'class': + stmt = self.parse_class_def() + is_simple = False + elif ts == 'global': + stmt = self.parse_global_decl() + elif ts == 'nonlocal' and self.pyversion[0] >= 3: + stmt = self.parse_nonlocal_decl() + elif ts == 'assert': + stmt = self.parse_assert_stmt() + elif ts == 'del': + stmt = self.parse_del_stmt() + elif ts == 'with': + stmt = self.parse_with_stmt() + is_simple = False + elif ts == '@': + stmt = self.parse_decorated_function_or_class() + is_simple = False + elif ts == 'print' and (self.pyversion[0] == 2 and + 'print_function' not in self.future_options): + stmt = self.parse_print_stmt() + elif ts == 'exec' and self.pyversion[0] == 2: + stmt = self.parse_exec_stmt() + else: + stmt = self.parse_expression_or_assignment() + if stmt is not None: + stmt.set_line(t) + return stmt, is_simple + + def parse_expression_or_assignment(self) -> Node: + expr = self.parse_expression(star_expr_allowed=True) + if self.current_str() == '=': + return self.parse_assignment(expr) + elif self.current_str() in op_assign: + # Operator assignment statement. + op = self.current_str()[:-1] + self.skip() + rvalue = self.parse_expression() + return OperatorAssignmentStmt(op, expr, rvalue) + else: + # Expression statement. + return ExpressionStmt(expr) + + def parse_assignment(self, lvalue: Any) -> Node: + """Parse an assignment statement. + + Assume that lvalue has been parsed already, and the current token is '='. + Also parse an optional '# type:' comment. + """ + self.expect('=') + lvalues = [lvalue] + expr = self.parse_expression(star_expr_allowed=True) + while self.current_str() == '=': + self.skip() + lvalues.append(expr) + expr = self.parse_expression(star_expr_allowed=True) + cur = self.current() + if isinstance(cur, Break): + type = self.parse_type_comment(cur, signature=False) + else: + type = None + return AssignmentStmt(lvalues, expr, type) + + def parse_return_stmt(self) -> ReturnStmt: + self.expect('return') + expr = None + current = self.current() + if current.string == 'yield': + self.parse_error() + if not isinstance(current, Break): + expr = self.parse_expression() + node = ReturnStmt(expr) + return node + + def parse_raise_stmt(self) -> RaiseStmt: + self.expect('raise') + expr = None + from_expr = None + if not isinstance(self.current(), Break): + expr = self.parse_expression() + if self.current_str() == 'from': + self.expect('from') + from_expr = self.parse_expression() + node = RaiseStmt(expr, from_expr) + return node + + def parse_assert_stmt(self) -> AssertStmt: + self.expect('assert') + expr = self.parse_expression() + node = AssertStmt(expr) + return node + + def parse_yield_or_yield_from_expr(self) -> Union[YieldFromExpr, YieldExpr]: + self.expect("yield") + expr = None + node = YieldExpr(expr) # type: Union[YieldFromExpr, YieldExpr] + if not isinstance(self.current(), Break): + if self.current_str() == "from": + self.expect("from") + expr = self.parse_expression() # when yield from is assigned to a variable + node = YieldFromExpr(expr) + else: + if self.current_str() == ')': + node = YieldExpr(None) + else: + expr = self.parse_expression() + node = YieldExpr(expr) + return node + + def parse_ellipsis(self) -> EllipsisExpr: + self.expect('...') + node = EllipsisExpr() + return node + + def parse_del_stmt(self) -> DelStmt: + self.expect('del') + expr = self.parse_expression() + node = DelStmt(expr) + return node + + def parse_break_stmt(self) -> BreakStmt: + self.expect('break') + node = BreakStmt() + return node + + def parse_continue_stmt(self) -> ContinueStmt: + self.expect('continue') + node = ContinueStmt() + return node + + def parse_pass_stmt(self) -> PassStmt: + self.expect('pass') + node = PassStmt() + return node + + def parse_global_decl(self) -> GlobalDecl: + self.expect('global') + names = self.parse_identifier_list() + node = GlobalDecl(names) + return node + + def parse_nonlocal_decl(self) -> NonlocalDecl: + self.expect('nonlocal') + names = self.parse_identifier_list() + node = NonlocalDecl(names) + return node + + def parse_identifier_list(self) -> List[str]: + names = [] + while True: + n = self.expect_type(Name) + names.append(n.string) + if self.current_str() != ',': + break + self.skip() + return names + + def parse_while_stmt(self) -> WhileStmt: + is_error = False + self.expect('while') + try: + expr = self.parse_expression() + except ParseError: + is_error = True + body, _ = self.parse_block() + if self.current_str() == 'else': + self.expect('else') + else_body, _ = self.parse_block() + else: + else_body = None + if is_error is not None: + node = WhileStmt(expr, body, else_body) + return node + else: + return None + + def parse_for_stmt(self) -> ForStmt: + self.expect('for') + index = self.parse_for_index_variables() + self.expect('in') + expr = self.parse_expression() + + body, _ = self.parse_block() + + if self.current_str() == 'else': + self.expect('else') + else_body, _ = self.parse_block() + else: + else_body = None + + node = ForStmt(index, expr, body, else_body) + return node + + def parse_for_index_variables(self) -> Node: + # Parse index variables of a 'for' statement. + index_items = [] + force_tuple = False + + while True: + v = self.parse_expression(precedence['in'], + star_expr_allowed=True) # Prevent parsing of for stmt 'in' + index_items.append(v) + if self.current_str() != ',': + break + self.skip() + if self.current_str() == 'in': + force_tuple = True + break + + if len(index_items) == 1 and not force_tuple: + index = index_items[0] + else: + index = TupleExpr(index_items) + index.set_line(index_items[0].get_line()) + + return index + + def parse_if_stmt(self) -> IfStmt: + is_error = False + + self.expect('if') + expr = [] + try: + expr.append(self.parse_expression()) + except ParseError: + is_error = True + + body = [self.parse_block()[0]] + + while self.current_str() == 'elif': + self.expect('elif') + try: + expr.append(self.parse_expression()) + except ParseError: + is_error = True + body.append(self.parse_block()[0]) + + if self.current_str() == 'else': + self.expect('else') + else_body, _ = self.parse_block() + else: + else_body = None + + if not is_error: + node = IfStmt(expr, body, else_body) + return node + else: + return None + + def parse_try_stmt(self) -> Node: + self.expect('try') + body, _ = self.parse_block() + is_error = False + vars = [] # type: List[NameExpr] + types = [] # type: List[Node] + handlers = [] # type: List[Block] + while self.current_str() == 'except': + self.expect('except') + if not isinstance(self.current(), Colon): + try: + t = self.current() + types.append(self.parse_expression(precedence[',']).set_line(t)) + if self.current_str() == 'as': + self.expect('as') + vars.append(self.parse_name_expr()) + elif self.pyversion[0] == 2 and self.current_str() == ',': + self.expect(',') + vars.append(self.parse_name_expr()) + else: + vars.append(None) + except ParseError: + is_error = True + else: + types.append(None) + vars.append(None) + handlers.append(self.parse_block()[0]) + if not is_error: + if self.current_str() == 'else': + self.skip() + else_body, _ = self.parse_block() + else: + else_body = None + if self.current_str() == 'finally': + self.expect('finally') + finally_body, _ = self.parse_block() + else: + finally_body = None + node = TryStmt(body, vars, types, handlers, else_body, + finally_body) + return node + else: + return None + + def parse_with_stmt(self) -> WithStmt: + self.expect('with') + exprs = [] + targets = [] + while True: + expr = self.parse_expression(precedence[',']) + if self.current_str() == 'as': + self.expect('as') + target = self.parse_expression(precedence[',']) + else: + target = None + exprs.append(expr) + targets.append(target) + if self.current_str() != ',': + break + self.expect(',') + body, _ = self.parse_block() + return WithStmt(exprs, targets, body) + + def parse_print_stmt(self) -> PrintStmt: + self.expect('print') + args = [] + target = None + if self.current_str() == '>>': + self.skip() + target = self.parse_expression(precedence[',']) + if self.current_str() == ',': + self.skip() + if isinstance(self.current(), Break): + self.parse_error() + else: + if not isinstance(self.current(), Break): + self.parse_error() + comma = False + while not isinstance(self.current(), Break): + args.append(self.parse_expression(precedence[','])) + if self.current_str() == ',': + comma = True + self.skip() + else: + comma = False + break + return PrintStmt(args, newline=not comma, target=target) + + def parse_exec_stmt(self) -> ExecStmt: + self.expect('exec') + expr = self.parse_expression(precedence['in']) + variables1 = None + variables2 = None + if self.current_str() == 'in': + self.skip() + variables1 = self.parse_expression(precedence[',']) + if self.current_str() == ',': + self.skip() + variables2 = self.parse_expression(precedence[',']) + return ExecStmt(expr, variables1, variables2) + + # Parsing expressions + + def parse_expression(self, prec: int = 0, star_expr_allowed: bool = False) -> Node: + """Parse a subexpression within a specific precedence context.""" + expr = None # type: Node + current = self.current() # Remember token for setting the line number. + + # Parse a "value" expression or unary operator expression and store + # that in expr. + s = self.current_str() + if s == '(': + # Parenthesised expression or cast. + expr = self.parse_parentheses() + elif s == '[': + expr = self.parse_list_expr() + elif s in ['-', '+', 'not', '~']: + # Unary operation. + expr = self.parse_unary_expr() + elif s == 'lambda': + expr = self.parse_lambda_expr() + elif s == '{': + expr = self.parse_dict_or_set_expr() + elif s == '*' and star_expr_allowed: + expr = self.parse_star_expr() + elif s == '`' and self.pyversion[0] == 2: + expr = self.parse_backquote_expr() + else: + if isinstance(current, Name): + # Name expression. + expr = self.parse_name_expr() + elif isinstance(current, IntLit): + expr = self.parse_int_expr() + elif isinstance(current, StrLit): + expr = self.parse_str_expr() + elif isinstance(current, BytesLit): + expr = self.parse_bytes_literal() + elif isinstance(current, UnicodeLit): + expr = self.parse_unicode_literal() + elif isinstance(current, FloatLit): + expr = self.parse_float_expr() + elif isinstance(current, ComplexLit): + expr = self.parse_complex_expr() + elif isinstance(current, Keyword) and s == "yield": + # The expression yield from and yield to assign + expr = self.parse_yield_or_yield_from_expr() + elif isinstance(current, EllipsisToken) and (self.pyversion[0] >= 3 + or self.is_stub_file): + expr = self.parse_ellipsis() + else: + # Invalid expression. + self.parse_error() + + # Set the line of the expression node, if not specified. This + # simplifies recording the line number as not every node type needs to + # deal with it separately. + if expr.line < 0: + expr.set_line(current) + + # Parse operations that require a left argument (stored in expr). + while True: + current = self.current() + s = self.current_str() + if s == '(': + # Call expression. + expr = self.parse_call_expr(expr) + elif s == '.': + # Member access expression. + expr = self.parse_member_expr(expr) + elif s == '[': + # Indexing expression. + expr = self.parse_index_expr(expr) + elif s == ',': + # The comma operator is used to build tuples. Comma also + # separates array items and function arguments, but in this + # case the precedence is too low to build a tuple. + if precedence[','] > prec: + expr = self.parse_tuple_expr(expr) + else: + break + elif s == 'for': + if precedence[''] > prec: + # List comprehension or generator expression. Parse as + # generator expression; it will be converted to list + # comprehension if needed elsewhere. + expr = self.parse_generator_expr(expr) + else: + break + elif s == 'if': + # Conditional expression. + if precedence[''] > prec: + expr = self.parse_conditional_expr(expr) + else: + break + else: + # Binary operation or a special case. + if isinstance(current, Op): + op = self.current_str() + op_prec = precedence[op] + if op == 'not': + # Either "not in" or an error. + op_prec = precedence['in'] + if op_prec > prec: + if op in op_comp: + expr = self.parse_comparison_expr(expr, op_prec) + else: + expr = self.parse_bin_op_expr(expr, op_prec) + else: + # The operation cannot be associated with the + # current left operand due to the precedence + # context; let the caller handle it. + break + else: + # Not an operation that accepts a left argument; let the + # caller handle the rest. + break + + # Set the line of the expression node, if not specified. This + # simplifies recording the line number as not every node type + # needs to deal with it separately. + if expr.line < 0: + expr.set_line(current) + + return expr + + def parse_parentheses(self) -> Node: + self.skip() + if self.current_str() == ')': + # Empty tuple (). + expr = self.parse_empty_tuple_expr() # type: Node + else: + # Parenthesised expression. + expr = self.parse_expression(0, star_expr_allowed=True) + self.expect(')') + return expr + + def parse_star_expr(self) -> Node: + star = self.expect('*') + expr = self.parse_expression(precedence['*u']) + expr = StarExpr(expr) + if expr.line < 0: + expr.set_line(star) + return expr + + def parse_empty_tuple_expr(self) -> TupleExpr: + self.expect(')') + node = TupleExpr([]) + return node + + def parse_list_expr(self) -> Node: + """Parse list literal or list comprehension.""" + items = [] + self.expect('[') + while self.current_str() != ']' and not self.eol(): + items.append(self.parse_expression(precedence[''], star_expr_allowed=True)) + if self.current_str() != ',': + break + self.expect(',') + if self.current_str() == 'for' and len(items) == 1: + items[0] = self.parse_generator_expr(items[0]) + self.expect(']') + if len(items) == 1 and isinstance(items[0], GeneratorExpr): + return ListComprehension(items[0]) + else: + expr = ListExpr(items) + return expr + + def parse_generator_expr(self, left_expr: Node) -> GeneratorExpr: + tok = self.current() + indices, sequences, condlists = self.parse_comp_for() + + gen = GeneratorExpr(left_expr, indices, sequences, condlists) + gen.set_line(tok) + return gen + + def parse_comp_for(self) -> Tuple[List[Node], List[Node], List[List[Node]]]: + indices = [] + sequences = [] + condlists = [] # type: List[List[Node]] + while self.current_str() == 'for': + conds = [] + self.expect('for') + index = self.parse_for_index_variables() + indices.append(index) + self.expect('in') + if self.pyversion[0] >= 3: + sequence = self.parse_expression(precedence['']) + else: + sequence = self.parse_expression_list() + sequences.append(sequence) + while self.current_str() == 'if': + self.skip() + conds.append(self.parse_expression(precedence[''])) + condlists.append(conds) + + return indices, sequences, condlists + + def parse_expression_list(self) -> Node: + prec = precedence[''] + expr = self.parse_expression(prec) + if self.current_str() != ',': + return expr + else: + t = self.current() + return self.parse_tuple_expr(expr, prec).set_line(t) + + def parse_conditional_expr(self, left_expr: Node) -> ConditionalExpr: + self.expect('if') + cond = self.parse_expression(precedence['']) + self.expect('else') + else_expr = self.parse_expression(precedence['']) + return ConditionalExpr(cond, left_expr, else_expr) + + def parse_dict_or_set_expr(self) -> Node: + items = [] # type: List[Tuple[Node, Node]] + self.expect('{') + while self.current_str() != '}' and not self.eol(): + key = self.parse_expression(precedence['']) + if self.current_str() in [',', '}'] and items == []: + return self.parse_set_expr(key) + elif self.current_str() == 'for' and items == []: + return self.parse_set_comprehension(key) + elif self.current_str() != ':': + self.parse_error() + colon = self.expect(':') + value = self.parse_expression(precedence['']) + if self.current_str() == 'for' and items == []: + return self.parse_dict_comprehension(key, value, colon) + items.append((key, value)) + if self.current_str() != ',': + break + self.expect(',') + self.expect('}') + node = DictExpr(items) + return node + + def parse_set_expr(self, first: Node) -> SetExpr: + items = [first] + while self.current_str() != '}' and not self.eol(): + self.expect(',') + if self.current_str() == '}': + break + items.append(self.parse_expression(precedence[','])) + self.expect('}') + expr = SetExpr(items) + return expr + + def parse_set_comprehension(self, expr: Node): + gen = self.parse_generator_expr(expr) + self.expect('}') + set_comp = SetComprehension(gen) + return set_comp + + def parse_dict_comprehension(self, key: Node, value: Node, + colon: Token) -> DictionaryComprehension: + indices, sequences, condlists = self.parse_comp_for() + dic = DictionaryComprehension(key, value, indices, sequences, condlists) + dic.set_line(colon) + self.expect('}') + return dic + + def parse_tuple_expr(self, expr: Node, + prec: int = precedence[',']) -> TupleExpr: + items = [expr] + while True: + self.expect(',') + if (self.current_str() in [')', ']', '=', ':'] or + isinstance(self.current(), Break)): + break + items.append(self.parse_expression(prec, star_expr_allowed=True)) + if self.current_str() != ',': break + node = TupleExpr(items) + return node + + def parse_backquote_expr(self) -> BackquoteExpr: + self.expect('`') + expr = self.parse_expression() + self.expect('`') + return BackquoteExpr(expr) + + def parse_name_expr(self) -> NameExpr: + tok = self.expect_type(Name) + node = NameExpr(tok.string) + node.set_line(tok) + return node + + octal_int = re.compile('0+[1-9]') + + def parse_int_expr(self) -> IntExpr: + tok = self.expect_type(IntLit) + string = tok.string.rstrip('lL') # Strip L prefix (Python 2 long literals) + if self.octal_int.match(string): + value = int(string, 8) + else: + value = int(string, 0) + node = IntExpr(value) + return node + + def parse_str_expr(self) -> Node: + # XXX \uxxxx literals + token = self.expect_type(StrLit) + value = cast(StrLit, token).parsed() + is_unicode = False + while isinstance(self.current(), (StrLit, UnicodeLit)): + token = self.skip() + if isinstance(token, StrLit): + value += token.parsed() + elif isinstance(token, UnicodeLit): + value += token.parsed() + is_unicode = True + if is_unicode or (self.pyversion[0] == 2 and 'unicode_literals' in self.future_options): + node = UnicodeExpr(value) # type: Node + else: + node = StrExpr(value) + return node + + def parse_bytes_literal(self) -> Node: + # XXX \uxxxx literals + tok = [self.expect_type(BytesLit)] + value = (cast(BytesLit, tok[0])).parsed() + while isinstance(self.current(), BytesLit): + t = cast(BytesLit, self.skip()) + value += t.parsed() + if self.pyversion[0] >= 3: + node = BytesExpr(value) # type: Node + else: + node = StrExpr(value) + return node + + def parse_unicode_literal(self) -> Node: + # XXX \uxxxx literals + token = self.expect_type(UnicodeLit) + value = cast(UnicodeLit, token).parsed() + while isinstance(self.current(), (UnicodeLit, StrLit)): + token = cast(Union[UnicodeLit, StrLit], self.skip()) + value += token.parsed() + if self.pyversion[0] >= 3: + # Python 3.3 supports u'...' as an alias of '...'. + node = StrExpr(value) # type: Node + else: + node = UnicodeExpr(value) + return node + + def parse_float_expr(self) -> FloatExpr: + tok = self.expect_type(FloatLit) + node = FloatExpr(float(tok.string)) + return node + + def parse_complex_expr(self) -> ComplexExpr: + tok = self.expect_type(ComplexLit) + node = ComplexExpr(complex(tok.string)) + return node + + def parse_call_expr(self, callee: Any) -> CallExpr: + self.expect('(') + args, kinds, names = self.parse_arg_expr() + self.expect(')') + node = CallExpr(callee, args, kinds, names) + return node + + def parse_arg_expr(self) -> Tuple[List[Node], List[int], List[str]]: + """Parse arguments in a call expression (within '(' and ')'). + + Return a tuple with these items: + argument expressions + argument kinds + argument names (for named arguments; None for ordinary args) + """ + args = [] # type: List[Node] + kinds = [] # type: List[int] + names = [] # type: List[str] + var_arg = False + dict_arg = False + named_args = False + while self.current_str() != ')' and not self.eol() and not dict_arg: + if isinstance(self.current(), Name) and self.peek().string == '=': + # Named argument + name = self.expect_type(Name) + self.expect('=') + kinds.append(nodes.ARG_NAMED) + names.append(name.string) + named_args = True + elif (self.current_str() == '*' and not var_arg and not dict_arg): + # *args + var_arg = True + self.expect('*') + kinds.append(nodes.ARG_STAR) + names.append(None) + elif self.current_str() == '**': + # **kwargs + self.expect('**') + dict_arg = True + kinds.append(nodes.ARG_STAR2) + names.append(None) + elif not var_arg and not named_args: + # Ordinary argument + kinds.append(nodes.ARG_POS) + names.append(None) + else: + self.parse_error() + args.append(self.parse_expression(precedence[','])) + if self.current_str() != ',': + break + self.expect(',') + return args, kinds, names + + def parse_member_expr(self, expr: Any) -> Node: + self.expect('.') + name = self.expect_type(Name) + if (isinstance(expr, CallExpr) and isinstance(expr.callee, NameExpr) + and expr.callee.name == 'super'): + # super() expression + node = SuperExpr(name.string) # type: Node + else: + node = MemberExpr(expr, name.string) + return node + + def parse_index_expr(self, base: Any) -> IndexExpr: + self.expect('[') + index = self.parse_slice_item() + if self.current_str() == ',': + # Extended slicing such as x[1:, :2]. + items = [index] + while self.current_str() == ',': + self.skip() + if self.current_str() == ']' or isinstance(self.current(), Break): + break + items.append(self.parse_slice_item()) + index = TupleExpr(items) + index.set_line(items[0].line) + self.expect(']') + node = IndexExpr(base, index) + return node + + def parse_slice_item(self) -> Node: + if self.current_str() != ':': + if self.current_str() == '...': + # Ellipsis is valid here even in Python 2 (but not elsewhere). + ellipsis = EllipsisExpr() + token = self.skip() + ellipsis.set_line(token) + return ellipsis + else: + item = self.parse_expression(precedence[',']) + else: + item = None + if self.current_str() == ':': + # Slice. + index = item + colon = self.expect(':') + if self.current_str() not in (']', ':', ','): + end_index = self.parse_expression(precedence[',']) + else: + end_index = None + stride = None + if self.current_str() == ':': + self.expect(':') + if self.current_str() not in (']', ','): + stride = self.parse_expression(precedence[',']) + item = SliceExpr(index, end_index, stride).set_line(colon.line) + return item + + def parse_bin_op_expr(self, left: Node, prec: int) -> OpExpr: + op = self.expect_type(Op) + op_str = op.string + if op_str == '~': + self.ind -= 1 + self.parse_error() + right = self.parse_expression(prec) + node = OpExpr(op_str, left, right) + return node + + def parse_comparison_expr(self, left: Node, prec: int) -> ComparisonExpr: + operators_str = [] + operands = [left] + + while True: + op = self.expect_type(Op) + op_str = op.string + if op_str == 'not': + if self.current_str() == 'in': + op_str = 'not in' + self.skip() + else: + self.parse_error() + elif op_str == 'is' and self.current_str() == 'not': + op_str = 'is not' + self.skip() + + operators_str.append(op_str) + operand = self.parse_expression(prec) + operands.append(operand) + + # Continue if next token is a comparison operator + self.current() + s = self.current_str() + if s not in op_comp: + break + + node = ComparisonExpr(operators_str, operands) + return node + + def parse_unary_expr(self) -> UnaryExpr: + op_tok = self.skip() + op = op_tok.string + if op == '-' or op == '+': + prec = precedence['-u'] + else: + prec = precedence[op] + expr = self.parse_expression(prec) + node = UnaryExpr(op, expr) + return node + + def parse_lambda_expr(self) -> FuncExpr: + lambda_tok = self.expect('lambda') + + args, extra_stmts = self.parse_arg_list(allow_signature=False) + + # Use 'object' as the placeholder return type; it will be inferred + # later. We can't use 'Any' since it could make type inference results + # less precise. + ret_type = UnboundType('__builtins__.object') + typ = self.build_func_annotation(ret_type, args, + lambda_tok.line, is_default_ret=True) + + colon = self.expect(':') + + expr = self.parse_expression(precedence[',']) + + nodes = [ReturnStmt(expr).set_line(lambda_tok)] + # Potentially insert extra assignment statements to the beginning of the + # body, used to decompose Python 2 tuple arguments. + nodes[:0] = extra_stmts + body = Block(nodes) + body.set_line(colon) + + return FuncExpr(args, body, typ) + + # Helper methods + + def skip(self) -> Token: + self.ind += 1 + return self.tok[self.ind - 1] + + def expect(self, string: str) -> Token: + if self.current_str() == string: + self.ind += 1 + return self.tok[self.ind - 1] + else: + self.parse_error() + + def expect_indent(self) -> Token: + if isinstance(self.current(), Indent): + return self.expect_type(Indent) + else: + self.fail('Expected an indented block', self.current().line) + return none + + def fail(self, msg: str, line: int) -> None: + self.errors.report(line, msg) + + def expect_type(self, typ: type) -> Token: + current = self.current() + if isinstance(current, typ): + self.ind += 1 + return current + else: + self.parse_error() + + def expect_colon_and_break(self) -> Tuple[Token, Token]: + return self.expect_type(Colon), self.expect_type(Break) + + def expect_break(self) -> Token: + return self.expect_type(Break) + + def current(self) -> Token: + return self.tok[self.ind] + + def current_str(self) -> str: + return self.current().string + + def peek(self) -> Token: + return self.tok[self.ind + 1] + + def parse_error(self) -> None: + self.parse_error_at(self.current()) + raise ParseError() + + def parse_error_at(self, tok: Token, skip: bool = True, reason: Optional[str] = None) -> None: + msg = '' + if isinstance(tok, LexError): + msg = token_repr(tok) + msg = msg[0].upper() + msg[1:] + elif isinstance(tok, Indent) or isinstance(tok, Dedent): + msg = 'Inconsistent indentation' + else: + formatted_reason = ": {}".format(reason) if reason else "" + msg = 'Parse error before {}{}'.format(token_repr(tok), formatted_reason) + + self.errors.report(tok.line, msg) + + if skip: + self.skip_until_next_line() + + def skip_until_break(self) -> None: + n = 0 + while (not isinstance(self.current(), Break) + and not isinstance(self.current(), Eof)): + self.skip() + n += 1 + if isinstance(self.tok[self.ind - 1], Colon) and n > 1: + self.ind -= 1 + + def skip_until_next_line(self) -> None: + self.skip_until_break() + if isinstance(self.current(), Break): + self.skip() + + def eol(self) -> bool: + return isinstance(self.current(), Break) or self.eof() + + def eof(self) -> bool: + return isinstance(self.current(), Eof) + + # Type annotation related functionality + + def parse_type(self) -> Type: + try: + typ, self.ind = parse_type(self.tok, self.ind) + except TypeParseError as e: + self.parse_error_at(e.token, reason=e.message) + raise ParseError() + return typ + + annotation_prefix_re = re.compile(r'#\s*type:') + ignore_prefix_re = re.compile(r'ignore\b') + + def parse_type_comment(self, token: Token, signature: bool) -> Type: + """Parse a '# type: ...' annotation. + + Return None if no annotation found. If signature is True, expect + a type signature of form (...) -> t. + """ + whitespace_or_comments = token.rep().strip() + if self.annotation_prefix_re.match(whitespace_or_comments): + type_as_str = whitespace_or_comments.split(':', 1)[1].strip() + if self.ignore_prefix_re.match(type_as_str): + # Actually a "# type: ignore" annotation -> not a type. + return None + tokens = lex.lex(type_as_str, token.line)[0] + if len(tokens) < 2: + # Empty annotation (only Eof token) + self.errors.report(token.line, 'Empty type annotation') + return None + try: + if not signature: + type, index = parse_types(tokens, 0) + else: + type, index = parse_signature(tokens) + except TypeParseError as e: + self.parse_error_at(e.token, skip=False, reason=e.message) + return None + if index < len(tokens) - 2: + self.parse_error_at(tokens[index], skip=False) + return None + return type + else: + return None + + +class ParseError(Exception): pass + + +def token_repr(tok: Token) -> str: + """Return a representation of a token for use in parse error messages.""" + if isinstance(tok, Break): + return 'end of line' + elif isinstance(tok, Eof): + return 'end of file' + elif isinstance(tok, Keyword) or isinstance(tok, Name): + return '"{}"'.format(tok.string) + elif isinstance(tok, IntLit) or isinstance(tok, FloatLit) or isinstance(tok, ComplexLit): + return 'numeric literal' + elif isinstance(tok, StrLit) or isinstance(tok, UnicodeLit): + return 'string literal' + elif (isinstance(tok, Punct) or isinstance(tok, Op) + or isinstance(tok, Colon)): + return tok.string + elif isinstance(tok, Bom): + return 'byte order mark' + elif isinstance(tok, Indent): + return 'indent' + elif isinstance(tok, Dedent): + return 'dedent' + elif isinstance(tok, EllipsisToken): + return '...' + else: + if isinstance(tok, LexError): + t = tok.type + if t == lex.NUMERIC_LITERAL_ERROR: + return 'invalid numeric literal' + elif t == lex.UNTERMINATED_STRING_LITERAL: + return 'unterminated string literal' + elif t == lex.INVALID_CHARACTER: + msg = 'unrecognized character' + if ord(tok.string) in range(33, 127): + msg += ' ' + tok.string + return msg + elif t == lex.INVALID_DEDENT: + return 'inconsistent indentation' + elif t == lex.DECODE_ERROR: + return tok.message + raise ValueError('Unknown token {}'.format(repr(tok))) + + +if __name__ == '__main__': + # Parse a file and dump the AST (or display errors). + import sys + + def usage(): + print('Usage: parse.py [--py2] [--quiet] FILE [...]', file=sys.stderr) + sys.exit(2) + + args = sys.argv[1:] + pyversion = defaults.PYTHON3_VERSION + quiet = False + while args and args[0].startswith('--'): + if args[0] == '--py2': + pyversion = defaults.PYTHON2_VERSION + elif args[0] == '--quiet': + quiet = True + else: + usage() + args = args[1:] + if len(args) < 1: + usage() + status = 0 + for fnam in args: + s = open(fnam, 'rb').read() + errors = Errors() + try: + options = Options() + options.python_version = pyversion + tree = parse(s, fnam, None, options=options) + if not quiet: + print(tree) + except CompileError as e: + for msg in e.messages: + sys.stderr.write('%s\n' % msg) + status = 1 + sys.exit(status) diff --git a/mypy/semanal.py b/mypy/semanal.py index 0f4331499d7cc..8ab29f560a4e7 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1,2839 +1,2839 @@ -"""The semantic analyzer. - -Bind names to definitions and do various other simple consistency -checks. For example, consider this program: - - x = 1 - y = x - -Here semantic analysis would detect that the assignment 'x = 1' -defines a new variable, the type of which is to be inferred (in a -later pass; type inference or type checking is not part of semantic -analysis). Also, it would bind both references to 'x' to the same -module-level variable node. The second assignment would also be -analyzed, and the type of 'y' marked as being inferred. - -Semantic analysis is the first analysis pass after parsing, and it is -subdivided into three passes: - - * FirstPass looks up externally visible names defined in a module but - ignores imports and local definitions. It helps enable (some) - cyclic references between modules, such as module 'a' that imports - module 'b' and used names defined in b *and* vice versa. The first - pass can be performed before dependent modules have been processed. - - * SemanticAnalyzer is the second pass. It does the bulk of the work. - It assumes that dependent modules have been semantically analyzed, - up to the second pass, unless there is a import cycle. - - * ThirdPass checks that type argument counts are valid; for example, - it will reject Dict[int]. We don't do this in the second pass, - since we infer the type argument counts of classes during this - pass, and it is possible to refer to classes defined later in a - file, which would not have the type argument count set yet. This - pass also recomputes the method resolution order of each class, in - case one of its bases belongs to a module involved in an import - loop. - -Semantic analysis of types is implemented in module mypy.typeanal. - -TODO: Check if the third pass slows down type checking significantly. - We could probably get rid of it -- for example, we could collect all - analyzed types in a collection and check them without having to - traverse the entire AST. -""" - -from typing import ( - List, Dict, Set, Tuple, cast, Any, overload, TypeVar, Union, Optional, Callable -) - -from mypy.nodes import ( - MypyFile, TypeInfo, Node, AssignmentStmt, FuncDef, OverloadedFuncDef, - ClassDef, Var, GDEF, MODULE_REF, FuncItem, Import, - ImportFrom, ImportAll, Block, LDEF, NameExpr, MemberExpr, - IndexExpr, TupleExpr, ListExpr, ExpressionStmt, ReturnStmt, - RaiseStmt, AssertStmt, OperatorAssignmentStmt, WhileStmt, - ForStmt, BreakStmt, ContinueStmt, IfStmt, TryStmt, WithStmt, DelStmt, - GlobalDecl, SuperExpr, DictExpr, CallExpr, RefExpr, OpExpr, UnaryExpr, - SliceExpr, CastExpr, RevealTypeExpr, TypeApplication, Context, SymbolTable, - SymbolTableNode, BOUND_TVAR, UNBOUND_TVAR, ListComprehension, GeneratorExpr, - FuncExpr, MDEF, FuncBase, Decorator, SetExpr, TypeVarExpr, - StrExpr, BytesExpr, PrintStmt, ConditionalExpr, PromoteExpr, - ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, MroError, type_aliases, - YieldFromExpr, NamedTupleExpr, NonlocalDecl, - SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr, - YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, COVARIANT, CONTRAVARIANT, - IntExpr, FloatExpr, UnicodeExpr, - INVARIANT, UNBOUND_IMPORTED, Expression, EllipsisExpr, NamedTupleTypeInfo -) -from mypy.visitor import NodeVisitor -from mypy.traverser import TraverserVisitor -from mypy.errors import Errors, report_internal_error -from mypy.types import ( - NoneTyp, CallableType, Overloaded, Instance, Type, TypeVarType, AnyType, - FunctionLike, UnboundType, TypeList, ErrorType, TypeVarDef, - replace_leading_arg_type, TupleType, UnionType, StarType, EllipsisType, - NamedTupleType -) -from mypy.nodes import function_type, implicit_module_attrs -from mypy.typeanal import TypeAnalyser, TypeAnalyserPass3, analyze_type_alias -from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError -from mypy.lex import lex -from mypy.parsetype import parse_type -from mypy.sametypes import is_same_type -from mypy.erasetype import erase_typevars -from mypy import defaults -from mypy.options import Options - - -T = TypeVar('T') - - -# Inferred value of an expression. -ALWAYS_TRUE = 0 -ALWAYS_FALSE = 1 -TRUTH_VALUE_UNKNOWN = 2 - -# Map from obsolete name to the current spelling. -obsolete_name_mapping = { - 'typing.Function': 'typing.Callable', - 'typing.typevar': 'typing.TypeVar', -} - -# Hard coded type promotions (shared between all Python versions). -# These add extra ad-hoc edges to the subtyping relation. For example, -# int is considered a subtype of float, even though there is no -# subclass relationship. -TYPE_PROMOTIONS = { - 'builtins.int': 'builtins.float', - 'builtins.float': 'builtins.complex', -} - -# Hard coded type promotions for Python 3. -# -# Note that the bytearray -> bytes promotion is a little unsafe -# as some functions only accept bytes objects. Here convenience -# trumps safety. -TYPE_PROMOTIONS_PYTHON3 = TYPE_PROMOTIONS.copy() -TYPE_PROMOTIONS_PYTHON3.update({ - 'builtins.bytearray': 'builtins.bytes', -}) - -# Hard coded type promotions for Python 2. -# -# These promotions are unsafe, but we are doing them anyway -# for convenience and also for Python 3 compatibility -# (bytearray -> str). -TYPE_PROMOTIONS_PYTHON2 = TYPE_PROMOTIONS.copy() -TYPE_PROMOTIONS_PYTHON2.update({ - 'builtins.str': 'builtins.unicode', - 'builtins.bytearray': 'builtins.str', -}) - -# When analyzing a function, should we analyze the whole function in one go, or -# should we only perform one phase of the analysis? The latter is used for -# nested functions. In the first phase we add the function to the symbol table -# but don't process body. In the second phase we process function body. This -# way we can have mutually recursive nested functions. -FUNCTION_BOTH_PHASES = 0 # Everthing in one go -FUNCTION_FIRST_PHASE_POSTPONE_SECOND = 1 # Add to symbol table but postpone body -FUNCTION_SECOND_PHASE = 2 # Only analyze body - - -class SemanticAnalyzer(NodeVisitor): - """Semantically analyze parsed mypy files. - - The analyzer binds names and does various consistency checks for a - parse tree. Note that type checking is performed as a separate - pass. - - This is the second phase of semantic analysis. - """ - - # Library search paths - lib_path = None # type: List[str] - # Module name space - modules = None # type: Dict[str, MypyFile] - # Global name space for current module - globals = None # type: SymbolTable - # Names declared using "global" (separate set for each scope) - global_decls = None # type: List[Set[str]] - # Names declated using "nonlocal" (separate set for each scope) - nonlocal_decls = None # type: List[Set[str]] - # Local names of function scopes; None for non-function scopes. - locals = None # type: List[SymbolTable] - # Nested block depths of scopes - block_depth = None # type: List[int] - # TypeInfo of directly enclosing class (or None) - type = None # type: TypeInfo - # Stack of outer classes (the second tuple item contains tvars). - type_stack = None # type: List[TypeInfo] - # Type variables that are bound by the directly enclosing class - bound_tvars = None # type: List[SymbolTableNode] - # Stack of type variables that were bound by outer classess - tvar_stack = None # type: List[List[SymbolTableNode]] - # Do weak type checking in this file - weak_opts = set() # type: Set[str] - - # Stack of functions being analyzed - function_stack = None # type: List[FuncItem] - # Stack of next available function type variable ids - next_function_tvar_id_stack = None # type: List[int] - - # Status of postponing analysis of nested function bodies. By using this we - # can have mutually recursive nested functions. Values are FUNCTION_x - # constants. Note that separate phasea are not used for methods. - postpone_nested_functions_stack = None # type: List[int] - # Postponed functions collected if - # postpone_nested_functions_stack[-1] == FUNCTION_FIRST_PHASE_POSTPONE_SECOND. - postponed_functions_stack = None # type: List[List[Node]] - - loop_depth = 0 # Depth of breakable loops - cur_mod_id = '' # Current module id (or None) (phase 2) - is_stub_file = False # Are we analyzing a stub file? - imports = None # type: Set[str] # Imported modules (during phase 2 analysis) - errors = None # type: Errors # Keeps track of generated errors - - def __init__(self, - lib_path: List[str], - errors: Errors, - options: Options) -> None: - """Construct semantic analyzer. - - Use lib_path to search for modules, and report analysis errors - using the Errors instance. - """ - self.locals = [None] - self.imports = set() - self.type = None - self.type_stack = [] - self.bound_tvars = None - self.tvar_stack = [] - self.function_stack = [] - self.next_function_tvar_id_stack = [-1] - self.block_depth = [0] - self.loop_depth = 0 - self.lib_path = lib_path - self.errors = errors - self.modules = {} - self.options = options - self.postpone_nested_functions_stack = [FUNCTION_BOTH_PHASES] - self.postponed_functions_stack = [] - self.all_exports = set() # type: Set[str] - - def visit_file(self, file_node: MypyFile, fnam: str) -> None: - self.errors.set_file(fnam) - self.cur_mod_node = file_node - self.cur_mod_id = file_node.fullname() - self.is_stub_file = fnam.lower().endswith('.pyi') - self.globals = file_node.names - self.weak_opts = file_node.weak_opts - - if 'builtins' in self.modules: - self.globals['__builtins__'] = SymbolTableNode( - MODULE_REF, self.modules['builtins'], self.cur_mod_id) - - for name in implicit_module_attrs: - v = self.globals[name].node - if isinstance(v, Var): - v.type = self.anal_type(v.type) - v.is_ready = True - - defs = file_node.defs - for d in defs: - self.accept(d) - - if self.cur_mod_id == 'builtins': - remove_imported_names_from_symtable(self.globals, 'builtins') - - if '__all__' in self.globals: - for name, g in self.globals.items(): - if name not in self.all_exports: - g.module_public = False - - def visit_func_def(self, defn: FuncDef) -> None: - phase_info = self.postpone_nested_functions_stack[-1] - if phase_info != FUNCTION_SECOND_PHASE: - self.function_stack.append(defn) - # First phase of analysis for function. - self.errors.push_function(defn.name()) - self.update_function_type_variables(defn) - self.errors.pop_function() - self.function_stack.pop() - - defn.is_conditional = self.block_depth[-1] > 0 - - # TODO(jukka): Figure out how to share the various cases. It doesn't - # make sense to have (almost) duplicate code (here and elsewhere) for - # 3 cases: module-level, class-level and local names. Maybe implement - # a common stack of namespaces. As the 3 kinds of namespaces have - # different semantics, this wouldn't always work, but it might still - # be a win. - if self.is_class_scope(): - # Method definition - defn.info = self.type - if not defn.is_decorated and not defn.is_overload: - if defn.name() in self.type.names: - # Redefinition. Conditional redefinition is okay. - n = self.type.names[defn.name()].node - if self.is_conditional_func(n, defn): - defn.original_def = cast(FuncDef, n) - else: - self.name_already_defined(defn.name(), defn) - self.type.names[defn.name()] = SymbolTableNode(MDEF, defn) - self.prepare_method_signature(defn) - elif self.is_func_scope(): - # Nested function - if not defn.is_decorated and not defn.is_overload: - if defn.name() in self.locals[-1]: - # Redefinition. Conditional redefinition is okay. - n = self.locals[-1][defn.name()].node - if self.is_conditional_func(n, defn): - defn.original_def = cast(FuncDef, n) - else: - self.name_already_defined(defn.name(), defn) - else: - self.add_local(defn, defn) - else: - # Top-level function - if not defn.is_decorated and not defn.is_overload: - symbol = self.globals.get(defn.name()) - if isinstance(symbol.node, FuncDef) and symbol.node != defn: - # This is redefinition. Conditional redefinition is okay. - original_def = symbol.node - if self.is_conditional_func(original_def, defn): - # Conditional function definition -- multiple defs are ok. - defn.original_def = original_def - else: - # Report error. - self.check_no_global(defn.name(), defn, True) - if phase_info == FUNCTION_FIRST_PHASE_POSTPONE_SECOND: - # Postpone this function (for the second phase). - self.postponed_functions_stack[-1].append(defn) - return - if phase_info != FUNCTION_FIRST_PHASE_POSTPONE_SECOND: - # Second phase of analysis for function. - self.errors.push_function(defn.name()) - self.analyze_function(defn) - self.errors.pop_function() - - def prepare_method_signature(self, func: FuncDef) -> None: - """Check basic signature validity and tweak annotation of self/cls argument.""" - # Only non-static methods are special. - if not func.is_static: - if not func.arguments: - self.fail('Method must have at least one argument', func) - elif func.type: - sig = cast(FunctionLike, func.type) - if func.is_class: - leading_type = self.class_type(self.type) - else: - leading_type = self_type(self.type) - func.type = replace_implicit_first_type(sig, leading_type) - - def is_conditional_func(self, previous: Node, new: FuncDef) -> bool: - """Does 'new' conditionally redefine 'previous'? - - We reject straight redefinitions of functions, as they are usually - a programming error. For example: - - . def f(): ... - . def f(): ... # Error: 'f' redefined - """ - return isinstance(previous, (FuncDef, Var)) and new.is_conditional - - def update_function_type_variables(self, defn: FuncDef) -> None: - """Make any type variables in the signature of defn explicit. - - Update the signature of defn to contain type variable definitions - if defn is generic. - """ - if defn.type: - functype = cast(CallableType, defn.type) - typevars = self.infer_type_variables(functype) - # Do not define a new type variable if already defined in scope. - typevars = [(name, tvar) for name, tvar in typevars - if not self.is_defined_type_var(name, defn)] - if typevars: - next_tvar_id = self.next_function_tvar_id() - defs = [TypeVarDef(tvar[0], next_tvar_id - i, - tvar[1].values, tvar[1].upper_bound, - tvar[1].variance) - for i, tvar in enumerate(typevars)] - functype.variables = defs - - def infer_type_variables(self, - type: CallableType) -> List[Tuple[str, TypeVarExpr]]: - """Return list of unique type variables referred to in a callable.""" - names = [] # type: List[str] - tvars = [] # type: List[TypeVarExpr] - for arg in type.arg_types + [type.ret_type]: - for name, tvar_expr in self.find_type_variables_in_type(arg): - if name not in names: - names.append(name) - tvars.append(tvar_expr) - return list(zip(names, tvars)) - - def find_type_variables_in_type( - self, type: Type) -> List[Tuple[str, TypeVarExpr]]: - """Return a list of all unique type variable references in type. - - This effectively does partial name binding, results of which are mostly thrown away. - """ - result = [] # type: List[Tuple[str, TypeVarExpr]] - if isinstance(type, UnboundType): - name = type.name - node = self.lookup_qualified(name, type) - if node and node.kind == UNBOUND_TVAR: - result.append((name, cast(TypeVarExpr, node.node))) - for arg in type.args: - result.extend(self.find_type_variables_in_type(arg)) - elif isinstance(type, TypeList): - for item in type.items: - result.extend(self.find_type_variables_in_type(item)) - elif isinstance(type, UnionType): - for item in type.items: - result.extend(self.find_type_variables_in_type(item)) - elif isinstance(type, AnyType): - pass - elif isinstance(type, EllipsisType) or isinstance(type, TupleType): - pass - else: - assert False, 'Unsupported type %s' % type - return result - - def is_defined_type_var(self, tvar: str, context: Node) -> bool: - return self.lookup_qualified(tvar, context).kind == BOUND_TVAR - - def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: - t = [] # type: List[CallableType] - for i, item in enumerate(defn.items): - # TODO support decorated overloaded functions properly - item.is_overload = True - item.func.is_overload = True - item.accept(self) - t.append(cast(CallableType, function_type(item.func, - self.builtin_type('builtins.function')))) - if item.func.is_property and i == 0: - # This defines a property, probably with a setter and/or deleter. - self.analyze_property_with_multi_part_definition(defn) - break - if not [dec for dec in item.decorators - if refers_to_fullname(dec, 'typing.overload')]: - self.fail("'overload' decorator expected", item) - - defn.type = Overloaded(t) - defn.type.line = defn.line - - if self.is_class_scope(): - self.type.names[defn.name()] = SymbolTableNode(MDEF, defn, - typ=defn.type) - defn.info = self.type - elif self.is_func_scope(): - self.add_local(defn, defn) - - def analyze_property_with_multi_part_definition(self, defn: OverloadedFuncDef) -> None: - """Analyze a property defined using multiple methods (e.g., using @x.setter). - - Assume that the first method (@property) has already been analyzed. - """ - defn.is_property = True - items = defn.items - for item in items[1:]: - if len(item.decorators) == 1: - node = item.decorators[0] - if isinstance(node, MemberExpr): - if node.name == 'setter': - # The first item represents the entire property. - defn.items[0].var.is_settable_property = True - # Get abstractness from the original definition. - item.func.is_abstract = items[0].func.is_abstract - else: - self.fail("Decorated property not supported", item) - item.func.accept(self) - - def next_function_tvar_id(self) -> int: - return self.next_function_tvar_id_stack[-1] - - def analyze_function(self, defn: FuncItem) -> None: - is_method = self.is_class_scope() - - tvarnodes = self.add_func_type_variables_to_symbol_table(defn) - next_function_tvar_id = min([self.next_function_tvar_id()] + - [n.tvar_def.id.raw_id - 1 for n in tvarnodes]) - self.next_function_tvar_id_stack.append(next_function_tvar_id) - - if defn.type: - # Signature must be analyzed in the surrounding scope so that - # class-level imported names and type variables are in scope. - defn.type = self.anal_type(defn.type) - self.check_function_signature(defn) - if isinstance(defn, FuncDef): - defn.type = set_callable_name(defn.type, defn) - for arg in defn.arguments: - if arg.initializer: - arg.initializer.accept(self) - self.function_stack.append(defn) - self.enter() - for arg in defn.arguments: - self.add_local(arg.variable, defn) - for arg in defn.arguments: - if arg.initialization_statement: - lvalue = arg.initialization_statement.lvalues[0] - lvalue.accept(self) - - # The first argument of a non-static, non-class method is like 'self' - # (though the name could be different), having the enclosing class's - # instance type. - if is_method and not defn.is_static and not defn.is_class and defn.arguments: - defn.arguments[0].variable.is_self = True - - # First analyze body of the function but ignore nested functions. - self.postpone_nested_functions_stack.append(FUNCTION_FIRST_PHASE_POSTPONE_SECOND) - self.postponed_functions_stack.append([]) - defn.body.accept(self) - - # Analyze nested functions (if any) as a second phase. - self.postpone_nested_functions_stack[-1] = FUNCTION_SECOND_PHASE - for postponed in self.postponed_functions_stack[-1]: - postponed.accept(self) - self.postpone_nested_functions_stack.pop() - self.postponed_functions_stack.pop() - - self.next_function_tvar_id_stack.pop() - disable_typevars(tvarnodes) - - self.leave() - self.function_stack.pop() - - def add_func_type_variables_to_symbol_table( - self, defn: FuncItem) -> List[SymbolTableNode]: - nodes = [] # type: List[SymbolTableNode] - if defn.type: - tt = defn.type - names = self.type_var_names() - items = cast(CallableType, tt).variables - for item in items: - name = item.name - if name in names: - self.name_already_defined(name, defn) - node = self.bind_type_var(name, item, defn) - nodes.append(node) - names.add(name) - return nodes - - def type_var_names(self) -> Set[str]: - if not self.type: - return set() - else: - return set(self.type.type_vars) - - def bind_type_var(self, fullname: str, tvar_def: TypeVarDef, - context: Context) -> SymbolTableNode: - node = self.lookup_qualified(fullname, context) - node.kind = BOUND_TVAR - node.tvar_def = tvar_def - return node - - def check_function_signature(self, fdef: FuncItem) -> None: - sig = cast(CallableType, fdef.type) - if len(sig.arg_types) < len(fdef.arguments): - self.fail('Type signature has too few arguments', fdef) - # Add dummy Any arguments to prevent crashes later. - extra_anys = [AnyType()] * (len(fdef.arguments) - len(sig.arg_types)) - sig.arg_types.extend(extra_anys) - elif len(sig.arg_types) > len(fdef.arguments): - self.fail('Type signature has too many arguments', fdef) - - def visit_class_def(self, defn: ClassDef) -> None: - self.clean_up_bases_and_infer_type_variables(defn) - self.setup_class_def_analysis(defn) - - self.bind_class_type_vars(defn) - - self.analyze_base_classes(defn) - self.analyze_metaclass(defn) - - for decorator in defn.decorators: - self.analyze_class_decorator(defn, decorator) - - self.enter_class(defn) - - self.setup_is_builtinclass(defn) - - # Analyze class body. - defn.defs.accept(self) - - self.calculate_abstract_status(defn.info) - self.setup_type_promotion(defn) - - self.leave_class() - self.unbind_class_type_vars() - - def enter_class(self, defn: ClassDef) -> None: - # Remember previous active class - self.type_stack.append(self.type) - self.locals.append(None) # Add class scope - self.block_depth.append(-1) # The class body increments this to 0 - self.postpone_nested_functions_stack.append(FUNCTION_BOTH_PHASES) - self.type = defn.info - - def leave_class(self) -> None: - """ Restore analyzer state. """ - self.postpone_nested_functions_stack.pop() - self.block_depth.pop() - self.locals.pop() - self.type = self.type_stack.pop() - - def bind_class_type_vars(self, defn: ClassDef) -> None: - """ Unbind type variables of previously active class and bind - the type variables for the active class. - """ - if self.bound_tvars: - disable_typevars(self.bound_tvars) - self.tvar_stack.append(self.bound_tvars) - self.bound_tvars = self.bind_class_type_variables_in_symbol_table(defn.info) - - def unbind_class_type_vars(self) -> None: - """ Unbind the active class' type vars and rebind the - type vars of the previously active class. - """ - disable_typevars(self.bound_tvars) - self.bound_tvars = self.tvar_stack.pop() - if self.bound_tvars: - enable_typevars(self.bound_tvars) - - def analyze_class_decorator(self, defn: ClassDef, decorator: Node) -> None: - decorator.accept(self) - - def setup_is_builtinclass(self, defn: ClassDef): - for decorator in defn.decorators: - if refers_to_fullname(decorator, 'typing.builtinclass'): - defn.is_builtinclass = True - if defn.fullname == 'builtins.object': - # Only 'object' is marked as a built-in class, as otherwise things elsewhere - # would break. We need a better way of dealing with built-in classes. - defn.is_builtinclass = True - - def calculate_abstract_status(self, typ: TypeInfo) -> None: - """Calculate abstract status of a class. - - Set is_abstract of the type to True if the type has an unimplemented - abstract attribute. Also compute a list of abstract attributes. - """ - concrete = set() # type: Set[str] - abstract = [] # type: List[str] - for base in typ.mro: - for name, symnode in base.names.items(): - node = symnode.node - if isinstance(node, OverloadedFuncDef): - # Unwrap an overloaded function definition. We can just - # check arbitrarily the first overload item. If the - # different items have a different abstract status, there - # should be an error reported elsewhere. - func = node.items[0] # type: Node - else: - func = node - if isinstance(func, Decorator): - fdef = func.func - if fdef.is_abstract and name not in concrete: - typ.is_abstract = True - abstract.append(name) - concrete.add(name) - typ.abstract_attributes = sorted(abstract) - - def setup_type_promotion(self, defn: ClassDef) -> None: - """Setup extra, ad-hoc subtyping relationships between classes (promotion). - - This includes things like 'int' being compatible with 'float'. - """ - promote_target = None # type: Type - for decorator in defn.decorators: - if isinstance(decorator, CallExpr): - analyzed = decorator.analyzed - if isinstance(analyzed, PromoteExpr): - # _promote class decorator (undocumented faeture). - promote_target = analyzed.type - if not promote_target: - promotions = (TYPE_PROMOTIONS_PYTHON3 if self.options.python_version[0] >= 3 - else TYPE_PROMOTIONS_PYTHON2) - if defn.fullname in promotions: - promote_target = self.named_type_or_none(promotions[defn.fullname]) - defn.info._promote = promote_target - - def clean_up_bases_and_infer_type_variables(self, defn: ClassDef) -> None: - """Remove extra base classes such as Generic and infer type vars. - - For example, consider this class: - - . class Foo(Bar, Generic[T]): ... - - Now we will remove Generic[T] from bases of Foo and infer that the - type variable 'T' is a type argument of Foo. - - Note that this is performed *before* semantic analysis. - """ - removed = [] # type: List[int] - type_vars = [] # type: List[TypeVarDef] - for i, base_expr in enumerate(defn.base_type_exprs): - try: - base = expr_to_unanalyzed_type(base_expr) - except TypeTranslationError: - # This error will be caught later. - continue - tvars = self.analyze_typevar_declaration(base) - if tvars is not None: - if type_vars: - self.fail('Duplicate Generic in bases', defn) - removed.append(i) - for j, (name, tvar_expr) in enumerate(tvars): - type_vars.append(TypeVarDef(name, j + 1, tvar_expr.values, - tvar_expr.upper_bound, tvar_expr.variance)) - if type_vars: - defn.type_vars = type_vars - if defn.info: - defn.info.type_vars = [tv.name for tv in type_vars] - for i in reversed(removed): - del defn.base_type_exprs[i] - - def analyze_typevar_declaration(self, t: Type) -> Optional[List[Tuple[str, TypeVarExpr]]]: - if not isinstance(t, UnboundType): - return None - unbound = t - sym = self.lookup_qualified(unbound.name, unbound) - if sym is None or sym.node is None: - return None - if sym.node.fullname() == 'typing.Generic': - tvars = [] # type: List[Tuple[str, TypeVarExpr]] - for arg in unbound.args: - tvar = self.analyze_unbound_tvar(arg) - if tvar: - tvars.append(tvar) - else: - self.fail('Free type variable expected in %s[...]' % - sym.node.name(), t) - return tvars - return None - - def analyze_unbound_tvar(self, t: Type) -> Tuple[str, TypeVarExpr]: - if not isinstance(t, UnboundType): - return None - unbound = t - sym = self.lookup_qualified(unbound.name, unbound) - if sym is not None and sym.kind == UNBOUND_TVAR: - return unbound.name, cast(TypeVarExpr, sym.node) - return None - - def setup_class_def_analysis(self, defn: ClassDef) -> None: - """Prepare for the analysis of a class definition.""" - if not defn.info: - defn.info = TypeInfo(SymbolTable(), defn) - defn.info._fullname = defn.info.name() - if self.is_func_scope() or self.type: - kind = MDEF - if self.is_func_scope(): - kind = LDEF - self.add_symbol(defn.name, SymbolTableNode(kind, defn.info), defn) - - def analyze_base_classes(self, defn: ClassDef) -> None: - """Analyze and set up base classes. - - This computes several attributes on the corresponding TypeInfo defn.info - related to the base classes: defn.info.bases, defn.info.mro, and - miscellaneous others (at least tuple_type, fallback_to_any, and is_enum.) - """ - - base_types = [] # type: List[Instance] - info = defn.info - for base_expr in defn.base_type_exprs: - try: - base = self.expr_to_analyzed_type(base_expr) - except TypeTranslationError: - self.fail('Invalid base class', base_expr) - info.fallback_to_any = True - continue - - if isinstance(base, TupleType): - if info.tuple_type: - self.fail("Class has two incompatible bases derived from tuple", defn) - defn.incompatible_baseclass = True - if (not self.is_stub_file - and not info.is_named_tuple - and base.fallback.type.fullname() == 'builtins.tuple'): - self.fail("Tuple[...] not supported as a base class outside a stub file", defn) - info.tuple_type = base - base_types.append(base.fallback) - elif isinstance(base, Instance): - base_types.append(base) - elif isinstance(base, AnyType): - info.fallback_to_any = True - else: - self.fail('Invalid base class', base_expr) - info.fallback_to_any = True - - # Add 'object' as implicit base if there is no other base class. - if (not base_types and defn.fullname != 'builtins.object'): - base_types.append(self.object_type()) - - info.bases = base_types - - # Calculate the MRO. It might be incomplete at this point if - # the bases of defn include classes imported from other - # modules in an import loop. We'll recompute it in ThirdPass. - if not self.verify_base_classes(defn): - info.mro = [] - return - calculate_class_mro(defn, self.fail) - # If there are cyclic imports, we may be missing 'object' in - # the MRO. Fix MRO if needed. - if info.mro and info.mro[-1].fullname() != 'builtins.object': - info.mro.append(self.object_type().type) - - def expr_to_analyzed_type(self, expr: Node) -> Type: - if isinstance(expr, CallExpr): - expr.accept(self) - info = self.check_namedtuple(expr) - if info is None: - # Some form of namedtuple is the only valid type that looks like a call - # expression. This isn't a valid type. - raise TypeTranslationError() - fallback = Instance(info, []) - return TupleType(info.tuple_type.items, fallback=fallback) - typ = expr_to_unanalyzed_type(expr) - return self.anal_type(typ) - - def verify_base_classes(self, defn: ClassDef) -> bool: - info = defn.info - for base in info.bases: - baseinfo = base.type - if self.is_base_class(info, baseinfo): - self.fail('Cycle in inheritance hierarchy', defn) - # Clear bases to forcefully get rid of the cycle. - info.bases = [] - if baseinfo.fullname() == 'builtins.bool': - self.fail("'%s' is not a valid base class" % - baseinfo.name(), defn) - return False - dup = find_duplicate(info.direct_base_classes()) - if dup: - self.fail('Duplicate base class "%s"' % dup.name(), defn) - return False - return True - - def is_base_class(self, t: TypeInfo, s: TypeInfo) -> bool: - """Determine if t is a base class of s (but do not use mro).""" - # Search the base class graph for t, starting from s. - worklist = [s] - visited = {s} - while worklist: - nxt = worklist.pop() - if nxt == t: - return True - for base in nxt.bases: - if base.type not in visited: - worklist.append(base.type) - visited.add(base.type) - return False - - def analyze_metaclass(self, defn: ClassDef) -> None: - if defn.metaclass: - sym = self.lookup_qualified(defn.metaclass, defn) - if sym is not None and not isinstance(sym.node, TypeInfo): - self.fail("Invalid metaclass '%s'" % defn.metaclass, defn) - - def object_type(self) -> Instance: - return self.named_type('__builtins__.object') - - def class_type(self, info: TypeInfo) -> Type: - # Construct a function type whose fallback is cls. - from mypy import checkmember # To avoid import cycle. - leading_type = checkmember.type_object_type(info, self.builtin_type) - if isinstance(leading_type, Overloaded): - # Overloaded __init__ is too complex to handle. Plus it's stubs only. - return AnyType() - else: - return leading_type - - def named_type(self, qualified_name: str, args: List[Type] = None) -> Instance: - sym = self.lookup_qualified(qualified_name, None) - return Instance(cast(TypeInfo, sym.node), args or []) - - def named_type_or_none(self, qualified_name: str) -> Instance: - sym = self.lookup_fully_qualified_or_none(qualified_name) - if not sym: - return None - return Instance(cast(TypeInfo, sym.node), []) - - def is_instance_type(self, t: Type) -> bool: - return isinstance(t, Instance) - - def bind_class_type_variables_in_symbol_table( - self, info: TypeInfo) -> List[SymbolTableNode]: - nodes = [] # type: List[SymbolTableNode] - for var, binder in zip(info.type_vars, info.defn.type_vars): - node = self.bind_type_var(var, binder, info) - nodes.append(node) - return nodes - - def visit_import(self, i: Import) -> None: - for id, as_id in i.ids: - if as_id is not None: - self.add_module_symbol(id, as_id, module_public=True, context=i) - else: - # Modules imported in a stub file without using 'as x' won't get exported when - # doing 'from m import *'. - module_public = not self.is_stub_file - base = id.split('.')[0] - self.add_module_symbol(base, base, module_public=module_public, - context=i) - - def add_module_symbol(self, id: str, as_id: str, module_public: bool, - context: Context) -> None: - if id in self.modules: - m = self.modules[id] - self.add_symbol(as_id, SymbolTableNode(MODULE_REF, m, self.cur_mod_id, - module_public=module_public), context) - else: - self.add_unknown_symbol(as_id, context) - - def visit_import_from(self, imp: ImportFrom) -> None: - import_id = self.correct_relative_import(imp) - if import_id in self.modules: - module = self.modules[import_id] - for id, as_id in imp.names: - node = module.names.get(id) - if node and node.kind != UNBOUND_IMPORTED: - node = self.normalize_type_alias(node, imp) - if not node: - return - imported_id = as_id or id - existing_symbol = self.globals.get(imported_id) - if existing_symbol: - # Import can redefine a variable. They get special treatment. - if self.process_import_over_existing_name( - imported_id, existing_symbol, node, imp): - continue - # 'from m import x as x' exports x in a stub file. - module_public = not self.is_stub_file or as_id is not None - symbol = SymbolTableNode(node.kind, node.node, - self.cur_mod_id, - node.type_override, - module_public=module_public) - self.add_symbol(imported_id, symbol, imp) - else: - message = "Module has no attribute '{}'".format(id) - extra = self.undefined_name_extra_info('{}.{}'.format(import_id, id)) - if extra: - message += " {}".format(extra) - self.fail(message, imp) - else: - # Missing module. - for id, as_id in imp.names: - self.add_unknown_symbol(as_id or id, imp) - - def process_import_over_existing_name(self, - imported_id: str, existing_symbol: SymbolTableNode, - module_symbol: SymbolTableNode, - import_node: ImportBase) -> bool: - if (existing_symbol.kind in (LDEF, GDEF, MDEF) and - isinstance(existing_symbol.node, (Var, FuncDef))): - # This is a valid import over an existing definition in the file. Construct a dummy - # assignment that we'll use to type check the import. - lvalue = NameExpr(imported_id) - lvalue.kind = existing_symbol.kind - lvalue.node = existing_symbol.node - rvalue = NameExpr(imported_id) - rvalue.kind = module_symbol.kind - rvalue.node = module_symbol.node - assignment = AssignmentStmt([lvalue], rvalue) - for node in assignment, lvalue, rvalue: - node.set_line(import_node) - import_node.assignments.append(assignment) - return True - return False - - def normalize_type_alias(self, node: SymbolTableNode, - ctx: Context) -> SymbolTableNode: - if node.fullname in type_aliases: - # Node refers to an aliased type such as typing.List; normalize. - node = self.lookup_qualified(type_aliases[node.fullname], ctx) - if node.fullname == 'typing.DefaultDict': - self.add_module_symbol('collections', '__mypy_collections__', False, ctx) - node = self.lookup_qualified('__mypy_collections__.defaultdict', ctx) - return node - - def correct_relative_import(self, node: Union[ImportFrom, ImportAll]) -> str: - if node.relative == 0: - return node.id - - parts = self.cur_mod_id.split(".") - cur_mod_id = self.cur_mod_id - - rel = node.relative - if self.cur_mod_node.is_package_init_file(): - rel -= 1 - if len(parts) < rel: - self.fail("Relative import climbs too many namespaces", node) - if rel != 0: - cur_mod_id = ".".join(parts[:-rel]) - - return cur_mod_id + (("." + node.id) if node.id else "") - - def visit_import_all(self, i: ImportAll) -> None: - i_id = self.correct_relative_import(i) - if i_id in self.modules: - m = self.modules[i_id] - for name, node in m.names.items(): - node = self.normalize_type_alias(node, i) - if not name.startswith('_') and node.module_public: - existing_symbol = self.globals.get(name) - if existing_symbol: - # Import can redefine a variable. They get special treatment. - if self.process_import_over_existing_name( - name, existing_symbol, node, i): - continue - self.add_symbol(name, SymbolTableNode(node.kind, node.node, - self.cur_mod_id), i) - else: - # Don't add any dummy symbols for 'from x import *' if 'x' is unknown. - pass - - def add_unknown_symbol(self, name: str, context: Context) -> None: - var = Var(name) - var._fullname = self.qualified_name(name) - var.is_ready = True - var.type = AnyType() - self.add_symbol(name, SymbolTableNode(GDEF, var, self.cur_mod_id), context) - - # - # Statements - # - - def visit_block(self, b: Block) -> None: - if b.is_unreachable: - return - self.block_depth[-1] += 1 - for s in b.body: - self.accept(s) - self.block_depth[-1] -= 1 - - def visit_block_maybe(self, b: Block) -> None: - if b: - self.visit_block(b) - - def anal_type(self, t: Type, allow_tuple_literal: bool = False) -> Type: - if t: - if allow_tuple_literal: - # Types such as (t1, t2, ...) only allowed in assignment statements. They'll - # generate errors elsewhere, and Tuple[t1, t2, ...] must be used instead. - if isinstance(t, TupleType): - # Unlike TypeAnalyser, also allow implicit tuple types (without Tuple[...]). - star_count = sum(1 for item in t.items if isinstance(item, StarType)) - if star_count > 1: - self.fail('At most one star type allowed in a tuple', t) - return TupleType([AnyType() for _ in t.items], - self.builtin_type('builtins.tuple'), t.line) - items = [self.anal_type(item, True) - for item in t.items] - return TupleType(items, self.builtin_type('builtins.tuple'), t.line) - a = TypeAnalyser(self.lookup_qualified, - self.lookup_fully_qualified, - self.fail) - return t.accept(a) - else: - return None - - def visit_assignment_stmt(self, s: AssignmentStmt) -> None: - for lval in s.lvalues: - self.analyze_lvalue(lval, explicit_type=s.type is not None) - s.rvalue.accept(self) - if s.type: - allow_tuple_literal = isinstance(s.lvalues[-1], (TupleExpr, ListExpr)) - s.type = self.anal_type(s.type, allow_tuple_literal) - else: - # For simple assignments, allow binding type aliases. - # Also set the type if the rvalue is a simple literal. - if (s.type is None and len(s.lvalues) == 1 and - isinstance(s.lvalues[0], NameExpr)): - if s.lvalues[0].is_def: - s.type = self.analyze_simple_literal_type(s.rvalue) - res = analyze_type_alias(s.rvalue, - self.lookup_qualified, - self.lookup_fully_qualified, - self.fail) - if res and (not isinstance(res, Instance) or cast(Instance, res).args): - # TODO: What if this gets reassigned? - name = s.lvalues[0] - node = self.lookup(name.name, name) - node.kind = TYPE_ALIAS - node.type_override = res - if isinstance(s.rvalue, IndexExpr): - s.rvalue.analyzed = TypeAliasExpr(res) - if s.type: - # Store type into nodes. - for lvalue in s.lvalues: - self.store_declared_types(lvalue, s.type) - self.check_and_set_up_type_alias(s) - self.process_typevar_declaration(s) - self.process_namedtuple_definition(s) - - if (len(s.lvalues) == 1 and isinstance(s.lvalues[0], NameExpr) and - s.lvalues[0].name == '__all__' and s.lvalues[0].kind == GDEF and - isinstance(s.rvalue, (ListExpr, TupleExpr))): - self.add_exports(*s.rvalue.items) - - def analyze_simple_literal_type(self, rvalue: Node) -> Optional[Type]: - """Return builtins.int if rvalue is an int literal, etc.""" - if self.weak_opts or self.options.semantic_analysis_only or self.function_stack: - # Skip this if any weak options are set. - # Also skip if we're only doing the semantic analysis pass. - # This is mostly to avoid breaking unit tests. - # Also skip inside a function; this is to avoid confusing - # the code that handles dead code due to isinstance() - # inside type variables with value restrictions (like - # AnyStr). - return None - if isinstance(rvalue, IntExpr): - return self.named_type_or_none('builtins.int') - if isinstance(rvalue, FloatExpr): - return self.named_type_or_none('builtins.float') - if isinstance(rvalue, StrExpr): - return self.named_type_or_none('builtins.str') - if isinstance(rvalue, BytesExpr): - return self.named_type_or_none('builtins.bytes') - if isinstance(rvalue, UnicodeExpr): - return self.named_type_or_none('builtins.unicode') - return None - - def check_and_set_up_type_alias(self, s: AssignmentStmt) -> None: - """Check if assignment creates a type alias and set it up as needed.""" - # For now, type aliases only work at the top level of a module. - if (len(s.lvalues) == 1 and not self.is_func_scope() and not self.type - and not s.type): - lvalue = s.lvalues[0] - if isinstance(lvalue, NameExpr): - if not lvalue.is_def: - # Only a definition can create a type alias, not regular assignment. - return - rvalue = s.rvalue - if isinstance(rvalue, RefExpr): - node = rvalue.node - if isinstance(node, TypeInfo): - # TODO: We should record the fact that this is a variable - # that refers to a type, rather than making this - # just an alias for the type. - self.globals[lvalue.name].node = node - - def analyze_lvalue(self, lval: Node, nested: bool = False, - add_global: bool = False, - explicit_type: bool = False) -> None: - """Analyze an lvalue or assignment target. - - Only if add_global is True, add name to globals table. If nested - is true, the lvalue is within a tuple or list lvalue expression. - """ - - if isinstance(lval, NameExpr): - # Top-level definitions within some statements (at least while) are - # not handled in the first pass, so they have to be added now. - nested_global = (not self.is_func_scope() and - self.block_depth[-1] > 0 and - not self.type) - if (add_global or nested_global) and lval.name not in self.globals: - # Define new global name. - v = Var(lval.name) - v._fullname = self.qualified_name(lval.name) - v.is_ready = False # Type not inferred yet - lval.node = v - lval.is_def = True - lval.kind = GDEF - lval.fullname = v._fullname - self.globals[lval.name] = SymbolTableNode(GDEF, v, - self.cur_mod_id) - elif isinstance(lval.node, Var) and lval.is_def: - # Since the is_def flag is set, this must have been analyzed - # already in the first pass and added to the symbol table. - assert lval.node.name() in self.globals - elif (self.is_func_scope() and lval.name not in self.locals[-1] and - lval.name not in self.global_decls[-1] and - lval.name not in self.nonlocal_decls[-1]): - # Define new local name. - v = Var(lval.name) - lval.node = v - lval.is_def = True - lval.kind = LDEF - lval.fullname = lval.name - self.add_local(v, lval) - elif not self.is_func_scope() and (self.type and - lval.name not in self.type.names): - # Define a new attribute within class body. - v = Var(lval.name) - v.info = self.type - v.is_initialized_in_class = True - v.set_line(lval) - lval.node = v - lval.is_def = True - lval.kind = MDEF - lval.fullname = lval.name - self.type.names[lval.name] = SymbolTableNode(MDEF, v) - else: - # Bind to an existing name. - if explicit_type: - self.name_already_defined(lval.name, lval) - lval.accept(self) - self.check_lvalue_validity(lval.node, lval) - elif isinstance(lval, MemberExpr): - if not add_global: - self.analyze_member_lvalue(lval) - if explicit_type and not self.is_self_member_ref(lval): - self.fail('Type cannot be declared in assignment to non-self ' - 'attribute', lval) - elif isinstance(lval, IndexExpr): - if explicit_type: - self.fail('Unexpected type declaration', lval) - if not add_global: - lval.accept(self) - elif (isinstance(lval, TupleExpr) or - isinstance(lval, ListExpr)): - items = cast(Any, lval).items - if len(items) == 0 and isinstance(lval, TupleExpr): - self.fail("Can't assign to ()", lval) - self.analyze_tuple_or_list_lvalue(cast(Union[ListExpr, TupleExpr], lval), - add_global, explicit_type) - elif isinstance(lval, StarExpr): - if nested: - self.analyze_lvalue(lval.expr, nested, add_global, explicit_type) - else: - self.fail('Starred assignment target must be in a list or tuple', lval) - else: - self.fail('Invalid assignment target', lval) - - def analyze_tuple_or_list_lvalue(self, lval: Union[ListExpr, TupleExpr], - add_global: bool = False, - explicit_type: bool = False) -> None: - """Analyze an lvalue or assignment target that is a list or tuple.""" - items = lval.items - star_exprs = [cast(StarExpr, item) - for item in items - if isinstance(item, StarExpr)] - - if len(star_exprs) > 1: - self.fail('Two starred expressions in assignment', lval) - else: - if len(star_exprs) == 1: - star_exprs[0].valid = True - for i in items: - self.analyze_lvalue(i, nested=True, add_global=add_global, - explicit_type = explicit_type) - - def analyze_member_lvalue(self, lval: MemberExpr) -> None: - lval.accept(self) - if (self.is_self_member_ref(lval) and - self.type.get(lval.name) is None): - # Implicit attribute definition in __init__. - lval.is_def = True - v = Var(lval.name) - v.info = self.type - v.is_ready = False - lval.def_var = v - lval.node = v - self.type.names[lval.name] = SymbolTableNode(MDEF, v) - self.check_lvalue_validity(lval.node, lval) - - def is_self_member_ref(self, memberexpr: MemberExpr) -> bool: - """Does memberexpr to refer to an attribute of self?""" - if not isinstance(memberexpr.expr, NameExpr): - return False - node = memberexpr.expr.node - return isinstance(node, Var) and node.is_self - - def check_lvalue_validity(self, node: Node, ctx: Context) -> None: - if isinstance(node, (TypeInfo, TypeVarExpr)): - self.fail('Invalid assignment target', ctx) - - def store_declared_types(self, lvalue: Node, typ: Type) -> None: - if isinstance(typ, StarType) and not isinstance(lvalue, StarExpr): - self.fail('Star type only allowed for starred expressions', lvalue) - if isinstance(lvalue, RefExpr): - lvalue.is_def = False - if isinstance(lvalue.node, Var): - var = lvalue.node - var.type = typ - var.is_ready = True - # If node is not a variable, we'll catch it elsewhere. - elif isinstance(lvalue, TupleExpr): - if isinstance(typ, TupleType): - if len(lvalue.items) != len(typ.items): - self.fail('Incompatible number of tuple items', lvalue) - return - for item, itemtype in zip(lvalue.items, typ.items): - self.store_declared_types(item, itemtype) - else: - self.fail('Tuple type expected for multiple variables', - lvalue) - elif isinstance(lvalue, StarExpr): - if isinstance(typ, StarType): - self.store_declared_types(lvalue.expr, typ.type) - else: - self.fail('Star type expected for starred expression', lvalue) - else: - # This has been flagged elsewhere as an error, so just ignore here. - pass - - def process_typevar_declaration(self, s: AssignmentStmt) -> None: - """Check if s declares a TypeVar; it yes, store it in symbol table.""" - call = self.get_typevar_declaration(s) - if not call: - return - - lvalue = cast(NameExpr, s.lvalues[0]) - name = lvalue.name - if not lvalue.is_def: - if s.type: - self.fail("Cannot declare the type of a type variable", s) - else: - self.fail("Cannot redefine '%s' as a type variable" % name, s) - return - - if not self.check_typevar_name(call, name, s): - return - - # Constraining types - n_values = call.arg_kinds[1:].count(ARG_POS) - values = self.analyze_types(call.args[1:1 + n_values]) - - res = self.process_typevar_parameters(call.args[1 + n_values:], - call.arg_names[1 + n_values:], - call.arg_kinds[1 + n_values:], - bool(values), - s) - if res is None: - return - variance, upper_bound = res - - # Yes, it's a valid type variable definition! Add it to the symbol table. - node = self.lookup(name, s) - node.kind = UNBOUND_TVAR - TypeVar = TypeVarExpr(name, node.fullname, values, upper_bound, variance) - TypeVar.line = call.line - call.analyzed = TypeVar - node.node = TypeVar - - def check_typevar_name(self, call: CallExpr, name: str, context: Context) -> bool: - if len(call.args) < 1: - self.fail("Too few arguments for TypeVar()", context) - return False - if not isinstance(call.args[0], (StrExpr, BytesExpr)) or not call.arg_kinds[0] == ARG_POS: - self.fail("TypeVar() expects a string literal as first argument", context) - return False - if cast(StrExpr, call.args[0]).value != name: - self.fail("Unexpected TypeVar() argument value", context) - return False - return True - - def get_typevar_declaration(self, s: AssignmentStmt) -> Optional[CallExpr]: - """Returns the TypeVar() call expression if `s` is a type var declaration - or None otherwise. - """ - if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): - return None - if not isinstance(s.rvalue, CallExpr): - return None - call = s.rvalue - if not isinstance(call.callee, RefExpr): - return None - callee = call.callee - if callee.fullname != 'typing.TypeVar': - return None - return call - - def process_typevar_parameters(self, args: List[Node], - names: List[Optional[str]], - kinds: List[int], - has_values: bool, - context: Context) -> Optional[Tuple[int, Type]]: - covariant = False - contravariant = False - upper_bound = self.object_type() # type: Type - for param_value, param_name, param_kind in zip(args, names, kinds): - if not param_kind == ARG_NAMED: - self.fail("Unexpected argument to TypeVar()", context) - return None - if param_name == 'covariant': - if isinstance(param_value, NameExpr): - if param_value.name == 'True': - covariant = True - else: - self.fail("TypeVar 'covariant' may only be 'True'", context) - return None - else: - self.fail("TypeVar 'covariant' may only be 'True'", context) - return None - elif param_name == 'contravariant': - if isinstance(param_value, NameExpr): - if param_value.name == 'True': - contravariant = True - else: - self.fail("TypeVar 'contravariant' may only be 'True'", context) - return None - else: - self.fail("TypeVar 'contravariant' may only be 'True'", context) - return None - elif param_name == 'bound': - if has_values: - self.fail("TypeVar cannot have both values and an upper bound", context) - return None - try: - upper_bound = self.expr_to_analyzed_type(param_value) - except TypeTranslationError: - self.fail("TypeVar 'bound' must be a type", param_value) - return None - elif param_name == 'values': - # Probably using obsolete syntax with values=(...). Explain the current syntax. - self.fail("TypeVar 'values' argument not supported", context) - self.fail("Use TypeVar('T', t, ...) instead of TypeVar('T', values=(t, ...))", - context) - return None - else: - self.fail("Unexpected argument to TypeVar(): {}".format(param_name), context) - return None - - if covariant and contravariant: - self.fail("TypeVar cannot be both covariant and contravariant", context) - return None - elif covariant: - variance = COVARIANT - elif contravariant: - variance = CONTRAVARIANT - else: - variance = INVARIANT - return (variance, upper_bound) - - def process_namedtuple_definition(self, s: AssignmentStmt) -> None: - """Check if s defines a namedtuple; if yes, store the definition in symbol table.""" - if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): - return - lvalue = cast(NameExpr, s.lvalues[0]) - name = lvalue.name - named_tuple = self.check_namedtuple(s.rvalue, name) - if named_tuple is None: - return - # Yes, it's a valid namedtuple definition. Add it to the symbol table. - node = self.lookup(name, s) - node.kind = GDEF # TODO locally defined namedtuple - # TODO call.analyzed - node.node = named_tuple - - def check_namedtuple(self, node: Node, var_name: str = None) -> TypeInfo: - """Check if a call defines a namedtuple. - - The optional var_name argument is the name of the variable to - which this is assigned, if any. - - If it does, return the corresponding TypeInfo. Return None otherwise. - - If the definition is invalid but looks like a namedtuple, - report errors but return (some) TypeInfo. - """ - if not isinstance(node, CallExpr): - return None - call = node - if not isinstance(call.callee, RefExpr): - return None - callee = call.callee - fullname = callee.fullname - if fullname not in ('collections.namedtuple', 'typing.NamedTuple'): - return None - items, types, ok = self.parse_namedtuple_args(call, fullname) - if not ok: - # Error. Construct dummy return value. - return self.build_namedtuple_typeinfo('namedtuple', [], []) - else: - # Give it a unique name derived from the line number. - name = cast(StrExpr, call.args[0]).value - if name != var_name: - name += '@' + str(call.line) - info = self.build_namedtuple_typeinfo(name, items, types) - # Store it as a global just in case it would remain anonymous. - self.globals[name] = SymbolTableNode(GDEF, info, self.cur_mod_id) - call.analyzed = NamedTupleExpr(info).set_line(call.line) - return info - - def parse_namedtuple_args(self, call: CallExpr, - fullname: str) -> Tuple[List[str], List[Type], bool]: - # TODO Share code with check_argument_count in checkexpr.py? - args = call.args - if len(args) < 2: - return self.fail_namedtuple_arg("Too few arguments for namedtuple()", call) - if len(args) > 2: - return self.fail_namedtuple_arg("Too many arguments for namedtuple()", call) - if call.arg_kinds != [ARG_POS, ARG_POS]: - return self.fail_namedtuple_arg("Unexpected arguments to namedtuple()", call) - if not isinstance(args[0], (StrExpr, BytesExpr)): - return self.fail_namedtuple_arg( - "namedtuple() expects a string literal as the first argument", call) - types = [] # type: List[Type] - ok = True - if not isinstance(args[1], ListExpr): - if fullname == 'collections.namedtuple' and isinstance(args[1], (StrExpr, BytesExpr)): - str_expr = cast(StrExpr, args[1]) - items = str_expr.value.split() - else: - return self.fail_namedtuple_arg( - "List literal expected as the second argument to namedtuple()", call) - else: - listexpr = args[1] - if fullname == 'collections.namedtuple': - # The fields argument contains just names, with implicit Any types. - if any(not isinstance(item, (StrExpr, BytesExpr)) for item in listexpr.items): - return self.fail_namedtuple_arg("String literal expected as namedtuple() item", - call) - items = [cast(StrExpr, item).value for item in listexpr.items] - else: - # The fields argument contains (name, type) tuples. - items, types, ok = self.parse_namedtuple_fields_with_types(listexpr.items, call) - if not types: - types = [AnyType() for _ in items] - return items, types, ok - - def parse_namedtuple_fields_with_types(self, nodes: List[Node], - context: Context) -> Tuple[List[str], List[Type], bool]: - items = [] # type: List[str] - types = [] # type: List[Type] - for item in nodes: - if isinstance(item, TupleExpr): - if len(item.items) != 2: - return self.fail_namedtuple_arg("Invalid NamedTuple field definition", - item) - name, type_node = item.items - if isinstance(name, (StrExpr, BytesExpr)): - items.append(name.value) - else: - return self.fail_namedtuple_arg("Invalid NamedTuple() field name", item) - try: - type = expr_to_unanalyzed_type(type_node) - except TypeTranslationError: - return self.fail_namedtuple_arg('Invalid field type', type_node) - types.append(self.anal_type(type)) - else: - return self.fail_namedtuple_arg("Tuple expected as NamedTuple() field", item) - return items, types, True - - def fail_namedtuple_arg(self, message: str, - context: Context) -> Tuple[List[str], List[Type], bool]: - self.fail(message, context) - return [], [], False - - def build_namedtuple_typeinfo(self, name: str, items: List[str], - types: List[Type]) -> TypeInfo: - symbols = SymbolTable() - tup = NamedTupleType(name, items, types, self.named_type('__builtins__.tuple', types)) - class_def = ClassDef(name, Block([])) - class_def.fullname = self.qualified_name(name) - info = NamedTupleTypeInfo(tup, symbols, class_def) - info.attrs = items - vars = [Var(item, typ) for item, typ in zip(items, types)] - # Add named tuple items as attributes. - # TODO: Make them read-only. - for var in vars: - var.info = info - symbols[var.name()] = SymbolTableNode(MDEF, var) - this_type = self_type(info) - # Add __init__, _replace, _asdict methods and _fields static field - self.add_namedtuple_method('_replace', symbols, info, this_type, this_type, - self.make_factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) - self.add_namedtuple_method('__init__', symbols, info, this_type, NoneTyp(), - self.make_factory_args(vars, ARG_POS), info.name()) - # TODO: refine to OrderedDict[Union[types]] - self.add_namedtuple_method('_asdict', symbols, info, this_type, AnyType(), []) - return info - - def make_factory_args(self, vars: List[Var], kind: int, - initializer: Expression = None) -> List[Argument]: - return [Argument(var, var.type, initializer, kind) for var in vars] - - def add_namedtuple_method(self, funcname: str, symbols: SymbolTable, info: TypeInfo, - this_type: Type, ret: Type, args: List[Argument], name=None) -> None: - args = [Argument(Var('self'), this_type, None, ARG_POS)] + args - types = [arg.type_annotation for arg in args] - items = [arg.variable.name() for arg in args] - arg_kinds = [arg.kind for arg in args] - # TODO: Make sure that the self argument name is not visible? - signature = CallableType(types, arg_kinds, items, ret, - self.named_type('__builtins__.function'), - name=name or info.name() + '.' + funcname) - func = FuncDef(funcname, - args, - Block([]), - typ=signature) - func.info = info - symbols[funcname] = SymbolTableNode(MDEF, func) - - def analyze_types(self, items: List[Node]) -> List[Type]: - result = [] # type: List[Type] - for node in items: - try: - result.append(self.anal_type(expr_to_unanalyzed_type(node))) - except TypeTranslationError: - self.fail('Type expected', node) - result.append(AnyType()) - return result - - def visit_decorator(self, dec: Decorator) -> None: - for d in dec.decorators: - d.accept(self) - removed = [] # type: List[int] - no_type_check = False - for i, d in enumerate(dec.decorators): - # A bunch of decorators are special cased here. - if refers_to_fullname(d, 'abc.abstractmethod'): - removed.append(i) - dec.func.is_abstract = True - self.check_decorated_function_is_method('abstractmethod', dec) - elif refers_to_fullname(d, 'asyncio.tasks.coroutine'): - removed.append(i) - elif refers_to_fullname(d, 'builtins.staticmethod'): - removed.append(i) - dec.func.is_static = True - dec.var.is_staticmethod = True - self.check_decorated_function_is_method('staticmethod', dec) - elif refers_to_fullname(d, 'builtins.classmethod'): - removed.append(i) - dec.func.is_class = True - dec.var.is_classmethod = True - self.check_decorated_function_is_method('classmethod', dec) - elif (refers_to_fullname(d, 'builtins.property') or - refers_to_fullname(d, 'abc.abstractproperty')): - removed.append(i) - dec.func.is_property = True - dec.var.is_property = True - if refers_to_fullname(d, 'abc.abstractproperty'): - dec.func.is_abstract = True - self.check_decorated_function_is_method('property', dec) - if len(dec.func.arguments) > 1: - self.fail('Too many arguments', dec.func) - elif refers_to_fullname(d, 'typing.no_type_check'): - dec.var.type = AnyType() - no_type_check = True - for i in reversed(removed): - del dec.decorators[i] - if not dec.is_overload or dec.var.is_property: - if self.is_func_scope(): - self.add_symbol(dec.var.name(), SymbolTableNode(LDEF, dec), - dec) - elif self.type: - dec.var.info = self.type - dec.var.is_initialized_in_class = True - self.add_symbol(dec.var.name(), SymbolTableNode(MDEF, dec), - dec) - if not no_type_check: - dec.func.accept(self) - if dec.decorators and dec.var.is_property: - self.fail('Decorated property not supported', dec) - - def check_decorated_function_is_method(self, decorator: str, - context: Context) -> None: - if not self.type or self.is_func_scope(): - self.fail("'%s' used with a non-method" % decorator, context) - - def visit_expression_stmt(self, s: ExpressionStmt) -> None: - s.expr.accept(self) - - def visit_return_stmt(self, s: ReturnStmt) -> None: - if not self.is_func_scope(): - self.fail("'return' outside function", s) - if s.expr: - s.expr.accept(self) - - def visit_raise_stmt(self, s: RaiseStmt) -> None: - if s.expr: - s.expr.accept(self) - if s.from_expr: - s.from_expr.accept(self) - - def visit_assert_stmt(self, s: AssertStmt) -> None: - if s.expr: - s.expr.accept(self) - - def visit_operator_assignment_stmt(self, - s: OperatorAssignmentStmt) -> None: - s.lvalue.accept(self) - s.rvalue.accept(self) - if (isinstance(s.lvalue, NameExpr) and s.lvalue.name == '__all__' and - s.lvalue.kind == GDEF and isinstance(s.rvalue, (ListExpr, TupleExpr))): - self.add_exports(*s.rvalue.items) - - def visit_while_stmt(self, s: WhileStmt) -> None: - s.expr.accept(self) - self.loop_depth += 1 - s.body.accept(self) - self.loop_depth -= 1 - self.visit_block_maybe(s.else_body) - - def visit_for_stmt(self, s: ForStmt) -> None: - s.expr.accept(self) - - # Bind index variables and check if they define new names. - self.analyze_lvalue(s.index) - - self.loop_depth += 1 - self.visit_block(s.body) - self.loop_depth -= 1 - - self.visit_block_maybe(s.else_body) - - def visit_break_stmt(self, s: BreakStmt) -> None: - if self.loop_depth == 0: - self.fail("'break' outside loop", s, True, blocker=True) - - def visit_continue_stmt(self, s: ContinueStmt) -> None: - if self.loop_depth == 0: - self.fail("'continue' outside loop", s, True, blocker=True) - - def visit_if_stmt(self, s: IfStmt) -> None: - infer_reachability_of_if_statement(s, pyversion=self.options.python_version) - for i in range(len(s.expr)): - s.expr[i].accept(self) - self.visit_block(s.body[i]) - self.visit_block_maybe(s.else_body) - - def visit_try_stmt(self, s: TryStmt) -> None: - self.analyze_try_stmt(s, self) - - def analyze_try_stmt(self, s: TryStmt, visitor: NodeVisitor, - add_global: bool = False) -> None: - s.body.accept(visitor) - for type, var, handler in zip(s.types, s.vars, s.handlers): - if type: - type.accept(visitor) - if var: - self.analyze_lvalue(var, add_global=add_global) - handler.accept(visitor) - if s.else_body: - s.else_body.accept(visitor) - if s.finally_body: - s.finally_body.accept(visitor) - - def visit_with_stmt(self, s: WithStmt) -> None: - for e, n in zip(s.expr, s.target): - e.accept(self) - if n: - self.analyze_lvalue(n) - self.visit_block(s.body) - - def visit_del_stmt(self, s: DelStmt) -> None: - s.expr.accept(self) - if not self.is_valid_del_target(s.expr): - self.fail('Invalid delete target', s) - - def is_valid_del_target(self, s: Node) -> bool: - if isinstance(s, (IndexExpr, NameExpr, MemberExpr)): - return True - elif isinstance(s, TupleExpr): - return all(self.is_valid_del_target(item) for item in s.items) - - def visit_global_decl(self, g: GlobalDecl) -> None: - for name in g.names: - if name in self.nonlocal_decls[-1]: - self.fail("Name '{}' is nonlocal and global".format(name), g) - self.global_decls[-1].add(name) - - def visit_nonlocal_decl(self, d: NonlocalDecl) -> None: - if not self.is_func_scope(): - self.fail("nonlocal declaration not allowed at module level", d) - else: - for name in d.names: - for table in reversed(self.locals[:-1]): - if table is not None and name in table: - break - else: - self.fail("No binding for nonlocal '{}' found".format(name), d) - - if self.locals[-1] is not None and name in self.locals[-1]: - self.fail("Name '{}' is already defined in local " - "scope before nonlocal declaration".format(name), d) - - if name in self.global_decls[-1]: - self.fail("Name '{}' is nonlocal and global".format(name), d) - self.nonlocal_decls[-1].add(name) - - def visit_print_stmt(self, s: PrintStmt) -> None: - for arg in s.args: - arg.accept(self) - if s.target: - s.target.accept(self) - - def visit_exec_stmt(self, s: ExecStmt) -> None: - s.expr.accept(self) - if s.variables1: - s.variables1.accept(self) - if s.variables2: - s.variables2.accept(self) - - # - # Expressions - # - - def visit_name_expr(self, expr: NameExpr) -> None: - n = self.lookup(expr.name, expr) - if n: - if n.kind == BOUND_TVAR: - self.fail("'{}' is a type variable and only valid in type " - "context".format(expr.name), expr) - else: - expr.kind = n.kind - expr.node = n.node - expr.fullname = n.fullname - - def visit_super_expr(self, expr: SuperExpr) -> None: - if not self.type: - self.fail('"super" used outside class', expr) - return - expr.info = self.type - - def visit_tuple_expr(self, expr: TupleExpr) -> None: - for item in expr.items: - item.accept(self) - - def visit_list_expr(self, expr: ListExpr) -> None: - for item in expr.items: - item.accept(self) - - def visit_set_expr(self, expr: SetExpr) -> None: - for item in expr.items: - item.accept(self) - - def visit_dict_expr(self, expr: DictExpr) -> None: - for key, value in expr.items: - key.accept(self) - value.accept(self) - - def visit_star_expr(self, expr: StarExpr) -> None: - if not expr.valid: - self.fail('Can use starred expression only as assignment target', expr) - else: - expr.expr.accept(self) - - def visit_yield_from_expr(self, e: YieldFromExpr) -> None: - if not self.is_func_scope(): # not sure - self.fail("'yield from' outside function", e, True, blocker=True) - else: - self.function_stack[-1].is_generator = True - if e.expr: - e.expr.accept(self) - - def visit_call_expr(self, expr: CallExpr) -> None: - """Analyze a call expression. - - Some call expressions are recognized as special forms, including - cast(...) and Any(...). - """ - expr.callee.accept(self) - if refers_to_fullname(expr.callee, 'typing.cast'): - # Special form cast(...). - if not self.check_fixed_args(expr, 2, 'cast'): - return - # Translate first argument to an unanalyzed type. - try: - target = expr_to_unanalyzed_type(expr.args[0]) - except TypeTranslationError: - self.fail('Cast target is not a type', expr) - return - # Piggyback CastExpr object to the CallExpr object; it takes - # precedence over the CallExpr semantics. - expr.analyzed = CastExpr(expr.args[1], target) - expr.analyzed.line = expr.line - expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'builtins.reveal_type'): - if not self.check_fixed_args(expr, 1, 'reveal_type'): - return - expr.analyzed = RevealTypeExpr(expr.args[0]) - expr.analyzed.line = expr.line - expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'typing.Any'): - # Special form Any(...). - if not self.check_fixed_args(expr, 1, 'Any'): - return - expr.analyzed = CastExpr(expr.args[0], AnyType()) - expr.analyzed.line = expr.line - expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'typing._promote'): - # Special form _promote(...). - if not self.check_fixed_args(expr, 1, '_promote'): - return - # Translate first argument to an unanalyzed type. - try: - target = expr_to_unanalyzed_type(expr.args[0]) - except TypeTranslationError: - self.fail('Argument 1 to _promote is not a type', expr) - return - expr.analyzed = PromoteExpr(target) - expr.analyzed.line = expr.line - expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'builtins.dict'): - expr.analyzed = self.translate_dict_call(expr) - else: - # Normal call expression. - for a in expr.args: - a.accept(self) - - if (isinstance(expr.callee, MemberExpr) and - isinstance(expr.callee.expr, NameExpr) and - expr.callee.expr.name == '__all__' and - expr.callee.expr.kind == GDEF and - expr.callee.name in ('append', 'extend')): - if expr.callee.name == 'append' and expr.args: - self.add_exports(expr.args[0]) - elif (expr.callee.name == 'extend' and expr.args and - isinstance(expr.args[0], (ListExpr, TupleExpr))): - self.add_exports(*expr.args[0].items) - - def translate_dict_call(self, call: CallExpr) -> Optional[DictExpr]: - """Translate 'dict(x=y, ...)' to {'x': y, ...}. - - For other variants of dict(...), return None. - """ - if not call.args: - return None - if not all(kind == ARG_NAMED for kind in call.arg_kinds): - # Must still accept those args. - for a in call.args: - a.accept(self) - return None - expr = DictExpr([(StrExpr(key), value) - for key, value in zip(call.arg_names, call.args)]) - expr.set_line(call) - expr.accept(self) - return expr - - def check_fixed_args(self, expr: CallExpr, numargs: int, - name: str) -> bool: - """Verify that expr has specified number of positional args. - - Return True if the arguments are valid. - """ - s = 's' - if numargs == 1: - s = '' - if len(expr.args) != numargs: - self.fail("'%s' expects %d argument%s" % (name, numargs, s), - expr) - return False - if expr.arg_kinds != [ARG_POS] * numargs: - self.fail("'%s' must be called with %s positional argument%s" % - (name, numargs, s), expr) - return False - return True - - def visit_member_expr(self, expr: MemberExpr) -> None: - base = expr.expr - base.accept(self) - # Bind references to module attributes. - if isinstance(base, RefExpr) and base.kind == MODULE_REF: - file = cast(MypyFile, base.node) - n = file.names.get(expr.name, None) if file is not None else None - if n: - n = self.normalize_type_alias(n, expr) - if not n: - return - expr.kind = n.kind - expr.fullname = n.fullname - expr.node = n.node - else: - # We only catch some errors here; the rest will be - # catched during type checking. - # - # This way we can report a larger number of errors in - # one type checker run. If we reported errors here, - # the build would terminate after semantic analysis - # and we wouldn't be able to report any type errors. - full_name = '%s.%s' % (file.fullname() if file is not None else None, expr.name) - if full_name in obsolete_name_mapping: - self.fail("Module has no attribute %r (it's now called %r)" % ( - expr.name, obsolete_name_mapping[full_name]), expr) - - def visit_op_expr(self, expr: OpExpr) -> None: - expr.left.accept(self) - expr.right.accept(self) - - def visit_comparison_expr(self, expr: ComparisonExpr) -> None: - for operand in expr.operands: - operand.accept(self) - - def visit_unary_expr(self, expr: UnaryExpr) -> None: - expr.expr.accept(self) - - def visit_index_expr(self, expr: IndexExpr) -> None: - expr.base.accept(self) - if refers_to_class_or_function(expr.base): - # Special form -- type application. - # Translate index to an unanalyzed type. - types = [] # type: List[Type] - if isinstance(expr.index, TupleExpr): - items = expr.index.items - else: - items = [expr.index] - for item in items: - try: - typearg = expr_to_unanalyzed_type(item) - except TypeTranslationError: - self.fail('Type expected within [...]', expr) - return - typearg = self.anal_type(typearg) - types.append(typearg) - expr.analyzed = TypeApplication(expr.base, types) - expr.analyzed.line = expr.line - else: - expr.index.accept(self) - - def visit_slice_expr(self, expr: SliceExpr) -> None: - if expr.begin_index: - expr.begin_index.accept(self) - if expr.end_index: - expr.end_index.accept(self) - if expr.stride: - expr.stride.accept(self) - - def visit_cast_expr(self, expr: CastExpr) -> None: - expr.expr.accept(self) - expr.type = self.anal_type(expr.type) - - def visit_reveal_type_expr(self, expr: RevealTypeExpr) -> None: - expr.expr.accept(self) - - def visit_type_application(self, expr: TypeApplication) -> None: - expr.expr.accept(self) - for i in range(len(expr.types)): - expr.types[i] = self.anal_type(expr.types[i]) - - def visit_list_comprehension(self, expr: ListComprehension) -> None: - expr.generator.accept(self) - - def visit_set_comprehension(self, expr: SetComprehension) -> None: - expr.generator.accept(self) - - def visit_dictionary_comprehension(self, expr: DictionaryComprehension) -> None: - self.enter() - self.analyze_comp_for(expr) - expr.key.accept(self) - expr.value.accept(self) - self.leave() - self.analyze_comp_for_2(expr) - - def visit_generator_expr(self, expr: GeneratorExpr) -> None: - self.enter() - self.analyze_comp_for(expr) - expr.left_expr.accept(self) - self.leave() - self.analyze_comp_for_2(expr) - - def analyze_comp_for(self, expr: Union[GeneratorExpr, - DictionaryComprehension]) -> None: - """Analyses the 'comp_for' part of comprehensions (part 1). - - That is the part after 'for' in (x for x in l if p). This analyzes - variables and conditions which are analyzed in a local scope. - """ - for i, (index, sequence, conditions) in enumerate(zip(expr.indices, - expr.sequences, - expr.condlists)): - if i > 0: - sequence.accept(self) - # Bind index variables. - self.analyze_lvalue(index) - for cond in conditions: - cond.accept(self) - - def analyze_comp_for_2(self, expr: Union[GeneratorExpr, - DictionaryComprehension]) -> None: - """Analyses the 'comp_for' part of comprehensions (part 2). - - That is the part after 'for' in (x for x in l if p). This analyzes - the 'l' part which is analyzed in the surrounding scope. - """ - expr.sequences[0].accept(self) - - def visit_func_expr(self, expr: FuncExpr) -> None: - self.analyze_function(expr) - - def visit_conditional_expr(self, expr: ConditionalExpr) -> None: - expr.if_expr.accept(self) - expr.cond.accept(self) - expr.else_expr.accept(self) - - def visit_backquote_expr(self, expr: BackquoteExpr) -> None: - expr.expr.accept(self) - - def visit__promote_expr(self, expr: PromoteExpr) -> None: - expr.type = self.anal_type(expr.type) - - def visit_yield_expr(self, expr: YieldExpr) -> None: - if not self.is_func_scope(): - self.fail("'yield' outside function", expr, True, blocker=True) - else: - self.function_stack[-1].is_generator = True - if expr.expr: - expr.expr.accept(self) - - # - # Helpers - # - - def lookup(self, name: str, ctx: Context) -> SymbolTableNode: - """Look up an unqualified name in all active namespaces.""" - # 1a. Name declared using 'global x' takes precedence - if name in self.global_decls[-1]: - if name in self.globals: - return self.globals[name] - else: - self.name_not_defined(name, ctx) - return None - # 1b. Name declared using 'nonlocal x' takes precedence - if name in self.nonlocal_decls[-1]: - for table in reversed(self.locals[:-1]): - if table is not None and name in table: - return table[name] - else: - self.name_not_defined(name, ctx) - return None - # 2. Class attributes (if within class definition) - if self.is_class_scope() and name in self.type.names: - return self.type[name] - # 3. Local (function) scopes - for table in reversed(self.locals): - if table is not None and name in table: - return table[name] - # 4. Current file global scope - if name in self.globals: - return self.globals[name] - # 5. Builtins - b = self.globals.get('__builtins__', None) - if b: - table = cast(MypyFile, b.node).names - if name in table: - if name[0] == "_" and name[1] != "_": - self.name_not_defined(name, ctx) - return None - node = table[name] - return node - # Give up. - self.name_not_defined(name, ctx) - self.check_for_obsolete_short_name(name, ctx) - return None - - def check_for_obsolete_short_name(self, name: str, ctx: Context) -> None: - matches = [obsolete_name - for obsolete_name in obsolete_name_mapping - if obsolete_name.rsplit('.', 1)[-1] == name] - if len(matches) == 1: - self.note("(Did you mean '{}'?)".format(obsolete_name_mapping[matches[0]]), ctx) - - def lookup_qualified(self, name: str, ctx: Context) -> SymbolTableNode: - if '.' not in name: - return self.lookup(name, ctx) - else: - parts = name.split('.') - n = self.lookup(parts[0], ctx) # type: SymbolTableNode - if n: - for i in range(1, len(parts)): - if isinstance(n.node, TypeInfo): - if n.node.mro is None: - # We haven't yet analyzed the class `n.node`. Fall back to direct - # lookup in the names declared directly under it, without its base - # classes. This can happen when we have a forward reference to a - # nested class, and the reference is bound before the outer class - # has been fully semantically analyzed. - # - # A better approach would be to introduce a new analysis pass or - # to move things around between passes, but this unblocks a common - # use case even though this is a little limited in case there is - # inheritance involved. - result = n.node.names.get(parts[i]) - else: - result = n.node.get(parts[i]) - n = result - elif isinstance(n.node, MypyFile): - n = n.node.names.get(parts[i], None) - # TODO: What if node is Var or FuncDef? - if not n: - self.name_not_defined(name, ctx) - break - if n: - n = self.normalize_type_alias(n, ctx) - return n - - def builtin_type(self, fully_qualified_name: str) -> Instance: - node = self.lookup_fully_qualified(fully_qualified_name) - info = cast(TypeInfo, node.node) - return Instance(info, []) - - def lookup_fully_qualified(self, name: str) -> SymbolTableNode: - """Lookup a fully qualified name. - - Assume that the name is defined. This happens in the global namespace -- the local - module namespace is ignored. - """ - assert '.' in name - parts = name.split('.') - n = self.modules[parts[0]] - for i in range(1, len(parts) - 1): - n = cast(MypyFile, n.names[parts[i]].node) - return n.names[parts[-1]] - - def lookup_fully_qualified_or_none(self, name: str) -> SymbolTableNode: - """Lookup a fully qualified name. - - Assume that the name is defined. This happens in the global namespace -- the local - module namespace is ignored. - """ - assert '.' in name - parts = name.split('.') - n = self.modules[parts[0]] - for i in range(1, len(parts) - 1): - next_sym = n.names.get(parts[i]) - if not next_sym: - return None - n = cast(MypyFile, next_sym.node) - return n.names.get(parts[-1]) - - def qualified_name(self, n: str) -> str: - return self.cur_mod_id + '.' + n - - def enter(self) -> None: - self.locals.append(SymbolTable()) - self.global_decls.append(set()) - self.nonlocal_decls.append(set()) - # -1 since entering block will increment this to 0. - self.block_depth.append(-1) - - def leave(self) -> None: - self.locals.pop() - self.global_decls.pop() - self.nonlocal_decls.pop() - self.block_depth.pop() - - def is_func_scope(self) -> bool: - return self.locals[-1] is not None - - def is_class_scope(self) -> bool: - return self.type is not None and not self.is_func_scope() - - def add_symbol(self, name: str, node: SymbolTableNode, - context: Context) -> None: - if self.is_func_scope(): - if name in self.locals[-1]: - # Flag redefinition unless this is a reimport of a module. - if not (node.kind == MODULE_REF and - self.locals[-1][name].node == node.node): - self.name_already_defined(name, context) - self.locals[-1][name] = node - elif self.type: - self.type.names[name] = node - else: - existing = self.globals.get(name) - if existing and (not isinstance(node.node, MypyFile) or - existing.node != node.node) and existing.kind != UNBOUND_IMPORTED: - # Modules can be imported multiple times to support import - # of multiple submodules of a package (e.g. a.x and a.y). - ok = False - # Only report an error if the symbol collision provides a different type. - if existing.type and node.type and is_same_type(existing.type, node.type): - ok = True - if not ok: - self.name_already_defined(name, context) - self.globals[name] = node - - def add_var(self, v: Var, ctx: Context) -> None: - if self.is_func_scope(): - self.add_local(v, ctx) - else: - self.globals[v.name()] = SymbolTableNode(GDEF, v, self.cur_mod_id) - v._fullname = self.qualified_name(v.name()) - - def add_local(self, node: Union[Var, FuncBase], ctx: Context) -> None: - name = node.name() - if name in self.locals[-1]: - self.name_already_defined(name, ctx) - node._fullname = name - self.locals[-1][name] = SymbolTableNode(LDEF, node) - - def add_exports(self, *exps: Node) -> None: - for exp in exps: - if isinstance(exp, StrExpr): - self.all_exports.add(exp.value) - - def check_no_global(self, n: str, ctx: Context, - is_overloaded_func: bool = False) -> None: - if n in self.globals: - prev_is_overloaded = isinstance(self.globals[n], OverloadedFuncDef) - if is_overloaded_func and prev_is_overloaded: - self.fail("Nonconsecutive overload {} found".format(n), ctx) - elif prev_is_overloaded: - self.fail("Definition of '{}' missing 'overload'".format(n), ctx) - else: - self.name_already_defined(n, ctx) - - def name_not_defined(self, name: str, ctx: Context) -> None: - message = "Name '{}' is not defined".format(name) - extra = self.undefined_name_extra_info(name) - if extra: - message += ' {}'.format(extra) - self.fail(message, ctx) - - def name_already_defined(self, name: str, ctx: Context) -> None: - self.fail("Name '{}' already defined".format(name), ctx) - - def fail(self, msg: str, ctx: Context, serious: bool = False, *, - blocker: bool = False) -> None: - if (not serious and - not self.options.check_untyped_defs and - self.function_stack and - self.function_stack[-1].is_dynamic()): - return - self.errors.report(ctx.get_line(), msg, blocker=blocker) - - def note(self, msg: str, ctx: Context) -> None: - if (not self.options.check_untyped_defs and - self.function_stack and - self.function_stack[-1].is_dynamic()): - return - self.errors.report(ctx.get_line(), msg, severity='note') - - def undefined_name_extra_info(self, fullname: str) -> Optional[str]: - if fullname in obsolete_name_mapping: - return "(it's now called '{}')".format(obsolete_name_mapping[fullname]) - else: - return None - - def accept(self, node: Node) -> None: - try: - node.accept(self) - except Exception as err: - report_internal_error(err, self.errors.file, node.line) - - -class FirstPass(NodeVisitor): - """First phase of semantic analysis. - - See docstring of 'analyze()' below for a description of what this does. - """ - - def __init__(self, sem: SemanticAnalyzer) -> None: - self.sem = sem - self.pyversion = sem.options.python_version - - def analyze(self, file: MypyFile, fnam: str, mod_id: str) -> None: - """Perform the first analysis pass. - - Populate module global table. Resolve the full names of - definitions not nested within functions and construct type - info structures, but do not resolve inter-definition - references such as base classes. - - Also add implicit definitions such as __name__. - - In this phase we don't resolve imports. For 'from ... import', - we generate dummy symbol table nodes for the imported names, - and these will get resolved in later phases of semantic - analysis. - """ - sem = self.sem - sem.cur_mod_id = mod_id - sem.errors.set_file(fnam) - sem.globals = SymbolTable() - sem.global_decls = [set()] - sem.nonlocal_decls = [set()] - sem.block_depth = [0] - - defs = file.defs - - # Add implicit definitions of module '__name__' etc. - for name, t in implicit_module_attrs.items(): - v = Var(name, UnboundType(t)) - v._fullname = self.sem.qualified_name(name) - self.sem.globals[name] = SymbolTableNode(GDEF, v, self.sem.cur_mod_id) - - for d in defs: - d.accept(self) - - # Add implicit definition of literals/keywords to builtins, as we - # cannot define a variable with them explicitly. - if mod_id == 'builtins': - literal_types = [ - ('None', NoneTyp()), - # reveal_type is a mypy-only function that gives an error with the type of its arg - ('reveal_type', AnyType()), - ] # type: List[Tuple[str, Type]] - - # TODO(ddfisher): This guard is only needed because mypy defines - # fake builtins for its tests which often don't define bool. If - # mypy is fast enough that we no longer need those, this - # conditional check should be removed. - if 'bool' in self.sem.globals: - bool_type = self.sem.named_type('bool') - literal_types.extend([ - ('True', bool_type), - ('False', bool_type), - ('__debug__', bool_type), - ]) - - for name, typ in literal_types: - v = Var(name, typ) - v._fullname = self.sem.qualified_name(name) - self.sem.globals[name] = SymbolTableNode(GDEF, v, self.sem.cur_mod_id) - - def visit_block(self, b: Block) -> None: - if b.is_unreachable: - return - self.sem.block_depth[-1] += 1 - for node in b.body: - node.accept(self) - self.sem.block_depth[-1] -= 1 - - def visit_assignment_stmt(self, s: AssignmentStmt) -> None: - for lval in s.lvalues: - self.analyze_lvalue(lval, explicit_type=s.type is not None) - - def visit_func_def(self, func: FuncDef) -> None: - sem = self.sem - func.is_conditional = sem.block_depth[-1] > 0 - func._fullname = sem.qualified_name(func.name()) - if func.name() in sem.globals: - # Already defined in this module. - original_sym = sem.globals[func.name()] - if original_sym.kind == UNBOUND_IMPORTED: - # Ah this is an imported name. We can't resolve them now, so we'll postpone - # this until the main phase of semantic analysis. - return - original_def = original_sym.node - if sem.is_conditional_func(original_def, func): - # Conditional function definition -- multiple defs are ok. - func.original_def = cast(FuncDef, original_def) - else: - # Report error. - sem.check_no_global(func.name(), func) - else: - sem.globals[func.name()] = SymbolTableNode(GDEF, func, sem.cur_mod_id) - - def visit_overloaded_func_def(self, func: OverloadedFuncDef) -> None: - self.sem.check_no_global(func.name(), func, True) - func._fullname = self.sem.qualified_name(func.name()) - self.sem.globals[func.name()] = SymbolTableNode(GDEF, func, - self.sem.cur_mod_id) - - def visit_class_def(self, cdef: ClassDef) -> None: - self.sem.check_no_global(cdef.name, cdef) - cdef.fullname = self.sem.qualified_name(cdef.name) - info = TypeInfo(SymbolTable(), cdef) - info.set_line(cdef.line) - cdef.info = info - self.sem.globals[cdef.name] = SymbolTableNode(GDEF, info, - self.sem.cur_mod_id) - self.process_nested_classes(cdef) - - def process_nested_classes(self, outer_def: ClassDef) -> None: - for node in outer_def.defs.body: - if isinstance(node, ClassDef): - node.info = TypeInfo(SymbolTable(), node) - if outer_def.fullname: - node.info._fullname = outer_def.fullname + '.' + node.info.name() - else: - node.info._fullname = node.info.name() - symbol = SymbolTableNode(MDEF, node.info) - outer_def.info.names[node.name] = symbol - self.process_nested_classes(node) - - def visit_import_from(self, node: ImportFrom) -> None: - # We can't bind module names during the first pass, as the target module might be - # unprocessed. However, we add dummy unbound imported names to the symbol table so - # that we at least know that the name refers to a module. - node.is_top_level = True - for name, as_name in node.names: - imported_name = as_name or name - if imported_name not in self.sem.globals: - self.sem.add_symbol(imported_name, SymbolTableNode(UNBOUND_IMPORTED, None), node) - - def visit_import(self, node: Import) -> None: - node.is_top_level = True - # This is similar to visit_import_from -- see the comment there. - for id, as_id in node.ids: - imported_id = as_id or id - if imported_id not in self.sem.globals: - self.sem.add_symbol(imported_id, SymbolTableNode(UNBOUND_IMPORTED, None), node) - else: - # If the previous symbol is a variable, this should take precedence. - self.sem.globals[imported_id] = SymbolTableNode(UNBOUND_IMPORTED, None) - - def visit_import_all(self, node: ImportAll) -> None: - node.is_top_level = True - - def visit_while_stmt(self, s: WhileStmt) -> None: - s.body.accept(self) - if s.else_body: - s.else_body.accept(self) - - def visit_for_stmt(self, s: ForStmt) -> None: - self.analyze_lvalue(s.index) - s.body.accept(self) - if s.else_body: - s.else_body.accept(self) - - def visit_with_stmt(self, s: WithStmt) -> None: - for n in s.target: - if n: - self.analyze_lvalue(n) - s.body.accept(self) - - def visit_decorator(self, d: Decorator) -> None: - d.var._fullname = self.sem.qualified_name(d.var.name()) - self.sem.add_symbol(d.var.name(), SymbolTableNode(GDEF, d.var), d) - - def visit_if_stmt(self, s: IfStmt) -> None: - infer_reachability_of_if_statement(s, pyversion=self.pyversion) - for node in s.body: - node.accept(self) - if s.else_body: - s.else_body.accept(self) - - def visit_try_stmt(self, s: TryStmt) -> None: - self.sem.analyze_try_stmt(s, self, add_global=True) - - def analyze_lvalue(self, lvalue: Node, explicit_type: bool = False) -> None: - self.sem.analyze_lvalue(lvalue, add_global=True, explicit_type=explicit_type) - - -class ThirdPass(TraverserVisitor): - """The third and final pass of semantic analysis. - - Check type argument counts and values of generic types, and perform some - straightforward type inference. - """ - - def __init__(self, modules: Dict[str, MypyFile], errors: Errors) -> None: - self.modules = modules - self.errors = errors - - def visit_file(self, file_node: MypyFile, fnam: str) -> None: - self.errors.set_file(fnam) - self.accept(file_node) - - def accept(self, node: Node) -> None: - try: - node.accept(self) - except Exception as err: - report_internal_error(err, self.errors.file, node.line) - - def visit_block(self, b: Block) -> None: - if b.is_unreachable: - return - super().visit_block(b) - - def visit_func_def(self, fdef: FuncDef) -> None: - self.errors.push_function(fdef.name()) - self.analyze(fdef.type) - super().visit_func_def(fdef) - self.errors.pop_function() - - def visit_class_def(self, tdef: ClassDef) -> None: - for type in tdef.info.bases: - self.analyze(type) - # Recompute MRO now that we have analyzed all modules, to pick - # up superclasses of bases imported from other modules in an - # import loop. (Only do so if we succeeded the first time.) - if tdef.info.mro: - tdef.info.mro = [] # Force recomputation - calculate_class_mro(tdef, self.fail) - super().visit_class_def(tdef) - - def visit_decorator(self, dec: Decorator) -> None: - """Try to infer the type of the decorated function. - - This lets us resolve references to decorated functions during - type checking when there are cyclic imports, as otherwise the - type might not be available when we need it. - - This basically uses a simple special-purpose type inference - engine just for decorators. - """ - super().visit_decorator(dec) - if dec.var.is_property: - # Decorators are expected to have a callable type (it's a little odd). - if dec.func.type is None: - dec.var.type = CallableType( - [AnyType()], - [ARG_POS], - [None], - AnyType(), - self.builtin_type('function'), - name=dec.var.name()) - elif isinstance(dec.func.type, CallableType): - dec.var.type = dec.func.type - return - decorator_preserves_type = True - for expr in dec.decorators: - preserve_type = False - if isinstance(expr, RefExpr) and isinstance(expr.node, FuncDef): - if is_identity_signature(expr.node.type): - preserve_type = True - if not preserve_type: - decorator_preserves_type = False - break - if decorator_preserves_type: - # No non-identity decorators left. We can trivially infer the type - # of the function here. - dec.var.type = function_type(dec.func, self.builtin_type('function')) - if dec.decorators: - if returns_any_if_called(dec.decorators[0]): - # The outermost decorator will return Any so we know the type of the - # decorated function. - dec.var.type = AnyType() - sig = find_fixed_callable_return(dec.decorators[0]) - if sig: - # The outermost decorator always returns the same kind of function, - # so we know that this is the type of the decoratored function. - orig_sig = function_type(dec.func, self.builtin_type('function')) - sig.name = orig_sig.items()[0].name - dec.var.type = sig - - def visit_assignment_stmt(self, s: AssignmentStmt) -> None: - self.analyze(s.type) - super().visit_assignment_stmt(s) - - def visit_cast_expr(self, e: CastExpr) -> None: - self.analyze(e.type) - super().visit_cast_expr(e) - - def visit_reveal_type_expr(self, e: RevealTypeExpr) -> None: - super().visit_reveal_type_expr(e) - - def visit_type_application(self, e: TypeApplication) -> None: - for type in e.types: - self.analyze(type) - super().visit_type_application(e) - - # Helpers - - def analyze(self, type: Type) -> None: - if type: - analyzer = TypeAnalyserPass3(self.fail) - type.accept(analyzer) - - def fail(self, msg: str, ctx: Context) -> None: - self.errors.report(ctx.get_line(), msg) - - def builtin_type(self, name: str, args: List[Type] = None) -> Instance: - names = self.modules['builtins'] - sym = names.names[name] - assert isinstance(sym.node, TypeInfo) - return Instance(sym.node, args or []) - - -def self_type(typ: TypeInfo) -> Union[Instance, TupleType]: - """For a non-generic type, return instance type representing the type. - For a generic G type with parameters T1, .., Tn, return G[T1, ..., Tn]. - """ - tv = [] # type: List[Type] - for i in range(len(typ.type_vars)): - tv.append(TypeVarType(typ.defn.type_vars[i])) - inst = Instance(typ, tv) - if typ.tuple_type is None: - return inst - return typ.tuple_type.self_type(inst) - - -def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: - if isinstance(sig, CallableType): - return replace_leading_arg_type(sig, new) - else: - sig = cast(Overloaded, sig) - return Overloaded([cast(CallableType, replace_implicit_first_type(i, new)) - for i in sig.items()]) - - -def set_callable_name(sig: Type, fdef: FuncDef) -> Type: - if isinstance(sig, FunctionLike): - if fdef.info: - return sig.with_name( - '"{}" of "{}"'.format(fdef.name(), fdef.info.name())) - else: - return sig.with_name('"{}"'.format(fdef.name())) - else: - return sig - - -def refers_to_fullname(node: Node, fullname: str) -> bool: - """Is node a name or member expression with the given full name?""" - return isinstance(node, RefExpr) and node.fullname == fullname - - -def refers_to_class_or_function(node: Node) -> bool: - """Does semantically analyzed node refer to a class?""" - return (isinstance(node, RefExpr) and - isinstance(node.node, (TypeInfo, FuncDef, OverloadedFuncDef))) - - -def calculate_class_mro(defn: ClassDef, fail: Callable[[str, Context], None]) -> None: - try: - defn.info.calculate_mro() - except MroError: - fail("Cannot determine consistent method resolution order " - '(MRO) for "%s"' % defn.name, defn) - defn.info.mro = [] - # The property of falling back to Any is inherited. - defn.info.fallback_to_any = any(baseinfo.fallback_to_any for baseinfo in defn.info.mro) - - -def find_duplicate(list: List[T]) -> T: - """If the list has duplicates, return one of the duplicates. - - Otherwise, return None. - """ - for i in range(1, len(list)): - if list[i] in list[:i]: - return list[i] - return None - - -def disable_typevars(nodes: List[SymbolTableNode]) -> None: - for node in nodes: - assert node.kind in (BOUND_TVAR, UNBOUND_TVAR) - node.kind = UNBOUND_TVAR - - -def enable_typevars(nodes: List[SymbolTableNode]) -> None: - for node in nodes: - assert node.kind in (BOUND_TVAR, UNBOUND_TVAR) - node.kind = BOUND_TVAR - - -def remove_imported_names_from_symtable(names: SymbolTable, - module: str) -> None: - """Remove all imported names from the symbol table of a module.""" - removed = [] # type: List[str] - for name, node in names.items(): - if node.node is None: - continue - fullname = node.node.fullname() - prefix = fullname[:fullname.rfind('.')] - if prefix != module: - removed.append(name) - for name in removed: - del names[name] - - -def infer_reachability_of_if_statement(s: IfStmt, - pyversion: Tuple[int, int]) -> None: - for i in range(len(s.expr)): - result = infer_if_condition_value(s.expr[i], pyversion) - if result == ALWAYS_FALSE: - # The condition is always false, so we skip the if/elif body. - mark_block_unreachable(s.body[i]) - elif result == ALWAYS_TRUE: - # This condition is always true, so all of the remaining - # elif/else bodies will never be executed. - for body in s.body[i + 1:]: - mark_block_unreachable(body) - if s.else_body: - mark_block_unreachable(s.else_body) - break - - -def infer_if_condition_value(expr: Node, pyversion: Tuple[int, int]) -> int: - """Infer whether if condition is always true/false. - - Return ALWAYS_TRUE if always true, ALWAYS_FALSE if always false, - and TRUTH_VALUE_UNKNOWN otherwise. - """ - name = '' - negated = False - alias = expr - if isinstance(alias, UnaryExpr): - if alias.op == 'not': - expr = alias.expr - negated = True - if isinstance(expr, NameExpr): - name = expr.name - elif isinstance(expr, MemberExpr): - name = expr.name - result = TRUTH_VALUE_UNKNOWN - if name == 'PY2': - result = ALWAYS_TRUE if pyversion[0] == 2 else ALWAYS_FALSE - elif name == 'PY3': - result = ALWAYS_TRUE if pyversion[0] == 3 else ALWAYS_FALSE - elif name == 'MYPY': - result = ALWAYS_TRUE - if negated: - if result == ALWAYS_TRUE: - result = ALWAYS_FALSE - elif result == ALWAYS_FALSE: - result = ALWAYS_TRUE - return result - - -def mark_block_unreachable(block: Block) -> None: - block.is_unreachable = True - block.accept(MarkImportsUnreachableVisitor()) - - -class MarkImportsUnreachableVisitor(TraverserVisitor): - """Visitor that flags all imports nested within a node as unreachable.""" - - def visit_import(self, node: Import) -> None: - node.is_unreachable = True - - def visit_import_from(self, node: ImportFrom) -> None: - node.is_unreachable = True - - def visit_import_all(self, node: ImportAll) -> None: - node.is_unreachable = True - - -def is_identity_signature(sig: Type) -> bool: - """Is type a callable of form T -> T (where T is a type variable)?""" - if isinstance(sig, CallableType) and sig.arg_kinds == [ARG_POS]: - if isinstance(sig.arg_types[0], TypeVarType) and isinstance(sig.ret_type, TypeVarType): - return sig.arg_types[0].id == sig.ret_type.id - return False - - -def returns_any_if_called(expr: Node) -> bool: - """Return True if we can predict that expr will return Any if called. - - This only uses information available during semantic analysis so this - will sometimes return False because of insufficient information (as - type inference hasn't run yet). - """ - if isinstance(expr, RefExpr): - if isinstance(expr.node, FuncDef): - typ = expr.node.type - if typ is None: - # No signature -> default to Any. - return True - # Explicit Any return? - return isinstance(typ, CallableType) and isinstance(typ.ret_type, AnyType) - elif isinstance(expr.node, Var): - typ = expr.node.type - return typ is None or isinstance(typ, AnyType) - elif isinstance(expr, CallExpr): - return returns_any_if_called(expr.callee) - return False - - -def find_fixed_callable_return(expr: Node) -> Optional[CallableType]: - if isinstance(expr, RefExpr): - if isinstance(expr.node, FuncDef): - typ = expr.node.type - if typ: - if isinstance(typ, CallableType) and has_no_typevars(typ.ret_type): - if isinstance(typ.ret_type, CallableType): - return typ.ret_type - elif isinstance(expr, CallExpr): - t = find_fixed_callable_return(expr.callee) - if t: - if isinstance(t.ret_type, CallableType): - return t.ret_type - return None - - -def has_no_typevars(typ: Type) -> bool: - return is_same_type(typ, erase_typevars(typ)) +"""The semantic analyzer. + +Bind names to definitions and do various other simple consistency +checks. For example, consider this program: + + x = 1 + y = x + +Here semantic analysis would detect that the assignment 'x = 1' +defines a new variable, the type of which is to be inferred (in a +later pass; type inference or type checking is not part of semantic +analysis). Also, it would bind both references to 'x' to the same +module-level variable node. The second assignment would also be +analyzed, and the type of 'y' marked as being inferred. + +Semantic analysis is the first analysis pass after parsing, and it is +subdivided into three passes: + + * FirstPass looks up externally visible names defined in a module but + ignores imports and local definitions. It helps enable (some) + cyclic references between modules, such as module 'a' that imports + module 'b' and used names defined in b *and* vice versa. The first + pass can be performed before dependent modules have been processed. + + * SemanticAnalyzer is the second pass. It does the bulk of the work. + It assumes that dependent modules have been semantically analyzed, + up to the second pass, unless there is a import cycle. + + * ThirdPass checks that type argument counts are valid; for example, + it will reject Dict[int]. We don't do this in the second pass, + since we infer the type argument counts of classes during this + pass, and it is possible to refer to classes defined later in a + file, which would not have the type argument count set yet. This + pass also recomputes the method resolution order of each class, in + case one of its bases belongs to a module involved in an import + loop. + +Semantic analysis of types is implemented in module mypy.typeanal. + +TODO: Check if the third pass slows down type checking significantly. + We could probably get rid of it -- for example, we could collect all + analyzed types in a collection and check them without having to + traverse the entire AST. +""" + +from typing import ( + List, Dict, Set, Tuple, cast, Any, overload, TypeVar, Union, Optional, Callable +) + +from mypy.nodes import ( + MypyFile, TypeInfo, Node, AssignmentStmt, FuncDef, OverloadedFuncDef, + ClassDef, Var, GDEF, MODULE_REF, FuncItem, Import, + ImportFrom, ImportAll, Block, LDEF, NameExpr, MemberExpr, + IndexExpr, TupleExpr, ListExpr, ExpressionStmt, ReturnStmt, + RaiseStmt, AssertStmt, OperatorAssignmentStmt, WhileStmt, + ForStmt, BreakStmt, ContinueStmt, IfStmt, TryStmt, WithStmt, DelStmt, + GlobalDecl, SuperExpr, DictExpr, CallExpr, RefExpr, OpExpr, UnaryExpr, + SliceExpr, CastExpr, RevealTypeExpr, TypeApplication, Context, SymbolTable, + SymbolTableNode, BOUND_TVAR, UNBOUND_TVAR, ListComprehension, GeneratorExpr, + FuncExpr, MDEF, FuncBase, Decorator, SetExpr, TypeVarExpr, + StrExpr, BytesExpr, PrintStmt, ConditionalExpr, PromoteExpr, + ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, MroError, type_aliases, + YieldFromExpr, NamedTupleExpr, NonlocalDecl, + SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr, + YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, COVARIANT, CONTRAVARIANT, + IntExpr, FloatExpr, UnicodeExpr, + INVARIANT, UNBOUND_IMPORTED, Expression, EllipsisExpr, NamedTupleTypeInfo +) +from mypy.visitor import NodeVisitor +from mypy.traverser import TraverserVisitor +from mypy.errors import Errors, report_internal_error +from mypy.types import ( + NoneTyp, CallableType, Overloaded, Instance, Type, TypeVarType, AnyType, + FunctionLike, UnboundType, TypeList, ErrorType, TypeVarDef, + replace_leading_arg_type, TupleType, UnionType, StarType, EllipsisType, + NamedTupleType +) +from mypy.nodes import function_type, implicit_module_attrs +from mypy.typeanal import TypeAnalyser, TypeAnalyserPass3, analyze_type_alias +from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError +from mypy.lex import lex +from mypy.parsetype import parse_type +from mypy.sametypes import is_same_type +from mypy.erasetype import erase_typevars +from mypy import defaults +from mypy.options import Options + + +T = TypeVar('T') + + +# Inferred value of an expression. +ALWAYS_TRUE = 0 +ALWAYS_FALSE = 1 +TRUTH_VALUE_UNKNOWN = 2 + +# Map from obsolete name to the current spelling. +obsolete_name_mapping = { + 'typing.Function': 'typing.Callable', + 'typing.typevar': 'typing.TypeVar', +} + +# Hard coded type promotions (shared between all Python versions). +# These add extra ad-hoc edges to the subtyping relation. For example, +# int is considered a subtype of float, even though there is no +# subclass relationship. +TYPE_PROMOTIONS = { + 'builtins.int': 'builtins.float', + 'builtins.float': 'builtins.complex', +} + +# Hard coded type promotions for Python 3. +# +# Note that the bytearray -> bytes promotion is a little unsafe +# as some functions only accept bytes objects. Here convenience +# trumps safety. +TYPE_PROMOTIONS_PYTHON3 = TYPE_PROMOTIONS.copy() +TYPE_PROMOTIONS_PYTHON3.update({ + 'builtins.bytearray': 'builtins.bytes', +}) + +# Hard coded type promotions for Python 2. +# +# These promotions are unsafe, but we are doing them anyway +# for convenience and also for Python 3 compatibility +# (bytearray -> str). +TYPE_PROMOTIONS_PYTHON2 = TYPE_PROMOTIONS.copy() +TYPE_PROMOTIONS_PYTHON2.update({ + 'builtins.str': 'builtins.unicode', + 'builtins.bytearray': 'builtins.str', +}) + +# When analyzing a function, should we analyze the whole function in one go, or +# should we only perform one phase of the analysis? The latter is used for +# nested functions. In the first phase we add the function to the symbol table +# but don't process body. In the second phase we process function body. This +# way we can have mutually recursive nested functions. +FUNCTION_BOTH_PHASES = 0 # Everthing in one go +FUNCTION_FIRST_PHASE_POSTPONE_SECOND = 1 # Add to symbol table but postpone body +FUNCTION_SECOND_PHASE = 2 # Only analyze body + + +class SemanticAnalyzer(NodeVisitor): + """Semantically analyze parsed mypy files. + + The analyzer binds names and does various consistency checks for a + parse tree. Note that type checking is performed as a separate + pass. + + This is the second phase of semantic analysis. + """ + + # Library search paths + lib_path = None # type: List[str] + # Module name space + modules = None # type: Dict[str, MypyFile] + # Global name space for current module + globals = None # type: SymbolTable + # Names declared using "global" (separate set for each scope) + global_decls = None # type: List[Set[str]] + # Names declated using "nonlocal" (separate set for each scope) + nonlocal_decls = None # type: List[Set[str]] + # Local names of function scopes; None for non-function scopes. + locals = None # type: List[SymbolTable] + # Nested block depths of scopes + block_depth = None # type: List[int] + # TypeInfo of directly enclosing class (or None) + type = None # type: TypeInfo + # Stack of outer classes (the second tuple item contains tvars). + type_stack = None # type: List[TypeInfo] + # Type variables that are bound by the directly enclosing class + bound_tvars = None # type: List[SymbolTableNode] + # Stack of type variables that were bound by outer classess + tvar_stack = None # type: List[List[SymbolTableNode]] + # Do weak type checking in this file + weak_opts = set() # type: Set[str] + + # Stack of functions being analyzed + function_stack = None # type: List[FuncItem] + # Stack of next available function type variable ids + next_function_tvar_id_stack = None # type: List[int] + + # Status of postponing analysis of nested function bodies. By using this we + # can have mutually recursive nested functions. Values are FUNCTION_x + # constants. Note that separate phasea are not used for methods. + postpone_nested_functions_stack = None # type: List[int] + # Postponed functions collected if + # postpone_nested_functions_stack[-1] == FUNCTION_FIRST_PHASE_POSTPONE_SECOND. + postponed_functions_stack = None # type: List[List[Node]] + + loop_depth = 0 # Depth of breakable loops + cur_mod_id = '' # Current module id (or None) (phase 2) + is_stub_file = False # Are we analyzing a stub file? + imports = None # type: Set[str] # Imported modules (during phase 2 analysis) + errors = None # type: Errors # Keeps track of generated errors + + def __init__(self, + lib_path: List[str], + errors: Errors, + options: Options) -> None: + """Construct semantic analyzer. + + Use lib_path to search for modules, and report analysis errors + using the Errors instance. + """ + self.locals = [None] + self.imports = set() + self.type = None + self.type_stack = [] + self.bound_tvars = None + self.tvar_stack = [] + self.function_stack = [] + self.next_function_tvar_id_stack = [-1] + self.block_depth = [0] + self.loop_depth = 0 + self.lib_path = lib_path + self.errors = errors + self.modules = {} + self.options = options + self.postpone_nested_functions_stack = [FUNCTION_BOTH_PHASES] + self.postponed_functions_stack = [] + self.all_exports = set() # type: Set[str] + + def visit_file(self, file_node: MypyFile, fnam: str) -> None: + self.errors.set_file(fnam) + self.cur_mod_node = file_node + self.cur_mod_id = file_node.fullname() + self.is_stub_file = fnam.lower().endswith('.pyi') + self.globals = file_node.names + self.weak_opts = file_node.weak_opts + + if 'builtins' in self.modules: + self.globals['__builtins__'] = SymbolTableNode( + MODULE_REF, self.modules['builtins'], self.cur_mod_id) + + for name in implicit_module_attrs: + v = self.globals[name].node + if isinstance(v, Var): + v.type = self.anal_type(v.type) + v.is_ready = True + + defs = file_node.defs + for d in defs: + self.accept(d) + + if self.cur_mod_id == 'builtins': + remove_imported_names_from_symtable(self.globals, 'builtins') + + if '__all__' in self.globals: + for name, g in self.globals.items(): + if name not in self.all_exports: + g.module_public = False + + def visit_func_def(self, defn: FuncDef) -> None: + phase_info = self.postpone_nested_functions_stack[-1] + if phase_info != FUNCTION_SECOND_PHASE: + self.function_stack.append(defn) + # First phase of analysis for function. + self.errors.push_function(defn.name()) + self.update_function_type_variables(defn) + self.errors.pop_function() + self.function_stack.pop() + + defn.is_conditional = self.block_depth[-1] > 0 + + # TODO(jukka): Figure out how to share the various cases. It doesn't + # make sense to have (almost) duplicate code (here and elsewhere) for + # 3 cases: module-level, class-level and local names. Maybe implement + # a common stack of namespaces. As the 3 kinds of namespaces have + # different semantics, this wouldn't always work, but it might still + # be a win. + if self.is_class_scope(): + # Method definition + defn.info = self.type + if not defn.is_decorated and not defn.is_overload: + if defn.name() in self.type.names: + # Redefinition. Conditional redefinition is okay. + n = self.type.names[defn.name()].node + if self.is_conditional_func(n, defn): + defn.original_def = cast(FuncDef, n) + else: + self.name_already_defined(defn.name(), defn) + self.type.names[defn.name()] = SymbolTableNode(MDEF, defn) + self.prepare_method_signature(defn) + elif self.is_func_scope(): + # Nested function + if not defn.is_decorated and not defn.is_overload: + if defn.name() in self.locals[-1]: + # Redefinition. Conditional redefinition is okay. + n = self.locals[-1][defn.name()].node + if self.is_conditional_func(n, defn): + defn.original_def = cast(FuncDef, n) + else: + self.name_already_defined(defn.name(), defn) + else: + self.add_local(defn, defn) + else: + # Top-level function + if not defn.is_decorated and not defn.is_overload: + symbol = self.globals.get(defn.name()) + if isinstance(symbol.node, FuncDef) and symbol.node != defn: + # This is redefinition. Conditional redefinition is okay. + original_def = symbol.node + if self.is_conditional_func(original_def, defn): + # Conditional function definition -- multiple defs are ok. + defn.original_def = original_def + else: + # Report error. + self.check_no_global(defn.name(), defn, True) + if phase_info == FUNCTION_FIRST_PHASE_POSTPONE_SECOND: + # Postpone this function (for the second phase). + self.postponed_functions_stack[-1].append(defn) + return + if phase_info != FUNCTION_FIRST_PHASE_POSTPONE_SECOND: + # Second phase of analysis for function. + self.errors.push_function(defn.name()) + self.analyze_function(defn) + self.errors.pop_function() + + def prepare_method_signature(self, func: FuncDef) -> None: + """Check basic signature validity and tweak annotation of self/cls argument.""" + # Only non-static methods are special. + if not func.is_static: + if not func.arguments: + self.fail('Method must have at least one argument', func) + elif func.type: + sig = cast(FunctionLike, func.type) + if func.is_class: + leading_type = self.class_type(self.type) + else: + leading_type = self_type(self.type) + func.type = replace_implicit_first_type(sig, leading_type) + + def is_conditional_func(self, previous: Node, new: FuncDef) -> bool: + """Does 'new' conditionally redefine 'previous'? + + We reject straight redefinitions of functions, as they are usually + a programming error. For example: + + . def f(): ... + . def f(): ... # Error: 'f' redefined + """ + return isinstance(previous, (FuncDef, Var)) and new.is_conditional + + def update_function_type_variables(self, defn: FuncDef) -> None: + """Make any type variables in the signature of defn explicit. + + Update the signature of defn to contain type variable definitions + if defn is generic. + """ + if defn.type: + functype = cast(CallableType, defn.type) + typevars = self.infer_type_variables(functype) + # Do not define a new type variable if already defined in scope. + typevars = [(name, tvar) for name, tvar in typevars + if not self.is_defined_type_var(name, defn)] + if typevars: + next_tvar_id = self.next_function_tvar_id() + defs = [TypeVarDef(tvar[0], next_tvar_id - i, + tvar[1].values, tvar[1].upper_bound, + tvar[1].variance) + for i, tvar in enumerate(typevars)] + functype.variables = defs + + def infer_type_variables(self, + type: CallableType) -> List[Tuple[str, TypeVarExpr]]: + """Return list of unique type variables referred to in a callable.""" + names = [] # type: List[str] + tvars = [] # type: List[TypeVarExpr] + for arg in type.arg_types + [type.ret_type]: + for name, tvar_expr in self.find_type_variables_in_type(arg): + if name not in names: + names.append(name) + tvars.append(tvar_expr) + return list(zip(names, tvars)) + + def find_type_variables_in_type( + self, type: Type) -> List[Tuple[str, TypeVarExpr]]: + """Return a list of all unique type variable references in type. + + This effectively does partial name binding, results of which are mostly thrown away. + """ + result = [] # type: List[Tuple[str, TypeVarExpr]] + if isinstance(type, UnboundType): + name = type.name + node = self.lookup_qualified(name, type) + if node and node.kind == UNBOUND_TVAR: + result.append((name, cast(TypeVarExpr, node.node))) + for arg in type.args: + result.extend(self.find_type_variables_in_type(arg)) + elif isinstance(type, TypeList): + for item in type.items: + result.extend(self.find_type_variables_in_type(item)) + elif isinstance(type, UnionType): + for item in type.items: + result.extend(self.find_type_variables_in_type(item)) + elif isinstance(type, AnyType): + pass + elif isinstance(type, EllipsisType) or isinstance(type, TupleType): + pass + else: + assert False, 'Unsupported type %s' % type + return result + + def is_defined_type_var(self, tvar: str, context: Node) -> bool: + return self.lookup_qualified(tvar, context).kind == BOUND_TVAR + + def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: + t = [] # type: List[CallableType] + for i, item in enumerate(defn.items): + # TODO support decorated overloaded functions properly + item.is_overload = True + item.func.is_overload = True + item.accept(self) + t.append(cast(CallableType, function_type(item.func, + self.builtin_type('builtins.function')))) + if item.func.is_property and i == 0: + # This defines a property, probably with a setter and/or deleter. + self.analyze_property_with_multi_part_definition(defn) + break + if not [dec for dec in item.decorators + if refers_to_fullname(dec, 'typing.overload')]: + self.fail("'overload' decorator expected", item) + + defn.type = Overloaded(t) + defn.type.line = defn.line + + if self.is_class_scope(): + self.type.names[defn.name()] = SymbolTableNode(MDEF, defn, + typ=defn.type) + defn.info = self.type + elif self.is_func_scope(): + self.add_local(defn, defn) + + def analyze_property_with_multi_part_definition(self, defn: OverloadedFuncDef) -> None: + """Analyze a property defined using multiple methods (e.g., using @x.setter). + + Assume that the first method (@property) has already been analyzed. + """ + defn.is_property = True + items = defn.items + for item in items[1:]: + if len(item.decorators) == 1: + node = item.decorators[0] + if isinstance(node, MemberExpr): + if node.name == 'setter': + # The first item represents the entire property. + defn.items[0].var.is_settable_property = True + # Get abstractness from the original definition. + item.func.is_abstract = items[0].func.is_abstract + else: + self.fail("Decorated property not supported", item) + item.func.accept(self) + + def next_function_tvar_id(self) -> int: + return self.next_function_tvar_id_stack[-1] + + def analyze_function(self, defn: FuncItem) -> None: + is_method = self.is_class_scope() + + tvarnodes = self.add_func_type_variables_to_symbol_table(defn) + next_function_tvar_id = min([self.next_function_tvar_id()] + + [n.tvar_def.id.raw_id - 1 for n in tvarnodes]) + self.next_function_tvar_id_stack.append(next_function_tvar_id) + + if defn.type: + # Signature must be analyzed in the surrounding scope so that + # class-level imported names and type variables are in scope. + defn.type = self.anal_type(defn.type) + self.check_function_signature(defn) + if isinstance(defn, FuncDef): + defn.type = set_callable_name(defn.type, defn) + for arg in defn.arguments: + if arg.initializer: + arg.initializer.accept(self) + self.function_stack.append(defn) + self.enter() + for arg in defn.arguments: + self.add_local(arg.variable, defn) + for arg in defn.arguments: + if arg.initialization_statement: + lvalue = arg.initialization_statement.lvalues[0] + lvalue.accept(self) + + # The first argument of a non-static, non-class method is like 'self' + # (though the name could be different), having the enclosing class's + # instance type. + if is_method and not defn.is_static and not defn.is_class and defn.arguments: + defn.arguments[0].variable.is_self = True + + # First analyze body of the function but ignore nested functions. + self.postpone_nested_functions_stack.append(FUNCTION_FIRST_PHASE_POSTPONE_SECOND) + self.postponed_functions_stack.append([]) + defn.body.accept(self) + + # Analyze nested functions (if any) as a second phase. + self.postpone_nested_functions_stack[-1] = FUNCTION_SECOND_PHASE + for postponed in self.postponed_functions_stack[-1]: + postponed.accept(self) + self.postpone_nested_functions_stack.pop() + self.postponed_functions_stack.pop() + + self.next_function_tvar_id_stack.pop() + disable_typevars(tvarnodes) + + self.leave() + self.function_stack.pop() + + def add_func_type_variables_to_symbol_table( + self, defn: FuncItem) -> List[SymbolTableNode]: + nodes = [] # type: List[SymbolTableNode] + if defn.type: + tt = defn.type + names = self.type_var_names() + items = cast(CallableType, tt).variables + for item in items: + name = item.name + if name in names: + self.name_already_defined(name, defn) + node = self.bind_type_var(name, item, defn) + nodes.append(node) + names.add(name) + return nodes + + def type_var_names(self) -> Set[str]: + if not self.type: + return set() + else: + return set(self.type.type_vars) + + def bind_type_var(self, fullname: str, tvar_def: TypeVarDef, + context: Context) -> SymbolTableNode: + node = self.lookup_qualified(fullname, context) + node.kind = BOUND_TVAR + node.tvar_def = tvar_def + return node + + def check_function_signature(self, fdef: FuncItem) -> None: + sig = cast(CallableType, fdef.type) + if len(sig.arg_types) < len(fdef.arguments): + self.fail('Type signature has too few arguments', fdef) + # Add dummy Any arguments to prevent crashes later. + extra_anys = [AnyType()] * (len(fdef.arguments) - len(sig.arg_types)) + sig.arg_types.extend(extra_anys) + elif len(sig.arg_types) > len(fdef.arguments): + self.fail('Type signature has too many arguments', fdef) + + def visit_class_def(self, defn: ClassDef) -> None: + self.clean_up_bases_and_infer_type_variables(defn) + self.setup_class_def_analysis(defn) + + self.bind_class_type_vars(defn) + + self.analyze_base_classes(defn) + self.analyze_metaclass(defn) + + for decorator in defn.decorators: + self.analyze_class_decorator(defn, decorator) + + self.enter_class(defn) + + self.setup_is_builtinclass(defn) + + # Analyze class body. + defn.defs.accept(self) + + self.calculate_abstract_status(defn.info) + self.setup_type_promotion(defn) + + self.leave_class() + self.unbind_class_type_vars() + + def enter_class(self, defn: ClassDef) -> None: + # Remember previous active class + self.type_stack.append(self.type) + self.locals.append(None) # Add class scope + self.block_depth.append(-1) # The class body increments this to 0 + self.postpone_nested_functions_stack.append(FUNCTION_BOTH_PHASES) + self.type = defn.info + + def leave_class(self) -> None: + """ Restore analyzer state. """ + self.postpone_nested_functions_stack.pop() + self.block_depth.pop() + self.locals.pop() + self.type = self.type_stack.pop() + + def bind_class_type_vars(self, defn: ClassDef) -> None: + """ Unbind type variables of previously active class and bind + the type variables for the active class. + """ + if self.bound_tvars: + disable_typevars(self.bound_tvars) + self.tvar_stack.append(self.bound_tvars) + self.bound_tvars = self.bind_class_type_variables_in_symbol_table(defn.info) + + def unbind_class_type_vars(self) -> None: + """ Unbind the active class' type vars and rebind the + type vars of the previously active class. + """ + disable_typevars(self.bound_tvars) + self.bound_tvars = self.tvar_stack.pop() + if self.bound_tvars: + enable_typevars(self.bound_tvars) + + def analyze_class_decorator(self, defn: ClassDef, decorator: Node) -> None: + decorator.accept(self) + + def setup_is_builtinclass(self, defn: ClassDef): + for decorator in defn.decorators: + if refers_to_fullname(decorator, 'typing.builtinclass'): + defn.is_builtinclass = True + if defn.fullname == 'builtins.object': + # Only 'object' is marked as a built-in class, as otherwise things elsewhere + # would break. We need a better way of dealing with built-in classes. + defn.is_builtinclass = True + + def calculate_abstract_status(self, typ: TypeInfo) -> None: + """Calculate abstract status of a class. + + Set is_abstract of the type to True if the type has an unimplemented + abstract attribute. Also compute a list of abstract attributes. + """ + concrete = set() # type: Set[str] + abstract = [] # type: List[str] + for base in typ.mro: + for name, symnode in base.names.items(): + node = symnode.node + if isinstance(node, OverloadedFuncDef): + # Unwrap an overloaded function definition. We can just + # check arbitrarily the first overload item. If the + # different items have a different abstract status, there + # should be an error reported elsewhere. + func = node.items[0] # type: Node + else: + func = node + if isinstance(func, Decorator): + fdef = func.func + if fdef.is_abstract and name not in concrete: + typ.is_abstract = True + abstract.append(name) + concrete.add(name) + typ.abstract_attributes = sorted(abstract) + + def setup_type_promotion(self, defn: ClassDef) -> None: + """Setup extra, ad-hoc subtyping relationships between classes (promotion). + + This includes things like 'int' being compatible with 'float'. + """ + promote_target = None # type: Type + for decorator in defn.decorators: + if isinstance(decorator, CallExpr): + analyzed = decorator.analyzed + if isinstance(analyzed, PromoteExpr): + # _promote class decorator (undocumented faeture). + promote_target = analyzed.type + if not promote_target: + promotions = (TYPE_PROMOTIONS_PYTHON3 if self.options.python_version[0] >= 3 + else TYPE_PROMOTIONS_PYTHON2) + if defn.fullname in promotions: + promote_target = self.named_type_or_none(promotions[defn.fullname]) + defn.info._promote = promote_target + + def clean_up_bases_and_infer_type_variables(self, defn: ClassDef) -> None: + """Remove extra base classes such as Generic and infer type vars. + + For example, consider this class: + + . class Foo(Bar, Generic[T]): ... + + Now we will remove Generic[T] from bases of Foo and infer that the + type variable 'T' is a type argument of Foo. + + Note that this is performed *before* semantic analysis. + """ + removed = [] # type: List[int] + type_vars = [] # type: List[TypeVarDef] + for i, base_expr in enumerate(defn.base_type_exprs): + try: + base = expr_to_unanalyzed_type(base_expr) + except TypeTranslationError: + # This error will be caught later. + continue + tvars = self.analyze_typevar_declaration(base) + if tvars is not None: + if type_vars: + self.fail('Duplicate Generic in bases', defn) + removed.append(i) + for j, (name, tvar_expr) in enumerate(tvars): + type_vars.append(TypeVarDef(name, j + 1, tvar_expr.values, + tvar_expr.upper_bound, tvar_expr.variance)) + if type_vars: + defn.type_vars = type_vars + if defn.info: + defn.info.type_vars = [tv.name for tv in type_vars] + for i in reversed(removed): + del defn.base_type_exprs[i] + + def analyze_typevar_declaration(self, t: Type) -> Optional[List[Tuple[str, TypeVarExpr]]]: + if not isinstance(t, UnboundType): + return None + unbound = t + sym = self.lookup_qualified(unbound.name, unbound) + if sym is None or sym.node is None: + return None + if sym.node.fullname() == 'typing.Generic': + tvars = [] # type: List[Tuple[str, TypeVarExpr]] + for arg in unbound.args: + tvar = self.analyze_unbound_tvar(arg) + if tvar: + tvars.append(tvar) + else: + self.fail('Free type variable expected in %s[...]' % + sym.node.name(), t) + return tvars + return None + + def analyze_unbound_tvar(self, t: Type) -> Tuple[str, TypeVarExpr]: + if not isinstance(t, UnboundType): + return None + unbound = t + sym = self.lookup_qualified(unbound.name, unbound) + if sym is not None and sym.kind == UNBOUND_TVAR: + return unbound.name, cast(TypeVarExpr, sym.node) + return None + + def setup_class_def_analysis(self, defn: ClassDef) -> None: + """Prepare for the analysis of a class definition.""" + if not defn.info: + defn.info = TypeInfo(SymbolTable(), defn) + defn.info._fullname = defn.info.name() + if self.is_func_scope() or self.type: + kind = MDEF + if self.is_func_scope(): + kind = LDEF + self.add_symbol(defn.name, SymbolTableNode(kind, defn.info), defn) + + def analyze_base_classes(self, defn: ClassDef) -> None: + """Analyze and set up base classes. + + This computes several attributes on the corresponding TypeInfo defn.info + related to the base classes: defn.info.bases, defn.info.mro, and + miscellaneous others (at least tuple_type, fallback_to_any, and is_enum.) + """ + + base_types = [] # type: List[Instance] + info = defn.info + for base_expr in defn.base_type_exprs: + try: + base = self.expr_to_analyzed_type(base_expr) + except TypeTranslationError: + self.fail('Invalid base class', base_expr) + info.fallback_to_any = True + continue + + if isinstance(base, TupleType): + if info.tuple_type: + self.fail("Class has two incompatible bases derived from tuple", defn) + defn.incompatible_baseclass = True + if (not self.is_stub_file + and not info.is_named_tuple + and base.fallback.type.fullname() == 'builtins.tuple'): + self.fail("Tuple[...] not supported as a base class outside a stub file", defn) + info.tuple_type = base + base_types.append(base.fallback) + elif isinstance(base, Instance): + base_types.append(base) + elif isinstance(base, AnyType): + info.fallback_to_any = True + else: + self.fail('Invalid base class', base_expr) + info.fallback_to_any = True + + # Add 'object' as implicit base if there is no other base class. + if (not base_types and defn.fullname != 'builtins.object'): + base_types.append(self.object_type()) + + info.bases = base_types + + # Calculate the MRO. It might be incomplete at this point if + # the bases of defn include classes imported from other + # modules in an import loop. We'll recompute it in ThirdPass. + if not self.verify_base_classes(defn): + info.mro = [] + return + calculate_class_mro(defn, self.fail) + # If there are cyclic imports, we may be missing 'object' in + # the MRO. Fix MRO if needed. + if info.mro and info.mro[-1].fullname() != 'builtins.object': + info.mro.append(self.object_type().type) + + def expr_to_analyzed_type(self, expr: Node) -> Type: + if isinstance(expr, CallExpr): + expr.accept(self) + info = self.check_namedtuple(expr) + if info is None: + # Some form of namedtuple is the only valid type that looks like a call + # expression. This isn't a valid type. + raise TypeTranslationError() + fallback = Instance(info, []) + return TupleType(info.tuple_type.items, fallback=fallback) + typ = expr_to_unanalyzed_type(expr) + return self.anal_type(typ) + + def verify_base_classes(self, defn: ClassDef) -> bool: + info = defn.info + for base in info.bases: + baseinfo = base.type + if self.is_base_class(info, baseinfo): + self.fail('Cycle in inheritance hierarchy', defn) + # Clear bases to forcefully get rid of the cycle. + info.bases = [] + if baseinfo.fullname() == 'builtins.bool': + self.fail("'%s' is not a valid base class" % + baseinfo.name(), defn) + return False + dup = find_duplicate(info.direct_base_classes()) + if dup: + self.fail('Duplicate base class "%s"' % dup.name(), defn) + return False + return True + + def is_base_class(self, t: TypeInfo, s: TypeInfo) -> bool: + """Determine if t is a base class of s (but do not use mro).""" + # Search the base class graph for t, starting from s. + worklist = [s] + visited = {s} + while worklist: + nxt = worklist.pop() + if nxt == t: + return True + for base in nxt.bases: + if base.type not in visited: + worklist.append(base.type) + visited.add(base.type) + return False + + def analyze_metaclass(self, defn: ClassDef) -> None: + if defn.metaclass: + sym = self.lookup_qualified(defn.metaclass, defn) + if sym is not None and not isinstance(sym.node, TypeInfo): + self.fail("Invalid metaclass '%s'" % defn.metaclass, defn) + + def object_type(self) -> Instance: + return self.named_type('__builtins__.object') + + def class_type(self, info: TypeInfo) -> Type: + # Construct a function type whose fallback is cls. + from mypy import checkmember # To avoid import cycle. + leading_type = checkmember.type_object_type(info, self.builtin_type) + if isinstance(leading_type, Overloaded): + # Overloaded __init__ is too complex to handle. Plus it's stubs only. + return AnyType() + else: + return leading_type + + def named_type(self, qualified_name: str, args: List[Type] = None) -> Instance: + sym = self.lookup_qualified(qualified_name, None) + return Instance(cast(TypeInfo, sym.node), args or []) + + def named_type_or_none(self, qualified_name: str) -> Instance: + sym = self.lookup_fully_qualified_or_none(qualified_name) + if not sym: + return None + return Instance(cast(TypeInfo, sym.node), []) + + def is_instance_type(self, t: Type) -> bool: + return isinstance(t, Instance) + + def bind_class_type_variables_in_symbol_table( + self, info: TypeInfo) -> List[SymbolTableNode]: + nodes = [] # type: List[SymbolTableNode] + for var, binder in zip(info.type_vars, info.defn.type_vars): + node = self.bind_type_var(var, binder, info) + nodes.append(node) + return nodes + + def visit_import(self, i: Import) -> None: + for id, as_id in i.ids: + if as_id is not None: + self.add_module_symbol(id, as_id, module_public=True, context=i) + else: + # Modules imported in a stub file without using 'as x' won't get exported when + # doing 'from m import *'. + module_public = not self.is_stub_file + base = id.split('.')[0] + self.add_module_symbol(base, base, module_public=module_public, + context=i) + + def add_module_symbol(self, id: str, as_id: str, module_public: bool, + context: Context) -> None: + if id in self.modules: + m = self.modules[id] + self.add_symbol(as_id, SymbolTableNode(MODULE_REF, m, self.cur_mod_id, + module_public=module_public), context) + else: + self.add_unknown_symbol(as_id, context) + + def visit_import_from(self, imp: ImportFrom) -> None: + import_id = self.correct_relative_import(imp) + if import_id in self.modules: + module = self.modules[import_id] + for id, as_id in imp.names: + node = module.names.get(id) + if node and node.kind != UNBOUND_IMPORTED: + node = self.normalize_type_alias(node, imp) + if not node: + return + imported_id = as_id or id + existing_symbol = self.globals.get(imported_id) + if existing_symbol: + # Import can redefine a variable. They get special treatment. + if self.process_import_over_existing_name( + imported_id, existing_symbol, node, imp): + continue + # 'from m import x as x' exports x in a stub file. + module_public = not self.is_stub_file or as_id is not None + symbol = SymbolTableNode(node.kind, node.node, + self.cur_mod_id, + node.type_override, + module_public=module_public) + self.add_symbol(imported_id, symbol, imp) + else: + message = "Module has no attribute '{}'".format(id) + extra = self.undefined_name_extra_info('{}.{}'.format(import_id, id)) + if extra: + message += " {}".format(extra) + self.fail(message, imp) + else: + # Missing module. + for id, as_id in imp.names: + self.add_unknown_symbol(as_id or id, imp) + + def process_import_over_existing_name(self, + imported_id: str, existing_symbol: SymbolTableNode, + module_symbol: SymbolTableNode, + import_node: ImportBase) -> bool: + if (existing_symbol.kind in (LDEF, GDEF, MDEF) and + isinstance(existing_symbol.node, (Var, FuncDef))): + # This is a valid import over an existing definition in the file. Construct a dummy + # assignment that we'll use to type check the import. + lvalue = NameExpr(imported_id) + lvalue.kind = existing_symbol.kind + lvalue.node = existing_symbol.node + rvalue = NameExpr(imported_id) + rvalue.kind = module_symbol.kind + rvalue.node = module_symbol.node + assignment = AssignmentStmt([lvalue], rvalue) + for node in assignment, lvalue, rvalue: + node.set_line(import_node) + import_node.assignments.append(assignment) + return True + return False + + def normalize_type_alias(self, node: SymbolTableNode, + ctx: Context) -> SymbolTableNode: + if node.fullname in type_aliases: + # Node refers to an aliased type such as typing.List; normalize. + node = self.lookup_qualified(type_aliases[node.fullname], ctx) + if node.fullname == 'typing.DefaultDict': + self.add_module_symbol('collections', '__mypy_collections__', False, ctx) + node = self.lookup_qualified('__mypy_collections__.defaultdict', ctx) + return node + + def correct_relative_import(self, node: Union[ImportFrom, ImportAll]) -> str: + if node.relative == 0: + return node.id + + parts = self.cur_mod_id.split(".") + cur_mod_id = self.cur_mod_id + + rel = node.relative + if self.cur_mod_node.is_package_init_file(): + rel -= 1 + if len(parts) < rel: + self.fail("Relative import climbs too many namespaces", node) + if rel != 0: + cur_mod_id = ".".join(parts[:-rel]) + + return cur_mod_id + (("." + node.id) if node.id else "") + + def visit_import_all(self, i: ImportAll) -> None: + i_id = self.correct_relative_import(i) + if i_id in self.modules: + m = self.modules[i_id] + for name, node in m.names.items(): + node = self.normalize_type_alias(node, i) + if not name.startswith('_') and node.module_public: + existing_symbol = self.globals.get(name) + if existing_symbol: + # Import can redefine a variable. They get special treatment. + if self.process_import_over_existing_name( + name, existing_symbol, node, i): + continue + self.add_symbol(name, SymbolTableNode(node.kind, node.node, + self.cur_mod_id), i) + else: + # Don't add any dummy symbols for 'from x import *' if 'x' is unknown. + pass + + def add_unknown_symbol(self, name: str, context: Context) -> None: + var = Var(name) + var._fullname = self.qualified_name(name) + var.is_ready = True + var.type = AnyType() + self.add_symbol(name, SymbolTableNode(GDEF, var, self.cur_mod_id), context) + + # + # Statements + # + + def visit_block(self, b: Block) -> None: + if b.is_unreachable: + return + self.block_depth[-1] += 1 + for s in b.body: + self.accept(s) + self.block_depth[-1] -= 1 + + def visit_block_maybe(self, b: Block) -> None: + if b: + self.visit_block(b) + + def anal_type(self, t: Type, allow_tuple_literal: bool = False) -> Type: + if t: + if allow_tuple_literal: + # Types such as (t1, t2, ...) only allowed in assignment statements. They'll + # generate errors elsewhere, and Tuple[t1, t2, ...] must be used instead. + if isinstance(t, TupleType): + # Unlike TypeAnalyser, also allow implicit tuple types (without Tuple[...]). + star_count = sum(1 for item in t.items if isinstance(item, StarType)) + if star_count > 1: + self.fail('At most one star type allowed in a tuple', t) + return TupleType([AnyType() for _ in t.items], + self.builtin_type('builtins.tuple'), t.line) + items = [self.anal_type(item, True) + for item in t.items] + return TupleType(items, self.builtin_type('builtins.tuple'), t.line) + a = TypeAnalyser(self.lookup_qualified, + self.lookup_fully_qualified, + self.fail) + return t.accept(a) + else: + return None + + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + for lval in s.lvalues: + self.analyze_lvalue(lval, explicit_type=s.type is not None) + s.rvalue.accept(self) + if s.type: + allow_tuple_literal = isinstance(s.lvalues[-1], (TupleExpr, ListExpr)) + s.type = self.anal_type(s.type, allow_tuple_literal) + else: + # For simple assignments, allow binding type aliases. + # Also set the type if the rvalue is a simple literal. + if (s.type is None and len(s.lvalues) == 1 and + isinstance(s.lvalues[0], NameExpr)): + if s.lvalues[0].is_def: + s.type = self.analyze_simple_literal_type(s.rvalue) + res = analyze_type_alias(s.rvalue, + self.lookup_qualified, + self.lookup_fully_qualified, + self.fail) + if res and (not isinstance(res, Instance) or cast(Instance, res).args): + # TODO: What if this gets reassigned? + name = s.lvalues[0] + node = self.lookup(name.name, name) + node.kind = TYPE_ALIAS + node.type_override = res + if isinstance(s.rvalue, IndexExpr): + s.rvalue.analyzed = TypeAliasExpr(res) + if s.type: + # Store type into nodes. + for lvalue in s.lvalues: + self.store_declared_types(lvalue, s.type) + self.check_and_set_up_type_alias(s) + self.process_typevar_declaration(s) + self.process_namedtuple_definition(s) + + if (len(s.lvalues) == 1 and isinstance(s.lvalues[0], NameExpr) and + s.lvalues[0].name == '__all__' and s.lvalues[0].kind == GDEF and + isinstance(s.rvalue, (ListExpr, TupleExpr))): + self.add_exports(*s.rvalue.items) + + def analyze_simple_literal_type(self, rvalue: Node) -> Optional[Type]: + """Return builtins.int if rvalue is an int literal, etc.""" + if self.weak_opts or self.options.semantic_analysis_only or self.function_stack: + # Skip this if any weak options are set. + # Also skip if we're only doing the semantic analysis pass. + # This is mostly to avoid breaking unit tests. + # Also skip inside a function; this is to avoid confusing + # the code that handles dead code due to isinstance() + # inside type variables with value restrictions (like + # AnyStr). + return None + if isinstance(rvalue, IntExpr): + return self.named_type_or_none('builtins.int') + if isinstance(rvalue, FloatExpr): + return self.named_type_or_none('builtins.float') + if isinstance(rvalue, StrExpr): + return self.named_type_or_none('builtins.str') + if isinstance(rvalue, BytesExpr): + return self.named_type_or_none('builtins.bytes') + if isinstance(rvalue, UnicodeExpr): + return self.named_type_or_none('builtins.unicode') + return None + + def check_and_set_up_type_alias(self, s: AssignmentStmt) -> None: + """Check if assignment creates a type alias and set it up as needed.""" + # For now, type aliases only work at the top level of a module. + if (len(s.lvalues) == 1 and not self.is_func_scope() and not self.type + and not s.type): + lvalue = s.lvalues[0] + if isinstance(lvalue, NameExpr): + if not lvalue.is_def: + # Only a definition can create a type alias, not regular assignment. + return + rvalue = s.rvalue + if isinstance(rvalue, RefExpr): + node = rvalue.node + if isinstance(node, TypeInfo): + # TODO: We should record the fact that this is a variable + # that refers to a type, rather than making this + # just an alias for the type. + self.globals[lvalue.name].node = node + + def analyze_lvalue(self, lval: Node, nested: bool = False, + add_global: bool = False, + explicit_type: bool = False) -> None: + """Analyze an lvalue or assignment target. + + Only if add_global is True, add name to globals table. If nested + is true, the lvalue is within a tuple or list lvalue expression. + """ + + if isinstance(lval, NameExpr): + # Top-level definitions within some statements (at least while) are + # not handled in the first pass, so they have to be added now. + nested_global = (not self.is_func_scope() and + self.block_depth[-1] > 0 and + not self.type) + if (add_global or nested_global) and lval.name not in self.globals: + # Define new global name. + v = Var(lval.name) + v._fullname = self.qualified_name(lval.name) + v.is_ready = False # Type not inferred yet + lval.node = v + lval.is_def = True + lval.kind = GDEF + lval.fullname = v._fullname + self.globals[lval.name] = SymbolTableNode(GDEF, v, + self.cur_mod_id) + elif isinstance(lval.node, Var) and lval.is_def: + # Since the is_def flag is set, this must have been analyzed + # already in the first pass and added to the symbol table. + assert lval.node.name() in self.globals + elif (self.is_func_scope() and lval.name not in self.locals[-1] and + lval.name not in self.global_decls[-1] and + lval.name not in self.nonlocal_decls[-1]): + # Define new local name. + v = Var(lval.name) + lval.node = v + lval.is_def = True + lval.kind = LDEF + lval.fullname = lval.name + self.add_local(v, lval) + elif not self.is_func_scope() and (self.type and + lval.name not in self.type.names): + # Define a new attribute within class body. + v = Var(lval.name) + v.info = self.type + v.is_initialized_in_class = True + v.set_line(lval) + lval.node = v + lval.is_def = True + lval.kind = MDEF + lval.fullname = lval.name + self.type.names[lval.name] = SymbolTableNode(MDEF, v) + else: + # Bind to an existing name. + if explicit_type: + self.name_already_defined(lval.name, lval) + lval.accept(self) + self.check_lvalue_validity(lval.node, lval) + elif isinstance(lval, MemberExpr): + if not add_global: + self.analyze_member_lvalue(lval) + if explicit_type and not self.is_self_member_ref(lval): + self.fail('Type cannot be declared in assignment to non-self ' + 'attribute', lval) + elif isinstance(lval, IndexExpr): + if explicit_type: + self.fail('Unexpected type declaration', lval) + if not add_global: + lval.accept(self) + elif (isinstance(lval, TupleExpr) or + isinstance(lval, ListExpr)): + items = cast(Any, lval).items + if len(items) == 0 and isinstance(lval, TupleExpr): + self.fail("Can't assign to ()", lval) + self.analyze_tuple_or_list_lvalue(cast(Union[ListExpr, TupleExpr], lval), + add_global, explicit_type) + elif isinstance(lval, StarExpr): + if nested: + self.analyze_lvalue(lval.expr, nested, add_global, explicit_type) + else: + self.fail('Starred assignment target must be in a list or tuple', lval) + else: + self.fail('Invalid assignment target', lval) + + def analyze_tuple_or_list_lvalue(self, lval: Union[ListExpr, TupleExpr], + add_global: bool = False, + explicit_type: bool = False) -> None: + """Analyze an lvalue or assignment target that is a list or tuple.""" + items = lval.items + star_exprs = [cast(StarExpr, item) + for item in items + if isinstance(item, StarExpr)] + + if len(star_exprs) > 1: + self.fail('Two starred expressions in assignment', lval) + else: + if len(star_exprs) == 1: + star_exprs[0].valid = True + for i in items: + self.analyze_lvalue(i, nested=True, add_global=add_global, + explicit_type = explicit_type) + + def analyze_member_lvalue(self, lval: MemberExpr) -> None: + lval.accept(self) + if (self.is_self_member_ref(lval) and + self.type.get(lval.name) is None): + # Implicit attribute definition in __init__. + lval.is_def = True + v = Var(lval.name) + v.info = self.type + v.is_ready = False + lval.def_var = v + lval.node = v + self.type.names[lval.name] = SymbolTableNode(MDEF, v) + self.check_lvalue_validity(lval.node, lval) + + def is_self_member_ref(self, memberexpr: MemberExpr) -> bool: + """Does memberexpr to refer to an attribute of self?""" + if not isinstance(memberexpr.expr, NameExpr): + return False + node = memberexpr.expr.node + return isinstance(node, Var) and node.is_self + + def check_lvalue_validity(self, node: Node, ctx: Context) -> None: + if isinstance(node, (TypeInfo, TypeVarExpr)): + self.fail('Invalid assignment target', ctx) + + def store_declared_types(self, lvalue: Node, typ: Type) -> None: + if isinstance(typ, StarType) and not isinstance(lvalue, StarExpr): + self.fail('Star type only allowed for starred expressions', lvalue) + if isinstance(lvalue, RefExpr): + lvalue.is_def = False + if isinstance(lvalue.node, Var): + var = lvalue.node + var.type = typ + var.is_ready = True + # If node is not a variable, we'll catch it elsewhere. + elif isinstance(lvalue, TupleExpr): + if isinstance(typ, TupleType): + if len(lvalue.items) != len(typ.items): + self.fail('Incompatible number of tuple items', lvalue) + return + for item, itemtype in zip(lvalue.items, typ.items): + self.store_declared_types(item, itemtype) + else: + self.fail('Tuple type expected for multiple variables', + lvalue) + elif isinstance(lvalue, StarExpr): + if isinstance(typ, StarType): + self.store_declared_types(lvalue.expr, typ.type) + else: + self.fail('Star type expected for starred expression', lvalue) + else: + # This has been flagged elsewhere as an error, so just ignore here. + pass + + def process_typevar_declaration(self, s: AssignmentStmt) -> None: + """Check if s declares a TypeVar; it yes, store it in symbol table.""" + call = self.get_typevar_declaration(s) + if not call: + return + + lvalue = cast(NameExpr, s.lvalues[0]) + name = lvalue.name + if not lvalue.is_def: + if s.type: + self.fail("Cannot declare the type of a type variable", s) + else: + self.fail("Cannot redefine '%s' as a type variable" % name, s) + return + + if not self.check_typevar_name(call, name, s): + return + + # Constraining types + n_values = call.arg_kinds[1:].count(ARG_POS) + values = self.analyze_types(call.args[1:1 + n_values]) + + res = self.process_typevar_parameters(call.args[1 + n_values:], + call.arg_names[1 + n_values:], + call.arg_kinds[1 + n_values:], + bool(values), + s) + if res is None: + return + variance, upper_bound = res + + # Yes, it's a valid type variable definition! Add it to the symbol table. + node = self.lookup(name, s) + node.kind = UNBOUND_TVAR + TypeVar = TypeVarExpr(name, node.fullname, values, upper_bound, variance) + TypeVar.line = call.line + call.analyzed = TypeVar + node.node = TypeVar + + def check_typevar_name(self, call: CallExpr, name: str, context: Context) -> bool: + if len(call.args) < 1: + self.fail("Too few arguments for TypeVar()", context) + return False + if not isinstance(call.args[0], (StrExpr, BytesExpr)) or not call.arg_kinds[0] == ARG_POS: + self.fail("TypeVar() expects a string literal as first argument", context) + return False + if cast(StrExpr, call.args[0]).value != name: + self.fail("Unexpected TypeVar() argument value", context) + return False + return True + + def get_typevar_declaration(self, s: AssignmentStmt) -> Optional[CallExpr]: + """Returns the TypeVar() call expression if `s` is a type var declaration + or None otherwise. + """ + if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): + return None + if not isinstance(s.rvalue, CallExpr): + return None + call = s.rvalue + if not isinstance(call.callee, RefExpr): + return None + callee = call.callee + if callee.fullname != 'typing.TypeVar': + return None + return call + + def process_typevar_parameters(self, args: List[Node], + names: List[Optional[str]], + kinds: List[int], + has_values: bool, + context: Context) -> Optional[Tuple[int, Type]]: + covariant = False + contravariant = False + upper_bound = self.object_type() # type: Type + for param_value, param_name, param_kind in zip(args, names, kinds): + if not param_kind == ARG_NAMED: + self.fail("Unexpected argument to TypeVar()", context) + return None + if param_name == 'covariant': + if isinstance(param_value, NameExpr): + if param_value.name == 'True': + covariant = True + else: + self.fail("TypeVar 'covariant' may only be 'True'", context) + return None + else: + self.fail("TypeVar 'covariant' may only be 'True'", context) + return None + elif param_name == 'contravariant': + if isinstance(param_value, NameExpr): + if param_value.name == 'True': + contravariant = True + else: + self.fail("TypeVar 'contravariant' may only be 'True'", context) + return None + else: + self.fail("TypeVar 'contravariant' may only be 'True'", context) + return None + elif param_name == 'bound': + if has_values: + self.fail("TypeVar cannot have both values and an upper bound", context) + return None + try: + upper_bound = self.expr_to_analyzed_type(param_value) + except TypeTranslationError: + self.fail("TypeVar 'bound' must be a type", param_value) + return None + elif param_name == 'values': + # Probably using obsolete syntax with values=(...). Explain the current syntax. + self.fail("TypeVar 'values' argument not supported", context) + self.fail("Use TypeVar('T', t, ...) instead of TypeVar('T', values=(t, ...))", + context) + return None + else: + self.fail("Unexpected argument to TypeVar(): {}".format(param_name), context) + return None + + if covariant and contravariant: + self.fail("TypeVar cannot be both covariant and contravariant", context) + return None + elif covariant: + variance = COVARIANT + elif contravariant: + variance = CONTRAVARIANT + else: + variance = INVARIANT + return (variance, upper_bound) + + def process_namedtuple_definition(self, s: AssignmentStmt) -> None: + """Check if s defines a namedtuple; if yes, store the definition in symbol table.""" + if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): + return + lvalue = cast(NameExpr, s.lvalues[0]) + name = lvalue.name + named_tuple = self.check_namedtuple(s.rvalue, name) + if named_tuple is None: + return + # Yes, it's a valid namedtuple definition. Add it to the symbol table. + node = self.lookup(name, s) + node.kind = GDEF # TODO locally defined namedtuple + # TODO call.analyzed + node.node = named_tuple + + def check_namedtuple(self, node: Node, var_name: str = None) -> TypeInfo: + """Check if a call defines a namedtuple. + + The optional var_name argument is the name of the variable to + which this is assigned, if any. + + If it does, return the corresponding TypeInfo. Return None otherwise. + + If the definition is invalid but looks like a namedtuple, + report errors but return (some) TypeInfo. + """ + if not isinstance(node, CallExpr): + return None + call = node + if not isinstance(call.callee, RefExpr): + return None + callee = call.callee + fullname = callee.fullname + if fullname not in ('collections.namedtuple', 'typing.NamedTuple'): + return None + items, types, ok = self.parse_namedtuple_args(call, fullname) + if not ok: + # Error. Construct dummy return value. + return self.build_namedtuple_typeinfo('namedtuple', [], []) + else: + # Give it a unique name derived from the line number. + name = cast(StrExpr, call.args[0]).value + if name != var_name: + name += '@' + str(call.line) + info = self.build_namedtuple_typeinfo(name, items, types) + # Store it as a global just in case it would remain anonymous. + self.globals[name] = SymbolTableNode(GDEF, info, self.cur_mod_id) + call.analyzed = NamedTupleExpr(info).set_line(call.line) + return info + + def parse_namedtuple_args(self, call: CallExpr, + fullname: str) -> Tuple[List[str], List[Type], bool]: + # TODO Share code with check_argument_count in checkexpr.py? + args = call.args + if len(args) < 2: + return self.fail_namedtuple_arg("Too few arguments for namedtuple()", call) + if len(args) > 2: + return self.fail_namedtuple_arg("Too many arguments for namedtuple()", call) + if call.arg_kinds != [ARG_POS, ARG_POS]: + return self.fail_namedtuple_arg("Unexpected arguments to namedtuple()", call) + if not isinstance(args[0], (StrExpr, BytesExpr)): + return self.fail_namedtuple_arg( + "namedtuple() expects a string literal as the first argument", call) + types = [] # type: List[Type] + ok = True + if not isinstance(args[1], ListExpr): + if fullname == 'collections.namedtuple' and isinstance(args[1], (StrExpr, BytesExpr)): + str_expr = cast(StrExpr, args[1]) + items = str_expr.value.split() + else: + return self.fail_namedtuple_arg( + "List literal expected as the second argument to namedtuple()", call) + else: + listexpr = args[1] + if fullname == 'collections.namedtuple': + # The fields argument contains just names, with implicit Any types. + if any(not isinstance(item, (StrExpr, BytesExpr)) for item in listexpr.items): + return self.fail_namedtuple_arg("String literal expected as namedtuple() item", + call) + items = [cast(StrExpr, item).value for item in listexpr.items] + else: + # The fields argument contains (name, type) tuples. + items, types, ok = self.parse_namedtuple_fields_with_types(listexpr.items, call) + if not types: + types = [AnyType() for _ in items] + return items, types, ok + + def parse_namedtuple_fields_with_types(self, nodes: List[Node], + context: Context) -> Tuple[List[str], List[Type], bool]: + items = [] # type: List[str] + types = [] # type: List[Type] + for item in nodes: + if isinstance(item, TupleExpr): + if len(item.items) != 2: + return self.fail_namedtuple_arg("Invalid NamedTuple field definition", + item) + name, type_node = item.items + if isinstance(name, (StrExpr, BytesExpr)): + items.append(name.value) + else: + return self.fail_namedtuple_arg("Invalid NamedTuple() field name", item) + try: + type = expr_to_unanalyzed_type(type_node) + except TypeTranslationError: + return self.fail_namedtuple_arg('Invalid field type', type_node) + types.append(self.anal_type(type)) + else: + return self.fail_namedtuple_arg("Tuple expected as NamedTuple() field", item) + return items, types, True + + def fail_namedtuple_arg(self, message: str, + context: Context) -> Tuple[List[str], List[Type], bool]: + self.fail(message, context) + return [], [], False + + def build_namedtuple_typeinfo(self, name: str, items: List[str], + types: List[Type]) -> TypeInfo: + symbols = SymbolTable() + tup = NamedTupleType(name, items, types, self.named_type('__builtins__.tuple', types)) + class_def = ClassDef(name, Block([])) + class_def.fullname = self.qualified_name(name) + info = NamedTupleTypeInfo(tup, symbols, class_def) + info.attrs = items + vars = [Var(item, typ) for item, typ in zip(items, types)] + # Add named tuple items as attributes. + # TODO: Make them read-only. + for var in vars: + var.info = info + symbols[var.name()] = SymbolTableNode(MDEF, var) + this_type = self_type(info) + # Add __init__, _replace, _asdict methods and _fields static field + self.add_namedtuple_method('_replace', symbols, info, this_type, this_type, + self.make_factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) + self.add_namedtuple_method('__init__', symbols, info, this_type, NoneTyp(), + self.make_factory_args(vars, ARG_POS), info.name()) + # TODO: refine to OrderedDict[Union[types]] + self.add_namedtuple_method('_asdict', symbols, info, this_type, AnyType(), []) + return info + + def make_factory_args(self, vars: List[Var], kind: int, + initializer: Expression = None) -> List[Argument]: + return [Argument(var, var.type, initializer, kind) for var in vars] + + def add_namedtuple_method(self, funcname: str, symbols: SymbolTable, info: TypeInfo, + this_type: Type, ret: Type, args: List[Argument], name=None) -> None: + args = [Argument(Var('self'), this_type, None, ARG_POS)] + args + types = [arg.type_annotation for arg in args] + items = [arg.variable.name() for arg in args] + arg_kinds = [arg.kind for arg in args] + # TODO: Make sure that the self argument name is not visible? + signature = CallableType(types, arg_kinds, items, ret, + self.named_type('__builtins__.function'), + name=name or info.name() + '.' + funcname) + func = FuncDef(funcname, + args, + Block([]), + typ=signature) + func.info = info + symbols[funcname] = SymbolTableNode(MDEF, func) + + def analyze_types(self, items: List[Node]) -> List[Type]: + result = [] # type: List[Type] + for node in items: + try: + result.append(self.anal_type(expr_to_unanalyzed_type(node))) + except TypeTranslationError: + self.fail('Type expected', node) + result.append(AnyType()) + return result + + def visit_decorator(self, dec: Decorator) -> None: + for d in dec.decorators: + d.accept(self) + removed = [] # type: List[int] + no_type_check = False + for i, d in enumerate(dec.decorators): + # A bunch of decorators are special cased here. + if refers_to_fullname(d, 'abc.abstractmethod'): + removed.append(i) + dec.func.is_abstract = True + self.check_decorated_function_is_method('abstractmethod', dec) + elif refers_to_fullname(d, 'asyncio.tasks.coroutine'): + removed.append(i) + elif refers_to_fullname(d, 'builtins.staticmethod'): + removed.append(i) + dec.func.is_static = True + dec.var.is_staticmethod = True + self.check_decorated_function_is_method('staticmethod', dec) + elif refers_to_fullname(d, 'builtins.classmethod'): + removed.append(i) + dec.func.is_class = True + dec.var.is_classmethod = True + self.check_decorated_function_is_method('classmethod', dec) + elif (refers_to_fullname(d, 'builtins.property') or + refers_to_fullname(d, 'abc.abstractproperty')): + removed.append(i) + dec.func.is_property = True + dec.var.is_property = True + if refers_to_fullname(d, 'abc.abstractproperty'): + dec.func.is_abstract = True + self.check_decorated_function_is_method('property', dec) + if len(dec.func.arguments) > 1: + self.fail('Too many arguments', dec.func) + elif refers_to_fullname(d, 'typing.no_type_check'): + dec.var.type = AnyType() + no_type_check = True + for i in reversed(removed): + del dec.decorators[i] + if not dec.is_overload or dec.var.is_property: + if self.is_func_scope(): + self.add_symbol(dec.var.name(), SymbolTableNode(LDEF, dec), + dec) + elif self.type: + dec.var.info = self.type + dec.var.is_initialized_in_class = True + self.add_symbol(dec.var.name(), SymbolTableNode(MDEF, dec), + dec) + if not no_type_check: + dec.func.accept(self) + if dec.decorators and dec.var.is_property: + self.fail('Decorated property not supported', dec) + + def check_decorated_function_is_method(self, decorator: str, + context: Context) -> None: + if not self.type or self.is_func_scope(): + self.fail("'%s' used with a non-method" % decorator, context) + + def visit_expression_stmt(self, s: ExpressionStmt) -> None: + s.expr.accept(self) + + def visit_return_stmt(self, s: ReturnStmt) -> None: + if not self.is_func_scope(): + self.fail("'return' outside function", s) + if s.expr: + s.expr.accept(self) + + def visit_raise_stmt(self, s: RaiseStmt) -> None: + if s.expr: + s.expr.accept(self) + if s.from_expr: + s.from_expr.accept(self) + + def visit_assert_stmt(self, s: AssertStmt) -> None: + if s.expr: + s.expr.accept(self) + + def visit_operator_assignment_stmt(self, + s: OperatorAssignmentStmt) -> None: + s.lvalue.accept(self) + s.rvalue.accept(self) + if (isinstance(s.lvalue, NameExpr) and s.lvalue.name == '__all__' and + s.lvalue.kind == GDEF and isinstance(s.rvalue, (ListExpr, TupleExpr))): + self.add_exports(*s.rvalue.items) + + def visit_while_stmt(self, s: WhileStmt) -> None: + s.expr.accept(self) + self.loop_depth += 1 + s.body.accept(self) + self.loop_depth -= 1 + self.visit_block_maybe(s.else_body) + + def visit_for_stmt(self, s: ForStmt) -> None: + s.expr.accept(self) + + # Bind index variables and check if they define new names. + self.analyze_lvalue(s.index) + + self.loop_depth += 1 + self.visit_block(s.body) + self.loop_depth -= 1 + + self.visit_block_maybe(s.else_body) + + def visit_break_stmt(self, s: BreakStmt) -> None: + if self.loop_depth == 0: + self.fail("'break' outside loop", s, True, blocker=True) + + def visit_continue_stmt(self, s: ContinueStmt) -> None: + if self.loop_depth == 0: + self.fail("'continue' outside loop", s, True, blocker=True) + + def visit_if_stmt(self, s: IfStmt) -> None: + infer_reachability_of_if_statement(s, pyversion=self.options.python_version) + for i in range(len(s.expr)): + s.expr[i].accept(self) + self.visit_block(s.body[i]) + self.visit_block_maybe(s.else_body) + + def visit_try_stmt(self, s: TryStmt) -> None: + self.analyze_try_stmt(s, self) + + def analyze_try_stmt(self, s: TryStmt, visitor: NodeVisitor, + add_global: bool = False) -> None: + s.body.accept(visitor) + for type, var, handler in zip(s.types, s.vars, s.handlers): + if type: + type.accept(visitor) + if var: + self.analyze_lvalue(var, add_global=add_global) + handler.accept(visitor) + if s.else_body: + s.else_body.accept(visitor) + if s.finally_body: + s.finally_body.accept(visitor) + + def visit_with_stmt(self, s: WithStmt) -> None: + for e, n in zip(s.expr, s.target): + e.accept(self) + if n: + self.analyze_lvalue(n) + self.visit_block(s.body) + + def visit_del_stmt(self, s: DelStmt) -> None: + s.expr.accept(self) + if not self.is_valid_del_target(s.expr): + self.fail('Invalid delete target', s) + + def is_valid_del_target(self, s: Node) -> bool: + if isinstance(s, (IndexExpr, NameExpr, MemberExpr)): + return True + elif isinstance(s, TupleExpr): + return all(self.is_valid_del_target(item) for item in s.items) + + def visit_global_decl(self, g: GlobalDecl) -> None: + for name in g.names: + if name in self.nonlocal_decls[-1]: + self.fail("Name '{}' is nonlocal and global".format(name), g) + self.global_decls[-1].add(name) + + def visit_nonlocal_decl(self, d: NonlocalDecl) -> None: + if not self.is_func_scope(): + self.fail("nonlocal declaration not allowed at module level", d) + else: + for name in d.names: + for table in reversed(self.locals[:-1]): + if table is not None and name in table: + break + else: + self.fail("No binding for nonlocal '{}' found".format(name), d) + + if self.locals[-1] is not None and name in self.locals[-1]: + self.fail("Name '{}' is already defined in local " + "scope before nonlocal declaration".format(name), d) + + if name in self.global_decls[-1]: + self.fail("Name '{}' is nonlocal and global".format(name), d) + self.nonlocal_decls[-1].add(name) + + def visit_print_stmt(self, s: PrintStmt) -> None: + for arg in s.args: + arg.accept(self) + if s.target: + s.target.accept(self) + + def visit_exec_stmt(self, s: ExecStmt) -> None: + s.expr.accept(self) + if s.variables1: + s.variables1.accept(self) + if s.variables2: + s.variables2.accept(self) + + # + # Expressions + # + + def visit_name_expr(self, expr: NameExpr) -> None: + n = self.lookup(expr.name, expr) + if n: + if n.kind == BOUND_TVAR: + self.fail("'{}' is a type variable and only valid in type " + "context".format(expr.name), expr) + else: + expr.kind = n.kind + expr.node = n.node + expr.fullname = n.fullname + + def visit_super_expr(self, expr: SuperExpr) -> None: + if not self.type: + self.fail('"super" used outside class', expr) + return + expr.info = self.type + + def visit_tuple_expr(self, expr: TupleExpr) -> None: + for item in expr.items: + item.accept(self) + + def visit_list_expr(self, expr: ListExpr) -> None: + for item in expr.items: + item.accept(self) + + def visit_set_expr(self, expr: SetExpr) -> None: + for item in expr.items: + item.accept(self) + + def visit_dict_expr(self, expr: DictExpr) -> None: + for key, value in expr.items: + key.accept(self) + value.accept(self) + + def visit_star_expr(self, expr: StarExpr) -> None: + if not expr.valid: + self.fail('Can use starred expression only as assignment target', expr) + else: + expr.expr.accept(self) + + def visit_yield_from_expr(self, e: YieldFromExpr) -> None: + if not self.is_func_scope(): # not sure + self.fail("'yield from' outside function", e, True, blocker=True) + else: + self.function_stack[-1].is_generator = True + if e.expr: + e.expr.accept(self) + + def visit_call_expr(self, expr: CallExpr) -> None: + """Analyze a call expression. + + Some call expressions are recognized as special forms, including + cast(...) and Any(...). + """ + expr.callee.accept(self) + if refers_to_fullname(expr.callee, 'typing.cast'): + # Special form cast(...). + if not self.check_fixed_args(expr, 2, 'cast'): + return + # Translate first argument to an unanalyzed type. + try: + target = expr_to_unanalyzed_type(expr.args[0]) + except TypeTranslationError: + self.fail('Cast target is not a type', expr) + return + # Piggyback CastExpr object to the CallExpr object; it takes + # precedence over the CallExpr semantics. + expr.analyzed = CastExpr(expr.args[1], target) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, 'builtins.reveal_type'): + if not self.check_fixed_args(expr, 1, 'reveal_type'): + return + expr.analyzed = RevealTypeExpr(expr.args[0]) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, 'typing.Any'): + # Special form Any(...). + if not self.check_fixed_args(expr, 1, 'Any'): + return + expr.analyzed = CastExpr(expr.args[0], AnyType()) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, 'typing._promote'): + # Special form _promote(...). + if not self.check_fixed_args(expr, 1, '_promote'): + return + # Translate first argument to an unanalyzed type. + try: + target = expr_to_unanalyzed_type(expr.args[0]) + except TypeTranslationError: + self.fail('Argument 1 to _promote is not a type', expr) + return + expr.analyzed = PromoteExpr(target) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, 'builtins.dict'): + expr.analyzed = self.translate_dict_call(expr) + else: + # Normal call expression. + for a in expr.args: + a.accept(self) + + if (isinstance(expr.callee, MemberExpr) and + isinstance(expr.callee.expr, NameExpr) and + expr.callee.expr.name == '__all__' and + expr.callee.expr.kind == GDEF and + expr.callee.name in ('append', 'extend')): + if expr.callee.name == 'append' and expr.args: + self.add_exports(expr.args[0]) + elif (expr.callee.name == 'extend' and expr.args and + isinstance(expr.args[0], (ListExpr, TupleExpr))): + self.add_exports(*expr.args[0].items) + + def translate_dict_call(self, call: CallExpr) -> Optional[DictExpr]: + """Translate 'dict(x=y, ...)' to {'x': y, ...}. + + For other variants of dict(...), return None. + """ + if not call.args: + return None + if not all(kind == ARG_NAMED for kind in call.arg_kinds): + # Must still accept those args. + for a in call.args: + a.accept(self) + return None + expr = DictExpr([(StrExpr(key), value) + for key, value in zip(call.arg_names, call.args)]) + expr.set_line(call) + expr.accept(self) + return expr + + def check_fixed_args(self, expr: CallExpr, numargs: int, + name: str) -> bool: + """Verify that expr has specified number of positional args. + + Return True if the arguments are valid. + """ + s = 's' + if numargs == 1: + s = '' + if len(expr.args) != numargs: + self.fail("'%s' expects %d argument%s" % (name, numargs, s), + expr) + return False + if expr.arg_kinds != [ARG_POS] * numargs: + self.fail("'%s' must be called with %s positional argument%s" % + (name, numargs, s), expr) + return False + return True + + def visit_member_expr(self, expr: MemberExpr) -> None: + base = expr.expr + base.accept(self) + # Bind references to module attributes. + if isinstance(base, RefExpr) and base.kind == MODULE_REF: + file = cast(MypyFile, base.node) + n = file.names.get(expr.name, None) if file is not None else None + if n: + n = self.normalize_type_alias(n, expr) + if not n: + return + expr.kind = n.kind + expr.fullname = n.fullname + expr.node = n.node + else: + # We only catch some errors here; the rest will be + # catched during type checking. + # + # This way we can report a larger number of errors in + # one type checker run. If we reported errors here, + # the build would terminate after semantic analysis + # and we wouldn't be able to report any type errors. + full_name = '%s.%s' % (file.fullname() if file is not None else None, expr.name) + if full_name in obsolete_name_mapping: + self.fail("Module has no attribute %r (it's now called %r)" % ( + expr.name, obsolete_name_mapping[full_name]), expr) + + def visit_op_expr(self, expr: OpExpr) -> None: + expr.left.accept(self) + expr.right.accept(self) + + def visit_comparison_expr(self, expr: ComparisonExpr) -> None: + for operand in expr.operands: + operand.accept(self) + + def visit_unary_expr(self, expr: UnaryExpr) -> None: + expr.expr.accept(self) + + def visit_index_expr(self, expr: IndexExpr) -> None: + expr.base.accept(self) + if refers_to_class_or_function(expr.base): + # Special form -- type application. + # Translate index to an unanalyzed type. + types = [] # type: List[Type] + if isinstance(expr.index, TupleExpr): + items = expr.index.items + else: + items = [expr.index] + for item in items: + try: + typearg = expr_to_unanalyzed_type(item) + except TypeTranslationError: + self.fail('Type expected within [...]', expr) + return + typearg = self.anal_type(typearg) + types.append(typearg) + expr.analyzed = TypeApplication(expr.base, types) + expr.analyzed.line = expr.line + else: + expr.index.accept(self) + + def visit_slice_expr(self, expr: SliceExpr) -> None: + if expr.begin_index: + expr.begin_index.accept(self) + if expr.end_index: + expr.end_index.accept(self) + if expr.stride: + expr.stride.accept(self) + + def visit_cast_expr(self, expr: CastExpr) -> None: + expr.expr.accept(self) + expr.type = self.anal_type(expr.type) + + def visit_reveal_type_expr(self, expr: RevealTypeExpr) -> None: + expr.expr.accept(self) + + def visit_type_application(self, expr: TypeApplication) -> None: + expr.expr.accept(self) + for i in range(len(expr.types)): + expr.types[i] = self.anal_type(expr.types[i]) + + def visit_list_comprehension(self, expr: ListComprehension) -> None: + expr.generator.accept(self) + + def visit_set_comprehension(self, expr: SetComprehension) -> None: + expr.generator.accept(self) + + def visit_dictionary_comprehension(self, expr: DictionaryComprehension) -> None: + self.enter() + self.analyze_comp_for(expr) + expr.key.accept(self) + expr.value.accept(self) + self.leave() + self.analyze_comp_for_2(expr) + + def visit_generator_expr(self, expr: GeneratorExpr) -> None: + self.enter() + self.analyze_comp_for(expr) + expr.left_expr.accept(self) + self.leave() + self.analyze_comp_for_2(expr) + + def analyze_comp_for(self, expr: Union[GeneratorExpr, + DictionaryComprehension]) -> None: + """Analyses the 'comp_for' part of comprehensions (part 1). + + That is the part after 'for' in (x for x in l if p). This analyzes + variables and conditions which are analyzed in a local scope. + """ + for i, (index, sequence, conditions) in enumerate(zip(expr.indices, + expr.sequences, + expr.condlists)): + if i > 0: + sequence.accept(self) + # Bind index variables. + self.analyze_lvalue(index) + for cond in conditions: + cond.accept(self) + + def analyze_comp_for_2(self, expr: Union[GeneratorExpr, + DictionaryComprehension]) -> None: + """Analyses the 'comp_for' part of comprehensions (part 2). + + That is the part after 'for' in (x for x in l if p). This analyzes + the 'l' part which is analyzed in the surrounding scope. + """ + expr.sequences[0].accept(self) + + def visit_func_expr(self, expr: FuncExpr) -> None: + self.analyze_function(expr) + + def visit_conditional_expr(self, expr: ConditionalExpr) -> None: + expr.if_expr.accept(self) + expr.cond.accept(self) + expr.else_expr.accept(self) + + def visit_backquote_expr(self, expr: BackquoteExpr) -> None: + expr.expr.accept(self) + + def visit__promote_expr(self, expr: PromoteExpr) -> None: + expr.type = self.anal_type(expr.type) + + def visit_yield_expr(self, expr: YieldExpr) -> None: + if not self.is_func_scope(): + self.fail("'yield' outside function", expr, True, blocker=True) + else: + self.function_stack[-1].is_generator = True + if expr.expr: + expr.expr.accept(self) + + # + # Helpers + # + + def lookup(self, name: str, ctx: Context) -> SymbolTableNode: + """Look up an unqualified name in all active namespaces.""" + # 1a. Name declared using 'global x' takes precedence + if name in self.global_decls[-1]: + if name in self.globals: + return self.globals[name] + else: + self.name_not_defined(name, ctx) + return None + # 1b. Name declared using 'nonlocal x' takes precedence + if name in self.nonlocal_decls[-1]: + for table in reversed(self.locals[:-1]): + if table is not None and name in table: + return table[name] + else: + self.name_not_defined(name, ctx) + return None + # 2. Class attributes (if within class definition) + if self.is_class_scope() and name in self.type.names: + return self.type[name] + # 3. Local (function) scopes + for table in reversed(self.locals): + if table is not None and name in table: + return table[name] + # 4. Current file global scope + if name in self.globals: + return self.globals[name] + # 5. Builtins + b = self.globals.get('__builtins__', None) + if b: + table = cast(MypyFile, b.node).names + if name in table: + if name[0] == "_" and name[1] != "_": + self.name_not_defined(name, ctx) + return None + node = table[name] + return node + # Give up. + self.name_not_defined(name, ctx) + self.check_for_obsolete_short_name(name, ctx) + return None + + def check_for_obsolete_short_name(self, name: str, ctx: Context) -> None: + matches = [obsolete_name + for obsolete_name in obsolete_name_mapping + if obsolete_name.rsplit('.', 1)[-1] == name] + if len(matches) == 1: + self.note("(Did you mean '{}'?)".format(obsolete_name_mapping[matches[0]]), ctx) + + def lookup_qualified(self, name: str, ctx: Context) -> SymbolTableNode: + if '.' not in name: + return self.lookup(name, ctx) + else: + parts = name.split('.') + n = self.lookup(parts[0], ctx) # type: SymbolTableNode + if n: + for i in range(1, len(parts)): + if isinstance(n.node, TypeInfo): + if n.node.mro is None: + # We haven't yet analyzed the class `n.node`. Fall back to direct + # lookup in the names declared directly under it, without its base + # classes. This can happen when we have a forward reference to a + # nested class, and the reference is bound before the outer class + # has been fully semantically analyzed. + # + # A better approach would be to introduce a new analysis pass or + # to move things around between passes, but this unblocks a common + # use case even though this is a little limited in case there is + # inheritance involved. + result = n.node.names.get(parts[i]) + else: + result = n.node.get(parts[i]) + n = result + elif isinstance(n.node, MypyFile): + n = n.node.names.get(parts[i], None) + # TODO: What if node is Var or FuncDef? + if not n: + self.name_not_defined(name, ctx) + break + if n: + n = self.normalize_type_alias(n, ctx) + return n + + def builtin_type(self, fully_qualified_name: str) -> Instance: + node = self.lookup_fully_qualified(fully_qualified_name) + info = cast(TypeInfo, node.node) + return Instance(info, []) + + def lookup_fully_qualified(self, name: str) -> SymbolTableNode: + """Lookup a fully qualified name. + + Assume that the name is defined. This happens in the global namespace -- the local + module namespace is ignored. + """ + assert '.' in name + parts = name.split('.') + n = self.modules[parts[0]] + for i in range(1, len(parts) - 1): + n = cast(MypyFile, n.names[parts[i]].node) + return n.names[parts[-1]] + + def lookup_fully_qualified_or_none(self, name: str) -> SymbolTableNode: + """Lookup a fully qualified name. + + Assume that the name is defined. This happens in the global namespace -- the local + module namespace is ignored. + """ + assert '.' in name + parts = name.split('.') + n = self.modules[parts[0]] + for i in range(1, len(parts) - 1): + next_sym = n.names.get(parts[i]) + if not next_sym: + return None + n = cast(MypyFile, next_sym.node) + return n.names.get(parts[-1]) + + def qualified_name(self, n: str) -> str: + return self.cur_mod_id + '.' + n + + def enter(self) -> None: + self.locals.append(SymbolTable()) + self.global_decls.append(set()) + self.nonlocal_decls.append(set()) + # -1 since entering block will increment this to 0. + self.block_depth.append(-1) + + def leave(self) -> None: + self.locals.pop() + self.global_decls.pop() + self.nonlocal_decls.pop() + self.block_depth.pop() + + def is_func_scope(self) -> bool: + return self.locals[-1] is not None + + def is_class_scope(self) -> bool: + return self.type is not None and not self.is_func_scope() + + def add_symbol(self, name: str, node: SymbolTableNode, + context: Context) -> None: + if self.is_func_scope(): + if name in self.locals[-1]: + # Flag redefinition unless this is a reimport of a module. + if not (node.kind == MODULE_REF and + self.locals[-1][name].node == node.node): + self.name_already_defined(name, context) + self.locals[-1][name] = node + elif self.type: + self.type.names[name] = node + else: + existing = self.globals.get(name) + if existing and (not isinstance(node.node, MypyFile) or + existing.node != node.node) and existing.kind != UNBOUND_IMPORTED: + # Modules can be imported multiple times to support import + # of multiple submodules of a package (e.g. a.x and a.y). + ok = False + # Only report an error if the symbol collision provides a different type. + if existing.type and node.type and is_same_type(existing.type, node.type): + ok = True + if not ok: + self.name_already_defined(name, context) + self.globals[name] = node + + def add_var(self, v: Var, ctx: Context) -> None: + if self.is_func_scope(): + self.add_local(v, ctx) + else: + self.globals[v.name()] = SymbolTableNode(GDEF, v, self.cur_mod_id) + v._fullname = self.qualified_name(v.name()) + + def add_local(self, node: Union[Var, FuncBase], ctx: Context) -> None: + name = node.name() + if name in self.locals[-1]: + self.name_already_defined(name, ctx) + node._fullname = name + self.locals[-1][name] = SymbolTableNode(LDEF, node) + + def add_exports(self, *exps: Node) -> None: + for exp in exps: + if isinstance(exp, StrExpr): + self.all_exports.add(exp.value) + + def check_no_global(self, n: str, ctx: Context, + is_overloaded_func: bool = False) -> None: + if n in self.globals: + prev_is_overloaded = isinstance(self.globals[n], OverloadedFuncDef) + if is_overloaded_func and prev_is_overloaded: + self.fail("Nonconsecutive overload {} found".format(n), ctx) + elif prev_is_overloaded: + self.fail("Definition of '{}' missing 'overload'".format(n), ctx) + else: + self.name_already_defined(n, ctx) + + def name_not_defined(self, name: str, ctx: Context) -> None: + message = "Name '{}' is not defined".format(name) + extra = self.undefined_name_extra_info(name) + if extra: + message += ' {}'.format(extra) + self.fail(message, ctx) + + def name_already_defined(self, name: str, ctx: Context) -> None: + self.fail("Name '{}' already defined".format(name), ctx) + + def fail(self, msg: str, ctx: Context, serious: bool = False, *, + blocker: bool = False) -> None: + if (not serious and + not self.options.check_untyped_defs and + self.function_stack and + self.function_stack[-1].is_dynamic()): + return + self.errors.report(ctx.get_line(), msg, blocker=blocker) + + def note(self, msg: str, ctx: Context) -> None: + if (not self.options.check_untyped_defs and + self.function_stack and + self.function_stack[-1].is_dynamic()): + return + self.errors.report(ctx.get_line(), msg, severity='note') + + def undefined_name_extra_info(self, fullname: str) -> Optional[str]: + if fullname in obsolete_name_mapping: + return "(it's now called '{}')".format(obsolete_name_mapping[fullname]) + else: + return None + + def accept(self, node: Node) -> None: + try: + node.accept(self) + except Exception as err: + report_internal_error(err, self.errors.file, node.line) + + +class FirstPass(NodeVisitor): + """First phase of semantic analysis. + + See docstring of 'analyze()' below for a description of what this does. + """ + + def __init__(self, sem: SemanticAnalyzer) -> None: + self.sem = sem + self.pyversion = sem.options.python_version + + def analyze(self, file: MypyFile, fnam: str, mod_id: str) -> None: + """Perform the first analysis pass. + + Populate module global table. Resolve the full names of + definitions not nested within functions and construct type + info structures, but do not resolve inter-definition + references such as base classes. + + Also add implicit definitions such as __name__. + + In this phase we don't resolve imports. For 'from ... import', + we generate dummy symbol table nodes for the imported names, + and these will get resolved in later phases of semantic + analysis. + """ + sem = self.sem + sem.cur_mod_id = mod_id + sem.errors.set_file(fnam) + sem.globals = SymbolTable() + sem.global_decls = [set()] + sem.nonlocal_decls = [set()] + sem.block_depth = [0] + + defs = file.defs + + # Add implicit definitions of module '__name__' etc. + for name, t in implicit_module_attrs.items(): + v = Var(name, UnboundType(t)) + v._fullname = self.sem.qualified_name(name) + self.sem.globals[name] = SymbolTableNode(GDEF, v, self.sem.cur_mod_id) + + for d in defs: + d.accept(self) + + # Add implicit definition of literals/keywords to builtins, as we + # cannot define a variable with them explicitly. + if mod_id == 'builtins': + literal_types = [ + ('None', NoneTyp()), + # reveal_type is a mypy-only function that gives an error with the type of its arg + ('reveal_type', AnyType()), + ] # type: List[Tuple[str, Type]] + + # TODO(ddfisher): This guard is only needed because mypy defines + # fake builtins for its tests which often don't define bool. If + # mypy is fast enough that we no longer need those, this + # conditional check should be removed. + if 'bool' in self.sem.globals: + bool_type = self.sem.named_type('bool') + literal_types.extend([ + ('True', bool_type), + ('False', bool_type), + ('__debug__', bool_type), + ]) + + for name, typ in literal_types: + v = Var(name, typ) + v._fullname = self.sem.qualified_name(name) + self.sem.globals[name] = SymbolTableNode(GDEF, v, self.sem.cur_mod_id) + + def visit_block(self, b: Block) -> None: + if b.is_unreachable: + return + self.sem.block_depth[-1] += 1 + for node in b.body: + node.accept(self) + self.sem.block_depth[-1] -= 1 + + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + for lval in s.lvalues: + self.analyze_lvalue(lval, explicit_type=s.type is not None) + + def visit_func_def(self, func: FuncDef) -> None: + sem = self.sem + func.is_conditional = sem.block_depth[-1] > 0 + func._fullname = sem.qualified_name(func.name()) + if func.name() in sem.globals: + # Already defined in this module. + original_sym = sem.globals[func.name()] + if original_sym.kind == UNBOUND_IMPORTED: + # Ah this is an imported name. We can't resolve them now, so we'll postpone + # this until the main phase of semantic analysis. + return + original_def = original_sym.node + if sem.is_conditional_func(original_def, func): + # Conditional function definition -- multiple defs are ok. + func.original_def = cast(FuncDef, original_def) + else: + # Report error. + sem.check_no_global(func.name(), func) + else: + sem.globals[func.name()] = SymbolTableNode(GDEF, func, sem.cur_mod_id) + + def visit_overloaded_func_def(self, func: OverloadedFuncDef) -> None: + self.sem.check_no_global(func.name(), func, True) + func._fullname = self.sem.qualified_name(func.name()) + self.sem.globals[func.name()] = SymbolTableNode(GDEF, func, + self.sem.cur_mod_id) + + def visit_class_def(self, cdef: ClassDef) -> None: + self.sem.check_no_global(cdef.name, cdef) + cdef.fullname = self.sem.qualified_name(cdef.name) + info = TypeInfo(SymbolTable(), cdef) + info.set_line(cdef.line) + cdef.info = info + self.sem.globals[cdef.name] = SymbolTableNode(GDEF, info, + self.sem.cur_mod_id) + self.process_nested_classes(cdef) + + def process_nested_classes(self, outer_def: ClassDef) -> None: + for node in outer_def.defs.body: + if isinstance(node, ClassDef): + node.info = TypeInfo(SymbolTable(), node) + if outer_def.fullname: + node.info._fullname = outer_def.fullname + '.' + node.info.name() + else: + node.info._fullname = node.info.name() + symbol = SymbolTableNode(MDEF, node.info) + outer_def.info.names[node.name] = symbol + self.process_nested_classes(node) + + def visit_import_from(self, node: ImportFrom) -> None: + # We can't bind module names during the first pass, as the target module might be + # unprocessed. However, we add dummy unbound imported names to the symbol table so + # that we at least know that the name refers to a module. + node.is_top_level = True + for name, as_name in node.names: + imported_name = as_name or name + if imported_name not in self.sem.globals: + self.sem.add_symbol(imported_name, SymbolTableNode(UNBOUND_IMPORTED, None), node) + + def visit_import(self, node: Import) -> None: + node.is_top_level = True + # This is similar to visit_import_from -- see the comment there. + for id, as_id in node.ids: + imported_id = as_id or id + if imported_id not in self.sem.globals: + self.sem.add_symbol(imported_id, SymbolTableNode(UNBOUND_IMPORTED, None), node) + else: + # If the previous symbol is a variable, this should take precedence. + self.sem.globals[imported_id] = SymbolTableNode(UNBOUND_IMPORTED, None) + + def visit_import_all(self, node: ImportAll) -> None: + node.is_top_level = True + + def visit_while_stmt(self, s: WhileStmt) -> None: + s.body.accept(self) + if s.else_body: + s.else_body.accept(self) + + def visit_for_stmt(self, s: ForStmt) -> None: + self.analyze_lvalue(s.index) + s.body.accept(self) + if s.else_body: + s.else_body.accept(self) + + def visit_with_stmt(self, s: WithStmt) -> None: + for n in s.target: + if n: + self.analyze_lvalue(n) + s.body.accept(self) + + def visit_decorator(self, d: Decorator) -> None: + d.var._fullname = self.sem.qualified_name(d.var.name()) + self.sem.add_symbol(d.var.name(), SymbolTableNode(GDEF, d.var), d) + + def visit_if_stmt(self, s: IfStmt) -> None: + infer_reachability_of_if_statement(s, pyversion=self.pyversion) + for node in s.body: + node.accept(self) + if s.else_body: + s.else_body.accept(self) + + def visit_try_stmt(self, s: TryStmt) -> None: + self.sem.analyze_try_stmt(s, self, add_global=True) + + def analyze_lvalue(self, lvalue: Node, explicit_type: bool = False) -> None: + self.sem.analyze_lvalue(lvalue, add_global=True, explicit_type=explicit_type) + + +class ThirdPass(TraverserVisitor): + """The third and final pass of semantic analysis. + + Check type argument counts and values of generic types, and perform some + straightforward type inference. + """ + + def __init__(self, modules: Dict[str, MypyFile], errors: Errors) -> None: + self.modules = modules + self.errors = errors + + def visit_file(self, file_node: MypyFile, fnam: str) -> None: + self.errors.set_file(fnam) + self.accept(file_node) + + def accept(self, node: Node) -> None: + try: + node.accept(self) + except Exception as err: + report_internal_error(err, self.errors.file, node.line) + + def visit_block(self, b: Block) -> None: + if b.is_unreachable: + return + super().visit_block(b) + + def visit_func_def(self, fdef: FuncDef) -> None: + self.errors.push_function(fdef.name()) + self.analyze(fdef.type) + super().visit_func_def(fdef) + self.errors.pop_function() + + def visit_class_def(self, tdef: ClassDef) -> None: + for type in tdef.info.bases: + self.analyze(type) + # Recompute MRO now that we have analyzed all modules, to pick + # up superclasses of bases imported from other modules in an + # import loop. (Only do so if we succeeded the first time.) + if tdef.info.mro: + tdef.info.mro = [] # Force recomputation + calculate_class_mro(tdef, self.fail) + super().visit_class_def(tdef) + + def visit_decorator(self, dec: Decorator) -> None: + """Try to infer the type of the decorated function. + + This lets us resolve references to decorated functions during + type checking when there are cyclic imports, as otherwise the + type might not be available when we need it. + + This basically uses a simple special-purpose type inference + engine just for decorators. + """ + super().visit_decorator(dec) + if dec.var.is_property: + # Decorators are expected to have a callable type (it's a little odd). + if dec.func.type is None: + dec.var.type = CallableType( + [AnyType()], + [ARG_POS], + [None], + AnyType(), + self.builtin_type('function'), + name=dec.var.name()) + elif isinstance(dec.func.type, CallableType): + dec.var.type = dec.func.type + return + decorator_preserves_type = True + for expr in dec.decorators: + preserve_type = False + if isinstance(expr, RefExpr) and isinstance(expr.node, FuncDef): + if is_identity_signature(expr.node.type): + preserve_type = True + if not preserve_type: + decorator_preserves_type = False + break + if decorator_preserves_type: + # No non-identity decorators left. We can trivially infer the type + # of the function here. + dec.var.type = function_type(dec.func, self.builtin_type('function')) + if dec.decorators: + if returns_any_if_called(dec.decorators[0]): + # The outermost decorator will return Any so we know the type of the + # decorated function. + dec.var.type = AnyType() + sig = find_fixed_callable_return(dec.decorators[0]) + if sig: + # The outermost decorator always returns the same kind of function, + # so we know that this is the type of the decoratored function. + orig_sig = function_type(dec.func, self.builtin_type('function')) + sig.name = orig_sig.items()[0].name + dec.var.type = sig + + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + self.analyze(s.type) + super().visit_assignment_stmt(s) + + def visit_cast_expr(self, e: CastExpr) -> None: + self.analyze(e.type) + super().visit_cast_expr(e) + + def visit_reveal_type_expr(self, e: RevealTypeExpr) -> None: + super().visit_reveal_type_expr(e) + + def visit_type_application(self, e: TypeApplication) -> None: + for type in e.types: + self.analyze(type) + super().visit_type_application(e) + + # Helpers + + def analyze(self, type: Type) -> None: + if type: + analyzer = TypeAnalyserPass3(self.fail) + type.accept(analyzer) + + def fail(self, msg: str, ctx: Context) -> None: + self.errors.report(ctx.get_line(), msg) + + def builtin_type(self, name: str, args: List[Type] = None) -> Instance: + names = self.modules['builtins'] + sym = names.names[name] + assert isinstance(sym.node, TypeInfo) + return Instance(sym.node, args or []) + + +def self_type(typ: TypeInfo) -> Union[Instance, TupleType]: + """For a non-generic type, return instance type representing the type. + For a generic G type with parameters T1, .., Tn, return G[T1, ..., Tn]. + """ + tv = [] # type: List[Type] + for i in range(len(typ.type_vars)): + tv.append(TypeVarType(typ.defn.type_vars[i])) + inst = Instance(typ, tv) + if typ.tuple_type is None: + return inst + return typ.tuple_type.self_type(inst) + + +def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: + if isinstance(sig, CallableType): + return replace_leading_arg_type(sig, new) + else: + sig = cast(Overloaded, sig) + return Overloaded([cast(CallableType, replace_implicit_first_type(i, new)) + for i in sig.items()]) + + +def set_callable_name(sig: Type, fdef: FuncDef) -> Type: + if isinstance(sig, FunctionLike): + if fdef.info: + return sig.with_name( + '"{}" of "{}"'.format(fdef.name(), fdef.info.name())) + else: + return sig.with_name('"{}"'.format(fdef.name())) + else: + return sig + + +def refers_to_fullname(node: Node, fullname: str) -> bool: + """Is node a name or member expression with the given full name?""" + return isinstance(node, RefExpr) and node.fullname == fullname + + +def refers_to_class_or_function(node: Node) -> bool: + """Does semantically analyzed node refer to a class?""" + return (isinstance(node, RefExpr) and + isinstance(node.node, (TypeInfo, FuncDef, OverloadedFuncDef))) + + +def calculate_class_mro(defn: ClassDef, fail: Callable[[str, Context], None]) -> None: + try: + defn.info.calculate_mro() + except MroError: + fail("Cannot determine consistent method resolution order " + '(MRO) for "%s"' % defn.name, defn) + defn.info.mro = [] + # The property of falling back to Any is inherited. + defn.info.fallback_to_any = any(baseinfo.fallback_to_any for baseinfo in defn.info.mro) + + +def find_duplicate(list: List[T]) -> T: + """If the list has duplicates, return one of the duplicates. + + Otherwise, return None. + """ + for i in range(1, len(list)): + if list[i] in list[:i]: + return list[i] + return None + + +def disable_typevars(nodes: List[SymbolTableNode]) -> None: + for node in nodes: + assert node.kind in (BOUND_TVAR, UNBOUND_TVAR) + node.kind = UNBOUND_TVAR + + +def enable_typevars(nodes: List[SymbolTableNode]) -> None: + for node in nodes: + assert node.kind in (BOUND_TVAR, UNBOUND_TVAR) + node.kind = BOUND_TVAR + + +def remove_imported_names_from_symtable(names: SymbolTable, + module: str) -> None: + """Remove all imported names from the symbol table of a module.""" + removed = [] # type: List[str] + for name, node in names.items(): + if node.node is None: + continue + fullname = node.node.fullname() + prefix = fullname[:fullname.rfind('.')] + if prefix != module: + removed.append(name) + for name in removed: + del names[name] + + +def infer_reachability_of_if_statement(s: IfStmt, + pyversion: Tuple[int, int]) -> None: + for i in range(len(s.expr)): + result = infer_if_condition_value(s.expr[i], pyversion) + if result == ALWAYS_FALSE: + # The condition is always false, so we skip the if/elif body. + mark_block_unreachable(s.body[i]) + elif result == ALWAYS_TRUE: + # This condition is always true, so all of the remaining + # elif/else bodies will never be executed. + for body in s.body[i + 1:]: + mark_block_unreachable(body) + if s.else_body: + mark_block_unreachable(s.else_body) + break + + +def infer_if_condition_value(expr: Node, pyversion: Tuple[int, int]) -> int: + """Infer whether if condition is always true/false. + + Return ALWAYS_TRUE if always true, ALWAYS_FALSE if always false, + and TRUTH_VALUE_UNKNOWN otherwise. + """ + name = '' + negated = False + alias = expr + if isinstance(alias, UnaryExpr): + if alias.op == 'not': + expr = alias.expr + negated = True + if isinstance(expr, NameExpr): + name = expr.name + elif isinstance(expr, MemberExpr): + name = expr.name + result = TRUTH_VALUE_UNKNOWN + if name == 'PY2': + result = ALWAYS_TRUE if pyversion[0] == 2 else ALWAYS_FALSE + elif name == 'PY3': + result = ALWAYS_TRUE if pyversion[0] == 3 else ALWAYS_FALSE + elif name == 'MYPY': + result = ALWAYS_TRUE + if negated: + if result == ALWAYS_TRUE: + result = ALWAYS_FALSE + elif result == ALWAYS_FALSE: + result = ALWAYS_TRUE + return result + + +def mark_block_unreachable(block: Block) -> None: + block.is_unreachable = True + block.accept(MarkImportsUnreachableVisitor()) + + +class MarkImportsUnreachableVisitor(TraverserVisitor): + """Visitor that flags all imports nested within a node as unreachable.""" + + def visit_import(self, node: Import) -> None: + node.is_unreachable = True + + def visit_import_from(self, node: ImportFrom) -> None: + node.is_unreachable = True + + def visit_import_all(self, node: ImportAll) -> None: + node.is_unreachable = True + + +def is_identity_signature(sig: Type) -> bool: + """Is type a callable of form T -> T (where T is a type variable)?""" + if isinstance(sig, CallableType) and sig.arg_kinds == [ARG_POS]: + if isinstance(sig.arg_types[0], TypeVarType) and isinstance(sig.ret_type, TypeVarType): + return sig.arg_types[0].id == sig.ret_type.id + return False + + +def returns_any_if_called(expr: Node) -> bool: + """Return True if we can predict that expr will return Any if called. + + This only uses information available during semantic analysis so this + will sometimes return False because of insufficient information (as + type inference hasn't run yet). + """ + if isinstance(expr, RefExpr): + if isinstance(expr.node, FuncDef): + typ = expr.node.type + if typ is None: + # No signature -> default to Any. + return True + # Explicit Any return? + return isinstance(typ, CallableType) and isinstance(typ.ret_type, AnyType) + elif isinstance(expr.node, Var): + typ = expr.node.type + return typ is None or isinstance(typ, AnyType) + elif isinstance(expr, CallExpr): + return returns_any_if_called(expr.callee) + return False + + +def find_fixed_callable_return(expr: Node) -> Optional[CallableType]: + if isinstance(expr, RefExpr): + if isinstance(expr.node, FuncDef): + typ = expr.node.type + if typ: + if isinstance(typ, CallableType) and has_no_typevars(typ.ret_type): + if isinstance(typ.ret_type, CallableType): + return typ.ret_type + elif isinstance(expr, CallExpr): + t = find_fixed_callable_return(expr.callee) + if t: + if isinstance(t.ret_type, CallableType): + return t.ret_type + return None + + +def has_no_typevars(typ: Type) -> bool: + return is_same_type(typ, erase_typevars(typ)) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 6872e297c3a64..42405695d5aed 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -1,409 +1,409 @@ -from typing import cast, List, Dict, Callable - -from mypy.types import ( - Type, AnyType, UnboundType, TypeVisitor, ErrorType, Void, NoneTyp, - Instance, TypeVarType, CallableType, TupleType, NamedTupleType, UnionType, Overloaded, ErasedType, TypeList, - PartialType, DeletedType, UninhabitedType, TypeType, is_named_instance -) -import mypy.applytype -import mypy.constraints -# Circular import; done in the function instead. -# import mypy.solve -from mypy import messages, sametypes -from mypy.nodes import CONTRAVARIANT, COVARIANT -from mypy.maptype import map_instance_to_supertype - -from mypy import experiments - - -TypeParameterChecker = Callable[[Type, Type, int], bool] - - -def check_type_parameter(lefta: Type, righta: Type, variance: int) -> bool: - if variance == COVARIANT: - return is_subtype(lefta, righta, check_type_parameter) - elif variance == CONTRAVARIANT: - return is_subtype(righta, lefta, check_type_parameter) - else: - return is_equivalent(lefta, righta, check_type_parameter) - - -def is_subtype(left: Type, right: Type, - type_parameter_checker: TypeParameterChecker = check_type_parameter) -> bool: - """Is 'left' subtype of 'right'? - - Also consider Any to be a subtype of any type, and vice versa. This - recursively applies to components of composite types (List[int] is subtype - of List[Any], for example). - - type_parameter_checker is used to check the type parameters (for example, - A with B in is_subtype(C[A], C[B]). The default checks for subtype relation - between the type arguments (e.g., A and B), taking the variance of the - type var into account. - """ - if (isinstance(right, AnyType) or isinstance(right, UnboundType) - or isinstance(right, ErasedType)): - return True - elif isinstance(right, UnionType) and not isinstance(left, UnionType): - return any(is_subtype(left, item, type_parameter_checker) - for item in right.items) - else: - return left.accept(SubtypeVisitor(right, type_parameter_checker)) - - -def is_subtype_ignoring_tvars(left: Type, right: Type) -> bool: - def ignore_tvars(s: Type, t: Type, v: int) -> bool: - return True - return is_subtype(left, right, ignore_tvars) - - -def is_equivalent(a: Type, b: Type, - type_parameter_checker=check_type_parameter) -> bool: - return is_subtype(a, b, type_parameter_checker) and is_subtype(b, a, type_parameter_checker) - - -def satisfies_upper_bound(a: Type, upper_bound: Type) -> bool: - """Is 'a' valid value for a type variable with the given 'upper_bound'? - - Same as is_subtype except that Void is considered to be a subtype of - any upper_bound. This is needed in a case like - - def f(g: Callable[[], T]) -> T: ... - def h() -> None: ... - f(h) - """ - return isinstance(a, Void) or is_subtype(a, upper_bound) - - -class SubtypeVisitor(TypeVisitor[bool]): - - def __init__(self, right: Type, - type_parameter_checker: TypeParameterChecker) -> None: - self.right = right - self.check_type_parameter = type_parameter_checker - - # visit_x(left) means: is left (which is an instance of X) a subtype of - # right? - - def visit_unbound_type(self, left: UnboundType) -> bool: - return True - - def visit_error_type(self, left: ErrorType) -> bool: - return False - - def visit_type_list(self, t: TypeList) -> bool: - assert False, 'Not supported' - - def visit_any(self, left: AnyType) -> bool: - return True - - def visit_void(self, left: Void) -> bool: - return isinstance(self.right, Void) - - def visit_none_type(self, left: NoneTyp) -> bool: - if experiments.STRICT_OPTIONAL: - return (isinstance(self.right, NoneTyp) or - is_named_instance(self.right, 'builtins.object')) - else: - return not isinstance(self.right, Void) - - def visit_uninhabited_type(self, left: UninhabitedType) -> bool: - return not isinstance(self.right, Void) - - def visit_erased_type(self, left: ErasedType) -> bool: - return True - - def visit_deleted_type(self, left: DeletedType) -> bool: - return True - - def visit_instance(self, left: Instance) -> bool: - if left.type.fallback_to_any: - return True - right = self.right - if isinstance(right, Instance): - if left.type._promote and is_subtype(left.type._promote, - self.right, - self.check_type_parameter): - return True - rname = right.type.fullname() - if not left.type.has_base(rname) and rname != 'builtins.object': - return False - - # Map left type to corresponding right instances. - t = map_instance_to_supertype(left, right.type) - - return all(self.check_type_parameter(lefta, righta, tvar.variance) - for lefta, righta, tvar in - zip(t.args, right.args, right.type.defn.type_vars)) - else: - return False - - def visit_type_var(self, left: TypeVarType) -> bool: - right = self.right - if isinstance(right, TypeVarType): - return left.id == right.id - else: - return is_subtype(left.upper_bound, self.right) - - def visit_callable_type(self, left: CallableType) -> bool: - right = self.right - if isinstance(right, CallableType): - return is_callable_subtype(left, right) - elif isinstance(right, Overloaded): - return all(is_subtype(left, item, self.check_type_parameter) - for item in right.items()) - elif isinstance(right, Instance): - return is_subtype(left.fallback, right) - elif isinstance(right, TypeType): - # This is unsound, we don't check the __init__ signature. - return left.is_type_obj() and is_subtype(left.ret_type, right.item) - else: - return False - - def visit_tuple_type(self, left: TupleType) -> bool: - right = self.right - if isinstance(right, Instance): - if is_named_instance(right, 'builtins.object'): - return True - if is_named_instance(right, 'builtins.tuple'): - target_item_type = right.args[0] - return all(is_subtype(item, target_item_type) - for item in left.items) - elif is_named_instance(right, 'typing.Sized'): - return True - elif (is_named_instance(right, 'typing.Iterable') or - is_named_instance(right, 'typing.Container') or - is_named_instance(right, 'typing.Sequence') or - is_named_instance(right, 'typing.Reversible')): - iter_type = right.args[0] - return all(is_subtype(li, iter_type) for li in left.items) - return False - elif isinstance(right, TupleType): - if len(left.items) != len(right.items): - return False - for l, r in zip(left.items, right.items): - if not is_subtype(l, r, self.check_type_parameter): - return False - if not is_subtype(left.fallback, right.fallback, self.check_type_parameter): - return False - return True - else: - return False - - def visit_namedtuple_type(self, left: NamedTupleType) -> bool: - if isinstance(self.right, NamedTupleType): - # left.attrs.issubset(right.attrs) is insufficient - # (a: int, b: str) is not subtype of (b: int, a: str) - if any(l != r for l, r in zip(left.attrs, self.right.attrs)): - return False - return self.visit_tuple_type(left) - - def visit_overloaded(self, left: Overloaded) -> bool: - right = self.right - if isinstance(right, Instance): - return is_subtype(left.fallback, right) - elif isinstance(right, CallableType) or is_named_instance( - right, 'builtins.type'): - for item in left.items(): - if is_subtype(item, right, self.check_type_parameter): - return True - return False - elif isinstance(right, Overloaded): - # TODO: this may be too restrictive - if len(left.items()) != len(right.items()): - return False - for i in range(len(left.items())): - if not is_subtype(left.items()[i], right.items()[i], self.check_type_parameter): - return False - return True - elif isinstance(right, UnboundType): - return True - elif isinstance(right, TypeType): - # All the items must have the same type object status, so - # it's sufficient to query only (any) one of them. - # This is unsound, we don't check the __init__ signature. - return left.is_type_obj() and is_subtype(left.items()[0].ret_type, right.item) - else: - return False - - def visit_union_type(self, left: UnionType) -> bool: - return all(is_subtype(item, self.right, self.check_type_parameter) - for item in left.items) - - def visit_partial_type(self, left: PartialType) -> bool: - # This is indeterminate as we don't really know the complete type yet. - raise RuntimeError - - def visit_type_type(self, left: TypeType) -> bool: - right = self.right - if isinstance(right, TypeType): - return is_subtype(left.item, right.item) - if isinstance(right, CallableType): - # This is unsound, we don't check the __init__ signature. - return right.is_type_obj() and is_subtype(left.item, right.ret_type) - if (isinstance(right, Instance) and - right.type.fullname() in ('builtins.type', 'builtins.object')): - # Treat builtins.type the same as Type[Any]; - # treat builtins.object the same as Any. - return True - return False - - -def is_callable_subtype(left: CallableType, right: CallableType, - ignore_return: bool = False) -> bool: - """Is left a subtype of right?""" - # TODO: Support named arguments, **args, etc. - # Non-type cannot be a subtype of type. - if right.is_type_obj() and not left.is_type_obj(): - return False - - # A callable L is a subtype of a generic callable R if L is a - # subtype of every type obtained from R by substituting types for - # the variables of R. We can check this by simply leaving the - # generic variables of R as type variables, effectively varying - # over all possible values. - - # It's okay even if these variables share ids with generic - # type variables of L, because generating and solving - # constraints for the variables of L to make L a subtype of R - # (below) treats type variables on the two sides as independent. - - if left.variables: - # Apply generic type variables away in left via type inference. - left = unify_generic_callable(left, right, ignore_return=ignore_return) - if left is None: - return False - - # Check return types. - if not ignore_return and not is_subtype(left.ret_type, right.ret_type): - return False - - if right.is_ellipsis_args: - return True - - # Check argument types. - if left.min_args > right.min_args: - return False - if left.is_var_arg: - return is_var_arg_callable_subtype_helper(left, right) - if right.is_var_arg: - return False - if len(left.arg_types) < len(right.arg_types): - return False - for i in range(len(right.arg_types)): - if not is_subtype(right.arg_types[i], left.arg_types[i]): - return False - return True - - -def is_var_arg_callable_subtype_helper(left: CallableType, right: CallableType) -> bool: - """Is left a subtype of right, assuming left has *args? - - See also is_callable_subtype for additional assumptions we can make. - """ - left_fixed = left.max_fixed_args() - right_fixed = right.max_fixed_args() - num_fixed_matching = min(left_fixed, right_fixed) - for i in range(num_fixed_matching): - if not is_subtype(right.arg_types[i], left.arg_types[i]): - return False - if not right.is_var_arg: - for i in range(num_fixed_matching, len(right.arg_types)): - if not is_subtype(right.arg_types[i], left.arg_types[-1]): - return False - return True - else: - for i in range(left_fixed, right_fixed): - if not is_subtype(right.arg_types[i], left.arg_types[-1]): - return False - for i in range(right_fixed, left_fixed): - if not is_subtype(right.arg_types[-1], left.arg_types[i]): - return False - return is_subtype(right.arg_types[-1], left.arg_types[-1]) - - -def unify_generic_callable(type: CallableType, target: CallableType, - ignore_return: bool) -> CallableType: - """Try to unify a generic callable type with another callable type. - - Return unified CallableType if successful; otherwise, return None. - """ - import mypy.solve - constraints = [] # type: List[mypy.constraints.Constraint] - for arg_type, target_arg_type in zip(type.arg_types, target.arg_types): - c = mypy.constraints.infer_constraints( - arg_type, target_arg_type, mypy.constraints.SUPERTYPE_OF) - constraints.extend(c) - if not ignore_return: - c = mypy.constraints.infer_constraints( - type.ret_type, target.ret_type, mypy.constraints.SUBTYPE_OF) - constraints.extend(c) - type_var_ids = [tvar.id for tvar in type.variables] - inferred_vars = mypy.solve.solve_constraints(type_var_ids, constraints) - if None in inferred_vars: - return None - msg = messages.temp_message_builder() - applied = mypy.applytype.apply_generic_arguments(type, inferred_vars, msg, context=target) - if msg.is_errors() or not isinstance(applied, CallableType): - return None - return cast(CallableType, applied) - - -def restrict_subtype_away(t: Type, s: Type) -> Type: - """Return a supertype of (t intersect not s) - - Currently just remove elements of a union type. - """ - if isinstance(t, UnionType): - new_items = [item for item in t.items if not is_subtype(item, s)] - return UnionType.make_union(new_items) - else: - return t - - -def is_proper_subtype(t: Type, s: Type) -> bool: - """Check if t is a proper subtype of s? - - For proper subtypes, there's no need to rely on compatibility due to - Any types. Any instance type t is also a proper subtype of t. - """ - # FIX tuple types - if isinstance(t, Instance): - if isinstance(s, Instance): - if not t.type.has_base(s.type.fullname()): - return False - - def check_argument(left: Type, right: Type, variance: int) -> bool: - if variance == COVARIANT: - return is_proper_subtype(left, right) - elif variance == CONTRAVARIANT: - return is_proper_subtype(right, left) - else: - return sametypes.is_same_type(left, right) - - # Map left type to corresponding right instances. - t = map_instance_to_supertype(t, s.type) - - return all(check_argument(ta, ra, tvar.variance) for ta, ra, tvar in - zip(t.args, s.args, s.type.defn.type_vars)) - return False - else: - return sametypes.is_same_type(t, s) - - -def is_more_precise(t: Type, s: Type) -> bool: - """Check if t is a more precise type than s. - - A t is a proper subtype of s, t is also more precise than s. Also, if - s is Any, t is more precise than s for any t. Finally, if t is the same - type as s, t is more precise than s. - """ - # TODO Should List[int] be more precise than List[Any]? - if isinstance(s, AnyType): - return True - if isinstance(s, Instance): - if isinstance(t, CallableType): - # Fall back to subclass check and ignore other properties of the callable. - return is_proper_subtype(t.fallback, s) - return is_proper_subtype(t, s) - return sametypes.is_same_type(t, s) +from typing import cast, List, Dict, Callable + +from mypy.types import ( + Type, AnyType, UnboundType, TypeVisitor, ErrorType, Void, NoneTyp, + Instance, TypeVarType, CallableType, TupleType, NamedTupleType, UnionType, Overloaded, ErasedType, TypeList, + PartialType, DeletedType, UninhabitedType, TypeType, is_named_instance +) +import mypy.applytype +import mypy.constraints +# Circular import; done in the function instead. +# import mypy.solve +from mypy import messages, sametypes +from mypy.nodes import CONTRAVARIANT, COVARIANT +from mypy.maptype import map_instance_to_supertype + +from mypy import experiments + + +TypeParameterChecker = Callable[[Type, Type, int], bool] + + +def check_type_parameter(lefta: Type, righta: Type, variance: int) -> bool: + if variance == COVARIANT: + return is_subtype(lefta, righta, check_type_parameter) + elif variance == CONTRAVARIANT: + return is_subtype(righta, lefta, check_type_parameter) + else: + return is_equivalent(lefta, righta, check_type_parameter) + + +def is_subtype(left: Type, right: Type, + type_parameter_checker: TypeParameterChecker = check_type_parameter) -> bool: + """Is 'left' subtype of 'right'? + + Also consider Any to be a subtype of any type, and vice versa. This + recursively applies to components of composite types (List[int] is subtype + of List[Any], for example). + + type_parameter_checker is used to check the type parameters (for example, + A with B in is_subtype(C[A], C[B]). The default checks for subtype relation + between the type arguments (e.g., A and B), taking the variance of the + type var into account. + """ + if (isinstance(right, AnyType) or isinstance(right, UnboundType) + or isinstance(right, ErasedType)): + return True + elif isinstance(right, UnionType) and not isinstance(left, UnionType): + return any(is_subtype(left, item, type_parameter_checker) + for item in right.items) + else: + return left.accept(SubtypeVisitor(right, type_parameter_checker)) + + +def is_subtype_ignoring_tvars(left: Type, right: Type) -> bool: + def ignore_tvars(s: Type, t: Type, v: int) -> bool: + return True + return is_subtype(left, right, ignore_tvars) + + +def is_equivalent(a: Type, b: Type, + type_parameter_checker=check_type_parameter) -> bool: + return is_subtype(a, b, type_parameter_checker) and is_subtype(b, a, type_parameter_checker) + + +def satisfies_upper_bound(a: Type, upper_bound: Type) -> bool: + """Is 'a' valid value for a type variable with the given 'upper_bound'? + + Same as is_subtype except that Void is considered to be a subtype of + any upper_bound. This is needed in a case like + + def f(g: Callable[[], T]) -> T: ... + def h() -> None: ... + f(h) + """ + return isinstance(a, Void) or is_subtype(a, upper_bound) + + +class SubtypeVisitor(TypeVisitor[bool]): + + def __init__(self, right: Type, + type_parameter_checker: TypeParameterChecker) -> None: + self.right = right + self.check_type_parameter = type_parameter_checker + + # visit_x(left) means: is left (which is an instance of X) a subtype of + # right? + + def visit_unbound_type(self, left: UnboundType) -> bool: + return True + + def visit_error_type(self, left: ErrorType) -> bool: + return False + + def visit_type_list(self, t: TypeList) -> bool: + assert False, 'Not supported' + + def visit_any(self, left: AnyType) -> bool: + return True + + def visit_void(self, left: Void) -> bool: + return isinstance(self.right, Void) + + def visit_none_type(self, left: NoneTyp) -> bool: + if experiments.STRICT_OPTIONAL: + return (isinstance(self.right, NoneTyp) or + is_named_instance(self.right, 'builtins.object')) + else: + return not isinstance(self.right, Void) + + def visit_uninhabited_type(self, left: UninhabitedType) -> bool: + return not isinstance(self.right, Void) + + def visit_erased_type(self, left: ErasedType) -> bool: + return True + + def visit_deleted_type(self, left: DeletedType) -> bool: + return True + + def visit_instance(self, left: Instance) -> bool: + if left.type.fallback_to_any: + return True + right = self.right + if isinstance(right, Instance): + if left.type._promote and is_subtype(left.type._promote, + self.right, + self.check_type_parameter): + return True + rname = right.type.fullname() + if not left.type.has_base(rname) and rname != 'builtins.object': + return False + + # Map left type to corresponding right instances. + t = map_instance_to_supertype(left, right.type) + + return all(self.check_type_parameter(lefta, righta, tvar.variance) + for lefta, righta, tvar in + zip(t.args, right.args, right.type.defn.type_vars)) + else: + return False + + def visit_type_var(self, left: TypeVarType) -> bool: + right = self.right + if isinstance(right, TypeVarType): + return left.id == right.id + else: + return is_subtype(left.upper_bound, self.right) + + def visit_callable_type(self, left: CallableType) -> bool: + right = self.right + if isinstance(right, CallableType): + return is_callable_subtype(left, right) + elif isinstance(right, Overloaded): + return all(is_subtype(left, item, self.check_type_parameter) + for item in right.items()) + elif isinstance(right, Instance): + return is_subtype(left.fallback, right) + elif isinstance(right, TypeType): + # This is unsound, we don't check the __init__ signature. + return left.is_type_obj() and is_subtype(left.ret_type, right.item) + else: + return False + + def visit_tuple_type(self, left: TupleType) -> bool: + right = self.right + if isinstance(right, Instance): + if is_named_instance(right, 'builtins.object'): + return True + if is_named_instance(right, 'builtins.tuple'): + target_item_type = right.args[0] + return all(is_subtype(item, target_item_type) + for item in left.items) + elif is_named_instance(right, 'typing.Sized'): + return True + elif (is_named_instance(right, 'typing.Iterable') or + is_named_instance(right, 'typing.Container') or + is_named_instance(right, 'typing.Sequence') or + is_named_instance(right, 'typing.Reversible')): + iter_type = right.args[0] + return all(is_subtype(li, iter_type) for li in left.items) + return False + elif isinstance(right, TupleType): + if len(left.items) != len(right.items): + return False + for l, r in zip(left.items, right.items): + if not is_subtype(l, r, self.check_type_parameter): + return False + if not is_subtype(left.fallback, right.fallback, self.check_type_parameter): + return False + return True + else: + return False + + def visit_namedtuple_type(self, left: NamedTupleType) -> bool: + if isinstance(self.right, NamedTupleType): + # left.attrs.issubset(right.attrs) is insufficient + # (a: int, b: str) is not subtype of (b: int, a: str) + if any(l != r for l, r in zip(left.attrs, self.right.attrs)): + return False + return self.visit_tuple_type(left) + + def visit_overloaded(self, left: Overloaded) -> bool: + right = self.right + if isinstance(right, Instance): + return is_subtype(left.fallback, right) + elif isinstance(right, CallableType) or is_named_instance( + right, 'builtins.type'): + for item in left.items(): + if is_subtype(item, right, self.check_type_parameter): + return True + return False + elif isinstance(right, Overloaded): + # TODO: this may be too restrictive + if len(left.items()) != len(right.items()): + return False + for i in range(len(left.items())): + if not is_subtype(left.items()[i], right.items()[i], self.check_type_parameter): + return False + return True + elif isinstance(right, UnboundType): + return True + elif isinstance(right, TypeType): + # All the items must have the same type object status, so + # it's sufficient to query only (any) one of them. + # This is unsound, we don't check the __init__ signature. + return left.is_type_obj() and is_subtype(left.items()[0].ret_type, right.item) + else: + return False + + def visit_union_type(self, left: UnionType) -> bool: + return all(is_subtype(item, self.right, self.check_type_parameter) + for item in left.items) + + def visit_partial_type(self, left: PartialType) -> bool: + # This is indeterminate as we don't really know the complete type yet. + raise RuntimeError + + def visit_type_type(self, left: TypeType) -> bool: + right = self.right + if isinstance(right, TypeType): + return is_subtype(left.item, right.item) + if isinstance(right, CallableType): + # This is unsound, we don't check the __init__ signature. + return right.is_type_obj() and is_subtype(left.item, right.ret_type) + if (isinstance(right, Instance) and + right.type.fullname() in ('builtins.type', 'builtins.object')): + # Treat builtins.type the same as Type[Any]; + # treat builtins.object the same as Any. + return True + return False + + +def is_callable_subtype(left: CallableType, right: CallableType, + ignore_return: bool = False) -> bool: + """Is left a subtype of right?""" + # TODO: Support named arguments, **args, etc. + # Non-type cannot be a subtype of type. + if right.is_type_obj() and not left.is_type_obj(): + return False + + # A callable L is a subtype of a generic callable R if L is a + # subtype of every type obtained from R by substituting types for + # the variables of R. We can check this by simply leaving the + # generic variables of R as type variables, effectively varying + # over all possible values. + + # It's okay even if these variables share ids with generic + # type variables of L, because generating and solving + # constraints for the variables of L to make L a subtype of R + # (below) treats type variables on the two sides as independent. + + if left.variables: + # Apply generic type variables away in left via type inference. + left = unify_generic_callable(left, right, ignore_return=ignore_return) + if left is None: + return False + + # Check return types. + if not ignore_return and not is_subtype(left.ret_type, right.ret_type): + return False + + if right.is_ellipsis_args: + return True + + # Check argument types. + if left.min_args > right.min_args: + return False + if left.is_var_arg: + return is_var_arg_callable_subtype_helper(left, right) + if right.is_var_arg: + return False + if len(left.arg_types) < len(right.arg_types): + return False + for i in range(len(right.arg_types)): + if not is_subtype(right.arg_types[i], left.arg_types[i]): + return False + return True + + +def is_var_arg_callable_subtype_helper(left: CallableType, right: CallableType) -> bool: + """Is left a subtype of right, assuming left has *args? + + See also is_callable_subtype for additional assumptions we can make. + """ + left_fixed = left.max_fixed_args() + right_fixed = right.max_fixed_args() + num_fixed_matching = min(left_fixed, right_fixed) + for i in range(num_fixed_matching): + if not is_subtype(right.arg_types[i], left.arg_types[i]): + return False + if not right.is_var_arg: + for i in range(num_fixed_matching, len(right.arg_types)): + if not is_subtype(right.arg_types[i], left.arg_types[-1]): + return False + return True + else: + for i in range(left_fixed, right_fixed): + if not is_subtype(right.arg_types[i], left.arg_types[-1]): + return False + for i in range(right_fixed, left_fixed): + if not is_subtype(right.arg_types[-1], left.arg_types[i]): + return False + return is_subtype(right.arg_types[-1], left.arg_types[-1]) + + +def unify_generic_callable(type: CallableType, target: CallableType, + ignore_return: bool) -> CallableType: + """Try to unify a generic callable type with another callable type. + + Return unified CallableType if successful; otherwise, return None. + """ + import mypy.solve + constraints = [] # type: List[mypy.constraints.Constraint] + for arg_type, target_arg_type in zip(type.arg_types, target.arg_types): + c = mypy.constraints.infer_constraints( + arg_type, target_arg_type, mypy.constraints.SUPERTYPE_OF) + constraints.extend(c) + if not ignore_return: + c = mypy.constraints.infer_constraints( + type.ret_type, target.ret_type, mypy.constraints.SUBTYPE_OF) + constraints.extend(c) + type_var_ids = [tvar.id for tvar in type.variables] + inferred_vars = mypy.solve.solve_constraints(type_var_ids, constraints) + if None in inferred_vars: + return None + msg = messages.temp_message_builder() + applied = mypy.applytype.apply_generic_arguments(type, inferred_vars, msg, context=target) + if msg.is_errors() or not isinstance(applied, CallableType): + return None + return cast(CallableType, applied) + + +def restrict_subtype_away(t: Type, s: Type) -> Type: + """Return a supertype of (t intersect not s) + + Currently just remove elements of a union type. + """ + if isinstance(t, UnionType): + new_items = [item for item in t.items if not is_subtype(item, s)] + return UnionType.make_union(new_items) + else: + return t + + +def is_proper_subtype(t: Type, s: Type) -> bool: + """Check if t is a proper subtype of s? + + For proper subtypes, there's no need to rely on compatibility due to + Any types. Any instance type t is also a proper subtype of t. + """ + # FIX tuple types + if isinstance(t, Instance): + if isinstance(s, Instance): + if not t.type.has_base(s.type.fullname()): + return False + + def check_argument(left: Type, right: Type, variance: int) -> bool: + if variance == COVARIANT: + return is_proper_subtype(left, right) + elif variance == CONTRAVARIANT: + return is_proper_subtype(right, left) + else: + return sametypes.is_same_type(left, right) + + # Map left type to corresponding right instances. + t = map_instance_to_supertype(t, s.type) + + return all(check_argument(ta, ra, tvar.variance) for ta, ra, tvar in + zip(t.args, s.args, s.type.defn.type_vars)) + return False + else: + return sametypes.is_same_type(t, s) + + +def is_more_precise(t: Type, s: Type) -> bool: + """Check if t is a more precise type than s. + + A t is a proper subtype of s, t is also more precise than s. Also, if + s is Any, t is more precise than s for any t. Finally, if t is the same + type as s, t is more precise than s. + """ + # TODO Should List[int] be more precise than List[Any]? + if isinstance(s, AnyType): + return True + if isinstance(s, Instance): + if isinstance(t, CallableType): + # Fall back to subclass check and ignore other properties of the callable. + return is_proper_subtype(t.fallback, s) + return is_proper_subtype(t, s) + return sametypes.is_same_type(t, s) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 1841f283a19e1..f10048c4f0fe4 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -1,430 +1,430 @@ -"""Semantic analysis of types""" - -from typing import Callable, cast, List, Tuple - -from mypy.types import ( - Type, UnboundType, TypeVarType, TupleType, NamedTupleType, UnionType, Instance, AnyType, CallableType, - Void, NoneTyp, DeletedType, TypeList, TypeVarDef, TypeVisitor, StarType, PartialType, - EllipsisType, UninhabitedType, TypeType -) -from mypy.nodes import ( - BOUND_TVAR, TYPE_ALIAS, UNBOUND_IMPORTED, - TypeInfo, Context, SymbolTableNode, TypeVarExpr, Var, Node, - IndexExpr, RefExpr -) -from mypy.sametypes import is_same_type -from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError -from mypy.subtypes import satisfies_upper_bound -from mypy import nodes -from mypy import experiments - - -type_constructors = {'typing.Tuple', 'typing.Union', 'typing.Callable', 'typing.Type'} - - -def analyze_type_alias(node: Node, - lookup_func: Callable[[str, Context], SymbolTableNode], - lookup_fqn_func: Callable[[str], SymbolTableNode], - fail_func: Callable[[str, Context], None]) -> Type: - """Return type if node is valid as a type alias rvalue. - - Return None otherwise. 'node' must have been semantically analyzed. - """ - # Quickly return None if the expression doesn't look like a type. Note - # that we don't support straight string literals as type aliases - # (only string literals within index expressions). - if isinstance(node, RefExpr): - if not (isinstance(node.node, TypeInfo) or - node.fullname == 'typing.Any' or - node.kind == TYPE_ALIAS): - return None - elif isinstance(node, IndexExpr): - base = node.base - if isinstance(base, RefExpr): - if not (isinstance(base.node, TypeInfo) or - base.fullname in type_constructors): - return None - else: - return None - else: - return None - - # It's a type alias (though it may be an invalid one). - try: - type = expr_to_unanalyzed_type(node) - except TypeTranslationError: - fail_func('Invalid type alias', node) - return None - analyzer = TypeAnalyser(lookup_func, lookup_fqn_func, fail_func) - return type.accept(analyzer) - - -class TypeAnalyser(TypeVisitor[Type]): - """Semantic analyzer for types (semantic analysis pass 2). - - Converts unbound types into bound types. - """ - - def __init__(self, - lookup_func: Callable[[str, Context], SymbolTableNode], - lookup_fqn_func: Callable[[str], SymbolTableNode], - fail_func: Callable[[str, Context], None]) -> None: - self.lookup = lookup_func - self.lookup_fqn_func = lookup_fqn_func - self.fail = fail_func - - def visit_unbound_type(self, t: UnboundType) -> Type: - if t.optional: - t.optional = False - # We don't need to worry about double-wrapping Optionals or - # wrapping Anys: Union simplification will take care of that. - return UnionType.make_simplified_union([self.visit_unbound_type(t), NoneTyp()]) - sym = self.lookup(t.name, t) - if sym is not None: - if sym.node is None: - # UNBOUND_IMPORTED can happen if an unknown name was imported. - if sym.kind != UNBOUND_IMPORTED: - self.fail('Internal error (node is None, kind={})'.format(sym.kind), t) - return AnyType() - fullname = sym.node.fullname() - if sym.kind == BOUND_TVAR: - if len(t.args) > 0: - self.fail('Type variable "{}" used with arguments'.format( - t.name), t) - assert sym.tvar_def is not None - return TypeVarType(sym.tvar_def, t.line) - elif fullname == 'builtins.None': - if experiments.STRICT_OPTIONAL: - if t.is_ret_type: - return Void() - else: - return NoneTyp() - else: - return Void() - elif fullname == 'typing.Any': - return AnyType() - elif fullname == 'typing.Tuple': - if len(t.args) == 2 and isinstance(t.args[1], EllipsisType): - # Tuple[T, ...] (uniform, variable-length tuple) - node = self.lookup_fqn_func('builtins.tuple') - tuple_info = cast(TypeInfo, node.node) - return Instance(tuple_info, [t.args[0].accept(self)], t.line) - return self.tuple_type(self.anal_array(t.args)) - elif fullname == 'typing.Union': - items = self.anal_array(t.args) - items = [item for item in items if not isinstance(item, Void)] - return UnionType.make_union(items) - elif fullname == 'typing.Optional': - if len(t.args) != 1: - self.fail('Optional[...] must have exactly one type argument', t) - return AnyType() - items = self.anal_array(t.args) - if experiments.STRICT_OPTIONAL: - return UnionType.make_simplified_union([items[0], NoneTyp()]) - else: - # Without strict Optional checking Optional[t] is just an alias for t. - return items[0] - elif fullname == 'typing.Callable': - return self.analyze_callable_type(t) - elif fullname == 'typing.Type': - if len(t.args) == 0: - return TypeType(AnyType(), line=t.line) - if len(t.args) != 1: - self.fail('Type[...] must have exactly one type argument', t) - items = self.anal_array(t.args) - item = items[0] - return TypeType(item, line=t.line) - elif sym.kind == TYPE_ALIAS: - # TODO: Generic type aliases. - return sym.type_override - elif not isinstance(sym.node, TypeInfo): - name = sym.fullname - if name is None: - name = sym.node.name() - if isinstance(sym.node, Var) and isinstance(sym.node.type, AnyType): - # Something with an Any type -- make it an alias for Any in a type - # context. This is slightly problematic as it allows using the type 'Any' - # as a base class -- however, this will fail soon at runtime so the problem - # is pretty minor. - return AnyType() - self.fail('Invalid type "{}"'.format(name), t) - return t - info = sym.node # type: TypeInfo - if len(t.args) > 0 and info.fullname() == 'builtins.tuple': - return TupleType(self.anal_array(t.args), - Instance(info, [AnyType()], t.line), - t.line) - else: - # Analyze arguments and construct Instance type. The - # number of type arguments and their values are - # checked only later, since we do not always know the - # valid count at this point. Thus we may construct an - # Instance with an invalid number of type arguments. - instance = Instance(info, self.anal_array(t.args), t.line) - if info.tuple_type is None: - return instance - else: - # The class has a Tuple[...] base class so it will be - # represented as a tuple type. - if t.args: - self.fail('Generic tuple types not supported', t) - return AnyType() - # import pdb; pdb.set_trace() - if info.is_named_tuple: - return NamedTupleType(info.name(), info.attrs, - self.anal_array(info.tuple_type.items), - fallback=instance, - line=t.line) - return TupleType(self.anal_array(info.tuple_type.items), - fallback=instance, - line=t.line) - else: - return AnyType() - - def visit_any(self, t: AnyType) -> Type: - return t - - def visit_void(self, t: Void) -> Type: - return t - - def visit_none_type(self, t: NoneTyp) -> Type: - return t - - def visit_uninhabited_type(self, t: UninhabitedType) -> Type: - return t - - def visit_deleted_type(self, t: DeletedType) -> Type: - return t - - def visit_type_list(self, t: TypeList) -> Type: - self.fail('Invalid type', t) - return AnyType() - - def visit_instance(self, t: Instance) -> Type: - return t - - def visit_type_var(self, t: TypeVarType) -> Type: - raise RuntimeError('TypeVarType is already analyzed') - - def visit_callable_type(self, t: CallableType) -> Type: - return t.copy_modified(arg_types=self.anal_array(t.arg_types), - ret_type=t.ret_type.accept(self), - fallback=t.fallback or self.builtin_type('builtins.function'), - variables=self.anal_var_defs(t.variables)) - - def visit_tuple_type(self, t: TupleType) -> Type: - if t.implicit: - self.fail('Invalid tuple literal type', t) - return AnyType() - star_count = sum(1 for item in t.items if isinstance(item, StarType)) - if star_count > 1: - self.fail('At most one star type allowed in a tuple', t) - return AnyType() - fallback = t.fallback if t.fallback else self.builtin_type('builtins.tuple', [AnyType()]) - return TupleType(self.anal_array(t.items), fallback, t.line) - - def visit_namedtuple_type(self, t: NamedTupleType) -> Type: - return NamedTupleType(t.name, t.attrs, - self.anal_array(t.items), - fallback=t.fallback, - line=t.line) - - def visit_star_type(self, t: StarType) -> Type: - return StarType(t.type.accept(self), t.line) - - def visit_union_type(self, t: UnionType) -> Type: - return UnionType(self.anal_array(t.items), t.line) - - def visit_partial_type(self, t: PartialType) -> Type: - assert False, "Internal error: Unexpected partial type" - - def visit_ellipsis_type(self, t: EllipsisType) -> Type: - self.fail("Unexpected '...'", t) - return AnyType() - - def visit_type_type(self, t: TypeType) -> Type: - return TypeType(t.item.accept(self), line=t.line) - - def analyze_callable_type(self, t: UnboundType) -> Type: - fallback = self.builtin_type('builtins.function') - if len(t.args) == 0: - # Callable (bare). Treat as Callable[..., Any]. - return CallableType([AnyType(), AnyType()], - [nodes.ARG_STAR, nodes.ARG_STAR2], - [None, None], - ret_type=AnyType(), - fallback=fallback, - is_ellipsis_args=True) - elif len(t.args) == 2: - ret_type = t.args[1].accept(self) - if isinstance(t.args[0], TypeList): - # Callable[[ARG, ...], RET] (ordinary callable type) - args = t.args[0].items - return CallableType(self.anal_array(args), - [nodes.ARG_POS] * len(args), - [None] * len(args), - ret_type=ret_type, - fallback=fallback) - elif isinstance(t.args[0], EllipsisType): - # Callable[..., RET] (with literal ellipsis; accept arbitrary arguments) - return CallableType([AnyType(), AnyType()], - [nodes.ARG_STAR, nodes.ARG_STAR2], - [None, None], - ret_type=ret_type, - fallback=fallback, - is_ellipsis_args=True) - else: - self.fail('The first argument to Callable must be a list of types or "..."', t) - return AnyType() - - self.fail('Invalid function type', t) - return AnyType() - - def anal_array(self, a: List[Type]) -> List[Type]: - res = [] # type: List[Type] - for t in a: - res.append(t.accept(self)) - return res - - def anal_var_defs(self, var_defs: List[TypeVarDef]) -> List[TypeVarDef]: - a = [] # type: List[TypeVarDef] - for vd in var_defs: - a.append(TypeVarDef(vd.name, vd.id.raw_id, self.anal_array(vd.values), - vd.upper_bound.accept(self), - vd.variance, - vd.line)) - return a - - def builtin_type(self, fully_qualified_name: str, args: List[Type] = None) -> Instance: - node = self.lookup_fqn_func(fully_qualified_name) - info = cast(TypeInfo, node.node) - return Instance(info, args or []) - - def tuple_type(self, items: List[Type]) -> TupleType: - return TupleType(items, fallback=self.builtin_type('builtins.tuple', [AnyType()])) - - -class TypeAnalyserPass3(TypeVisitor[None]): - """Analyze type argument counts and values of generic types. - - This is semantic analysis pass 3 for types. - - Perform these operations: - - * Report error for invalid type argument counts, such as List[x, y]. - * Make implicit Any type arguments explicit my modifying types - in-place. For example, modify Foo into Foo[Any] if Foo expects a single - type argument. - * If a type variable has a value restriction, ensure that the value is - valid. For example, reject IO[int] if the type argument must be str - or bytes. - - We can't do this earlier than the third pass, since type argument counts - are only determined in pass 2, and we have to support forward references - to types. - """ - - def __init__(self, fail_func: Callable[[str, Context], None]) -> None: - self.fail = fail_func - - def visit_instance(self, t: Instance) -> None: - info = t.type - # Check type argument count. - if len(t.args) != len(info.type_vars): - if len(t.args) == 0: - # Insert implicit 'Any' type arguments. - t.args = [AnyType()] * len(info.type_vars) - return - # Invalid number of type parameters. - n = len(info.type_vars) - s = '{} type arguments'.format(n) - if n == 0: - s = 'no type arguments' - elif n == 1: - s = '1 type argument' - act = str(len(t.args)) - if act == '0': - act = 'none' - self.fail('"{}" expects {}, but {} given'.format( - info.name(), s, act), t) - # Construct the correct number of type arguments, as - # otherwise the type checker may crash as it expects - # things to be right. - t.args = [AnyType() for _ in info.type_vars] - elif info.defn.type_vars: - # Check type argument values. - for arg, TypeVar in zip(t.args, info.defn.type_vars): - if TypeVar.values: - if isinstance(arg, TypeVarType): - arg_values = arg.values - if not arg_values: - self.fail('Type variable "{}" not valid as type ' - 'argument value for "{}"'.format( - arg.name, info.name()), t) - continue - else: - arg_values = [arg] - self.check_type_var_values(info, arg_values, - TypeVar.values, t) - if not satisfies_upper_bound(arg, TypeVar.upper_bound): - self.fail('Type argument "{}" of "{}" must be ' - 'a subtype of "{}"'.format( - arg, info.name(), TypeVar.upper_bound), t) - for arg in t.args: - arg.accept(self) - - def check_type_var_values(self, type: TypeInfo, actuals: List[Type], - valids: List[Type], context: Context) -> None: - for actual in actuals: - if (not isinstance(actual, AnyType) and - not any(is_same_type(actual, value) for value in valids)): - self.fail('Invalid type argument value for "{}"'.format( - type.name()), context) - - def visit_callable_type(self, t: CallableType) -> None: - t.ret_type.accept(self) - for arg_type in t.arg_types: - arg_type.accept(self) - - def visit_tuple_type(self, t: TupleType) -> None: - for item in t.items: - item.accept(self) - - def visit_union_type(self, t: UnionType) -> None: - for item in t.items: - item.accept(self) - - def visit_star_type(self, t: StarType) -> None: - t.type.accept(self) - - # Other kinds of type are trivial, since they are atomic (or invalid). - - def visit_unbound_type(self, t: UnboundType) -> None: - pass - - def visit_any(self, t: AnyType) -> None: - pass - - def visit_void(self, t: Void) -> None: - pass - - def visit_none_type(self, t: NoneTyp) -> None: - pass - - def visit_uninhabited_type(self, t: UninhabitedType) -> None: - pass - - def visit_deleted_type(self, t: DeletedType) -> None: - pass - - def visit_type_list(self, t: TypeList) -> None: - self.fail('Invalid type', t) - - def visit_type_var(self, t: TypeVarType) -> None: - pass - - def visit_partial_type(self, t: PartialType) -> None: - pass - - def visit_type_type(self, t: TypeType) -> None: - pass +"""Semantic analysis of types""" + +from typing import Callable, cast, List, Tuple + +from mypy.types import ( + Type, UnboundType, TypeVarType, TupleType, NamedTupleType, UnionType, Instance, AnyType, CallableType, + Void, NoneTyp, DeletedType, TypeList, TypeVarDef, TypeVisitor, StarType, PartialType, + EllipsisType, UninhabitedType, TypeType +) +from mypy.nodes import ( + BOUND_TVAR, TYPE_ALIAS, UNBOUND_IMPORTED, + TypeInfo, Context, SymbolTableNode, TypeVarExpr, Var, Node, + IndexExpr, RefExpr +) +from mypy.sametypes import is_same_type +from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError +from mypy.subtypes import satisfies_upper_bound +from mypy import nodes +from mypy import experiments + + +type_constructors = {'typing.Tuple', 'typing.Union', 'typing.Callable', 'typing.Type'} + + +def analyze_type_alias(node: Node, + lookup_func: Callable[[str, Context], SymbolTableNode], + lookup_fqn_func: Callable[[str], SymbolTableNode], + fail_func: Callable[[str, Context], None]) -> Type: + """Return type if node is valid as a type alias rvalue. + + Return None otherwise. 'node' must have been semantically analyzed. + """ + # Quickly return None if the expression doesn't look like a type. Note + # that we don't support straight string literals as type aliases + # (only string literals within index expressions). + if isinstance(node, RefExpr): + if not (isinstance(node.node, TypeInfo) or + node.fullname == 'typing.Any' or + node.kind == TYPE_ALIAS): + return None + elif isinstance(node, IndexExpr): + base = node.base + if isinstance(base, RefExpr): + if not (isinstance(base.node, TypeInfo) or + base.fullname in type_constructors): + return None + else: + return None + else: + return None + + # It's a type alias (though it may be an invalid one). + try: + type = expr_to_unanalyzed_type(node) + except TypeTranslationError: + fail_func('Invalid type alias', node) + return None + analyzer = TypeAnalyser(lookup_func, lookup_fqn_func, fail_func) + return type.accept(analyzer) + + +class TypeAnalyser(TypeVisitor[Type]): + """Semantic analyzer for types (semantic analysis pass 2). + + Converts unbound types into bound types. + """ + + def __init__(self, + lookup_func: Callable[[str, Context], SymbolTableNode], + lookup_fqn_func: Callable[[str], SymbolTableNode], + fail_func: Callable[[str, Context], None]) -> None: + self.lookup = lookup_func + self.lookup_fqn_func = lookup_fqn_func + self.fail = fail_func + + def visit_unbound_type(self, t: UnboundType) -> Type: + if t.optional: + t.optional = False + # We don't need to worry about double-wrapping Optionals or + # wrapping Anys: Union simplification will take care of that. + return UnionType.make_simplified_union([self.visit_unbound_type(t), NoneTyp()]) + sym = self.lookup(t.name, t) + if sym is not None: + if sym.node is None: + # UNBOUND_IMPORTED can happen if an unknown name was imported. + if sym.kind != UNBOUND_IMPORTED: + self.fail('Internal error (node is None, kind={})'.format(sym.kind), t) + return AnyType() + fullname = sym.node.fullname() + if sym.kind == BOUND_TVAR: + if len(t.args) > 0: + self.fail('Type variable "{}" used with arguments'.format( + t.name), t) + assert sym.tvar_def is not None + return TypeVarType(sym.tvar_def, t.line) + elif fullname == 'builtins.None': + if experiments.STRICT_OPTIONAL: + if t.is_ret_type: + return Void() + else: + return NoneTyp() + else: + return Void() + elif fullname == 'typing.Any': + return AnyType() + elif fullname == 'typing.Tuple': + if len(t.args) == 2 and isinstance(t.args[1], EllipsisType): + # Tuple[T, ...] (uniform, variable-length tuple) + node = self.lookup_fqn_func('builtins.tuple') + tuple_info = cast(TypeInfo, node.node) + return Instance(tuple_info, [t.args[0].accept(self)], t.line) + return self.tuple_type(self.anal_array(t.args)) + elif fullname == 'typing.Union': + items = self.anal_array(t.args) + items = [item for item in items if not isinstance(item, Void)] + return UnionType.make_union(items) + elif fullname == 'typing.Optional': + if len(t.args) != 1: + self.fail('Optional[...] must have exactly one type argument', t) + return AnyType() + items = self.anal_array(t.args) + if experiments.STRICT_OPTIONAL: + return UnionType.make_simplified_union([items[0], NoneTyp()]) + else: + # Without strict Optional checking Optional[t] is just an alias for t. + return items[0] + elif fullname == 'typing.Callable': + return self.analyze_callable_type(t) + elif fullname == 'typing.Type': + if len(t.args) == 0: + return TypeType(AnyType(), line=t.line) + if len(t.args) != 1: + self.fail('Type[...] must have exactly one type argument', t) + items = self.anal_array(t.args) + item = items[0] + return TypeType(item, line=t.line) + elif sym.kind == TYPE_ALIAS: + # TODO: Generic type aliases. + return sym.type_override + elif not isinstance(sym.node, TypeInfo): + name = sym.fullname + if name is None: + name = sym.node.name() + if isinstance(sym.node, Var) and isinstance(sym.node.type, AnyType): + # Something with an Any type -- make it an alias for Any in a type + # context. This is slightly problematic as it allows using the type 'Any' + # as a base class -- however, this will fail soon at runtime so the problem + # is pretty minor. + return AnyType() + self.fail('Invalid type "{}"'.format(name), t) + return t + info = sym.node # type: TypeInfo + if len(t.args) > 0 and info.fullname() == 'builtins.tuple': + return TupleType(self.anal_array(t.args), + Instance(info, [AnyType()], t.line), + t.line) + else: + # Analyze arguments and construct Instance type. The + # number of type arguments and their values are + # checked only later, since we do not always know the + # valid count at this point. Thus we may construct an + # Instance with an invalid number of type arguments. + instance = Instance(info, self.anal_array(t.args), t.line) + if info.tuple_type is None: + return instance + else: + # The class has a Tuple[...] base class so it will be + # represented as a tuple type. + if t.args: + self.fail('Generic tuple types not supported', t) + return AnyType() + # import pdb; pdb.set_trace() + if info.is_named_tuple: + return NamedTupleType(info.name(), info.attrs, + self.anal_array(info.tuple_type.items), + fallback=instance, + line=t.line) + return TupleType(self.anal_array(info.tuple_type.items), + fallback=instance, + line=t.line) + else: + return AnyType() + + def visit_any(self, t: AnyType) -> Type: + return t + + def visit_void(self, t: Void) -> Type: + return t + + def visit_none_type(self, t: NoneTyp) -> Type: + return t + + def visit_uninhabited_type(self, t: UninhabitedType) -> Type: + return t + + def visit_deleted_type(self, t: DeletedType) -> Type: + return t + + def visit_type_list(self, t: TypeList) -> Type: + self.fail('Invalid type', t) + return AnyType() + + def visit_instance(self, t: Instance) -> Type: + return t + + def visit_type_var(self, t: TypeVarType) -> Type: + raise RuntimeError('TypeVarType is already analyzed') + + def visit_callable_type(self, t: CallableType) -> Type: + return t.copy_modified(arg_types=self.anal_array(t.arg_types), + ret_type=t.ret_type.accept(self), + fallback=t.fallback or self.builtin_type('builtins.function'), + variables=self.anal_var_defs(t.variables)) + + def visit_tuple_type(self, t: TupleType) -> Type: + if t.implicit: + self.fail('Invalid tuple literal type', t) + return AnyType() + star_count = sum(1 for item in t.items if isinstance(item, StarType)) + if star_count > 1: + self.fail('At most one star type allowed in a tuple', t) + return AnyType() + fallback = t.fallback if t.fallback else self.builtin_type('builtins.tuple', [AnyType()]) + return TupleType(self.anal_array(t.items), fallback, t.line) + + def visit_namedtuple_type(self, t: NamedTupleType) -> Type: + return NamedTupleType(t.name, t.attrs, + self.anal_array(t.items), + fallback=t.fallback, + line=t.line) + + def visit_star_type(self, t: StarType) -> Type: + return StarType(t.type.accept(self), t.line) + + def visit_union_type(self, t: UnionType) -> Type: + return UnionType(self.anal_array(t.items), t.line) + + def visit_partial_type(self, t: PartialType) -> Type: + assert False, "Internal error: Unexpected partial type" + + def visit_ellipsis_type(self, t: EllipsisType) -> Type: + self.fail("Unexpected '...'", t) + return AnyType() + + def visit_type_type(self, t: TypeType) -> Type: + return TypeType(t.item.accept(self), line=t.line) + + def analyze_callable_type(self, t: UnboundType) -> Type: + fallback = self.builtin_type('builtins.function') + if len(t.args) == 0: + # Callable (bare). Treat as Callable[..., Any]. + return CallableType([AnyType(), AnyType()], + [nodes.ARG_STAR, nodes.ARG_STAR2], + [None, None], + ret_type=AnyType(), + fallback=fallback, + is_ellipsis_args=True) + elif len(t.args) == 2: + ret_type = t.args[1].accept(self) + if isinstance(t.args[0], TypeList): + # Callable[[ARG, ...], RET] (ordinary callable type) + args = t.args[0].items + return CallableType(self.anal_array(args), + [nodes.ARG_POS] * len(args), + [None] * len(args), + ret_type=ret_type, + fallback=fallback) + elif isinstance(t.args[0], EllipsisType): + # Callable[..., RET] (with literal ellipsis; accept arbitrary arguments) + return CallableType([AnyType(), AnyType()], + [nodes.ARG_STAR, nodes.ARG_STAR2], + [None, None], + ret_type=ret_type, + fallback=fallback, + is_ellipsis_args=True) + else: + self.fail('The first argument to Callable must be a list of types or "..."', t) + return AnyType() + + self.fail('Invalid function type', t) + return AnyType() + + def anal_array(self, a: List[Type]) -> List[Type]: + res = [] # type: List[Type] + for t in a: + res.append(t.accept(self)) + return res + + def anal_var_defs(self, var_defs: List[TypeVarDef]) -> List[TypeVarDef]: + a = [] # type: List[TypeVarDef] + for vd in var_defs: + a.append(TypeVarDef(vd.name, vd.id.raw_id, self.anal_array(vd.values), + vd.upper_bound.accept(self), + vd.variance, + vd.line)) + return a + + def builtin_type(self, fully_qualified_name: str, args: List[Type] = None) -> Instance: + node = self.lookup_fqn_func(fully_qualified_name) + info = cast(TypeInfo, node.node) + return Instance(info, args or []) + + def tuple_type(self, items: List[Type]) -> TupleType: + return TupleType(items, fallback=self.builtin_type('builtins.tuple', [AnyType()])) + + +class TypeAnalyserPass3(TypeVisitor[None]): + """Analyze type argument counts and values of generic types. + + This is semantic analysis pass 3 for types. + + Perform these operations: + + * Report error for invalid type argument counts, such as List[x, y]. + * Make implicit Any type arguments explicit my modifying types + in-place. For example, modify Foo into Foo[Any] if Foo expects a single + type argument. + * If a type variable has a value restriction, ensure that the value is + valid. For example, reject IO[int] if the type argument must be str + or bytes. + + We can't do this earlier than the third pass, since type argument counts + are only determined in pass 2, and we have to support forward references + to types. + """ + + def __init__(self, fail_func: Callable[[str, Context], None]) -> None: + self.fail = fail_func + + def visit_instance(self, t: Instance) -> None: + info = t.type + # Check type argument count. + if len(t.args) != len(info.type_vars): + if len(t.args) == 0: + # Insert implicit 'Any' type arguments. + t.args = [AnyType()] * len(info.type_vars) + return + # Invalid number of type parameters. + n = len(info.type_vars) + s = '{} type arguments'.format(n) + if n == 0: + s = 'no type arguments' + elif n == 1: + s = '1 type argument' + act = str(len(t.args)) + if act == '0': + act = 'none' + self.fail('"{}" expects {}, but {} given'.format( + info.name(), s, act), t) + # Construct the correct number of type arguments, as + # otherwise the type checker may crash as it expects + # things to be right. + t.args = [AnyType() for _ in info.type_vars] + elif info.defn.type_vars: + # Check type argument values. + for arg, TypeVar in zip(t.args, info.defn.type_vars): + if TypeVar.values: + if isinstance(arg, TypeVarType): + arg_values = arg.values + if not arg_values: + self.fail('Type variable "{}" not valid as type ' + 'argument value for "{}"'.format( + arg.name, info.name()), t) + continue + else: + arg_values = [arg] + self.check_type_var_values(info, arg_values, + TypeVar.values, t) + if not satisfies_upper_bound(arg, TypeVar.upper_bound): + self.fail('Type argument "{}" of "{}" must be ' + 'a subtype of "{}"'.format( + arg, info.name(), TypeVar.upper_bound), t) + for arg in t.args: + arg.accept(self) + + def check_type_var_values(self, type: TypeInfo, actuals: List[Type], + valids: List[Type], context: Context) -> None: + for actual in actuals: + if (not isinstance(actual, AnyType) and + not any(is_same_type(actual, value) for value in valids)): + self.fail('Invalid type argument value for "{}"'.format( + type.name()), context) + + def visit_callable_type(self, t: CallableType) -> None: + t.ret_type.accept(self) + for arg_type in t.arg_types: + arg_type.accept(self) + + def visit_tuple_type(self, t: TupleType) -> None: + for item in t.items: + item.accept(self) + + def visit_union_type(self, t: UnionType) -> None: + for item in t.items: + item.accept(self) + + def visit_star_type(self, t: StarType) -> None: + t.type.accept(self) + + # Other kinds of type are trivial, since they are atomic (or invalid). + + def visit_unbound_type(self, t: UnboundType) -> None: + pass + + def visit_any(self, t: AnyType) -> None: + pass + + def visit_void(self, t: Void) -> None: + pass + + def visit_none_type(self, t: NoneTyp) -> None: + pass + + def visit_uninhabited_type(self, t: UninhabitedType) -> None: + pass + + def visit_deleted_type(self, t: DeletedType) -> None: + pass + + def visit_type_list(self, t: TypeList) -> None: + self.fail('Invalid type', t) + + def visit_type_var(self, t: TypeVarType) -> None: + pass + + def visit_partial_type(self, t: PartialType) -> None: + pass + + def visit_type_type(self, t: TypeType) -> None: + pass diff --git a/mypy/types.py b/mypy/types.py index 7d8dcb1c1a603..57bc7bd6b26f7 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1,1451 +1,1451 @@ -"""Classes for representing mypy types.""" - -from abc import abstractmethod -from typing import ( - Any, TypeVar, Dict, List, Tuple, cast, Generic, Set, Sequence, Optional, Union -) - -import mypy.nodes -from mypy.nodes import INVARIANT, SymbolNode - -from mypy import experiments - - -T = TypeVar('T') - -JsonDict = Dict[str, Any] - - -class Type(mypy.nodes.Context): - """Abstract base class for all types.""" - - line = 0 - - def __init__(self, line: int = -1) -> None: - self.line = line - - def get_line(self) -> int: - return self.line - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - raise RuntimeError('Not implemented') - - def __repr__(self) -> str: - return self.accept(TypeStrVisitor()) - - def serialize(self) -> JsonDict: - raise NotImplementedError('Cannot serialize {} instance'.format(self.__class__.__name__)) - - @classmethod - def deserialize(cls, data: JsonDict) -> 'Type': - classname = data['.class'] - glo = globals() - if classname in glo: - cl = glo[classname] - if 'deserialize' in cl.__dict__: - return cl.deserialize(data) - raise NotImplementedError('unexpected .class {}'.format(classname)) - - -class TypeVarId: - # A type variable is uniquely identified by its raw id and meta level. - - # For plain variables (type parameters of generic classes and - # functions) raw ids are allocated by semantic analysis, using - # positive ids 1, 2, ... for generic class parameters and negative - # ids -1, ... for generic function type arguments. This convention - # is only used to keep type variable ids distinct when allocating - # them; the type checker makes no distinction between class and - # function type variables. - - # Metavariables are allocated unique ids starting from 1. - raw_id = 0 # type: int - - # Level of the variable in type inference. Currently either 0 for - # declared types, or 1 for type inference metavariables. - meta_level = 0 # type: int - - # Class variable used for allocating fresh ids for metavariables. - next_raw_id = 1 # type: int - - def __init__(self, raw_id: int, meta_level: int = 0) -> None: - self.raw_id = raw_id - self.meta_level = meta_level - - @staticmethod - def new(meta_level: int) -> 'TypeVarId': - raw_id = TypeVarId.next_raw_id - TypeVarId.next_raw_id += 1 - return TypeVarId(raw_id, meta_level) - - def __repr__(self) -> str: - return self.raw_id.__repr__() - - def __eq__(self, other: object) -> bool: - if isinstance(other, TypeVarId): - return (self.raw_id == other.raw_id and - self.meta_level == other.meta_level) - else: - return False - - def __ne__(self, other: object) -> bool: - return not (self == other) - - def __hash__(self) -> int: - return hash((self.raw_id, self.meta_level)) - - def is_meta_var(self) -> bool: - return self.meta_level > 0 - - -class TypeVarDef(mypy.nodes.Context): - """Definition of a single type variable.""" - - name = '' - id = None # type: TypeVarId - values = None # type: List[Type] # Value restriction, empty list if no restriction - upper_bound = None # type: Type - variance = INVARIANT # type: int - line = 0 - - def __init__(self, name: str, id: Union[TypeVarId, int], values: Optional[List[Type]], - upper_bound: Type, variance: int = INVARIANT, line: int = -1) -> None: - self.name = name - if isinstance(id, int): - id = TypeVarId(id) - self.id = id - self.values = values - self.upper_bound = upper_bound - self.variance = variance - self.line = line - - @staticmethod - def new_unification_variable(old: 'TypeVarDef') -> 'TypeVarDef': - new_id = TypeVarId.new(meta_level=1) - return TypeVarDef(old.name, new_id, old.values, - old.upper_bound, old.variance, old.line) - - def get_line(self) -> int: - return self.line - - def __repr__(self) -> str: - if self.values: - return '{} in {}'.format(self.name, tuple(self.values)) - elif not is_named_instance(self.upper_bound, 'builtins.object'): - return '{} <: {}'.format(self.name, self.upper_bound) - else: - return self.name - - def serialize(self) -> JsonDict: - assert not self.id.is_meta_var() - return {'.class': 'TypeVarDef', - 'name': self.name, - 'id': self.id.raw_id, - 'values': None if self.values is None else [v.serialize() for v in self.values], - 'upper_bound': self.upper_bound.serialize(), - 'variance': self.variance, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'TypeVarDef': - assert data['.class'] == 'TypeVarDef' - return TypeVarDef(data['name'], - data['id'], - None if data['values'] is None - else [Type.deserialize(v) for v in data['values']], - Type.deserialize(data['upper_bound']), - data['variance'], - ) - - -class UnboundType(Type): - """Instance type that has not been bound during semantic analysis.""" - - name = '' - args = None # type: List[Type] - # should this type be wrapped in an Optional? - optional = False - # is this type a return type? - is_ret_type = False - - def __init__(self, - name: str, - args: List[Type] = None, - line: int = -1, - optional: bool = False, - is_ret_type: bool = False) -> None: - if not args: - args = [] - self.name = name - self.args = args - self.optional = optional - self.is_ret_type = is_ret_type - super().__init__(line) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_unbound_type(self) - - def serialize(self) -> JsonDict: - return {'.class': 'UnboundType', - 'name': self.name, - 'args': [a.serialize() for a in self.args], - } - - @classmethod - def deserialize(self, data: JsonDict) -> 'UnboundType': - assert data['.class'] == 'UnboundType' - return UnboundType(data['name'], - [Type.deserialize(a) for a in data['args']]) - - -class ErrorType(Type): - """The error type is used as the result of failed type operations.""" - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_error_type(self) - - -class TypeList(Type): - """A list of types [...]. - - This is only used for the arguments of a Callable type, i.e. for - [arg, ...] in Callable[[arg, ...], ret]. This is not a real type - but a syntactic AST construct. - """ - - items = None # type: List[Type] - - def __init__(self, items: List[Type], line: int = -1) -> None: - super().__init__(line) - self.items = items - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_type_list(self) - - def serialize(self) -> JsonDict: - return {'.class': 'TypeList', - 'items': [t.serialize() for t in self.items], - } - - @classmethod - def deserialize(self, data: JsonDict) -> 'TypeList': - assert data['.class'] == 'TypeList' - return TypeList([Type.deserialize(t) for t in data['items']]) - - -class AnyType(Type): - """The type 'Any'.""" - - def __init__(self, implicit=False, line: int = -1) -> None: - super().__init__(line) - self.implicit = implicit - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_any(self) - - def serialize(self) -> JsonDict: - return {'.class': 'AnyType'} - - @classmethod - def deserialize(cls, data: JsonDict) -> 'AnyType': - assert data['.class'] == 'AnyType' - return AnyType() - - -class Void(Type): - """The return type 'None'. - - This can only be used as the return type in a callable type and as - the result type of calling such callable. - """ - - source = '' # May be None; function that generated this value - - def __init__(self, source: str = None, line: int = -1) -> None: - self.source = source - super().__init__(line) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_void(self) - - def with_source(self, source: str) -> 'Void': - return Void(source, self.line) - - def serialize(self) -> JsonDict: - return {'.class': 'Void'} - - @classmethod - def deserialize(cls, data: JsonDict) -> 'Void': - assert data['.class'] == 'Void' - return Void() - - -class UninhabitedType(Type): - """This type has no members. - - This type is almost the bottom type, except it is not a subtype of Void. - With strict Optional checking, it is the only common subtype between all - other types, which allows `meet` to be well defined. Without strict - Optional checking, NoneTyp fills this role. - - In general, for any type T that isn't Void: - join(UninhabitedType, T) = T - meet(UninhabitedType, T) = UninhabitedType - is_subtype(UninhabitedType, T) = True - """ - - def __init__(self, line: int = -1) -> None: - super().__init__(line) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_uninhabited_type(self) - - def serialize(self) -> JsonDict: - return {'.class': 'UninhabitedType'} - - @classmethod - def deserialize(cls, data: JsonDict) -> 'UninhabitedType': - assert data['.class'] == 'UninhabitedType' - return UninhabitedType() - - -class NoneTyp(Type): - """The type of 'None'. - - Without strict Optional checking: - This is only used internally during type inference. Programs - cannot declare a variable of this type, and the type checker - refuses to infer this type for a variable. However, subexpressions - often have this type. Note that this is not used as the result - type when calling a function with a void type, even though - semantically such a function returns a None value; the void type - is used instead so that we can report an error if the caller tries - to do anything with the return value. - - With strict Optional checking: - This type can be written by users as 'None', except as the return value - of a function, where 'None' means Void. - """ - - def __init__(self, line: int = -1) -> None: - super().__init__(line) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_none_type(self) - - def serialize(self) -> JsonDict: - return {'.class': 'NoneTyp'} - - @classmethod - def deserialize(self, data: JsonDict) -> 'NoneTyp': - assert data['.class'] == 'NoneTyp' - return NoneTyp() - - -class ErasedType(Type): - """Placeholder for an erased type. - - This is used during type inference. This has the special property that - it is ignored during type inference. - """ - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_erased_type(self) - - -class DeletedType(Type): - """Type of deleted variables. - - These can be used as lvalues but not rvalues. - """ - - source = '' # May be None; name that generated this value - - def __init__(self, source: str = None, line: int = -1) -> None: - self.source = source - super().__init__(line) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_deleted_type(self) - - def serialize(self) -> JsonDict: - return {'.class': 'DeletedType', - 'source': self.source} - - @classmethod - def deserialize(self, data: JsonDict) -> 'DeletedType': - assert data['.class'] == 'DeletedType' - return DeletedType(data['source']) - - -class Instance(Type): - """An instance type of form C[T1, ..., Tn]. - - The list of type variables may be empty. - """ - - type = None # type: mypy.nodes.TypeInfo - args = None # type: List[Type] - erased = False # True if result of type variable substitution - - def __init__(self, typ: mypy.nodes.TypeInfo, args: List[Type], - line: int = -1, erased: bool = False) -> None: - self.type = typ - self.args = args - self.erased = erased - super().__init__(line) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_instance(self) - - type_ref = None # type: str - - def serialize(self) -> JsonDict: - data = {'.class': 'Instance', - } # type: JsonDict - assert self.type is not None - data['type_ref'] = self.type.alt_fullname or self.type.fullname() - if self.args: - data['args'] = [arg.serialize() for arg in self.args] - return data - - @classmethod - def deserialize(cls, data: JsonDict) -> 'Instance': - assert data['.class'] == 'Instance' - args = [] # type: List[Type] - if 'args' in data: - args_list = data['args'] - assert isinstance(args_list, list) - args = [Type.deserialize(arg) for arg in args_list] - inst = Instance(None, args) - inst.type_ref = data['type_ref'] # Will be fixed up by fixup.py later. - return inst - - -class TypeVarType(Type): - """A type variable type. - - This refers to either a class type variable (id > 0) or a function - type variable (id < 0). - """ - - name = '' # Name of the type variable (for messages and debugging) - id = None # type: TypeVarId - values = None # type: List[Type] # Value restriction, empty list if no restriction - upper_bound = None # type: Type # Upper bound for values - # See comments in TypeVarDef for more about variance. - variance = INVARIANT # type: int - - def __init__(self, binder: TypeVarDef, line: int = -1) -> None: - self.name = binder.name - self.id = binder.id - self.values = binder.values - self.upper_bound = binder.upper_bound - self.variance = binder.variance - super().__init__(line) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_type_var(self) - - def erase_to_union_or_bound(self) -> Type: - if self.values: - return UnionType.make_simplified_union(self.values) - else: - return self.upper_bound - - def serialize(self) -> JsonDict: - assert not self.id.is_meta_var() - return {'.class': 'TypeVarType', - 'name': self.name, - 'id': self.id.raw_id, - 'values': [v.serialize() for v in self.values], - 'upper_bound': self.upper_bound.serialize(), - 'variance': self.variance, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'TypeVarType': - assert data['.class'] == 'TypeVarType' - tvdef = TypeVarDef(data['name'], - data['id'], - [Type.deserialize(v) for v in data['values']], - Type.deserialize(data['upper_bound']), - data['variance']) - return TypeVarType(tvdef) - - -class FunctionLike(Type): - """Abstract base class for function types.""" - - @abstractmethod - def is_type_obj(self) -> bool: pass - - def is_concrete_type_obj(self) -> bool: - return self.is_type_obj() - - @abstractmethod - def type_object(self) -> mypy.nodes.TypeInfo: pass - - @abstractmethod - def items(self) -> List['CallableType']: pass - - @abstractmethod - def with_name(self, name: str) -> 'FunctionLike': pass - - # Corresponding instance type (e.g. builtins.type) - fallback = None # type: Instance - - @classmethod - def deserialize(cls, data: JsonDict) -> 'FunctionLike': - return cast(FunctionLike, super().deserialize(data)) - - -_dummy = object() # type: Any - - -class CallableType(FunctionLike): - """Type of a non-overloaded callable object (function).""" - - arg_types = None # type: List[Type] # Types of function arguments - arg_kinds = None # type: List[int] # mypy.nodes.ARG_ constants - arg_names = None # type: List[str] # None if not a keyword argument - min_args = 0 # Minimum number of arguments; derived from arg_kinds - is_var_arg = False # Is it a varargs function? derived from arg_kinds - ret_type = None # type: Type # Return value type - name = '' # Name (may be None; for error messages) - definition = None # type: SymbolNode # For error messages. May be None. - # Type variables for a generic function - variables = None # type: List[TypeVarDef] - - # Is this Callable[..., t] (with literal '...')? - is_ellipsis_args = False - # Is this callable constructed for the benefit of a classmethod's 'cls' argument? - is_classmethod_class = False - # Was this type implicitly generated instead of explicitly specified by the user? - implicit = False - # Defined for signatures that require special handling (currently only value is 'dict' - # for a signature similar to 'dict') - special_sig = None # type: Optional[str] - - def __init__(self, - arg_types: List[Type], - arg_kinds: List[int], - arg_names: List[str], - ret_type: Type, - fallback: Instance, - name: str = None, - definition: SymbolNode = None, - variables: List[TypeVarDef] = None, - line: int = -1, - is_ellipsis_args: bool = False, - implicit=False, - is_classmethod_class=False, - special_sig=None, - ) -> None: - if variables is None: - variables = [] - self.arg_types = arg_types - self.arg_kinds = arg_kinds - self.arg_names = arg_names - self.min_args = arg_kinds.count(mypy.nodes.ARG_POS) - self.is_var_arg = mypy.nodes.ARG_STAR in arg_kinds - self.ret_type = ret_type - self.fallback = fallback - assert not name or ' 'CallableType': - return CallableType( - arg_types=arg_types if arg_types is not _dummy else self.arg_types, - arg_kinds=arg_kinds if arg_kinds is not _dummy else self.arg_kinds, - arg_names=arg_names if arg_names is not _dummy else self.arg_names, - ret_type=ret_type if ret_type is not _dummy else self.ret_type, - fallback=fallback if fallback is not _dummy else self.fallback, - name=name if name is not _dummy else self.name, - definition=definition if definition is not _dummy else self.definition, - variables=variables if variables is not _dummy else self.variables, - line=line if line is not _dummy else self.line, - is_ellipsis_args=( - is_ellipsis_args if is_ellipsis_args is not _dummy else self.is_ellipsis_args), - implicit=self.implicit, - is_classmethod_class=self.is_classmethod_class, - special_sig=special_sig if special_sig is not _dummy else self.special_sig, - ) - - def is_type_obj(self) -> bool: - return self.fallback.type is not None and self.fallback.type.fullname() == 'builtins.type' - - def is_concrete_type_obj(self) -> bool: - return self.is_type_obj() and self.is_classmethod_class - - def type_object(self) -> mypy.nodes.TypeInfo: - assert self.is_type_obj() - ret = self.ret_type - if isinstance(ret, TupleType): - ret = ret.fallback - return cast(Instance, ret).type - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_callable_type(self) - - def with_name(self, name: str) -> 'CallableType': - """Return a copy of this type with the specified name.""" - ret = self.ret_type - if isinstance(ret, Void): - ret = ret.with_source(name) - return self.copy_modified(ret_type=ret, name=name) - - def max_fixed_args(self) -> int: - n = len(self.arg_types) - if self.is_var_arg: - n -= 1 - return n - - def items(self) -> List['CallableType']: - return [self] - - def is_generic(self) -> bool: - return bool(self.variables) - - def type_var_ids(self) -> List[TypeVarId]: - a = [] # type: List[TypeVarId] - for tv in self.variables: - a.append(tv.id) - return a - - def serialize(self) -> JsonDict: - # TODO: As an optimization, leave out everything related to - # generic functions for non-generic functions. - return {'.class': 'CallableType', - 'arg_types': [(None if t is None else t.serialize()) - for t in self.arg_types], - 'arg_kinds': self.arg_kinds, - 'arg_names': self.arg_names, - 'ret_type': self.ret_type.serialize(), - 'fallback': self.fallback.serialize(), - 'name': self.name, - # We don't serialize the definition (only used for error messages). - 'variables': [v.serialize() for v in self.variables], - 'is_ellipsis_args': self.is_ellipsis_args, - 'implicit': self.implicit, - 'is_classmethod_class': self.is_classmethod_class, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'CallableType': - assert data['.class'] == 'CallableType' - # TODO: Set definition to the containing SymbolNode? - return CallableType([(None if t is None else Type.deserialize(t)) - for t in data['arg_types']], - data['arg_kinds'], - data['arg_names'], - Type.deserialize(data['ret_type']), - Instance.deserialize(data['fallback']), - name=data['name'], - variables=[TypeVarDef.deserialize(v) for v in data['variables']], - is_ellipsis_args=data['is_ellipsis_args'], - implicit=data['implicit'], - is_classmethod_class=data['is_classmethod_class'], - ) - - -class Overloaded(FunctionLike): - """Overloaded function type T1, ... Tn, where each Ti is CallableType. - - The variant to call is chosen based on static argument - types. Overloaded function types can only be defined in stub - files, and thus there is no explicit runtime dispatch - implementation. - """ - - _items = None # type: List[CallableType] # Must not be empty - - def __init__(self, items: List[CallableType]) -> None: - self._items = items - self.fallback = items[0].fallback - super().__init__(items[0].line) - - def items(self) -> List[CallableType]: - return self._items - - def name(self) -> str: - return self._items[0].name - - def is_type_obj(self) -> bool: - # All the items must have the same type object status, so it's - # sufficient to query only (any) one of them. - return self._items[0].is_type_obj() - - def type_object(self) -> mypy.nodes.TypeInfo: - # All the items must have the same type object, so it's sufficient to - # query only (any) one of them. - return self._items[0].type_object() - - def with_name(self, name: str) -> 'Overloaded': - ni = [] # type: List[CallableType] - for it in self._items: - ni.append(it.with_name(name)) - return Overloaded(ni) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_overloaded(self) - - def serialize(self) -> JsonDict: - return {'.class': 'Overloaded', - 'items': [t.serialize() for t in self.items()], - } - - @classmethod - def deserialize(self, data: JsonDict) -> 'Overloaded': - assert data['.class'] == 'Overloaded' - return Overloaded([CallableType.deserialize(t) for t in data['items']]) - - -class TupleType(Type): - """The tuple type Tuple[T1, ..., Tn] (at least one type argument). - - Instance variables: - items: tuple item types - fallback: the underlying instance type that is used for non-tuple methods - (this is currently always builtins.tuple, but it could be different for named - tuples, for example) - implicit: if True, derived from a tuple expression (t,....) instead of Tuple[t, ...] - """ - - items = None # type: List[Type] - fallback = None # type: Instance - implicit = False - - def __init__(self, items: List[Type], fallback: Instance, line: int = -1, - implicit: bool = False) -> None: - #import pdb; pdb.set_trace() - self.items = items - self.fallback = fallback - self.implicit = implicit - super().__init__(line) - - def length(self) -> int: - return len(self.items) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_tuple_type(self) - - def serialize(self) -> JsonDict: - return {'.class': 'TupleType', - 'items': [t.serialize() for t in self.items], - 'fallback': self.fallback.serialize(), - 'implicit': self.implicit, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'TupleType': - assert data['.class'] == 'TupleType' - return TupleType([Type.deserialize(t) for t in data['items']], - Instance.deserialize(data['fallback']), - implicit=data['implicit']) - - def self_type(self, inst: Instance) -> 'TupleType': - return TupleType(self.items, inst, self.line, self.implicit) - - def with_types(self, items: List[Type]) -> 'TupleType': - return TupleType(items, self.fallback, self.line, self.implicit) - - def slice(self, begin: int, stride: int, end: int) -> 'TupleType': - return TupleType(self.items[begin:end:stride], self.fallback, - self.line, self.implicit) - - -class NamedTupleType(TupleType): - attrs = None # type: List[str] - name = None # type: str - - def __init__(self, name: str, attrs: List[str], *args, **kwargs) -> None: - self.attrs = attrs - self.name = name - super().__init__(*args, **kwargs) - - def self_type(self, inst: Instance) -> 'NamedTupleType': - return NamedTupleType(self.name, self.attrs, self.items, inst, self.line) - - def with_types(self, items: List[Type]) -> 'NamedTupleType': - return NamedTupleType(self.name, self.attrs, items, - self.fallback, self.line, self.implicit) - - def __eq__(self, other) -> bool: - return self.serialize() == other.serialize() - - def serialize(self) -> JsonDict: - d = super().serialize() - d.update({'.class': 'NamedTupleType', - 'name': self.name, - 'attrs': self.attrs}) - return d - - @classmethod - def deserialize(cls, data: JsonDict) -> 'NamedTupleType': - assert data['.class'] == 'NamedTupleType' - data['.class'] = 'TupleType' - tup = TupleType.deserialize(data) - return NamedTupleType(data['name'], - data['attrs'], - tup.items, - tup.fallback, - implicit=tup.implicit) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - try: - return visitor.visit_namedtuple_type(self) - except: - return super().accept(visitor) - - -class StarType(Type): - """The star type *type_parameter. - - This is not a real type but a syntactic AST construct. - """ - - type = None # type: Type - - def __init__(self, type: Type, line: int = -1) -> None: - self.type = type - super().__init__(line) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_star_type(self) - - -class UnionType(Type): - """The union type Union[T1, ..., Tn] (at least one type argument).""" - - items = None # type: List[Type] - - def __init__(self, items: List[Type], line: int = -1) -> None: - self.items = items - super().__init__(line) - - @staticmethod - def make_union(items: List[Type], line: int = -1) -> Type: - if len(items) > 1: - return UnionType(items, line) - elif len(items) == 1: - return items[0] - else: - if experiments.STRICT_OPTIONAL: - return UninhabitedType() - else: - return Void() - - @staticmethod - def make_simplified_union(items: List[Type], line: int = -1) -> Type: - while any(isinstance(typ, UnionType) for typ in items): - all_items = [] # type: List[Type] - for typ in items: - if isinstance(typ, UnionType): - all_items.extend(typ.items) - else: - all_items.append(typ) - items = all_items - - if any(isinstance(typ, AnyType) for typ in items): - return AnyType() - - from mypy.subtypes import is_subtype - removed = set() # type: Set[int] - for i in range(len(items)): - if any(is_subtype(items[i], items[j]) for j in range(len(items)) - if j not in removed and j != i): - removed.add(i) - - simplified_set = [items[i] for i in range(len(items)) if i not in removed] - return UnionType.make_union(simplified_set) - - def length(self) -> int: - return len(self.items) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_union_type(self) - - def has_readable_member(self, name: str) -> bool: - """For a tree of unions of instances, check whether all instances have a given member. - - TODO: Deal with attributes of TupleType etc. - TODO: This should probably be refactored to go elsewhere. - """ - return all((isinstance(x, UnionType) and x.has_readable_member(name)) or - (isinstance(x, Instance) and x.type.has_readable_member(name)) - for x in self.items) - - def serialize(self) -> JsonDict: - return {'.class': 'UnionType', - 'items': [t.serialize() for t in self.items], - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'UnionType': - assert data['.class'] == 'UnionType' - return UnionType([Type.deserialize(t) for t in data['items']]) - - -class PartialType(Type): - """Type such as List[?] where type arguments are unknown, or partial None type. - - These are used for inferring types in multiphase initialization such as this: - - x = [] # x gets a partial type List[?], as item type is unknown - x.append(1) # partial type gets replaced with normal type List[int] - - Or with None: - - x = None # x gets a partial type None - if c: - x = 1 # Infer actual type int for x - """ - - # None for the 'None' partial type; otherwise a generic class - type = None # type: Optional[mypy.nodes.TypeInfo] - var = None # type: mypy.nodes.Var - inner_types = None # type: List[Type] - - def __init__(self, - type: Optional['mypy.nodes.TypeInfo'], - var: 'mypy.nodes.Var', - inner_types: List[Type]) -> None: - self.type = type - self.var = var - self.inner_types = inner_types - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_partial_type(self) - - -class EllipsisType(Type): - """The type ... (ellipsis). - - This is not a real type but a syntactic AST construct, used in Callable[..., T], for example. - - A semantically analyzed type will never have ellipsis types. - """ - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_ellipsis_type(self) - - def serialize(self) -> JsonDict: - return {'.class': 'EllipsisType'} - - @classmethod - def deserialize(self, data: JsonDict) -> 'EllipsisType': - assert data['.class'] == 'EllipsisType' - return EllipsisType() - - -class TypeType(Type): - """For types like Type[User]. - - This annotates variables that are class objects, constrained by - the type argument. See PEP 484 for more details. - - We may encounter expressions whose values are specific classes; - those are represented as callables (possibly overloaded) - corresponding to the class's constructor's signature and returning - an instance of that class. The difference with Type[C] is that - those callables always represent the exact class given as the - return type; Type[C] represents any class that's a subclass of C, - and C may also be a type variable or a union (or Any). - - Many questions around subtype relationships between Type[C1] and - def(...) -> C2 are answered by looking at the subtype - relationships between C1 and C2, since Type[] is considered - covariant. - - There's an unsolved problem with constructor signatures (also - unsolved in PEP 484): calling a variable whose type is Type[C] - assumes the constructor signature for C, even though a subclass of - C might completely change the constructor signature. For now we - just assume that users of Type[C] are careful not to do that (in - the future we might detect when they are violating that - assumption). - """ - - # This can't be everything, but it can be a class reference, - # a generic class instance, a union, Any, a type variable... - item = None # type: Type - - def __init__(self, item: Type, *, line: int = -1) -> None: - super().__init__(line) - self.item = item - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_type_type(self) - - def serialize(self) -> JsonDict: - return {'.class': 'TypeType', 'item': self.item.serialize()} - - def deserialize(self, data: JsonDict) -> 'TypeType': - assert data['.class'] == 'TypeType' - return TypeType(Type.deserialize(data['item'])) - - -# -# Visitor-related classes -# - - -class TypeVisitor(Generic[T]): - """Visitor class for types (Type subclasses). - - The parameter T is the return type of the visit methods. - """ - - def _notimplemented_helper(self, name: str) -> NotImplementedError: - return NotImplementedError("Method {}.visit_{}() not implemented\n" - .format(type(self).__name__, name) - + "This is a known bug, track development in " - + "'https://github.com/JukkaL/mypy/issues/730'") - - @abstractmethod - def visit_unbound_type(self, t: UnboundType) -> T: - pass - - def visit_type_list(self, t: TypeList) -> T: - raise self._notimplemented_helper('type_list') - - def visit_error_type(self, t: ErrorType) -> T: - raise self._notimplemented_helper('error_type') - - @abstractmethod - def visit_any(self, t: AnyType) -> T: - pass - - @abstractmethod - def visit_void(self, t: Void) -> T: - pass - - @abstractmethod - def visit_none_type(self, t: NoneTyp) -> T: - pass - - @abstractmethod - def visit_uninhabited_type(self, t: UninhabitedType) -> T: - pass - - def visit_erased_type(self, t: ErasedType) -> T: - raise self._notimplemented_helper('erased_type') - - @abstractmethod - def visit_deleted_type(self, t: DeletedType) -> T: - pass - - @abstractmethod - def visit_type_var(self, t: TypeVarType) -> T: - pass - - @abstractmethod - def visit_instance(self, t: Instance) -> T: - pass - - @abstractmethod - def visit_callable_type(self, t: CallableType) -> T: - pass - - def visit_overloaded(self, t: Overloaded) -> T: - raise self._notimplemented_helper('overloaded') - - @abstractmethod - def visit_tuple_type(self, t: TupleType) -> T: - pass - - def visit_star_type(self, t: StarType) -> T: - raise self._notimplemented_helper('star_type') - - @abstractmethod - def visit_union_type(self, t: UnionType) -> T: - pass - - @abstractmethod - def visit_partial_type(self, t: PartialType) -> T: - pass - - def visit_ellipsis_type(self, t: EllipsisType) -> T: - raise self._notimplemented_helper('ellipsis_type') - - @abstractmethod - def visit_type_type(self, t: TypeType) -> T: - pass - - -class TypeTranslator(TypeVisitor[Type]): - """Identity type transformation. - - Subclass this and override some methods to implement a non-trivial - transformation. - """ - - def visit_unbound_type(self, t: UnboundType) -> Type: - return t - - def visit_type_list(self, t: TypeList) -> Type: - return t - - def visit_error_type(self, t: ErrorType) -> Type: - return t - - def visit_any(self, t: AnyType) -> Type: - return t - - def visit_void(self, t: Void) -> Type: - return t - - def visit_none_type(self, t: NoneTyp) -> Type: - return t - - def visit_uninhabited_type(self, t: UninhabitedType) -> Type: - return t - - def visit_erased_type(self, t: ErasedType) -> Type: - return t - - def visit_deleted_type(self, t: DeletedType) -> Type: - return t - - def visit_instance(self, t: Instance) -> Type: - return Instance(t.type, self.translate_types(t.args), t.line) - - def visit_type_var(self, t: TypeVarType) -> Type: - return t - - def visit_partial_type(self, t: PartialType) -> Type: - return t - - def visit_callable_type(self, t: CallableType) -> Type: - return t.copy_modified(arg_types=self.translate_types(t.arg_types), - ret_type=t.ret_type.accept(self), - variables=self.translate_variables(t.variables)) - - def visit_tuple_type(self, t: TupleType) -> Type: - return TupleType(self.translate_types(t.items), - cast(Any, t.fallback.accept(self)), - t.line) - - def visit_star_type(self, t: StarType) -> Type: - return StarType(t.type.accept(self), t.line) - - def visit_union_type(self, t: UnionType) -> Type: - return UnionType(self.translate_types(t.items), t.line) - - def visit_ellipsis_type(self, t: EllipsisType) -> Type: - return t - - def translate_types(self, types: List[Type]) -> List[Type]: - return [t.accept(self) for t in types] - - def translate_variables(self, - variables: List[TypeVarDef]) -> List[TypeVarDef]: - return variables - - def visit_overloaded(self, t: Overloaded) -> Type: - items = [] # type: List[CallableType] - for item in t.items(): - new = item.accept(self) - if isinstance(new, CallableType): - items.append(new) - else: - raise RuntimeError('CallableType expectected, but got {}'.format(type(new))) - return Overloaded(items=items) - - def visit_type_type(self, t: TypeType) -> Type: - return TypeType(t.item.accept(self), line=t.line) - - -class TypeStrVisitor(TypeVisitor[str]): - """Visitor for pretty-printing types into strings. - - This is mostly for debugging/testing. - - Do not preserve original formatting. - - Notes: - - Represent unbound types as Foo? or Foo?[...]. - - Represent the NoneTyp type as None. - """ - - def visit_unbound_type(self, t): - s = t.name + '?' - if t.args != []: - s += '[{}]'.format(self.list_str(t.args)) - return s - - def visit_type_list(self, t): - return ''.format(self.list_str(t.items)) - - def visit_error_type(self, t): - return '' - - def visit_any(self, t): - return 'Any' - - def visit_void(self, t): - return 'void' - - def visit_none_type(self, t): - # Fully qualify to make this distinct from the None value. - return "builtins.None" - - def visit_uninhabited_type(self, t): - return "" - - def visit_erased_type(self, t): - return "" - - def visit_deleted_type(self, t): - if t.source is None: - return "" - else: - return "".format(t.source) - - def visit_instance(self, t): - s = t.type.fullname() if t.type is not None else '' - if t.erased: - s += '*' - if t.args != []: - s += '[{}]'.format(self.list_str(t.args)) - return s - - def visit_type_var(self, t): - if t.name is None: - # Anonymous type variable type (only numeric id). - return '`{}'.format(t.id) - else: - # Named type variable type. - return '{}`{}'.format(t.name, t.id) - - def visit_callable_type(self, t): - s = '' - bare_asterisk = False - for i in range(len(t.arg_types)): - if s != '': - s += ', ' - if t.arg_kinds[i] == mypy.nodes.ARG_NAMED and not bare_asterisk: - s += '*, ' - bare_asterisk = True - if t.arg_kinds[i] == mypy.nodes.ARG_STAR: - s += '*' - if t.arg_kinds[i] == mypy.nodes.ARG_STAR2: - s += '**' - if t.arg_names[i]: - s += t.arg_names[i] + ': ' - s += str(t.arg_types[i]) - if t.arg_kinds[i] == mypy.nodes.ARG_OPT: - s += ' =' - - s = '({})'.format(s) - - if not isinstance(t.ret_type, Void): - s += ' -> {}'.format(t.ret_type) - - if t.variables: - s = '{} {}'.format(t.variables, s) - - return 'def {}'.format(s) - - def visit_overloaded(self, t): - a = [] - for i in t.items(): - a.append(i.accept(self)) - return 'Overload({})'.format(', '.join(a)) - - def visit_tuple_type(self, t): - s = self.list_str(t.items) - if t.fallback and t.fallback.type: - fallback_name = t.fallback.type.fullname() - if fallback_name != 'builtins.tuple': - return 'Tuple[{}, fallback={}]'.format(s, t.fallback.accept(self)) - return 'Tuple[{}]'.format(s) - - def visit_namedtuple_type(self, t): - s = self.list_str(zip(t.attrs, t.items)) - return '{}[{}]'.format(t.name, s) - - def visit_star_type(self, t): - s = t.type.accept(self) - return '*{}'.format(s) - - def visit_union_type(self, t): - s = self.list_str(t.items) - return 'Union[{}]'.format(s) - - def visit_partial_type(self, t: PartialType) -> str: - if t.type is None: - return '' - else: - return ''.format(t.type.name(), - ', '.join(['?'] * len(t.type.type_vars))) - - def visit_ellipsis_type(self, t): - return '...' - - def visit_type_type(self, t): - return 'Type[{}]'.format(t.item.accept(self)) - - def list_str(self, a): - """Convert items of an array to strings (pretty-print types) - and join the results with commas. - """ - res = [] - for t in a: - if isinstance(t, Type): - res.append(t.accept(self)) - else: - res.append(str(t)) - return ', '.join(res) - - -# These constants define the method used by TypeQuery to combine multiple -# query results, e.g. for tuple types. The strategy is not used for empty -# result lists; in that case the default value takes precedence. -ANY_TYPE_STRATEGY = 0 # Return True if any of the results are True. -ALL_TYPES_STRATEGY = 1 # Return True if all of the results are True. - - -class TypeQuery(TypeVisitor[bool]): - """Visitor for performing simple boolean queries of types. - - This class allows defining the default value for leafs to simplify the - implementation of many queries. - """ - - default = False # Default result - strategy = 0 # Strategy for combining multiple values (ANY_TYPE_STRATEGY or ALL_TYPES_...). - - def __init__(self, default: bool, strategy: int) -> None: - """Construct a query visitor. - - Use the given default result and strategy for combining - multiple results. The strategy must be either - ANY_TYPE_STRATEGY or ALL_TYPES_STRATEGY. - """ - self.default = default - self.strategy = strategy - - def visit_unbound_type(self, t: UnboundType) -> bool: - return self.default - - def visit_type_list(self, t: TypeList) -> bool: - return self.default - - def visit_error_type(self, t: ErrorType) -> bool: - return self.default - - def visit_any(self, t: AnyType) -> bool: - return self.default - - def visit_void(self, t: Void) -> bool: - return self.default - - def visit_uninhabited_type(self, t: UninhabitedType) -> bool: - return self.default - - def visit_none_type(self, t: NoneTyp) -> bool: - return self.default - - def visit_erased_type(self, t: ErasedType) -> bool: - return self.default - - def visit_deleted_type(self, t: DeletedType) -> bool: - return self.default - - def visit_type_var(self, t: TypeVarType) -> bool: - return self.default - - def visit_partial_type(self, t: PartialType) -> bool: - return self.default - - def visit_instance(self, t: Instance) -> bool: - return self.query_types(t.args) - - def visit_callable_type(self, t: CallableType) -> bool: - # FIX generics - return self.query_types(t.arg_types + [t.ret_type]) - - def visit_tuple_type(self, t: TupleType) -> bool: - return self.query_types(t.items) - - def visit_star_type(self, t: StarType) -> bool: - return t.type.accept(self) - - def visit_union_type(self, t: UnionType) -> bool: - return self.query_types(t.items) - - def visit_overloaded(self, t: Overloaded) -> bool: - return self.query_types(t.items()) - - def visit_type_type(self, t: TypeType) -> bool: - return t.item.accept(self) - - def query_types(self, types: Sequence[Type]) -> bool: - """Perform a query for a list of types. - - Use the strategy constant to combine the results. - """ - if not types: - # Use default result for empty list. - return self.default - if self.strategy == ANY_TYPE_STRATEGY: - # Return True if at least one component is true. - res = False - for t in types: - res = res or t.accept(self) - if res: - break - return res - else: - # Return True if all components are true. - res = True - for t in types: - res = res and t.accept(self) - if not res: - break - return res - - -def strip_type(typ: Type) -> Type: - """Make a copy of type without 'debugging info' (function name).""" - - if isinstance(typ, CallableType): - return typ.copy_modified(name=None) - elif isinstance(typ, Overloaded): - return Overloaded([cast(CallableType, strip_type(item)) - for item in typ.items()]) - else: - return typ - - -def replace_leading_arg_type(t: CallableType, self_type: Type) -> CallableType: - """Return a copy of a callable type with a different self argument type. - - Assume that the callable is the signature of a method. - """ - return t.copy_modified(arg_types=[self_type] + t.arg_types[1:]) - - -def is_named_instance(t: Type, fullname: str) -> bool: - return (isinstance(t, Instance) and - t.type is not None and - t.type.fullname() == fullname) +"""Classes for representing mypy types.""" + +from abc import abstractmethod +from typing import ( + Any, TypeVar, Dict, List, Tuple, cast, Generic, Set, Sequence, Optional, Union +) + +import mypy.nodes +from mypy.nodes import INVARIANT, SymbolNode + +from mypy import experiments + + +T = TypeVar('T') + +JsonDict = Dict[str, Any] + + +class Type(mypy.nodes.Context): + """Abstract base class for all types.""" + + line = 0 + + def __init__(self, line: int = -1) -> None: + self.line = line + + def get_line(self) -> int: + return self.line + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + raise RuntimeError('Not implemented') + + def __repr__(self) -> str: + return self.accept(TypeStrVisitor()) + + def serialize(self) -> JsonDict: + raise NotImplementedError('Cannot serialize {} instance'.format(self.__class__.__name__)) + + @classmethod + def deserialize(cls, data: JsonDict) -> 'Type': + classname = data['.class'] + glo = globals() + if classname in glo: + cl = glo[classname] + if 'deserialize' in cl.__dict__: + return cl.deserialize(data) + raise NotImplementedError('unexpected .class {}'.format(classname)) + + +class TypeVarId: + # A type variable is uniquely identified by its raw id and meta level. + + # For plain variables (type parameters of generic classes and + # functions) raw ids are allocated by semantic analysis, using + # positive ids 1, 2, ... for generic class parameters and negative + # ids -1, ... for generic function type arguments. This convention + # is only used to keep type variable ids distinct when allocating + # them; the type checker makes no distinction between class and + # function type variables. + + # Metavariables are allocated unique ids starting from 1. + raw_id = 0 # type: int + + # Level of the variable in type inference. Currently either 0 for + # declared types, or 1 for type inference metavariables. + meta_level = 0 # type: int + + # Class variable used for allocating fresh ids for metavariables. + next_raw_id = 1 # type: int + + def __init__(self, raw_id: int, meta_level: int = 0) -> None: + self.raw_id = raw_id + self.meta_level = meta_level + + @staticmethod + def new(meta_level: int) -> 'TypeVarId': + raw_id = TypeVarId.next_raw_id + TypeVarId.next_raw_id += 1 + return TypeVarId(raw_id, meta_level) + + def __repr__(self) -> str: + return self.raw_id.__repr__() + + def __eq__(self, other: object) -> bool: + if isinstance(other, TypeVarId): + return (self.raw_id == other.raw_id and + self.meta_level == other.meta_level) + else: + return False + + def __ne__(self, other: object) -> bool: + return not (self == other) + + def __hash__(self) -> int: + return hash((self.raw_id, self.meta_level)) + + def is_meta_var(self) -> bool: + return self.meta_level > 0 + + +class TypeVarDef(mypy.nodes.Context): + """Definition of a single type variable.""" + + name = '' + id = None # type: TypeVarId + values = None # type: List[Type] # Value restriction, empty list if no restriction + upper_bound = None # type: Type + variance = INVARIANT # type: int + line = 0 + + def __init__(self, name: str, id: Union[TypeVarId, int], values: Optional[List[Type]], + upper_bound: Type, variance: int = INVARIANT, line: int = -1) -> None: + self.name = name + if isinstance(id, int): + id = TypeVarId(id) + self.id = id + self.values = values + self.upper_bound = upper_bound + self.variance = variance + self.line = line + + @staticmethod + def new_unification_variable(old: 'TypeVarDef') -> 'TypeVarDef': + new_id = TypeVarId.new(meta_level=1) + return TypeVarDef(old.name, new_id, old.values, + old.upper_bound, old.variance, old.line) + + def get_line(self) -> int: + return self.line + + def __repr__(self) -> str: + if self.values: + return '{} in {}'.format(self.name, tuple(self.values)) + elif not is_named_instance(self.upper_bound, 'builtins.object'): + return '{} <: {}'.format(self.name, self.upper_bound) + else: + return self.name + + def serialize(self) -> JsonDict: + assert not self.id.is_meta_var() + return {'.class': 'TypeVarDef', + 'name': self.name, + 'id': self.id.raw_id, + 'values': None if self.values is None else [v.serialize() for v in self.values], + 'upper_bound': self.upper_bound.serialize(), + 'variance': self.variance, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'TypeVarDef': + assert data['.class'] == 'TypeVarDef' + return TypeVarDef(data['name'], + data['id'], + None if data['values'] is None + else [Type.deserialize(v) for v in data['values']], + Type.deserialize(data['upper_bound']), + data['variance'], + ) + + +class UnboundType(Type): + """Instance type that has not been bound during semantic analysis.""" + + name = '' + args = None # type: List[Type] + # should this type be wrapped in an Optional? + optional = False + # is this type a return type? + is_ret_type = False + + def __init__(self, + name: str, + args: List[Type] = None, + line: int = -1, + optional: bool = False, + is_ret_type: bool = False) -> None: + if not args: + args = [] + self.name = name + self.args = args + self.optional = optional + self.is_ret_type = is_ret_type + super().__init__(line) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_unbound_type(self) + + def serialize(self) -> JsonDict: + return {'.class': 'UnboundType', + 'name': self.name, + 'args': [a.serialize() for a in self.args], + } + + @classmethod + def deserialize(self, data: JsonDict) -> 'UnboundType': + assert data['.class'] == 'UnboundType' + return UnboundType(data['name'], + [Type.deserialize(a) for a in data['args']]) + + +class ErrorType(Type): + """The error type is used as the result of failed type operations.""" + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_error_type(self) + + +class TypeList(Type): + """A list of types [...]. + + This is only used for the arguments of a Callable type, i.e. for + [arg, ...] in Callable[[arg, ...], ret]. This is not a real type + but a syntactic AST construct. + """ + + items = None # type: List[Type] + + def __init__(self, items: List[Type], line: int = -1) -> None: + super().__init__(line) + self.items = items + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_type_list(self) + + def serialize(self) -> JsonDict: + return {'.class': 'TypeList', + 'items': [t.serialize() for t in self.items], + } + + @classmethod + def deserialize(self, data: JsonDict) -> 'TypeList': + assert data['.class'] == 'TypeList' + return TypeList([Type.deserialize(t) for t in data['items']]) + + +class AnyType(Type): + """The type 'Any'.""" + + def __init__(self, implicit=False, line: int = -1) -> None: + super().__init__(line) + self.implicit = implicit + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_any(self) + + def serialize(self) -> JsonDict: + return {'.class': 'AnyType'} + + @classmethod + def deserialize(cls, data: JsonDict) -> 'AnyType': + assert data['.class'] == 'AnyType' + return AnyType() + + +class Void(Type): + """The return type 'None'. + + This can only be used as the return type in a callable type and as + the result type of calling such callable. + """ + + source = '' # May be None; function that generated this value + + def __init__(self, source: str = None, line: int = -1) -> None: + self.source = source + super().__init__(line) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_void(self) + + def with_source(self, source: str) -> 'Void': + return Void(source, self.line) + + def serialize(self) -> JsonDict: + return {'.class': 'Void'} + + @classmethod + def deserialize(cls, data: JsonDict) -> 'Void': + assert data['.class'] == 'Void' + return Void() + + +class UninhabitedType(Type): + """This type has no members. + + This type is almost the bottom type, except it is not a subtype of Void. + With strict Optional checking, it is the only common subtype between all + other types, which allows `meet` to be well defined. Without strict + Optional checking, NoneTyp fills this role. + + In general, for any type T that isn't Void: + join(UninhabitedType, T) = T + meet(UninhabitedType, T) = UninhabitedType + is_subtype(UninhabitedType, T) = True + """ + + def __init__(self, line: int = -1) -> None: + super().__init__(line) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_uninhabited_type(self) + + def serialize(self) -> JsonDict: + return {'.class': 'UninhabitedType'} + + @classmethod + def deserialize(cls, data: JsonDict) -> 'UninhabitedType': + assert data['.class'] == 'UninhabitedType' + return UninhabitedType() + + +class NoneTyp(Type): + """The type of 'None'. + + Without strict Optional checking: + This is only used internally during type inference. Programs + cannot declare a variable of this type, and the type checker + refuses to infer this type for a variable. However, subexpressions + often have this type. Note that this is not used as the result + type when calling a function with a void type, even though + semantically such a function returns a None value; the void type + is used instead so that we can report an error if the caller tries + to do anything with the return value. + + With strict Optional checking: + This type can be written by users as 'None', except as the return value + of a function, where 'None' means Void. + """ + + def __init__(self, line: int = -1) -> None: + super().__init__(line) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_none_type(self) + + def serialize(self) -> JsonDict: + return {'.class': 'NoneTyp'} + + @classmethod + def deserialize(self, data: JsonDict) -> 'NoneTyp': + assert data['.class'] == 'NoneTyp' + return NoneTyp() + + +class ErasedType(Type): + """Placeholder for an erased type. + + This is used during type inference. This has the special property that + it is ignored during type inference. + """ + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_erased_type(self) + + +class DeletedType(Type): + """Type of deleted variables. + + These can be used as lvalues but not rvalues. + """ + + source = '' # May be None; name that generated this value + + def __init__(self, source: str = None, line: int = -1) -> None: + self.source = source + super().__init__(line) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_deleted_type(self) + + def serialize(self) -> JsonDict: + return {'.class': 'DeletedType', + 'source': self.source} + + @classmethod + def deserialize(self, data: JsonDict) -> 'DeletedType': + assert data['.class'] == 'DeletedType' + return DeletedType(data['source']) + + +class Instance(Type): + """An instance type of form C[T1, ..., Tn]. + + The list of type variables may be empty. + """ + + type = None # type: mypy.nodes.TypeInfo + args = None # type: List[Type] + erased = False # True if result of type variable substitution + + def __init__(self, typ: mypy.nodes.TypeInfo, args: List[Type], + line: int = -1, erased: bool = False) -> None: + self.type = typ + self.args = args + self.erased = erased + super().__init__(line) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_instance(self) + + type_ref = None # type: str + + def serialize(self) -> JsonDict: + data = {'.class': 'Instance', + } # type: JsonDict + assert self.type is not None + data['type_ref'] = self.type.alt_fullname or self.type.fullname() + if self.args: + data['args'] = [arg.serialize() for arg in self.args] + return data + + @classmethod + def deserialize(cls, data: JsonDict) -> 'Instance': + assert data['.class'] == 'Instance' + args = [] # type: List[Type] + if 'args' in data: + args_list = data['args'] + assert isinstance(args_list, list) + args = [Type.deserialize(arg) for arg in args_list] + inst = Instance(None, args) + inst.type_ref = data['type_ref'] # Will be fixed up by fixup.py later. + return inst + + +class TypeVarType(Type): + """A type variable type. + + This refers to either a class type variable (id > 0) or a function + type variable (id < 0). + """ + + name = '' # Name of the type variable (for messages and debugging) + id = None # type: TypeVarId + values = None # type: List[Type] # Value restriction, empty list if no restriction + upper_bound = None # type: Type # Upper bound for values + # See comments in TypeVarDef for more about variance. + variance = INVARIANT # type: int + + def __init__(self, binder: TypeVarDef, line: int = -1) -> None: + self.name = binder.name + self.id = binder.id + self.values = binder.values + self.upper_bound = binder.upper_bound + self.variance = binder.variance + super().__init__(line) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_type_var(self) + + def erase_to_union_or_bound(self) -> Type: + if self.values: + return UnionType.make_simplified_union(self.values) + else: + return self.upper_bound + + def serialize(self) -> JsonDict: + assert not self.id.is_meta_var() + return {'.class': 'TypeVarType', + 'name': self.name, + 'id': self.id.raw_id, + 'values': [v.serialize() for v in self.values], + 'upper_bound': self.upper_bound.serialize(), + 'variance': self.variance, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'TypeVarType': + assert data['.class'] == 'TypeVarType' + tvdef = TypeVarDef(data['name'], + data['id'], + [Type.deserialize(v) for v in data['values']], + Type.deserialize(data['upper_bound']), + data['variance']) + return TypeVarType(tvdef) + + +class FunctionLike(Type): + """Abstract base class for function types.""" + + @abstractmethod + def is_type_obj(self) -> bool: pass + + def is_concrete_type_obj(self) -> bool: + return self.is_type_obj() + + @abstractmethod + def type_object(self) -> mypy.nodes.TypeInfo: pass + + @abstractmethod + def items(self) -> List['CallableType']: pass + + @abstractmethod + def with_name(self, name: str) -> 'FunctionLike': pass + + # Corresponding instance type (e.g. builtins.type) + fallback = None # type: Instance + + @classmethod + def deserialize(cls, data: JsonDict) -> 'FunctionLike': + return cast(FunctionLike, super().deserialize(data)) + + +_dummy = object() # type: Any + + +class CallableType(FunctionLike): + """Type of a non-overloaded callable object (function).""" + + arg_types = None # type: List[Type] # Types of function arguments + arg_kinds = None # type: List[int] # mypy.nodes.ARG_ constants + arg_names = None # type: List[str] # None if not a keyword argument + min_args = 0 # Minimum number of arguments; derived from arg_kinds + is_var_arg = False # Is it a varargs function? derived from arg_kinds + ret_type = None # type: Type # Return value type + name = '' # Name (may be None; for error messages) + definition = None # type: SymbolNode # For error messages. May be None. + # Type variables for a generic function + variables = None # type: List[TypeVarDef] + + # Is this Callable[..., t] (with literal '...')? + is_ellipsis_args = False + # Is this callable constructed for the benefit of a classmethod's 'cls' argument? + is_classmethod_class = False + # Was this type implicitly generated instead of explicitly specified by the user? + implicit = False + # Defined for signatures that require special handling (currently only value is 'dict' + # for a signature similar to 'dict') + special_sig = None # type: Optional[str] + + def __init__(self, + arg_types: List[Type], + arg_kinds: List[int], + arg_names: List[str], + ret_type: Type, + fallback: Instance, + name: str = None, + definition: SymbolNode = None, + variables: List[TypeVarDef] = None, + line: int = -1, + is_ellipsis_args: bool = False, + implicit=False, + is_classmethod_class=False, + special_sig=None, + ) -> None: + if variables is None: + variables = [] + self.arg_types = arg_types + self.arg_kinds = arg_kinds + self.arg_names = arg_names + self.min_args = arg_kinds.count(mypy.nodes.ARG_POS) + self.is_var_arg = mypy.nodes.ARG_STAR in arg_kinds + self.ret_type = ret_type + self.fallback = fallback + assert not name or ' 'CallableType': + return CallableType( + arg_types=arg_types if arg_types is not _dummy else self.arg_types, + arg_kinds=arg_kinds if arg_kinds is not _dummy else self.arg_kinds, + arg_names=arg_names if arg_names is not _dummy else self.arg_names, + ret_type=ret_type if ret_type is not _dummy else self.ret_type, + fallback=fallback if fallback is not _dummy else self.fallback, + name=name if name is not _dummy else self.name, + definition=definition if definition is not _dummy else self.definition, + variables=variables if variables is not _dummy else self.variables, + line=line if line is not _dummy else self.line, + is_ellipsis_args=( + is_ellipsis_args if is_ellipsis_args is not _dummy else self.is_ellipsis_args), + implicit=self.implicit, + is_classmethod_class=self.is_classmethod_class, + special_sig=special_sig if special_sig is not _dummy else self.special_sig, + ) + + def is_type_obj(self) -> bool: + return self.fallback.type is not None and self.fallback.type.fullname() == 'builtins.type' + + def is_concrete_type_obj(self) -> bool: + return self.is_type_obj() and self.is_classmethod_class + + def type_object(self) -> mypy.nodes.TypeInfo: + assert self.is_type_obj() + ret = self.ret_type + if isinstance(ret, TupleType): + ret = ret.fallback + return cast(Instance, ret).type + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_callable_type(self) + + def with_name(self, name: str) -> 'CallableType': + """Return a copy of this type with the specified name.""" + ret = self.ret_type + if isinstance(ret, Void): + ret = ret.with_source(name) + return self.copy_modified(ret_type=ret, name=name) + + def max_fixed_args(self) -> int: + n = len(self.arg_types) + if self.is_var_arg: + n -= 1 + return n + + def items(self) -> List['CallableType']: + return [self] + + def is_generic(self) -> bool: + return bool(self.variables) + + def type_var_ids(self) -> List[TypeVarId]: + a = [] # type: List[TypeVarId] + for tv in self.variables: + a.append(tv.id) + return a + + def serialize(self) -> JsonDict: + # TODO: As an optimization, leave out everything related to + # generic functions for non-generic functions. + return {'.class': 'CallableType', + 'arg_types': [(None if t is None else t.serialize()) + for t in self.arg_types], + 'arg_kinds': self.arg_kinds, + 'arg_names': self.arg_names, + 'ret_type': self.ret_type.serialize(), + 'fallback': self.fallback.serialize(), + 'name': self.name, + # We don't serialize the definition (only used for error messages). + 'variables': [v.serialize() for v in self.variables], + 'is_ellipsis_args': self.is_ellipsis_args, + 'implicit': self.implicit, + 'is_classmethod_class': self.is_classmethod_class, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'CallableType': + assert data['.class'] == 'CallableType' + # TODO: Set definition to the containing SymbolNode? + return CallableType([(None if t is None else Type.deserialize(t)) + for t in data['arg_types']], + data['arg_kinds'], + data['arg_names'], + Type.deserialize(data['ret_type']), + Instance.deserialize(data['fallback']), + name=data['name'], + variables=[TypeVarDef.deserialize(v) for v in data['variables']], + is_ellipsis_args=data['is_ellipsis_args'], + implicit=data['implicit'], + is_classmethod_class=data['is_classmethod_class'], + ) + + +class Overloaded(FunctionLike): + """Overloaded function type T1, ... Tn, where each Ti is CallableType. + + The variant to call is chosen based on static argument + types. Overloaded function types can only be defined in stub + files, and thus there is no explicit runtime dispatch + implementation. + """ + + _items = None # type: List[CallableType] # Must not be empty + + def __init__(self, items: List[CallableType]) -> None: + self._items = items + self.fallback = items[0].fallback + super().__init__(items[0].line) + + def items(self) -> List[CallableType]: + return self._items + + def name(self) -> str: + return self._items[0].name + + def is_type_obj(self) -> bool: + # All the items must have the same type object status, so it's + # sufficient to query only (any) one of them. + return self._items[0].is_type_obj() + + def type_object(self) -> mypy.nodes.TypeInfo: + # All the items must have the same type object, so it's sufficient to + # query only (any) one of them. + return self._items[0].type_object() + + def with_name(self, name: str) -> 'Overloaded': + ni = [] # type: List[CallableType] + for it in self._items: + ni.append(it.with_name(name)) + return Overloaded(ni) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_overloaded(self) + + def serialize(self) -> JsonDict: + return {'.class': 'Overloaded', + 'items': [t.serialize() for t in self.items()], + } + + @classmethod + def deserialize(self, data: JsonDict) -> 'Overloaded': + assert data['.class'] == 'Overloaded' + return Overloaded([CallableType.deserialize(t) for t in data['items']]) + + +class TupleType(Type): + """The tuple type Tuple[T1, ..., Tn] (at least one type argument). + + Instance variables: + items: tuple item types + fallback: the underlying instance type that is used for non-tuple methods + (this is currently always builtins.tuple, but it could be different for named + tuples, for example) + implicit: if True, derived from a tuple expression (t,....) instead of Tuple[t, ...] + """ + + items = None # type: List[Type] + fallback = None # type: Instance + implicit = False + + def __init__(self, items: List[Type], fallback: Instance, line: int = -1, + implicit: bool = False) -> None: + #import pdb; pdb.set_trace() + self.items = items + self.fallback = fallback + self.implicit = implicit + super().__init__(line) + + def length(self) -> int: + return len(self.items) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_tuple_type(self) + + def serialize(self) -> JsonDict: + return {'.class': 'TupleType', + 'items': [t.serialize() for t in self.items], + 'fallback': self.fallback.serialize(), + 'implicit': self.implicit, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'TupleType': + assert data['.class'] == 'TupleType' + return TupleType([Type.deserialize(t) for t in data['items']], + Instance.deserialize(data['fallback']), + implicit=data['implicit']) + + def self_type(self, inst: Instance) -> 'TupleType': + return TupleType(self.items, inst, self.line, self.implicit) + + def with_types(self, items: List[Type]) -> 'TupleType': + return TupleType(items, self.fallback, self.line, self.implicit) + + def slice(self, begin: int, stride: int, end: int) -> 'TupleType': + return TupleType(self.items[begin:end:stride], self.fallback, + self.line, self.implicit) + + +class NamedTupleType(TupleType): + attrs = None # type: List[str] + name = None # type: str + + def __init__(self, name: str, attrs: List[str], *args, **kwargs) -> None: + self.attrs = attrs + self.name = name + super().__init__(*args, **kwargs) + + def self_type(self, inst: Instance) -> 'NamedTupleType': + return NamedTupleType(self.name, self.attrs, self.items, inst, self.line) + + def with_types(self, items: List[Type]) -> 'NamedTupleType': + return NamedTupleType(self.name, self.attrs, items, + self.fallback, self.line, self.implicit) + + def __eq__(self, other) -> bool: + return self.serialize() == other.serialize() + + def serialize(self) -> JsonDict: + d = super().serialize() + d.update({'.class': 'NamedTupleType', + 'name': self.name, + 'attrs': self.attrs}) + return d + + @classmethod + def deserialize(cls, data: JsonDict) -> 'NamedTupleType': + assert data['.class'] == 'NamedTupleType' + data['.class'] = 'TupleType' + tup = TupleType.deserialize(data) + return NamedTupleType(data['name'], + data['attrs'], + tup.items, + tup.fallback, + implicit=tup.implicit) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + try: + return visitor.visit_namedtuple_type(self) + except: + return super().accept(visitor) + + +class StarType(Type): + """The star type *type_parameter. + + This is not a real type but a syntactic AST construct. + """ + + type = None # type: Type + + def __init__(self, type: Type, line: int = -1) -> None: + self.type = type + super().__init__(line) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_star_type(self) + + +class UnionType(Type): + """The union type Union[T1, ..., Tn] (at least one type argument).""" + + items = None # type: List[Type] + + def __init__(self, items: List[Type], line: int = -1) -> None: + self.items = items + super().__init__(line) + + @staticmethod + def make_union(items: List[Type], line: int = -1) -> Type: + if len(items) > 1: + return UnionType(items, line) + elif len(items) == 1: + return items[0] + else: + if experiments.STRICT_OPTIONAL: + return UninhabitedType() + else: + return Void() + + @staticmethod + def make_simplified_union(items: List[Type], line: int = -1) -> Type: + while any(isinstance(typ, UnionType) for typ in items): + all_items = [] # type: List[Type] + for typ in items: + if isinstance(typ, UnionType): + all_items.extend(typ.items) + else: + all_items.append(typ) + items = all_items + + if any(isinstance(typ, AnyType) for typ in items): + return AnyType() + + from mypy.subtypes import is_subtype + removed = set() # type: Set[int] + for i in range(len(items)): + if any(is_subtype(items[i], items[j]) for j in range(len(items)) + if j not in removed and j != i): + removed.add(i) + + simplified_set = [items[i] for i in range(len(items)) if i not in removed] + return UnionType.make_union(simplified_set) + + def length(self) -> int: + return len(self.items) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_union_type(self) + + def has_readable_member(self, name: str) -> bool: + """For a tree of unions of instances, check whether all instances have a given member. + + TODO: Deal with attributes of TupleType etc. + TODO: This should probably be refactored to go elsewhere. + """ + return all((isinstance(x, UnionType) and x.has_readable_member(name)) or + (isinstance(x, Instance) and x.type.has_readable_member(name)) + for x in self.items) + + def serialize(self) -> JsonDict: + return {'.class': 'UnionType', + 'items': [t.serialize() for t in self.items], + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'UnionType': + assert data['.class'] == 'UnionType' + return UnionType([Type.deserialize(t) for t in data['items']]) + + +class PartialType(Type): + """Type such as List[?] where type arguments are unknown, or partial None type. + + These are used for inferring types in multiphase initialization such as this: + + x = [] # x gets a partial type List[?], as item type is unknown + x.append(1) # partial type gets replaced with normal type List[int] + + Or with None: + + x = None # x gets a partial type None + if c: + x = 1 # Infer actual type int for x + """ + + # None for the 'None' partial type; otherwise a generic class + type = None # type: Optional[mypy.nodes.TypeInfo] + var = None # type: mypy.nodes.Var + inner_types = None # type: List[Type] + + def __init__(self, + type: Optional['mypy.nodes.TypeInfo'], + var: 'mypy.nodes.Var', + inner_types: List[Type]) -> None: + self.type = type + self.var = var + self.inner_types = inner_types + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_partial_type(self) + + +class EllipsisType(Type): + """The type ... (ellipsis). + + This is not a real type but a syntactic AST construct, used in Callable[..., T], for example. + + A semantically analyzed type will never have ellipsis types. + """ + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_ellipsis_type(self) + + def serialize(self) -> JsonDict: + return {'.class': 'EllipsisType'} + + @classmethod + def deserialize(self, data: JsonDict) -> 'EllipsisType': + assert data['.class'] == 'EllipsisType' + return EllipsisType() + + +class TypeType(Type): + """For types like Type[User]. + + This annotates variables that are class objects, constrained by + the type argument. See PEP 484 for more details. + + We may encounter expressions whose values are specific classes; + those are represented as callables (possibly overloaded) + corresponding to the class's constructor's signature and returning + an instance of that class. The difference with Type[C] is that + those callables always represent the exact class given as the + return type; Type[C] represents any class that's a subclass of C, + and C may also be a type variable or a union (or Any). + + Many questions around subtype relationships between Type[C1] and + def(...) -> C2 are answered by looking at the subtype + relationships between C1 and C2, since Type[] is considered + covariant. + + There's an unsolved problem with constructor signatures (also + unsolved in PEP 484): calling a variable whose type is Type[C] + assumes the constructor signature for C, even though a subclass of + C might completely change the constructor signature. For now we + just assume that users of Type[C] are careful not to do that (in + the future we might detect when they are violating that + assumption). + """ + + # This can't be everything, but it can be a class reference, + # a generic class instance, a union, Any, a type variable... + item = None # type: Type + + def __init__(self, item: Type, *, line: int = -1) -> None: + super().__init__(line) + self.item = item + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_type_type(self) + + def serialize(self) -> JsonDict: + return {'.class': 'TypeType', 'item': self.item.serialize()} + + def deserialize(self, data: JsonDict) -> 'TypeType': + assert data['.class'] == 'TypeType' + return TypeType(Type.deserialize(data['item'])) + + +# +# Visitor-related classes +# + + +class TypeVisitor(Generic[T]): + """Visitor class for types (Type subclasses). + + The parameter T is the return type of the visit methods. + """ + + def _notimplemented_helper(self, name: str) -> NotImplementedError: + return NotImplementedError("Method {}.visit_{}() not implemented\n" + .format(type(self).__name__, name) + + "This is a known bug, track development in " + + "'https://github.com/JukkaL/mypy/issues/730'") + + @abstractmethod + def visit_unbound_type(self, t: UnboundType) -> T: + pass + + def visit_type_list(self, t: TypeList) -> T: + raise self._notimplemented_helper('type_list') + + def visit_error_type(self, t: ErrorType) -> T: + raise self._notimplemented_helper('error_type') + + @abstractmethod + def visit_any(self, t: AnyType) -> T: + pass + + @abstractmethod + def visit_void(self, t: Void) -> T: + pass + + @abstractmethod + def visit_none_type(self, t: NoneTyp) -> T: + pass + + @abstractmethod + def visit_uninhabited_type(self, t: UninhabitedType) -> T: + pass + + def visit_erased_type(self, t: ErasedType) -> T: + raise self._notimplemented_helper('erased_type') + + @abstractmethod + def visit_deleted_type(self, t: DeletedType) -> T: + pass + + @abstractmethod + def visit_type_var(self, t: TypeVarType) -> T: + pass + + @abstractmethod + def visit_instance(self, t: Instance) -> T: + pass + + @abstractmethod + def visit_callable_type(self, t: CallableType) -> T: + pass + + def visit_overloaded(self, t: Overloaded) -> T: + raise self._notimplemented_helper('overloaded') + + @abstractmethod + def visit_tuple_type(self, t: TupleType) -> T: + pass + + def visit_star_type(self, t: StarType) -> T: + raise self._notimplemented_helper('star_type') + + @abstractmethod + def visit_union_type(self, t: UnionType) -> T: + pass + + @abstractmethod + def visit_partial_type(self, t: PartialType) -> T: + pass + + def visit_ellipsis_type(self, t: EllipsisType) -> T: + raise self._notimplemented_helper('ellipsis_type') + + @abstractmethod + def visit_type_type(self, t: TypeType) -> T: + pass + + +class TypeTranslator(TypeVisitor[Type]): + """Identity type transformation. + + Subclass this and override some methods to implement a non-trivial + transformation. + """ + + def visit_unbound_type(self, t: UnboundType) -> Type: + return t + + def visit_type_list(self, t: TypeList) -> Type: + return t + + def visit_error_type(self, t: ErrorType) -> Type: + return t + + def visit_any(self, t: AnyType) -> Type: + return t + + def visit_void(self, t: Void) -> Type: + return t + + def visit_none_type(self, t: NoneTyp) -> Type: + return t + + def visit_uninhabited_type(self, t: UninhabitedType) -> Type: + return t + + def visit_erased_type(self, t: ErasedType) -> Type: + return t + + def visit_deleted_type(self, t: DeletedType) -> Type: + return t + + def visit_instance(self, t: Instance) -> Type: + return Instance(t.type, self.translate_types(t.args), t.line) + + def visit_type_var(self, t: TypeVarType) -> Type: + return t + + def visit_partial_type(self, t: PartialType) -> Type: + return t + + def visit_callable_type(self, t: CallableType) -> Type: + return t.copy_modified(arg_types=self.translate_types(t.arg_types), + ret_type=t.ret_type.accept(self), + variables=self.translate_variables(t.variables)) + + def visit_tuple_type(self, t: TupleType) -> Type: + return TupleType(self.translate_types(t.items), + cast(Any, t.fallback.accept(self)), + t.line) + + def visit_star_type(self, t: StarType) -> Type: + return StarType(t.type.accept(self), t.line) + + def visit_union_type(self, t: UnionType) -> Type: + return UnionType(self.translate_types(t.items), t.line) + + def visit_ellipsis_type(self, t: EllipsisType) -> Type: + return t + + def translate_types(self, types: List[Type]) -> List[Type]: + return [t.accept(self) for t in types] + + def translate_variables(self, + variables: List[TypeVarDef]) -> List[TypeVarDef]: + return variables + + def visit_overloaded(self, t: Overloaded) -> Type: + items = [] # type: List[CallableType] + for item in t.items(): + new = item.accept(self) + if isinstance(new, CallableType): + items.append(new) + else: + raise RuntimeError('CallableType expectected, but got {}'.format(type(new))) + return Overloaded(items=items) + + def visit_type_type(self, t: TypeType) -> Type: + return TypeType(t.item.accept(self), line=t.line) + + +class TypeStrVisitor(TypeVisitor[str]): + """Visitor for pretty-printing types into strings. + + This is mostly for debugging/testing. + + Do not preserve original formatting. + + Notes: + - Represent unbound types as Foo? or Foo?[...]. + - Represent the NoneTyp type as None. + """ + + def visit_unbound_type(self, t): + s = t.name + '?' + if t.args != []: + s += '[{}]'.format(self.list_str(t.args)) + return s + + def visit_type_list(self, t): + return ''.format(self.list_str(t.items)) + + def visit_error_type(self, t): + return '' + + def visit_any(self, t): + return 'Any' + + def visit_void(self, t): + return 'void' + + def visit_none_type(self, t): + # Fully qualify to make this distinct from the None value. + return "builtins.None" + + def visit_uninhabited_type(self, t): + return "" + + def visit_erased_type(self, t): + return "" + + def visit_deleted_type(self, t): + if t.source is None: + return "" + else: + return "".format(t.source) + + def visit_instance(self, t): + s = t.type.fullname() if t.type is not None else '' + if t.erased: + s += '*' + if t.args != []: + s += '[{}]'.format(self.list_str(t.args)) + return s + + def visit_type_var(self, t): + if t.name is None: + # Anonymous type variable type (only numeric id). + return '`{}'.format(t.id) + else: + # Named type variable type. + return '{}`{}'.format(t.name, t.id) + + def visit_callable_type(self, t): + s = '' + bare_asterisk = False + for i in range(len(t.arg_types)): + if s != '': + s += ', ' + if t.arg_kinds[i] == mypy.nodes.ARG_NAMED and not bare_asterisk: + s += '*, ' + bare_asterisk = True + if t.arg_kinds[i] == mypy.nodes.ARG_STAR: + s += '*' + if t.arg_kinds[i] == mypy.nodes.ARG_STAR2: + s += '**' + if t.arg_names[i]: + s += t.arg_names[i] + ': ' + s += str(t.arg_types[i]) + if t.arg_kinds[i] == mypy.nodes.ARG_OPT: + s += ' =' + + s = '({})'.format(s) + + if not isinstance(t.ret_type, Void): + s += ' -> {}'.format(t.ret_type) + + if t.variables: + s = '{} {}'.format(t.variables, s) + + return 'def {}'.format(s) + + def visit_overloaded(self, t): + a = [] + for i in t.items(): + a.append(i.accept(self)) + return 'Overload({})'.format(', '.join(a)) + + def visit_tuple_type(self, t): + s = self.list_str(t.items) + if t.fallback and t.fallback.type: + fallback_name = t.fallback.type.fullname() + if fallback_name != 'builtins.tuple': + return 'Tuple[{}, fallback={}]'.format(s, t.fallback.accept(self)) + return 'Tuple[{}]'.format(s) + + def visit_namedtuple_type(self, t): + s = self.list_str(zip(t.attrs, t.items)) + return '{}[{}]'.format(t.name, s) + + def visit_star_type(self, t): + s = t.type.accept(self) + return '*{}'.format(s) + + def visit_union_type(self, t): + s = self.list_str(t.items) + return 'Union[{}]'.format(s) + + def visit_partial_type(self, t: PartialType) -> str: + if t.type is None: + return '' + else: + return ''.format(t.type.name(), + ', '.join(['?'] * len(t.type.type_vars))) + + def visit_ellipsis_type(self, t): + return '...' + + def visit_type_type(self, t): + return 'Type[{}]'.format(t.item.accept(self)) + + def list_str(self, a): + """Convert items of an array to strings (pretty-print types) + and join the results with commas. + """ + res = [] + for t in a: + if isinstance(t, Type): + res.append(t.accept(self)) + else: + res.append(str(t)) + return ', '.join(res) + + +# These constants define the method used by TypeQuery to combine multiple +# query results, e.g. for tuple types. The strategy is not used for empty +# result lists; in that case the default value takes precedence. +ANY_TYPE_STRATEGY = 0 # Return True if any of the results are True. +ALL_TYPES_STRATEGY = 1 # Return True if all of the results are True. + + +class TypeQuery(TypeVisitor[bool]): + """Visitor for performing simple boolean queries of types. + + This class allows defining the default value for leafs to simplify the + implementation of many queries. + """ + + default = False # Default result + strategy = 0 # Strategy for combining multiple values (ANY_TYPE_STRATEGY or ALL_TYPES_...). + + def __init__(self, default: bool, strategy: int) -> None: + """Construct a query visitor. + + Use the given default result and strategy for combining + multiple results. The strategy must be either + ANY_TYPE_STRATEGY or ALL_TYPES_STRATEGY. + """ + self.default = default + self.strategy = strategy + + def visit_unbound_type(self, t: UnboundType) -> bool: + return self.default + + def visit_type_list(self, t: TypeList) -> bool: + return self.default + + def visit_error_type(self, t: ErrorType) -> bool: + return self.default + + def visit_any(self, t: AnyType) -> bool: + return self.default + + def visit_void(self, t: Void) -> bool: + return self.default + + def visit_uninhabited_type(self, t: UninhabitedType) -> bool: + return self.default + + def visit_none_type(self, t: NoneTyp) -> bool: + return self.default + + def visit_erased_type(self, t: ErasedType) -> bool: + return self.default + + def visit_deleted_type(self, t: DeletedType) -> bool: + return self.default + + def visit_type_var(self, t: TypeVarType) -> bool: + return self.default + + def visit_partial_type(self, t: PartialType) -> bool: + return self.default + + def visit_instance(self, t: Instance) -> bool: + return self.query_types(t.args) + + def visit_callable_type(self, t: CallableType) -> bool: + # FIX generics + return self.query_types(t.arg_types + [t.ret_type]) + + def visit_tuple_type(self, t: TupleType) -> bool: + return self.query_types(t.items) + + def visit_star_type(self, t: StarType) -> bool: + return t.type.accept(self) + + def visit_union_type(self, t: UnionType) -> bool: + return self.query_types(t.items) + + def visit_overloaded(self, t: Overloaded) -> bool: + return self.query_types(t.items()) + + def visit_type_type(self, t: TypeType) -> bool: + return t.item.accept(self) + + def query_types(self, types: Sequence[Type]) -> bool: + """Perform a query for a list of types. + + Use the strategy constant to combine the results. + """ + if not types: + # Use default result for empty list. + return self.default + if self.strategy == ANY_TYPE_STRATEGY: + # Return True if at least one component is true. + res = False + for t in types: + res = res or t.accept(self) + if res: + break + return res + else: + # Return True if all components are true. + res = True + for t in types: + res = res and t.accept(self) + if not res: + break + return res + + +def strip_type(typ: Type) -> Type: + """Make a copy of type without 'debugging info' (function name).""" + + if isinstance(typ, CallableType): + return typ.copy_modified(name=None) + elif isinstance(typ, Overloaded): + return Overloaded([cast(CallableType, strip_type(item)) + for item in typ.items()]) + else: + return typ + + +def replace_leading_arg_type(t: CallableType, self_type: Type) -> CallableType: + """Return a copy of a callable type with a different self argument type. + + Assume that the callable is the signature of a method. + """ + return t.copy_modified(arg_types=[self_type] + t.arg_types[1:]) + + +def is_named_instance(t: Type, fullname: str) -> bool: + return (isinstance(t, Instance) and + t.type is not None and + t.type.fullname() == fullname) diff --git a/test-data/unit/check-lists.test b/test-data/unit/check-lists.test index 032e02314f5c4..a6df5ae740c61 100644 --- a/test-data/unit/check-lists.test +++ b/test-data/unit/check-lists.test @@ -1,66 +1,63 @@ --- Nested list assignment --- ----------------------------- -[case testListSimplest] -l = [1, 1] - - -[case testNestedListAssignment] -from typing import List -a1, b1, c1 = None, None, None # type: (A, B, C) -a2, b2, c2 = None, None, None # type: (A, B, C) - -a1, [b1, c1] = a2, [b2, c2] -a1, [a1, [b1, c1]] = a2, [a2, [b2, c2]] -a1, [a1, [a1, b1]] = a1, [a1, [a1, c1]] # E: Incompatible types in assignment (expression has type "C", variable has type "B") - -class A: pass -class B: pass -class C: pass -[builtins fixtures/list.py] -[out] - -[case testNestedListAssignmentToTuple] -from typing import List -a, b, c = None, None, None # type: (A, B, C) - -a, b = [a, b] -a, b = [a] # E: Need more than 1 value to unpack (2 expected) -a, b = [a, b, c] # E: Too many values to unpack (2 expected, 3 provided) - -class A: pass -class B: pass -class C: pass -[builtins fixtures/list.py] -[out] - -[case testListAssignmentFromTuple] -from typing import List -a, b, c = None, None, None # type: (A, B, C) -t = a, b - -[a, b], c = t, c -[a, c], c = t, c # E: Incompatible types in assignment (expression has type "B", variable has type "C") -[a, a, a], c = t, c # E: Need more than 2 values to unpack (3 expected) -[a], c = t, c # E: Too many values to unpack (1 expected, 2 provided) - -class A: pass -class B: pass -class C: pass -[builtins fixtures/list.py] -[out] - -[case testListAssignmentUnequalAmountToUnpack] -from typing import List -a, b, c = None, None, None # type: (A, B, C) - -def f() -> None: # needed because test parser tries to parse [a, b] as section header - [a, b] = [a, b] - [a, b] = [a] # E: Need more than 1 value to unpack (2 expected) - [a, b] = [a, b, c] # E: Too many values to unpack (2 expected, 3 provided) - -class A: pass -class B: pass -class C: pass -[builtins fixtures/list.py] -[out] -main: note: In function "f": +-- Nested list assignment +-- ----------------------------- + +[case testNestedListAssignment] +from typing import List +a1, b1, c1 = None, None, None # type: (A, B, C) +a2, b2, c2 = None, None, None # type: (A, B, C) + +a1, [b1, c1] = a2, [b2, c2] +a1, [a1, [b1, c1]] = a2, [a2, [b2, c2]] +a1, [a1, [a1, b1]] = a1, [a1, [a1, c1]] # E: Incompatible types in assignment (expression has type "C", variable has type "B") + +class A: pass +class B: pass +class C: pass +[builtins fixtures/list.py] +[out] + +[case testNestedListAssignmentToTuple] +from typing import List +a, b, c = None, None, None # type: (A, B, C) + +a, b = [a, b] +a, b = [a] # E: Need more than 1 value to unpack (2 expected) +a, b = [a, b, c] # E: Too many values to unpack (2 expected, 3 provided) + +class A: pass +class B: pass +class C: pass +[builtins fixtures/list.py] +[out] + +[case testListAssignmentFromTuple] +from typing import List +a, b, c = None, None, None # type: (A, B, C) +t = a, b + +[a, b], c = t, c +[a, c], c = t, c # E: Incompatible types in assignment (expression has type "B", variable has type "C") +[a, a, a], c = t, c # E: Need more than 2 values to unpack (3 expected) +[a], c = t, c # E: Too many values to unpack (1 expected, 2 provided) + +class A: pass +class B: pass +class C: pass +[builtins fixtures/list.py] +[out] + +[case testListAssignmentUnequalAmountToUnpack] +from typing import List +a, b, c = None, None, None # type: (A, B, C) + +def f() -> None: # needed because test parser tries to parse [a, b] as section header + [a, b] = [a, b] + [a, b] = [a] # E: Need more than 1 value to unpack (2 expected) + [a, b] = [a, b, c] # E: Too many values to unpack (2 expected, 3 provided) + +class A: pass +class B: pass +class C: pass +[builtins fixtures/list.py] +[out] +main: note: In function "f": diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index adb07ec522705..86c19956ca634 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -1,223 +1,223 @@ -[case testNamedTupleDeclaration] -from typing import NamedTuple - -A = NamedTuple('A', [('x', str), ('y', int)]) - -[case testNamedTupleUsedAsTuple] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -a, b = x -b = x[0] -a = x[1] -a, b, c = x # E: Need more than 2 values to unpack (3 expected) -x[2] # E: Tuple index out of range - -[case testAccessingNamedTupleAttributes] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -x.x -x.y -x.z # E: "X" has no attribute "z" - -[case testCreateNamedTupleWithPositionalArguments] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = X(1, 'x') -x.x -x.z # E: "X" has no attribute "z" -x = X(1) # E: Too few arguments for "X" -x = X(1, 2, 3) # E: Too many arguments for "X" - -[case testCreateNamedTupleWithKeywordArguments] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = X(x=1, y='x') -x = X(1, y='x') -x = X(x=1, z=1) # E: Unexpected keyword argument "z" for "X" -x = X(y=1) # E: Missing positional argument "x" in call to "X" - -[case testCreateNamedTupleAndUseAsTuple] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = X(1, 'x') -a, b = x -a, b, c = x # E: Need more than 2 values to unpack (3 expected) - -[case testNamedTupleWithItemTypes] -from typing import NamedTuple -N = NamedTuple('N', [('a', int), - ('b', str)]) -n = N(1, 'x') -s = n.a # type: str # E: Incompatible types in assignment (expression has type "int", \ - variable has type "str") -i = n.b # type: int # E: Incompatible types in assignment (expression has type "str", \ - variable has type "int") -x, y = n -x = y # E: Incompatible types in assignment (expression has type "str", variable has type "int") - -[case testNamedTupleConstructorArgumentTypes] -from typing import NamedTuple -N = NamedTuple('N', [('a', int), - ('b', str)]) -n = N('x', 'x') # E: Argument 1 to "N" has incompatible type "str"; expected "int" -n = N(1, b=2) # E: Argument 2 to "N" has incompatible type "int"; expected "str" -N(1, 'x') -N(b='x', a=1) - -[case testNamedTupleAsBaseClass] -from typing import NamedTuple -N = NamedTuple('N', [('a', int), - ('b', str)]) -class X(N): - pass -x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" -s = '' -i = 0 -s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") -i, s = x -s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") - -[case testNamedTupleAsBaseClass2] -from typing import NamedTuple -class X(NamedTuple('N', [('a', int), - ('b', str)])): - pass -x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" -s = '' -i = 0 -s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") -i, s = x -s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") - -[case testTwoNamedTuplesAsBaseClasses] -from typing import NamedTuple -A = NamedTuple('A', [('a', int)]) -B = NamedTuple('B', [('a', int)]) -class X(A, B): # E: Class has two incompatible bases derived from tuple - pass - -[case testTwoNamedTuplesAsBaseClasses2] -from typing import NamedTuple -A = NamedTuple('A', [('a', int)]) -class X(A, NamedTuple('B', [('a', int)])): # E: Class has two incompatible bases derived from tuple - pass - -[case testSelfTypeWithNamedTupleAsBase] -from typing import NamedTuple -A = NamedTuple('A', [('a', int), ('b', str)]) -class B(A): - def f(self, x: int) -> None: - self.f(self.a) - self.f(self.b) # E: Argument 1 to "f" of "B" has incompatible type "str"; expected "int" - i = 0 - s = '' - i, s = self - i, i = self # E: Incompatible types in assignment (expression has type "str", \ - variable has type "int") -[out] -main: note: In member "f" of class "B": - -[case testTypeReferenceToClassDerivedFromNamedTuple] -from typing import NamedTuple -A = NamedTuple('A', [('a', int), ('b', str)]) -class B(A): - def f(self, x: 'B') -> None: - i = 0 - s = '' - self = x - i, s = x - i, s = x.a, x.b - i, s = x.a, x.a # E: Incompatible types in assignment (expression has type "int", \ - variable has type "str") - i, i = self # E: Incompatible types in assignment (expression has type "str", \ - variable has type "int") -[out] -main: note: In member "f" of class "B": - -[case testNamedTupleSubtyping] -from typing import NamedTuple, Tuple -A = NamedTuple('A', [('a', int), ('b', str)]) -class B(A): pass -a = A(1, '') -b = B(1, '') -t = None # type: Tuple[int, str] -b = a # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "A") -b = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "B") -t = a -t = (1, '') -t = b -a = b - -[case testNamedTupleSimpleTypeInference] -from typing import NamedTuple, Tuple -A = NamedTuple('A', [('a', int)]) -l = [A(1), A(2)] -a = A(1) -a = l[0] -(i,) = l[0] -i, i = l[0] # E: Need more than 1 value to unpack (2 expected) -a = (1,) # E: Incompatible types in assignment (expression has type "Tuple[int]", \ - variable has type "A") -[builtins fixtures/list.py] - -[case testNamedTupleMissingClassAttribute] -import collections -MyNamedTuple = collections.namedtuple('MyNamedTuple', ['spam', 'eggs']) -MyNamedTuple.x # E: "MyNamedTuple" has no attribute "x" - -[case testNamedTupleEmptyItems] -from typing import NamedTuple -A = NamedTuple('A', []) -[builtins fixtures/list.py] - -[case testNamedTupleProperty] -from typing import NamedTuple -A = NamedTuple('A', [('a', int)]) -class B(A): - @property - def b(self) -> int: - return self.a -class C(B): pass -B(1).b -C(2).b -[builtins fixtures/property.py] - -[case testNamedTupleAsDict] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -a = x._asdict() - -[case testNamedTupleReplace] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -x = x._replace(y=5) -x = x._replace(z=5) # E: Unexpected keyword argument "z" for X._replace -x = x._replace(5) # E: Too many positional arguments for X._replace - -[case testNamedTupleReplaceAsClass] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -X._replace(x, x=1, y=2) -X._replace(x=1, y=2) # E: Missing positional argument "self" in call to X._replace - -[case testNamedTupleReplaceTyped] -from typing import NamedTuple - -X = NamedTuple('X', [('x', int), ('y', str)]) -x = None # type: X -x = x._replace(x=5) -x = x._replace(y=5) # E: Argument 1 to X._replace has incompatible type "int"; expected "str" +[case testNamedTupleDeclaration] +from typing import NamedTuple + +A = NamedTuple('A', [('x', str), ('y', int)]) + +[case testNamedTupleUsedAsTuple] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +a, b = x +b = x[0] +a = x[1] +a, b, c = x # E: Need more than 2 values to unpack (3 expected) +x[2] # E: Tuple index out of range + +[case testAccessingNamedTupleAttributes] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +x.x +x.y +x.z # E: "X" has no attribute "z" + +[case testCreateNamedTupleWithPositionalArguments] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = X(1, 'x') +x.x +x.z # E: "X" has no attribute "z" +x = X(1) # E: Too few arguments for "X" +x = X(1, 2, 3) # E: Too many arguments for "X" + +[case testCreateNamedTupleWithKeywordArguments] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = X(x=1, y='x') +x = X(1, y='x') +x = X(x=1, z=1) # E: Unexpected keyword argument "z" for "X" +x = X(y=1) # E: Missing positional argument "x" in call to "X" + +[case testCreateNamedTupleAndUseAsTuple] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = X(1, 'x') +a, b = x +a, b, c = x # E: Need more than 2 values to unpack (3 expected) + +[case testNamedTupleWithItemTypes] +from typing import NamedTuple +N = NamedTuple('N', [('a', int), + ('b', str)]) +n = N(1, 'x') +s = n.a # type: str # E: Incompatible types in assignment (expression has type "int", \ + variable has type "str") +i = n.b # type: int # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +x, y = n +x = y # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testNamedTupleConstructorArgumentTypes] +from typing import NamedTuple +N = NamedTuple('N', [('a', int), + ('b', str)]) +n = N('x', 'x') # E: Argument 1 to "N" has incompatible type "str"; expected "int" +n = N(1, b=2) # E: Argument 2 to "N" has incompatible type "int"; expected "str" +N(1, 'x') +N(b='x', a=1) + +[case testNamedTupleAsBaseClass] +from typing import NamedTuple +N = NamedTuple('N', [('a', int), + ('b', str)]) +class X(N): + pass +x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" +s = '' +i = 0 +s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") +i, s = x +s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testNamedTupleAsBaseClass2] +from typing import NamedTuple +class X(NamedTuple('N', [('a', int), + ('b', str)])): + pass +x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" +s = '' +i = 0 +s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") +i, s = x +s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testTwoNamedTuplesAsBaseClasses] +from typing import NamedTuple +A = NamedTuple('A', [('a', int)]) +B = NamedTuple('B', [('a', int)]) +class X(A, B): # E: Class has two incompatible bases derived from tuple + pass + +[case testTwoNamedTuplesAsBaseClasses2] +from typing import NamedTuple +A = NamedTuple('A', [('a', int)]) +class X(A, NamedTuple('B', [('a', int)])): # E: Class has two incompatible bases derived from tuple + pass + +[case testSelfTypeWithNamedTupleAsBase] +from typing import NamedTuple +A = NamedTuple('A', [('a', int), ('b', str)]) +class B(A): + def f(self, x: int) -> None: + self.f(self.a) + self.f(self.b) # E: Argument 1 to "f" of "B" has incompatible type "str"; expected "int" + i = 0 + s = '' + i, s = self + i, i = self # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +[out] +main: note: In member "f" of class "B": + +[case testTypeReferenceToClassDerivedFromNamedTuple] +from typing import NamedTuple +A = NamedTuple('A', [('a', int), ('b', str)]) +class B(A): + def f(self, x: 'B') -> None: + i = 0 + s = '' + self = x + i, s = x + i, s = x.a, x.b + i, s = x.a, x.a # E: Incompatible types in assignment (expression has type "int", \ + variable has type "str") + i, i = self # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +[out] +main: note: In member "f" of class "B": + +[case testNamedTupleSubtyping] +from typing import NamedTuple, Tuple +A = NamedTuple('A', [('a', int), ('b', str)]) +class B(A): pass +a = A(1, '') +b = B(1, '') +t = None # type: Tuple[int, str] +b = a # E: Incompatible types in assignment (expression has type "A", variable has type "B") +a = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "A") +b = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "B") +t = a +t = (1, '') +t = b +a = b + +[case testNamedTupleSimpleTypeInference] +from typing import NamedTuple, Tuple +A = NamedTuple('A', [('a', int)]) +l = [A(1), A(2)] +a = A(1) +a = l[0] +(i,) = l[0] +i, i = l[0] # E: Need more than 1 value to unpack (2 expected) +a = (1,) # E: Incompatible types in assignment (expression has type "Tuple[int]", \ + variable has type "A") +[builtins fixtures/list.py] + +[case testNamedTupleMissingClassAttribute] +import collections +MyNamedTuple = collections.namedtuple('MyNamedTuple', ['spam', 'eggs']) +MyNamedTuple.x # E: "MyNamedTuple" has no attribute "x" + +[case testNamedTupleEmptyItems] +from typing import NamedTuple +A = NamedTuple('A', []) +[builtins fixtures/list.py] + +[case testNamedTupleProperty] +from typing import NamedTuple +A = NamedTuple('A', [('a', int)]) +class B(A): + @property + def b(self) -> int: + return self.a +class C(B): pass +B(1).b +C(2).b +[builtins fixtures/property.py] + +[case testNamedTupleAsDict] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +a = x._asdict() + +[case testNamedTupleReplace] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +x = x._replace(y=5) +x = x._replace(z=5) # E: Unexpected keyword argument "z" for X._replace +x = x._replace(5) # E: Too many positional arguments for X._replace + +[case testNamedTupleReplaceAsClass] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +X._replace(x, x=1, y=2) +X._replace(x=1, y=2) # E: Missing positional argument "self" in call to X._replace + +[case testNamedTupleReplaceTyped] +from typing import NamedTuple + +X = NamedTuple('X', [('x', int), ('y', str)]) +x = None # type: X +x = x._replace(x=5) +x = x._replace(y=5) # E: Argument 1 to X._replace has incompatible type "int"; expected "str" diff --git a/test-data/unit/check-tuples.test b/test-data/unit/check-tuples.test index 50d1b3e64598d..f946c451123fd 100644 --- a/test-data/unit/check-tuples.test +++ b/test-data/unit/check-tuples.test @@ -1,770 +1,770 @@ --- Normal assignment and subtyping --- ------------------------------- - - -[case testTupleAssignmentWithTupleTypes] -from typing import Tuple -t1 = None # type: Tuple[A] -t2 = None # type: Tuple[B] -t3 = None # type: Tuple[A, A] -t4 = None # type: Tuple[A, B] -t5 = None # type: Tuple[B, A] - -t1 = t2 # E: Incompatible types in assignment (expression has type "Tuple[B]", variable has type "Tuple[A]") -t1 = t3 # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A]") -t3 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A]", variable has type "Tuple[A, A]") -t3 = t4 # E: Incompatible types in assignment (expression has type "Tuple[A, B]", variable has type "Tuple[A, A]") -t3 = t5 # E: Incompatible types in assignment (expression has type "Tuple[B, A]", variable has type "Tuple[A, A]") - -# Ok -t1 = t1 -t2 = t2 -t3 = t3 -t4 = t4 -t5 = t5 - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - -[case testTupleSubtyping] -from typing import Tuple -t1 = None # type: Tuple[A, A] -t2 = None # type: Tuple[A, B] -t3 = None # type: Tuple[B, A] - -t2 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A, B]") -t2 = t3 # E: Incompatible types in assignment (expression has type "Tuple[B, A]", variable has type "Tuple[A, B]") -t3 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[B, A]") -t3 = t2 # E: Incompatible types in assignment (expression has type "Tuple[A, B]", variable has type "Tuple[B, A]") - -t1 = t2 -t1 = t3 - -class A: pass -class B(A): pass -[builtins fixtures/tuple.py] - -[case testTupleCompatibilityWithOtherTypes] -from typing import Tuple -a, o = None, None # type: (A, object) -t = None # type: Tuple[A, A] - -a = t # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "A") -t = o # E: Incompatible types in assignment (expression has type "object", variable has type "Tuple[A, A]") -t = a # E: Incompatible types in assignment (expression has type "A", variable has type "Tuple[A, A]") -# TODO: callable types + tuples - -# Ok -o = t -t = None - -class A: pass -[builtins fixtures/tuple.py] - -[case testNestedTupleTypes] -from typing import Tuple -t1 = None # type: Tuple[A, Tuple[A, A]] -t2 = None # type: Tuple[B, Tuple[B, B]] - -t2 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, Tuple[A, A]]", variable has type "Tuple[B, Tuple[B, B]]") -t1 = t2 - -class A: pass -class B(A): pass -[builtins fixtures/tuple.py] - -[case testNestedTupleTypes2] -from typing import Tuple -t1 = None # type: Tuple[A, Tuple[A, A]] -t2 = None # type: Tuple[B, Tuple[B, B]] - -t2 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, Tuple[A, A]]", variable has type "Tuple[B, Tuple[B, B]]") -t1 = t2 - -class A: pass -class B(A): pass -[builtins fixtures/tuple.py] - -[case testSubtypingWithNamedTupleType] -from typing import Tuple -t1 = None # type: Tuple[A, A] -t2 = None # type: tuple - -t1 = t2 # E: Incompatible types in assignment (expression has type Tuple[Any, ...], variable has type "Tuple[A, A]") -t2 = t1 - -class A: pass -[builtins fixtures/tuple.py] - -[case testTupleInitializationWithNone] -from typing import Tuple -t = None # type: Tuple[A, A] -t = None -class A: pass -[builtins fixtures/tuple.py] - - --- Tuple expressions --- ----------------- - - -[case testTupleExpressions] -from typing import Tuple -t1 = None # type: tuple -t2 = None # type: Tuple[A] -t3 = None # type: Tuple[A, B] - -a, b, c = None, None, None # type: (A, B, C) - -t2 = () # E: Incompatible types in assignment (expression has type "Tuple[]", variable has type "Tuple[A]") -t2 = (a, a) # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A]") -t3 = (a, a) # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A, B]") -t3 = (b, b) # E: Incompatible types in assignment (expression has type "Tuple[B, B]", variable has type "Tuple[A, B]") -t3 = (a, b, a) # E: Incompatible types in assignment (expression has type "Tuple[A, B, A]", variable has type "Tuple[A, B]") - -t1 = () -t1 = (a,) -t2 = (a,) -t3 = (a, b) -t3 = (a, c) -t3 = (None, None) - -class A: pass -class B: pass -class C(B): pass -[builtins fixtures/tuple.py] - -[case testVoidValueInTuple] -import typing -(None, f()) # E: "f" does not return a value -(f(), None) # E: "f" does not return a value - -def f() -> None: pass -[builtins fixtures/tuple.py] - - --- Indexing --- -------- - - -[case testIndexingTuples] -from typing import Tuple -t1 = None # type: Tuple[A, B] -t2 = None # type: Tuple[A] -t3 = None # type: Tuple[A, B, C, D, E] -a, b = None, None # type: (A, B) -x = None # type: Tuple[A, B, C] -y = None # type: Tuple[A, C, E] -n = 0 - -a = t1[1] # E: Incompatible types in assignment (expression has type "B", variable has type "A") -b = t1[0] # E: Incompatible types in assignment (expression has type "A", variable has type "B") -t1[2] # E: Tuple index out of range -t1[3] # E: Tuple index out of range -t2[1] # E: Tuple index out of range -t1[n] # E: Tuple index must be an integer literal -t3[n:] # E: Tuple slice must be an integer literal -b = t1[(0)] # E: Incompatible types in assignment (expression has type "A", variable has type "B") - -a = t1[0] -b = t1[1] -b = t1[-1] -a = t1[(0)] -x = t3[0:3] # type (A, B, C) -y = t3[0:5:2] # type (A, C, E) -x = t3[:-2] # type (A, B, C) - -class A: pass -class B: pass -class C: pass -class D: pass -class E: pass -[builtins fixtures/tuple.py] - -[case testIndexingTuplesWithNegativeIntegers] -from typing import Tuple -t1 = None # type: Tuple[A, B] -t2 = None # type: Tuple[A] -a, b = None, None # type: A, B - -a = t1[-1] # E: Incompatible types in assignment (expression has type "B", variable has type "A") -b = t1[-2] # E: Incompatible types in assignment (expression has type "A", variable has type "B") -t1[-3] # E: Tuple index out of range -t1[-4] # E: Tuple index out of range -b = t2[(-1)] # E: Incompatible types in assignment (expression has type "A", variable has type "B") - -a = t1[-2] -b = t1[-1] -a = t2[(-1)] - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - -[case testAssigningToTupleItems] -from typing import Tuple -t = None # type: Tuple[A, B] -n = 0 - -t[0] = A() # E: Unsupported target for indexed assignment -t[2] = A() # E: Unsupported target for indexed assignment -t[n] = A() # E: Unsupported target for indexed assignment - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - - --- Multiple assignment --- ------------------- - - -[case testMultipleAssignmentWithTuples] -from typing import Tuple -t1 = None # type: Tuple[A, B] -t2 = None # type: Tuple[A, B, A] -a, b = None, None # type: (A, B) - -a, a = t1 # E: Incompatible types in assignment (expression has type "B", variable has type "A") -b, b = t1 # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a, b, b = t2 # E: Incompatible types in assignment (expression has type "A", variable has type "B") - -a, b = t1 -a, b, a = t2 - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - -[case testMultipleAssignmentWithInvalidNumberOfValues] -from typing import Tuple -t1 = None # type: Tuple[A, A, A] -a = None # type: A - -a, a = t1 # E: Too many values to unpack (2 expected, 3 provided) -a, a, a, a = t1 # E: Need more than 3 values to unpack (4 expected) - -a, a, a = t1 - -class A: pass -[builtins fixtures/tuple.py] - -[case testMultipleAssignmentWithTupleExpressionRvalue] - -a, b = None, None # type: (A, B) - -a, b = a, a # Fail -a, b = b, a # Fail - -a, b = a, b -a, a = a, a - -class A: pass -class B: pass -[builtins fixtures/tuple.py] -[out] -main:4: error: Incompatible types in assignment (expression has type "A", variable has type "B") -main:5: error: Incompatible types in assignment (expression has type "B", variable has type "A") -main:5: error: Incompatible types in assignment (expression has type "A", variable has type "B") - -[case testSubtypingInMultipleAssignment] - -a, b = None, None # type: (A, B) - -b, b = a, b # E: Incompatible types in assignment (expression has type "A", variable has type "B") -b, b = b, a # E: Incompatible types in assignment (expression has type "A", variable has type "B") - -a, b = b, b -b, a = b, b - -class A: pass -class B(A): pass -[builtins fixtures/tuple.py] - -[case testInitializationWithMultipleValues] - -a, b = None, None # type: (A, B) - -a1, b1 = a, a # type: (A, B) # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a2, b2 = b, b # type: (A, B) # E: Incompatible types in assignment (expression has type "B", variable has type "A") -a3, b3 = a # type: (A, B) # E: '__main__.A' object is not iterable -a4, b4 = None # type: (A, B) # E: 'builtins.None' object is not iterable -a5, b5 = a, b, a # type: (A, B) # E: Too many values to unpack (2 expected, 3 provided) - -ax, bx = a, b # type: (A, B) - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - -[case testMultipleAssignmentWithNonTupleRvalue] - -a, b = None, None # type: (A, B) -def f(): pass - -a, b = None # E: 'builtins.None' object is not iterable -a, b = a # E: '__main__.A' object is not iterable -a, b = f # E: 'def () -> Any' object is not iterable - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - -[case testMultipleAssignmentWithIndexedLvalues] - -a, b = None, None # type: (A, B) -aa, bb = None, None # type: (AA, BB) - -a[a], b[b] = a, bb # E: Incompatible types in assignment (expression has type "A", target has type "AA") -a[a], b[b] = aa, b # E: Incompatible types in assignment (expression has type "B", target has type "BB") -a[aa], b[b] = aa, bb # E: Invalid index type "AA" for "A" -a[a], b[bb] = aa, bb # E: Invalid index type "BB" for "B" -a[a], b[b] = aa, bb - -class A: - def __setitem__(self, x: 'A', y: 'AA') -> None: pass -class B: - def __setitem__(self, x: 'B', y: 'BB') -> None: pass - -class AA: pass -class BB: pass -[builtins fixtures/tuple.py] - -[case testMultipleDeclarationWithParentheses] - -(a, b) = (None, None) # type: int, str -a = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -b = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") -a = 1 -b = '' - -[case testMultipleAssignmentWithExtraParentheses] - -a, b = None, None # type: (A, B) - -(a, b) = (a, a) # E: Incompatible types in assignment (expression has type "A", variable has type "B") -(a, b) = (b, b) # E: Incompatible types in assignment (expression has type "B", variable has type "A") -((a), (b)) = ((a), (a)) # E: Incompatible types in assignment (expression has type "A", variable has type "B") -((a), (b)) = ((b), (b)) # E: Incompatible types in assignment (expression has type "B", variable has type "A") -[a, b] = a, a # E: Incompatible types in assignment (expression has type "A", variable has type "B") -[a, b] = b, b # E: Incompatible types in assignment (expression has type "B", variable has type "A") - -(a, b) = (a, b) -((a), (b)) = ((a), (b)) -[a, b] = a, b - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - -[case testMultipleAssignmentUsingSingleTupleType] -from typing import Tuple -a, b = None, None # type: Tuple[int, str] -a = 1 -b = '' -a = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -b = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") - -[case testMultipleAssignmentWithMixedVariables] -a = b, c = 1, 1 -x, y = p, q = 1, 1 -u, v, w = r, s = 1, 1 # E: Need more than 2 values to unpack (3 expected) -d, e = f, g, h = 1, 1 # E: Need more than 2 values to unpack (3 expected) - - --- Assignment to starred expressions --- --------------------------------- - - -[case testAssignmentToStarMissingAnnotation] -from typing import List -t = 1, 2 -a, b, *c = 1, 2 # E: Need type annotation for variable -aa, bb, *cc = t # E: Need type annotation for variable -[builtins fixtures/list.py] - -[case testAssignmentToStarAnnotation] -from typing import List -li, lo = None, None # type: List[int], List[object] -a, b, *c = 1, 2 # type: int, int, *List[int] -c = lo # E: Incompatible types in assignment (expression has type List[object], variable has type List[int]) -c = li -[builtins fixtures/list.py] - -[case testAssignmentToStarCount1] -from typing import List -ca = None # type: List[int] -c = [1] -a, b, *c = 1, # E: Need more than 1 value to unpack (2 expected) -a, b, *c = 1, 2 -a, b, *c = 1, 2, 3 -a, b, *c = 1, 2, 3, 4 -[builtins fixtures/list.py] - -[case testAssignmentToStarCount2] -from typing import List -ca = None # type: List[int] -t1 = 1, -t2 = 1, 2 -t3 = 1, 2, 3 -t4 = 1, 2, 3, 4 -c = [1] -a, b, *c = t1 # E: Need more than 1 value to unpack (2 expected) -a, b, *c = t2 -a, b, *c = t3 -a, b, *c = t4 -[builtins fixtures/list.py] - -[case testAssignmentToStarFromAny] -from typing import Any -a, c = Any(1), C() -p, *q = a -c = a -c = q - -class C: pass - -[case testAssignmentToComplexStar] -from typing import List -li = None # type: List[int] -a, *(li) = 1, -a, *(b, c) = 1, 2 # E: Need more than 1 value to unpack (2 expected) -a, *(b, c) = 1, 2, 3 -a, *(b, c) = 1, 2, 3, 4 # E: Too many values to unpack (2 expected, 3 provided) -[builtins fixtures/list.py] - -[case testAssignmentToStarFromTupleType] -from typing import List, Tuple -li = None # type: List[int] -la = None # type: List[A] -ta = None # type: Tuple[A, A, A] -a, *la = ta -a, *li = ta # E -a, *na = ta -na = la -na = a # E - -class A: pass -[builtins fixtures/list.py] -[out] -main:6: error: List item 0 has incompatible type "A" -main:6: error: List item 1 has incompatible type "A" -main:9: error: Incompatible types in assignment (expression has type "A", variable has type List[A]) - -[case testAssignmentToStarFromTupleInference] -from typing import List -li = None # type: List[int] -la = None # type: List[A] -a, *l = A(), A() -l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) -l = la - -class A: pass -[builtins fixtures/list.py] -[out] - -[case testAssignmentToStarFromListInference] -from typing import List -li = None # type: List[int] -la = None # type: List[A] -a, *l = [A(), A()] -l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) -l = la - -class A: pass -[builtins fixtures/list.py] -[out] - -[case testAssignmentToStarFromTupleTypeInference] -from typing import List, Tuple -li = None # type: List[int] -la = None # type: List[A] -ta = None # type: Tuple[A, A, A] -a, *l = ta -l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) -l = la - -class A: pass -[builtins fixtures/list.py] -[out] - -[case testAssignmentToStarFromListTypeInference] -from typing import List -li = None # type: List[int] -la = None # type: List[A] -a, *l = la -l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) -l = la - -class A: pass -[builtins fixtures/list.py] -[out] - - --- Nested tuple assignment --- ---------------------------- - - -[case testNestedTupleAssignment1] - -a1, b1, c1 = None, None, None # type: (A, B, C) -a2, b2, c2 = None, None, None # type: (A, B, C) - -a1, (b1, c1) = a2, (b2, c2) -a1, (a1, (b1, c1)) = a2, (a2, (b2, c2)) -a1, (a1, (a1, b1)) = a1, (a1, (a1, c1)) # Fail - -class A: pass -class B: pass -class C: pass -[out] -main:7: error: Incompatible types in assignment (expression has type "C", variable has type "B") - -[case testNestedTupleAssignment2] - -a1, b1, c1 = None, None, None # type: (A, B, C) -a2, b2, c2 = None, None, None # type: (A, B, C) -t = a1, b1 - -a2, b2 = t -(a2, b2), c2 = t, c1 -(a2, c2), c2 = t, c1 # Fail -t, c2 = (a2, b2), c2 -t, c2 = (a2, a2), c2 # Fail -t = a1, a1, a1 # Fail -t = a1 # Fail -a2, a2, a2 = t # Fail -a2, = t # Fail -a2 = t # Fail - -class A: pass -class B: pass -class C: pass -[out] -main:8: error: Incompatible types in assignment (expression has type "B", variable has type "C") -main:10: error: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A, B]") -main:11: error: Incompatible types in assignment (expression has type "Tuple[A, A, A]", variable has type "Tuple[A, B]") -main:12: error: Incompatible types in assignment (expression has type "A", variable has type "Tuple[A, B]") -main:13: error: Need more than 2 values to unpack (3 expected) -main:14: error: Too many values to unpack (1 expected, 2 provided) -main:15: error: Incompatible types in assignment (expression has type "Tuple[A, B]", variable has type "A") - - --- Error messages --- -------------- - - -[case testTupleErrorMessages] - -a = None # type: A - -(a, a) + a # E: Unsupported left operand type for + (Tuple[A, ...]) -a + (a, a) # E: Unsupported operand types for + ("A" and "Tuple[A, A]") -f((a, a)) # E: Argument 1 to "f" has incompatible type "Tuple[A, A]"; expected "A" -(a, a).foo # E: Tuple[A, ...] has no attribute "foo" - -def f(x: 'A') -> None: pass - -class A: - def __add__(self, x: 'A') -> 'A': pass -[builtins fixtures/tuple.py] - -[case testLargeTuplesInErrorMessages] - -a = None # type: LongTypeName -a + (a, a, a, a, a, a, a) # Fail - -class LongTypeName: - def __add__(self, x: 'LongTypeName') -> 'LongTypeName': pass -[builtins fixtures/tuple.py] -[out] -main:3: error: Unsupported operand types for + ("LongTypeName" and tuple(length 7)) - - --- Tuple methods --- ------------- - - -[case testTupleMethods] -from typing import Tuple -t = None # type: Tuple[int, str] -i = 0 -s = '' -b = bool() - -s = t.__len__() # E: Incompatible types in assignment (expression has type "int", variable has type "str") -i = t.__str__() # E: Incompatible types in assignment (expression has type "str", variable has type "int") -i = s in t # E: Incompatible types in assignment (expression has type "bool", variable has type "int") -t.foo # E: Tuple[Any, ...] has no attribute "foo" - -i = t.__len__() -s = t.__str__() -b = s in t - -[file builtins.py] -from typing import TypeVar, Generic -_T = TypeVar('_T') -class object: - def __init__(self) -> None: pass -class tuple(Generic[_T]): - def __len__(self) -> int: pass - def __str__(self) -> str: pass - def __contains__(self, o: object) -> bool: pass -class int: pass -class str: pass -class bool: pass -class type: pass -class function: pass - - --- For loop over tuple --- ------------------- - - -[case testForLoopOverTuple] -import typing -t = 1, 2 -for x in t: - x = 1 - x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -[builtins fixtures/for.py] - -[case testForLoopOverEmptyTuple] -import typing -t = () -for x in t: pass # E: Need type annotation for variable -[builtins fixtures/for.py] - -[case testForLoopOverNoneValuedTuple] -import typing -t = () -for x in None, None: pass # E: Need type annotation for variable -[builtins fixtures/for.py] - -[case testForLoopOverTupleAndSubtyping] -import typing -class A: pass -class B(A): pass -for x in B(), A(): - x = A() - x = B() - x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "A") -[builtins fixtures/for.py] - -[case testTupleIterable] -y = 'a' -x = sum((1,2)) -y = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") -[builtins fixtures/tuple.py] - - --- Tuple as a base type --- -------------------- - - -[case testTupleBaseClass] -import m -[file m.pyi] -from typing import Tuple -class A(Tuple[int, str]): - def f(self, x: int) -> None: - a, b = 1, '' - a, b = self - b, a = self # Error - self.f('') # Error -[builtins fixtures/tuple.py] -[out] -main:1: note: In module imported here: -tmp/m.pyi: note: In member "f" of class "A": -tmp/m.pyi:6: error: Incompatible types in assignment (expression has type "int", variable has type "str") -tmp/m.pyi:6: error: Incompatible types in assignment (expression has type "str", variable has type "int") -tmp/m.pyi:7: error: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" - -[case testInvalidTupleBaseClass] -from typing import Tuple -class A(Tuple[int, str]): pass # E: Tuple[...] not supported as a base class outside a stub file -[builtins fixtures/tuple.py] -[out] - -[case testValidTupleBaseClass] -from typing import Tuple -class A(tuple): pass -[out] - -[case testTupleBaseClass2-skip] -import m -[file m.pyi] -# This doesn't work correctly -- no errors are reported (#867) -from typing import Tuple -a = None # type: A -class A(Tuple[int, str]): pass -x, y = a -x() # Expected: "int" not callable -y() # Expected: "str" not callable -[out] -(should fail) - -[case testGenericClassWithTupleBaseClass] -from typing import TypeVar, Generic, Tuple -T = TypeVar('T') -class Test(Generic[T], Tuple[T]): pass -x = Test() # type: Test[int] -[builtins fixtures/tuple.py] -[out] -main:3: error: Tuple[...] not supported as a base class outside a stub file -main:4: error: Generic tuple types not supported - - --- Variable-length tuples (Tuple[t, ...] with literal '...') --- --------------------------------------------------------- - - -[case testIndexingVariableLengthTuple] -from typing import Tuple -x = () # type: Tuple[str, ...] -n = 5 -x[n]() # E: "str" not callable -x[3]() # E: "str" not callable -[builtins fixtures/tuple.py] - -[case testSubtypingVariableLengthTuple] -from typing import Tuple -class A: pass -class B(A): pass -def fa(t: Tuple[A, ...]) -> None: pass -def fb(t: Tuple[B, ...]) -> None: pass -ta = () # type: Tuple[A, ...] -tb = () # type: Tuple[B, ...] -fa(ta) -fa(tb) -fb(tb) -fb(ta) # E: Argument 1 to "fb" has incompatible type Tuple[A, ...]; expected Tuple[B, ...] -[builtins fixtures/tuple.py] - -[case testSubtypingFixedAndVariableLengthTuples] -from typing import Tuple -class A: pass -class B(A): pass -def fa(t: Tuple[A, ...]) -> None: pass -def fb(t: Tuple[B, ...]) -> None: pass -aa = (A(), A()) -ab = (A(), B()) -bb = (B(), B()) -fa(aa) -fa(ab) -fa(bb) -fb(bb) -fb(ab) # E: Argument 1 to "fb" has incompatible type "Tuple[A, B]"; expected Tuple[B, ...] -fb(aa) # E: Argument 1 to "fb" has incompatible type "Tuple[A, A]"; expected Tuple[B, ...] -[builtins fixtures/tuple.py] - -[case testSubtypingTupleIsContainer] -from typing import Container -a = None # type: Container[str] -a = () - -[case testSubtypingTupleIsSized] -from typing import Sized -a = None # type: Sized -a = () +-- Normal assignment and subtyping +-- ------------------------------- + + +[case testTupleAssignmentWithTupleTypes] +from typing import Tuple +t1 = None # type: Tuple[A] +t2 = None # type: Tuple[B] +t3 = None # type: Tuple[A, A] +t4 = None # type: Tuple[A, B] +t5 = None # type: Tuple[B, A] + +t1 = t2 # E: Incompatible types in assignment (expression has type "Tuple[B]", variable has type "Tuple[A]") +t1 = t3 # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A]") +t3 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A]", variable has type "Tuple[A, A]") +t3 = t4 # E: Incompatible types in assignment (expression has type "Tuple[A, B]", variable has type "Tuple[A, A]") +t3 = t5 # E: Incompatible types in assignment (expression has type "Tuple[B, A]", variable has type "Tuple[A, A]") + +# Ok +t1 = t1 +t2 = t2 +t3 = t3 +t4 = t4 +t5 = t5 + +class A: pass +class B: pass +[builtins fixtures/tuple.py] + +[case testTupleSubtyping] +from typing import Tuple +t1 = None # type: Tuple[A, A] +t2 = None # type: Tuple[A, B] +t3 = None # type: Tuple[B, A] + +t2 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A, B]") +t2 = t3 # E: Incompatible types in assignment (expression has type "Tuple[B, A]", variable has type "Tuple[A, B]") +t3 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[B, A]") +t3 = t2 # E: Incompatible types in assignment (expression has type "Tuple[A, B]", variable has type "Tuple[B, A]") + +t1 = t2 +t1 = t3 + +class A: pass +class B(A): pass +[builtins fixtures/tuple.py] + +[case testTupleCompatibilityWithOtherTypes] +from typing import Tuple +a, o = None, None # type: (A, object) +t = None # type: Tuple[A, A] + +a = t # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "A") +t = o # E: Incompatible types in assignment (expression has type "object", variable has type "Tuple[A, A]") +t = a # E: Incompatible types in assignment (expression has type "A", variable has type "Tuple[A, A]") +# TODO: callable types + tuples + +# Ok +o = t +t = None + +class A: pass +[builtins fixtures/tuple.py] + +[case testNestedTupleTypes] +from typing import Tuple +t1 = None # type: Tuple[A, Tuple[A, A]] +t2 = None # type: Tuple[B, Tuple[B, B]] + +t2 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, Tuple[A, A]]", variable has type "Tuple[B, Tuple[B, B]]") +t1 = t2 + +class A: pass +class B(A): pass +[builtins fixtures/tuple.py] + +[case testNestedTupleTypes2] +from typing import Tuple +t1 = None # type: Tuple[A, Tuple[A, A]] +t2 = None # type: Tuple[B, Tuple[B, B]] + +t2 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, Tuple[A, A]]", variable has type "Tuple[B, Tuple[B, B]]") +t1 = t2 + +class A: pass +class B(A): pass +[builtins fixtures/tuple.py] + +[case testSubtypingWithNamedTupleType] +from typing import Tuple +t1 = None # type: Tuple[A, A] +t2 = None # type: tuple + +t1 = t2 # E: Incompatible types in assignment (expression has type Tuple[Any, ...], variable has type "Tuple[A, A]") +t2 = t1 + +class A: pass +[builtins fixtures/tuple.py] + +[case testTupleInitializationWithNone] +from typing import Tuple +t = None # type: Tuple[A, A] +t = None +class A: pass +[builtins fixtures/tuple.py] + + +-- Tuple expressions +-- ----------------- + + +[case testTupleExpressions] +from typing import Tuple +t1 = None # type: tuple +t2 = None # type: Tuple[A] +t3 = None # type: Tuple[A, B] + +a, b, c = None, None, None # type: (A, B, C) + +t2 = () # E: Incompatible types in assignment (expression has type "Tuple[]", variable has type "Tuple[A]") +t2 = (a, a) # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A]") +t3 = (a, a) # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A, B]") +t3 = (b, b) # E: Incompatible types in assignment (expression has type "Tuple[B, B]", variable has type "Tuple[A, B]") +t3 = (a, b, a) # E: Incompatible types in assignment (expression has type "Tuple[A, B, A]", variable has type "Tuple[A, B]") + +t1 = () +t1 = (a,) +t2 = (a,) +t3 = (a, b) +t3 = (a, c) +t3 = (None, None) + +class A: pass +class B: pass +class C(B): pass +[builtins fixtures/tuple.py] + +[case testVoidValueInTuple] +import typing +(None, f()) # E: "f" does not return a value +(f(), None) # E: "f" does not return a value + +def f() -> None: pass +[builtins fixtures/tuple.py] + + +-- Indexing +-- -------- + + +[case testIndexingTuples] +from typing import Tuple +t1 = None # type: Tuple[A, B] +t2 = None # type: Tuple[A] +t3 = None # type: Tuple[A, B, C, D, E] +a, b = None, None # type: (A, B) +x = None # type: Tuple[A, B, C] +y = None # type: Tuple[A, C, E] +n = 0 + +a = t1[1] # E: Incompatible types in assignment (expression has type "B", variable has type "A") +b = t1[0] # E: Incompatible types in assignment (expression has type "A", variable has type "B") +t1[2] # E: Tuple index out of range +t1[3] # E: Tuple index out of range +t2[1] # E: Tuple index out of range +t1[n] # E: Tuple index must be an integer literal +t3[n:] # E: Tuple slice must be an integer literal +b = t1[(0)] # E: Incompatible types in assignment (expression has type "A", variable has type "B") + +a = t1[0] +b = t1[1] +b = t1[-1] +a = t1[(0)] +x = t3[0:3] # type (A, B, C) +y = t3[0:5:2] # type (A, C, E) +x = t3[:-2] # type (A, B, C) + +class A: pass +class B: pass +class C: pass +class D: pass +class E: pass +[builtins fixtures/tuple.py] + +[case testIndexingTuplesWithNegativeIntegers] +from typing import Tuple +t1 = None # type: Tuple[A, B] +t2 = None # type: Tuple[A] +a, b = None, None # type: A, B + +a = t1[-1] # E: Incompatible types in assignment (expression has type "B", variable has type "A") +b = t1[-2] # E: Incompatible types in assignment (expression has type "A", variable has type "B") +t1[-3] # E: Tuple index out of range +t1[-4] # E: Tuple index out of range +b = t2[(-1)] # E: Incompatible types in assignment (expression has type "A", variable has type "B") + +a = t1[-2] +b = t1[-1] +a = t2[(-1)] + +class A: pass +class B: pass +[builtins fixtures/tuple.py] + +[case testAssigningToTupleItems] +from typing import Tuple +t = None # type: Tuple[A, B] +n = 0 + +t[0] = A() # E: Unsupported target for indexed assignment +t[2] = A() # E: Unsupported target for indexed assignment +t[n] = A() # E: Unsupported target for indexed assignment + +class A: pass +class B: pass +[builtins fixtures/tuple.py] + + +-- Multiple assignment +-- ------------------- + + +[case testMultipleAssignmentWithTuples] +from typing import Tuple +t1 = None # type: Tuple[A, B] +t2 = None # type: Tuple[A, B, A] +a, b = None, None # type: (A, B) + +a, a = t1 # E: Incompatible types in assignment (expression has type "B", variable has type "A") +b, b = t1 # E: Incompatible types in assignment (expression has type "A", variable has type "B") +a, b, b = t2 # E: Incompatible types in assignment (expression has type "A", variable has type "B") + +a, b = t1 +a, b, a = t2 + +class A: pass +class B: pass +[builtins fixtures/tuple.py] + +[case testMultipleAssignmentWithInvalidNumberOfValues] +from typing import Tuple +t1 = None # type: Tuple[A, A, A] +a = None # type: A + +a, a = t1 # E: Too many values to unpack (2 expected, 3 provided) +a, a, a, a = t1 # E: Need more than 3 values to unpack (4 expected) + +a, a, a = t1 + +class A: pass +[builtins fixtures/tuple.py] + +[case testMultipleAssignmentWithTupleExpressionRvalue] + +a, b = None, None # type: (A, B) + +a, b = a, a # Fail +a, b = b, a # Fail + +a, b = a, b +a, a = a, a + +class A: pass +class B: pass +[builtins fixtures/tuple.py] +[out] +main:4: error: Incompatible types in assignment (expression has type "A", variable has type "B") +main:5: error: Incompatible types in assignment (expression has type "B", variable has type "A") +main:5: error: Incompatible types in assignment (expression has type "A", variable has type "B") + +[case testSubtypingInMultipleAssignment] + +a, b = None, None # type: (A, B) + +b, b = a, b # E: Incompatible types in assignment (expression has type "A", variable has type "B") +b, b = b, a # E: Incompatible types in assignment (expression has type "A", variable has type "B") + +a, b = b, b +b, a = b, b + +class A: pass +class B(A): pass +[builtins fixtures/tuple.py] + +[case testInitializationWithMultipleValues] + +a, b = None, None # type: (A, B) + +a1, b1 = a, a # type: (A, B) # E: Incompatible types in assignment (expression has type "A", variable has type "B") +a2, b2 = b, b # type: (A, B) # E: Incompatible types in assignment (expression has type "B", variable has type "A") +a3, b3 = a # type: (A, B) # E: '__main__.A' object is not iterable +a4, b4 = None # type: (A, B) # E: 'builtins.None' object is not iterable +a5, b5 = a, b, a # type: (A, B) # E: Too many values to unpack (2 expected, 3 provided) + +ax, bx = a, b # type: (A, B) + +class A: pass +class B: pass +[builtins fixtures/tuple.py] + +[case testMultipleAssignmentWithNonTupleRvalue] + +a, b = None, None # type: (A, B) +def f(): pass + +a, b = None # E: 'builtins.None' object is not iterable +a, b = a # E: '__main__.A' object is not iterable +a, b = f # E: 'def () -> Any' object is not iterable + +class A: pass +class B: pass +[builtins fixtures/tuple.py] + +[case testMultipleAssignmentWithIndexedLvalues] + +a, b = None, None # type: (A, B) +aa, bb = None, None # type: (AA, BB) + +a[a], b[b] = a, bb # E: Incompatible types in assignment (expression has type "A", target has type "AA") +a[a], b[b] = aa, b # E: Incompatible types in assignment (expression has type "B", target has type "BB") +a[aa], b[b] = aa, bb # E: Invalid index type "AA" for "A" +a[a], b[bb] = aa, bb # E: Invalid index type "BB" for "B" +a[a], b[b] = aa, bb + +class A: + def __setitem__(self, x: 'A', y: 'AA') -> None: pass +class B: + def __setitem__(self, x: 'B', y: 'BB') -> None: pass + +class AA: pass +class BB: pass +[builtins fixtures/tuple.py] + +[case testMultipleDeclarationWithParentheses] + +(a, b) = (None, None) # type: int, str +a = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +b = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") +a = 1 +b = '' + +[case testMultipleAssignmentWithExtraParentheses] + +a, b = None, None # type: (A, B) + +(a, b) = (a, a) # E: Incompatible types in assignment (expression has type "A", variable has type "B") +(a, b) = (b, b) # E: Incompatible types in assignment (expression has type "B", variable has type "A") +((a), (b)) = ((a), (a)) # E: Incompatible types in assignment (expression has type "A", variable has type "B") +((a), (b)) = ((b), (b)) # E: Incompatible types in assignment (expression has type "B", variable has type "A") +[a, b] = a, a # E: Incompatible types in assignment (expression has type "A", variable has type "B") +[a, b] = b, b # E: Incompatible types in assignment (expression has type "B", variable has type "A") + +(a, b) = (a, b) +((a), (b)) = ((a), (b)) +[a, b] = a, b + +class A: pass +class B: pass +[builtins fixtures/tuple.py] + +[case testMultipleAssignmentUsingSingleTupleType] +from typing import Tuple +a, b = None, None # type: Tuple[int, str] +a = 1 +b = '' +a = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +b = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testMultipleAssignmentWithMixedVariables] +a = b, c = 1, 1 +x, y = p, q = 1, 1 +u, v, w = r, s = 1, 1 # E: Need more than 2 values to unpack (3 expected) +d, e = f, g, h = 1, 1 # E: Need more than 2 values to unpack (3 expected) + + +-- Assignment to starred expressions +-- --------------------------------- + + +[case testAssignmentToStarMissingAnnotation] +from typing import List +t = 1, 2 +a, b, *c = 1, 2 # E: Need type annotation for variable +aa, bb, *cc = t # E: Need type annotation for variable +[builtins fixtures/list.py] + +[case testAssignmentToStarAnnotation] +from typing import List +li, lo = None, None # type: List[int], List[object] +a, b, *c = 1, 2 # type: int, int, *List[int] +c = lo # E: Incompatible types in assignment (expression has type List[object], variable has type List[int]) +c = li +[builtins fixtures/list.py] + +[case testAssignmentToStarCount1] +from typing import List +ca = None # type: List[int] +c = [1] +a, b, *c = 1, # E: Need more than 1 value to unpack (2 expected) +a, b, *c = 1, 2 +a, b, *c = 1, 2, 3 +a, b, *c = 1, 2, 3, 4 +[builtins fixtures/list.py] + +[case testAssignmentToStarCount2] +from typing import List +ca = None # type: List[int] +t1 = 1, +t2 = 1, 2 +t3 = 1, 2, 3 +t4 = 1, 2, 3, 4 +c = [1] +a, b, *c = t1 # E: Need more than 1 value to unpack (2 expected) +a, b, *c = t2 +a, b, *c = t3 +a, b, *c = t4 +[builtins fixtures/list.py] + +[case testAssignmentToStarFromAny] +from typing import Any +a, c = Any(1), C() +p, *q = a +c = a +c = q + +class C: pass + +[case testAssignmentToComplexStar] +from typing import List +li = None # type: List[int] +a, *(li) = 1, +a, *(b, c) = 1, 2 # E: Need more than 1 value to unpack (2 expected) +a, *(b, c) = 1, 2, 3 +a, *(b, c) = 1, 2, 3, 4 # E: Too many values to unpack (2 expected, 3 provided) +[builtins fixtures/list.py] + +[case testAssignmentToStarFromTupleType] +from typing import List, Tuple +li = None # type: List[int] +la = None # type: List[A] +ta = None # type: Tuple[A, A, A] +a, *la = ta +a, *li = ta # E +a, *na = ta +na = la +na = a # E + +class A: pass +[builtins fixtures/list.py] +[out] +main:6: error: List item 0 has incompatible type "A" +main:6: error: List item 1 has incompatible type "A" +main:9: error: Incompatible types in assignment (expression has type "A", variable has type List[A]) + +[case testAssignmentToStarFromTupleInference] +from typing import List +li = None # type: List[int] +la = None # type: List[A] +a, *l = A(), A() +l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) +l = la + +class A: pass +[builtins fixtures/list.py] +[out] + +[case testAssignmentToStarFromListInference] +from typing import List +li = None # type: List[int] +la = None # type: List[A] +a, *l = [A(), A()] +l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) +l = la + +class A: pass +[builtins fixtures/list.py] +[out] + +[case testAssignmentToStarFromTupleTypeInference] +from typing import List, Tuple +li = None # type: List[int] +la = None # type: List[A] +ta = None # type: Tuple[A, A, A] +a, *l = ta +l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) +l = la + +class A: pass +[builtins fixtures/list.py] +[out] + +[case testAssignmentToStarFromListTypeInference] +from typing import List +li = None # type: List[int] +la = None # type: List[A] +a, *l = la +l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) +l = la + +class A: pass +[builtins fixtures/list.py] +[out] + + +-- Nested tuple assignment +-- ---------------------------- + + +[case testNestedTupleAssignment1] + +a1, b1, c1 = None, None, None # type: (A, B, C) +a2, b2, c2 = None, None, None # type: (A, B, C) + +a1, (b1, c1) = a2, (b2, c2) +a1, (a1, (b1, c1)) = a2, (a2, (b2, c2)) +a1, (a1, (a1, b1)) = a1, (a1, (a1, c1)) # Fail + +class A: pass +class B: pass +class C: pass +[out] +main:7: error: Incompatible types in assignment (expression has type "C", variable has type "B") + +[case testNestedTupleAssignment2] + +a1, b1, c1 = None, None, None # type: (A, B, C) +a2, b2, c2 = None, None, None # type: (A, B, C) +t = a1, b1 + +a2, b2 = t +(a2, b2), c2 = t, c1 +(a2, c2), c2 = t, c1 # Fail +t, c2 = (a2, b2), c2 +t, c2 = (a2, a2), c2 # Fail +t = a1, a1, a1 # Fail +t = a1 # Fail +a2, a2, a2 = t # Fail +a2, = t # Fail +a2 = t # Fail + +class A: pass +class B: pass +class C: pass +[out] +main:8: error: Incompatible types in assignment (expression has type "B", variable has type "C") +main:10: error: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A, B]") +main:11: error: Incompatible types in assignment (expression has type "Tuple[A, A, A]", variable has type "Tuple[A, B]") +main:12: error: Incompatible types in assignment (expression has type "A", variable has type "Tuple[A, B]") +main:13: error: Need more than 2 values to unpack (3 expected) +main:14: error: Too many values to unpack (1 expected, 2 provided) +main:15: error: Incompatible types in assignment (expression has type "Tuple[A, B]", variable has type "A") + + +-- Error messages +-- -------------- + + +[case testTupleErrorMessages] + +a = None # type: A + +(a, a) + a # E: Unsupported left operand type for + (Tuple[A, ...]) +a + (a, a) # E: Unsupported operand types for + ("A" and "Tuple[A, A]") +f((a, a)) # E: Argument 1 to "f" has incompatible type "Tuple[A, A]"; expected "A" +(a, a).foo # E: Tuple[A, ...] has no attribute "foo" + +def f(x: 'A') -> None: pass + +class A: + def __add__(self, x: 'A') -> 'A': pass +[builtins fixtures/tuple.py] + +[case testLargeTuplesInErrorMessages] + +a = None # type: LongTypeName +a + (a, a, a, a, a, a, a) # Fail + +class LongTypeName: + def __add__(self, x: 'LongTypeName') -> 'LongTypeName': pass +[builtins fixtures/tuple.py] +[out] +main:3: error: Unsupported operand types for + ("LongTypeName" and tuple(length 7)) + + +-- Tuple methods +-- ------------- + + +[case testTupleMethods] +from typing import Tuple +t = None # type: Tuple[int, str] +i = 0 +s = '' +b = bool() + +s = t.__len__() # E: Incompatible types in assignment (expression has type "int", variable has type "str") +i = t.__str__() # E: Incompatible types in assignment (expression has type "str", variable has type "int") +i = s in t # E: Incompatible types in assignment (expression has type "bool", variable has type "int") +t.foo # E: Tuple[Any, ...] has no attribute "foo" + +i = t.__len__() +s = t.__str__() +b = s in t + +[file builtins.py] +from typing import TypeVar, Generic +_T = TypeVar('_T') +class object: + def __init__(self) -> None: pass +class tuple(Generic[_T]): + def __len__(self) -> int: pass + def __str__(self) -> str: pass + def __contains__(self, o: object) -> bool: pass +class int: pass +class str: pass +class bool: pass +class type: pass +class function: pass + + +-- For loop over tuple +-- ------------------- + + +[case testForLoopOverTuple] +import typing +t = 1, 2 +for x in t: + x = 1 + x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +[builtins fixtures/for.py] + +[case testForLoopOverEmptyTuple] +import typing +t = () +for x in t: pass # E: Need type annotation for variable +[builtins fixtures/for.py] + +[case testForLoopOverNoneValuedTuple] +import typing +t = () +for x in None, None: pass # E: Need type annotation for variable +[builtins fixtures/for.py] + +[case testForLoopOverTupleAndSubtyping] +import typing +class A: pass +class B(A): pass +for x in B(), A(): + x = A() + x = B() + x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "A") +[builtins fixtures/for.py] + +[case testTupleIterable] +y = 'a' +x = sum((1,2)) +y = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") +[builtins fixtures/tuple.py] + + +-- Tuple as a base type +-- -------------------- + + +[case testTupleBaseClass] +import m +[file m.pyi] +from typing import Tuple +class A(Tuple[int, str]): + def f(self, x: int) -> None: + a, b = 1, '' + a, b = self + b, a = self # Error + self.f('') # Error +[builtins fixtures/tuple.py] +[out] +main:1: note: In module imported here: +tmp/m.pyi: note: In member "f" of class "A": +tmp/m.pyi:6: error: Incompatible types in assignment (expression has type "int", variable has type "str") +tmp/m.pyi:6: error: Incompatible types in assignment (expression has type "str", variable has type "int") +tmp/m.pyi:7: error: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" + +[case testInvalidTupleBaseClass] +from typing import Tuple +class A(Tuple[int, str]): pass # E: Tuple[...] not supported as a base class outside a stub file +[builtins fixtures/tuple.py] +[out] + +[case testValidTupleBaseClass] +from typing import Tuple +class A(tuple): pass +[out] + +[case testTupleBaseClass2-skip] +import m +[file m.pyi] +# This doesn't work correctly -- no errors are reported (#867) +from typing import Tuple +a = None # type: A +class A(Tuple[int, str]): pass +x, y = a +x() # Expected: "int" not callable +y() # Expected: "str" not callable +[out] +(should fail) + +[case testGenericClassWithTupleBaseClass] +from typing import TypeVar, Generic, Tuple +T = TypeVar('T') +class Test(Generic[T], Tuple[T]): pass +x = Test() # type: Test[int] +[builtins fixtures/tuple.py] +[out] +main:3: error: Tuple[...] not supported as a base class outside a stub file +main:4: error: Generic tuple types not supported + + +-- Variable-length tuples (Tuple[t, ...] with literal '...') +-- --------------------------------------------------------- + + +[case testIndexingVariableLengthTuple] +from typing import Tuple +x = () # type: Tuple[str, ...] +n = 5 +x[n]() # E: "str" not callable +x[3]() # E: "str" not callable +[builtins fixtures/tuple.py] + +[case testSubtypingVariableLengthTuple] +from typing import Tuple +class A: pass +class B(A): pass +def fa(t: Tuple[A, ...]) -> None: pass +def fb(t: Tuple[B, ...]) -> None: pass +ta = () # type: Tuple[A, ...] +tb = () # type: Tuple[B, ...] +fa(ta) +fa(tb) +fb(tb) +fb(ta) # E: Argument 1 to "fb" has incompatible type Tuple[A, ...]; expected Tuple[B, ...] +[builtins fixtures/tuple.py] + +[case testSubtypingFixedAndVariableLengthTuples] +from typing import Tuple +class A: pass +class B(A): pass +def fa(t: Tuple[A, ...]) -> None: pass +def fb(t: Tuple[B, ...]) -> None: pass +aa = (A(), A()) +ab = (A(), B()) +bb = (B(), B()) +fa(aa) +fa(ab) +fa(bb) +fb(bb) +fb(ab) # E: Argument 1 to "fb" has incompatible type "Tuple[A, B]"; expected Tuple[B, ...] +fb(aa) # E: Argument 1 to "fb" has incompatible type "Tuple[A, A]"; expected Tuple[B, ...] +[builtins fixtures/tuple.py] + +[case testSubtypingTupleIsContainer] +from typing import Container +a = None # type: Container[str] +a = () + +[case testSubtypingTupleIsSized] +from typing import Sized +a = None # type: Sized +a = () From dc9d8daae1f3249f14a26df3f62165107d6aff9a Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Sun, 14 Aug 2016 19:12:09 +0300 Subject: [PATCH 20/74] fix line endings --- mypy/build.py | 3356 +++++++-------- mypy/checker.py | 4918 +++++++++++----------- mypy/checkexpr.py | 3644 ++++++++--------- mypy/expandtype.py | 224 +- mypy/join.py | 734 ++-- mypy/meet.py | 570 +-- mypy/messages.py | 1840 ++++----- mypy/nodes.py | 4484 ++++++++++---------- mypy/parse.py | 4068 +++++++++--------- mypy/semanal.py | 5678 +++++++++++++------------- mypy/subtypes.py | 818 ++-- mypy/typeanal.py | 860 ++-- mypy/types.py | 2902 ++++++------- test-data/unit/check-lists.test | 126 +- test-data/unit/check-namedtuple.test | 446 +- test-data/unit/check-tuples.test | 1540 +++---- 16 files changed, 18104 insertions(+), 18104 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index a930614989f90..e8a84c97c7c76 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -1,1678 +1,1678 @@ -"""Facilities to analyze entire programs, including imported modules. - -Parse and analyze the source files of a program in the correct order -(based on file dependencies), and collect the results. - -This module only directs a build, which is performed in multiple passes per -file. The individual passes are implemented in separate modules. - -The function build() is the main interface to this module. -""" -# TODO: More consistent terminology, e.g. path/fnam, module/id, state/file - -import binascii -import collections -import contextlib -import json -import os -import os.path -import sys -import time -from os.path import dirname, basename - -from typing import (AbstractSet, Dict, Iterable, Iterator, List, - NamedTuple, Optional, Set, Tuple, Union, Mapping) - -from mypy.types import Type -from mypy.nodes import (MypyFile, Node, Import, ImportFrom, ImportAll, - SymbolTableNode, MODULE_REF) -from mypy.semanal import FirstPass, SemanticAnalyzer, ThirdPass -from mypy.checker import TypeChecker -from mypy.errors import Errors, CompileError, DecodeError, report_internal_error -from mypy import fixup -from mypy.report import Reports -from mypy import defaults -from mypy import moduleinfo -from mypy import util -from mypy.fixup import fixup_module_pass_one, fixup_module_pass_two -from mypy.options import Options -from mypy.parse import parse -from mypy.stats import dump_type_stats - - -# We need to know the location of this file to load data, but -# until Python 3.4, __file__ is relative. -__file__ = os.path.realpath(__file__) - -PYTHON_EXTENSIONS = ['.pyi', '.py'] - - -class BuildResult: - """The result of a successful build. - - Attributes: - manager: The build manager. - files: Dictionary from module name to related AST node. - types: Dictionary from parse tree node to its inferred type. - errors: List of error messages. - """ - - def __init__(self, manager: 'BuildManager') -> None: - self.manager = manager - self.files = manager.modules - self.types = manager.type_checker.type_map - self.errors = manager.errors.messages() - - -class BuildSource: - def __init__(self, path: Optional[str], module: Optional[str], - text: Optional[str]) -> None: - self.path = path - self.module = module or '__main__' - self.text = text - - @property - def effective_path(self) -> str: - """Return the effective path (ie, if its from in memory)""" - return self.path or '' - - -class BuildSourceSet: - """Efficiently test a file's membership in the set of build sources.""" - - def __init__(self, sources: List[BuildSource]) -> None: - self.source_text_present = False - self.source_modules = set() # type: Set[str] - self.source_paths = set() # type: Set[str] - - for source in sources: - if source.text is not None: - self.source_text_present = True - elif source.path: - self.source_paths.add(source.path) - else: - self.source_modules.add(source.module) - - def is_source(self, file: MypyFile) -> bool: - if file.path and file.path in self.source_paths: - return True - elif file._fullname in self.source_modules: - return True - elif file.path is None and self.source_text_present: - return True - else: - return False - - -def build(sources: List[BuildSource], - options: Options, - alt_lib_path: str = None, - bin_dir: str = None) -> BuildResult: - """Analyze a program. - - A single call to build performs parsing, semantic analysis and optionally - type checking for the program *and* all imported modules, recursively. - - Return BuildResult if successful or only non-blocking errors were found; - otherwise raise CompileError. - - Args: - sources: list of sources to build - options: build options - alt_lib_path: an additional directory for looking up library modules - (takes precedence over other directories) - bin_dir: directory containing the mypy script, used for finding data - directories; if omitted, use '.' as the data directory - """ - - data_dir = default_data_dir(bin_dir) - - find_module_clear_caches() - - # Determine the default module search path. - lib_path = default_lib_path(data_dir, options.python_version) - - if options.use_builtins_fixtures: - # Use stub builtins (to speed up test cases and to make them easier to - # debug). This is a test-only feature, so assume our files are laid out - # as in the source tree. - root_dir = os.path.dirname(os.path.dirname(__file__)) - lib_path.insert(0, os.path.join(root_dir, 'test-data', 'unit', 'lib-stub')) - else: - for source in sources: - if source.path: - # Include directory of the program file in the module search path. - dir = remove_cwd_prefix_from_path(dirname(source.path)) - if dir not in lib_path: - lib_path.insert(0, dir) - - # Do this even if running as a file, for sanity (mainly because with - # multiple builds, there could be a mix of files/modules, so its easier - # to just define the semantics that we always add the current director - # to the lib_path - lib_path.insert(0, os.getcwd()) - - # Add MYPYPATH environment variable to front of library path, if defined. - lib_path[:0] = mypy_path() - - # If provided, insert the caller-supplied extra module path to the - # beginning (highest priority) of the search path. - if alt_lib_path: - lib_path.insert(0, alt_lib_path) - - reports = Reports(data_dir, options.report_dirs) - - source_set = BuildSourceSet(sources) - - # Construct a build manager object to hold state during the build. - # - # Ignore current directory prefix in error messages. - manager = BuildManager(data_dir, lib_path, - ignore_prefix=os.getcwd(), - source_set=source_set, - reports=reports, - options=options) - - try: - dispatch(sources, manager) - return BuildResult(manager) - finally: - manager.log("Build finished with %d modules, %d types, and %d errors" % - (len(manager.modules), - len(manager.type_checker.type_map), - manager.errors.num_messages())) - # Finish the HTML or XML reports even if CompileError was raised. - reports.finish() - - -def default_data_dir(bin_dir: Optional[str]) -> str: - """Returns directory containing typeshed directory - - Args: - bin_dir: directory containing the mypy script - """ - if not bin_dir: - mypy_package = os.path.dirname(__file__) - parent = os.path.dirname(mypy_package) - if (os.path.basename(parent) == 'site-packages' or - os.path.basename(parent) == 'dist-packages'): - # Installed in site-packages or dist-packages, but invoked with python3 -m mypy; - # __file__ is .../blah/lib/python3.N/site-packages/mypy/build.py - # or .../blah/lib/python3.N/dist-packages/mypy/build.py (Debian) - # or .../blah/lib/site-packages/mypy/build.py (Windows) - # blah may be a virtualenv or /usr/local. We want .../blah/lib/mypy. - lib = parent - for i in range(2): - lib = os.path.dirname(lib) - if os.path.basename(lib) == 'lib': - return os.path.join(lib, 'mypy') - subdir = os.path.join(parent, 'lib', 'mypy') - if os.path.isdir(subdir): - # If installed via buildout, the __file__ is - # somewhere/mypy/__init__.py and what we want is - # somewhere/lib/mypy. - return subdir - # Default to directory containing this file's parent. - return parent - base = os.path.basename(bin_dir) - dir = os.path.dirname(bin_dir) - if (sys.platform == 'win32' and base.lower() == 'scripts' - and not os.path.isdir(os.path.join(dir, 'typeshed'))): - # Installed, on Windows. - return os.path.join(dir, 'Lib', 'mypy') - elif base == 'scripts': - # Assume that we have a repo check out or unpacked source tarball. - return dir - elif base == 'bin': - # Installed to somewhere (can be under /usr/local or anywhere). - return os.path.join(dir, 'lib', 'mypy') - elif base == 'python3': - # Assume we installed python3 with brew on os x - return os.path.join(os.path.dirname(dir), 'lib', 'mypy') - elif dir.endswith('python-exec'): - # Gentoo uses a python wrapper in /usr/lib to which mypy is a symlink. - return os.path.join(os.path.dirname(dir), 'mypy') - else: - # Don't know where to find the data files! - raise RuntimeError("Broken installation: can't determine base dir") - - -def mypy_path() -> List[str]: - path_env = os.getenv('MYPYPATH') - if not path_env: - return [] - return path_env.split(os.pathsep) - - -def default_lib_path(data_dir: str, pyversion: Tuple[int, int]) -> List[str]: - """Return default standard library search paths.""" - # IDEA: Make this more portable. - path = [] # type: List[str] - - auto = os.path.join(data_dir, 'stubs-auto') - if os.path.isdir(auto): - data_dir = auto - - # We allow a module for e.g. version 3.5 to be in 3.4/. The assumption - # is that a module added with 3.4 will still be present in Python 3.5. - versions = ["%d.%d" % (pyversion[0], minor) - for minor in reversed(range(pyversion[1] + 1))] - # E.g. for Python 3.2, try 3.2/, 3.1/, 3.0/, 3/, 2and3/. - # (Note that 3.1 and 3.0 aren't really supported, but we don't care.) - for v in versions + [str(pyversion[0]), '2and3']: - for lib_type in ['stdlib', 'third_party']: - stubdir = os.path.join(data_dir, 'typeshed', lib_type, v) - if os.path.isdir(stubdir): - path.append(stubdir) - - # Add fallback path that can be used if we have a broken installation. - if sys.platform != 'win32': - path.append('/usr/local/lib/mypy') - - return path - - -CacheMeta = NamedTuple('CacheMeta', - [('id', str), - ('path', str), - ('mtime', float), - ('size', int), - ('dependencies', List[str]), # names of imported modules - ('data_mtime', float), # mtime of data_json - ('data_json', str), # path of .data.json - ('suppressed', List[str]), # dependencies that weren't imported - ('options', Optional[Dict[str, bool]]), # build options - ('dep_prios', List[int]), - ]) -# NOTE: dependencies + suppressed == all reachable imports; -# suppressed contains those reachable imports that were prevented by -# --silent-imports or simply not found. - - -# Priorities used for imports. (Here, top-level includes inside a class.) -# These are used to determine a more predictable order in which the -# nodes in an import cycle are processed. -PRI_HIGH = 5 # top-level "from X import blah" -PRI_MED = 10 # top-level "import X" -PRI_LOW = 20 # either form inside a function -PRI_ALL = 99 # include all priorities - - -class BuildManager: - """This class holds shared state for building a mypy program. - - It is used to coordinate parsing, import processing, semantic - analysis and type checking. The actual build steps are carried - out by dispatch(). - - Attributes: - data_dir: Mypy data directory (contains stubs) - lib_path: Library path for looking up modules - modules: Mapping of module ID to MypyFile (shared by the passes) - semantic_analyzer: - Semantic analyzer, pass 2 - semantic_analyzer_pass3: - Semantic analyzer, pass 3 - type_checker: Type checker - errors: Used for reporting all errors - options: Build options - missing_modules: Set of modules that could not be imported encountered so far - """ - - def __init__(self, data_dir: str, - lib_path: List[str], - ignore_prefix: str, - source_set: BuildSourceSet, - reports: Reports, - options: Options) -> None: - self.start_time = time.time() - self.data_dir = data_dir - self.errors = Errors() - self.errors.set_ignore_prefix(ignore_prefix) - self.lib_path = tuple(lib_path) - self.source_set = source_set - self.reports = reports - self.options = options - self.semantic_analyzer = SemanticAnalyzer(lib_path, self.errors, options=options) - self.modules = self.semantic_analyzer.modules - self.semantic_analyzer_pass3 = ThirdPass(self.modules, self.errors) - self.type_checker = TypeChecker(self.errors, self.modules, options=options) - self.missing_modules = set() # type: Set[str] - - def all_imported_modules_in_file(self, - file: MypyFile) -> List[Tuple[int, str, int]]: - """Find all reachable import statements in a file. - - Return list of tuples (priority, module id, import line number) - for all modules imported in file; lower numbers == higher priority. - """ - - def correct_rel_imp(imp: Union[ImportFrom, ImportAll]) -> str: - """Function to correct for relative imports.""" - file_id = file.fullname() - rel = imp.relative - if rel == 0: - return imp.id - if os.path.basename(file.path).startswith('__init__.'): - rel -= 1 - if rel != 0: - file_id = ".".join(file_id.split(".")[:-rel]) - new_id = file_id + "." + imp.id if imp.id else file_id - - return new_id - - res = [] # type: List[Tuple[int, str, int]] - for imp in file.imports: - if not imp.is_unreachable: - if isinstance(imp, Import): - pri = PRI_MED if imp.is_top_level else PRI_LOW - for id, _ in imp.ids: - res.append((pri, id, imp.line)) - elif isinstance(imp, ImportFrom): - cur_id = correct_rel_imp(imp) - pos = len(res) - all_are_submodules = True - # Also add any imported names that are submodules. - pri = PRI_MED if imp.is_top_level else PRI_LOW - for name, __ in imp.names: - sub_id = cur_id + '.' + name - if self.is_module(sub_id): - res.append((pri, sub_id, imp.line)) - else: - all_are_submodules = False - # If all imported names are submodules, don't add - # cur_id as a dependency. Otherwise (i.e., if at - # least one imported name isn't a submodule) - # cur_id is also a dependency, and we should - # insert it *before* any submodules. - if not all_are_submodules: - pri = PRI_HIGH if imp.is_top_level else PRI_LOW - res.insert(pos, ((pri, cur_id, imp.line))) - elif isinstance(imp, ImportAll): - pri = PRI_HIGH if imp.is_top_level else PRI_LOW - res.append((pri, correct_rel_imp(imp), imp.line)) - - return res - - def is_module(self, id: str) -> bool: - """Is there a file in the file system corresponding to module id?""" - return find_module(id, self.lib_path) is not None - - def parse_file(self, id: str, path: str, source: str) -> MypyFile: - """Parse the source of a file with the given name. - - Raise CompileError if there is a parse error. - """ - num_errs = self.errors.num_messages() - tree = parse(source, path, self.errors, options=self.options) - tree._fullname = id - - # We don't want to warn about 'type: ignore' comments on - # imports, but we're about to modify tree.imports, so grab - # these first. - import_lines = set(node.line for node in tree.imports) - - # Skip imports that have been ignored (so that we can ignore a C extension module without - # stub, for example), except for 'from x import *', because we wouldn't be able to - # determine which names should be defined unless we process the module. We can still - # ignore errors such as redefinitions when using the latter form. - imports = [node for node in tree.imports - if node.line not in tree.ignored_lines or isinstance(node, ImportAll)] - tree.imports = imports - - if self.errors.num_messages() != num_errs: - self.log("Bailing due to parse errors") - self.errors.raise_error() - - self.errors.set_file_ignored_lines(path, tree.ignored_lines) - self.errors.mark_file_ignored_lines_used(path, import_lines) - return tree - - def module_not_found(self, path: str, line: int, id: str) -> None: - self.errors.set_file(path) - stub_msg = "(Stub files are from https://github.com/python/typeshed)" - if ((self.options.python_version[0] == 2 and moduleinfo.is_py2_std_lib_module(id)) or - (self.options.python_version[0] >= 3 and moduleinfo.is_py3_std_lib_module(id))): - self.errors.report( - line, "No library stub file for standard library module '{}'".format(id)) - self.errors.report(line, stub_msg, severity='note', only_once=True) - elif moduleinfo.is_third_party_module(id): - self.errors.report(line, "No library stub file for module '{}'".format(id)) - self.errors.report(line, stub_msg, severity='note', only_once=True) - else: - self.errors.report(line, "Cannot find module named '{}'".format(id)) - self.errors.report(line, '(Perhaps setting MYPYPATH ' - 'or using the "--silent-imports" flag would help)', - severity='note', only_once=True) - - def report_file(self, file: MypyFile) -> None: - if self.source_set.is_source(file): - self.reports.file(file, type_map=self.type_checker.type_map) - - def log(self, *message: str) -> None: - if self.options.verbosity >= 1: - print('%.3f:LOG: ' % (time.time() - self.start_time), *message, file=sys.stderr) - sys.stderr.flush() - - def trace(self, *message: str) -> None: - if self.options.verbosity >= 2: - print('%.3f:TRACE:' % (time.time() - self.start_time), *message, file=sys.stderr) - sys.stderr.flush() - - -def remove_cwd_prefix_from_path(p: str) -> str: - """Remove current working directory prefix from p, if present. - - Also crawl up until a directory without __init__.py is found. - - If the result would be empty, return '.' instead. - """ - cur = os.getcwd() - # Add separator to the end of the path, unless one is already present. - if basename(cur) != '': - cur += os.sep - # Compute root path. - while (p and - (os.path.isfile(os.path.join(p, '__init__.py')) or - os.path.isfile(os.path.join(p, '__init__.pyi')))): - dir, base = os.path.split(p) - if not base: - break - p = dir - # Remove current directory prefix from the path, if present. - if p.startswith(cur): - p = p[len(cur):] - # Avoid returning an empty path; replace that with '.'. - if p == '': - p = '.' - return p - - -# Cache find_module: (id, lib_path) -> result. -find_module_cache = {} # type: Dict[Tuple[str, Tuple[str, ...]], str] - -# Cache some repeated work within distinct find_module calls: finding which -# elements of lib_path have even the subdirectory they'd need for the module -# to exist. This is shared among different module ids when they differ only -# in the last component. -find_module_dir_cache = {} # type: Dict[Tuple[str, Tuple[str, ...]], List[str]] - -# Cache directory listings. We assume that while one os.listdir() -# call may be more expensive than one os.stat() call, a small number -# of os.stat() calls is quickly more expensive than caching the -# os.listdir() outcome, and the advantage of the latter is that it -# gives us the case-correct filename on Windows and Mac. -find_module_listdir_cache = {} # type: Dict[str, Optional[List[str]]] - - -def find_module_clear_caches(): - find_module_cache.clear() - find_module_dir_cache.clear() - find_module_listdir_cache.clear() - - -def list_dir(path: str) -> Optional[List[str]]: - """Return a cached directory listing. - - Returns None if the path doesn't exist or isn't a directory. - """ - if path in find_module_listdir_cache: - return find_module_listdir_cache[path] - try: - res = os.listdir(path) - except OSError: - res = None - find_module_listdir_cache[path] = res - return res - - -def is_file(path: str) -> bool: - """Return whether path exists and is a file. - - On case-insensitive filesystems (like Mac or Windows) this returns - False if the case of the path's last component does not exactly - match the case found in the filesystem. - """ - head, tail = os.path.split(path) - if not tail: - return False - names = list_dir(head) - if not names: - return False - if tail not in names: - return False - return os.path.isfile(path) - - -def find_module(id: str, lib_path: Iterable[str]) -> str: - """Return the path of the module source file, or None if not found.""" - if not isinstance(lib_path, tuple): - lib_path = tuple(lib_path) - - def find(): - # If we're looking for a module like 'foo.bar.baz', it's likely that most of the - # many elements of lib_path don't even have a subdirectory 'foo/bar'. Discover - # that only once and cache it for when we look for modules like 'foo.bar.blah' - # that will require the same subdirectory. - components = id.split('.') - dir_chain = os.sep.join(components[:-1]) # e.g., 'foo/bar' - if (dir_chain, lib_path) not in find_module_dir_cache: - dirs = [] - for pathitem in lib_path: - # e.g., '/usr/lib/python3.4/foo/bar' - dir = os.path.normpath(os.path.join(pathitem, dir_chain)) - if os.path.isdir(dir): - dirs.append(dir) - find_module_dir_cache[dir_chain, lib_path] = dirs - candidate_base_dirs = find_module_dir_cache[dir_chain, lib_path] - - # If we're looking for a module like 'foo.bar.baz', then candidate_base_dirs now - # contains just the subdirectories 'foo/bar' that actually exist under the - # elements of lib_path. This is probably much shorter than lib_path itself. - # Now just look for 'baz.pyi', 'baz/__init__.py', etc., inside those directories. - seplast = os.sep + components[-1] # so e.g. '/baz' - sepinit = os.sep + '__init__' - for base_dir in candidate_base_dirs: - base_path = base_dir + seplast # so e.g. '/usr/lib/python3.4/foo/bar/baz' - # Prefer package over module, i.e. baz/__init__.py* over baz.py*. - for extension in PYTHON_EXTENSIONS: - path = base_path + sepinit + extension - if is_file(path) and verify_module(id, path): - return path - # No package, look for module. - for extension in PYTHON_EXTENSIONS: - path = base_path + extension - if is_file(path) and verify_module(id, path): - return path - return None - - key = (id, lib_path) - if key not in find_module_cache: - find_module_cache[key] = find() - return find_module_cache[key] - - -def find_modules_recursive(module: str, lib_path: List[str]) -> List[BuildSource]: - module_path = find_module(module, lib_path) - if not module_path: - return [] - result = [BuildSource(module_path, module, None)] - if module_path.endswith(('__init__.py', '__init__.pyi')): - # Subtle: this code prefers the .pyi over the .py if both - # exists, and also prefers packages over modules if both x/ - # and x.py* exist. How? We sort the directory items, so x - # comes before x.py and x.pyi. But the preference for .pyi - # over .py is encoded in find_module(); even though we see - # x.py before x.pyi, find_module() will find x.pyi first. We - # use hits to avoid adding it a second time when we see x.pyi. - # This also avoids both x.py and x.pyi when x/ was seen first. - hits = set() # type: Set[str] - for item in sorted(os.listdir(os.path.dirname(module_path))): - abs_path = os.path.join(os.path.dirname(module_path), item) - if os.path.isdir(abs_path) and \ - (os.path.isfile(os.path.join(abs_path, '__init__.py')) or - os.path.isfile(os.path.join(abs_path, '__init__.pyi'))): - hits.add(item) - result += find_modules_recursive(module + '.' + item, lib_path) - elif item != '__init__.py' and item != '__init__.pyi' and \ - item.endswith(('.py', '.pyi')): - mod = item.split('.')[0] - if mod not in hits: - hits.add(mod) - result += find_modules_recursive( - module + '.' + mod, lib_path) - return result - - -def verify_module(id: str, path: str) -> bool: - """Check that all packages containing id have a __init__ file.""" - if path.endswith(('__init__.py', '__init__.pyi')): - path = dirname(path) - for i in range(id.count('.')): - path = dirname(path) - if not any(os.path.isfile(os.path.join(path, '__init__{}'.format(extension))) - for extension in PYTHON_EXTENSIONS): - return False - return True - - -def read_with_python_encoding(path: str, pyversion: Tuple[int, int]) -> str: - """Read the Python file with while obeying PEP-263 encoding detection""" - source_bytearray = bytearray() - encoding = 'utf8' if pyversion[0] >= 3 else 'ascii' - - with open(path, 'rb') as f: - # read first two lines and check if PEP-263 coding is present - source_bytearray.extend(f.readline()) - source_bytearray.extend(f.readline()) - - # check for BOM UTF-8 encoding and strip it out if present - if source_bytearray.startswith(b'\xef\xbb\xbf'): - encoding = 'utf8' - source_bytearray = source_bytearray[3:] - else: - _encoding, _ = util.find_python_encoding(source_bytearray, pyversion) - # check that the coding isn't mypy. We skip it since - # registering may not have happened yet - if _encoding != 'mypy': - encoding = _encoding - - source_bytearray.extend(f.read()) - try: - source_bytearray.decode(encoding) - except LookupError as lookuperr: - raise DecodeError(str(lookuperr)) - return source_bytearray.decode(encoding) - - -MYPY_CACHE = '.mypy_cache' - - -def get_cache_names(id: str, path: str, pyversion: Tuple[int, int]) -> Tuple[str, str]: - """Return the file names for the cache files. - - Args: - id: module ID - path: module path (used to recognize packages) - pyversion: Python version (major, minor) - - Returns: - A tuple with the file names to be used for the meta JSON and the - data JSON, respectively. - """ - prefix = os.path.join(MYPY_CACHE, '%d.%d' % pyversion, *id.split('.')) - is_package = os.path.basename(path).startswith('__init__.py') - if is_package: - prefix = os.path.join(prefix, '__init__') - return (prefix + '.meta.json', prefix + '.data.json') - - -def find_cache_meta(id: str, path: str, manager: BuildManager) -> Optional[CacheMeta]: - """Find cache data for a module. - - Args: - id: module ID - path: module path - manager: the build manager (for pyversion, log/trace, and build options) - - Returns: - A CacheMeta instance if the cache data was found and appears - valid; otherwise None. - """ - # TODO: May need to take more build options into account - meta_json, data_json = get_cache_names(id, path, manager.options.python_version) - manager.trace('Looking for {} {}'.format(id, data_json)) - if not os.path.exists(meta_json): - return None - with open(meta_json, 'r') as f: - meta_str = f.read() - manager.trace('Meta {} {}'.format(id, meta_str.rstrip())) - meta = json.loads(meta_str) # TODO: Errors - if not isinstance(meta, dict): - return None - path = os.path.abspath(path) - m = CacheMeta( - meta.get('id'), - meta.get('path'), - meta.get('mtime'), - meta.get('size'), - meta.get('dependencies', []), - meta.get('data_mtime'), - data_json, - meta.get('suppressed', []), - meta.get('options'), - meta.get('dep_prios', []), - ) - if (m.id != id or m.path != path or - m.mtime is None or m.size is None or - m.dependencies is None or m.data_mtime is None): - return None - - # Ignore cache if generated by an older mypy version. - if m.options is None or len(m.dependencies) != len(m.dep_prios): - return None - - # Ignore cache if (relevant) options aren't the same. - cached_options = m.options - current_options = select_options_affecting_cache(manager.options) - if cached_options != current_options: - return None - - # TODO: Share stat() outcome with find_module() - st = os.stat(path) # TODO: Errors - if st.st_mtime != m.mtime or st.st_size != m.size: - manager.log('Metadata abandoned because of modified file {}'.format(path)) - return None - # It's a match on (id, path, mtime, size). - # Check data_json; assume if its mtime matches it's good. - # TODO: stat() errors - if os.path.getmtime(data_json) != m.data_mtime: - return None - manager.log('Found {} {}'.format(id, meta_json)) - return m - - -def select_options_affecting_cache(options: Options) -> Mapping[str, bool]: - return {opt: getattr(options, opt) for opt in OPTIONS_AFFECTING_CACHE} - - -OPTIONS_AFFECTING_CACHE = [ - "silent_imports", - "almost_silent", - "disallow_untyped_calls", - "disallow_untyped_defs", - "check_untyped_defs", -] - - -def random_string(): - return binascii.hexlify(os.urandom(8)).decode('ascii') - - -def write_cache(id: str, path: str, tree: MypyFile, - dependencies: List[str], suppressed: List[str], - dep_prios: List[int], - manager: BuildManager) -> None: - """Write cache files for a module. - - Args: - id: module ID - path: module path - tree: the fully checked module data - dependencies: module IDs on which this module depends - suppressed: module IDs which were suppressed as dependencies - dep_prios: priorities (parallel array to dependencies) - manager: the build manager (for pyversion, log/trace) - """ - path = os.path.abspath(path) - manager.trace('Dumping {} {}'.format(id, path)) - st = os.stat(path) # TODO: Errors - mtime = st.st_mtime - size = st.st_size - meta_json, data_json = get_cache_names(id, path, manager.options.python_version) - manager.log('Writing {} {} {}'.format(id, meta_json, data_json)) - data = tree.serialize() - parent = os.path.dirname(data_json) - if not os.path.isdir(parent): - os.makedirs(parent) - assert os.path.dirname(meta_json) == parent - nonce = '.' + random_string() - data_json_tmp = data_json + nonce - meta_json_tmp = meta_json + nonce - with open(data_json_tmp, 'w') as f: - json.dump(data, f, indent=2, sort_keys=True) - f.write('\n') - data_mtime = os.path.getmtime(data_json_tmp) - meta = {'id': id, - 'path': path, - 'mtime': mtime, - 'size': size, - 'data_mtime': data_mtime, - 'dependencies': dependencies, - 'suppressed': suppressed, - 'options': select_options_affecting_cache(manager.options), - 'dep_prios': dep_prios, - } - with open(meta_json_tmp, 'w') as f: - json.dump(meta, f, sort_keys=True) - f.write('\n') - os.replace(data_json_tmp, data_json) - os.replace(meta_json_tmp, meta_json) - - -"""Dependency manager. - -Design -====== - -Ideally -------- - -A. Collapse cycles (each SCC -- strongly connected component -- - becomes one "supernode"). - -B. Topologically sort nodes based on dependencies. - -C. Process from leaves towards roots. - -Wrinkles --------- - -a. Need to parse source modules to determine dependencies. - -b. Processing order for modules within an SCC. - -c. Must order mtimes of files to decide whether to re-process; depends - on clock never resetting. - -d. from P import M; checks filesystem whether module P.M exists in - filesystem. - -e. Race conditions, where somebody modifies a file while we're - processing. I propose not to modify the algorithm to handle this, - but to detect when this could lead to inconsistencies. (For - example, when we decide on the dependencies based on cache - metadata, and then we decide to re-parse a file because of a stale - dependency, if the re-parsing leads to a different list of - dependencies we should warn the user or start over.) - -Steps ------ - -1. For each explicitly given module find the source file location. - -2. For each such module load and check the cache metadata, and decide - whether it's valid. - -3. Now recursively (or iteratively) find dependencies and add those to - the graph: - - - for cached nodes use the list of dependencies from the cache - metadata (this will be valid even if we later end up re-parsing - the same source); - - - for uncached nodes parse the file and process all imports found, - taking care of (a) above. - -Step 3 should also address (d) above. - -Once step 3 terminates we have the entire dependency graph, and for -each module we've either loaded the cache metadata or parsed the -source code. (However, we may still need to parse those modules for -which we have cache metadata but that depend, directly or indirectly, -on at least one module for which the cache metadata is stale.) - -Now we can execute steps A-C from the first section. Finding SCCs for -step A shouldn't be hard; there's a recipe here: -http://code.activestate.com/recipes/578507/. There's also a plethora -of topsort recipes, e.g. http://code.activestate.com/recipes/577413/. - -For single nodes, processing is simple. If the node was cached, we -deserialize the cache data and fix up cross-references. Otherwise, we -do semantic analysis followed by type checking. We also handle (c) -above; if a module has valid cache data *but* any of its -dependencies was processed from source, then the module should be -processed from source. - -A relatively simple optimization (outside SCCs) we might do in the -future is as follows: if a node's cache data is valid, but one or more -of its dependencies are out of date so we have to re-parse the node -from source, once we have fully type-checked the node, we can decide -whether its symbol table actually changed compared to the cache data -(by reading the cache data and comparing it to the data we would be -writing). If there is no change we can declare the node up to date, -and any node that depends (and for which we have cached data, and -whose other dependencies are up to date) on it won't need to be -re-parsed from source. - -Import cycles -------------- - -Finally we have to decide how to handle (c), import cycles. Here -we'll need a modified version of the original state machine -(build.py), but we only need to do this per SCC, and we won't have to -deal with changes to the list of nodes while we're processing it. - -If all nodes in the SCC have valid cache metadata and all dependencies -outside the SCC are still valid, we can proceed as follows: - - 1. Load cache data for all nodes in the SCC. - - 2. Fix up cross-references for all nodes in the SCC. - -Otherwise, the simplest (but potentially slow) way to proceed is to -invalidate all cache data in the SCC and re-parse all nodes in the SCC -from source. We can do this as follows: - - 1. Parse source for all nodes in the SCC. - - 2. Semantic analysis for all nodes in the SCC. - - 3. Type check all nodes in the SCC. - -(If there are more passes the process is the same -- each pass should -be done for all nodes before starting the next pass for any nodes in -the SCC.) - -We could process the nodes in the SCC in any order. For sentimental -reasons, I've decided to process them in the reverse order in which we -encountered them when originally constructing the graph. That's how -the old build.py deals with cycles, and at least this reproduces the -previous implementation more accurately. - -Can we do better than re-parsing all nodes in the SCC when any of its -dependencies are out of date? It's doubtful. The optimization -mentioned at the end of the previous section would require re-parsing -and type-checking a node and then comparing its symbol table to the -cached data; but because the node is part of a cycle we can't -technically type-check it until the semantic analysis of all other -nodes in the cycle has completed. (This is an important issue because -Dropbox has a very large cycle in production code. But I'd like to -deal with it later.) - -Additional wrinkles -------------------- - -During implementation more wrinkles were found. - -- When a submodule of a package (e.g. x.y) is encountered, the parent - package (e.g. x) must also be loaded, but it is not strictly a - dependency. See State.add_ancestors() below. -""" - - -class ModuleNotFound(Exception): - """Control flow exception to signal that a module was not found.""" - - -class State: - """The state for a module. - - The source is only used for the -c command line option; in that - case path is None. Otherwise source is None and path isn't. - """ - - manager = None # type: BuildManager - order_counter = 0 # Class variable - order = None # type: int # Order in which modules were encountered - id = None # type: str # Fully qualified module name - path = None # type: Optional[str] # Path to module source - xpath = None # type: str # Path or '' - source = None # type: Optional[str] # Module source code - meta = None # type: Optional[CacheMeta] - data = None # type: Optional[str] - tree = None # type: Optional[MypyFile] - dependencies = None # type: List[str] - suppressed = None # type: List[str] # Suppressed/missing dependencies - priorities = None # type: Dict[str, int] - - # Map each dependency to the line number where it is first imported - dep_line_map = None # type: Dict[str, int] - - # Parent package, its parent, etc. - ancestors = None # type: Optional[List[str]] - - # List of (path, line number) tuples giving context for import - import_context = None # type: List[Tuple[str, int]] - - # The State from which this module was imported, if any - caller_state = None # type: Optional[State] - - # If caller_state is set, the line number in the caller where the import occurred - caller_line = 0 - - def __init__(self, - id: Optional[str], - path: Optional[str], - source: Optional[str], - manager: BuildManager, - caller_state: 'State' = None, - caller_line: int = 0, - ancestor_for: 'State' = None, - ) -> None: - assert id or path or source is not None, "Neither id, path nor source given" - self.manager = manager - State.order_counter += 1 - self.order = State.order_counter - self.caller_state = caller_state - self.caller_line = caller_line - if caller_state: - self.import_context = caller_state.import_context[:] - self.import_context.append((caller_state.xpath, caller_line)) - else: - self.import_context = [] - self.id = id or '__main__' - if not path and source is None: - file_id = id - if id == 'builtins' and manager.options.python_version[0] == 2: - # The __builtin__ module is called internally by mypy - # 'builtins' in Python 2 mode (similar to Python 3), - # but the stub file is __builtin__.pyi. The reason is - # that a lot of code hard-codes 'builtins.x' and it's - # easier to work it around like this. It also means - # that the implementation can mostly ignore the - # difference and just assume 'builtins' everywhere, - # which simplifies code. - file_id = '__builtin__' - path = find_module(file_id, manager.lib_path) - if path: - # In silent mode, don't import .py files, except from stubs. - if (manager.options.silent_imports and - path.endswith('.py') and (caller_state or ancestor_for)): - # (Never silence builtins, even if it's a .py file; - # this can happen in tests!) - if (id != 'builtins' and - not ((caller_state and - caller_state.tree and - caller_state.tree.is_stub))): - if manager.options.almost_silent: - if ancestor_for: - self.skipping_ancestor(id, path, ancestor_for) - else: - self.skipping_module(id, path) - path = None - manager.missing_modules.add(id) - raise ModuleNotFound - else: - # Could not find a module. Typically the reason is a - # misspelled module name, missing stub, module not in - # search path or the module has not been installed. - if caller_state: - suppress_message = ((manager.options.silent_imports and - not manager.options.almost_silent) or - (caller_state.tree is not None and - 'import' in caller_state.tree.weak_opts)) - if not suppress_message: - save_import_context = manager.errors.import_context() - manager.errors.set_import_context(caller_state.import_context) - manager.module_not_found(caller_state.xpath, caller_line, id) - manager.errors.set_import_context(save_import_context) - manager.missing_modules.add(id) - raise ModuleNotFound - else: - # If we can't find a root source it's always fatal. - # TODO: This might hide non-fatal errors from - # root sources processed earlier. - raise CompileError(["mypy: can't find module '%s'" % id]) - self.path = path - self.xpath = path or '' - self.source = source - if path and source is None and manager.options.incremental: - self.meta = find_cache_meta(self.id, self.path, manager) - # TODO: Get mtime if not cached. - self.add_ancestors() - if self.meta: - # Make copies, since we may modify these and want to - # compare them to the originals later. - self.dependencies = list(self.meta.dependencies) - self.suppressed = list(self.meta.suppressed) - assert len(self.meta.dependencies) == len(self.meta.dep_prios) - self.priorities = {id: pri - for id, pri in zip(self.meta.dependencies, self.meta.dep_prios)} - self.dep_line_map = {} - else: - # Parse the file (and then some) to get the dependencies. - self.parse_file() - self.suppressed = [] - - def skipping_ancestor(self, id: str, path: str, ancestor_for: 'State') -> None: - # TODO: Read the path (the __init__.py file) and return - # immediately if it's empty or only contains comments. - # But beware, some package may be the ancestor of many modules, - # so we'd need to cache the decision. - manager = self.manager - manager.errors.set_import_context([]) - manager.errors.set_file(ancestor_for.xpath) - manager.errors.report(-1, "Ancestor package '%s' silently ignored" % (id,), - severity='note', only_once=True) - manager.errors.report(-1, "(Using --silent-imports, submodule passed on command line)", - severity='note', only_once=True) - manager.errors.report(-1, "(This note brought to you by --almost-silent)", - severity='note', only_once=True) - - def skipping_module(self, id: str, path: str) -> None: - assert self.caller_state, (id, path) - manager = self.manager - save_import_context = manager.errors.import_context() - manager.errors.set_import_context(self.caller_state.import_context) - manager.errors.set_file(self.caller_state.xpath) - line = self.caller_line - manager.errors.report(line, "Import of '%s' silently ignored" % (id,), - severity='note') - manager.errors.report(line, "(Using --silent-imports, module not passed on command line)", - severity='note', only_once=True) - manager.errors.report(line, "(This note courtesy of --almost-silent)", - severity='note', only_once=True) - manager.errors.set_import_context(save_import_context) - - def add_ancestors(self) -> None: - # All parent packages are new ancestors. - ancestors = [] - parent = self.id - while '.' in parent: - parent, _ = parent.rsplit('.', 1) - ancestors.append(parent) - self.ancestors = ancestors - - def is_fresh(self) -> bool: - """Return whether the cache data for this file is fresh.""" - # NOTE: self.dependencies may differ from - # self.meta.dependencies when a dependency is dropped due to - # suppression by --silent-imports. However when a suppressed - # dependency is added back we find out later in the process. - return self.meta is not None and self.dependencies == self.meta.dependencies - - def mark_stale(self) -> None: - """Throw away the cache data for this file, marking it as stale.""" - self.meta = None - - def check_blockers(self) -> None: - """Raise CompileError if a blocking error is detected.""" - if self.manager.errors.is_blockers(): - self.manager.log("Bailing due to blocking errors") - self.manager.errors.raise_error() - - @contextlib.contextmanager - def wrap_context(self) -> Iterator[None]: - save_import_context = self.manager.errors.import_context() - self.manager.errors.set_import_context(self.import_context) - try: - yield - except CompileError: - raise - except Exception as err: - report_internal_error(err, self.path, 0) - self.manager.errors.set_import_context(save_import_context) - self.check_blockers() - - # Methods for processing cached modules. - - def load_tree(self) -> None: - with open(self.meta.data_json) as f: - data = json.load(f) - # TODO: Assert data file wasn't changed. - self.tree = MypyFile.deserialize(data) - self.manager.modules[self.id] = self.tree - - def fix_cross_refs(self) -> None: - fixup_module_pass_one(self.tree, self.manager.modules) - - def calculate_mros(self) -> None: - fixup_module_pass_two(self.tree, self.manager.modules) - - # Methods for processing modules from source code. - - def parse_file(self) -> None: - if self.tree is not None: - # The file was already parsed (in __init__()). - return - - manager = self.manager - modules = manager.modules - manager.log("Parsing %s (%s)" % (self.xpath, self.id)) - - with self.wrap_context(): - source = self.source - self.source = None # We won't need it again. - if self.path and source is None: - try: - source = read_with_python_encoding(self.path, manager.options.python_version) - except IOError as ioerr: - raise CompileError([ - "mypy: can't read file '{}': {}".format(self.path, ioerr.strerror)]) - except (UnicodeDecodeError, DecodeError) as decodeerr: - raise CompileError([ - "mypy: can't decode file '{}': {}".format(self.path, str(decodeerr))]) - self.tree = manager.parse_file(self.id, self.xpath, source) - - modules[self.id] = self.tree - - # Do the first pass of semantic analysis: add top-level - # definitions in the file to the symbol table. We must do - # this before processing imports, since this may mark some - # import statements as unreachable. - first = FirstPass(manager.semantic_analyzer) - first.analyze(self.tree, self.xpath, self.id) - - # Initialize module symbol table, which was populated by the - # semantic analyzer. - # TODO: Why can't FirstPass .analyze() do this? - self.tree.names = manager.semantic_analyzer.globals - - # Compute (direct) dependencies. - # Add all direct imports (this is why we needed the first pass). - # Also keep track of each dependency's source line. - dependencies = [] - suppressed = [] - priorities = {} # type: Dict[str, int] # id -> priority - dep_line_map = {} # type: Dict[str, int] # id -> line - for pri, id, line in manager.all_imported_modules_in_file(self.tree): - priorities[id] = min(pri, priorities.get(id, PRI_ALL)) - if id == self.id: - continue - # Omit missing modules, as otherwise we could not type-check - # programs with missing modules. - if id in manager.missing_modules: - if id not in dep_line_map: - suppressed.append(id) - dep_line_map[id] = line - continue - if id == '': - # Must be from a relative import. - manager.errors.set_file(self.xpath) - manager.errors.report(line, "No parent module -- cannot perform relative import", - blocker=True) - continue - if id not in dep_line_map: - dependencies.append(id) - dep_line_map[id] = line - # Every module implicitly depends on builtins. - if self.id != 'builtins' and 'builtins' not in dep_line_map: - dependencies.append('builtins') - - # If self.dependencies is already set, it was read from the - # cache, but for some reason we're re-parsing the file. - # NOTE: What to do about race conditions (like editing the - # file while mypy runs)? A previous version of this code - # explicitly checked for this, but ran afoul of other reasons - # for differences (e.g. --silent-imports). - self.dependencies = dependencies - self.suppressed = suppressed - self.priorities = priorities - self.dep_line_map = dep_line_map - self.check_blockers() - - def patch_parent(self) -> None: - # Include module in the symbol table of the enclosing package. - if '.' not in self.id: - return - manager = self.manager - modules = manager.modules - parent, child = self.id.rsplit('.', 1) - if parent in modules: - manager.trace("Added %s.%s" % (parent, child)) - modules[parent].names[child] = SymbolTableNode(MODULE_REF, self.tree, parent) - else: - manager.log("Hm... couldn't add %s.%s" % (parent, child)) - - def semantic_analysis(self) -> None: - with self.wrap_context(): - self.manager.semantic_analyzer.visit_file(self.tree, self.xpath) - - def semantic_analysis_pass_three(self) -> None: - with self.wrap_context(): - self.manager.semantic_analyzer_pass3.visit_file(self.tree, self.xpath) - if self.manager.options.dump_type_stats: - dump_type_stats(self.tree, self.xpath) - - def type_check(self) -> None: - manager = self.manager - if manager.options.semantic_analysis_only: - return - with self.wrap_context(): - manager.type_checker.visit_file(self.tree, self.xpath) - if manager.options.dump_inference_stats: - dump_type_stats(self.tree, self.xpath, inferred=True, - typemap=manager.type_checker.type_map) - manager.report_file(self.tree) - - def write_cache(self) -> None: - if self.path and self.manager.options.incremental and not self.manager.errors.is_errors(): - dep_prios = [self.priorities.get(dep, PRI_HIGH) for dep in self.dependencies] - write_cache(self.id, self.path, self.tree, - list(self.dependencies), list(self.suppressed), - dep_prios, - self.manager) - - -Graph = Dict[str, State] - - -def dispatch(sources: List[BuildSource], manager: BuildManager) -> None: - manager.log("Using new dependency manager") - graph = load_graph(sources, manager) - manager.log("Loaded graph with %d nodes" % len(graph)) - process_graph(graph, manager) - if manager.options.warn_unused_ignores: - manager.errors.generate_unused_ignore_notes() - - -def load_graph(sources: List[BuildSource], manager: BuildManager) -> Graph: - """Given some source files, load the full dependency graph.""" - graph = {} # type: Graph - # The deque is used to implement breadth-first traversal. - # TODO: Consider whether to go depth-first instead. This may - # affect the order in which we process files within import cycles. - new = collections.deque() # type: collections.deque[State] - # Seed the graph with the initial root sources. - for bs in sources: - try: - st = State(id=bs.module, path=bs.path, source=bs.text, manager=manager) - except ModuleNotFound: - continue - if st.id in graph: - manager.errors.set_file(st.xpath) - manager.errors.report(1, "Duplicate module named '%s'" % st.id) - manager.errors.raise_error() - graph[st.id] = st - new.append(st) - # Collect dependencies. We go breadth-first. - while new: - st = new.popleft() - for dep in st.ancestors + st.dependencies: - if dep not in graph: - try: - if dep in st.ancestors: - # TODO: Why not 'if dep not in st.dependencies' ? - # Ancestors don't have import context. - newst = State(id=dep, path=None, source=None, manager=manager, - ancestor_for=st) - else: - newst = State(id=dep, path=None, source=None, manager=manager, - caller_state=st, caller_line=st.dep_line_map.get(dep, 1)) - except ModuleNotFound: - if dep in st.dependencies: - st.dependencies.remove(dep) - st.suppressed.append(dep) - else: - assert newst.id not in graph, newst.id - graph[newst.id] = newst - new.append(newst) - return graph - - -def process_graph(graph: Graph, manager: BuildManager) -> None: - """Process everything in dependency order.""" - sccs = sorted_components(graph) - manager.log("Found %d SCCs; largest has %d nodes" % - (len(sccs), max(len(scc) for scc in sccs))) - # We're processing SCCs from leaves (those without further - # dependencies) to roots (those from which everything else can be - # reached). - for ascc in sccs: - # Order the SCC's nodes using a heuristic. - # Note that ascc is a set, and scc is a list. - scc = order_ascc(graph, ascc) - # If builtins is in the list, move it last. (This is a bit of - # a hack, but it's necessary because the builtins module is - # part of a small cycle involving at least {builtins, abc, - # typing}. Of these, builtins must be processed last or else - # some builtin objects will be incompletely processed.) - if 'builtins' in ascc: - scc.remove('builtins') - scc.append('builtins') - if manager.options.verbosity >= 2: - for id in scc: - manager.trace("Priorities for %s:" % id, - " ".join("%s:%d" % (x, graph[id].priorities[x]) - for x in graph[id].dependencies - if x in ascc and x in graph[id].priorities)) - # Because the SCCs are presented in topological sort order, we - # don't need to look at dependencies recursively for staleness - # -- the immediate dependencies are sufficient. - stale_scc = {id for id in scc if not graph[id].is_fresh()} - fresh = not stale_scc - deps = set() - for id in scc: - deps.update(graph[id].dependencies) - deps -= ascc - stale_deps = {id for id in deps if not graph[id].is_fresh()} - fresh = fresh and not stale_deps - undeps = set() - if fresh: - # Check if any dependencies that were suppressed according - # to the cache have heen added back in this run. - # NOTE: Newly suppressed dependencies are handled by is_fresh(). - for id in scc: - undeps.update(graph[id].suppressed) - undeps &= graph.keys() - if undeps: - fresh = False - if fresh: - # All cache files are fresh. Check that no dependency's - # cache file is newer than any scc node's cache file. - oldest_in_scc = min(graph[id].meta.data_mtime for id in scc) - newest_in_deps = 0 if not deps else max(graph[dep].meta.data_mtime for dep in deps) - if manager.options.verbosity >= 3: # Dump all mtimes for extreme debugging. - all_ids = sorted(ascc | deps, key=lambda id: graph[id].meta.data_mtime) - for id in all_ids: - if id in scc: - if graph[id].meta.data_mtime < newest_in_deps: - key = "*id:" - else: - key = "id:" - else: - if graph[id].meta.data_mtime > oldest_in_scc: - key = "+dep:" - else: - key = "dep:" - manager.trace(" %5s %.0f %s" % (key, graph[id].meta.data_mtime, id)) - # If equal, give the benefit of the doubt, due to 1-sec time granularity - # (on some platforms). - if oldest_in_scc < newest_in_deps: - fresh = False - fresh_msg = "out of date by %.0f seconds" % (newest_in_deps - oldest_in_scc) - else: - fresh_msg = "fresh" - elif undeps: - fresh_msg = "stale due to changed suppression (%s)" % " ".join(sorted(undeps)) - elif stale_scc: - fresh_msg = "inherently stale (%s)" % " ".join(sorted(stale_scc)) - if stale_deps: - fresh_msg += " with stale deps (%s)" % " ".join(sorted(stale_deps)) - else: - fresh_msg = "stale due to deps (%s)" % " ".join(sorted(stale_deps)) - if len(scc) == 1: - manager.log("Processing SCC singleton (%s) as %s" % (" ".join(scc), fresh_msg)) - else: - manager.log("Processing SCC of size %d (%s) as %s" % - (len(scc), " ".join(scc), fresh_msg)) - if fresh: - process_fresh_scc(graph, scc) - else: - process_stale_scc(graph, scc) - - -def order_ascc(graph: Graph, ascc: AbstractSet[str], pri_max: int = PRI_ALL) -> List[str]: - """Come up with the ideal processing order within an SCC. - - Using the priorities assigned by all_imported_modules_in_file(), - try to reduce the cycle to a DAG, by omitting arcs representing - dependencies of lower priority. - - In the simplest case, if we have A <--> B where A has a top-level - "import B" (medium priority) but B only has the reverse "import A" - inside a function (low priority), we turn the cycle into a DAG by - dropping the B --> A arc, which leaves only A --> B. - - If all arcs have the same priority, we fall back to sorting by - reverse global order (the order in which modules were first - encountered). - - The algorithm is recursive, as follows: when as arcs of different - priorities are present, drop all arcs of the lowest priority, - identify SCCs in the resulting graph, and apply the algorithm to - each SCC thus found. The recursion is bounded because at each - recursion the spread in priorities is (at least) one less. - - In practice there are only a few priority levels (currently - N=3) and in the worst case we just carry out the same algorithm - for finding SCCs N times. Thus the complexity is no worse than - the complexity of the original SCC-finding algorithm -- see - strongly_connected_components() below for a reference. - """ - if len(ascc) == 1: - return [s for s in ascc] - pri_spread = set() - for id in ascc: - state = graph[id] - for dep in state.dependencies: - if dep in ascc: - pri = state.priorities.get(dep, PRI_HIGH) - if pri < pri_max: - pri_spread.add(pri) - if len(pri_spread) == 1: - # Filtered dependencies are uniform -- order by global order. - return sorted(ascc, key=lambda id: -graph[id].order) - pri_max = max(pri_spread) - sccs = sorted_components(graph, ascc, pri_max) - # The recursion is bounded by the len(pri_spread) check above. - return [s for ss in sccs for s in order_ascc(graph, ss, pri_max)] - - -def process_fresh_scc(graph: Graph, scc: List[str]) -> None: - """Process the modules in one SCC from their cached data.""" - for id in scc: - graph[id].load_tree() - for id in scc: - graph[id].patch_parent() - for id in scc: - graph[id].fix_cross_refs() - for id in scc: - graph[id].calculate_mros() - - -def process_stale_scc(graph: Graph, scc: List[str]) -> None: - """Process the modules in one SCC from source code.""" - for id in scc: - graph[id].mark_stale() - for id in scc: - # We may already have parsed the module, or not. - # If the former, parse_file() is a no-op. - graph[id].parse_file() - for id in scc: - graph[id].patch_parent() - for id in scc: - graph[id].semantic_analysis() - for id in scc: - graph[id].semantic_analysis_pass_three() - for id in scc: - graph[id].type_check() - graph[id].write_cache() - - -def sorted_components(graph: Graph, - vertices: Optional[AbstractSet[str]] = None, - pri_max: int = PRI_ALL) -> List[AbstractSet[str]]: - """Return the graph's SCCs, topologically sorted by dependencies. - - The sort order is from leaves (nodes without dependencies) to - roots (nodes on which no other nodes depend). - - This works for a subset of the full dependency graph too; - dependencies that aren't present in graph.keys() are ignored. - """ - # Compute SCCs. - if vertices is None: - vertices = set(graph) - edges = {id: deps_filtered(graph, vertices, id, pri_max) for id in vertices} - sccs = list(strongly_connected_components(vertices, edges)) - # Topsort. - sccsmap = {id: frozenset(scc) for scc in sccs for id in scc} - data = {} # type: Dict[AbstractSet[str], Set[AbstractSet[str]]] - for scc in sccs: - deps = set() # type: Set[AbstractSet[str]] - for id in scc: - deps.update(sccsmap[x] for x in deps_filtered(graph, vertices, id, pri_max)) - data[frozenset(scc)] = deps - res = [] - for ready in topsort(data): - # Sort the sets in ready by reversed smallest State.order. Examples: - # - # - If ready is [{x}, {y}], x.order == 1, y.order == 2, we get - # [{y}, {x}]. - # - # - If ready is [{a, b}, {c, d}], a.order == 1, b.order == 3, - # c.order == 2, d.order == 4, the sort keys become [1, 2] - # and the result is [{c, d}, {a, b}]. - res.extend(sorted(ready, - key=lambda scc: -min(graph[id].order for id in scc))) - return res - - -def deps_filtered(graph: Graph, vertices: AbstractSet[str], id: str, pri_max: int) -> List[str]: - """Filter dependencies for id with pri < pri_max.""" - if id not in vertices: - return [] - state = graph[id] - return [dep - for dep in state.dependencies - if dep in vertices and state.priorities.get(dep, PRI_HIGH) < pri_max] - - -def strongly_connected_components(vertices: AbstractSet[str], - edges: Dict[str, List[str]]) -> Iterator[Set[str]]: - """Compute Strongly Connected Components of a directed graph. - - Args: - vertices: the labels for the vertices - edges: for each vertex, gives the target vertices of its outgoing edges - - Returns: - An iterator yielding strongly connected components, each - represented as a set of vertices. Each input vertex will occur - exactly once; vertices not part of a SCC are returned as - singleton sets. - - From http://code.activestate.com/recipes/578507/. - """ - identified = set() # type: Set[str] - stack = [] # type: List[str] - index = {} # type: Dict[str, int] - boundaries = [] # type: List[int] - - def dfs(v: str) -> Iterator[Set[str]]: - index[v] = len(stack) - stack.append(v) - boundaries.append(index[v]) - - for w in edges[v]: - if w not in index: - # For Python >= 3.3, replace with "yield from dfs(w)" - for scc in dfs(w): - yield scc - elif w not in identified: - while index[w] < boundaries[-1]: - boundaries.pop() - - if boundaries[-1] == index[v]: - boundaries.pop() - scc = set(stack[index[v]:]) - del stack[index[v]:] - identified.update(scc) - yield scc - - for v in vertices: - if v not in index: - # For Python >= 3.3, replace with "yield from dfs(v)" - for scc in dfs(v): - yield scc - - -def topsort(data: Dict[AbstractSet[str], - Set[AbstractSet[str]]]) -> Iterable[Set[AbstractSet[str]]]: - """Topological sort. - - Args: - data: A map from SCCs (represented as frozen sets of strings) to - sets of SCCs, its dependencies. NOTE: This data structure - is modified in place -- for normalization purposes, - self-dependencies are removed and entries representing - orphans are added. - - Returns: - An iterator yielding sets of SCCs that have an equivalent - ordering. NOTE: The algorithm doesn't care about the internal - structure of SCCs. - - Example: - Suppose the input has the following structure: - - {A: {B, C}, B: {D}, C: {D}} - - This is normalized to: - - {A: {B, C}, B: {D}, C: {D}, D: {}} - - The algorithm will yield the following values: - - {D} - {B, C} - {A} - - From http://code.activestate.com/recipes/577413/. - """ - # TODO: Use a faster algorithm? - for k, v in data.items(): - v.discard(k) # Ignore self dependencies. - for item in set.union(*data.values()) - set(data.keys()): - data[item] = set() - while True: - ready = {item for item, dep in data.items() if not dep} - if not ready: - break - yield ready - data = {item: (dep - ready) - for item, dep in data.items() - if item not in ready} - assert not data, "A cyclic dependency exists amongst %r" % data +"""Facilities to analyze entire programs, including imported modules. + +Parse and analyze the source files of a program in the correct order +(based on file dependencies), and collect the results. + +This module only directs a build, which is performed in multiple passes per +file. The individual passes are implemented in separate modules. + +The function build() is the main interface to this module. +""" +# TODO: More consistent terminology, e.g. path/fnam, module/id, state/file + +import binascii +import collections +import contextlib +import json +import os +import os.path +import sys +import time +from os.path import dirname, basename + +from typing import (AbstractSet, Dict, Iterable, Iterator, List, + NamedTuple, Optional, Set, Tuple, Union, Mapping) + +from mypy.types import Type +from mypy.nodes import (MypyFile, Node, Import, ImportFrom, ImportAll, + SymbolTableNode, MODULE_REF) +from mypy.semanal import FirstPass, SemanticAnalyzer, ThirdPass +from mypy.checker import TypeChecker +from mypy.errors import Errors, CompileError, DecodeError, report_internal_error +from mypy import fixup +from mypy.report import Reports +from mypy import defaults +from mypy import moduleinfo +from mypy import util +from mypy.fixup import fixup_module_pass_one, fixup_module_pass_two +from mypy.options import Options +from mypy.parse import parse +from mypy.stats import dump_type_stats + + +# We need to know the location of this file to load data, but +# until Python 3.4, __file__ is relative. +__file__ = os.path.realpath(__file__) + +PYTHON_EXTENSIONS = ['.pyi', '.py'] + + +class BuildResult: + """The result of a successful build. + + Attributes: + manager: The build manager. + files: Dictionary from module name to related AST node. + types: Dictionary from parse tree node to its inferred type. + errors: List of error messages. + """ + + def __init__(self, manager: 'BuildManager') -> None: + self.manager = manager + self.files = manager.modules + self.types = manager.type_checker.type_map + self.errors = manager.errors.messages() + + +class BuildSource: + def __init__(self, path: Optional[str], module: Optional[str], + text: Optional[str]) -> None: + self.path = path + self.module = module or '__main__' + self.text = text + + @property + def effective_path(self) -> str: + """Return the effective path (ie, if its from in memory)""" + return self.path or '' + + +class BuildSourceSet: + """Efficiently test a file's membership in the set of build sources.""" + + def __init__(self, sources: List[BuildSource]) -> None: + self.source_text_present = False + self.source_modules = set() # type: Set[str] + self.source_paths = set() # type: Set[str] + + for source in sources: + if source.text is not None: + self.source_text_present = True + elif source.path: + self.source_paths.add(source.path) + else: + self.source_modules.add(source.module) + + def is_source(self, file: MypyFile) -> bool: + if file.path and file.path in self.source_paths: + return True + elif file._fullname in self.source_modules: + return True + elif file.path is None and self.source_text_present: + return True + else: + return False + + +def build(sources: List[BuildSource], + options: Options, + alt_lib_path: str = None, + bin_dir: str = None) -> BuildResult: + """Analyze a program. + + A single call to build performs parsing, semantic analysis and optionally + type checking for the program *and* all imported modules, recursively. + + Return BuildResult if successful or only non-blocking errors were found; + otherwise raise CompileError. + + Args: + sources: list of sources to build + options: build options + alt_lib_path: an additional directory for looking up library modules + (takes precedence over other directories) + bin_dir: directory containing the mypy script, used for finding data + directories; if omitted, use '.' as the data directory + """ + + data_dir = default_data_dir(bin_dir) + + find_module_clear_caches() + + # Determine the default module search path. + lib_path = default_lib_path(data_dir, options.python_version) + + if options.use_builtins_fixtures: + # Use stub builtins (to speed up test cases and to make them easier to + # debug). This is a test-only feature, so assume our files are laid out + # as in the source tree. + root_dir = os.path.dirname(os.path.dirname(__file__)) + lib_path.insert(0, os.path.join(root_dir, 'test-data', 'unit', 'lib-stub')) + else: + for source in sources: + if source.path: + # Include directory of the program file in the module search path. + dir = remove_cwd_prefix_from_path(dirname(source.path)) + if dir not in lib_path: + lib_path.insert(0, dir) + + # Do this even if running as a file, for sanity (mainly because with + # multiple builds, there could be a mix of files/modules, so its easier + # to just define the semantics that we always add the current director + # to the lib_path + lib_path.insert(0, os.getcwd()) + + # Add MYPYPATH environment variable to front of library path, if defined. + lib_path[:0] = mypy_path() + + # If provided, insert the caller-supplied extra module path to the + # beginning (highest priority) of the search path. + if alt_lib_path: + lib_path.insert(0, alt_lib_path) + + reports = Reports(data_dir, options.report_dirs) + + source_set = BuildSourceSet(sources) + + # Construct a build manager object to hold state during the build. + # + # Ignore current directory prefix in error messages. + manager = BuildManager(data_dir, lib_path, + ignore_prefix=os.getcwd(), + source_set=source_set, + reports=reports, + options=options) + + try: + dispatch(sources, manager) + return BuildResult(manager) + finally: + manager.log("Build finished with %d modules, %d types, and %d errors" % + (len(manager.modules), + len(manager.type_checker.type_map), + manager.errors.num_messages())) + # Finish the HTML or XML reports even if CompileError was raised. + reports.finish() + + +def default_data_dir(bin_dir: Optional[str]) -> str: + """Returns directory containing typeshed directory + + Args: + bin_dir: directory containing the mypy script + """ + if not bin_dir: + mypy_package = os.path.dirname(__file__) + parent = os.path.dirname(mypy_package) + if (os.path.basename(parent) == 'site-packages' or + os.path.basename(parent) == 'dist-packages'): + # Installed in site-packages or dist-packages, but invoked with python3 -m mypy; + # __file__ is .../blah/lib/python3.N/site-packages/mypy/build.py + # or .../blah/lib/python3.N/dist-packages/mypy/build.py (Debian) + # or .../blah/lib/site-packages/mypy/build.py (Windows) + # blah may be a virtualenv or /usr/local. We want .../blah/lib/mypy. + lib = parent + for i in range(2): + lib = os.path.dirname(lib) + if os.path.basename(lib) == 'lib': + return os.path.join(lib, 'mypy') + subdir = os.path.join(parent, 'lib', 'mypy') + if os.path.isdir(subdir): + # If installed via buildout, the __file__ is + # somewhere/mypy/__init__.py and what we want is + # somewhere/lib/mypy. + return subdir + # Default to directory containing this file's parent. + return parent + base = os.path.basename(bin_dir) + dir = os.path.dirname(bin_dir) + if (sys.platform == 'win32' and base.lower() == 'scripts' + and not os.path.isdir(os.path.join(dir, 'typeshed'))): + # Installed, on Windows. + return os.path.join(dir, 'Lib', 'mypy') + elif base == 'scripts': + # Assume that we have a repo check out or unpacked source tarball. + return dir + elif base == 'bin': + # Installed to somewhere (can be under /usr/local or anywhere). + return os.path.join(dir, 'lib', 'mypy') + elif base == 'python3': + # Assume we installed python3 with brew on os x + return os.path.join(os.path.dirname(dir), 'lib', 'mypy') + elif dir.endswith('python-exec'): + # Gentoo uses a python wrapper in /usr/lib to which mypy is a symlink. + return os.path.join(os.path.dirname(dir), 'mypy') + else: + # Don't know where to find the data files! + raise RuntimeError("Broken installation: can't determine base dir") + + +def mypy_path() -> List[str]: + path_env = os.getenv('MYPYPATH') + if not path_env: + return [] + return path_env.split(os.pathsep) + + +def default_lib_path(data_dir: str, pyversion: Tuple[int, int]) -> List[str]: + """Return default standard library search paths.""" + # IDEA: Make this more portable. + path = [] # type: List[str] + + auto = os.path.join(data_dir, 'stubs-auto') + if os.path.isdir(auto): + data_dir = auto + + # We allow a module for e.g. version 3.5 to be in 3.4/. The assumption + # is that a module added with 3.4 will still be present in Python 3.5. + versions = ["%d.%d" % (pyversion[0], minor) + for minor in reversed(range(pyversion[1] + 1))] + # E.g. for Python 3.2, try 3.2/, 3.1/, 3.0/, 3/, 2and3/. + # (Note that 3.1 and 3.0 aren't really supported, but we don't care.) + for v in versions + [str(pyversion[0]), '2and3']: + for lib_type in ['stdlib', 'third_party']: + stubdir = os.path.join(data_dir, 'typeshed', lib_type, v) + if os.path.isdir(stubdir): + path.append(stubdir) + + # Add fallback path that can be used if we have a broken installation. + if sys.platform != 'win32': + path.append('/usr/local/lib/mypy') + + return path + + +CacheMeta = NamedTuple('CacheMeta', + [('id', str), + ('path', str), + ('mtime', float), + ('size', int), + ('dependencies', List[str]), # names of imported modules + ('data_mtime', float), # mtime of data_json + ('data_json', str), # path of .data.json + ('suppressed', List[str]), # dependencies that weren't imported + ('options', Optional[Dict[str, bool]]), # build options + ('dep_prios', List[int]), + ]) +# NOTE: dependencies + suppressed == all reachable imports; +# suppressed contains those reachable imports that were prevented by +# --silent-imports or simply not found. + + +# Priorities used for imports. (Here, top-level includes inside a class.) +# These are used to determine a more predictable order in which the +# nodes in an import cycle are processed. +PRI_HIGH = 5 # top-level "from X import blah" +PRI_MED = 10 # top-level "import X" +PRI_LOW = 20 # either form inside a function +PRI_ALL = 99 # include all priorities + + +class BuildManager: + """This class holds shared state for building a mypy program. + + It is used to coordinate parsing, import processing, semantic + analysis and type checking. The actual build steps are carried + out by dispatch(). + + Attributes: + data_dir: Mypy data directory (contains stubs) + lib_path: Library path for looking up modules + modules: Mapping of module ID to MypyFile (shared by the passes) + semantic_analyzer: + Semantic analyzer, pass 2 + semantic_analyzer_pass3: + Semantic analyzer, pass 3 + type_checker: Type checker + errors: Used for reporting all errors + options: Build options + missing_modules: Set of modules that could not be imported encountered so far + """ + + def __init__(self, data_dir: str, + lib_path: List[str], + ignore_prefix: str, + source_set: BuildSourceSet, + reports: Reports, + options: Options) -> None: + self.start_time = time.time() + self.data_dir = data_dir + self.errors = Errors() + self.errors.set_ignore_prefix(ignore_prefix) + self.lib_path = tuple(lib_path) + self.source_set = source_set + self.reports = reports + self.options = options + self.semantic_analyzer = SemanticAnalyzer(lib_path, self.errors, options=options) + self.modules = self.semantic_analyzer.modules + self.semantic_analyzer_pass3 = ThirdPass(self.modules, self.errors) + self.type_checker = TypeChecker(self.errors, self.modules, options=options) + self.missing_modules = set() # type: Set[str] + + def all_imported_modules_in_file(self, + file: MypyFile) -> List[Tuple[int, str, int]]: + """Find all reachable import statements in a file. + + Return list of tuples (priority, module id, import line number) + for all modules imported in file; lower numbers == higher priority. + """ + + def correct_rel_imp(imp: Union[ImportFrom, ImportAll]) -> str: + """Function to correct for relative imports.""" + file_id = file.fullname() + rel = imp.relative + if rel == 0: + return imp.id + if os.path.basename(file.path).startswith('__init__.'): + rel -= 1 + if rel != 0: + file_id = ".".join(file_id.split(".")[:-rel]) + new_id = file_id + "." + imp.id if imp.id else file_id + + return new_id + + res = [] # type: List[Tuple[int, str, int]] + for imp in file.imports: + if not imp.is_unreachable: + if isinstance(imp, Import): + pri = PRI_MED if imp.is_top_level else PRI_LOW + for id, _ in imp.ids: + res.append((pri, id, imp.line)) + elif isinstance(imp, ImportFrom): + cur_id = correct_rel_imp(imp) + pos = len(res) + all_are_submodules = True + # Also add any imported names that are submodules. + pri = PRI_MED if imp.is_top_level else PRI_LOW + for name, __ in imp.names: + sub_id = cur_id + '.' + name + if self.is_module(sub_id): + res.append((pri, sub_id, imp.line)) + else: + all_are_submodules = False + # If all imported names are submodules, don't add + # cur_id as a dependency. Otherwise (i.e., if at + # least one imported name isn't a submodule) + # cur_id is also a dependency, and we should + # insert it *before* any submodules. + if not all_are_submodules: + pri = PRI_HIGH if imp.is_top_level else PRI_LOW + res.insert(pos, ((pri, cur_id, imp.line))) + elif isinstance(imp, ImportAll): + pri = PRI_HIGH if imp.is_top_level else PRI_LOW + res.append((pri, correct_rel_imp(imp), imp.line)) + + return res + + def is_module(self, id: str) -> bool: + """Is there a file in the file system corresponding to module id?""" + return find_module(id, self.lib_path) is not None + + def parse_file(self, id: str, path: str, source: str) -> MypyFile: + """Parse the source of a file with the given name. + + Raise CompileError if there is a parse error. + """ + num_errs = self.errors.num_messages() + tree = parse(source, path, self.errors, options=self.options) + tree._fullname = id + + # We don't want to warn about 'type: ignore' comments on + # imports, but we're about to modify tree.imports, so grab + # these first. + import_lines = set(node.line for node in tree.imports) + + # Skip imports that have been ignored (so that we can ignore a C extension module without + # stub, for example), except for 'from x import *', because we wouldn't be able to + # determine which names should be defined unless we process the module. We can still + # ignore errors such as redefinitions when using the latter form. + imports = [node for node in tree.imports + if node.line not in tree.ignored_lines or isinstance(node, ImportAll)] + tree.imports = imports + + if self.errors.num_messages() != num_errs: + self.log("Bailing due to parse errors") + self.errors.raise_error() + + self.errors.set_file_ignored_lines(path, tree.ignored_lines) + self.errors.mark_file_ignored_lines_used(path, import_lines) + return tree + + def module_not_found(self, path: str, line: int, id: str) -> None: + self.errors.set_file(path) + stub_msg = "(Stub files are from https://github.com/python/typeshed)" + if ((self.options.python_version[0] == 2 and moduleinfo.is_py2_std_lib_module(id)) or + (self.options.python_version[0] >= 3 and moduleinfo.is_py3_std_lib_module(id))): + self.errors.report( + line, "No library stub file for standard library module '{}'".format(id)) + self.errors.report(line, stub_msg, severity='note', only_once=True) + elif moduleinfo.is_third_party_module(id): + self.errors.report(line, "No library stub file for module '{}'".format(id)) + self.errors.report(line, stub_msg, severity='note', only_once=True) + else: + self.errors.report(line, "Cannot find module named '{}'".format(id)) + self.errors.report(line, '(Perhaps setting MYPYPATH ' + 'or using the "--silent-imports" flag would help)', + severity='note', only_once=True) + + def report_file(self, file: MypyFile) -> None: + if self.source_set.is_source(file): + self.reports.file(file, type_map=self.type_checker.type_map) + + def log(self, *message: str) -> None: + if self.options.verbosity >= 1: + print('%.3f:LOG: ' % (time.time() - self.start_time), *message, file=sys.stderr) + sys.stderr.flush() + + def trace(self, *message: str) -> None: + if self.options.verbosity >= 2: + print('%.3f:TRACE:' % (time.time() - self.start_time), *message, file=sys.stderr) + sys.stderr.flush() + + +def remove_cwd_prefix_from_path(p: str) -> str: + """Remove current working directory prefix from p, if present. + + Also crawl up until a directory without __init__.py is found. + + If the result would be empty, return '.' instead. + """ + cur = os.getcwd() + # Add separator to the end of the path, unless one is already present. + if basename(cur) != '': + cur += os.sep + # Compute root path. + while (p and + (os.path.isfile(os.path.join(p, '__init__.py')) or + os.path.isfile(os.path.join(p, '__init__.pyi')))): + dir, base = os.path.split(p) + if not base: + break + p = dir + # Remove current directory prefix from the path, if present. + if p.startswith(cur): + p = p[len(cur):] + # Avoid returning an empty path; replace that with '.'. + if p == '': + p = '.' + return p + + +# Cache find_module: (id, lib_path) -> result. +find_module_cache = {} # type: Dict[Tuple[str, Tuple[str, ...]], str] + +# Cache some repeated work within distinct find_module calls: finding which +# elements of lib_path have even the subdirectory they'd need for the module +# to exist. This is shared among different module ids when they differ only +# in the last component. +find_module_dir_cache = {} # type: Dict[Tuple[str, Tuple[str, ...]], List[str]] + +# Cache directory listings. We assume that while one os.listdir() +# call may be more expensive than one os.stat() call, a small number +# of os.stat() calls is quickly more expensive than caching the +# os.listdir() outcome, and the advantage of the latter is that it +# gives us the case-correct filename on Windows and Mac. +find_module_listdir_cache = {} # type: Dict[str, Optional[List[str]]] + + +def find_module_clear_caches(): + find_module_cache.clear() + find_module_dir_cache.clear() + find_module_listdir_cache.clear() + + +def list_dir(path: str) -> Optional[List[str]]: + """Return a cached directory listing. + + Returns None if the path doesn't exist or isn't a directory. + """ + if path in find_module_listdir_cache: + return find_module_listdir_cache[path] + try: + res = os.listdir(path) + except OSError: + res = None + find_module_listdir_cache[path] = res + return res + + +def is_file(path: str) -> bool: + """Return whether path exists and is a file. + + On case-insensitive filesystems (like Mac or Windows) this returns + False if the case of the path's last component does not exactly + match the case found in the filesystem. + """ + head, tail = os.path.split(path) + if not tail: + return False + names = list_dir(head) + if not names: + return False + if tail not in names: + return False + return os.path.isfile(path) + + +def find_module(id: str, lib_path: Iterable[str]) -> str: + """Return the path of the module source file, or None if not found.""" + if not isinstance(lib_path, tuple): + lib_path = tuple(lib_path) + + def find(): + # If we're looking for a module like 'foo.bar.baz', it's likely that most of the + # many elements of lib_path don't even have a subdirectory 'foo/bar'. Discover + # that only once and cache it for when we look for modules like 'foo.bar.blah' + # that will require the same subdirectory. + components = id.split('.') + dir_chain = os.sep.join(components[:-1]) # e.g., 'foo/bar' + if (dir_chain, lib_path) not in find_module_dir_cache: + dirs = [] + for pathitem in lib_path: + # e.g., '/usr/lib/python3.4/foo/bar' + dir = os.path.normpath(os.path.join(pathitem, dir_chain)) + if os.path.isdir(dir): + dirs.append(dir) + find_module_dir_cache[dir_chain, lib_path] = dirs + candidate_base_dirs = find_module_dir_cache[dir_chain, lib_path] + + # If we're looking for a module like 'foo.bar.baz', then candidate_base_dirs now + # contains just the subdirectories 'foo/bar' that actually exist under the + # elements of lib_path. This is probably much shorter than lib_path itself. + # Now just look for 'baz.pyi', 'baz/__init__.py', etc., inside those directories. + seplast = os.sep + components[-1] # so e.g. '/baz' + sepinit = os.sep + '__init__' + for base_dir in candidate_base_dirs: + base_path = base_dir + seplast # so e.g. '/usr/lib/python3.4/foo/bar/baz' + # Prefer package over module, i.e. baz/__init__.py* over baz.py*. + for extension in PYTHON_EXTENSIONS: + path = base_path + sepinit + extension + if is_file(path) and verify_module(id, path): + return path + # No package, look for module. + for extension in PYTHON_EXTENSIONS: + path = base_path + extension + if is_file(path) and verify_module(id, path): + return path + return None + + key = (id, lib_path) + if key not in find_module_cache: + find_module_cache[key] = find() + return find_module_cache[key] + + +def find_modules_recursive(module: str, lib_path: List[str]) -> List[BuildSource]: + module_path = find_module(module, lib_path) + if not module_path: + return [] + result = [BuildSource(module_path, module, None)] + if module_path.endswith(('__init__.py', '__init__.pyi')): + # Subtle: this code prefers the .pyi over the .py if both + # exists, and also prefers packages over modules if both x/ + # and x.py* exist. How? We sort the directory items, so x + # comes before x.py and x.pyi. But the preference for .pyi + # over .py is encoded in find_module(); even though we see + # x.py before x.pyi, find_module() will find x.pyi first. We + # use hits to avoid adding it a second time when we see x.pyi. + # This also avoids both x.py and x.pyi when x/ was seen first. + hits = set() # type: Set[str] + for item in sorted(os.listdir(os.path.dirname(module_path))): + abs_path = os.path.join(os.path.dirname(module_path), item) + if os.path.isdir(abs_path) and \ + (os.path.isfile(os.path.join(abs_path, '__init__.py')) or + os.path.isfile(os.path.join(abs_path, '__init__.pyi'))): + hits.add(item) + result += find_modules_recursive(module + '.' + item, lib_path) + elif item != '__init__.py' and item != '__init__.pyi' and \ + item.endswith(('.py', '.pyi')): + mod = item.split('.')[0] + if mod not in hits: + hits.add(mod) + result += find_modules_recursive( + module + '.' + mod, lib_path) + return result + + +def verify_module(id: str, path: str) -> bool: + """Check that all packages containing id have a __init__ file.""" + if path.endswith(('__init__.py', '__init__.pyi')): + path = dirname(path) + for i in range(id.count('.')): + path = dirname(path) + if not any(os.path.isfile(os.path.join(path, '__init__{}'.format(extension))) + for extension in PYTHON_EXTENSIONS): + return False + return True + + +def read_with_python_encoding(path: str, pyversion: Tuple[int, int]) -> str: + """Read the Python file with while obeying PEP-263 encoding detection""" + source_bytearray = bytearray() + encoding = 'utf8' if pyversion[0] >= 3 else 'ascii' + + with open(path, 'rb') as f: + # read first two lines and check if PEP-263 coding is present + source_bytearray.extend(f.readline()) + source_bytearray.extend(f.readline()) + + # check for BOM UTF-8 encoding and strip it out if present + if source_bytearray.startswith(b'\xef\xbb\xbf'): + encoding = 'utf8' + source_bytearray = source_bytearray[3:] + else: + _encoding, _ = util.find_python_encoding(source_bytearray, pyversion) + # check that the coding isn't mypy. We skip it since + # registering may not have happened yet + if _encoding != 'mypy': + encoding = _encoding + + source_bytearray.extend(f.read()) + try: + source_bytearray.decode(encoding) + except LookupError as lookuperr: + raise DecodeError(str(lookuperr)) + return source_bytearray.decode(encoding) + + +MYPY_CACHE = '.mypy_cache' + + +def get_cache_names(id: str, path: str, pyversion: Tuple[int, int]) -> Tuple[str, str]: + """Return the file names for the cache files. + + Args: + id: module ID + path: module path (used to recognize packages) + pyversion: Python version (major, minor) + + Returns: + A tuple with the file names to be used for the meta JSON and the + data JSON, respectively. + """ + prefix = os.path.join(MYPY_CACHE, '%d.%d' % pyversion, *id.split('.')) + is_package = os.path.basename(path).startswith('__init__.py') + if is_package: + prefix = os.path.join(prefix, '__init__') + return (prefix + '.meta.json', prefix + '.data.json') + + +def find_cache_meta(id: str, path: str, manager: BuildManager) -> Optional[CacheMeta]: + """Find cache data for a module. + + Args: + id: module ID + path: module path + manager: the build manager (for pyversion, log/trace, and build options) + + Returns: + A CacheMeta instance if the cache data was found and appears + valid; otherwise None. + """ + # TODO: May need to take more build options into account + meta_json, data_json = get_cache_names(id, path, manager.options.python_version) + manager.trace('Looking for {} {}'.format(id, data_json)) + if not os.path.exists(meta_json): + return None + with open(meta_json, 'r') as f: + meta_str = f.read() + manager.trace('Meta {} {}'.format(id, meta_str.rstrip())) + meta = json.loads(meta_str) # TODO: Errors + if not isinstance(meta, dict): + return None + path = os.path.abspath(path) + m = CacheMeta( + meta.get('id'), + meta.get('path'), + meta.get('mtime'), + meta.get('size'), + meta.get('dependencies', []), + meta.get('data_mtime'), + data_json, + meta.get('suppressed', []), + meta.get('options'), + meta.get('dep_prios', []), + ) + if (m.id != id or m.path != path or + m.mtime is None or m.size is None or + m.dependencies is None or m.data_mtime is None): + return None + + # Ignore cache if generated by an older mypy version. + if m.options is None or len(m.dependencies) != len(m.dep_prios): + return None + + # Ignore cache if (relevant) options aren't the same. + cached_options = m.options + current_options = select_options_affecting_cache(manager.options) + if cached_options != current_options: + return None + + # TODO: Share stat() outcome with find_module() + st = os.stat(path) # TODO: Errors + if st.st_mtime != m.mtime or st.st_size != m.size: + manager.log('Metadata abandoned because of modified file {}'.format(path)) + return None + # It's a match on (id, path, mtime, size). + # Check data_json; assume if its mtime matches it's good. + # TODO: stat() errors + if os.path.getmtime(data_json) != m.data_mtime: + return None + manager.log('Found {} {}'.format(id, meta_json)) + return m + + +def select_options_affecting_cache(options: Options) -> Mapping[str, bool]: + return {opt: getattr(options, opt) for opt in OPTIONS_AFFECTING_CACHE} + + +OPTIONS_AFFECTING_CACHE = [ + "silent_imports", + "almost_silent", + "disallow_untyped_calls", + "disallow_untyped_defs", + "check_untyped_defs", +] + + +def random_string(): + return binascii.hexlify(os.urandom(8)).decode('ascii') + + +def write_cache(id: str, path: str, tree: MypyFile, + dependencies: List[str], suppressed: List[str], + dep_prios: List[int], + manager: BuildManager) -> None: + """Write cache files for a module. + + Args: + id: module ID + path: module path + tree: the fully checked module data + dependencies: module IDs on which this module depends + suppressed: module IDs which were suppressed as dependencies + dep_prios: priorities (parallel array to dependencies) + manager: the build manager (for pyversion, log/trace) + """ + path = os.path.abspath(path) + manager.trace('Dumping {} {}'.format(id, path)) + st = os.stat(path) # TODO: Errors + mtime = st.st_mtime + size = st.st_size + meta_json, data_json = get_cache_names(id, path, manager.options.python_version) + manager.log('Writing {} {} {}'.format(id, meta_json, data_json)) + data = tree.serialize() + parent = os.path.dirname(data_json) + if not os.path.isdir(parent): + os.makedirs(parent) + assert os.path.dirname(meta_json) == parent + nonce = '.' + random_string() + data_json_tmp = data_json + nonce + meta_json_tmp = meta_json + nonce + with open(data_json_tmp, 'w') as f: + json.dump(data, f, indent=2, sort_keys=True) + f.write('\n') + data_mtime = os.path.getmtime(data_json_tmp) + meta = {'id': id, + 'path': path, + 'mtime': mtime, + 'size': size, + 'data_mtime': data_mtime, + 'dependencies': dependencies, + 'suppressed': suppressed, + 'options': select_options_affecting_cache(manager.options), + 'dep_prios': dep_prios, + } + with open(meta_json_tmp, 'w') as f: + json.dump(meta, f, sort_keys=True) + f.write('\n') + os.replace(data_json_tmp, data_json) + os.replace(meta_json_tmp, meta_json) + + +"""Dependency manager. + +Design +====== + +Ideally +------- + +A. Collapse cycles (each SCC -- strongly connected component -- + becomes one "supernode"). + +B. Topologically sort nodes based on dependencies. + +C. Process from leaves towards roots. + +Wrinkles +-------- + +a. Need to parse source modules to determine dependencies. + +b. Processing order for modules within an SCC. + +c. Must order mtimes of files to decide whether to re-process; depends + on clock never resetting. + +d. from P import M; checks filesystem whether module P.M exists in + filesystem. + +e. Race conditions, where somebody modifies a file while we're + processing. I propose not to modify the algorithm to handle this, + but to detect when this could lead to inconsistencies. (For + example, when we decide on the dependencies based on cache + metadata, and then we decide to re-parse a file because of a stale + dependency, if the re-parsing leads to a different list of + dependencies we should warn the user or start over.) + +Steps +----- + +1. For each explicitly given module find the source file location. + +2. For each such module load and check the cache metadata, and decide + whether it's valid. + +3. Now recursively (or iteratively) find dependencies and add those to + the graph: + + - for cached nodes use the list of dependencies from the cache + metadata (this will be valid even if we later end up re-parsing + the same source); + + - for uncached nodes parse the file and process all imports found, + taking care of (a) above. + +Step 3 should also address (d) above. + +Once step 3 terminates we have the entire dependency graph, and for +each module we've either loaded the cache metadata or parsed the +source code. (However, we may still need to parse those modules for +which we have cache metadata but that depend, directly or indirectly, +on at least one module for which the cache metadata is stale.) + +Now we can execute steps A-C from the first section. Finding SCCs for +step A shouldn't be hard; there's a recipe here: +http://code.activestate.com/recipes/578507/. There's also a plethora +of topsort recipes, e.g. http://code.activestate.com/recipes/577413/. + +For single nodes, processing is simple. If the node was cached, we +deserialize the cache data and fix up cross-references. Otherwise, we +do semantic analysis followed by type checking. We also handle (c) +above; if a module has valid cache data *but* any of its +dependencies was processed from source, then the module should be +processed from source. + +A relatively simple optimization (outside SCCs) we might do in the +future is as follows: if a node's cache data is valid, but one or more +of its dependencies are out of date so we have to re-parse the node +from source, once we have fully type-checked the node, we can decide +whether its symbol table actually changed compared to the cache data +(by reading the cache data and comparing it to the data we would be +writing). If there is no change we can declare the node up to date, +and any node that depends (and for which we have cached data, and +whose other dependencies are up to date) on it won't need to be +re-parsed from source. + +Import cycles +------------- + +Finally we have to decide how to handle (c), import cycles. Here +we'll need a modified version of the original state machine +(build.py), but we only need to do this per SCC, and we won't have to +deal with changes to the list of nodes while we're processing it. + +If all nodes in the SCC have valid cache metadata and all dependencies +outside the SCC are still valid, we can proceed as follows: + + 1. Load cache data for all nodes in the SCC. + + 2. Fix up cross-references for all nodes in the SCC. + +Otherwise, the simplest (but potentially slow) way to proceed is to +invalidate all cache data in the SCC and re-parse all nodes in the SCC +from source. We can do this as follows: + + 1. Parse source for all nodes in the SCC. + + 2. Semantic analysis for all nodes in the SCC. + + 3. Type check all nodes in the SCC. + +(If there are more passes the process is the same -- each pass should +be done for all nodes before starting the next pass for any nodes in +the SCC.) + +We could process the nodes in the SCC in any order. For sentimental +reasons, I've decided to process them in the reverse order in which we +encountered them when originally constructing the graph. That's how +the old build.py deals with cycles, and at least this reproduces the +previous implementation more accurately. + +Can we do better than re-parsing all nodes in the SCC when any of its +dependencies are out of date? It's doubtful. The optimization +mentioned at the end of the previous section would require re-parsing +and type-checking a node and then comparing its symbol table to the +cached data; but because the node is part of a cycle we can't +technically type-check it until the semantic analysis of all other +nodes in the cycle has completed. (This is an important issue because +Dropbox has a very large cycle in production code. But I'd like to +deal with it later.) + +Additional wrinkles +------------------- + +During implementation more wrinkles were found. + +- When a submodule of a package (e.g. x.y) is encountered, the parent + package (e.g. x) must also be loaded, but it is not strictly a + dependency. See State.add_ancestors() below. +""" + + +class ModuleNotFound(Exception): + """Control flow exception to signal that a module was not found.""" + + +class State: + """The state for a module. + + The source is only used for the -c command line option; in that + case path is None. Otherwise source is None and path isn't. + """ + + manager = None # type: BuildManager + order_counter = 0 # Class variable + order = None # type: int # Order in which modules were encountered + id = None # type: str # Fully qualified module name + path = None # type: Optional[str] # Path to module source + xpath = None # type: str # Path or '' + source = None # type: Optional[str] # Module source code + meta = None # type: Optional[CacheMeta] + data = None # type: Optional[str] + tree = None # type: Optional[MypyFile] + dependencies = None # type: List[str] + suppressed = None # type: List[str] # Suppressed/missing dependencies + priorities = None # type: Dict[str, int] + + # Map each dependency to the line number where it is first imported + dep_line_map = None # type: Dict[str, int] + + # Parent package, its parent, etc. + ancestors = None # type: Optional[List[str]] + + # List of (path, line number) tuples giving context for import + import_context = None # type: List[Tuple[str, int]] + + # The State from which this module was imported, if any + caller_state = None # type: Optional[State] + + # If caller_state is set, the line number in the caller where the import occurred + caller_line = 0 + + def __init__(self, + id: Optional[str], + path: Optional[str], + source: Optional[str], + manager: BuildManager, + caller_state: 'State' = None, + caller_line: int = 0, + ancestor_for: 'State' = None, + ) -> None: + assert id or path or source is not None, "Neither id, path nor source given" + self.manager = manager + State.order_counter += 1 + self.order = State.order_counter + self.caller_state = caller_state + self.caller_line = caller_line + if caller_state: + self.import_context = caller_state.import_context[:] + self.import_context.append((caller_state.xpath, caller_line)) + else: + self.import_context = [] + self.id = id or '__main__' + if not path and source is None: + file_id = id + if id == 'builtins' and manager.options.python_version[0] == 2: + # The __builtin__ module is called internally by mypy + # 'builtins' in Python 2 mode (similar to Python 3), + # but the stub file is __builtin__.pyi. The reason is + # that a lot of code hard-codes 'builtins.x' and it's + # easier to work it around like this. It also means + # that the implementation can mostly ignore the + # difference and just assume 'builtins' everywhere, + # which simplifies code. + file_id = '__builtin__' + path = find_module(file_id, manager.lib_path) + if path: + # In silent mode, don't import .py files, except from stubs. + if (manager.options.silent_imports and + path.endswith('.py') and (caller_state or ancestor_for)): + # (Never silence builtins, even if it's a .py file; + # this can happen in tests!) + if (id != 'builtins' and + not ((caller_state and + caller_state.tree and + caller_state.tree.is_stub))): + if manager.options.almost_silent: + if ancestor_for: + self.skipping_ancestor(id, path, ancestor_for) + else: + self.skipping_module(id, path) + path = None + manager.missing_modules.add(id) + raise ModuleNotFound + else: + # Could not find a module. Typically the reason is a + # misspelled module name, missing stub, module not in + # search path or the module has not been installed. + if caller_state: + suppress_message = ((manager.options.silent_imports and + not manager.options.almost_silent) or + (caller_state.tree is not None and + 'import' in caller_state.tree.weak_opts)) + if not suppress_message: + save_import_context = manager.errors.import_context() + manager.errors.set_import_context(caller_state.import_context) + manager.module_not_found(caller_state.xpath, caller_line, id) + manager.errors.set_import_context(save_import_context) + manager.missing_modules.add(id) + raise ModuleNotFound + else: + # If we can't find a root source it's always fatal. + # TODO: This might hide non-fatal errors from + # root sources processed earlier. + raise CompileError(["mypy: can't find module '%s'" % id]) + self.path = path + self.xpath = path or '' + self.source = source + if path and source is None and manager.options.incremental: + self.meta = find_cache_meta(self.id, self.path, manager) + # TODO: Get mtime if not cached. + self.add_ancestors() + if self.meta: + # Make copies, since we may modify these and want to + # compare them to the originals later. + self.dependencies = list(self.meta.dependencies) + self.suppressed = list(self.meta.suppressed) + assert len(self.meta.dependencies) == len(self.meta.dep_prios) + self.priorities = {id: pri + for id, pri in zip(self.meta.dependencies, self.meta.dep_prios)} + self.dep_line_map = {} + else: + # Parse the file (and then some) to get the dependencies. + self.parse_file() + self.suppressed = [] + + def skipping_ancestor(self, id: str, path: str, ancestor_for: 'State') -> None: + # TODO: Read the path (the __init__.py file) and return + # immediately if it's empty or only contains comments. + # But beware, some package may be the ancestor of many modules, + # so we'd need to cache the decision. + manager = self.manager + manager.errors.set_import_context([]) + manager.errors.set_file(ancestor_for.xpath) + manager.errors.report(-1, "Ancestor package '%s' silently ignored" % (id,), + severity='note', only_once=True) + manager.errors.report(-1, "(Using --silent-imports, submodule passed on command line)", + severity='note', only_once=True) + manager.errors.report(-1, "(This note brought to you by --almost-silent)", + severity='note', only_once=True) + + def skipping_module(self, id: str, path: str) -> None: + assert self.caller_state, (id, path) + manager = self.manager + save_import_context = manager.errors.import_context() + manager.errors.set_import_context(self.caller_state.import_context) + manager.errors.set_file(self.caller_state.xpath) + line = self.caller_line + manager.errors.report(line, "Import of '%s' silently ignored" % (id,), + severity='note') + manager.errors.report(line, "(Using --silent-imports, module not passed on command line)", + severity='note', only_once=True) + manager.errors.report(line, "(This note courtesy of --almost-silent)", + severity='note', only_once=True) + manager.errors.set_import_context(save_import_context) + + def add_ancestors(self) -> None: + # All parent packages are new ancestors. + ancestors = [] + parent = self.id + while '.' in parent: + parent, _ = parent.rsplit('.', 1) + ancestors.append(parent) + self.ancestors = ancestors + + def is_fresh(self) -> bool: + """Return whether the cache data for this file is fresh.""" + # NOTE: self.dependencies may differ from + # self.meta.dependencies when a dependency is dropped due to + # suppression by --silent-imports. However when a suppressed + # dependency is added back we find out later in the process. + return self.meta is not None and self.dependencies == self.meta.dependencies + + def mark_stale(self) -> None: + """Throw away the cache data for this file, marking it as stale.""" + self.meta = None + + def check_blockers(self) -> None: + """Raise CompileError if a blocking error is detected.""" + if self.manager.errors.is_blockers(): + self.manager.log("Bailing due to blocking errors") + self.manager.errors.raise_error() + + @contextlib.contextmanager + def wrap_context(self) -> Iterator[None]: + save_import_context = self.manager.errors.import_context() + self.manager.errors.set_import_context(self.import_context) + try: + yield + except CompileError: + raise + except Exception as err: + report_internal_error(err, self.path, 0) + self.manager.errors.set_import_context(save_import_context) + self.check_blockers() + + # Methods for processing cached modules. + + def load_tree(self) -> None: + with open(self.meta.data_json) as f: + data = json.load(f) + # TODO: Assert data file wasn't changed. + self.tree = MypyFile.deserialize(data) + self.manager.modules[self.id] = self.tree + + def fix_cross_refs(self) -> None: + fixup_module_pass_one(self.tree, self.manager.modules) + + def calculate_mros(self) -> None: + fixup_module_pass_two(self.tree, self.manager.modules) + + # Methods for processing modules from source code. + + def parse_file(self) -> None: + if self.tree is not None: + # The file was already parsed (in __init__()). + return + + manager = self.manager + modules = manager.modules + manager.log("Parsing %s (%s)" % (self.xpath, self.id)) + + with self.wrap_context(): + source = self.source + self.source = None # We won't need it again. + if self.path and source is None: + try: + source = read_with_python_encoding(self.path, manager.options.python_version) + except IOError as ioerr: + raise CompileError([ + "mypy: can't read file '{}': {}".format(self.path, ioerr.strerror)]) + except (UnicodeDecodeError, DecodeError) as decodeerr: + raise CompileError([ + "mypy: can't decode file '{}': {}".format(self.path, str(decodeerr))]) + self.tree = manager.parse_file(self.id, self.xpath, source) + + modules[self.id] = self.tree + + # Do the first pass of semantic analysis: add top-level + # definitions in the file to the symbol table. We must do + # this before processing imports, since this may mark some + # import statements as unreachable. + first = FirstPass(manager.semantic_analyzer) + first.analyze(self.tree, self.xpath, self.id) + + # Initialize module symbol table, which was populated by the + # semantic analyzer. + # TODO: Why can't FirstPass .analyze() do this? + self.tree.names = manager.semantic_analyzer.globals + + # Compute (direct) dependencies. + # Add all direct imports (this is why we needed the first pass). + # Also keep track of each dependency's source line. + dependencies = [] + suppressed = [] + priorities = {} # type: Dict[str, int] # id -> priority + dep_line_map = {} # type: Dict[str, int] # id -> line + for pri, id, line in manager.all_imported_modules_in_file(self.tree): + priorities[id] = min(pri, priorities.get(id, PRI_ALL)) + if id == self.id: + continue + # Omit missing modules, as otherwise we could not type-check + # programs with missing modules. + if id in manager.missing_modules: + if id not in dep_line_map: + suppressed.append(id) + dep_line_map[id] = line + continue + if id == '': + # Must be from a relative import. + manager.errors.set_file(self.xpath) + manager.errors.report(line, "No parent module -- cannot perform relative import", + blocker=True) + continue + if id not in dep_line_map: + dependencies.append(id) + dep_line_map[id] = line + # Every module implicitly depends on builtins. + if self.id != 'builtins' and 'builtins' not in dep_line_map: + dependencies.append('builtins') + + # If self.dependencies is already set, it was read from the + # cache, but for some reason we're re-parsing the file. + # NOTE: What to do about race conditions (like editing the + # file while mypy runs)? A previous version of this code + # explicitly checked for this, but ran afoul of other reasons + # for differences (e.g. --silent-imports). + self.dependencies = dependencies + self.suppressed = suppressed + self.priorities = priorities + self.dep_line_map = dep_line_map + self.check_blockers() + + def patch_parent(self) -> None: + # Include module in the symbol table of the enclosing package. + if '.' not in self.id: + return + manager = self.manager + modules = manager.modules + parent, child = self.id.rsplit('.', 1) + if parent in modules: + manager.trace("Added %s.%s" % (parent, child)) + modules[parent].names[child] = SymbolTableNode(MODULE_REF, self.tree, parent) + else: + manager.log("Hm... couldn't add %s.%s" % (parent, child)) + + def semantic_analysis(self) -> None: + with self.wrap_context(): + self.manager.semantic_analyzer.visit_file(self.tree, self.xpath) + + def semantic_analysis_pass_three(self) -> None: + with self.wrap_context(): + self.manager.semantic_analyzer_pass3.visit_file(self.tree, self.xpath) + if self.manager.options.dump_type_stats: + dump_type_stats(self.tree, self.xpath) + + def type_check(self) -> None: + manager = self.manager + if manager.options.semantic_analysis_only: + return + with self.wrap_context(): + manager.type_checker.visit_file(self.tree, self.xpath) + if manager.options.dump_inference_stats: + dump_type_stats(self.tree, self.xpath, inferred=True, + typemap=manager.type_checker.type_map) + manager.report_file(self.tree) + + def write_cache(self) -> None: + if self.path and self.manager.options.incremental and not self.manager.errors.is_errors(): + dep_prios = [self.priorities.get(dep, PRI_HIGH) for dep in self.dependencies] + write_cache(self.id, self.path, self.tree, + list(self.dependencies), list(self.suppressed), + dep_prios, + self.manager) + + +Graph = Dict[str, State] + + +def dispatch(sources: List[BuildSource], manager: BuildManager) -> None: + manager.log("Using new dependency manager") + graph = load_graph(sources, manager) + manager.log("Loaded graph with %d nodes" % len(graph)) + process_graph(graph, manager) + if manager.options.warn_unused_ignores: + manager.errors.generate_unused_ignore_notes() + + +def load_graph(sources: List[BuildSource], manager: BuildManager) -> Graph: + """Given some source files, load the full dependency graph.""" + graph = {} # type: Graph + # The deque is used to implement breadth-first traversal. + # TODO: Consider whether to go depth-first instead. This may + # affect the order in which we process files within import cycles. + new = collections.deque() # type: collections.deque[State] + # Seed the graph with the initial root sources. + for bs in sources: + try: + st = State(id=bs.module, path=bs.path, source=bs.text, manager=manager) + except ModuleNotFound: + continue + if st.id in graph: + manager.errors.set_file(st.xpath) + manager.errors.report(1, "Duplicate module named '%s'" % st.id) + manager.errors.raise_error() + graph[st.id] = st + new.append(st) + # Collect dependencies. We go breadth-first. + while new: + st = new.popleft() + for dep in st.ancestors + st.dependencies: + if dep not in graph: + try: + if dep in st.ancestors: + # TODO: Why not 'if dep not in st.dependencies' ? + # Ancestors don't have import context. + newst = State(id=dep, path=None, source=None, manager=manager, + ancestor_for=st) + else: + newst = State(id=dep, path=None, source=None, manager=manager, + caller_state=st, caller_line=st.dep_line_map.get(dep, 1)) + except ModuleNotFound: + if dep in st.dependencies: + st.dependencies.remove(dep) + st.suppressed.append(dep) + else: + assert newst.id not in graph, newst.id + graph[newst.id] = newst + new.append(newst) + return graph + + +def process_graph(graph: Graph, manager: BuildManager) -> None: + """Process everything in dependency order.""" + sccs = sorted_components(graph) + manager.log("Found %d SCCs; largest has %d nodes" % + (len(sccs), max(len(scc) for scc in sccs))) + # We're processing SCCs from leaves (those without further + # dependencies) to roots (those from which everything else can be + # reached). + for ascc in sccs: + # Order the SCC's nodes using a heuristic. + # Note that ascc is a set, and scc is a list. + scc = order_ascc(graph, ascc) + # If builtins is in the list, move it last. (This is a bit of + # a hack, but it's necessary because the builtins module is + # part of a small cycle involving at least {builtins, abc, + # typing}. Of these, builtins must be processed last or else + # some builtin objects will be incompletely processed.) + if 'builtins' in ascc: + scc.remove('builtins') + scc.append('builtins') + if manager.options.verbosity >= 2: + for id in scc: + manager.trace("Priorities for %s:" % id, + " ".join("%s:%d" % (x, graph[id].priorities[x]) + for x in graph[id].dependencies + if x in ascc and x in graph[id].priorities)) + # Because the SCCs are presented in topological sort order, we + # don't need to look at dependencies recursively for staleness + # -- the immediate dependencies are sufficient. + stale_scc = {id for id in scc if not graph[id].is_fresh()} + fresh = not stale_scc + deps = set() + for id in scc: + deps.update(graph[id].dependencies) + deps -= ascc + stale_deps = {id for id in deps if not graph[id].is_fresh()} + fresh = fresh and not stale_deps + undeps = set() + if fresh: + # Check if any dependencies that were suppressed according + # to the cache have heen added back in this run. + # NOTE: Newly suppressed dependencies are handled by is_fresh(). + for id in scc: + undeps.update(graph[id].suppressed) + undeps &= graph.keys() + if undeps: + fresh = False + if fresh: + # All cache files are fresh. Check that no dependency's + # cache file is newer than any scc node's cache file. + oldest_in_scc = min(graph[id].meta.data_mtime for id in scc) + newest_in_deps = 0 if not deps else max(graph[dep].meta.data_mtime for dep in deps) + if manager.options.verbosity >= 3: # Dump all mtimes for extreme debugging. + all_ids = sorted(ascc | deps, key=lambda id: graph[id].meta.data_mtime) + for id in all_ids: + if id in scc: + if graph[id].meta.data_mtime < newest_in_deps: + key = "*id:" + else: + key = "id:" + else: + if graph[id].meta.data_mtime > oldest_in_scc: + key = "+dep:" + else: + key = "dep:" + manager.trace(" %5s %.0f %s" % (key, graph[id].meta.data_mtime, id)) + # If equal, give the benefit of the doubt, due to 1-sec time granularity + # (on some platforms). + if oldest_in_scc < newest_in_deps: + fresh = False + fresh_msg = "out of date by %.0f seconds" % (newest_in_deps - oldest_in_scc) + else: + fresh_msg = "fresh" + elif undeps: + fresh_msg = "stale due to changed suppression (%s)" % " ".join(sorted(undeps)) + elif stale_scc: + fresh_msg = "inherently stale (%s)" % " ".join(sorted(stale_scc)) + if stale_deps: + fresh_msg += " with stale deps (%s)" % " ".join(sorted(stale_deps)) + else: + fresh_msg = "stale due to deps (%s)" % " ".join(sorted(stale_deps)) + if len(scc) == 1: + manager.log("Processing SCC singleton (%s) as %s" % (" ".join(scc), fresh_msg)) + else: + manager.log("Processing SCC of size %d (%s) as %s" % + (len(scc), " ".join(scc), fresh_msg)) + if fresh: + process_fresh_scc(graph, scc) + else: + process_stale_scc(graph, scc) + + +def order_ascc(graph: Graph, ascc: AbstractSet[str], pri_max: int = PRI_ALL) -> List[str]: + """Come up with the ideal processing order within an SCC. + + Using the priorities assigned by all_imported_modules_in_file(), + try to reduce the cycle to a DAG, by omitting arcs representing + dependencies of lower priority. + + In the simplest case, if we have A <--> B where A has a top-level + "import B" (medium priority) but B only has the reverse "import A" + inside a function (low priority), we turn the cycle into a DAG by + dropping the B --> A arc, which leaves only A --> B. + + If all arcs have the same priority, we fall back to sorting by + reverse global order (the order in which modules were first + encountered). + + The algorithm is recursive, as follows: when as arcs of different + priorities are present, drop all arcs of the lowest priority, + identify SCCs in the resulting graph, and apply the algorithm to + each SCC thus found. The recursion is bounded because at each + recursion the spread in priorities is (at least) one less. + + In practice there are only a few priority levels (currently + N=3) and in the worst case we just carry out the same algorithm + for finding SCCs N times. Thus the complexity is no worse than + the complexity of the original SCC-finding algorithm -- see + strongly_connected_components() below for a reference. + """ + if len(ascc) == 1: + return [s for s in ascc] + pri_spread = set() + for id in ascc: + state = graph[id] + for dep in state.dependencies: + if dep in ascc: + pri = state.priorities.get(dep, PRI_HIGH) + if pri < pri_max: + pri_spread.add(pri) + if len(pri_spread) == 1: + # Filtered dependencies are uniform -- order by global order. + return sorted(ascc, key=lambda id: -graph[id].order) + pri_max = max(pri_spread) + sccs = sorted_components(graph, ascc, pri_max) + # The recursion is bounded by the len(pri_spread) check above. + return [s for ss in sccs for s in order_ascc(graph, ss, pri_max)] + + +def process_fresh_scc(graph: Graph, scc: List[str]) -> None: + """Process the modules in one SCC from their cached data.""" + for id in scc: + graph[id].load_tree() + for id in scc: + graph[id].patch_parent() + for id in scc: + graph[id].fix_cross_refs() + for id in scc: + graph[id].calculate_mros() + + +def process_stale_scc(graph: Graph, scc: List[str]) -> None: + """Process the modules in one SCC from source code.""" + for id in scc: + graph[id].mark_stale() + for id in scc: + # We may already have parsed the module, or not. + # If the former, parse_file() is a no-op. + graph[id].parse_file() + for id in scc: + graph[id].patch_parent() + for id in scc: + graph[id].semantic_analysis() + for id in scc: + graph[id].semantic_analysis_pass_three() + for id in scc: + graph[id].type_check() + graph[id].write_cache() + + +def sorted_components(graph: Graph, + vertices: Optional[AbstractSet[str]] = None, + pri_max: int = PRI_ALL) -> List[AbstractSet[str]]: + """Return the graph's SCCs, topologically sorted by dependencies. + + The sort order is from leaves (nodes without dependencies) to + roots (nodes on which no other nodes depend). + + This works for a subset of the full dependency graph too; + dependencies that aren't present in graph.keys() are ignored. + """ + # Compute SCCs. + if vertices is None: + vertices = set(graph) + edges = {id: deps_filtered(graph, vertices, id, pri_max) for id in vertices} + sccs = list(strongly_connected_components(vertices, edges)) + # Topsort. + sccsmap = {id: frozenset(scc) for scc in sccs for id in scc} + data = {} # type: Dict[AbstractSet[str], Set[AbstractSet[str]]] + for scc in sccs: + deps = set() # type: Set[AbstractSet[str]] + for id in scc: + deps.update(sccsmap[x] for x in deps_filtered(graph, vertices, id, pri_max)) + data[frozenset(scc)] = deps + res = [] + for ready in topsort(data): + # Sort the sets in ready by reversed smallest State.order. Examples: + # + # - If ready is [{x}, {y}], x.order == 1, y.order == 2, we get + # [{y}, {x}]. + # + # - If ready is [{a, b}, {c, d}], a.order == 1, b.order == 3, + # c.order == 2, d.order == 4, the sort keys become [1, 2] + # and the result is [{c, d}, {a, b}]. + res.extend(sorted(ready, + key=lambda scc: -min(graph[id].order for id in scc))) + return res + + +def deps_filtered(graph: Graph, vertices: AbstractSet[str], id: str, pri_max: int) -> List[str]: + """Filter dependencies for id with pri < pri_max.""" + if id not in vertices: + return [] + state = graph[id] + return [dep + for dep in state.dependencies + if dep in vertices and state.priorities.get(dep, PRI_HIGH) < pri_max] + + +def strongly_connected_components(vertices: AbstractSet[str], + edges: Dict[str, List[str]]) -> Iterator[Set[str]]: + """Compute Strongly Connected Components of a directed graph. + + Args: + vertices: the labels for the vertices + edges: for each vertex, gives the target vertices of its outgoing edges + + Returns: + An iterator yielding strongly connected components, each + represented as a set of vertices. Each input vertex will occur + exactly once; vertices not part of a SCC are returned as + singleton sets. + + From http://code.activestate.com/recipes/578507/. + """ + identified = set() # type: Set[str] + stack = [] # type: List[str] + index = {} # type: Dict[str, int] + boundaries = [] # type: List[int] + + def dfs(v: str) -> Iterator[Set[str]]: + index[v] = len(stack) + stack.append(v) + boundaries.append(index[v]) + + for w in edges[v]: + if w not in index: + # For Python >= 3.3, replace with "yield from dfs(w)" + for scc in dfs(w): + yield scc + elif w not in identified: + while index[w] < boundaries[-1]: + boundaries.pop() + + if boundaries[-1] == index[v]: + boundaries.pop() + scc = set(stack[index[v]:]) + del stack[index[v]:] + identified.update(scc) + yield scc + + for v in vertices: + if v not in index: + # For Python >= 3.3, replace with "yield from dfs(v)" + for scc in dfs(v): + yield scc + + +def topsort(data: Dict[AbstractSet[str], + Set[AbstractSet[str]]]) -> Iterable[Set[AbstractSet[str]]]: + """Topological sort. + + Args: + data: A map from SCCs (represented as frozen sets of strings) to + sets of SCCs, its dependencies. NOTE: This data structure + is modified in place -- for normalization purposes, + self-dependencies are removed and entries representing + orphans are added. + + Returns: + An iterator yielding sets of SCCs that have an equivalent + ordering. NOTE: The algorithm doesn't care about the internal + structure of SCCs. + + Example: + Suppose the input has the following structure: + + {A: {B, C}, B: {D}, C: {D}} + + This is normalized to: + + {A: {B, C}, B: {D}, C: {D}, D: {}} + + The algorithm will yield the following values: + + {D} + {B, C} + {A} + + From http://code.activestate.com/recipes/577413/. + """ + # TODO: Use a faster algorithm? + for k, v in data.items(): + v.discard(k) # Ignore self dependencies. + for item in set.union(*data.values()) - set(data.keys()): + data[item] = set() + while True: + ready = {item for item, dep in data.items() if not dep} + if not ready: + break + yield ready + data = {item: (dep - ready) + for item, dep in data.items() + if item not in ready} + assert not data, "A cyclic dependency exists amongst %r" % data diff --git a/mypy/checker.py b/mypy/checker.py index 62f4c250ea272..7063758096dfa 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -1,2459 +1,2459 @@ -"""Mypy type checker.""" - -import itertools -import contextlib -import os -import os.path - -from typing import ( - Any, Dict, Set, List, cast, Tuple, TypeVar, Union, Optional, NamedTuple -) - -from mypy.errors import Errors, report_internal_error -from mypy.nodes import ( - SymbolTable, Node, MypyFile, Var, - OverloadedFuncDef, FuncDef, FuncItem, FuncBase, TypeInfo, - ClassDef, GDEF, Block, AssignmentStmt, NameExpr, MemberExpr, IndexExpr, - TupleExpr, ListExpr, ExpressionStmt, ReturnStmt, IfStmt, - WhileStmt, OperatorAssignmentStmt, WithStmt, AssertStmt, - RaiseStmt, TryStmt, ForStmt, DelStmt, CallExpr, IntExpr, StrExpr, - BytesExpr, UnicodeExpr, FloatExpr, OpExpr, UnaryExpr, CastExpr, RevealTypeExpr, SuperExpr, - TypeApplication, DictExpr, SliceExpr, FuncExpr, TempNode, SymbolTableNode, - Context, ListComprehension, ConditionalExpr, GeneratorExpr, - Decorator, SetExpr, TypeVarExpr, PrintStmt, - LITERAL_TYPE, BreakStmt, ContinueStmt, ComparisonExpr, StarExpr, - YieldFromExpr, NamedTupleExpr, SetComprehension, - DictionaryComprehension, ComplexExpr, EllipsisExpr, TypeAliasExpr, - RefExpr, YieldExpr, BackquoteExpr, ImportFrom, ImportAll, ImportBase, - CONTRAVARIANT, COVARIANT -) -from mypy.nodes import function_type, method_type, method_type_with_fallback -from mypy import nodes -from mypy.types import ( - Type, AnyType, CallableType, Void, FunctionLike, Overloaded, TupleType, - Instance, NoneTyp, ErrorType, strip_type, - UnionType, TypeVarId, TypeVarType, PartialType, DeletedType, UninhabitedType -) -from mypy.sametypes import is_same_type -from mypy.messages import MessageBuilder -import mypy.checkexpr -from mypy.checkmember import map_type_from_supertype -from mypy import defaults -from mypy import messages -from mypy.subtypes import ( - is_subtype, is_equivalent, is_proper_subtype, - is_more_precise, restrict_subtype_away -) -from mypy.maptype import map_instance_to_supertype -from mypy.semanal import self_type, set_callable_name, refers_to_fullname -from mypy.erasetype import erase_typevars -from mypy.expandtype import expand_type_by_instance, expand_type -from mypy.visitor import NodeVisitor -from mypy.join import join_types -from mypy.treetransform import TransformVisitor -from mypy.meet import meet_simple, nearest_builtin_ancestor, is_overlapping_types -from mypy.binder import ConditionalTypeBinder -from mypy.options import Options - -from mypy import experiments - - -T = TypeVar('T') - - -# A node which is postponed to be type checked during the next pass. -DeferredNode = NamedTuple( - 'DeferredNode', - [ - ('node', Node), - ('context_type_name', Optional[str]), # Name of the surrounding class (for error messages) - ]) - - -class TypeChecker(NodeVisitor[Type]): - """Mypy type checker. - - Type check mypy source files that have been semantically analyzed. - """ - - # Are we type checking a stub? - is_stub = False - # Error message reporter - errors = None # type: Errors - # Utility for generating messages - msg = None # type: MessageBuilder - # Types of type checked nodes - type_map = None # type: Dict[Node, Type] - - # Helper for managing conditional types - binder = None # type: ConditionalTypeBinder - # Helper for type checking expressions - expr_checker = None # type: mypy.checkexpr.ExpressionChecker - - # Stack of function return types - return_types = None # type: List[Type] - # Type context for type inference - type_context = None # type: List[Type] - # Flags; true for dynamically typed functions - dynamic_funcs = None # type: List[bool] - # Stack of functions being type checked - function_stack = None # type: List[FuncItem] - # Do weak type checking in this file - weak_opts = set() # type: Set[str] - # Stack of collections of variables with partial types - partial_types = None # type: List[Dict[Var, Context]] - globals = None # type: SymbolTable - modules = None # type: Dict[str, MypyFile] - # Nodes that couldn't be checked because some types weren't available. We'll run - # another pass and try these again. - deferred_nodes = None # type: List[DeferredNode] - # Type checking pass number (0 = first pass) - pass_num = 0 - # Have we deferred the current function? If yes, don't infer additional - # types during this pass within the function. - current_node_deferred = False - is_typeshed_stub = False - options = None # type: Options - - def __init__(self, errors: Errors, modules: Dict[str, MypyFile], options: Options) -> None: - """Construct a type checker. - - Use errors to report type check errors. - """ - self.errors = errors - self.modules = modules - self.options = options - self.msg = MessageBuilder(errors, modules) - self.type_map = {} - self.binder = ConditionalTypeBinder() - self.expr_checker = mypy.checkexpr.ExpressionChecker(self, self.msg) - self.return_types = [] - self.type_context = [] - self.dynamic_funcs = [] - self.function_stack = [] - self.weak_opts = set() # type: Set[str] - self.partial_types = [] - self.deferred_nodes = [] - self.pass_num = 0 - self.current_node_deferred = False - - def visit_file(self, file_node: MypyFile, path: str) -> None: - """Type check a mypy file with the given path.""" - self.pass_num = 0 - self.is_stub = file_node.is_stub - self.errors.set_file(path) - self.globals = file_node.names - self.weak_opts = file_node.weak_opts - self.enter_partial_types() - # gross, but no other clear way to tell - self.is_typeshed_stub = self.is_stub and 'typeshed' in os.path.normpath(path).split(os.sep) - - for d in file_node.defs: - self.accept(d) - - self.leave_partial_types() - - if self.deferred_nodes: - self.check_second_pass() - - self.current_node_deferred = False - - all_ = self.globals.get('__all__') - if all_ is not None and all_.type is not None: - seq_str = self.named_generic_type('typing.Sequence', - [self.named_type('builtins.str')]) - if not is_subtype(all_.type, seq_str): - str_seq_s, all_s = self.msg.format_distinctly(seq_str, all_.type) - self.fail(messages.ALL_MUST_BE_SEQ_STR.format(str_seq_s, all_s), - all_.node) - - def check_second_pass(self): - """Run second pass of type checking which goes through deferred nodes.""" - self.pass_num = 1 - for node, type_name in self.deferred_nodes: - if type_name: - self.errors.push_type(type_name) - self.accept(node) - if type_name: - self.errors.pop_type() - self.deferred_nodes = [] - - def handle_cannot_determine_type(self, name: str, context: Context) -> None: - if self.pass_num == 0 and self.function_stack: - # Don't report an error yet. Just defer. - node = self.function_stack[-1] - if self.errors.type_name: - type_name = self.errors.type_name[-1] - else: - type_name = None - self.deferred_nodes.append(DeferredNode(node, type_name)) - # Set a marker so that we won't infer additional types in this - # function. Any inferred types could be bogus, because there's at - # least one type that we don't know. - self.current_node_deferred = True - else: - self.msg.cannot_determine_type(name, context) - - def accept(self, node: Node, type_context: Type = None) -> Type: - """Type check a node in the given type context.""" - self.type_context.append(type_context) - try: - typ = node.accept(self) - except Exception as err: - report_internal_error(err, self.errors.file, node.line) - self.type_context.pop() - self.store_type(node, typ) - if self.typing_mode_none(): - return AnyType() - else: - return typ - - def accept_loop(self, body: Node, else_body: Node = None) -> Type: - """Repeatedly type check a loop body until the frame doesn't change. - - Then check the else_body. - """ - # The outer frame accumulates the results of all iterations - with self.binder.frame_context(1) as outer_frame: - self.binder.push_loop_frame() - while True: - with self.binder.frame_context(1): - # We may skip each iteration - self.binder.options_on_return[-1].append(outer_frame) - self.accept(body) - if not self.binder.last_pop_changed: - break - self.binder.pop_loop_frame() - if else_body: - self.accept(else_body) - - # - # Definitions - # - - def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> Type: - num_abstract = 0 - if defn.is_property: - # HACK: Infer the type of the property. - self.visit_decorator(defn.items[0]) - for fdef in defn.items: - self.check_func_item(fdef.func, name=fdef.func.name()) - if fdef.func.is_abstract: - num_abstract += 1 - if num_abstract not in (0, len(defn.items)): - self.fail(messages.INCONSISTENT_ABSTRACT_OVERLOAD, defn) - if defn.info: - self.check_method_override(defn) - self.check_inplace_operator_method(defn) - self.check_overlapping_overloads(defn) - - def check_overlapping_overloads(self, defn: OverloadedFuncDef) -> None: - for i, item in enumerate(defn.items): - for j, item2 in enumerate(defn.items[i + 1:]): - # TODO overloads involving decorators - sig1 = self.function_type(item.func) - sig2 = self.function_type(item2.func) - if is_unsafe_overlapping_signatures(sig1, sig2): - self.msg.overloaded_signatures_overlap(i + 1, i + j + 2, - item.func) - - def is_generator_return_type(self, typ: Type) -> bool: - return is_subtype(self.named_generic_type('typing.Generator', - [AnyType(), AnyType(), AnyType()]), - typ) - - def get_generator_yield_type(self, return_type: Type) -> Type: - if isinstance(return_type, AnyType): - return AnyType() - elif not self.is_generator_return_type(return_type): - # If the function doesn't have a proper Generator (or superclass) return type, anything - # is permissible. - return AnyType() - elif not isinstance(return_type, Instance): - # Same as above, but written as a separate branch so the typechecker can understand. - return AnyType() - elif return_type.args: - return return_type.args[0] - else: - # If the function's declared supertype of Generator has no type - # parameters (i.e. is `object`), then the yielded values can't - # be accessed so any type is acceptable. - return AnyType() - - def get_generator_receive_type(self, return_type: Type) -> Type: - if isinstance(return_type, AnyType): - return AnyType() - elif not self.is_generator_return_type(return_type): - # If the function doesn't have a proper Generator (or superclass) return type, anything - # is permissible. - return AnyType() - elif not isinstance(return_type, Instance): - # Same as above, but written as a separate branch so the typechecker can understand. - return AnyType() - elif return_type.type.fullname() == 'typing.Generator': - # Generator is the only type which specifies the type of values it can receive. - return return_type.args[1] - else: - # `return_type` is a supertype of Generator, so callers won't be able to send it - # values. - return Void() - - def get_generator_return_type(self, return_type: Type) -> Type: - if isinstance(return_type, AnyType): - return AnyType() - elif not self.is_generator_return_type(return_type): - # If the function doesn't have a proper Generator (or superclass) return type, anything - # is permissible. - return AnyType() - elif not isinstance(return_type, Instance): - # Same as above, but written as a separate branch so the typechecker can understand. - return AnyType() - elif return_type.type.fullname() == 'typing.Generator': - # Generator is the only type which specifies the type of values it returns into - # `yield from` expressions. - return return_type.args[2] - else: - # `return_type` is supertype of Generator, so callers won't be able to see the return - # type when used in a `yield from` expression. - return AnyType() - - def visit_func_def(self, defn: FuncDef) -> Type: - """Type check a function definition.""" - self.check_func_item(defn, name=defn.name()) - if defn.info: - if not defn.is_dynamic(): - self.check_method_override(defn) - self.check_inplace_operator_method(defn) - if defn.original_def: - # Override previous definition. - new_type = self.function_type(defn) - if isinstance(defn.original_def, FuncDef): - # Function definition overrides function definition. - if not is_same_type(new_type, self.function_type(defn.original_def)): - self.msg.incompatible_conditional_function_def(defn) - else: - # Function definition overrides a variable initialized via assignment. - orig_type = defn.original_def.type - if orig_type is None: - # XXX This can be None, as happens in - # test_testcheck_TypeCheckSuite.testRedefinedFunctionInTryWithElse - self.msg.note("Internal mypy error checking function redefinition.", defn) - return None - if isinstance(orig_type, PartialType): - if orig_type.type is None: - # Ah this is a partial type. Give it the type of the function. - var = defn.original_def - partial_types = self.find_partial_types(var) - if partial_types is not None: - var.type = new_type - del partial_types[var] - else: - # Trying to redefine something like partial empty list as function. - self.fail(messages.INCOMPATIBLE_REDEFINITION, defn) - else: - # TODO: Update conditional type binder. - self.check_subtype(new_type, orig_type, defn, - messages.INCOMPATIBLE_REDEFINITION, - 'redefinition with type', - 'original type') - - def check_func_item(self, defn: FuncItem, - type_override: CallableType = None, - name: str = None) -> Type: - """Type check a function. - - If type_override is provided, use it as the function type. - """ - # We may be checking a function definition or an anonymous function. In - # the first case, set up another reference with the precise type. - fdef = None # type: FuncDef - if isinstance(defn, FuncDef): - fdef = defn - - self.function_stack.append(defn) - self.dynamic_funcs.append(defn.is_dynamic() and not type_override) - - if fdef: - self.errors.push_function(fdef.name()) - - self.enter_partial_types() - - typ = self.function_type(defn) - if type_override: - typ = type_override - if isinstance(typ, CallableType): - self.check_func_def(defn, typ, name) - else: - raise RuntimeError('Not supported') - - self.leave_partial_types() - - if fdef: - self.errors.pop_function() - - self.dynamic_funcs.pop() - self.function_stack.pop() - self.current_node_deferred = False - - def check_func_def(self, defn: FuncItem, typ: CallableType, name: str) -> None: - """Type check a function definition.""" - # Expand type variables with value restrictions to ordinary types. - for item, typ in self.expand_typevars(defn, typ): - old_binder = self.binder - self.binder = ConditionalTypeBinder() - with self.binder.frame_context(): - defn.expanded.append(item) - - # We may be checking a function definition or an anonymous - # function. In the first case, set up another reference with the - # precise type. - if isinstance(item, FuncDef): - fdef = item - else: - fdef = None - - if fdef: - # Check if __init__ has an invalid, non-None return type. - if (fdef.info and fdef.name() == '__init__' and - not isinstance(typ.ret_type, Void) and - not self.dynamic_funcs[-1]): - self.fail(messages.INIT_MUST_HAVE_NONE_RETURN_TYPE, - item.type) - - show_untyped = not self.is_typeshed_stub or self.options.warn_incomplete_stub - if self.options.disallow_untyped_defs and show_untyped: - # Check for functions with unspecified/not fully specified types. - def is_implicit_any(t: Type) -> bool: - return isinstance(t, AnyType) and t.implicit - - if fdef.type is None: - self.fail(messages.FUNCTION_TYPE_EXPECTED, fdef) - elif isinstance(fdef.type, CallableType): - if is_implicit_any(fdef.type.ret_type): - self.fail(messages.RETURN_TYPE_EXPECTED, fdef) - if any(is_implicit_any(t) for t in fdef.type.arg_types): - self.fail(messages.ARGUMENT_TYPE_EXPECTED, fdef) - - if name in nodes.reverse_op_method_set: - self.check_reverse_op_method(item, typ, name) - elif name == '__getattr__': - self.check_getattr_method(typ, defn) - - # Refuse contravariant return type variable - if isinstance(typ.ret_type, TypeVarType): - if typ.ret_type.variance == CONTRAVARIANT: - self.fail(messages.RETURN_TYPE_CANNOT_BE_CONTRAVARIANT, - typ.ret_type) - - # Check that Generator functions have the appropriate return type. - if defn.is_generator: - if not self.is_generator_return_type(typ.ret_type): - self.fail(messages.INVALID_RETURN_TYPE_FOR_GENERATOR, typ) - - # Python 2 generators aren't allowed to return values. - if (self.options.python_version[0] == 2 and - isinstance(typ.ret_type, Instance) and - typ.ret_type.type.fullname() == 'typing.Generator'): - if not (isinstance(typ.ret_type.args[2], Void) - or isinstance(typ.ret_type.args[2], AnyType)): - self.fail(messages.INVALID_GENERATOR_RETURN_ITEM_TYPE, typ) - - # Push return type. - self.return_types.append(typ.ret_type) - - # Store argument types. - for i in range(len(typ.arg_types)): - arg_type = typ.arg_types[i] - - # Refuse covariant parameter type variables - if isinstance(arg_type, TypeVarType): - if arg_type.variance == COVARIANT: - self.fail(messages.FUNCTION_PARAMETER_CANNOT_BE_COVARIANT, - arg_type) - - if typ.arg_kinds[i] == nodes.ARG_STAR: - # builtins.tuple[T] is typing.Tuple[T, ...] - arg_type = self.named_generic_type('builtins.tuple', - [arg_type]) - elif typ.arg_kinds[i] == nodes.ARG_STAR2: - arg_type = self.named_generic_type('builtins.dict', - [self.str_type(), - arg_type]) - item.arguments[i].variable.type = arg_type - - # Type check initialization expressions. - for arg in item.arguments: - init = arg.initialization_statement - if init: - self.accept(init) - - # Type check body in a new scope. - with self.binder.frame_context(): - self.accept(item.body) - - self.return_types.pop() - - self.binder = old_binder - - def check_reverse_op_method(self, defn: FuncItem, typ: CallableType, - method: str) -> None: - """Check a reverse operator method such as __radd__.""" - - # This used to check for some very obscure scenario. It now - # just decides whether it's worth calling - # check_overlapping_op_methods(). - - if method in ('__eq__', '__ne__'): - # These are defined for all objects => can't cause trouble. - return - - # With 'Any' or 'object' return type we are happy, since any possible - # return value is valid. - ret_type = typ.ret_type - if isinstance(ret_type, AnyType): - return - if isinstance(ret_type, Instance): - if ret_type.type.fullname() == 'builtins.object': - return - # Plausibly the method could have too few arguments, which would result - # in an error elsewhere. - if len(typ.arg_types) <= 2: - # TODO check self argument kind - - # Check for the issue described above. - arg_type = typ.arg_types[1] - other_method = nodes.normal_from_reverse_op[method] - if isinstance(arg_type, Instance): - if not arg_type.type.has_readable_member(other_method): - return - elif isinstance(arg_type, AnyType): - return - elif isinstance(arg_type, UnionType): - if not arg_type.has_readable_member(other_method): - return - else: - return - - typ2 = self.expr_checker.analyze_external_member_access( - other_method, arg_type, defn) - self.check_overlapping_op_methods( - typ, method, defn.info, - typ2, other_method, cast(Instance, arg_type), - defn) - - def check_overlapping_op_methods(self, - reverse_type: CallableType, - reverse_name: str, - reverse_class: TypeInfo, - forward_type: Type, - forward_name: str, - forward_base: Instance, - context: Context) -> None: - """Check for overlapping method and reverse method signatures. - - Assume reverse method has valid argument count and kinds. - """ - - # Reverse operator method that overlaps unsafely with the - # forward operator method can result in type unsafety. This is - # similar to overlapping overload variants. - # - # This example illustrates the issue: - # - # class X: pass - # class A: - # def __add__(self, x: X) -> int: - # if isinstance(x, X): - # return 1 - # return NotImplemented - # class B: - # def __radd__(self, x: A) -> str: return 'x' - # class C(X, B): pass - # def f(b: B) -> None: - # A() + b # Result is 1, even though static type seems to be str! - # f(C()) - # - # The reason for the problem is that B and X are overlapping - # types, and the return types are different. Also, if the type - # of x in __radd__ would not be A, the methods could be - # non-overlapping. - - if isinstance(forward_type, CallableType): - # TODO check argument kinds - if len(forward_type.arg_types) < 1: - # Not a valid operator method -- can't succeed anyway. - return - - # Construct normalized function signatures corresponding to the - # operator methods. The first argument is the left operand and the - # second operand is the right argument -- we switch the order of - # the arguments of the reverse method. - forward_tweaked = CallableType( - [forward_base, forward_type.arg_types[0]], - [nodes.ARG_POS] * 2, - [None] * 2, - forward_type.ret_type, - forward_type.fallback, - name=forward_type.name) - reverse_args = reverse_type.arg_types - reverse_tweaked = CallableType( - [reverse_args[1], reverse_args[0]], - [nodes.ARG_POS] * 2, - [None] * 2, - reverse_type.ret_type, - fallback=self.named_type('builtins.function'), - name=reverse_type.name) - - if is_unsafe_overlapping_signatures(forward_tweaked, - reverse_tweaked): - self.msg.operator_method_signatures_overlap( - reverse_class.name(), reverse_name, - forward_base.type.name(), forward_name, context) - elif isinstance(forward_type, Overloaded): - for item in forward_type.items(): - self.check_overlapping_op_methods( - reverse_type, reverse_name, reverse_class, - item, forward_name, forward_base, context) - elif not isinstance(forward_type, AnyType): - self.msg.forward_operator_not_callable(forward_name, context) - - def check_inplace_operator_method(self, defn: FuncBase) -> None: - """Check an inplace operator method such as __iadd__. - - They cannot arbitrarily overlap with __add__. - """ - method = defn.name() - if method not in nodes.inplace_operator_methods: - return - typ = self.method_type(defn) - cls = defn.info - other_method = '__' + method[3:] - if cls.has_readable_member(other_method): - instance = self_type(cls) - typ2 = self.expr_checker.analyze_external_member_access( - other_method, instance, defn) - fail = False - if isinstance(typ2, FunctionLike): - if not is_more_general_arg_prefix(typ, typ2): - fail = True - else: - # TODO overloads - fail = True - if fail: - self.msg.signatures_incompatible(method, other_method, defn) - - def check_getattr_method(self, typ: CallableType, context: Context) -> None: - method_type = CallableType([AnyType(), self.named_type('builtins.str')], - [nodes.ARG_POS, nodes.ARG_POS], - [None], - AnyType(), - self.named_type('builtins.function')) - if not is_subtype(typ, method_type): - self.msg.invalid_signature(typ, context) - - def expand_typevars(self, defn: FuncItem, - typ: CallableType) -> List[Tuple[FuncItem, CallableType]]: - # TODO use generator - subst = [] # type: List[List[Tuple[TypeVarId, Type]]] - tvars = typ.variables or [] - tvars = tvars[:] - if defn.info: - # Class type variables - tvars += defn.info.defn.type_vars or [] - for tvar in tvars: - if tvar.values: - subst.append([(tvar.id, value) - for value in tvar.values]) - if subst: - result = [] # type: List[Tuple[FuncItem, CallableType]] - for substitutions in itertools.product(*subst): - mapping = dict(substitutions) - expanded = cast(CallableType, expand_type(typ, mapping)) - result.append((expand_func(defn, mapping), expanded)) - return result - else: - return [(defn, typ)] - - def check_method_override(self, defn: FuncBase) -> None: - """Check if function definition is compatible with base classes.""" - # Check against definitions in base classes. - for base in defn.info.mro[1:]: - self.check_method_or_accessor_override_for_base(defn, base) - - def check_method_or_accessor_override_for_base(self, defn: FuncBase, - base: TypeInfo) -> None: - """Check if method definition is compatible with a base class.""" - if base: - name = defn.name() - if name not in ('__init__', '__new__'): - # Check method override (__init__ and __new__ are special). - self.check_method_override_for_base_with_name(defn, name, base) - if name in nodes.inplace_operator_methods: - # Figure out the name of the corresponding operator method. - method = '__' + name[3:] - # An inplace operator method such as __iadd__ might not be - # always introduced safely if a base class defined __add__. - # TODO can't come up with an example where this is - # necessary; now it's "just in case" - self.check_method_override_for_base_with_name(defn, method, - base) - - def check_method_override_for_base_with_name( - self, defn: FuncBase, name: str, base: TypeInfo) -> None: - base_attr = base.names.get(name) - if base_attr: - # The name of the method is defined in the base class. - - # Construct the type of the overriding method. - typ = self.method_type(defn) - # Map the overridden method type to subtype context so that - # it can be checked for compatibility. - original_type = base_attr.type - if original_type is None and isinstance(base_attr.node, - FuncDef): - original_type = self.function_type(base_attr.node) - if isinstance(original_type, FunctionLike): - original = map_type_from_supertype( - method_type(original_type), - defn.info, base) - # Check that the types are compatible. - # TODO overloaded signatures - self.check_override(typ, - cast(FunctionLike, original), - defn.name(), - name, - base.name(), - defn) - else: - assert original_type is not None - self.msg.signature_incompatible_with_supertype( - defn.name(), name, base.name(), defn) - - def check_override(self, override: FunctionLike, original: FunctionLike, - name: str, name_in_super: str, supertype: str, - node: Context) -> None: - """Check a method override with given signatures. - - Arguments: - override: The signature of the overriding method. - original: The signature of the original supertype method. - name: The name of the subtype. This and the next argument are - only used for generating error messages. - supertype: The name of the supertype. - """ - # Use boolean variable to clarify code. - fail = False - if not is_subtype(override, original): - fail = True - elif (not isinstance(original, Overloaded) and - isinstance(override, Overloaded) and - name in nodes.reverse_op_methods.keys()): - # Operator method overrides cannot introduce overloading, as - # this could be unsafe with reverse operator methods. - fail = True - - if fail: - emitted_msg = False - if (isinstance(override, CallableType) and - isinstance(original, CallableType) and - len(override.arg_types) == len(original.arg_types) and - override.min_args == original.min_args): - # Give more detailed messages for the common case of both - # signatures having the same number of arguments and no - # overloads. - - # override might have its own generic function type - # variables. If an argument or return type of override - # does not have the correct subtyping relationship - # with the original type even after these variables - # are erased, then it is definitely an incompatiblity. - - override_ids = override.type_var_ids() - - def erase_override(t: Type) -> Type: - return erase_typevars(t, ids_to_erase=override_ids) - - for i in range(len(override.arg_types)): - if not is_subtype(original.arg_types[i], - erase_override(override.arg_types[i])): - self.msg.argument_incompatible_with_supertype( - i + 1, name, name_in_super, supertype, node) - emitted_msg = True - - if not is_subtype(erase_override(override.ret_type), - original.ret_type): - self.msg.return_type_incompatible_with_supertype( - name, name_in_super, supertype, node) - emitted_msg = True - - if not emitted_msg: - # Fall back to generic incompatibility message. - self.msg.signature_incompatible_with_supertype( - name, name_in_super, supertype, node) - - def visit_class_def(self, defn: ClassDef) -> Type: - """Type check a class definition.""" - typ = defn.info - self.errors.push_type(defn.name) - self.enter_partial_types() - old_binder = self.binder - self.binder = ConditionalTypeBinder() - with self.binder.frame_context(): - self.accept(defn.defs) - self.binder = old_binder - if not hasattr(defn, 'incompatible_baseclass'): - # Otherwise we've already found errors; more errors are not useful - self.check_multiple_inheritance(typ) - self.leave_partial_types() - self.errors.pop_type() - - def check_multiple_inheritance(self, typ: TypeInfo) -> None: - """Check for multiple inheritance related errors.""" - - if len(typ.bases) <= 1: - # No multiple inheritance. - return - # Verify that inherited attributes are compatible. - mro = typ.mro[1:] - for i, base in enumerate(mro): - for name in base.names: - for base2 in mro[i + 1:]: - # We only need to check compatibility of attributes from classes not - # in a subclass relationship. For subclasses, normal (single inheritance) - # checks suffice (these are implemented elsewhere). - if name in base2.names and base2 not in base.mro: - self.check_compatibility(name, base, base2, typ) - # Verify that base class layouts are compatible. - builtin_bases = [nearest_builtin_ancestor(base.type) - for base in typ.bases] - for base1 in builtin_bases: - for base2 in builtin_bases: - if not (base1 in base2.mro or base2 in base1.mro): - self.fail(messages.INSTANCE_LAYOUT_CONFLICT, typ) - - def check_compatibility(self, name: str, base1: TypeInfo, - base2: TypeInfo, ctx: Context) -> None: - """Check if attribute name in base1 is compatible with base2 in multiple inheritance. - - Assume base1 comes before base2 in the MRO, and that base1 and base2 don't have - a direct subclass relationship (i.e., the compatibility requirement only derives from - multiple inheritance). - """ - if name == '__init__': - # __init__ can be incompatible -- it's a special case. - return - first = base1[name] - second = base2[name] - first_type = first.type - if first_type is None and isinstance(first.node, FuncDef): - first_type = self.function_type(first.node) - second_type = second.type - if second_type is None and isinstance(second.node, FuncDef): - second_type = self.function_type(second.node) - # TODO: What if some classes are generic? - if (isinstance(first_type, FunctionLike) and - isinstance(second_type, FunctionLike)): - # Method override - first_sig = method_type(first_type) - second_sig = method_type(second_type) - ok = is_subtype(first_sig, second_sig) - elif first_type and second_type: - ok = is_equivalent(first_type, second_type) - else: - if first_type is None: - self.msg.cannot_determine_type_in_base(name, base1.name(), ctx) - if second_type is None: - self.msg.cannot_determine_type_in_base(name, base2.name(), ctx) - ok = True - if not ok: - self.msg.base_class_definitions_incompatible(name, base1, base2, - ctx) - - def visit_import_from(self, node: ImportFrom) -> Type: - self.check_import(node) - - def visit_import_all(self, node: ImportAll) -> Type: - self.check_import(node) - - def check_import(self, node: ImportBase) -> Type: - for assign in node.assignments: - lvalue = assign.lvalues[0] - lvalue_type, _, __ = self.check_lvalue(lvalue) - if lvalue_type is None: - # TODO: This is broken. - lvalue_type = AnyType() - message = '{} "{}"'.format(messages.INCOMPATIBLE_IMPORT_OF, - cast(NameExpr, assign.rvalue).name) - self.check_simple_assignment(lvalue_type, assign.rvalue, node, - msg=message, lvalue_name='local name', - rvalue_name='imported name') - - # - # Statements - # - - def visit_block(self, b: Block) -> Type: - if b.is_unreachable: - return None - for s in b.body: - self.accept(s) - if self.binder.breaking_out: - break - - def visit_assignment_stmt(self, s: AssignmentStmt) -> Type: - """Type check an assignment statement. - - Handle all kinds of assignment statements (simple, indexed, multiple). - """ - self.check_assignment(s.lvalues[-1], s.rvalue, s.type is None) - - if len(s.lvalues) > 1: - # Chained assignment (e.g. x = y = ...). - # Make sure that rvalue type will not be reinferred. - if s.rvalue not in self.type_map: - self.accept(s.rvalue) - rvalue = self.temp_node(self.type_map[s.rvalue], s) - for lv in s.lvalues[:-1]: - self.check_assignment(lv, rvalue, s.type is None) - - def check_assignment(self, lvalue: Node, rvalue: Node, infer_lvalue_type: bool = True) -> None: - """Type check a single assignment: lvalue = rvalue.""" - if isinstance(lvalue, TupleExpr) or isinstance(lvalue, ListExpr): - ltuple = cast(Union[TupleExpr, ListExpr], lvalue) - - self.check_assignment_to_multiple_lvalues(ltuple.items, rvalue, lvalue, - infer_lvalue_type) - else: - lvalue_type, index_lvalue, inferred = self.check_lvalue(lvalue) - if lvalue_type: - if isinstance(lvalue_type, PartialType) and lvalue_type.type is None: - # Try to infer a proper type for a variable with a partial None type. - rvalue_type = self.accept(rvalue) - if isinstance(rvalue_type, NoneTyp): - # This doesn't actually provide any additional information -- multiple - # None initializers preserve the partial None type. - return - if is_valid_inferred_type(rvalue_type): - var = lvalue_type.var - partial_types = self.find_partial_types(var) - if partial_types is not None: - if not self.current_node_deferred: - if experiments.STRICT_OPTIONAL: - var.type = UnionType.make_simplified_union( - [rvalue_type, NoneTyp()]) - else: - var.type = rvalue_type - else: - var.type = None - del partial_types[var] - # Try to infer a partial type. No need to check the return value, as - # an error will be reported elsewhere. - self.infer_partial_type(lvalue_type.var, lvalue, rvalue_type) - return - if (is_literal_none(rvalue) and - isinstance(lvalue, NameExpr) and - isinstance(lvalue.node, Var) and - lvalue.node.is_initialized_in_class): - # Allow None's to be assigned to class variables with non-Optional types. - rvalue_type = lvalue_type - else: - rvalue_type = self.check_simple_assignment(lvalue_type, rvalue, lvalue) - - if rvalue_type and infer_lvalue_type: - self.binder.assign_type(lvalue, - rvalue_type, - lvalue_type, - self.typing_mode_weak()) - elif index_lvalue: - self.check_indexed_assignment(index_lvalue, rvalue, rvalue) - - if inferred: - self.infer_variable_type(inferred, lvalue, self.accept(rvalue), - rvalue) - - def check_assignment_to_multiple_lvalues(self, lvalues: List[Node], rvalue: Node, - context: Context, - infer_lvalue_type: bool = True) -> None: - if isinstance(rvalue, TupleExpr) or isinstance(rvalue, ListExpr): - # Recursively go into Tuple or List expression rhs instead of - # using the type of rhs, because this allowed more fine grained - # control in cases like: a, b = [int, str] where rhs would get - # type List[object] - - rvalues = cast(Union[TupleExpr, ListExpr], rvalue).items - - if self.check_rvalue_count_in_assignment(lvalues, len(rvalues), context): - star_index = next((i for i, lv in enumerate(lvalues) if - isinstance(lv, StarExpr)), len(lvalues)) - - left_lvs = lvalues[:star_index] - star_lv = cast(StarExpr, - lvalues[star_index]) if star_index != len(lvalues) else None - right_lvs = lvalues[star_index + 1:] - - left_rvs, star_rvs, right_rvs = self.split_around_star( - rvalues, star_index, len(lvalues)) - - lr_pairs = list(zip(left_lvs, left_rvs)) - if star_lv: - rv_list = ListExpr(star_rvs) - rv_list.set_line(rvalue.get_line()) - lr_pairs.append((star_lv.expr, rv_list)) - lr_pairs.extend(zip(right_lvs, right_rvs)) - - for lv, rv in lr_pairs: - self.check_assignment(lv, rv, infer_lvalue_type) - else: - self.check_multi_assignment(lvalues, rvalue, context, infer_lvalue_type) - - def check_rvalue_count_in_assignment(self, lvalues: List[Node], rvalue_count: int, - context: Context) -> bool: - if any(isinstance(lvalue, StarExpr) for lvalue in lvalues): - if len(lvalues) - 1 > rvalue_count: - self.msg.wrong_number_values_to_unpack(rvalue_count, - len(lvalues) - 1, context) - return False - elif rvalue_count != len(lvalues): - self.msg.wrong_number_values_to_unpack(rvalue_count, - len(lvalues), context) - return False - return True - - def check_multi_assignment(self, lvalues: List[Node], - rvalue: Node, - context: Context, - infer_lvalue_type: bool = True, - msg: str = None) -> None: - """Check the assignment of one rvalue to a number of lvalues.""" - - # Infer the type of an ordinary rvalue expression. - rvalue_type = self.accept(rvalue) # TODO maybe elsewhere; redundant - undefined_rvalue = False - - if isinstance(rvalue_type, AnyType): - for lv in lvalues: - if isinstance(lv, StarExpr): - lv = lv.expr - self.check_assignment(lv, self.temp_node(AnyType(), context), infer_lvalue_type) - elif isinstance(rvalue_type, TupleType): - self.check_multi_assignment_from_tuple(lvalues, rvalue, rvalue_type, - context, undefined_rvalue, infer_lvalue_type) - else: - self.check_multi_assignment_from_iterable(lvalues, rvalue_type, - context, infer_lvalue_type) - - def check_multi_assignment_from_tuple(self, lvalues: List[Node], rvalue: Node, - rvalue_type: TupleType, context: Context, - undefined_rvalue: bool, - infer_lvalue_type: bool = True) -> None: - if self.check_rvalue_count_in_assignment(lvalues, len(rvalue_type.items), context): - star_index = next((i for i, lv in enumerate(lvalues) - if isinstance(lv, StarExpr)), len(lvalues)) - - left_lvs = lvalues[:star_index] - star_lv = cast(StarExpr, lvalues[star_index]) if star_index != len(lvalues) else None - right_lvs = lvalues[star_index + 1:] - - if not undefined_rvalue: - # Infer rvalue again, now in the correct type context. - lvalue_type = self.lvalue_type_for_inference(lvalues, rvalue_type) - rvalue_type = cast(TupleType, self.accept(rvalue, lvalue_type)) - - left_rv_types, star_rv_types, right_rv_types = self.split_around_star( - rvalue_type.items, star_index, len(lvalues)) - - for lv, rv_type in zip(left_lvs, left_rv_types): - self.check_assignment(lv, self.temp_node(rv_type, context), infer_lvalue_type) - if star_lv: - nodes = [self.temp_node(rv_type, context) for rv_type in star_rv_types] - list_expr = ListExpr(nodes) - list_expr.set_line(context.get_line()) - self.check_assignment(star_lv.expr, list_expr, infer_lvalue_type) - for lv, rv_type in zip(right_lvs, right_rv_types): - self.check_assignment(lv, self.temp_node(rv_type, context), infer_lvalue_type) - - def lvalue_type_for_inference(self, lvalues: List[Node], rvalue_type: TupleType) -> Type: - star_index = next((i for i, lv in enumerate(lvalues) - if isinstance(lv, StarExpr)), len(lvalues)) - left_lvs = lvalues[:star_index] - star_lv = cast(StarExpr, lvalues[star_index]) if star_index != len(lvalues) else None - right_lvs = lvalues[star_index + 1:] - left_rv_types, star_rv_types, right_rv_types = self.split_around_star( - rvalue_type.items, star_index, len(lvalues)) - - type_parameters = [] # type: List[Type] - - def append_types_for_inference(lvs: List[Node], rv_types: List[Type]) -> None: - for lv, rv_type in zip(lvs, rv_types): - sub_lvalue_type, index_expr, inferred = self.check_lvalue(lv) - if sub_lvalue_type: - type_parameters.append(sub_lvalue_type) - else: # index lvalue - # TODO Figure out more precise type context, probably - # based on the type signature of the _set method. - type_parameters.append(rv_type) - - append_types_for_inference(left_lvs, left_rv_types) - - if star_lv: - sub_lvalue_type, index_expr, inferred = self.check_lvalue(star_lv.expr) - if sub_lvalue_type: - type_parameters.extend([sub_lvalue_type] * len(star_rv_types)) - else: # index lvalue - # TODO Figure out more precise type context, probably - # based on the type signature of the _set method. - type_parameters.extend(star_rv_types) - - append_types_for_inference(right_lvs, right_rv_types) - - return TupleType(type_parameters, self.named_type('builtins.tuple')) - - def split_around_star(self, items: List[T], star_index: int, - length: int) -> Tuple[List[T], List[T], List[T]]: - """Splits a list of items in three to match another list of length 'length' - that contains a starred expression at 'star_index' in the following way: - - star_index = 2, length = 5 (i.e., [a,b,*,c,d]), items = [1,2,3,4,5,6,7] - returns in: ([1,2], [3,4,5], [6,7]) - """ - nr_right_of_star = length - star_index - 1 - right_index = nr_right_of_star if -nr_right_of_star != 0 else len(items) - left = items[:star_index] - star = items[star_index:right_index] - right = items[right_index:] - return (left, star, right) - - def type_is_iterable(self, type: Type) -> bool: - return (is_subtype(type, self.named_generic_type('typing.Iterable', - [AnyType()])) and - isinstance(type, Instance)) - - def check_multi_assignment_from_iterable(self, lvalues: List[Node], rvalue_type: Type, - context: Context, - infer_lvalue_type: bool = True) -> None: - if self.type_is_iterable(rvalue_type): - item_type = self.iterable_item_type(cast(Instance, rvalue_type)) - for lv in lvalues: - if isinstance(lv, StarExpr): - self.check_assignment(lv.expr, self.temp_node(rvalue_type, context), - infer_lvalue_type) - else: - self.check_assignment(lv, self.temp_node(item_type, context), - infer_lvalue_type) - else: - self.msg.type_not_iterable(rvalue_type, context) - - def check_lvalue(self, lvalue: Node) -> Tuple[Type, IndexExpr, Var]: - lvalue_type = None # type: Type - index_lvalue = None # type: IndexExpr - inferred = None # type: Var - - if self.is_definition(lvalue): - if isinstance(lvalue, NameExpr): - inferred = cast(Var, lvalue.node) - assert isinstance(inferred, Var) - else: - m = cast(MemberExpr, lvalue) - self.accept(m.expr) - inferred = m.def_var - elif isinstance(lvalue, IndexExpr): - index_lvalue = lvalue - elif isinstance(lvalue, MemberExpr): - lvalue_type = self.expr_checker.analyze_ordinary_member_access(lvalue, - True) - self.store_type(lvalue, lvalue_type) - elif isinstance(lvalue, NameExpr): - lvalue_type = self.expr_checker.analyze_ref_expr(lvalue, lvalue=True) - self.store_type(lvalue, lvalue_type) - elif isinstance(lvalue, TupleExpr) or isinstance(lvalue, ListExpr): - lv = cast(Union[TupleExpr, ListExpr], lvalue) - types = [self.check_lvalue(sub_expr)[0] for sub_expr in lv.items] - lvalue_type = TupleType(types, self.named_type('builtins.tuple')) - else: - lvalue_type = self.accept(lvalue) - - return lvalue_type, index_lvalue, inferred - - def is_definition(self, s: Node) -> bool: - if isinstance(s, NameExpr): - if s.is_def: - return True - # If the node type is not defined, this must the first assignment - # that we process => this is a definition, even though the semantic - # analyzer did not recognize this as such. This can arise in code - # that uses isinstance checks, if type checking of the primary - # definition is skipped due to an always False type check. - node = s.node - if isinstance(node, Var): - return node.type is None - elif isinstance(s, MemberExpr): - return s.is_def - return False - - def infer_variable_type(self, name: Var, lvalue: Node, - init_type: Type, context: Context) -> None: - """Infer the type of initialized variables from initializer type.""" - if self.typing_mode_weak(): - self.set_inferred_type(name, lvalue, AnyType()) - self.binder.assign_type(lvalue, init_type, self.binder.get_declaration(lvalue), True) - elif isinstance(init_type, Void): - self.check_not_void(init_type, context) - self.set_inference_error_fallback_type(name, lvalue, init_type, context) - elif isinstance(init_type, DeletedType): - self.msg.deleted_as_rvalue(init_type, context) - elif not is_valid_inferred_type(init_type): - # We cannot use the type of the initialization expression for full type - # inference (it's not specific enough), but we might be able to give - # partial type which will be made more specific later. A partial type - # gets generated in assignment like 'x = []' where item type is not known. - if not self.infer_partial_type(name, lvalue, init_type): - self.fail(messages.NEED_ANNOTATION_FOR_VAR, context) - self.set_inference_error_fallback_type(name, lvalue, init_type, context) - else: - # Infer type of the target. - - # Make the type more general (strip away function names etc.). - init_type = strip_type(init_type) - - self.set_inferred_type(name, lvalue, init_type) - - def infer_partial_type(self, name: Var, lvalue: Node, init_type: Type) -> bool: - if isinstance(init_type, (NoneTyp, UninhabitedType)): - partial_type = PartialType(None, name, [init_type]) - elif isinstance(init_type, Instance): - fullname = init_type.type.fullname() - if (isinstance(lvalue, NameExpr) and - (fullname == 'builtins.list' or - fullname == 'builtins.set' or - fullname == 'builtins.dict') and - all(isinstance(t, (NoneTyp, UninhabitedType)) for t in init_type.args)): - partial_type = PartialType(init_type.type, name, init_type.args) - else: - return False - else: - return False - self.set_inferred_type(name, lvalue, partial_type) - self.partial_types[-1][name] = lvalue - return True - - def set_inferred_type(self, var: Var, lvalue: Node, type: Type) -> None: - """Store inferred variable type. - - Store the type to both the variable node and the expression node that - refers to the variable (lvalue). If var is None, do nothing. - """ - if var and not self.current_node_deferred: - var.type = type - self.store_type(lvalue, type) - - def set_inference_error_fallback_type(self, var: Var, lvalue: Node, type: Type, - context: Context) -> None: - """If errors on context line are ignored, store dummy type for variable. - - If a program ignores error on type inference error, the variable should get some - inferred type so that if can used later on in the program. Example: - - x = [] # type: ignore - x.append(1) # Should be ok! - - We implement this here by giving x a valid type (Any). - """ - if context.get_line() in self.errors.ignored_lines[self.errors.file]: - self.set_inferred_type(var, lvalue, AnyType()) - - def narrow_type_from_binder(self, expr: Node, known_type: Type) -> Type: - if expr.literal >= LITERAL_TYPE: - restriction = self.binder.get(expr) - if restriction: - ans = meet_simple(known_type, restriction) - return ans - return known_type - - def check_simple_assignment(self, lvalue_type: Type, rvalue: Node, - context: Node, - msg: str = messages.INCOMPATIBLE_TYPES_IN_ASSIGNMENT, - lvalue_name: str = 'variable', - rvalue_name: str = 'expression') -> Type: - if self.is_stub and isinstance(rvalue, EllipsisExpr): - # '...' is always a valid initializer in a stub. - return AnyType() - else: - rvalue_type = self.accept(rvalue, lvalue_type) - if isinstance(rvalue_type, DeletedType): - self.msg.deleted_as_rvalue(rvalue_type, context) - if self.typing_mode_weak(): - return rvalue_type - if isinstance(lvalue_type, DeletedType): - self.msg.deleted_as_lvalue(lvalue_type, context) - else: - self.check_subtype(rvalue_type, lvalue_type, context, msg, - '{} has type'.format(rvalue_name), - '{} has type'.format(lvalue_name)) - return rvalue_type - - def check_indexed_assignment(self, lvalue: IndexExpr, - rvalue: Node, context: Context) -> None: - """Type check indexed assignment base[index] = rvalue. - - The lvalue argument is the base[index] expression. - """ - self.try_infer_partial_type_from_indexed_assignment(lvalue, rvalue) - basetype = self.accept(lvalue.base) - method_type = self.expr_checker.analyze_external_member_access( - '__setitem__', basetype, context) - lvalue.method_type = method_type - self.expr_checker.check_call(method_type, [lvalue.index, rvalue], - [nodes.ARG_POS, nodes.ARG_POS], - context) - - def try_infer_partial_type_from_indexed_assignment( - self, lvalue: IndexExpr, rvalue: Node) -> None: - # TODO: Should we share some of this with try_infer_partial_type? - if isinstance(lvalue.base, RefExpr) and isinstance(lvalue.base.node, Var): - var = lvalue.base.node - if isinstance(var.type, PartialType): - type_type = var.type.type - if type_type is None: - return # The partial type is None. - partial_types = self.find_partial_types(var) - if partial_types is None: - return - typename = type_type.fullname() - if typename == 'builtins.dict': - # TODO: Don't infer things twice. - key_type = self.accept(lvalue.index) - value_type = self.accept(rvalue) - full_key_type = UnionType.make_simplified_union( - [key_type, var.type.inner_types[0]]) - full_value_type = UnionType.make_simplified_union( - [value_type, var.type.inner_types[1]]) - if (is_valid_inferred_type(full_key_type) and - is_valid_inferred_type(full_value_type)): - if not self.current_node_deferred: - var.type = self.named_generic_type('builtins.dict', - [full_key_type, full_value_type]) - del partial_types[var] - - def visit_expression_stmt(self, s: ExpressionStmt) -> Type: - self.accept(s.expr) - - def visit_return_stmt(self, s: ReturnStmt) -> Type: - """Type check a return statement.""" - self.binder.breaking_out = True - if self.is_within_function(): - if self.function_stack[-1].is_generator: - return_type = self.get_generator_return_type(self.return_types[-1]) - else: - return_type = self.return_types[-1] - - if s.expr: - # Return with a value. - typ = self.accept(s.expr, return_type) - # Returning a value of type Any is always fine. - if isinstance(typ, AnyType): - return None - - if isinstance(return_type, Void): - # Lambdas are allowed to have a Void return. - # Functions returning a value of type None are allowed to have a Void return. - if isinstance(self.function_stack[-1], FuncExpr) or isinstance(typ, NoneTyp): - return None - self.fail(messages.NO_RETURN_VALUE_EXPECTED, s) - else: - self.check_subtype( - subtype_label='got', - subtype=typ, - supertype_label='expected', - supertype=return_type, - context=s, - msg=messages.INCOMPATIBLE_RETURN_VALUE_TYPE) - else: - # Empty returns are valid in Generators with Any typed returns. - if (self.function_stack[-1].is_generator and isinstance(return_type, AnyType)): - return None - - if isinstance(return_type, Void): - return None - - if isinstance(return_type, AnyType): - return None - - if self.typing_mode_full(): - self.fail(messages.RETURN_VALUE_EXPECTED, s) - - def wrap_generic_type(self, typ: Instance, rtyp: Instance, check_type: - str, context: Context) -> Type: - n_diff = self.count_nested_types(rtyp, check_type) - self.count_nested_types(typ, - check_type) - if n_diff == 1: - return self.named_generic_type(check_type, [typ]) - elif n_diff == 0 or n_diff > 1: - self.fail(messages.INCOMPATIBLE_RETURN_VALUE_TYPE - + ": expected {}, got {}".format(rtyp, typ), context) - return typ - return typ - - def count_nested_types(self, typ: Instance, check_type: str) -> int: - c = 0 - while is_subtype(typ, self.named_type(check_type)): - c += 1 - typ = map_instance_to_supertype(self.named_generic_type(check_type, typ.args), - self.lookup_typeinfo(check_type)) - if typ.args: - typ = cast(Instance, typ.args[0]) - else: - return c - return c - - def visit_if_stmt(self, s: IfStmt) -> Type: - """Type check an if statement.""" - breaking_out = True - # This frame records the knowledge from previous if/elif clauses not being taken. - with self.binder.frame_context(): - for e, b in zip(s.expr, s.body): - t = self.accept(e) - self.check_not_void(t, e) - if_map, else_map = find_isinstance_check( - e, self.type_map, - self.typing_mode_weak() - ) - if if_map is None: - # The condition is always false - # XXX should issue a warning? - pass - else: - # Only type check body if the if condition can be true. - with self.binder.frame_context(2): - if if_map: - for var, type in if_map.items(): - self.binder.push(var, type) - - self.accept(b) - breaking_out = breaking_out and self.binder.last_pop_breaking_out - - if else_map: - for var, type in else_map.items(): - self.binder.push(var, type) - if else_map is None: - # The condition is always true => remaining elif/else blocks - # can never be reached. - - # Might also want to issue a warning - # print("Warning: isinstance always true") - break - else: # Didn't break => can't prove one of the conditions is always true - with self.binder.frame_context(2): - if s.else_body: - self.accept(s.else_body) - breaking_out = breaking_out and self.binder.last_pop_breaking_out - if breaking_out: - self.binder.breaking_out = True - return None - - def visit_while_stmt(self, s: WhileStmt) -> Type: - """Type check a while statement.""" - self.accept_loop(IfStmt([s.expr], [s.body], None), s.else_body) - - def visit_operator_assignment_stmt(self, - s: OperatorAssignmentStmt) -> Type: - """Type check an operator assignment statement, e.g. x += 1.""" - lvalue_type = self.accept(s.lvalue) - method = infer_operator_assignment_method(lvalue_type, s.op) - rvalue_type, method_type = self.expr_checker.check_op( - method, lvalue_type, s.rvalue, s) - - if isinstance(s.lvalue, IndexExpr): - self.check_indexed_assignment(s.lvalue, s.rvalue, s.rvalue) - else: - if not is_subtype(rvalue_type, lvalue_type): - self.msg.incompatible_operator_assignment(s.op, s) - - def visit_assert_stmt(self, s: AssertStmt) -> Type: - self.accept(s.expr) - - # If this is asserting some isinstance check, bind that type in the following code - true_map, _ = find_isinstance_check( - s.expr, self.type_map, - self.typing_mode_weak() - ) - - if true_map: - for var, type in true_map.items(): - self.binder.push(var, type) - - def visit_raise_stmt(self, s: RaiseStmt) -> Type: - """Type check a raise statement.""" - self.binder.breaking_out = True - if s.expr: - self.type_check_raise(s.expr, s) - if s.from_expr: - self.type_check_raise(s.from_expr, s) - - def type_check_raise(self, e: Node, s: RaiseStmt) -> None: - typ = self.accept(e) - if isinstance(typ, FunctionLike): - if typ.is_type_obj(): - # Cases like "raise/from ExceptionClass". - typeinfo = typ.type_object() - base = self.lookup_typeinfo('builtins.BaseException') - if base in typeinfo.mro or typeinfo.fallback_to_any: - # Good! - return - # Else fall back to the checks below (which will fail). - if isinstance(typ, TupleType) and self.options.python_version[0] == 2: - # allow `raise type, value, traceback` - # https://docs.python.org/2/reference/simple_stmts.html#the-raise-statement - # TODO: Also check tuple item types. - if len(typ.items) in (2, 3): - return - if isinstance(typ, Instance) and typ.type.fallback_to_any: - # OK! - return - self.check_subtype(typ, - self.named_type('builtins.BaseException'), s, - messages.INVALID_EXCEPTION) - - def visit_try_stmt(self, s: TryStmt) -> Type: - """Type check a try statement.""" - # Our enclosing frame will get the result if the try/except falls through. - # This one gets all possible intermediate states - with self.binder.frame_context(): - if s.finally_body: - self.binder.try_frames.add(len(self.binder.frames) - 1) - breaking_out = self.visit_try_without_finally(s) - self.binder.try_frames.remove(len(self.binder.frames) - 1) - # First we check finally_body is type safe for all intermediate frames - self.accept(s.finally_body) - breaking_out = breaking_out or self.binder.breaking_out - else: - breaking_out = self.visit_try_without_finally(s) - - if not breaking_out and s.finally_body: - # Then we try again for the more restricted set of options that can fall through - self.accept(s.finally_body) - self.binder.breaking_out = breaking_out - return None - - def visit_try_without_finally(self, s: TryStmt) -> bool: - """Type check a try statement, ignoring the finally block. - - Return whether we are guaranteed to be breaking out. - Otherwise, it will place the results possible frames of - that don't break out into self.binder.frames[-2]. - """ - breaking_out = True - # This frame records the possible states that exceptions can leave variables in - # during the try: block - with self.binder.frame_context(): - with self.binder.frame_context(3): - self.binder.try_frames.add(len(self.binder.frames) - 2) - self.accept(s.body) - self.binder.try_frames.remove(len(self.binder.frames) - 2) - if s.else_body: - self.accept(s.else_body) - breaking_out = breaking_out and self.binder.last_pop_breaking_out - for i in range(len(s.handlers)): - with self.binder.frame_context(3): - if s.types[i]: - t = self.visit_except_handler_test(s.types[i]) - if s.vars[i]: - # To support local variables, we make this a definition line, - # causing assignment to set the variable's type. - s.vars[i].is_def = True - self.check_assignment(s.vars[i], self.temp_node(t, s.vars[i])) - self.accept(s.handlers[i]) - if s.vars[i]: - # Exception variables are deleted in python 3 but not python 2. - # But, since it's bad form in python 2 and the type checking - # wouldn't work very well, we delete it anyway. - - # Unfortunately, this doesn't let us detect usage before the - # try/except block. - if self.options.python_version[0] >= 3: - source = s.vars[i].name - else: - source = ('(exception variable "{}", which we do not accept outside' - 'except: blocks even in python 2)'.format(s.vars[i].name)) - var = cast(Var, s.vars[i].node) - var.type = DeletedType(source=source) - self.binder.cleanse(s.vars[i]) - breaking_out = breaking_out and self.binder.last_pop_breaking_out - return breaking_out - - def visit_except_handler_test(self, n: Node) -> Type: - """Type check an exception handler test clause.""" - type = self.accept(n) - - all_types = [] # type: List[Type] - test_types = type.items if isinstance(type, TupleType) else [type] - - for ttype in test_types: - if isinstance(ttype, AnyType): - all_types.append(ttype) - continue - - if not isinstance(ttype, FunctionLike): - self.fail(messages.INVALID_EXCEPTION_TYPE, n) - return AnyType() - - item = ttype.items()[0] - ret_type = item.ret_type - if not (is_subtype(ret_type, self.named_type('builtins.BaseException')) - and item.is_type_obj()): - self.fail(messages.INVALID_EXCEPTION_TYPE, n) - return AnyType() - - all_types.append(ret_type) - - return UnionType.make_simplified_union(all_types) - - def visit_for_stmt(self, s: ForStmt) -> Type: - """Type check a for statement.""" - item_type = self.analyze_iterable_item_type(s.expr) - self.analyze_index_variables(s.index, item_type, s) - self.accept_loop(s.body, s.else_body) - - def analyze_iterable_item_type(self, expr: Node) -> Type: - """Analyse iterable expression and return iterator item type.""" - iterable = self.accept(expr) - - self.check_not_void(iterable, expr) - if isinstance(iterable, TupleType): - if experiments.STRICT_OPTIONAL: - joined = UninhabitedType() # type: Type - else: - joined = NoneTyp() - for item in iterable.items: - joined = join_types(joined, item) - if isinstance(joined, ErrorType): - self.fail(messages.CANNOT_INFER_ITEM_TYPE, expr) - return AnyType() - return joined - else: - # Non-tuple iterable. - self.check_subtype(iterable, - self.named_generic_type('typing.Iterable', - [AnyType()]), - expr, messages.ITERABLE_EXPECTED) - - echk = self.expr_checker - method = echk.analyze_external_member_access('__iter__', iterable, - expr) - iterator = echk.check_call(method, [], [], expr)[0] - if self.options.python_version[0] >= 3: - nextmethod = '__next__' - else: - nextmethod = 'next' - method = echk.analyze_external_member_access(nextmethod, iterator, - expr) - return echk.check_call(method, [], [], expr)[0] - - def analyze_index_variables(self, index: Node, item_type: Type, - context: Context) -> None: - """Type check or infer for loop or list comprehension index vars.""" - self.check_assignment(index, self.temp_node(item_type, context)) - - def visit_del_stmt(self, s: DelStmt) -> Type: - if isinstance(s.expr, IndexExpr): - e = s.expr - m = MemberExpr(e.base, '__delitem__') - m.line = s.line - c = CallExpr(m, [e.index], [nodes.ARG_POS], [None]) - c.line = s.line - return c.accept(self) - else: - def flatten(t: Node) -> List[Node]: - """Flatten a nested sequence of tuples/lists into one list of nodes.""" - if isinstance(t, TupleExpr) or isinstance(t, ListExpr): - t = cast(Union[TupleExpr, ListExpr], t) - return [b for a in t.items for b in flatten(a)] - else: - return [t] - - s.expr.accept(self) - for elt in flatten(s.expr): - if isinstance(elt, NameExpr): - self.binder.assign_type(elt, - DeletedType(source=elt.name), - self.binder.get_declaration(elt), - self.typing_mode_weak()) - return None - - def visit_decorator(self, e: Decorator) -> Type: - for d in e.decorators: - if isinstance(d, RefExpr): - if d.fullname == 'typing.no_type_check': - e.var.type = AnyType() - e.var.is_ready = True - return NoneTyp() - - e.func.accept(self) - sig = self.function_type(e.func) # type: Type - # Process decorators from the inside out. - for i in range(len(e.decorators)): - n = len(e.decorators) - 1 - i - d = e.decorators[n] - if isinstance(d, NameExpr) and d.fullname == 'typing.overload': - self.fail('Single overload definition, multiple required', e) - continue - dec = self.accept(d) - temp = self.temp_node(sig) - sig, t2 = self.expr_checker.check_call(dec, [temp], - [nodes.ARG_POS], e) - sig = cast(FunctionLike, sig) - sig = set_callable_name(sig, e.func) - e.var.type = sig - e.var.is_ready = True - if e.func.is_property: - self.check_incompatible_property_override(e) - - def check_incompatible_property_override(self, e: Decorator) -> None: - if not e.var.is_settable_property and e.func.info is not None: - name = e.func.name() - for base in e.func.info.mro[1:]: - base_attr = base.names.get(name) - if not base_attr: - continue - if (isinstance(base_attr.node, OverloadedFuncDef) and - base_attr.node.is_property and - base_attr.node.items[0].var.is_settable_property): - self.fail(messages.READ_ONLY_PROPERTY_OVERRIDES_READ_WRITE, e) - - def visit_with_stmt(self, s: WithStmt) -> Type: - echk = self.expr_checker - for expr, target in zip(s.expr, s.target): - ctx = self.accept(expr) - enter = echk.analyze_external_member_access('__enter__', ctx, expr) - obj = echk.check_call(enter, [], [], expr)[0] - if target: - self.check_assignment(target, self.temp_node(obj, expr)) - exit = echk.analyze_external_member_access('__exit__', ctx, expr) - arg = self.temp_node(AnyType(), expr) - echk.check_call(exit, [arg] * 3, [nodes.ARG_POS] * 3, expr) - self.accept(s.body) - - def visit_print_stmt(self, s: PrintStmt) -> Type: - for arg in s.args: - self.accept(arg) - if s.target: - target_type = self.accept(s.target) - if not isinstance(target_type, NoneTyp): - # TODO: Also verify the type of 'write'. - self.expr_checker.analyze_external_member_access('write', target_type, s.target) - - # - # Expressions - # - - def visit_name_expr(self, e: NameExpr) -> Type: - return self.expr_checker.visit_name_expr(e) - - def visit_call_expr(self, e: CallExpr) -> Type: - return self.expr_checker.visit_call_expr(e) - - def visit_yield_from_expr(self, e: YieldFromExpr) -> Type: - return_type = self.return_types[-1] - subexpr_type = self.accept(e.expr, return_type) - iter_type = None # type: Type - - # Check that the expr is an instance of Iterable and get the type of the iterator produced - # by __iter__. - if isinstance(subexpr_type, AnyType): - iter_type = AnyType() - elif (isinstance(subexpr_type, Instance) and - is_subtype(subexpr_type, self.named_type('typing.Iterable'))): - iter_method_type = self.expr_checker.analyze_external_member_access( - '__iter__', - subexpr_type, - AnyType()) - - generic_generator_type = self.named_generic_type('typing.Generator', - [AnyType(), AnyType(), AnyType()]) - iter_type, _ = self.expr_checker.check_call(iter_method_type, [], [], - context=generic_generator_type) - else: - self.msg.yield_from_invalid_operand_type(subexpr_type, e) - iter_type = AnyType() - - # Check that the iterator's item type matches the type yielded by the Generator function - # containing this `yield from` expression. - expected_item_type = self.get_generator_yield_type(return_type) - actual_item_type = self.get_generator_yield_type(iter_type) - - self.check_subtype(actual_item_type, expected_item_type, e, - messages.INCOMPATIBLE_TYPES_IN_YIELD_FROM, - 'actual type', 'expected type') - - # Determine the type of the entire yield from expression. - if (isinstance(iter_type, Instance) and - iter_type.type.fullname() == 'typing.Generator'): - return self.get_generator_return_type(iter_type) - else: - # Non-Generators don't return anything from `yield from` expressions. - return Void() - - def visit_member_expr(self, e: MemberExpr) -> Type: - return self.expr_checker.visit_member_expr(e) - - def visit_break_stmt(self, s: BreakStmt) -> Type: - self.binder.breaking_out = True - self.binder.allow_jump(self.binder.loop_frames[-1] - 1) - return None - - def visit_continue_stmt(self, s: ContinueStmt) -> Type: - self.binder.breaking_out = True - self.binder.allow_jump(self.binder.loop_frames[-1]) - return None - - def visit_int_expr(self, e: IntExpr) -> Type: - return self.expr_checker.visit_int_expr(e) - - def visit_str_expr(self, e: StrExpr) -> Type: - return self.expr_checker.visit_str_expr(e) - - def visit_bytes_expr(self, e: BytesExpr) -> Type: - return self.expr_checker.visit_bytes_expr(e) - - def visit_unicode_expr(self, e: UnicodeExpr) -> Type: - return self.expr_checker.visit_unicode_expr(e) - - def visit_float_expr(self, e: FloatExpr) -> Type: - return self.expr_checker.visit_float_expr(e) - - def visit_complex_expr(self, e: ComplexExpr) -> Type: - return self.expr_checker.visit_complex_expr(e) - - def visit_ellipsis(self, e: EllipsisExpr) -> Type: - return self.expr_checker.visit_ellipsis(e) - - def visit_op_expr(self, e: OpExpr) -> Type: - return self.expr_checker.visit_op_expr(e) - - def visit_comparison_expr(self, e: ComparisonExpr) -> Type: - return self.expr_checker.visit_comparison_expr(e) - - def visit_unary_expr(self, e: UnaryExpr) -> Type: - return self.expr_checker.visit_unary_expr(e) - - def visit_index_expr(self, e: IndexExpr) -> Type: - return self.expr_checker.visit_index_expr(e) - - def visit_cast_expr(self, e: CastExpr) -> Type: - return self.expr_checker.visit_cast_expr(e) - - def visit_reveal_type_expr(self, e: RevealTypeExpr) -> Type: - return self.expr_checker.visit_reveal_type_expr(e) - - def visit_super_expr(self, e: SuperExpr) -> Type: - return self.expr_checker.visit_super_expr(e) - - def visit_type_application(self, e: TypeApplication) -> Type: - return self.expr_checker.visit_type_application(e) - - def visit_type_alias_expr(self, e: TypeAliasExpr) -> Type: - return self.expr_checker.visit_type_alias_expr(e) - - def visit_type_var_expr(self, e: TypeVarExpr) -> Type: - # TODO: Perhaps return a special type used for type variables only? - return AnyType() - - def visit_namedtuple_expr(self, e: NamedTupleExpr) -> Type: - # TODO: Perhaps return a type object type? - return AnyType() - - def visit_list_expr(self, e: ListExpr) -> Type: - return self.expr_checker.visit_list_expr(e) - - def visit_set_expr(self, e: SetExpr) -> Type: - return self.expr_checker.visit_set_expr(e) - - def visit_tuple_expr(self, e: TupleExpr) -> Type: - return self.expr_checker.visit_tuple_expr(e) - - def visit_dict_expr(self, e: DictExpr) -> Type: - return self.expr_checker.visit_dict_expr(e) - - def visit_slice_expr(self, e: SliceExpr) -> Type: - return self.expr_checker.visit_slice_expr(e) - - def visit_func_expr(self, e: FuncExpr) -> Type: - return self.expr_checker.visit_func_expr(e) - - def visit_list_comprehension(self, e: ListComprehension) -> Type: - return self.expr_checker.visit_list_comprehension(e) - - def visit_set_comprehension(self, e: SetComprehension) -> Type: - return self.expr_checker.visit_set_comprehension(e) - - def visit_generator_expr(self, e: GeneratorExpr) -> Type: - return self.expr_checker.visit_generator_expr(e) - - def visit_dictionary_comprehension(self, e: DictionaryComprehension) -> Type: - return self.expr_checker.visit_dictionary_comprehension(e) - - def visit_temp_node(self, e: TempNode) -> Type: - return e.type - - def visit_conditional_expr(self, e: ConditionalExpr) -> Type: - return self.expr_checker.visit_conditional_expr(e) - - def visit_backquote_expr(self, e: BackquoteExpr) -> Type: - return self.expr_checker.visit_backquote_expr(e) - - def visit_yield_expr(self, e: YieldExpr) -> Type: - return_type = self.return_types[-1] - expected_item_type = self.get_generator_yield_type(return_type) - if e.expr is None: - if (not (isinstance(expected_item_type, Void) or - isinstance(expected_item_type, AnyType)) - and self.typing_mode_full()): - self.fail(messages.YIELD_VALUE_EXPECTED, e) - else: - actual_item_type = self.accept(e.expr, expected_item_type) - self.check_subtype(actual_item_type, expected_item_type, e, - messages.INCOMPATIBLE_TYPES_IN_YIELD, - 'actual type', 'expected type') - return self.get_generator_receive_type(return_type) - - # - # Helpers - # - - def check_subtype(self, subtype: Type, supertype: Type, context: Context, - msg: str = messages.INCOMPATIBLE_TYPES, - subtype_label: str = None, - supertype_label: str = None) -> None: - """Generate an error if the subtype is not compatible with - supertype.""" - if not is_subtype(subtype, supertype): - if isinstance(subtype, Void): - self.msg.does_not_return_value(subtype, context) - else: - extra_info = [] # type: List[str] - if subtype_label is not None or supertype_label is not None: - subtype_str, supertype_str = self.msg.format_distinctly(subtype, supertype) - if subtype_label is not None: - extra_info.append(subtype_label + ' ' + subtype_str) - if supertype_label is not None: - extra_info.append(supertype_label + ' ' + supertype_str) - if extra_info: - msg += ' (' + ', '.join(extra_info) + ')' - self.fail(msg, context) - - def named_type(self, name: str) -> Instance: - """Return an instance type with type given by the name and no - type arguments. For example, named_type('builtins.object') - produces the object type. - """ - # Assume that the name refers to a type. - sym = self.lookup_qualified(name) - return Instance(cast(TypeInfo, sym.node), []) - - def named_generic_type(self, name: str, args: List[Type]) -> Instance: - """Return an instance with the given name and type arguments. - - Assume that the number of arguments is correct. Assume that - the name refers to a compatible generic type. - """ - return Instance(self.lookup_typeinfo(name), args) - - def lookup_typeinfo(self, fullname: str) -> TypeInfo: - # Assume that the name refers to a class. - sym = self.lookup_qualified(fullname) - return cast(TypeInfo, sym.node) - - def type_type(self) -> Instance: - """Return instance type 'type'.""" - return self.named_type('builtins.type') - - def object_type(self) -> Instance: - """Return instance type 'object'.""" - return self.named_type('builtins.object') - - def bool_type(self) -> Instance: - """Return instance type 'bool'.""" - return self.named_type('builtins.bool') - - def str_type(self) -> Instance: - """Return instance type 'str'.""" - return self.named_type('builtins.str') - - def check_type_equivalency(self, t1: Type, t2: Type, node: Context, - msg: str = messages.INCOMPATIBLE_TYPES) -> None: - """Generate an error if the types are not equivalent. The - dynamic type is equivalent with all types. - """ - if not is_equivalent(t1, t2): - self.fail(msg, node) - - def store_type(self, node: Node, typ: Type) -> None: - """Store the type of a node in the type map.""" - self.type_map[node] = typ - - def typing_mode_none(self) -> bool: - if self.is_dynamic_function() and not self.options.check_untyped_defs: - return not self.weak_opts - elif self.function_stack: - return False - else: - return False - - def typing_mode_weak(self) -> bool: - if self.is_dynamic_function() and not self.options.check_untyped_defs: - return bool(self.weak_opts) - elif self.function_stack: - return False - else: - return 'global' in self.weak_opts - - def typing_mode_full(self) -> bool: - if self.is_dynamic_function() and not self.options.check_untyped_defs: - return False - elif self.function_stack: - return True - else: - return 'global' not in self.weak_opts - - def is_dynamic_function(self) -> bool: - return len(self.dynamic_funcs) > 0 and self.dynamic_funcs[-1] - - def lookup(self, name: str, kind: int) -> SymbolTableNode: - """Look up a definition from the symbol table with the given name. - TODO remove kind argument - """ - if name in self.globals: - return self.globals[name] - else: - b = self.globals.get('__builtins__', None) - if b: - table = cast(MypyFile, b.node).names - if name in table: - return table[name] - raise KeyError('Failed lookup: {}'.format(name)) - - def lookup_qualified(self, name: str) -> SymbolTableNode: - if '.' not in name: - return self.lookup(name, GDEF) # FIX kind - else: - parts = name.split('.') - n = self.modules[parts[0]] - for i in range(1, len(parts) - 1): - n = cast(MypyFile, n.names.get(parts[i], None).node) - return n.names[parts[-1]] - - def enter_partial_types(self) -> None: - """Push a new scope for collecting partial types.""" - self.partial_types.append({}) - - def leave_partial_types(self) -> None: - """Pop partial type scope. - - Also report errors for variables which still have partial - types, i.e. we couldn't infer a complete type. - """ - partial_types = self.partial_types.pop() - if not self.current_node_deferred: - for var, context in partial_types.items(): - if experiments.STRICT_OPTIONAL and cast(PartialType, var.type).type is None: - # None partial type: assume variable is intended to have type None - var.type = NoneTyp() - else: - self.msg.fail(messages.NEED_ANNOTATION_FOR_VAR, context) - var.type = AnyType() - - def find_partial_types(self, var: Var) -> Optional[Dict[Var, Context]]: - for partial_types in reversed(self.partial_types): - if var in partial_types: - return partial_types - return None - - def is_within_function(self) -> bool: - """Are we currently type checking within a function? - - I.e. not at class body or at the top level. - """ - return self.return_types != [] - - def check_not_void(self, typ: Type, context: Context) -> None: - """Generate an error if the type is Void.""" - if isinstance(typ, Void): - self.msg.does_not_return_value(typ, context) - - def temp_node(self, t: Type, context: Context = None) -> Node: - """Create a temporary node with the given, fixed type.""" - temp = TempNode(t) - if context: - temp.set_line(context.get_line()) - return temp - - def fail(self, msg: str, context: Context) -> None: - """Produce an error message.""" - self.msg.fail(msg, context) - - def iterable_item_type(self, instance: Instance) -> Type: - iterable = map_instance_to_supertype( - instance, - self.lookup_typeinfo('typing.Iterable')) - return iterable.args[0] - - def function_type(self, func: FuncBase) -> FunctionLike: - return function_type(func, self.named_type('builtins.function')) - - def method_type(self, func: FuncBase) -> FunctionLike: - return method_type_with_fallback(func, self.named_type('builtins.function')) - - -# Data structure returned by find_isinstance_check representing -# information learned from the truth or falsehood of a condition. The -# dict maps nodes representing expressions like 'a[0].x' to their -# refined types under the assumption that the condition has a -# particular truth value. A value of None means that the condition can -# never have that truth value. - -# NB: The keys of this dict are nodes in the original source program, -# which are compared by reference equality--effectively, being *the -# same* expression of the program, not just two identical expressions -# (such as two references to the same variable). TODO: it would -# probably be better to have the dict keyed by the nodes' literal_hash -# field instead. - -# NB: This should be `TypeMap = Optional[Dict[Node, Type]]`! -# But see https://github.com/python/mypy/issues/1637 -TypeMap = Dict[Node, Type] - - -def conditional_type_map(expr: Node, - current_type: Optional[Type], - proposed_type: Optional[Type], - *, - weak: bool = False - ) -> Tuple[TypeMap, TypeMap]: - """Takes in an expression, the current type of the expression, and a - proposed type of that expression. - - Returns a 2-tuple: The first element is a map from the expression to - the proposed type, if the expression can be the proposed type. The - second element is a map from the expression to the type it would hold - if it was not the proposed type, if any.""" - if proposed_type: - if current_type: - if is_proper_subtype(current_type, proposed_type): - return {expr: proposed_type}, None - elif not is_overlapping_types(current_type, proposed_type): - return None, {expr: current_type} - else: - remaining_type = restrict_subtype_away(current_type, proposed_type) - return {expr: proposed_type}, {expr: remaining_type} - else: - return {expr: proposed_type}, {} - else: - # An isinstance check, but we don't understand the type - if weak: - return {expr: AnyType()}, {expr: current_type} - else: - return {}, {} - - -def is_literal_none(n: Node) -> bool: - return isinstance(n, NameExpr) and n.fullname == 'builtins.None' - - -def and_conditional_maps(m1: TypeMap, m2: TypeMap) -> TypeMap: - """Calculate what information we can learn from the truth of (e1 and e2) - in terms of the information that we can learn from the truth of e1 and - the truth of e2. - """ - - if m1 is None or m2 is None: - # One of the conditions can never be true. - return None - # Both conditions can be true; combine the information. Anything - # we learn from either conditions's truth is valid. If the same - # expression's type is refined by both conditions, we somewhat - # arbitrarily give precedence to m2. (In the future, we could use - # an intersection type.) - result = m2.copy() - m2_keys = set(n2.literal_hash for n2 in m2) - for n1 in m1: - if n1.literal_hash not in m2_keys: - result[n1] = m1[n1] - return result - - -def or_conditional_maps(m1: TypeMap, m2: TypeMap) -> TypeMap: - """Calculate what information we can learn from the truth of (e1 or e2) - in terms of the information that we can learn from the truth of e1 and - the truth of e2. - """ - - if m1 is None: - return m2 - if m2 is None: - return m1 - # Both conditions can be true. Combine information about - # expressions whose type is refined by both conditions. (We do not - # learn anything about expressions whose type is refined by only - # one condition.) - result = {} - for n1 in m1: - for n2 in m2: - if n1.literal_hash == n2.literal_hash: - result[n1] = UnionType.make_simplified_union([m1[n1], m2[n2]]) - return result - - -def find_isinstance_check(node: Node, - type_map: Dict[Node, Type], - weak: bool=False - ) -> Tuple[TypeMap, TypeMap]: - """Find any isinstance checks (within a chain of ands). Includes - implicit and explicit checks for None. - - Return value is a map of variables to their types if the condition - is true and a map of variables to their types if the condition is false. - - If either of the values in the tuple is None, then that particular - branch can never occur. - - Guaranteed to not return None, None. (But may return {}, {}) - """ - if isinstance(node, CallExpr): - if refers_to_fullname(node.callee, 'builtins.isinstance'): - expr = node.args[0] - if expr.literal == LITERAL_TYPE: - vartype = type_map[expr] - type = get_isinstance_type(node.args[1], type_map) - return conditional_type_map(expr, vartype, type, weak=weak) - elif (isinstance(node, ComparisonExpr) and any(is_literal_none(n) for n in node.operands) and - experiments.STRICT_OPTIONAL): - # Check for `x is None` and `x is not None`. - is_not = node.operators == ['is not'] - if is_not or node.operators == ['is']: - if_vars = {} # type: Dict[Node, Type] - else_vars = {} # type: Dict[Node, Type] - for expr in node.operands: - if expr.literal == LITERAL_TYPE and not is_literal_none(expr) and expr in type_map: - # This should only be true at most once: there should be - # two elements in node.operands, and at least one of them - # should represent a None. - vartype = type_map[expr] - if_vars, else_vars = conditional_type_map(expr, vartype, NoneTyp(), weak=weak) - break - - if is_not: - if_vars, else_vars = else_vars, if_vars - return if_vars, else_vars - elif isinstance(node, RefExpr) and experiments.STRICT_OPTIONAL: - # The type could be falsy, so we can't deduce anything new about the else branch - vartype = type_map[node] - _, if_vars = conditional_type_map(node, vartype, NoneTyp(), weak=weak) - return if_vars, {} - elif isinstance(node, OpExpr) and node.op == 'and': - left_if_vars, left_else_vars = find_isinstance_check( - node.left, - type_map, - weak, - ) - - right_if_vars, right_else_vars = find_isinstance_check( - node.right, - type_map, - weak, - ) - - # (e1 and e2) is true if both e1 and e2 are true, - # and false if at least one of e1 and e2 is false. - return (and_conditional_maps(left_if_vars, right_if_vars), - or_conditional_maps(left_else_vars, right_else_vars)) - elif isinstance(node, OpExpr) and node.op == 'or': - left_if_vars, left_else_vars = find_isinstance_check( - node.left, - type_map, - weak, - ) - - right_if_vars, right_else_vars = find_isinstance_check( - node.right, - type_map, - weak, - ) - - # (e1 or e2) is true if at least one of e1 or e2 is true, - # and false if both e1 and e2 are false. - return (or_conditional_maps(left_if_vars, right_if_vars), - and_conditional_maps(left_else_vars, right_else_vars)) - elif isinstance(node, UnaryExpr) and node.op == 'not': - left, right = find_isinstance_check(node.expr, type_map, weak) - return right, left - - # Not a supported isinstance check - return {}, {} - - -def get_isinstance_type(node: Node, type_map: Dict[Node, Type]) -> Type: - type = type_map[node] - - if isinstance(type, TupleType): - all_types = type.items - else: - all_types = [type] - - types = [] # type: List[Type] - - for type in all_types: - if isinstance(type, FunctionLike): - if type.is_type_obj(): - # Type variables may be present -- erase them, which is the best - # we can do (outside disallowing them here). - type = erase_typevars(type.items()[0].ret_type) - - types.append(type) - - if len(types) == 0: - return None - elif len(types) == 1: - return types[0] - else: - return UnionType(types) - - -def expand_node(defn: Node, map: Dict[TypeVarId, Type]) -> Node: - visitor = TypeTransformVisitor(map) - return defn.accept(visitor) - - -def expand_func(defn: FuncItem, map: Dict[TypeVarId, Type]) -> FuncItem: - return cast(FuncItem, expand_node(defn, map)) - - -class TypeTransformVisitor(TransformVisitor): - def __init__(self, map: Dict[TypeVarId, Type]) -> None: - super().__init__() - self.map = map - - def type(self, type: Type) -> Type: - return expand_type(type, self.map) - - -def is_unsafe_overlapping_signatures(signature: Type, other: Type) -> bool: - """Check if two signatures may be unsafely overlapping. - - Two signatures s and t are overlapping if both can be valid for the same - statically typed values and the return types are incompatible. - - Assume calls are first checked against 'signature', then against 'other'. - Thus if 'signature' is more general than 'other', there is no unsafe - overlapping. - - TODO If argument types vary covariantly, the return type may vary - covariantly as well. - """ - if isinstance(signature, CallableType): - if isinstance(other, CallableType): - # TODO varargs - # TODO keyword args - # TODO erasure - # TODO allow to vary covariantly - # Check if the argument counts are overlapping. - min_args = max(signature.min_args, other.min_args) - max_args = min(len(signature.arg_types), len(other.arg_types)) - if min_args > max_args: - # Argument counts are not overlapping. - return False - # Signatures are overlapping iff if they are overlapping for the - # smallest common argument count. - for i in range(min_args): - t1 = signature.arg_types[i] - t2 = other.arg_types[i] - if not is_overlapping_types(t1, t2): - return False - # All arguments types for the smallest common argument count are - # overlapping => the signature is overlapping. The overlapping is - # safe if the return types are identical. - if is_same_type(signature.ret_type, other.ret_type): - return False - # If the first signature has more general argument types, the - # latter will never be called - if is_more_general_arg_prefix(signature, other): - return False - return not is_more_precise_signature(signature, other) - return True - - -def is_more_general_arg_prefix(t: FunctionLike, s: FunctionLike) -> bool: - """Does t have wider arguments than s?""" - # TODO should an overload with additional items be allowed to be more - # general than one with fewer items (or just one item)? - # TODO check argument kinds - if isinstance(t, CallableType): - if isinstance(s, CallableType): - return all(is_proper_subtype(args, argt) - for argt, args in zip(t.arg_types, s.arg_types)) - elif isinstance(t, FunctionLike): - if isinstance(s, FunctionLike): - if len(t.items()) == len(s.items()): - return all(is_same_arg_prefix(items, itemt) - for items, itemt in zip(t.items(), s.items())) - return False - - -def is_same_arg_prefix(t: CallableType, s: CallableType) -> bool: - # TODO check argument kinds - return all(is_same_type(argt, args) - for argt, args in zip(t.arg_types, s.arg_types)) - - -def is_more_precise_signature(t: CallableType, s: CallableType) -> bool: - """Is t more precise than s? - - A signature t is more precise than s if all argument types and the return - type of t are more precise than the corresponding types in s. - - Assume that the argument kinds and names are compatible, and that the - argument counts are overlapping. - """ - # TODO generic function types - # Only consider the common prefix of argument types. - for argt, args in zip(t.arg_types, s.arg_types): - if not is_more_precise(argt, args): - return False - return is_more_precise(t.ret_type, s.ret_type) - - -def infer_operator_assignment_method(type: Type, operator: str) -> str: - """Return the method used for operator assignment for given value type. - - For example, if operator is '+', return '__iadd__' or '__add__' depending - on which method is supported by the type. - """ - method = nodes.op_methods[operator] - if isinstance(type, Instance): - if operator in nodes.ops_with_inplace_method: - inplace = '__i' + method[2:] - if type.type.has_readable_member(inplace): - method = inplace - return method - - -def is_valid_inferred_type(typ: Type) -> bool: - """Is an inferred type valid? - - Examples of invalid types include the None type or a type with a None component. - """ - if is_same_type(typ, NoneTyp()): - # With strict Optional checking, we *may* eventually infer NoneTyp, but - # we only do that if we can't infer a specific Optional type. This - # resolution happens in leave_partial_types when we pop a partial types - # scope. - return False - if is_same_type(typ, UninhabitedType()): - return False - elif isinstance(typ, Instance): - for arg in typ.args: - if not is_valid_inferred_type(arg): - return False - elif isinstance(typ, TupleType): - for item in typ.items: - if not is_valid_inferred_type(item): - return False - return True +"""Mypy type checker.""" + +import itertools +import contextlib +import os +import os.path + +from typing import ( + Any, Dict, Set, List, cast, Tuple, TypeVar, Union, Optional, NamedTuple +) + +from mypy.errors import Errors, report_internal_error +from mypy.nodes import ( + SymbolTable, Node, MypyFile, Var, + OverloadedFuncDef, FuncDef, FuncItem, FuncBase, TypeInfo, + ClassDef, GDEF, Block, AssignmentStmt, NameExpr, MemberExpr, IndexExpr, + TupleExpr, ListExpr, ExpressionStmt, ReturnStmt, IfStmt, + WhileStmt, OperatorAssignmentStmt, WithStmt, AssertStmt, + RaiseStmt, TryStmt, ForStmt, DelStmt, CallExpr, IntExpr, StrExpr, + BytesExpr, UnicodeExpr, FloatExpr, OpExpr, UnaryExpr, CastExpr, RevealTypeExpr, SuperExpr, + TypeApplication, DictExpr, SliceExpr, FuncExpr, TempNode, SymbolTableNode, + Context, ListComprehension, ConditionalExpr, GeneratorExpr, + Decorator, SetExpr, TypeVarExpr, PrintStmt, + LITERAL_TYPE, BreakStmt, ContinueStmt, ComparisonExpr, StarExpr, + YieldFromExpr, NamedTupleExpr, SetComprehension, + DictionaryComprehension, ComplexExpr, EllipsisExpr, TypeAliasExpr, + RefExpr, YieldExpr, BackquoteExpr, ImportFrom, ImportAll, ImportBase, + CONTRAVARIANT, COVARIANT +) +from mypy.nodes import function_type, method_type, method_type_with_fallback +from mypy import nodes +from mypy.types import ( + Type, AnyType, CallableType, Void, FunctionLike, Overloaded, TupleType, + Instance, NoneTyp, ErrorType, strip_type, + UnionType, TypeVarId, TypeVarType, PartialType, DeletedType, UninhabitedType +) +from mypy.sametypes import is_same_type +from mypy.messages import MessageBuilder +import mypy.checkexpr +from mypy.checkmember import map_type_from_supertype +from mypy import defaults +from mypy import messages +from mypy.subtypes import ( + is_subtype, is_equivalent, is_proper_subtype, + is_more_precise, restrict_subtype_away +) +from mypy.maptype import map_instance_to_supertype +from mypy.semanal import self_type, set_callable_name, refers_to_fullname +from mypy.erasetype import erase_typevars +from mypy.expandtype import expand_type_by_instance, expand_type +from mypy.visitor import NodeVisitor +from mypy.join import join_types +from mypy.treetransform import TransformVisitor +from mypy.meet import meet_simple, nearest_builtin_ancestor, is_overlapping_types +from mypy.binder import ConditionalTypeBinder +from mypy.options import Options + +from mypy import experiments + + +T = TypeVar('T') + + +# A node which is postponed to be type checked during the next pass. +DeferredNode = NamedTuple( + 'DeferredNode', + [ + ('node', Node), + ('context_type_name', Optional[str]), # Name of the surrounding class (for error messages) + ]) + + +class TypeChecker(NodeVisitor[Type]): + """Mypy type checker. + + Type check mypy source files that have been semantically analyzed. + """ + + # Are we type checking a stub? + is_stub = False + # Error message reporter + errors = None # type: Errors + # Utility for generating messages + msg = None # type: MessageBuilder + # Types of type checked nodes + type_map = None # type: Dict[Node, Type] + + # Helper for managing conditional types + binder = None # type: ConditionalTypeBinder + # Helper for type checking expressions + expr_checker = None # type: mypy.checkexpr.ExpressionChecker + + # Stack of function return types + return_types = None # type: List[Type] + # Type context for type inference + type_context = None # type: List[Type] + # Flags; true for dynamically typed functions + dynamic_funcs = None # type: List[bool] + # Stack of functions being type checked + function_stack = None # type: List[FuncItem] + # Do weak type checking in this file + weak_opts = set() # type: Set[str] + # Stack of collections of variables with partial types + partial_types = None # type: List[Dict[Var, Context]] + globals = None # type: SymbolTable + modules = None # type: Dict[str, MypyFile] + # Nodes that couldn't be checked because some types weren't available. We'll run + # another pass and try these again. + deferred_nodes = None # type: List[DeferredNode] + # Type checking pass number (0 = first pass) + pass_num = 0 + # Have we deferred the current function? If yes, don't infer additional + # types during this pass within the function. + current_node_deferred = False + is_typeshed_stub = False + options = None # type: Options + + def __init__(self, errors: Errors, modules: Dict[str, MypyFile], options: Options) -> None: + """Construct a type checker. + + Use errors to report type check errors. + """ + self.errors = errors + self.modules = modules + self.options = options + self.msg = MessageBuilder(errors, modules) + self.type_map = {} + self.binder = ConditionalTypeBinder() + self.expr_checker = mypy.checkexpr.ExpressionChecker(self, self.msg) + self.return_types = [] + self.type_context = [] + self.dynamic_funcs = [] + self.function_stack = [] + self.weak_opts = set() # type: Set[str] + self.partial_types = [] + self.deferred_nodes = [] + self.pass_num = 0 + self.current_node_deferred = False + + def visit_file(self, file_node: MypyFile, path: str) -> None: + """Type check a mypy file with the given path.""" + self.pass_num = 0 + self.is_stub = file_node.is_stub + self.errors.set_file(path) + self.globals = file_node.names + self.weak_opts = file_node.weak_opts + self.enter_partial_types() + # gross, but no other clear way to tell + self.is_typeshed_stub = self.is_stub and 'typeshed' in os.path.normpath(path).split(os.sep) + + for d in file_node.defs: + self.accept(d) + + self.leave_partial_types() + + if self.deferred_nodes: + self.check_second_pass() + + self.current_node_deferred = False + + all_ = self.globals.get('__all__') + if all_ is not None and all_.type is not None: + seq_str = self.named_generic_type('typing.Sequence', + [self.named_type('builtins.str')]) + if not is_subtype(all_.type, seq_str): + str_seq_s, all_s = self.msg.format_distinctly(seq_str, all_.type) + self.fail(messages.ALL_MUST_BE_SEQ_STR.format(str_seq_s, all_s), + all_.node) + + def check_second_pass(self): + """Run second pass of type checking which goes through deferred nodes.""" + self.pass_num = 1 + for node, type_name in self.deferred_nodes: + if type_name: + self.errors.push_type(type_name) + self.accept(node) + if type_name: + self.errors.pop_type() + self.deferred_nodes = [] + + def handle_cannot_determine_type(self, name: str, context: Context) -> None: + if self.pass_num == 0 and self.function_stack: + # Don't report an error yet. Just defer. + node = self.function_stack[-1] + if self.errors.type_name: + type_name = self.errors.type_name[-1] + else: + type_name = None + self.deferred_nodes.append(DeferredNode(node, type_name)) + # Set a marker so that we won't infer additional types in this + # function. Any inferred types could be bogus, because there's at + # least one type that we don't know. + self.current_node_deferred = True + else: + self.msg.cannot_determine_type(name, context) + + def accept(self, node: Node, type_context: Type = None) -> Type: + """Type check a node in the given type context.""" + self.type_context.append(type_context) + try: + typ = node.accept(self) + except Exception as err: + report_internal_error(err, self.errors.file, node.line) + self.type_context.pop() + self.store_type(node, typ) + if self.typing_mode_none(): + return AnyType() + else: + return typ + + def accept_loop(self, body: Node, else_body: Node = None) -> Type: + """Repeatedly type check a loop body until the frame doesn't change. + + Then check the else_body. + """ + # The outer frame accumulates the results of all iterations + with self.binder.frame_context(1) as outer_frame: + self.binder.push_loop_frame() + while True: + with self.binder.frame_context(1): + # We may skip each iteration + self.binder.options_on_return[-1].append(outer_frame) + self.accept(body) + if not self.binder.last_pop_changed: + break + self.binder.pop_loop_frame() + if else_body: + self.accept(else_body) + + # + # Definitions + # + + def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> Type: + num_abstract = 0 + if defn.is_property: + # HACK: Infer the type of the property. + self.visit_decorator(defn.items[0]) + for fdef in defn.items: + self.check_func_item(fdef.func, name=fdef.func.name()) + if fdef.func.is_abstract: + num_abstract += 1 + if num_abstract not in (0, len(defn.items)): + self.fail(messages.INCONSISTENT_ABSTRACT_OVERLOAD, defn) + if defn.info: + self.check_method_override(defn) + self.check_inplace_operator_method(defn) + self.check_overlapping_overloads(defn) + + def check_overlapping_overloads(self, defn: OverloadedFuncDef) -> None: + for i, item in enumerate(defn.items): + for j, item2 in enumerate(defn.items[i + 1:]): + # TODO overloads involving decorators + sig1 = self.function_type(item.func) + sig2 = self.function_type(item2.func) + if is_unsafe_overlapping_signatures(sig1, sig2): + self.msg.overloaded_signatures_overlap(i + 1, i + j + 2, + item.func) + + def is_generator_return_type(self, typ: Type) -> bool: + return is_subtype(self.named_generic_type('typing.Generator', + [AnyType(), AnyType(), AnyType()]), + typ) + + def get_generator_yield_type(self, return_type: Type) -> Type: + if isinstance(return_type, AnyType): + return AnyType() + elif not self.is_generator_return_type(return_type): + # If the function doesn't have a proper Generator (or superclass) return type, anything + # is permissible. + return AnyType() + elif not isinstance(return_type, Instance): + # Same as above, but written as a separate branch so the typechecker can understand. + return AnyType() + elif return_type.args: + return return_type.args[0] + else: + # If the function's declared supertype of Generator has no type + # parameters (i.e. is `object`), then the yielded values can't + # be accessed so any type is acceptable. + return AnyType() + + def get_generator_receive_type(self, return_type: Type) -> Type: + if isinstance(return_type, AnyType): + return AnyType() + elif not self.is_generator_return_type(return_type): + # If the function doesn't have a proper Generator (or superclass) return type, anything + # is permissible. + return AnyType() + elif not isinstance(return_type, Instance): + # Same as above, but written as a separate branch so the typechecker can understand. + return AnyType() + elif return_type.type.fullname() == 'typing.Generator': + # Generator is the only type which specifies the type of values it can receive. + return return_type.args[1] + else: + # `return_type` is a supertype of Generator, so callers won't be able to send it + # values. + return Void() + + def get_generator_return_type(self, return_type: Type) -> Type: + if isinstance(return_type, AnyType): + return AnyType() + elif not self.is_generator_return_type(return_type): + # If the function doesn't have a proper Generator (or superclass) return type, anything + # is permissible. + return AnyType() + elif not isinstance(return_type, Instance): + # Same as above, but written as a separate branch so the typechecker can understand. + return AnyType() + elif return_type.type.fullname() == 'typing.Generator': + # Generator is the only type which specifies the type of values it returns into + # `yield from` expressions. + return return_type.args[2] + else: + # `return_type` is supertype of Generator, so callers won't be able to see the return + # type when used in a `yield from` expression. + return AnyType() + + def visit_func_def(self, defn: FuncDef) -> Type: + """Type check a function definition.""" + self.check_func_item(defn, name=defn.name()) + if defn.info: + if not defn.is_dynamic(): + self.check_method_override(defn) + self.check_inplace_operator_method(defn) + if defn.original_def: + # Override previous definition. + new_type = self.function_type(defn) + if isinstance(defn.original_def, FuncDef): + # Function definition overrides function definition. + if not is_same_type(new_type, self.function_type(defn.original_def)): + self.msg.incompatible_conditional_function_def(defn) + else: + # Function definition overrides a variable initialized via assignment. + orig_type = defn.original_def.type + if orig_type is None: + # XXX This can be None, as happens in + # test_testcheck_TypeCheckSuite.testRedefinedFunctionInTryWithElse + self.msg.note("Internal mypy error checking function redefinition.", defn) + return None + if isinstance(orig_type, PartialType): + if orig_type.type is None: + # Ah this is a partial type. Give it the type of the function. + var = defn.original_def + partial_types = self.find_partial_types(var) + if partial_types is not None: + var.type = new_type + del partial_types[var] + else: + # Trying to redefine something like partial empty list as function. + self.fail(messages.INCOMPATIBLE_REDEFINITION, defn) + else: + # TODO: Update conditional type binder. + self.check_subtype(new_type, orig_type, defn, + messages.INCOMPATIBLE_REDEFINITION, + 'redefinition with type', + 'original type') + + def check_func_item(self, defn: FuncItem, + type_override: CallableType = None, + name: str = None) -> Type: + """Type check a function. + + If type_override is provided, use it as the function type. + """ + # We may be checking a function definition or an anonymous function. In + # the first case, set up another reference with the precise type. + fdef = None # type: FuncDef + if isinstance(defn, FuncDef): + fdef = defn + + self.function_stack.append(defn) + self.dynamic_funcs.append(defn.is_dynamic() and not type_override) + + if fdef: + self.errors.push_function(fdef.name()) + + self.enter_partial_types() + + typ = self.function_type(defn) + if type_override: + typ = type_override + if isinstance(typ, CallableType): + self.check_func_def(defn, typ, name) + else: + raise RuntimeError('Not supported') + + self.leave_partial_types() + + if fdef: + self.errors.pop_function() + + self.dynamic_funcs.pop() + self.function_stack.pop() + self.current_node_deferred = False + + def check_func_def(self, defn: FuncItem, typ: CallableType, name: str) -> None: + """Type check a function definition.""" + # Expand type variables with value restrictions to ordinary types. + for item, typ in self.expand_typevars(defn, typ): + old_binder = self.binder + self.binder = ConditionalTypeBinder() + with self.binder.frame_context(): + defn.expanded.append(item) + + # We may be checking a function definition or an anonymous + # function. In the first case, set up another reference with the + # precise type. + if isinstance(item, FuncDef): + fdef = item + else: + fdef = None + + if fdef: + # Check if __init__ has an invalid, non-None return type. + if (fdef.info and fdef.name() == '__init__' and + not isinstance(typ.ret_type, Void) and + not self.dynamic_funcs[-1]): + self.fail(messages.INIT_MUST_HAVE_NONE_RETURN_TYPE, + item.type) + + show_untyped = not self.is_typeshed_stub or self.options.warn_incomplete_stub + if self.options.disallow_untyped_defs and show_untyped: + # Check for functions with unspecified/not fully specified types. + def is_implicit_any(t: Type) -> bool: + return isinstance(t, AnyType) and t.implicit + + if fdef.type is None: + self.fail(messages.FUNCTION_TYPE_EXPECTED, fdef) + elif isinstance(fdef.type, CallableType): + if is_implicit_any(fdef.type.ret_type): + self.fail(messages.RETURN_TYPE_EXPECTED, fdef) + if any(is_implicit_any(t) for t in fdef.type.arg_types): + self.fail(messages.ARGUMENT_TYPE_EXPECTED, fdef) + + if name in nodes.reverse_op_method_set: + self.check_reverse_op_method(item, typ, name) + elif name == '__getattr__': + self.check_getattr_method(typ, defn) + + # Refuse contravariant return type variable + if isinstance(typ.ret_type, TypeVarType): + if typ.ret_type.variance == CONTRAVARIANT: + self.fail(messages.RETURN_TYPE_CANNOT_BE_CONTRAVARIANT, + typ.ret_type) + + # Check that Generator functions have the appropriate return type. + if defn.is_generator: + if not self.is_generator_return_type(typ.ret_type): + self.fail(messages.INVALID_RETURN_TYPE_FOR_GENERATOR, typ) + + # Python 2 generators aren't allowed to return values. + if (self.options.python_version[0] == 2 and + isinstance(typ.ret_type, Instance) and + typ.ret_type.type.fullname() == 'typing.Generator'): + if not (isinstance(typ.ret_type.args[2], Void) + or isinstance(typ.ret_type.args[2], AnyType)): + self.fail(messages.INVALID_GENERATOR_RETURN_ITEM_TYPE, typ) + + # Push return type. + self.return_types.append(typ.ret_type) + + # Store argument types. + for i in range(len(typ.arg_types)): + arg_type = typ.arg_types[i] + + # Refuse covariant parameter type variables + if isinstance(arg_type, TypeVarType): + if arg_type.variance == COVARIANT: + self.fail(messages.FUNCTION_PARAMETER_CANNOT_BE_COVARIANT, + arg_type) + + if typ.arg_kinds[i] == nodes.ARG_STAR: + # builtins.tuple[T] is typing.Tuple[T, ...] + arg_type = self.named_generic_type('builtins.tuple', + [arg_type]) + elif typ.arg_kinds[i] == nodes.ARG_STAR2: + arg_type = self.named_generic_type('builtins.dict', + [self.str_type(), + arg_type]) + item.arguments[i].variable.type = arg_type + + # Type check initialization expressions. + for arg in item.arguments: + init = arg.initialization_statement + if init: + self.accept(init) + + # Type check body in a new scope. + with self.binder.frame_context(): + self.accept(item.body) + + self.return_types.pop() + + self.binder = old_binder + + def check_reverse_op_method(self, defn: FuncItem, typ: CallableType, + method: str) -> None: + """Check a reverse operator method such as __radd__.""" + + # This used to check for some very obscure scenario. It now + # just decides whether it's worth calling + # check_overlapping_op_methods(). + + if method in ('__eq__', '__ne__'): + # These are defined for all objects => can't cause trouble. + return + + # With 'Any' or 'object' return type we are happy, since any possible + # return value is valid. + ret_type = typ.ret_type + if isinstance(ret_type, AnyType): + return + if isinstance(ret_type, Instance): + if ret_type.type.fullname() == 'builtins.object': + return + # Plausibly the method could have too few arguments, which would result + # in an error elsewhere. + if len(typ.arg_types) <= 2: + # TODO check self argument kind + + # Check for the issue described above. + arg_type = typ.arg_types[1] + other_method = nodes.normal_from_reverse_op[method] + if isinstance(arg_type, Instance): + if not arg_type.type.has_readable_member(other_method): + return + elif isinstance(arg_type, AnyType): + return + elif isinstance(arg_type, UnionType): + if not arg_type.has_readable_member(other_method): + return + else: + return + + typ2 = self.expr_checker.analyze_external_member_access( + other_method, arg_type, defn) + self.check_overlapping_op_methods( + typ, method, defn.info, + typ2, other_method, cast(Instance, arg_type), + defn) + + def check_overlapping_op_methods(self, + reverse_type: CallableType, + reverse_name: str, + reverse_class: TypeInfo, + forward_type: Type, + forward_name: str, + forward_base: Instance, + context: Context) -> None: + """Check for overlapping method and reverse method signatures. + + Assume reverse method has valid argument count and kinds. + """ + + # Reverse operator method that overlaps unsafely with the + # forward operator method can result in type unsafety. This is + # similar to overlapping overload variants. + # + # This example illustrates the issue: + # + # class X: pass + # class A: + # def __add__(self, x: X) -> int: + # if isinstance(x, X): + # return 1 + # return NotImplemented + # class B: + # def __radd__(self, x: A) -> str: return 'x' + # class C(X, B): pass + # def f(b: B) -> None: + # A() + b # Result is 1, even though static type seems to be str! + # f(C()) + # + # The reason for the problem is that B and X are overlapping + # types, and the return types are different. Also, if the type + # of x in __radd__ would not be A, the methods could be + # non-overlapping. + + if isinstance(forward_type, CallableType): + # TODO check argument kinds + if len(forward_type.arg_types) < 1: + # Not a valid operator method -- can't succeed anyway. + return + + # Construct normalized function signatures corresponding to the + # operator methods. The first argument is the left operand and the + # second operand is the right argument -- we switch the order of + # the arguments of the reverse method. + forward_tweaked = CallableType( + [forward_base, forward_type.arg_types[0]], + [nodes.ARG_POS] * 2, + [None] * 2, + forward_type.ret_type, + forward_type.fallback, + name=forward_type.name) + reverse_args = reverse_type.arg_types + reverse_tweaked = CallableType( + [reverse_args[1], reverse_args[0]], + [nodes.ARG_POS] * 2, + [None] * 2, + reverse_type.ret_type, + fallback=self.named_type('builtins.function'), + name=reverse_type.name) + + if is_unsafe_overlapping_signatures(forward_tweaked, + reverse_tweaked): + self.msg.operator_method_signatures_overlap( + reverse_class.name(), reverse_name, + forward_base.type.name(), forward_name, context) + elif isinstance(forward_type, Overloaded): + for item in forward_type.items(): + self.check_overlapping_op_methods( + reverse_type, reverse_name, reverse_class, + item, forward_name, forward_base, context) + elif not isinstance(forward_type, AnyType): + self.msg.forward_operator_not_callable(forward_name, context) + + def check_inplace_operator_method(self, defn: FuncBase) -> None: + """Check an inplace operator method such as __iadd__. + + They cannot arbitrarily overlap with __add__. + """ + method = defn.name() + if method not in nodes.inplace_operator_methods: + return + typ = self.method_type(defn) + cls = defn.info + other_method = '__' + method[3:] + if cls.has_readable_member(other_method): + instance = self_type(cls) + typ2 = self.expr_checker.analyze_external_member_access( + other_method, instance, defn) + fail = False + if isinstance(typ2, FunctionLike): + if not is_more_general_arg_prefix(typ, typ2): + fail = True + else: + # TODO overloads + fail = True + if fail: + self.msg.signatures_incompatible(method, other_method, defn) + + def check_getattr_method(self, typ: CallableType, context: Context) -> None: + method_type = CallableType([AnyType(), self.named_type('builtins.str')], + [nodes.ARG_POS, nodes.ARG_POS], + [None], + AnyType(), + self.named_type('builtins.function')) + if not is_subtype(typ, method_type): + self.msg.invalid_signature(typ, context) + + def expand_typevars(self, defn: FuncItem, + typ: CallableType) -> List[Tuple[FuncItem, CallableType]]: + # TODO use generator + subst = [] # type: List[List[Tuple[TypeVarId, Type]]] + tvars = typ.variables or [] + tvars = tvars[:] + if defn.info: + # Class type variables + tvars += defn.info.defn.type_vars or [] + for tvar in tvars: + if tvar.values: + subst.append([(tvar.id, value) + for value in tvar.values]) + if subst: + result = [] # type: List[Tuple[FuncItem, CallableType]] + for substitutions in itertools.product(*subst): + mapping = dict(substitutions) + expanded = cast(CallableType, expand_type(typ, mapping)) + result.append((expand_func(defn, mapping), expanded)) + return result + else: + return [(defn, typ)] + + def check_method_override(self, defn: FuncBase) -> None: + """Check if function definition is compatible with base classes.""" + # Check against definitions in base classes. + for base in defn.info.mro[1:]: + self.check_method_or_accessor_override_for_base(defn, base) + + def check_method_or_accessor_override_for_base(self, defn: FuncBase, + base: TypeInfo) -> None: + """Check if method definition is compatible with a base class.""" + if base: + name = defn.name() + if name not in ('__init__', '__new__'): + # Check method override (__init__ and __new__ are special). + self.check_method_override_for_base_with_name(defn, name, base) + if name in nodes.inplace_operator_methods: + # Figure out the name of the corresponding operator method. + method = '__' + name[3:] + # An inplace operator method such as __iadd__ might not be + # always introduced safely if a base class defined __add__. + # TODO can't come up with an example where this is + # necessary; now it's "just in case" + self.check_method_override_for_base_with_name(defn, method, + base) + + def check_method_override_for_base_with_name( + self, defn: FuncBase, name: str, base: TypeInfo) -> None: + base_attr = base.names.get(name) + if base_attr: + # The name of the method is defined in the base class. + + # Construct the type of the overriding method. + typ = self.method_type(defn) + # Map the overridden method type to subtype context so that + # it can be checked for compatibility. + original_type = base_attr.type + if original_type is None and isinstance(base_attr.node, + FuncDef): + original_type = self.function_type(base_attr.node) + if isinstance(original_type, FunctionLike): + original = map_type_from_supertype( + method_type(original_type), + defn.info, base) + # Check that the types are compatible. + # TODO overloaded signatures + self.check_override(typ, + cast(FunctionLike, original), + defn.name(), + name, + base.name(), + defn) + else: + assert original_type is not None + self.msg.signature_incompatible_with_supertype( + defn.name(), name, base.name(), defn) + + def check_override(self, override: FunctionLike, original: FunctionLike, + name: str, name_in_super: str, supertype: str, + node: Context) -> None: + """Check a method override with given signatures. + + Arguments: + override: The signature of the overriding method. + original: The signature of the original supertype method. + name: The name of the subtype. This and the next argument are + only used for generating error messages. + supertype: The name of the supertype. + """ + # Use boolean variable to clarify code. + fail = False + if not is_subtype(override, original): + fail = True + elif (not isinstance(original, Overloaded) and + isinstance(override, Overloaded) and + name in nodes.reverse_op_methods.keys()): + # Operator method overrides cannot introduce overloading, as + # this could be unsafe with reverse operator methods. + fail = True + + if fail: + emitted_msg = False + if (isinstance(override, CallableType) and + isinstance(original, CallableType) and + len(override.arg_types) == len(original.arg_types) and + override.min_args == original.min_args): + # Give more detailed messages for the common case of both + # signatures having the same number of arguments and no + # overloads. + + # override might have its own generic function type + # variables. If an argument or return type of override + # does not have the correct subtyping relationship + # with the original type even after these variables + # are erased, then it is definitely an incompatiblity. + + override_ids = override.type_var_ids() + + def erase_override(t: Type) -> Type: + return erase_typevars(t, ids_to_erase=override_ids) + + for i in range(len(override.arg_types)): + if not is_subtype(original.arg_types[i], + erase_override(override.arg_types[i])): + self.msg.argument_incompatible_with_supertype( + i + 1, name, name_in_super, supertype, node) + emitted_msg = True + + if not is_subtype(erase_override(override.ret_type), + original.ret_type): + self.msg.return_type_incompatible_with_supertype( + name, name_in_super, supertype, node) + emitted_msg = True + + if not emitted_msg: + # Fall back to generic incompatibility message. + self.msg.signature_incompatible_with_supertype( + name, name_in_super, supertype, node) + + def visit_class_def(self, defn: ClassDef) -> Type: + """Type check a class definition.""" + typ = defn.info + self.errors.push_type(defn.name) + self.enter_partial_types() + old_binder = self.binder + self.binder = ConditionalTypeBinder() + with self.binder.frame_context(): + self.accept(defn.defs) + self.binder = old_binder + if not hasattr(defn, 'incompatible_baseclass'): + # Otherwise we've already found errors; more errors are not useful + self.check_multiple_inheritance(typ) + self.leave_partial_types() + self.errors.pop_type() + + def check_multiple_inheritance(self, typ: TypeInfo) -> None: + """Check for multiple inheritance related errors.""" + + if len(typ.bases) <= 1: + # No multiple inheritance. + return + # Verify that inherited attributes are compatible. + mro = typ.mro[1:] + for i, base in enumerate(mro): + for name in base.names: + for base2 in mro[i + 1:]: + # We only need to check compatibility of attributes from classes not + # in a subclass relationship. For subclasses, normal (single inheritance) + # checks suffice (these are implemented elsewhere). + if name in base2.names and base2 not in base.mro: + self.check_compatibility(name, base, base2, typ) + # Verify that base class layouts are compatible. + builtin_bases = [nearest_builtin_ancestor(base.type) + for base in typ.bases] + for base1 in builtin_bases: + for base2 in builtin_bases: + if not (base1 in base2.mro or base2 in base1.mro): + self.fail(messages.INSTANCE_LAYOUT_CONFLICT, typ) + + def check_compatibility(self, name: str, base1: TypeInfo, + base2: TypeInfo, ctx: Context) -> None: + """Check if attribute name in base1 is compatible with base2 in multiple inheritance. + + Assume base1 comes before base2 in the MRO, and that base1 and base2 don't have + a direct subclass relationship (i.e., the compatibility requirement only derives from + multiple inheritance). + """ + if name == '__init__': + # __init__ can be incompatible -- it's a special case. + return + first = base1[name] + second = base2[name] + first_type = first.type + if first_type is None and isinstance(first.node, FuncDef): + first_type = self.function_type(first.node) + second_type = second.type + if second_type is None and isinstance(second.node, FuncDef): + second_type = self.function_type(second.node) + # TODO: What if some classes are generic? + if (isinstance(first_type, FunctionLike) and + isinstance(second_type, FunctionLike)): + # Method override + first_sig = method_type(first_type) + second_sig = method_type(second_type) + ok = is_subtype(first_sig, second_sig) + elif first_type and second_type: + ok = is_equivalent(first_type, second_type) + else: + if first_type is None: + self.msg.cannot_determine_type_in_base(name, base1.name(), ctx) + if second_type is None: + self.msg.cannot_determine_type_in_base(name, base2.name(), ctx) + ok = True + if not ok: + self.msg.base_class_definitions_incompatible(name, base1, base2, + ctx) + + def visit_import_from(self, node: ImportFrom) -> Type: + self.check_import(node) + + def visit_import_all(self, node: ImportAll) -> Type: + self.check_import(node) + + def check_import(self, node: ImportBase) -> Type: + for assign in node.assignments: + lvalue = assign.lvalues[0] + lvalue_type, _, __ = self.check_lvalue(lvalue) + if lvalue_type is None: + # TODO: This is broken. + lvalue_type = AnyType() + message = '{} "{}"'.format(messages.INCOMPATIBLE_IMPORT_OF, + cast(NameExpr, assign.rvalue).name) + self.check_simple_assignment(lvalue_type, assign.rvalue, node, + msg=message, lvalue_name='local name', + rvalue_name='imported name') + + # + # Statements + # + + def visit_block(self, b: Block) -> Type: + if b.is_unreachable: + return None + for s in b.body: + self.accept(s) + if self.binder.breaking_out: + break + + def visit_assignment_stmt(self, s: AssignmentStmt) -> Type: + """Type check an assignment statement. + + Handle all kinds of assignment statements (simple, indexed, multiple). + """ + self.check_assignment(s.lvalues[-1], s.rvalue, s.type is None) + + if len(s.lvalues) > 1: + # Chained assignment (e.g. x = y = ...). + # Make sure that rvalue type will not be reinferred. + if s.rvalue not in self.type_map: + self.accept(s.rvalue) + rvalue = self.temp_node(self.type_map[s.rvalue], s) + for lv in s.lvalues[:-1]: + self.check_assignment(lv, rvalue, s.type is None) + + def check_assignment(self, lvalue: Node, rvalue: Node, infer_lvalue_type: bool = True) -> None: + """Type check a single assignment: lvalue = rvalue.""" + if isinstance(lvalue, TupleExpr) or isinstance(lvalue, ListExpr): + ltuple = cast(Union[TupleExpr, ListExpr], lvalue) + + self.check_assignment_to_multiple_lvalues(ltuple.items, rvalue, lvalue, + infer_lvalue_type) + else: + lvalue_type, index_lvalue, inferred = self.check_lvalue(lvalue) + if lvalue_type: + if isinstance(lvalue_type, PartialType) and lvalue_type.type is None: + # Try to infer a proper type for a variable with a partial None type. + rvalue_type = self.accept(rvalue) + if isinstance(rvalue_type, NoneTyp): + # This doesn't actually provide any additional information -- multiple + # None initializers preserve the partial None type. + return + if is_valid_inferred_type(rvalue_type): + var = lvalue_type.var + partial_types = self.find_partial_types(var) + if partial_types is not None: + if not self.current_node_deferred: + if experiments.STRICT_OPTIONAL: + var.type = UnionType.make_simplified_union( + [rvalue_type, NoneTyp()]) + else: + var.type = rvalue_type + else: + var.type = None + del partial_types[var] + # Try to infer a partial type. No need to check the return value, as + # an error will be reported elsewhere. + self.infer_partial_type(lvalue_type.var, lvalue, rvalue_type) + return + if (is_literal_none(rvalue) and + isinstance(lvalue, NameExpr) and + isinstance(lvalue.node, Var) and + lvalue.node.is_initialized_in_class): + # Allow None's to be assigned to class variables with non-Optional types. + rvalue_type = lvalue_type + else: + rvalue_type = self.check_simple_assignment(lvalue_type, rvalue, lvalue) + + if rvalue_type and infer_lvalue_type: + self.binder.assign_type(lvalue, + rvalue_type, + lvalue_type, + self.typing_mode_weak()) + elif index_lvalue: + self.check_indexed_assignment(index_lvalue, rvalue, rvalue) + + if inferred: + self.infer_variable_type(inferred, lvalue, self.accept(rvalue), + rvalue) + + def check_assignment_to_multiple_lvalues(self, lvalues: List[Node], rvalue: Node, + context: Context, + infer_lvalue_type: bool = True) -> None: + if isinstance(rvalue, TupleExpr) or isinstance(rvalue, ListExpr): + # Recursively go into Tuple or List expression rhs instead of + # using the type of rhs, because this allowed more fine grained + # control in cases like: a, b = [int, str] where rhs would get + # type List[object] + + rvalues = cast(Union[TupleExpr, ListExpr], rvalue).items + + if self.check_rvalue_count_in_assignment(lvalues, len(rvalues), context): + star_index = next((i for i, lv in enumerate(lvalues) if + isinstance(lv, StarExpr)), len(lvalues)) + + left_lvs = lvalues[:star_index] + star_lv = cast(StarExpr, + lvalues[star_index]) if star_index != len(lvalues) else None + right_lvs = lvalues[star_index + 1:] + + left_rvs, star_rvs, right_rvs = self.split_around_star( + rvalues, star_index, len(lvalues)) + + lr_pairs = list(zip(left_lvs, left_rvs)) + if star_lv: + rv_list = ListExpr(star_rvs) + rv_list.set_line(rvalue.get_line()) + lr_pairs.append((star_lv.expr, rv_list)) + lr_pairs.extend(zip(right_lvs, right_rvs)) + + for lv, rv in lr_pairs: + self.check_assignment(lv, rv, infer_lvalue_type) + else: + self.check_multi_assignment(lvalues, rvalue, context, infer_lvalue_type) + + def check_rvalue_count_in_assignment(self, lvalues: List[Node], rvalue_count: int, + context: Context) -> bool: + if any(isinstance(lvalue, StarExpr) for lvalue in lvalues): + if len(lvalues) - 1 > rvalue_count: + self.msg.wrong_number_values_to_unpack(rvalue_count, + len(lvalues) - 1, context) + return False + elif rvalue_count != len(lvalues): + self.msg.wrong_number_values_to_unpack(rvalue_count, + len(lvalues), context) + return False + return True + + def check_multi_assignment(self, lvalues: List[Node], + rvalue: Node, + context: Context, + infer_lvalue_type: bool = True, + msg: str = None) -> None: + """Check the assignment of one rvalue to a number of lvalues.""" + + # Infer the type of an ordinary rvalue expression. + rvalue_type = self.accept(rvalue) # TODO maybe elsewhere; redundant + undefined_rvalue = False + + if isinstance(rvalue_type, AnyType): + for lv in lvalues: + if isinstance(lv, StarExpr): + lv = lv.expr + self.check_assignment(lv, self.temp_node(AnyType(), context), infer_lvalue_type) + elif isinstance(rvalue_type, TupleType): + self.check_multi_assignment_from_tuple(lvalues, rvalue, rvalue_type, + context, undefined_rvalue, infer_lvalue_type) + else: + self.check_multi_assignment_from_iterable(lvalues, rvalue_type, + context, infer_lvalue_type) + + def check_multi_assignment_from_tuple(self, lvalues: List[Node], rvalue: Node, + rvalue_type: TupleType, context: Context, + undefined_rvalue: bool, + infer_lvalue_type: bool = True) -> None: + if self.check_rvalue_count_in_assignment(lvalues, len(rvalue_type.items), context): + star_index = next((i for i, lv in enumerate(lvalues) + if isinstance(lv, StarExpr)), len(lvalues)) + + left_lvs = lvalues[:star_index] + star_lv = cast(StarExpr, lvalues[star_index]) if star_index != len(lvalues) else None + right_lvs = lvalues[star_index + 1:] + + if not undefined_rvalue: + # Infer rvalue again, now in the correct type context. + lvalue_type = self.lvalue_type_for_inference(lvalues, rvalue_type) + rvalue_type = cast(TupleType, self.accept(rvalue, lvalue_type)) + + left_rv_types, star_rv_types, right_rv_types = self.split_around_star( + rvalue_type.items, star_index, len(lvalues)) + + for lv, rv_type in zip(left_lvs, left_rv_types): + self.check_assignment(lv, self.temp_node(rv_type, context), infer_lvalue_type) + if star_lv: + nodes = [self.temp_node(rv_type, context) for rv_type in star_rv_types] + list_expr = ListExpr(nodes) + list_expr.set_line(context.get_line()) + self.check_assignment(star_lv.expr, list_expr, infer_lvalue_type) + for lv, rv_type in zip(right_lvs, right_rv_types): + self.check_assignment(lv, self.temp_node(rv_type, context), infer_lvalue_type) + + def lvalue_type_for_inference(self, lvalues: List[Node], rvalue_type: TupleType) -> Type: + star_index = next((i for i, lv in enumerate(lvalues) + if isinstance(lv, StarExpr)), len(lvalues)) + left_lvs = lvalues[:star_index] + star_lv = cast(StarExpr, lvalues[star_index]) if star_index != len(lvalues) else None + right_lvs = lvalues[star_index + 1:] + left_rv_types, star_rv_types, right_rv_types = self.split_around_star( + rvalue_type.items, star_index, len(lvalues)) + + type_parameters = [] # type: List[Type] + + def append_types_for_inference(lvs: List[Node], rv_types: List[Type]) -> None: + for lv, rv_type in zip(lvs, rv_types): + sub_lvalue_type, index_expr, inferred = self.check_lvalue(lv) + if sub_lvalue_type: + type_parameters.append(sub_lvalue_type) + else: # index lvalue + # TODO Figure out more precise type context, probably + # based on the type signature of the _set method. + type_parameters.append(rv_type) + + append_types_for_inference(left_lvs, left_rv_types) + + if star_lv: + sub_lvalue_type, index_expr, inferred = self.check_lvalue(star_lv.expr) + if sub_lvalue_type: + type_parameters.extend([sub_lvalue_type] * len(star_rv_types)) + else: # index lvalue + # TODO Figure out more precise type context, probably + # based on the type signature of the _set method. + type_parameters.extend(star_rv_types) + + append_types_for_inference(right_lvs, right_rv_types) + + return TupleType(type_parameters, self.named_type('builtins.tuple')) + + def split_around_star(self, items: List[T], star_index: int, + length: int) -> Tuple[List[T], List[T], List[T]]: + """Splits a list of items in three to match another list of length 'length' + that contains a starred expression at 'star_index' in the following way: + + star_index = 2, length = 5 (i.e., [a,b,*,c,d]), items = [1,2,3,4,5,6,7] + returns in: ([1,2], [3,4,5], [6,7]) + """ + nr_right_of_star = length - star_index - 1 + right_index = nr_right_of_star if -nr_right_of_star != 0 else len(items) + left = items[:star_index] + star = items[star_index:right_index] + right = items[right_index:] + return (left, star, right) + + def type_is_iterable(self, type: Type) -> bool: + return (is_subtype(type, self.named_generic_type('typing.Iterable', + [AnyType()])) and + isinstance(type, Instance)) + + def check_multi_assignment_from_iterable(self, lvalues: List[Node], rvalue_type: Type, + context: Context, + infer_lvalue_type: bool = True) -> None: + if self.type_is_iterable(rvalue_type): + item_type = self.iterable_item_type(cast(Instance, rvalue_type)) + for lv in lvalues: + if isinstance(lv, StarExpr): + self.check_assignment(lv.expr, self.temp_node(rvalue_type, context), + infer_lvalue_type) + else: + self.check_assignment(lv, self.temp_node(item_type, context), + infer_lvalue_type) + else: + self.msg.type_not_iterable(rvalue_type, context) + + def check_lvalue(self, lvalue: Node) -> Tuple[Type, IndexExpr, Var]: + lvalue_type = None # type: Type + index_lvalue = None # type: IndexExpr + inferred = None # type: Var + + if self.is_definition(lvalue): + if isinstance(lvalue, NameExpr): + inferred = cast(Var, lvalue.node) + assert isinstance(inferred, Var) + else: + m = cast(MemberExpr, lvalue) + self.accept(m.expr) + inferred = m.def_var + elif isinstance(lvalue, IndexExpr): + index_lvalue = lvalue + elif isinstance(lvalue, MemberExpr): + lvalue_type = self.expr_checker.analyze_ordinary_member_access(lvalue, + True) + self.store_type(lvalue, lvalue_type) + elif isinstance(lvalue, NameExpr): + lvalue_type = self.expr_checker.analyze_ref_expr(lvalue, lvalue=True) + self.store_type(lvalue, lvalue_type) + elif isinstance(lvalue, TupleExpr) or isinstance(lvalue, ListExpr): + lv = cast(Union[TupleExpr, ListExpr], lvalue) + types = [self.check_lvalue(sub_expr)[0] for sub_expr in lv.items] + lvalue_type = TupleType(types, self.named_type('builtins.tuple')) + else: + lvalue_type = self.accept(lvalue) + + return lvalue_type, index_lvalue, inferred + + def is_definition(self, s: Node) -> bool: + if isinstance(s, NameExpr): + if s.is_def: + return True + # If the node type is not defined, this must the first assignment + # that we process => this is a definition, even though the semantic + # analyzer did not recognize this as such. This can arise in code + # that uses isinstance checks, if type checking of the primary + # definition is skipped due to an always False type check. + node = s.node + if isinstance(node, Var): + return node.type is None + elif isinstance(s, MemberExpr): + return s.is_def + return False + + def infer_variable_type(self, name: Var, lvalue: Node, + init_type: Type, context: Context) -> None: + """Infer the type of initialized variables from initializer type.""" + if self.typing_mode_weak(): + self.set_inferred_type(name, lvalue, AnyType()) + self.binder.assign_type(lvalue, init_type, self.binder.get_declaration(lvalue), True) + elif isinstance(init_type, Void): + self.check_not_void(init_type, context) + self.set_inference_error_fallback_type(name, lvalue, init_type, context) + elif isinstance(init_type, DeletedType): + self.msg.deleted_as_rvalue(init_type, context) + elif not is_valid_inferred_type(init_type): + # We cannot use the type of the initialization expression for full type + # inference (it's not specific enough), but we might be able to give + # partial type which will be made more specific later. A partial type + # gets generated in assignment like 'x = []' where item type is not known. + if not self.infer_partial_type(name, lvalue, init_type): + self.fail(messages.NEED_ANNOTATION_FOR_VAR, context) + self.set_inference_error_fallback_type(name, lvalue, init_type, context) + else: + # Infer type of the target. + + # Make the type more general (strip away function names etc.). + init_type = strip_type(init_type) + + self.set_inferred_type(name, lvalue, init_type) + + def infer_partial_type(self, name: Var, lvalue: Node, init_type: Type) -> bool: + if isinstance(init_type, (NoneTyp, UninhabitedType)): + partial_type = PartialType(None, name, [init_type]) + elif isinstance(init_type, Instance): + fullname = init_type.type.fullname() + if (isinstance(lvalue, NameExpr) and + (fullname == 'builtins.list' or + fullname == 'builtins.set' or + fullname == 'builtins.dict') and + all(isinstance(t, (NoneTyp, UninhabitedType)) for t in init_type.args)): + partial_type = PartialType(init_type.type, name, init_type.args) + else: + return False + else: + return False + self.set_inferred_type(name, lvalue, partial_type) + self.partial_types[-1][name] = lvalue + return True + + def set_inferred_type(self, var: Var, lvalue: Node, type: Type) -> None: + """Store inferred variable type. + + Store the type to both the variable node and the expression node that + refers to the variable (lvalue). If var is None, do nothing. + """ + if var and not self.current_node_deferred: + var.type = type + self.store_type(lvalue, type) + + def set_inference_error_fallback_type(self, var: Var, lvalue: Node, type: Type, + context: Context) -> None: + """If errors on context line are ignored, store dummy type for variable. + + If a program ignores error on type inference error, the variable should get some + inferred type so that if can used later on in the program. Example: + + x = [] # type: ignore + x.append(1) # Should be ok! + + We implement this here by giving x a valid type (Any). + """ + if context.get_line() in self.errors.ignored_lines[self.errors.file]: + self.set_inferred_type(var, lvalue, AnyType()) + + def narrow_type_from_binder(self, expr: Node, known_type: Type) -> Type: + if expr.literal >= LITERAL_TYPE: + restriction = self.binder.get(expr) + if restriction: + ans = meet_simple(known_type, restriction) + return ans + return known_type + + def check_simple_assignment(self, lvalue_type: Type, rvalue: Node, + context: Node, + msg: str = messages.INCOMPATIBLE_TYPES_IN_ASSIGNMENT, + lvalue_name: str = 'variable', + rvalue_name: str = 'expression') -> Type: + if self.is_stub and isinstance(rvalue, EllipsisExpr): + # '...' is always a valid initializer in a stub. + return AnyType() + else: + rvalue_type = self.accept(rvalue, lvalue_type) + if isinstance(rvalue_type, DeletedType): + self.msg.deleted_as_rvalue(rvalue_type, context) + if self.typing_mode_weak(): + return rvalue_type + if isinstance(lvalue_type, DeletedType): + self.msg.deleted_as_lvalue(lvalue_type, context) + else: + self.check_subtype(rvalue_type, lvalue_type, context, msg, + '{} has type'.format(rvalue_name), + '{} has type'.format(lvalue_name)) + return rvalue_type + + def check_indexed_assignment(self, lvalue: IndexExpr, + rvalue: Node, context: Context) -> None: + """Type check indexed assignment base[index] = rvalue. + + The lvalue argument is the base[index] expression. + """ + self.try_infer_partial_type_from_indexed_assignment(lvalue, rvalue) + basetype = self.accept(lvalue.base) + method_type = self.expr_checker.analyze_external_member_access( + '__setitem__', basetype, context) + lvalue.method_type = method_type + self.expr_checker.check_call(method_type, [lvalue.index, rvalue], + [nodes.ARG_POS, nodes.ARG_POS], + context) + + def try_infer_partial_type_from_indexed_assignment( + self, lvalue: IndexExpr, rvalue: Node) -> None: + # TODO: Should we share some of this with try_infer_partial_type? + if isinstance(lvalue.base, RefExpr) and isinstance(lvalue.base.node, Var): + var = lvalue.base.node + if isinstance(var.type, PartialType): + type_type = var.type.type + if type_type is None: + return # The partial type is None. + partial_types = self.find_partial_types(var) + if partial_types is None: + return + typename = type_type.fullname() + if typename == 'builtins.dict': + # TODO: Don't infer things twice. + key_type = self.accept(lvalue.index) + value_type = self.accept(rvalue) + full_key_type = UnionType.make_simplified_union( + [key_type, var.type.inner_types[0]]) + full_value_type = UnionType.make_simplified_union( + [value_type, var.type.inner_types[1]]) + if (is_valid_inferred_type(full_key_type) and + is_valid_inferred_type(full_value_type)): + if not self.current_node_deferred: + var.type = self.named_generic_type('builtins.dict', + [full_key_type, full_value_type]) + del partial_types[var] + + def visit_expression_stmt(self, s: ExpressionStmt) -> Type: + self.accept(s.expr) + + def visit_return_stmt(self, s: ReturnStmt) -> Type: + """Type check a return statement.""" + self.binder.breaking_out = True + if self.is_within_function(): + if self.function_stack[-1].is_generator: + return_type = self.get_generator_return_type(self.return_types[-1]) + else: + return_type = self.return_types[-1] + + if s.expr: + # Return with a value. + typ = self.accept(s.expr, return_type) + # Returning a value of type Any is always fine. + if isinstance(typ, AnyType): + return None + + if isinstance(return_type, Void): + # Lambdas are allowed to have a Void return. + # Functions returning a value of type None are allowed to have a Void return. + if isinstance(self.function_stack[-1], FuncExpr) or isinstance(typ, NoneTyp): + return None + self.fail(messages.NO_RETURN_VALUE_EXPECTED, s) + else: + self.check_subtype( + subtype_label='got', + subtype=typ, + supertype_label='expected', + supertype=return_type, + context=s, + msg=messages.INCOMPATIBLE_RETURN_VALUE_TYPE) + else: + # Empty returns are valid in Generators with Any typed returns. + if (self.function_stack[-1].is_generator and isinstance(return_type, AnyType)): + return None + + if isinstance(return_type, Void): + return None + + if isinstance(return_type, AnyType): + return None + + if self.typing_mode_full(): + self.fail(messages.RETURN_VALUE_EXPECTED, s) + + def wrap_generic_type(self, typ: Instance, rtyp: Instance, check_type: + str, context: Context) -> Type: + n_diff = self.count_nested_types(rtyp, check_type) - self.count_nested_types(typ, + check_type) + if n_diff == 1: + return self.named_generic_type(check_type, [typ]) + elif n_diff == 0 or n_diff > 1: + self.fail(messages.INCOMPATIBLE_RETURN_VALUE_TYPE + + ": expected {}, got {}".format(rtyp, typ), context) + return typ + return typ + + def count_nested_types(self, typ: Instance, check_type: str) -> int: + c = 0 + while is_subtype(typ, self.named_type(check_type)): + c += 1 + typ = map_instance_to_supertype(self.named_generic_type(check_type, typ.args), + self.lookup_typeinfo(check_type)) + if typ.args: + typ = cast(Instance, typ.args[0]) + else: + return c + return c + + def visit_if_stmt(self, s: IfStmt) -> Type: + """Type check an if statement.""" + breaking_out = True + # This frame records the knowledge from previous if/elif clauses not being taken. + with self.binder.frame_context(): + for e, b in zip(s.expr, s.body): + t = self.accept(e) + self.check_not_void(t, e) + if_map, else_map = find_isinstance_check( + e, self.type_map, + self.typing_mode_weak() + ) + if if_map is None: + # The condition is always false + # XXX should issue a warning? + pass + else: + # Only type check body if the if condition can be true. + with self.binder.frame_context(2): + if if_map: + for var, type in if_map.items(): + self.binder.push(var, type) + + self.accept(b) + breaking_out = breaking_out and self.binder.last_pop_breaking_out + + if else_map: + for var, type in else_map.items(): + self.binder.push(var, type) + if else_map is None: + # The condition is always true => remaining elif/else blocks + # can never be reached. + + # Might also want to issue a warning + # print("Warning: isinstance always true") + break + else: # Didn't break => can't prove one of the conditions is always true + with self.binder.frame_context(2): + if s.else_body: + self.accept(s.else_body) + breaking_out = breaking_out and self.binder.last_pop_breaking_out + if breaking_out: + self.binder.breaking_out = True + return None + + def visit_while_stmt(self, s: WhileStmt) -> Type: + """Type check a while statement.""" + self.accept_loop(IfStmt([s.expr], [s.body], None), s.else_body) + + def visit_operator_assignment_stmt(self, + s: OperatorAssignmentStmt) -> Type: + """Type check an operator assignment statement, e.g. x += 1.""" + lvalue_type = self.accept(s.lvalue) + method = infer_operator_assignment_method(lvalue_type, s.op) + rvalue_type, method_type = self.expr_checker.check_op( + method, lvalue_type, s.rvalue, s) + + if isinstance(s.lvalue, IndexExpr): + self.check_indexed_assignment(s.lvalue, s.rvalue, s.rvalue) + else: + if not is_subtype(rvalue_type, lvalue_type): + self.msg.incompatible_operator_assignment(s.op, s) + + def visit_assert_stmt(self, s: AssertStmt) -> Type: + self.accept(s.expr) + + # If this is asserting some isinstance check, bind that type in the following code + true_map, _ = find_isinstance_check( + s.expr, self.type_map, + self.typing_mode_weak() + ) + + if true_map: + for var, type in true_map.items(): + self.binder.push(var, type) + + def visit_raise_stmt(self, s: RaiseStmt) -> Type: + """Type check a raise statement.""" + self.binder.breaking_out = True + if s.expr: + self.type_check_raise(s.expr, s) + if s.from_expr: + self.type_check_raise(s.from_expr, s) + + def type_check_raise(self, e: Node, s: RaiseStmt) -> None: + typ = self.accept(e) + if isinstance(typ, FunctionLike): + if typ.is_type_obj(): + # Cases like "raise/from ExceptionClass". + typeinfo = typ.type_object() + base = self.lookup_typeinfo('builtins.BaseException') + if base in typeinfo.mro or typeinfo.fallback_to_any: + # Good! + return + # Else fall back to the checks below (which will fail). + if isinstance(typ, TupleType) and self.options.python_version[0] == 2: + # allow `raise type, value, traceback` + # https://docs.python.org/2/reference/simple_stmts.html#the-raise-statement + # TODO: Also check tuple item types. + if len(typ.items) in (2, 3): + return + if isinstance(typ, Instance) and typ.type.fallback_to_any: + # OK! + return + self.check_subtype(typ, + self.named_type('builtins.BaseException'), s, + messages.INVALID_EXCEPTION) + + def visit_try_stmt(self, s: TryStmt) -> Type: + """Type check a try statement.""" + # Our enclosing frame will get the result if the try/except falls through. + # This one gets all possible intermediate states + with self.binder.frame_context(): + if s.finally_body: + self.binder.try_frames.add(len(self.binder.frames) - 1) + breaking_out = self.visit_try_without_finally(s) + self.binder.try_frames.remove(len(self.binder.frames) - 1) + # First we check finally_body is type safe for all intermediate frames + self.accept(s.finally_body) + breaking_out = breaking_out or self.binder.breaking_out + else: + breaking_out = self.visit_try_without_finally(s) + + if not breaking_out and s.finally_body: + # Then we try again for the more restricted set of options that can fall through + self.accept(s.finally_body) + self.binder.breaking_out = breaking_out + return None + + def visit_try_without_finally(self, s: TryStmt) -> bool: + """Type check a try statement, ignoring the finally block. + + Return whether we are guaranteed to be breaking out. + Otherwise, it will place the results possible frames of + that don't break out into self.binder.frames[-2]. + """ + breaking_out = True + # This frame records the possible states that exceptions can leave variables in + # during the try: block + with self.binder.frame_context(): + with self.binder.frame_context(3): + self.binder.try_frames.add(len(self.binder.frames) - 2) + self.accept(s.body) + self.binder.try_frames.remove(len(self.binder.frames) - 2) + if s.else_body: + self.accept(s.else_body) + breaking_out = breaking_out and self.binder.last_pop_breaking_out + for i in range(len(s.handlers)): + with self.binder.frame_context(3): + if s.types[i]: + t = self.visit_except_handler_test(s.types[i]) + if s.vars[i]: + # To support local variables, we make this a definition line, + # causing assignment to set the variable's type. + s.vars[i].is_def = True + self.check_assignment(s.vars[i], self.temp_node(t, s.vars[i])) + self.accept(s.handlers[i]) + if s.vars[i]: + # Exception variables are deleted in python 3 but not python 2. + # But, since it's bad form in python 2 and the type checking + # wouldn't work very well, we delete it anyway. + + # Unfortunately, this doesn't let us detect usage before the + # try/except block. + if self.options.python_version[0] >= 3: + source = s.vars[i].name + else: + source = ('(exception variable "{}", which we do not accept outside' + 'except: blocks even in python 2)'.format(s.vars[i].name)) + var = cast(Var, s.vars[i].node) + var.type = DeletedType(source=source) + self.binder.cleanse(s.vars[i]) + breaking_out = breaking_out and self.binder.last_pop_breaking_out + return breaking_out + + def visit_except_handler_test(self, n: Node) -> Type: + """Type check an exception handler test clause.""" + type = self.accept(n) + + all_types = [] # type: List[Type] + test_types = type.items if isinstance(type, TupleType) else [type] + + for ttype in test_types: + if isinstance(ttype, AnyType): + all_types.append(ttype) + continue + + if not isinstance(ttype, FunctionLike): + self.fail(messages.INVALID_EXCEPTION_TYPE, n) + return AnyType() + + item = ttype.items()[0] + ret_type = item.ret_type + if not (is_subtype(ret_type, self.named_type('builtins.BaseException')) + and item.is_type_obj()): + self.fail(messages.INVALID_EXCEPTION_TYPE, n) + return AnyType() + + all_types.append(ret_type) + + return UnionType.make_simplified_union(all_types) + + def visit_for_stmt(self, s: ForStmt) -> Type: + """Type check a for statement.""" + item_type = self.analyze_iterable_item_type(s.expr) + self.analyze_index_variables(s.index, item_type, s) + self.accept_loop(s.body, s.else_body) + + def analyze_iterable_item_type(self, expr: Node) -> Type: + """Analyse iterable expression and return iterator item type.""" + iterable = self.accept(expr) + + self.check_not_void(iterable, expr) + if isinstance(iterable, TupleType): + if experiments.STRICT_OPTIONAL: + joined = UninhabitedType() # type: Type + else: + joined = NoneTyp() + for item in iterable.items: + joined = join_types(joined, item) + if isinstance(joined, ErrorType): + self.fail(messages.CANNOT_INFER_ITEM_TYPE, expr) + return AnyType() + return joined + else: + # Non-tuple iterable. + self.check_subtype(iterable, + self.named_generic_type('typing.Iterable', + [AnyType()]), + expr, messages.ITERABLE_EXPECTED) + + echk = self.expr_checker + method = echk.analyze_external_member_access('__iter__', iterable, + expr) + iterator = echk.check_call(method, [], [], expr)[0] + if self.options.python_version[0] >= 3: + nextmethod = '__next__' + else: + nextmethod = 'next' + method = echk.analyze_external_member_access(nextmethod, iterator, + expr) + return echk.check_call(method, [], [], expr)[0] + + def analyze_index_variables(self, index: Node, item_type: Type, + context: Context) -> None: + """Type check or infer for loop or list comprehension index vars.""" + self.check_assignment(index, self.temp_node(item_type, context)) + + def visit_del_stmt(self, s: DelStmt) -> Type: + if isinstance(s.expr, IndexExpr): + e = s.expr + m = MemberExpr(e.base, '__delitem__') + m.line = s.line + c = CallExpr(m, [e.index], [nodes.ARG_POS], [None]) + c.line = s.line + return c.accept(self) + else: + def flatten(t: Node) -> List[Node]: + """Flatten a nested sequence of tuples/lists into one list of nodes.""" + if isinstance(t, TupleExpr) or isinstance(t, ListExpr): + t = cast(Union[TupleExpr, ListExpr], t) + return [b for a in t.items for b in flatten(a)] + else: + return [t] + + s.expr.accept(self) + for elt in flatten(s.expr): + if isinstance(elt, NameExpr): + self.binder.assign_type(elt, + DeletedType(source=elt.name), + self.binder.get_declaration(elt), + self.typing_mode_weak()) + return None + + def visit_decorator(self, e: Decorator) -> Type: + for d in e.decorators: + if isinstance(d, RefExpr): + if d.fullname == 'typing.no_type_check': + e.var.type = AnyType() + e.var.is_ready = True + return NoneTyp() + + e.func.accept(self) + sig = self.function_type(e.func) # type: Type + # Process decorators from the inside out. + for i in range(len(e.decorators)): + n = len(e.decorators) - 1 - i + d = e.decorators[n] + if isinstance(d, NameExpr) and d.fullname == 'typing.overload': + self.fail('Single overload definition, multiple required', e) + continue + dec = self.accept(d) + temp = self.temp_node(sig) + sig, t2 = self.expr_checker.check_call(dec, [temp], + [nodes.ARG_POS], e) + sig = cast(FunctionLike, sig) + sig = set_callable_name(sig, e.func) + e.var.type = sig + e.var.is_ready = True + if e.func.is_property: + self.check_incompatible_property_override(e) + + def check_incompatible_property_override(self, e: Decorator) -> None: + if not e.var.is_settable_property and e.func.info is not None: + name = e.func.name() + for base in e.func.info.mro[1:]: + base_attr = base.names.get(name) + if not base_attr: + continue + if (isinstance(base_attr.node, OverloadedFuncDef) and + base_attr.node.is_property and + base_attr.node.items[0].var.is_settable_property): + self.fail(messages.READ_ONLY_PROPERTY_OVERRIDES_READ_WRITE, e) + + def visit_with_stmt(self, s: WithStmt) -> Type: + echk = self.expr_checker + for expr, target in zip(s.expr, s.target): + ctx = self.accept(expr) + enter = echk.analyze_external_member_access('__enter__', ctx, expr) + obj = echk.check_call(enter, [], [], expr)[0] + if target: + self.check_assignment(target, self.temp_node(obj, expr)) + exit = echk.analyze_external_member_access('__exit__', ctx, expr) + arg = self.temp_node(AnyType(), expr) + echk.check_call(exit, [arg] * 3, [nodes.ARG_POS] * 3, expr) + self.accept(s.body) + + def visit_print_stmt(self, s: PrintStmt) -> Type: + for arg in s.args: + self.accept(arg) + if s.target: + target_type = self.accept(s.target) + if not isinstance(target_type, NoneTyp): + # TODO: Also verify the type of 'write'. + self.expr_checker.analyze_external_member_access('write', target_type, s.target) + + # + # Expressions + # + + def visit_name_expr(self, e: NameExpr) -> Type: + return self.expr_checker.visit_name_expr(e) + + def visit_call_expr(self, e: CallExpr) -> Type: + return self.expr_checker.visit_call_expr(e) + + def visit_yield_from_expr(self, e: YieldFromExpr) -> Type: + return_type = self.return_types[-1] + subexpr_type = self.accept(e.expr, return_type) + iter_type = None # type: Type + + # Check that the expr is an instance of Iterable and get the type of the iterator produced + # by __iter__. + if isinstance(subexpr_type, AnyType): + iter_type = AnyType() + elif (isinstance(subexpr_type, Instance) and + is_subtype(subexpr_type, self.named_type('typing.Iterable'))): + iter_method_type = self.expr_checker.analyze_external_member_access( + '__iter__', + subexpr_type, + AnyType()) + + generic_generator_type = self.named_generic_type('typing.Generator', + [AnyType(), AnyType(), AnyType()]) + iter_type, _ = self.expr_checker.check_call(iter_method_type, [], [], + context=generic_generator_type) + else: + self.msg.yield_from_invalid_operand_type(subexpr_type, e) + iter_type = AnyType() + + # Check that the iterator's item type matches the type yielded by the Generator function + # containing this `yield from` expression. + expected_item_type = self.get_generator_yield_type(return_type) + actual_item_type = self.get_generator_yield_type(iter_type) + + self.check_subtype(actual_item_type, expected_item_type, e, + messages.INCOMPATIBLE_TYPES_IN_YIELD_FROM, + 'actual type', 'expected type') + + # Determine the type of the entire yield from expression. + if (isinstance(iter_type, Instance) and + iter_type.type.fullname() == 'typing.Generator'): + return self.get_generator_return_type(iter_type) + else: + # Non-Generators don't return anything from `yield from` expressions. + return Void() + + def visit_member_expr(self, e: MemberExpr) -> Type: + return self.expr_checker.visit_member_expr(e) + + def visit_break_stmt(self, s: BreakStmt) -> Type: + self.binder.breaking_out = True + self.binder.allow_jump(self.binder.loop_frames[-1] - 1) + return None + + def visit_continue_stmt(self, s: ContinueStmt) -> Type: + self.binder.breaking_out = True + self.binder.allow_jump(self.binder.loop_frames[-1]) + return None + + def visit_int_expr(self, e: IntExpr) -> Type: + return self.expr_checker.visit_int_expr(e) + + def visit_str_expr(self, e: StrExpr) -> Type: + return self.expr_checker.visit_str_expr(e) + + def visit_bytes_expr(self, e: BytesExpr) -> Type: + return self.expr_checker.visit_bytes_expr(e) + + def visit_unicode_expr(self, e: UnicodeExpr) -> Type: + return self.expr_checker.visit_unicode_expr(e) + + def visit_float_expr(self, e: FloatExpr) -> Type: + return self.expr_checker.visit_float_expr(e) + + def visit_complex_expr(self, e: ComplexExpr) -> Type: + return self.expr_checker.visit_complex_expr(e) + + def visit_ellipsis(self, e: EllipsisExpr) -> Type: + return self.expr_checker.visit_ellipsis(e) + + def visit_op_expr(self, e: OpExpr) -> Type: + return self.expr_checker.visit_op_expr(e) + + def visit_comparison_expr(self, e: ComparisonExpr) -> Type: + return self.expr_checker.visit_comparison_expr(e) + + def visit_unary_expr(self, e: UnaryExpr) -> Type: + return self.expr_checker.visit_unary_expr(e) + + def visit_index_expr(self, e: IndexExpr) -> Type: + return self.expr_checker.visit_index_expr(e) + + def visit_cast_expr(self, e: CastExpr) -> Type: + return self.expr_checker.visit_cast_expr(e) + + def visit_reveal_type_expr(self, e: RevealTypeExpr) -> Type: + return self.expr_checker.visit_reveal_type_expr(e) + + def visit_super_expr(self, e: SuperExpr) -> Type: + return self.expr_checker.visit_super_expr(e) + + def visit_type_application(self, e: TypeApplication) -> Type: + return self.expr_checker.visit_type_application(e) + + def visit_type_alias_expr(self, e: TypeAliasExpr) -> Type: + return self.expr_checker.visit_type_alias_expr(e) + + def visit_type_var_expr(self, e: TypeVarExpr) -> Type: + # TODO: Perhaps return a special type used for type variables only? + return AnyType() + + def visit_namedtuple_expr(self, e: NamedTupleExpr) -> Type: + # TODO: Perhaps return a type object type? + return AnyType() + + def visit_list_expr(self, e: ListExpr) -> Type: + return self.expr_checker.visit_list_expr(e) + + def visit_set_expr(self, e: SetExpr) -> Type: + return self.expr_checker.visit_set_expr(e) + + def visit_tuple_expr(self, e: TupleExpr) -> Type: + return self.expr_checker.visit_tuple_expr(e) + + def visit_dict_expr(self, e: DictExpr) -> Type: + return self.expr_checker.visit_dict_expr(e) + + def visit_slice_expr(self, e: SliceExpr) -> Type: + return self.expr_checker.visit_slice_expr(e) + + def visit_func_expr(self, e: FuncExpr) -> Type: + return self.expr_checker.visit_func_expr(e) + + def visit_list_comprehension(self, e: ListComprehension) -> Type: + return self.expr_checker.visit_list_comprehension(e) + + def visit_set_comprehension(self, e: SetComprehension) -> Type: + return self.expr_checker.visit_set_comprehension(e) + + def visit_generator_expr(self, e: GeneratorExpr) -> Type: + return self.expr_checker.visit_generator_expr(e) + + def visit_dictionary_comprehension(self, e: DictionaryComprehension) -> Type: + return self.expr_checker.visit_dictionary_comprehension(e) + + def visit_temp_node(self, e: TempNode) -> Type: + return e.type + + def visit_conditional_expr(self, e: ConditionalExpr) -> Type: + return self.expr_checker.visit_conditional_expr(e) + + def visit_backquote_expr(self, e: BackquoteExpr) -> Type: + return self.expr_checker.visit_backquote_expr(e) + + def visit_yield_expr(self, e: YieldExpr) -> Type: + return_type = self.return_types[-1] + expected_item_type = self.get_generator_yield_type(return_type) + if e.expr is None: + if (not (isinstance(expected_item_type, Void) or + isinstance(expected_item_type, AnyType)) + and self.typing_mode_full()): + self.fail(messages.YIELD_VALUE_EXPECTED, e) + else: + actual_item_type = self.accept(e.expr, expected_item_type) + self.check_subtype(actual_item_type, expected_item_type, e, + messages.INCOMPATIBLE_TYPES_IN_YIELD, + 'actual type', 'expected type') + return self.get_generator_receive_type(return_type) + + # + # Helpers + # + + def check_subtype(self, subtype: Type, supertype: Type, context: Context, + msg: str = messages.INCOMPATIBLE_TYPES, + subtype_label: str = None, + supertype_label: str = None) -> None: + """Generate an error if the subtype is not compatible with + supertype.""" + if not is_subtype(subtype, supertype): + if isinstance(subtype, Void): + self.msg.does_not_return_value(subtype, context) + else: + extra_info = [] # type: List[str] + if subtype_label is not None or supertype_label is not None: + subtype_str, supertype_str = self.msg.format_distinctly(subtype, supertype) + if subtype_label is not None: + extra_info.append(subtype_label + ' ' + subtype_str) + if supertype_label is not None: + extra_info.append(supertype_label + ' ' + supertype_str) + if extra_info: + msg += ' (' + ', '.join(extra_info) + ')' + self.fail(msg, context) + + def named_type(self, name: str) -> Instance: + """Return an instance type with type given by the name and no + type arguments. For example, named_type('builtins.object') + produces the object type. + """ + # Assume that the name refers to a type. + sym = self.lookup_qualified(name) + return Instance(cast(TypeInfo, sym.node), []) + + def named_generic_type(self, name: str, args: List[Type]) -> Instance: + """Return an instance with the given name and type arguments. + + Assume that the number of arguments is correct. Assume that + the name refers to a compatible generic type. + """ + return Instance(self.lookup_typeinfo(name), args) + + def lookup_typeinfo(self, fullname: str) -> TypeInfo: + # Assume that the name refers to a class. + sym = self.lookup_qualified(fullname) + return cast(TypeInfo, sym.node) + + def type_type(self) -> Instance: + """Return instance type 'type'.""" + return self.named_type('builtins.type') + + def object_type(self) -> Instance: + """Return instance type 'object'.""" + return self.named_type('builtins.object') + + def bool_type(self) -> Instance: + """Return instance type 'bool'.""" + return self.named_type('builtins.bool') + + def str_type(self) -> Instance: + """Return instance type 'str'.""" + return self.named_type('builtins.str') + + def check_type_equivalency(self, t1: Type, t2: Type, node: Context, + msg: str = messages.INCOMPATIBLE_TYPES) -> None: + """Generate an error if the types are not equivalent. The + dynamic type is equivalent with all types. + """ + if not is_equivalent(t1, t2): + self.fail(msg, node) + + def store_type(self, node: Node, typ: Type) -> None: + """Store the type of a node in the type map.""" + self.type_map[node] = typ + + def typing_mode_none(self) -> bool: + if self.is_dynamic_function() and not self.options.check_untyped_defs: + return not self.weak_opts + elif self.function_stack: + return False + else: + return False + + def typing_mode_weak(self) -> bool: + if self.is_dynamic_function() and not self.options.check_untyped_defs: + return bool(self.weak_opts) + elif self.function_stack: + return False + else: + return 'global' in self.weak_opts + + def typing_mode_full(self) -> bool: + if self.is_dynamic_function() and not self.options.check_untyped_defs: + return False + elif self.function_stack: + return True + else: + return 'global' not in self.weak_opts + + def is_dynamic_function(self) -> bool: + return len(self.dynamic_funcs) > 0 and self.dynamic_funcs[-1] + + def lookup(self, name: str, kind: int) -> SymbolTableNode: + """Look up a definition from the symbol table with the given name. + TODO remove kind argument + """ + if name in self.globals: + return self.globals[name] + else: + b = self.globals.get('__builtins__', None) + if b: + table = cast(MypyFile, b.node).names + if name in table: + return table[name] + raise KeyError('Failed lookup: {}'.format(name)) + + def lookup_qualified(self, name: str) -> SymbolTableNode: + if '.' not in name: + return self.lookup(name, GDEF) # FIX kind + else: + parts = name.split('.') + n = self.modules[parts[0]] + for i in range(1, len(parts) - 1): + n = cast(MypyFile, n.names.get(parts[i], None).node) + return n.names[parts[-1]] + + def enter_partial_types(self) -> None: + """Push a new scope for collecting partial types.""" + self.partial_types.append({}) + + def leave_partial_types(self) -> None: + """Pop partial type scope. + + Also report errors for variables which still have partial + types, i.e. we couldn't infer a complete type. + """ + partial_types = self.partial_types.pop() + if not self.current_node_deferred: + for var, context in partial_types.items(): + if experiments.STRICT_OPTIONAL and cast(PartialType, var.type).type is None: + # None partial type: assume variable is intended to have type None + var.type = NoneTyp() + else: + self.msg.fail(messages.NEED_ANNOTATION_FOR_VAR, context) + var.type = AnyType() + + def find_partial_types(self, var: Var) -> Optional[Dict[Var, Context]]: + for partial_types in reversed(self.partial_types): + if var in partial_types: + return partial_types + return None + + def is_within_function(self) -> bool: + """Are we currently type checking within a function? + + I.e. not at class body or at the top level. + """ + return self.return_types != [] + + def check_not_void(self, typ: Type, context: Context) -> None: + """Generate an error if the type is Void.""" + if isinstance(typ, Void): + self.msg.does_not_return_value(typ, context) + + def temp_node(self, t: Type, context: Context = None) -> Node: + """Create a temporary node with the given, fixed type.""" + temp = TempNode(t) + if context: + temp.set_line(context.get_line()) + return temp + + def fail(self, msg: str, context: Context) -> None: + """Produce an error message.""" + self.msg.fail(msg, context) + + def iterable_item_type(self, instance: Instance) -> Type: + iterable = map_instance_to_supertype( + instance, + self.lookup_typeinfo('typing.Iterable')) + return iterable.args[0] + + def function_type(self, func: FuncBase) -> FunctionLike: + return function_type(func, self.named_type('builtins.function')) + + def method_type(self, func: FuncBase) -> FunctionLike: + return method_type_with_fallback(func, self.named_type('builtins.function')) + + +# Data structure returned by find_isinstance_check representing +# information learned from the truth or falsehood of a condition. The +# dict maps nodes representing expressions like 'a[0].x' to their +# refined types under the assumption that the condition has a +# particular truth value. A value of None means that the condition can +# never have that truth value. + +# NB: The keys of this dict are nodes in the original source program, +# which are compared by reference equality--effectively, being *the +# same* expression of the program, not just two identical expressions +# (such as two references to the same variable). TODO: it would +# probably be better to have the dict keyed by the nodes' literal_hash +# field instead. + +# NB: This should be `TypeMap = Optional[Dict[Node, Type]]`! +# But see https://github.com/python/mypy/issues/1637 +TypeMap = Dict[Node, Type] + + +def conditional_type_map(expr: Node, + current_type: Optional[Type], + proposed_type: Optional[Type], + *, + weak: bool = False + ) -> Tuple[TypeMap, TypeMap]: + """Takes in an expression, the current type of the expression, and a + proposed type of that expression. + + Returns a 2-tuple: The first element is a map from the expression to + the proposed type, if the expression can be the proposed type. The + second element is a map from the expression to the type it would hold + if it was not the proposed type, if any.""" + if proposed_type: + if current_type: + if is_proper_subtype(current_type, proposed_type): + return {expr: proposed_type}, None + elif not is_overlapping_types(current_type, proposed_type): + return None, {expr: current_type} + else: + remaining_type = restrict_subtype_away(current_type, proposed_type) + return {expr: proposed_type}, {expr: remaining_type} + else: + return {expr: proposed_type}, {} + else: + # An isinstance check, but we don't understand the type + if weak: + return {expr: AnyType()}, {expr: current_type} + else: + return {}, {} + + +def is_literal_none(n: Node) -> bool: + return isinstance(n, NameExpr) and n.fullname == 'builtins.None' + + +def and_conditional_maps(m1: TypeMap, m2: TypeMap) -> TypeMap: + """Calculate what information we can learn from the truth of (e1 and e2) + in terms of the information that we can learn from the truth of e1 and + the truth of e2. + """ + + if m1 is None or m2 is None: + # One of the conditions can never be true. + return None + # Both conditions can be true; combine the information. Anything + # we learn from either conditions's truth is valid. If the same + # expression's type is refined by both conditions, we somewhat + # arbitrarily give precedence to m2. (In the future, we could use + # an intersection type.) + result = m2.copy() + m2_keys = set(n2.literal_hash for n2 in m2) + for n1 in m1: + if n1.literal_hash not in m2_keys: + result[n1] = m1[n1] + return result + + +def or_conditional_maps(m1: TypeMap, m2: TypeMap) -> TypeMap: + """Calculate what information we can learn from the truth of (e1 or e2) + in terms of the information that we can learn from the truth of e1 and + the truth of e2. + """ + + if m1 is None: + return m2 + if m2 is None: + return m1 + # Both conditions can be true. Combine information about + # expressions whose type is refined by both conditions. (We do not + # learn anything about expressions whose type is refined by only + # one condition.) + result = {} + for n1 in m1: + for n2 in m2: + if n1.literal_hash == n2.literal_hash: + result[n1] = UnionType.make_simplified_union([m1[n1], m2[n2]]) + return result + + +def find_isinstance_check(node: Node, + type_map: Dict[Node, Type], + weak: bool=False + ) -> Tuple[TypeMap, TypeMap]: + """Find any isinstance checks (within a chain of ands). Includes + implicit and explicit checks for None. + + Return value is a map of variables to their types if the condition + is true and a map of variables to their types if the condition is false. + + If either of the values in the tuple is None, then that particular + branch can never occur. + + Guaranteed to not return None, None. (But may return {}, {}) + """ + if isinstance(node, CallExpr): + if refers_to_fullname(node.callee, 'builtins.isinstance'): + expr = node.args[0] + if expr.literal == LITERAL_TYPE: + vartype = type_map[expr] + type = get_isinstance_type(node.args[1], type_map) + return conditional_type_map(expr, vartype, type, weak=weak) + elif (isinstance(node, ComparisonExpr) and any(is_literal_none(n) for n in node.operands) and + experiments.STRICT_OPTIONAL): + # Check for `x is None` and `x is not None`. + is_not = node.operators == ['is not'] + if is_not or node.operators == ['is']: + if_vars = {} # type: Dict[Node, Type] + else_vars = {} # type: Dict[Node, Type] + for expr in node.operands: + if expr.literal == LITERAL_TYPE and not is_literal_none(expr) and expr in type_map: + # This should only be true at most once: there should be + # two elements in node.operands, and at least one of them + # should represent a None. + vartype = type_map[expr] + if_vars, else_vars = conditional_type_map(expr, vartype, NoneTyp(), weak=weak) + break + + if is_not: + if_vars, else_vars = else_vars, if_vars + return if_vars, else_vars + elif isinstance(node, RefExpr) and experiments.STRICT_OPTIONAL: + # The type could be falsy, so we can't deduce anything new about the else branch + vartype = type_map[node] + _, if_vars = conditional_type_map(node, vartype, NoneTyp(), weak=weak) + return if_vars, {} + elif isinstance(node, OpExpr) and node.op == 'and': + left_if_vars, left_else_vars = find_isinstance_check( + node.left, + type_map, + weak, + ) + + right_if_vars, right_else_vars = find_isinstance_check( + node.right, + type_map, + weak, + ) + + # (e1 and e2) is true if both e1 and e2 are true, + # and false if at least one of e1 and e2 is false. + return (and_conditional_maps(left_if_vars, right_if_vars), + or_conditional_maps(left_else_vars, right_else_vars)) + elif isinstance(node, OpExpr) and node.op == 'or': + left_if_vars, left_else_vars = find_isinstance_check( + node.left, + type_map, + weak, + ) + + right_if_vars, right_else_vars = find_isinstance_check( + node.right, + type_map, + weak, + ) + + # (e1 or e2) is true if at least one of e1 or e2 is true, + # and false if both e1 and e2 are false. + return (or_conditional_maps(left_if_vars, right_if_vars), + and_conditional_maps(left_else_vars, right_else_vars)) + elif isinstance(node, UnaryExpr) and node.op == 'not': + left, right = find_isinstance_check(node.expr, type_map, weak) + return right, left + + # Not a supported isinstance check + return {}, {} + + +def get_isinstance_type(node: Node, type_map: Dict[Node, Type]) -> Type: + type = type_map[node] + + if isinstance(type, TupleType): + all_types = type.items + else: + all_types = [type] + + types = [] # type: List[Type] + + for type in all_types: + if isinstance(type, FunctionLike): + if type.is_type_obj(): + # Type variables may be present -- erase them, which is the best + # we can do (outside disallowing them here). + type = erase_typevars(type.items()[0].ret_type) + + types.append(type) + + if len(types) == 0: + return None + elif len(types) == 1: + return types[0] + else: + return UnionType(types) + + +def expand_node(defn: Node, map: Dict[TypeVarId, Type]) -> Node: + visitor = TypeTransformVisitor(map) + return defn.accept(visitor) + + +def expand_func(defn: FuncItem, map: Dict[TypeVarId, Type]) -> FuncItem: + return cast(FuncItem, expand_node(defn, map)) + + +class TypeTransformVisitor(TransformVisitor): + def __init__(self, map: Dict[TypeVarId, Type]) -> None: + super().__init__() + self.map = map + + def type(self, type: Type) -> Type: + return expand_type(type, self.map) + + +def is_unsafe_overlapping_signatures(signature: Type, other: Type) -> bool: + """Check if two signatures may be unsafely overlapping. + + Two signatures s and t are overlapping if both can be valid for the same + statically typed values and the return types are incompatible. + + Assume calls are first checked against 'signature', then against 'other'. + Thus if 'signature' is more general than 'other', there is no unsafe + overlapping. + + TODO If argument types vary covariantly, the return type may vary + covariantly as well. + """ + if isinstance(signature, CallableType): + if isinstance(other, CallableType): + # TODO varargs + # TODO keyword args + # TODO erasure + # TODO allow to vary covariantly + # Check if the argument counts are overlapping. + min_args = max(signature.min_args, other.min_args) + max_args = min(len(signature.arg_types), len(other.arg_types)) + if min_args > max_args: + # Argument counts are not overlapping. + return False + # Signatures are overlapping iff if they are overlapping for the + # smallest common argument count. + for i in range(min_args): + t1 = signature.arg_types[i] + t2 = other.arg_types[i] + if not is_overlapping_types(t1, t2): + return False + # All arguments types for the smallest common argument count are + # overlapping => the signature is overlapping. The overlapping is + # safe if the return types are identical. + if is_same_type(signature.ret_type, other.ret_type): + return False + # If the first signature has more general argument types, the + # latter will never be called + if is_more_general_arg_prefix(signature, other): + return False + return not is_more_precise_signature(signature, other) + return True + + +def is_more_general_arg_prefix(t: FunctionLike, s: FunctionLike) -> bool: + """Does t have wider arguments than s?""" + # TODO should an overload with additional items be allowed to be more + # general than one with fewer items (or just one item)? + # TODO check argument kinds + if isinstance(t, CallableType): + if isinstance(s, CallableType): + return all(is_proper_subtype(args, argt) + for argt, args in zip(t.arg_types, s.arg_types)) + elif isinstance(t, FunctionLike): + if isinstance(s, FunctionLike): + if len(t.items()) == len(s.items()): + return all(is_same_arg_prefix(items, itemt) + for items, itemt in zip(t.items(), s.items())) + return False + + +def is_same_arg_prefix(t: CallableType, s: CallableType) -> bool: + # TODO check argument kinds + return all(is_same_type(argt, args) + for argt, args in zip(t.arg_types, s.arg_types)) + + +def is_more_precise_signature(t: CallableType, s: CallableType) -> bool: + """Is t more precise than s? + + A signature t is more precise than s if all argument types and the return + type of t are more precise than the corresponding types in s. + + Assume that the argument kinds and names are compatible, and that the + argument counts are overlapping. + """ + # TODO generic function types + # Only consider the common prefix of argument types. + for argt, args in zip(t.arg_types, s.arg_types): + if not is_more_precise(argt, args): + return False + return is_more_precise(t.ret_type, s.ret_type) + + +def infer_operator_assignment_method(type: Type, operator: str) -> str: + """Return the method used for operator assignment for given value type. + + For example, if operator is '+', return '__iadd__' or '__add__' depending + on which method is supported by the type. + """ + method = nodes.op_methods[operator] + if isinstance(type, Instance): + if operator in nodes.ops_with_inplace_method: + inplace = '__i' + method[2:] + if type.type.has_readable_member(inplace): + method = inplace + return method + + +def is_valid_inferred_type(typ: Type) -> bool: + """Is an inferred type valid? + + Examples of invalid types include the None type or a type with a None component. + """ + if is_same_type(typ, NoneTyp()): + # With strict Optional checking, we *may* eventually infer NoneTyp, but + # we only do that if we can't infer a specific Optional type. This + # resolution happens in leave_partial_types when we pop a partial types + # scope. + return False + if is_same_type(typ, UninhabitedType()): + return False + elif isinstance(typ, Instance): + for arg in typ.args: + if not is_valid_inferred_type(arg): + return False + elif isinstance(typ, TupleType): + for item in typ.items: + if not is_valid_inferred_type(item): + return False + return True diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 2fee197d9c86d..97a50142901eb 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1,1822 +1,1822 @@ -"""Expression type checker. This file is conceptually part of TypeChecker.""" - -from typing import cast, Dict, List, Tuple, Callable, Union, Optional - -from mypy.types import ( - Type, AnyType, CallableType, Overloaded, NoneTyp, Void, TypeVarDef, - TupleType, Instance, TypeVarId, TypeVarType, ErasedType, UnionType, - PartialType, DeletedType, UnboundType, UninhabitedType, TypeType -) -from mypy.nodes import ( - NameExpr, RefExpr, Var, FuncDef, OverloadedFuncDef, TypeInfo, CallExpr, - Node, MemberExpr, IntExpr, StrExpr, BytesExpr, UnicodeExpr, FloatExpr, - OpExpr, UnaryExpr, IndexExpr, CastExpr, RevealTypeExpr, TypeApplication, ListExpr, - TupleExpr, DictExpr, FuncExpr, SuperExpr, SliceExpr, Context, - ListComprehension, GeneratorExpr, SetExpr, MypyFile, Decorator, - ConditionalExpr, ComparisonExpr, TempNode, SetComprehension, - DictionaryComprehension, ComplexExpr, EllipsisExpr, - TypeAliasExpr, BackquoteExpr, ARG_POS, ARG_NAMED, ARG_STAR2 -) -from mypy.nodes import function_type -from mypy import nodes -import mypy.checker -from mypy import types -from mypy.sametypes import is_same_type -from mypy.erasetype import replace_meta_vars -from mypy.messages import MessageBuilder -from mypy import messages -from mypy.infer import infer_type_arguments, infer_function_type_arguments -from mypy import join -from mypy.subtypes import is_subtype, is_equivalent -from mypy import applytype -from mypy import erasetype -from mypy.checkmember import analyze_member_access, type_object_type -from mypy.semanal import self_type -from mypy.constraints import get_actual_type -from mypy.checkstrformat import StringFormatterChecker -from mypy.expandtype import expand_type -import mypy.checkexpr - -from mypy import experiments - -# Type of callback user for checking individual function arguments. See -# check_args() below for details. -ArgChecker = Callable[[Type, Type, int, Type, int, int, CallableType, Context, MessageBuilder], - None] - - -class Finished(Exception): - """Raised if we can terminate overload argument check early (no match).""" - - -class ExpressionChecker: - """Expression type checker. - - This class works closely together with checker.TypeChecker. - """ - - # Some services are provided by a TypeChecker instance. - chk = None # type: mypy.checker.TypeChecker - # This is shared with TypeChecker, but stored also here for convenience. - msg = None # type: MessageBuilder - - strfrm_checker = None # type: mypy.checkstrformat.StringFormatterChecker - - def __init__(self, - chk: 'mypy.checker.TypeChecker', - msg: MessageBuilder) -> None: - """Construct an expression type checker.""" - self.chk = chk - self.msg = msg - self.strfrm_checker = mypy.checkexpr.StringFormatterChecker(self, self.chk, self.msg) - - def visit_name_expr(self, e: NameExpr) -> Type: - """Type check a name expression. - - It can be of any kind: local, member or global. - """ - result = self.analyze_ref_expr(e) - return self.chk.narrow_type_from_binder(e, result) - - def analyze_ref_expr(self, e: RefExpr, lvalue: bool = False) -> Type: - result = None # type: Type - node = e.node - if isinstance(node, Var): - # Variable reference. - result = self.analyze_var_ref(node, e) - if isinstance(result, PartialType): - if result.type is None: - # 'None' partial type. It has a well-defined type. In an lvalue context - # we want to preserve the knowledge of it being a partial type. - if not lvalue: - result = NoneTyp() - else: - partial_types = self.chk.find_partial_types(node) - if partial_types is not None and not self.chk.current_node_deferred: - context = partial_types[node] - self.msg.fail(messages.NEED_ANNOTATION_FOR_VAR, context) - result = AnyType() - elif isinstance(node, FuncDef): - # Reference to a global function. - result = function_type(node, self.named_type('builtins.function')) - elif isinstance(node, OverloadedFuncDef): - result = node.type - elif isinstance(node, TypeInfo): - # Reference to a type object. - result = type_object_type(node, self.named_type) - elif isinstance(node, MypyFile): - # Reference to a module object. - result = self.named_type('builtins.module') - elif isinstance(node, Decorator): - result = self.analyze_var_ref(node.var, e) - else: - # Unknown reference; use any type implicitly to avoid - # generating extra type errors. - result = AnyType() - return result - - def analyze_var_ref(self, var: Var, context: Context) -> Type: - if not var.type: - if not var.is_ready and self.chk.typing_mode_full(): - self.chk.handle_cannot_determine_type(var.name(), context) - # Implicit 'Any' type. - return AnyType() - else: - # Look up local type of variable with type (inferred or explicit). - val = self.chk.binder.get(var) - if val is None: - return var.type - else: - return val - - def visit_call_expr(self, e: CallExpr) -> Type: - """Type check a call expression.""" - if e.analyzed: - # It's really a special form that only looks like a call. - return self.accept(e.analyzed, self.chk.type_context[-1]) - self.try_infer_partial_type(e) - callee_type = self.accept(e.callee) - if (self.chk.options.disallow_untyped_calls and - self.chk.typing_mode_full() and - isinstance(callee_type, CallableType) - and callee_type.implicit): - return self.msg.untyped_function_call(callee_type, e) - return self.check_call_expr_with_callee_type(callee_type, e) - - # Types and methods that can be used to infer partial types. - item_args = {'builtins.list': ['append'], - 'builtins.set': ['add', 'discard'], - } - container_args = {'builtins.list': {'extend': ['builtins.list']}, - 'builtins.dict': {'update': ['builtins.dict']}, - 'builtins.set': {'update': ['builtins.set', 'builtins.list']}, - } - - def try_infer_partial_type(self, e: CallExpr) -> None: - if isinstance(e.callee, MemberExpr) and isinstance(e.callee.expr, RefExpr): - var = cast(Var, e.callee.expr.node) - partial_types = self.chk.find_partial_types(var) - if partial_types is not None and not self.chk.current_node_deferred: - partial_type = cast(PartialType, var.type) - if partial_type is None or partial_type.type is None: - # A partial None type -> can't infer anything. - return - typename = partial_type.type.fullname() - methodname = e.callee.name - # Sometimes we can infer a full type for a partial List, Dict or Set type. - # TODO: Don't infer argument expression twice. - if (typename in self.item_args and methodname in self.item_args[typename] - and e.arg_kinds == [ARG_POS]): - item_type = self.accept(e.args[0]) - full_item_type = UnionType.make_simplified_union( - [item_type, partial_type.inner_types[0]]) - if mypy.checker.is_valid_inferred_type(full_item_type): - var.type = self.chk.named_generic_type(typename, [full_item_type]) - del partial_types[var] - elif (typename in self.container_args - and methodname in self.container_args[typename] - and e.arg_kinds == [ARG_POS]): - arg_type = self.accept(e.args[0]) - if isinstance(arg_type, Instance): - arg_typename = arg_type.type.fullname() - if arg_typename in self.container_args[typename][methodname]: - full_item_types = [ - UnionType.make_simplified_union([item_type, prev_type]) - for item_type, prev_type - in zip(arg_type.args, partial_type.inner_types) - ] - if all(mypy.checker.is_valid_inferred_type(item_type) - for item_type in full_item_types): - var.type = self.chk.named_generic_type(typename, - list(full_item_types)) - del partial_types[var] - - def check_call_expr_with_callee_type(self, callee_type: Type, - e: CallExpr) -> Type: - """Type check call expression. - - The given callee type overrides the type of the callee - expression. - """ - return self.check_call(callee_type, e.args, e.arg_kinds, e, - e.arg_names, callable_node=e.callee)[0] - - def check_call(self, callee: Type, args: List[Node], - arg_kinds: List[int], context: Context, - arg_names: List[str] = None, - callable_node: Node = None, - arg_messages: MessageBuilder = None) -> Tuple[Type, Type]: - """Type check a call. - - Also infer type arguments if the callee is a generic function. - - Return (result type, inferred callee type). - - Arguments: - callee: type of the called value - args: actual argument expressions - arg_kinds: contains nodes.ARG_* constant for each argument in args - describing whether the argument is positional, *arg, etc. - arg_names: names of arguments (optional) - callable_node: associate the inferred callable type to this node, - if specified - arg_messages: TODO - """ - arg_messages = arg_messages or self.msg - if isinstance(callee, CallableType): - if callee.is_concrete_type_obj() and callee.type_object().is_abstract: - type = callee.type_object() - self.msg.cannot_instantiate_abstract_class( - callee.type_object().name(), type.abstract_attributes, - context) - - formal_to_actual = map_actuals_to_formals( - arg_kinds, arg_names, - callee.arg_kinds, callee.arg_names, - lambda i: self.accept(args[i])) - - if callee.is_generic(): - callee = freshen_generic_callable(callee) - callee = self.infer_function_type_arguments_using_context( - callee, context) - callee = self.infer_function_type_arguments( - callee, args, arg_kinds, formal_to_actual, context) - - arg_types = self.infer_arg_types_in_context2( - callee, args, arg_kinds, formal_to_actual) - - self.check_argument_count(callee, arg_types, arg_kinds, - arg_names, formal_to_actual, context, self.msg) - - self.check_argument_types(arg_types, arg_kinds, callee, - formal_to_actual, context, - messages=arg_messages) - - if (callee.is_type_obj() and (len(arg_types) == 1) - and is_equivalent(callee.ret_type, self.named_type('builtins.type'))): - callee = callee.copy_modified(ret_type=TypeType(arg_types[0])) - - if callable_node: - # Store the inferred callable type. - self.chk.store_type(callable_node, callee) - return callee.ret_type, callee - elif isinstance(callee, Overloaded): - # Type check arguments in empty context. They will be checked again - # later in a context derived from the signature; these types are - # only used to pick a signature variant. - self.msg.disable_errors() - arg_types = self.infer_arg_types_in_context(None, args) - self.msg.enable_errors() - - target = self.overload_call_target(arg_types, arg_kinds, arg_names, - callee, context, - messages=arg_messages) - return self.check_call(target, args, arg_kinds, context, arg_names, - arg_messages=arg_messages) - elif isinstance(callee, AnyType) or self.chk.typing_mode_none(): - self.infer_arg_types_in_context(None, args) - return AnyType(), AnyType() - elif isinstance(callee, UnionType): - self.msg.disable_type_names += 1 - results = [self.check_call(subtype, args, arg_kinds, context, arg_names, - arg_messages=arg_messages) - for subtype in callee.items] - self.msg.disable_type_names -= 1 - return (UnionType.make_simplified_union([res[0] for res in results]), - callee) - elif isinstance(callee, Instance): - call_function = analyze_member_access('__call__', callee, context, - False, False, self.named_type, self.not_ready_callback, - self.msg) - return self.check_call(call_function, args, arg_kinds, context, arg_names, - callable_node, arg_messages) - elif isinstance(callee, TypeVarType): - return self.check_call(callee.upper_bound, args, arg_kinds, context, arg_names, - callable_node, arg_messages) - elif isinstance(callee, TypeType): - # Pass the original Type[] as context since that's where errors should go. - item = self.analyze_type_type_callee(callee.item, callee) - return self.check_call(item, args, arg_kinds, context, arg_names, - callable_node, arg_messages) - else: - return self.msg.not_callable(callee, context), AnyType() - - def analyze_type_type_callee(self, item: Type, context: Context) -> Type: - """Analyze the callee X in X(...) where X is Type[item]. - - Return a Y that we can pass to check_call(Y, ...). - """ - if isinstance(item, AnyType): - return AnyType() - if isinstance(item, Instance): - return type_object_type(item.type, self.named_type) - if isinstance(item, UnionType): - return UnionType([self.analyze_type_type_callee(item, context) - for item in item.items], item.line) - if isinstance(item, TypeVarType): - # Pretend we're calling the typevar's upper bound, - # i.e. its constructor (a poor approximation for reality, - # but better than AnyType...), but replace the return type - # with typevar. - callee = self.analyze_type_type_callee(item.upper_bound, context) - if isinstance(callee, CallableType): - if callee.is_generic(): - callee = None - else: - callee = callee.copy_modified(ret_type=item) - elif isinstance(callee, Overloaded): - if callee.items()[0].is_generic(): - callee = None - else: - callee = Overloaded([c.copy_modified(ret_type=item) - for c in callee.items()]) - if callee: - return callee - - self.msg.unsupported_type_type(item, context) - return AnyType() - - def infer_arg_types_in_context(self, callee: Optional[CallableType], - args: List[Node]) -> List[Type]: - """Infer argument expression types using a callable type as context. - - For example, if callee argument 2 has type List[int], infer the - argument expression with List[int] type context. - """ - # TODO Always called with callee as None, i.e. empty context. - res = [] # type: List[Type] - - fixed = len(args) - if callee: - fixed = min(fixed, callee.max_fixed_args()) - - arg_type = None # type: Type - ctx = None # type: Type - for i, arg in enumerate(args): - if i < fixed: - if callee and i < len(callee.arg_types): - ctx = callee.arg_types[i] - arg_type = self.accept(arg, ctx) - else: - if callee and callee.is_var_arg: - arg_type = self.accept(arg, callee.arg_types[-1]) - else: - arg_type = self.accept(arg) - if has_erased_component(arg_type): - res.append(NoneTyp()) - else: - res.append(arg_type) - return res - - def infer_arg_types_in_context2( - self, callee: CallableType, args: List[Node], arg_kinds: List[int], - formal_to_actual: List[List[int]]) -> List[Type]: - """Infer argument expression types using a callable type as context. - - For example, if callee argument 2 has type List[int], infer the - argument expression with List[int] type context. - - Returns the inferred types of *actual arguments*. - """ - res = [None] * len(args) # type: List[Type] - - for i, actuals in enumerate(formal_to_actual): - for ai in actuals: - if arg_kinds[ai] not in (nodes.ARG_STAR, nodes.ARG_STAR2): - res[ai] = self.accept(args[ai], callee.arg_types[i]) - - # Fill in the rest of the argument types. - for i, t in enumerate(res): - if not t: - res[i] = self.accept(args[i]) - return res - - def infer_function_type_arguments_using_context( - self, callable: CallableType, error_context: Context) -> CallableType: - """Unify callable return type to type context to infer type vars. - - For example, if the return type is set[t] where 't' is a type variable - of callable, and if the context is set[int], return callable modified - by substituting 't' with 'int'. - """ - ctx = self.chk.type_context[-1] - if not ctx: - return callable - # The return type may have references to type metavariables that - # we are inferring right now. We must consider them as indeterminate - # and they are not potential results; thus we replace them with the - # special ErasedType type. On the other hand, class type variables are - # valid results. - erased_ctx = replace_meta_vars(ctx, ErasedType()) - ret_type = callable.ret_type - if isinstance(ret_type, TypeVarType): - if ret_type.values or (not isinstance(ctx, Instance) or - not cast(Instance, ctx).args): - # The return type is a type variable. If it has values, we can't easily restrict - # type inference to conform to the valid values. If it's unrestricted, we could - # infer a too general type for the type variable if we use context, and this could - # result in confusing and spurious type errors elsewhere. - # - # Give up and just use function arguments for type inference. As an exception, - # if the context is a generic instance type, actually use it as context, as - # this *seems* to usually be the reasonable thing to do. - # - # See also github issues #462 and #360. - ret_type = NoneTyp() - args = infer_type_arguments(callable.type_var_ids(), ret_type, erased_ctx) - # Only substitute non-None and non-erased types. - new_args = [] # type: List[Type] - for arg in args: - if isinstance(arg, (NoneTyp, UninhabitedType)) or has_erased_component(arg): - new_args.append(None) - else: - new_args.append(arg) - return cast(CallableType, self.apply_generic_arguments(callable, new_args, - error_context)) - - def infer_function_type_arguments(self, callee_type: CallableType, - args: List[Node], - arg_kinds: List[int], - formal_to_actual: List[List[int]], - context: Context) -> CallableType: - """Infer the type arguments for a generic callee type. - - Infer based on the types of arguments. - - Return a derived callable type that has the arguments applied. - """ - if not self.chk.typing_mode_none(): - # Disable type errors during type inference. There may be errors - # due to partial available context information at this time, but - # these errors can be safely ignored as the arguments will be - # inferred again later. - self.msg.disable_errors() - - arg_types = self.infer_arg_types_in_context2( - callee_type, args, arg_kinds, formal_to_actual) - - self.msg.enable_errors() - - arg_pass_nums = self.get_arg_infer_passes( - callee_type.arg_types, formal_to_actual, len(args)) - - pass1_args = [] # type: List[Optional[Type]] - for i, arg in enumerate(arg_types): - if arg_pass_nums[i] > 1: - pass1_args.append(None) - else: - pass1_args.append(arg) - - inferred_args = infer_function_type_arguments( - callee_type, pass1_args, arg_kinds, formal_to_actual, - strict=self.chk.typing_mode_full()) # type: List[Type] - - if 2 in arg_pass_nums: - # Second pass of type inference. - (callee_type, - inferred_args) = self.infer_function_type_arguments_pass2( - callee_type, args, arg_kinds, formal_to_actual, - inferred_args, context) - - if callee_type.special_sig == 'dict' and len(inferred_args) == 2 and ( - ARG_NAMED in arg_kinds or ARG_STAR2 in arg_kinds): - # HACK: Infer str key type for dict(...) with keyword args. The type system - # can't represent this so we special case it, as this is a pretty common - # thing. This doesn't quite work with all possible subclasses of dict - # if they shuffle type variables around, as we assume that there is a 1-1 - # correspondence with dict type variables. This is a marginal issue and - # a little tricky to fix so it's left unfixed for now. - if isinstance(inferred_args[0], (NoneTyp, UninhabitedType)): - inferred_args[0] = self.named_type('builtins.str') - elif not is_subtype(self.named_type('builtins.str'), inferred_args[0]): - self.msg.fail(messages.KEYWORD_ARGUMENT_REQUIRES_STR_KEY_TYPE, - context) - else: - # In dynamically typed functions use implicit 'Any' types for - # type variables. - inferred_args = [AnyType()] * len(callee_type.variables) - return self.apply_inferred_arguments(callee_type, inferred_args, - context) - - def infer_function_type_arguments_pass2( - self, callee_type: CallableType, - args: List[Node], - arg_kinds: List[int], - formal_to_actual: List[List[int]], - inferred_args: List[Type], - context: Context) -> Tuple[CallableType, List[Type]]: - """Perform second pass of generic function type argument inference. - - The second pass is needed for arguments with types such as Callable[[T], S], - where both T and S are type variables, when the actual argument is a - lambda with inferred types. The idea is to infer the type variable T - in the first pass (based on the types of other arguments). This lets - us infer the argument and return type of the lambda expression and - thus also the type variable S in this second pass. - - Return (the callee with type vars applied, inferred actual arg types). - """ - # None or erased types in inferred types mean that there was not enough - # information to infer the argument. Replace them with None values so - # that they are not applied yet below. - for i, arg in enumerate(inferred_args): - if isinstance(arg, (NoneTyp, UninhabitedType)) or has_erased_component(arg): - inferred_args[i] = None - - callee_type = cast(CallableType, self.apply_generic_arguments( - callee_type, inferred_args, context)) - arg_types = self.infer_arg_types_in_context2( - callee_type, args, arg_kinds, formal_to_actual) - - inferred_args = infer_function_type_arguments( - callee_type, arg_types, arg_kinds, formal_to_actual) - - return callee_type, inferred_args - - def get_arg_infer_passes(self, arg_types: List[Type], - formal_to_actual: List[List[int]], - num_actuals: int) -> List[int]: - """Return pass numbers for args for two-pass argument type inference. - - For each actual, the pass number is either 1 (first pass) or 2 (second - pass). - - Two-pass argument type inference primarily lets us infer types of - lambdas more effectively. - """ - res = [1] * num_actuals - for i, arg in enumerate(arg_types): - if arg.accept(ArgInferSecondPassQuery()): - for j in formal_to_actual[i]: - res[j] = 2 - return res - - def apply_inferred_arguments(self, callee_type: CallableType, - inferred_args: List[Type], - context: Context) -> CallableType: - """Apply inferred values of type arguments to a generic function. - - Inferred_args contains the values of function type arguments. - """ - # Report error if some of the variables could not be solved. In that - # case assume that all variables have type Any to avoid extra - # bogus error messages. - for i, inferred_type in enumerate(inferred_args): - if not inferred_type or has_erased_component(inferred_type): - # Could not infer a non-trivial type for a type variable. - self.msg.could_not_infer_type_arguments( - callee_type, i + 1, context) - inferred_args = [AnyType()] * len(inferred_args) - # Apply the inferred types to the function type. In this case the - # return type must be CallableType, since we give the right number of type - # arguments. - return cast(CallableType, self.apply_generic_arguments(callee_type, - inferred_args, context)) - - def check_argument_count(self, callee: CallableType, actual_types: List[Type], - actual_kinds: List[int], actual_names: List[str], - formal_to_actual: List[List[int]], - context: Context, - messages: Optional[MessageBuilder]) -> bool: - """Check that there is a value for all required arguments to a function. - - Also check that there are no duplicate values for arguments. Report found errors - using 'messages' if it's not None. - - Return False if there were any errors. Otherwise return True - """ - # TODO(jukka): We could return as soon as we find an error if messages is None. - formal_kinds = callee.arg_kinds - - # Collect list of all actual arguments matched to formal arguments. - all_actuals = [] # type: List[int] - for actuals in formal_to_actual: - all_actuals.extend(actuals) - - is_unexpected_arg_error = False # Keep track of errors to avoid duplicate errors. - ok = True # False if we've found any error. - for i, kind in enumerate(actual_kinds): - if i not in all_actuals and ( - kind != nodes.ARG_STAR or - not is_empty_tuple(actual_types[i])): - # Extra actual: not matched by a formal argument. - ok = False - if kind != nodes.ARG_NAMED: - if messages: - messages.too_many_arguments(callee, context) - else: - if messages: - messages.unexpected_keyword_argument( - callee, actual_names[i], context) - is_unexpected_arg_error = True - elif kind == nodes.ARG_STAR and ( - nodes.ARG_STAR not in formal_kinds): - actual_type = actual_types[i] - if isinstance(actual_type, TupleType): - if all_actuals.count(i) < len(actual_type.items): - # Too many tuple items as some did not match. - if messages: - messages.too_many_arguments(callee, context) - ok = False - # *args can be applied even if the function takes a fixed - # number of positional arguments. This may succeed at runtime. - - for i, kind in enumerate(formal_kinds): - if kind == nodes.ARG_POS and (not formal_to_actual[i] and - not is_unexpected_arg_error): - # No actual for a mandatory positional formal. - if messages: - messages.too_few_arguments(callee, context, actual_names) - ok = False - elif kind in [nodes.ARG_POS, nodes.ARG_OPT, - nodes.ARG_NAMED] and is_duplicate_mapping( - formal_to_actual[i], actual_kinds): - if (self.chk.typing_mode_full() or - isinstance(actual_types[formal_to_actual[i][0]], TupleType)): - if messages: - messages.duplicate_argument_value(callee, i, context) - ok = False - elif (kind == nodes.ARG_NAMED and formal_to_actual[i] and - actual_kinds[formal_to_actual[i][0]] != nodes.ARG_NAMED): - # Positional argument when expecting a keyword argument. - if messages: - messages.too_many_positional_arguments(callee, context) - ok = False - return ok - - def check_argument_types(self, arg_types: List[Type], arg_kinds: List[int], - callee: CallableType, - formal_to_actual: List[List[int]], - context: Context, - messages: MessageBuilder = None, - check_arg: ArgChecker = None) -> None: - """Check argument types against a callable type. - - Report errors if the argument types are not compatible. - """ - messages = messages or self.msg - check_arg = check_arg or self.check_arg - # Keep track of consumed tuple *arg items. - tuple_counter = [0] - for i, actuals in enumerate(formal_to_actual): - for actual in actuals: - arg_type = arg_types[actual] - # Check that a *arg is valid as varargs. - if (arg_kinds[actual] == nodes.ARG_STAR and - not self.is_valid_var_arg(arg_type)): - messages.invalid_var_arg(arg_type, context) - if (arg_kinds[actual] == nodes.ARG_STAR2 and - not self.is_valid_keyword_var_arg(arg_type)): - messages.invalid_keyword_var_arg(arg_type, context) - # Get the type of an individual actual argument (for *args - # and **args this is the item type, not the collection type). - actual_type = get_actual_type(arg_type, arg_kinds[actual], - tuple_counter) - check_arg(actual_type, arg_type, arg_kinds[actual], - callee.arg_types[i], - actual + 1, i + 1, callee, context, messages) - - # There may be some remaining tuple varargs items that haven't - # been checked yet. Handle them. - if (callee.arg_kinds[i] == nodes.ARG_STAR and - arg_kinds[actual] == nodes.ARG_STAR and - isinstance(arg_types[actual], TupleType)): - tuplet = cast(TupleType, arg_types[actual]) - while tuple_counter[0] < len(tuplet.items): - actual_type = get_actual_type(arg_type, - arg_kinds[actual], - tuple_counter) - check_arg(actual_type, arg_type, arg_kinds[actual], - callee.arg_types[i], - actual + 1, i + 1, callee, context, messages) - - def check_arg(self, caller_type: Type, original_caller_type: Type, - caller_kind: int, - callee_type: Type, n: int, m: int, callee: CallableType, - context: Context, messages: MessageBuilder) -> None: - """Check the type of a single argument in a call.""" - if isinstance(caller_type, Void): - messages.does_not_return_value(caller_type, context) - elif isinstance(caller_type, DeletedType): - messages.deleted_as_rvalue(caller_type, context) - elif not is_subtype(caller_type, callee_type): - messages.incompatible_argument(n, m, callee, original_caller_type, - caller_kind, context) - - def overload_call_target(self, arg_types: List[Type], arg_kinds: List[int], - arg_names: List[str], - overload: Overloaded, context: Context, - messages: MessageBuilder = None) -> Type: - """Infer the correct overload item to call with given argument types. - - The return value may be CallableType or AnyType (if an unique item - could not be determined). - """ - messages = messages or self.msg - # TODO: For overlapping signatures we should try to get a more precise - # result than 'Any'. - match = [] # type: List[CallableType] - best_match = 0 - for typ in overload.items(): - similarity = self.erased_signature_similarity(arg_types, arg_kinds, arg_names, - typ, context=context) - if similarity > 0 and similarity >= best_match: - if (match and not is_same_type(match[-1].ret_type, - typ.ret_type) and - not mypy.checker.is_more_precise_signature( - match[-1], typ)): - # Ambiguous return type. Either the function overload is - # overlapping (which we don't handle very well here) or the - # caller has provided some Any argument types; in either - # case we'll fall back to Any. It's okay to use Any types - # in calls. - # - # Overlapping overload items are generally fine if the - # overlapping is only possible when there is multiple - # inheritance, as this is rare. See docstring of - # mypy.meet.is_overlapping_types for more about this. - # - # Note that there is no ambiguity if the items are - # covariant in both argument types and return types with - # respect to type precision. We'll pick the best/closest - # match. - # - # TODO: Consider returning a union type instead if the - # overlapping is NOT due to Any types? - return AnyType() - else: - match.append(typ) - best_match = max(best_match, similarity) - if not match: - messages.no_variant_matches_arguments(overload, arg_types, context) - return AnyType() - else: - if len(match) == 1: - return match[0] - else: - # More than one signature matches. Pick the first *non-erased* - # matching signature, or default to the first one if none - # match. - for m in match: - if self.match_signature_types(arg_types, arg_kinds, arg_names, m, - context=context): - return m - return match[0] - - def erased_signature_similarity(self, arg_types: List[Type], arg_kinds: List[int], - arg_names: List[str], callee: CallableType, - context: Context) -> int: - """Determine whether arguments could match the signature at runtime. - - Return similarity level (0 = no match, 1 = can match, 2 = non-promotion match). See - overload_arg_similarity for a discussion of similarity levels. - """ - formal_to_actual = map_actuals_to_formals(arg_kinds, - arg_names, - callee.arg_kinds, - callee.arg_names, - lambda i: arg_types[i]) - - if not self.check_argument_count(callee, arg_types, arg_kinds, arg_names, - formal_to_actual, None, None): - # Too few or many arguments -> no match. - return 0 - - similarity = 2 - - def check_arg(caller_type: Type, original_caller_type: Type, caller_kind: int, - callee_type: Type, n: int, m: int, callee: CallableType, - context: Context, messages: MessageBuilder) -> None: - nonlocal similarity - similarity = min(similarity, - overload_arg_similarity(caller_type, callee_type)) - if similarity == 0: - # No match -- exit early since none of the remaining work can change - # the result. - raise Finished - - try: - self.check_argument_types(arg_types, arg_kinds, callee, formal_to_actual, - context=context, check_arg=check_arg) - except Finished: - pass - - return similarity - - def match_signature_types(self, arg_types: List[Type], arg_kinds: List[int], - arg_names: List[str], callee: CallableType, - context: Context) -> bool: - """Determine whether arguments types match the signature. - - Assume that argument counts are compatible. - - Return True if arguments match. - """ - formal_to_actual = map_actuals_to_formals(arg_kinds, - arg_names, - callee.arg_kinds, - callee.arg_names, - lambda i: arg_types[i]) - ok = True - - def check_arg(caller_type: Type, original_caller_type: Type, caller_kind: int, - callee_type: Type, n: int, m: int, callee: CallableType, - context: Context, messages: MessageBuilder) -> None: - nonlocal ok - if not is_subtype(caller_type, callee_type): - ok = False - - self.check_argument_types(arg_types, arg_kinds, callee, formal_to_actual, - context=context, check_arg=check_arg) - return ok - - def apply_generic_arguments(self, callable: CallableType, types: List[Type], - context: Context) -> Type: - """Simple wrapper around mypy.applytype.apply_generic_arguments.""" - return applytype.apply_generic_arguments(callable, types, self.msg, context) - - def apply_generic_arguments2(self, overload: Overloaded, types: List[Type], - context: Context) -> Type: - items = [] # type: List[CallableType] - for item in overload.items(): - applied = self.apply_generic_arguments(item, types, context) - if isinstance(applied, CallableType): - items.append(applied) - else: - # There was an error. - return AnyType() - return Overloaded(items) - - def visit_member_expr(self, e: MemberExpr) -> Type: - """Visit member expression (of form e.id).""" - result = self.analyze_ordinary_member_access(e, False) - return self.chk.narrow_type_from_binder(e, result) - - def analyze_ordinary_member_access(self, e: MemberExpr, - is_lvalue: bool) -> Type: - """Analyse member expression or member lvalue.""" - if e.kind is not None: - # This is a reference to a module attribute. - return self.analyze_ref_expr(e) - else: - # This is a reference to a non-module attribute. - return analyze_member_access(e.name, self.accept(e.expr), e, - is_lvalue, False, - self.named_type, self.not_ready_callback, self.msg) - - def analyze_external_member_access(self, member: str, base_type: Type, - context: Context) -> Type: - """Analyse member access that is external, i.e. it cannot - refer to private definitions. Return the result type. - """ - # TODO remove; no private definitions in mypy - return analyze_member_access(member, base_type, context, False, False, - self.named_type, self.not_ready_callback, self.msg) - - def visit_int_expr(self, e: IntExpr) -> Type: - """Type check an integer literal (trivial).""" - return self.named_type('builtins.int') - - def visit_str_expr(self, e: StrExpr) -> Type: - """Type check a string literal (trivial).""" - return self.named_type('builtins.str') - - def visit_bytes_expr(self, e: BytesExpr) -> Type: - """Type check a bytes literal (trivial).""" - return self.named_type('builtins.bytes') - - def visit_unicode_expr(self, e: UnicodeExpr) -> Type: - """Type check a unicode literal (trivial).""" - return self.named_type('builtins.unicode') - - def visit_float_expr(self, e: FloatExpr) -> Type: - """Type check a float literal (trivial).""" - return self.named_type('builtins.float') - - def visit_complex_expr(self, e: ComplexExpr) -> Type: - """Type check a complex literal.""" - return self.named_type('builtins.complex') - - def visit_ellipsis(self, e: EllipsisExpr) -> Type: - """Type check '...'.""" - if self.chk.options.python_version[0] >= 3: - return self.named_type('builtins.ellipsis') - else: - # '...' is not valid in normal Python 2 code, but it can - # be used in stubs. The parser makes sure that we only - # get this far if we are in a stub, and we can safely - # return 'object' as ellipsis is special cased elsewhere. - # The builtins.ellipsis type does not exist in Python 2. - return self.named_type('builtins.object') - - def visit_op_expr(self, e: OpExpr) -> Type: - """Type check a binary operator expression.""" - if e.op == 'and' or e.op == 'or': - return self.check_boolean_op(e, e) - if e.op == '*' and isinstance(e.left, ListExpr): - # Expressions of form [...] * e get special type inference. - return self.check_list_multiply(e) - if e.op == '%' and isinstance(e.left, (StrExpr, BytesExpr)): - return self.strfrm_checker.check_str_interpolation(cast(StrExpr, e.left), e.right) - left_type = self.accept(e.left) - - if e.op in nodes.op_methods: - method = self.get_operator_method(e.op) - result, method_type = self.check_op(method, left_type, e.right, e, - allow_reverse=True) - e.method_type = method_type - return result - else: - raise RuntimeError('Unknown operator {}'.format(e.op)) - - def visit_comparison_expr(self, e: ComparisonExpr) -> Type: - """Type check a comparison expression. - - Comparison expressions are type checked consecutive-pair-wise - That is, 'a < b > c == d' is check as 'a < b and b > c and c == d' - """ - result = None # type: mypy.types.Type - - # Check each consecutive operand pair and their operator - for left, right, operator in zip(e.operands, e.operands[1:], e.operators): - left_type = self.accept(left) - - method_type = None # type: mypy.types.Type - - if operator == 'in' or operator == 'not in': - right_type = self.accept(right) # TODO only evaluate if needed - - # Keep track of whether we get type check errors (these won't be reported, they - # are just to verify whether something is valid typing wise). - local_errors = self.msg.copy() - local_errors.disable_count = 0 - sub_result, method_type = self.check_op_local('__contains__', right_type, - left, e, local_errors) - if isinstance(right_type, PartialType): - # We don't really know if this is an error or not, so just shut up. - pass - elif (local_errors.is_errors() and - # is_valid_var_arg is True for any Iterable - self.is_valid_var_arg(right_type)): - itertype = self.chk.analyze_iterable_item_type(right) - method_type = CallableType( - [left_type], - [nodes.ARG_POS], - [None], - self.chk.bool_type(), - self.named_type('builtins.function')) - sub_result = self.chk.bool_type() - if not is_subtype(left_type, itertype): - self.msg.unsupported_operand_types('in', left_type, right_type, e) - else: - self.msg.add_errors(local_errors) - if operator == 'not in': - sub_result = self.chk.bool_type() - elif operator in nodes.op_methods: - method = self.get_operator_method(operator) - sub_result, method_type = self.check_op(method, left_type, right, e, - allow_reverse=True) - - elif operator == 'is' or operator == 'is not': - sub_result = self.chk.bool_type() - method_type = None - else: - raise RuntimeError('Unknown comparison operator {}'.format(operator)) - - e.method_types.append(method_type) - - # Determine type of boolean-and of result and sub_result - if result is None: - result = sub_result - else: - # TODO: check on void needed? - self.check_not_void(sub_result, e) - result = join.join_types(result, sub_result) - - return result - - def get_operator_method(self, op: str) -> str: - if op == '/' and self.chk.options.python_version[0] == 2: - # TODO also check for "from __future__ import division" - return '__div__' - else: - return nodes.op_methods[op] - - def check_op_local(self, method: str, base_type: Type, arg: Node, - context: Context, local_errors: MessageBuilder) -> Tuple[Type, Type]: - """Type check a binary operation which maps to a method call. - - Return tuple (result type, inferred operator method type). - """ - method_type = analyze_member_access(method, base_type, context, False, False, - self.named_type, self.not_ready_callback, local_errors) - return self.check_call(method_type, [arg], [nodes.ARG_POS], - context, arg_messages=local_errors) - - def check_op(self, method: str, base_type: Type, arg: Node, - context: Context, - allow_reverse: bool = False) -> Tuple[Type, Type]: - """Type check a binary operation which maps to a method call. - - Return tuple (result type, inferred operator method type). - """ - # Use a local error storage for errors related to invalid argument - # type (but NOT other errors). This error may need to be suppressed - # for operators which support __rX methods. - local_errors = self.msg.copy() - local_errors.disable_count = 0 - if not allow_reverse or self.has_member(base_type, method): - result = self.check_op_local(method, base_type, arg, context, - local_errors) - if allow_reverse: - arg_type = self.chk.type_map[arg] - if isinstance(arg_type, AnyType): - # If the right operand has type Any, we can't make any - # conjectures about the type of the result, since the - # operand could have a __r method that returns anything. - - # However, in weak mode, we do make conjectures. - if not self.chk.typing_mode_weak(): - result = AnyType(), result[1] - success = not local_errors.is_errors() - else: - result = AnyType(), AnyType() - success = False - if success or not allow_reverse or isinstance(base_type, AnyType): - # We were able to call the normal variant of the operator method, - # or there was some problem not related to argument type - # validity, or the operator has no __rX method. In any case, we - # don't need to consider the __rX method. - self.msg.add_errors(local_errors) - return result - else: - # Calling the operator method was unsuccessful. Try the __rX - # method of the other operand instead. - rmethod = self.get_reverse_op_method(method) - arg_type = self.accept(arg) - if self.has_member(arg_type, rmethod): - method_type = self.analyze_external_member_access( - rmethod, arg_type, context) - temp = TempNode(base_type) - return self.check_call(method_type, [temp], [nodes.ARG_POS], - context) - else: - # No __rX method either. Do deferred type checking to produce - # error message that we may have missed previously. - # TODO Fix type checking an expression more than once. - return self.check_op_local(method, base_type, arg, context, - self.msg) - - def get_reverse_op_method(self, method: str) -> str: - if method == '__div__' and self.chk.options.python_version[0] == 2: - return '__rdiv__' - else: - return nodes.reverse_op_methods[method] - - def check_boolean_op(self, e: OpExpr, context: Context) -> Type: - """Type check a boolean operation ('and' or 'or').""" - - # A boolean operation can evaluate to either of the operands. - - # We use the current type context to guide the type inference of of - # the left operand. We also use the left operand type to guide the type - # inference of the right operand so that expressions such as - # '[1] or []' are inferred correctly. - ctx = self.chk.type_context[-1] - left_type = self.accept(e.left, ctx) - - if e.op == 'and': - right_map, _ = \ - mypy.checker.find_isinstance_check(e.left, self.chk.type_map, - self.chk.typing_mode_weak()) - elif e.op == 'or': - _, right_map = \ - mypy.checker.find_isinstance_check(e.left, self.chk.type_map, - self.chk.typing_mode_weak()) - else: - right_map = None - - with self.chk.binder.frame_context(): - if right_map: - for var, type in right_map.items(): - self.chk.binder.push(var, type) - - right_type = self.accept(e.right, left_type) - - self.check_not_void(left_type, context) - self.check_not_void(right_type, context) - return UnionType.make_simplified_union([left_type, right_type]) - - def check_list_multiply(self, e: OpExpr) -> Type: - """Type check an expression of form '[...] * e'. - - Type inference is special-cased for this common construct. - """ - right_type = self.accept(e.right) - if is_subtype(right_type, self.named_type('builtins.int')): - # Special case: [...] * . Use the type context of the - # OpExpr, since the multiplication does not affect the type. - left_type = self.accept(e.left, context=self.chk.type_context[-1]) - else: - left_type = self.accept(e.left) - result, method_type = self.check_op('__mul__', left_type, e.right, e) - e.method_type = method_type - return result - - def visit_unary_expr(self, e: UnaryExpr) -> Type: - """Type check an unary operation ('not', '-', '+' or '~').""" - operand_type = self.accept(e.expr) - op = e.op - if op == 'not': - self.check_not_void(operand_type, e) - result = self.chk.bool_type() # type: Type - elif op == '-': - method_type = self.analyze_external_member_access('__neg__', - operand_type, e) - result, method_type = self.check_call(method_type, [], [], e) - e.method_type = method_type - elif op == '+': - method_type = self.analyze_external_member_access('__pos__', - operand_type, e) - result, method_type = self.check_call(method_type, [], [], e) - e.method_type = method_type - else: - assert op == '~', "unhandled unary operator" - method_type = self.analyze_external_member_access('__invert__', - operand_type, e) - result, method_type = self.check_call(method_type, [], [], e) - e.method_type = method_type - return result - - def visit_index_expr(self, e: IndexExpr) -> Type: - """Type check an index expression (base[index]). - - It may also represent type application. - """ - result = self.visit_index_expr_helper(e) - return self.chk.narrow_type_from_binder(e, result) - - def visit_index_expr_helper(self, e: IndexExpr) -> Type: - if e.analyzed: - # It's actually a type application. - return self.accept(e.analyzed) - left_type = self.accept(e.base) - if isinstance(left_type, TupleType) and self.chk.typing_mode_full(): - # Special case for tuples. They support indexing only by integer - # literals. (Except in weak type checking mode.) - index = e.index - if isinstance(index, SliceExpr): - return self.visit_tuple_slice_helper(left_type, index) - - ok = False - if isinstance(index, IntExpr): - n = index.value - ok = True - elif isinstance(index, UnaryExpr): - if index.op == '-': - operand = index.expr - if isinstance(operand, IntExpr): - n = len(left_type.items) - operand.value - ok = True - if ok: - if n >= 0 and n < len(left_type.items): - return left_type.items[n] - else: - self.chk.fail(messages.TUPLE_INDEX_OUT_OF_RANGE, e) - return AnyType() - else: - self.chk.fail(messages.TUPLE_INDEX_MUST_BE_AN_INT_LITERAL, e) - return AnyType() - else: - result, method_type = self.check_op('__getitem__', left_type, e.index, e) - e.method_type = method_type - return result - - def visit_tuple_slice_helper(self, left_type: TupleType, slic: SliceExpr): - begin = None # type: int - end = None # type: int - stride = None # type:int - - if slic.begin_index: - begin = self._get_value(slic.begin_index) - if begin is None: - self.chk.fail( - messages.TUPLE_SLICE_MUST_BE_AN_INT_LITERAL, - slic.begin_index) - return AnyType() - - if slic.end_index: - end = self._get_value(slic.end_index) - if end is None: - self.chk.fail( - messages.TUPLE_SLICE_MUST_BE_AN_INT_LITERAL, - slic.end_index) - return AnyType() - - if slic.stride: - stride = self._get_value(slic.stride) - if stride is None: - self.chk.fail( - messages.TUPLE_SLICE_MUST_BE_AN_INT_LITERAL, - slic.stride) - return AnyType() - - return left_type.slice(begin, stride, end) - - def _get_value(self, index: Node) -> Optional[int]: - if isinstance(index, IntExpr): - return index.value - elif isinstance(index, UnaryExpr): - if index.op == '-': - operand = index.expr - if isinstance(operand, IntExpr): - return -1 * operand.value - return None - - def visit_cast_expr(self, expr: CastExpr) -> Type: - """Type check a cast expression.""" - source_type = self.accept(expr.expr, context=AnyType()) - target_type = expr.type - if self.chk.options.warn_redundant_casts and is_same_type(source_type, target_type): - self.msg.redundant_cast(target_type, expr) - if not self.is_valid_cast(source_type, target_type): - self.msg.invalid_cast(target_type, source_type, expr) - return target_type - - def is_valid_cast(self, source_type: Type, target_type: Type) -> bool: - """Is a cast from source_type to target_type meaningful?""" - return (isinstance(target_type, AnyType) or - (not isinstance(source_type, Void) and - not isinstance(target_type, Void))) - - def visit_reveal_type_expr(self, expr: RevealTypeExpr) -> Type: - """Type check a reveal_type expression.""" - revealed_type = self.accept(expr.expr) - self.msg.reveal_type(revealed_type, expr) - return revealed_type - - def visit_type_application(self, tapp: TypeApplication) -> Type: - """Type check a type application (expr[type, ...]).""" - self.chk.fail(messages.GENERIC_TYPE_NOT_VALID_AS_EXPRESSION, tapp) - return AnyType() - - def visit_type_alias_expr(self, alias: TypeAliasExpr) -> Type: - return AnyType() - - def visit_list_expr(self, e: ListExpr) -> Type: - """Type check a list expression [...].""" - return self.check_list_or_set_expr(e.items, 'builtins.list', '', - e) - - def visit_set_expr(self, e: SetExpr) -> Type: - return self.check_list_or_set_expr(e.items, 'builtins.set', '', e) - - def check_list_or_set_expr(self, items: List[Node], fullname: str, - tag: str, context: Context) -> Type: - # Translate into type checking a generic function call. - tvdef = TypeVarDef('T', -1, [], self.chk.object_type()) - tv = TypeVarType(tvdef) - constructor = CallableType( - [tv], - [nodes.ARG_STAR], - [None], - self.chk.named_generic_type(fullname, [tv]), - self.named_type('builtins.function'), - name=tag, - variables=[tvdef]) - return self.check_call(constructor, - items, - [nodes.ARG_POS] * len(items), context)[0] - - def visit_tuple_expr(self, e: TupleExpr) -> Type: - """Type check a tuple expression.""" - ctx = None # type: TupleType - # Try to determine type context for type inference. - if isinstance(self.chk.type_context[-1], TupleType): - t = self.chk.type_context[-1] - if len(t.items) == len(e.items): - ctx = t - # Infer item types. - items = [] # type: List[Type] - for i in range(len(e.items)): - item = e.items[i] - tt = None # type: Type - if not ctx: - tt = self.accept(item) - else: - tt = self.accept(item, ctx.items[i]) - self.check_not_void(tt, e) - items.append(tt) - fallback_item = join.join_type_list(items) - return TupleType(items, self.chk.named_generic_type('builtins.tuple', [fallback_item])) - - def visit_dict_expr(self, e: DictExpr) -> Type: - # Translate into type checking a generic function call. - ktdef = TypeVarDef('KT', -1, [], self.chk.object_type()) - vtdef = TypeVarDef('VT', -2, [], self.chk.object_type()) - kt = TypeVarType(ktdef) - vt = TypeVarType(vtdef) - # The callable type represents a function like this: - # - # def (*v: Tuple[kt, vt]) -> Dict[kt, vt]: ... - constructor = CallableType( - [TupleType([kt, vt], self.named_type('builtins.tuple'))], - [nodes.ARG_STAR], - [None], - self.chk.named_generic_type('builtins.dict', [kt, vt]), - self.named_type('builtins.function'), - name='', - variables=[ktdef, vtdef]) - # Synthesize function arguments. - args = [] # type: List[Node] - for key, value in e.items: - args.append(TupleExpr([key, value])) - return self.check_call(constructor, - args, - [nodes.ARG_POS] * len(args), e)[0] - - def visit_func_expr(self, e: FuncExpr) -> Type: - """Type check lambda expression.""" - inferred_type = self.infer_lambda_type_using_context(e) - if not inferred_type: - # No useful type context. - ret_type = e.expr().accept(self.chk) - if not e.arguments: - # Form 'lambda: e'; just use the inferred return type. - return CallableType([], [], [], ret_type, self.named_type('builtins.function')) - else: - # TODO: Consider reporting an error. However, this is fine if - # we are just doing the first pass in contextual type - # inference. - return AnyType() - else: - # Type context available. - self.chk.check_func_item(e, type_override=inferred_type) - if e.expr() not in self.chk.type_map: - self.accept(e.expr()) - ret_type = self.chk.type_map[e.expr()] - if isinstance(ret_type, NoneTyp): - # For "lambda ...: None", just use type from the context. - # Important when the context is Callable[..., None] which - # really means Void. See #1425. - return inferred_type - return replace_callable_return_type(inferred_type, ret_type) - - def infer_lambda_type_using_context(self, e: FuncExpr) -> CallableType: - """Try to infer lambda expression type using context. - - Return None if could not infer type. - """ - # TODO also accept 'Any' context - ctx = self.chk.type_context[-1] - if not ctx or not isinstance(ctx, CallableType): - return None - - # The context may have function type variables in it. We replace them - # since these are the type variables we are ultimately trying to infer; - # they must be considered as indeterminate. We use ErasedType since it - # does not affect type inference results (it is for purposes like this - # only). - ctx = replace_meta_vars(ctx, ErasedType()) - - callable_ctx = cast(CallableType, ctx) - - arg_kinds = [arg.kind for arg in e.arguments] - - if callable_ctx.is_ellipsis_args: - # Fill in Any arguments to match the arguments of the lambda. - callable_ctx = callable_ctx.copy_modified( - is_ellipsis_args=False, - arg_types=[AnyType()] * len(arg_kinds), - arg_kinds=arg_kinds - ) - - if callable_ctx.arg_kinds != arg_kinds: - # Incompatible context; cannot use it to infer types. - self.chk.fail(messages.CANNOT_INFER_LAMBDA_TYPE, e) - return None - - return callable_ctx - - def visit_super_expr(self, e: SuperExpr) -> Type: - """Type check a super expression (non-lvalue).""" - t = self.analyze_super(e, False) - return t - - def analyze_super(self, e: SuperExpr, is_lvalue: bool) -> Type: - """Type check a super expression.""" - if e.info and e.info.bases: - # TODO fix multiple inheritance etc - if len(e.info.mro) < 2: - self.chk.fail('Internal error: unexpected mro for {}: {}'.format( - e.info.name(), e.info.mro), e) - return AnyType() - for base in e.info.mro[1:]: - if e.name in base.names or base == e.info.mro[-1]: - if e.info.fallback_to_any and base == e.info.mro[-1]: - # There's an undefined base class, and we're - # at the end of the chain. That's not an error. - return AnyType() - if not self.chk.typing_mode_full(): - return AnyType() - return analyze_member_access(e.name, self_type(e.info), e, - is_lvalue, True, - self.named_type, self.not_ready_callback, - self.msg, base) - else: - # Invalid super. This has been reported by the semantic analyzer. - return AnyType() - - def visit_slice_expr(self, e: SliceExpr) -> Type: - for index in [e.begin_index, e.end_index, e.stride]: - if index: - t = self.accept(index) - self.chk.check_subtype(t, self.named_type('builtins.int'), - index, messages.INVALID_SLICE_INDEX) - return self.named_type('builtins.slice') - - def visit_list_comprehension(self, e: ListComprehension) -> Type: - return self.check_generator_or_comprehension( - e.generator, 'builtins.list', '') - - def visit_set_comprehension(self, e: SetComprehension) -> Type: - return self.check_generator_or_comprehension( - e.generator, 'builtins.set', '') - - def visit_generator_expr(self, e: GeneratorExpr) -> Type: - return self.check_generator_or_comprehension(e, 'typing.Iterator', - '') - - def check_generator_or_comprehension(self, gen: GeneratorExpr, - type_name: str, - id_for_messages: str) -> Type: - """Type check a generator expression or a list comprehension.""" - self.check_for_comp(gen) - - # Infer the type of the list comprehension by using a synthetic generic - # callable type. - tvdef = TypeVarDef('T', -1, [], self.chk.object_type()) - tv = TypeVarType(tvdef) - constructor = CallableType( - [tv], - [nodes.ARG_POS], - [None], - self.chk.named_generic_type(type_name, [tv]), - self.chk.named_type('builtins.function'), - name=id_for_messages, - variables=[tvdef]) - return self.check_call(constructor, - [gen.left_expr], [nodes.ARG_POS], gen)[0] - - def visit_dictionary_comprehension(self, e: DictionaryComprehension): - """Type check a dictionary comprehension.""" - self.check_for_comp(e) - - # Infer the type of the list comprehension by using a synthetic generic - # callable type. - ktdef = TypeVarDef('KT', -1, [], self.chk.object_type()) - vtdef = TypeVarDef('VT', -2, [], self.chk.object_type()) - kt = TypeVarType(ktdef) - vt = TypeVarType(vtdef) - constructor = CallableType( - [kt, vt], - [nodes.ARG_POS, nodes.ARG_POS], - [None, None], - self.chk.named_generic_type('builtins.dict', [kt, vt]), - self.chk.named_type('builtins.function'), - name='', - variables=[ktdef, vtdef]) - return self.check_call(constructor, - [e.key, e.value], [nodes.ARG_POS, nodes.ARG_POS], e)[0] - - def check_for_comp(self, e: Union[GeneratorExpr, DictionaryComprehension]) -> None: - """Check the for_comp part of comprehensions. That is the part from 'for': - ... for x in y if z - """ - with self.chk.binder.frame_context(): - for index, sequence, conditions in zip(e.indices, e.sequences, - e.condlists): - sequence_type = self.chk.analyze_iterable_item_type(sequence) - self.chk.analyze_index_variables(index, sequence_type, e) - for condition in conditions: - self.accept(condition) - - def visit_conditional_expr(self, e: ConditionalExpr) -> Type: - cond_type = self.accept(e.cond) - self.check_not_void(cond_type, e) - ctx = self.chk.type_context[-1] - - # Gain type information from isinstance if it is there - # but only for the current expression - if_map, else_map = mypy.checker.find_isinstance_check( - e.cond, - self.chk.type_map, - self.chk.typing_mode_weak()) - - if_type = self.analyze_cond_branch(if_map, e.if_expr, context=ctx) - - if not mypy.checker.is_valid_inferred_type(if_type): - # Analyze the right branch disregarding the left branch. - else_type = self.analyze_cond_branch(else_map, e.else_expr, context=ctx) - - # If it would make a difference, re-analyze the left - # branch using the right branch's type as context. - if ctx is None or not is_equivalent(else_type, ctx): - # TODO: If it's possible that the previous analysis of - # the left branch produced errors that are avoided - # using this context, suppress those errors. - if_type = self.analyze_cond_branch(if_map, e.if_expr, context=else_type) - - else: - # Analyze the right branch in the context of the left - # branch's type. - else_type = self.analyze_cond_branch(else_map, e.else_expr, context=if_type) - - res = join.join_types(if_type, else_type) - - return res - - def analyze_cond_branch(self, map: Optional[Dict[Node, Type]], - node: Node, context: Optional[Type]) -> Type: - with self.chk.binder.frame_context(): - if map: - for var, type in map.items(): - self.chk.binder.push(var, type) - return self.accept(node, context=context) - - def visit_backquote_expr(self, e: BackquoteExpr) -> Type: - self.accept(e.expr) - return self.named_type('builtins.str') - - # - # Helpers - # - - def accept(self, node: Node, context: Type = None) -> Type: - """Type check a node. Alias for TypeChecker.accept.""" - return self.chk.accept(node, context) - - def check_not_void(self, typ: Type, context: Context) -> None: - """Generate an error if type is Void.""" - self.chk.check_not_void(typ, context) - - def is_boolean(self, typ: Type) -> bool: - """Is type compatible with bool?""" - return is_subtype(typ, self.chk.bool_type()) - - def named_type(self, name: str) -> Instance: - """Return an instance type with type given by the name and no type - arguments. Alias for TypeChecker.named_type. - """ - return self.chk.named_type(name) - - def is_valid_var_arg(self, typ: Type) -> bool: - """Is a type valid as a *args argument?""" - return (isinstance(typ, TupleType) or - is_subtype(typ, self.chk.named_generic_type('typing.Iterable', - [AnyType()])) or - isinstance(typ, AnyType)) - - def is_valid_keyword_var_arg(self, typ: Type) -> bool: - """Is a type valid as a **kwargs argument?""" - return is_subtype(typ, self.chk.named_generic_type( - 'builtins.dict', [self.named_type('builtins.str'), AnyType()])) - - def has_non_method(self, typ: Type, member: str) -> bool: - """Does type have a member variable / property with the given name?""" - if isinstance(typ, Instance): - return (not typ.type.has_method(member) and - typ.type.has_readable_member(member)) - else: - return False - - def has_member(self, typ: Type, member: str) -> bool: - """Does type have member with the given name?""" - # TODO TupleType => also consider tuple attributes - if isinstance(typ, Instance): - return typ.type.has_readable_member(member) - elif isinstance(typ, AnyType): - return True - elif isinstance(typ, UnionType): - result = all(self.has_member(x, member) for x in typ.items) - return result - elif isinstance(typ, TupleType): - return self.has_member(typ.fallback, member) - else: - return False - - def not_ready_callback(self, name: str, context: Context) -> None: - """Called when we can't infer the type of a variable because it's not ready yet. - - Either defer type checking of the enclosing function to the next - pass or report an error. - """ - self.chk.handle_cannot_determine_type(name, context) - - -def map_actuals_to_formals(caller_kinds: List[int], - caller_names: List[str], - callee_kinds: List[int], - callee_names: List[str], - caller_arg_type: Callable[[int], - Type]) -> List[List[int]]: - """Calculate mapping between actual (caller) args and formals. - - The result contains a list of caller argument indexes mapping to each - callee argument index, indexed by callee index. - - The caller_arg_type argument should evaluate to the type of the actual - argument type with the given index. - """ - ncallee = len(callee_kinds) - map = [None] * ncallee # type: List[List[int]] - for i in range(ncallee): - map[i] = [] - j = 0 - for i, kind in enumerate(caller_kinds): - if kind == nodes.ARG_POS: - if j < ncallee: - if callee_kinds[j] in [nodes.ARG_POS, nodes.ARG_OPT, - nodes.ARG_NAMED]: - map[j].append(i) - j += 1 - elif callee_kinds[j] == nodes.ARG_STAR: - map[j].append(i) - elif kind == nodes.ARG_STAR: - # We need to know the actual type to map varargs. - argt = caller_arg_type(i) - if isinstance(argt, TupleType): - # A tuple actual maps to a fixed number of formals. - for _ in range(len(argt.items)): - if j < ncallee: - if callee_kinds[j] != nodes.ARG_STAR2: - map[j].append(i) - else: - raise NotImplementedError() - j += 1 - else: - # Assume that it is an iterable (if it isn't, there will be - # an error later). - while j < ncallee: - if callee_kinds[j] in (nodes.ARG_NAMED, nodes.ARG_STAR2): - break - else: - map[j].append(i) - j += 1 - elif kind == nodes.ARG_NAMED: - name = caller_names[i] - if name in callee_names: - map[callee_names.index(name)].append(i) - elif nodes.ARG_STAR2 in callee_kinds: - map[callee_kinds.index(nodes.ARG_STAR2)].append(i) - else: - assert kind == nodes.ARG_STAR2 - for j in range(ncallee): - # TODO tuple varargs complicate this - no_certain_match = ( - not map[j] or caller_kinds[map[j][0]] == nodes.ARG_STAR) - if ((callee_names[j] and no_certain_match) - or callee_kinds[j] == nodes.ARG_STAR2): - map[j].append(i) - return map - - -def is_empty_tuple(t: Type) -> bool: - return isinstance(t, TupleType) and not t.items - - -def is_duplicate_mapping(mapping: List[int], actual_kinds: List[int]) -> bool: - # Multiple actuals can map to the same formal only if they both come from - # varargs (*args and **kwargs); in this case at runtime it is possible that - # there are no duplicates. We need to allow this, as the convention - # f(..., *args, **kwargs) is common enough. - return len(mapping) > 1 and not ( - len(mapping) == 2 and - actual_kinds[mapping[0]] == nodes.ARG_STAR and - actual_kinds[mapping[1]] == nodes.ARG_STAR2) - - -def replace_callable_return_type(c: CallableType, new_ret_type: Type) -> CallableType: - """Return a copy of a callable type with a different return type.""" - return c.copy_modified(ret_type=new_ret_type) - - -class ArgInferSecondPassQuery(types.TypeQuery): - """Query whether an argument type should be inferred in the second pass. - - The result is True if the type has a type variable in a callable return - type anywhere. For example, the result for Callable[[], T] is True if t is - a type variable. - """ - def __init__(self) -> None: - super().__init__(False, types.ANY_TYPE_STRATEGY) - - def visit_callable_type(self, t: CallableType) -> bool: - return self.query_types(t.arg_types) or t.accept(HasTypeVarQuery()) - - -class HasTypeVarQuery(types.TypeQuery): - """Visitor for querying whether a type has a type variable component.""" - def __init__(self) -> None: - super().__init__(False, types.ANY_TYPE_STRATEGY) - - def visit_type_var(self, t: TypeVarType) -> bool: - return True - - -def has_erased_component(t: Type) -> bool: - return t is not None and t.accept(HasErasedComponentsQuery()) - - -class HasErasedComponentsQuery(types.TypeQuery): - """Visitor for querying whether a type has an erased component.""" - def __init__(self) -> None: - super().__init__(False, types.ANY_TYPE_STRATEGY) - - def visit_erased_type(self, t: ErasedType) -> bool: - return True - - -def overload_arg_similarity(actual: Type, formal: Type) -> int: - """Return if caller argument (actual) is compatible with overloaded signature arg (formal). - - Return a similarity level: - 0: no match - 1: actual is compatible, but only using type promotions (e.g. int vs float) - 2: actual is compatible without type promotions (e.g. int vs object) - - The distinction is important in cases where multiple overload items match. We want - give priority to higher similarity matches. - """ - # Replace type variables with their upper bounds. Overloading - # resolution is based on runtime behavior which erases type - # parameters, so no need to handle type variables occurring within - # a type. - if isinstance(actual, TypeVarType): - actual = actual.erase_to_union_or_bound() - if isinstance(formal, TypeVarType): - formal = formal.erase_to_union_or_bound() - if (isinstance(actual, UninhabitedType) or isinstance(actual, AnyType) or - isinstance(formal, AnyType) or isinstance(formal, CallableType) or - (isinstance(actual, Instance) and actual.type.fallback_to_any)): - # These could match anything at runtime. - return 2 - if not experiments.STRICT_OPTIONAL and isinstance(actual, NoneTyp): - # NoneTyp matches anything if we're not doing strict Optional checking - return 2 - if isinstance(actual, UnionType): - return max(overload_arg_similarity(item, formal) - for item in actual.items) - if isinstance(formal, UnionType): - return max(overload_arg_similarity(actual, item) - for item in formal.items) - if isinstance(formal, TypeType): - if isinstance(actual, TypeType): - # Since Type[T] is covariant, check if actual = Type[A] is - # a subtype of formal = Type[F]. - return overload_arg_similarity(actual.item, formal.item) - elif isinstance(actual, CallableType) and actual.is_type_obj(): - # Check if the actual is a constructor of some sort. - # Note that this is this unsound, since we don't check the __init__ signature. - return overload_arg_similarity(actual.ret_type, formal.item) - else: - return 0 - if isinstance(formal, Instance): - if isinstance(actual, CallableType): - actual = actual.fallback - if isinstance(actual, Overloaded): - actual = actual.items()[0].fallback - if isinstance(actual, TupleType): - actual = actual.fallback - if isinstance(actual, Instance): - # First perform a quick check (as an optimization) and fall back to generic - # subtyping algorithm if type promotions are possible (e.g., int vs. float). - if formal.type in actual.type.mro: - return 2 - elif actual.type._promote and is_subtype(actual, formal): - return 1 - else: - return 0 - elif isinstance(actual, TypeType): - if formal.type.fullname() in {"builtins.object", "builtins.type"}: - return 2 - else: - return 0 - else: - return 0 - if isinstance(actual, UnboundType) or isinstance(formal, UnboundType): - # Either actual or formal is the result of an error; shut up. - return 2 - # Fall back to a conservative equality check for the remaining kinds of type. - return 2 if is_same_type(erasetype.erase_type(actual), erasetype.erase_type(formal)) else 0 - - -def freshen_generic_callable(callee: CallableType) -> CallableType: - tvdefs = [] - tvmap = {} # type: Dict[TypeVarId, Type] - for v in callee.variables: - tvdef = TypeVarDef.new_unification_variable(v) - tvdefs.append(tvdef) - tvmap[v.id] = TypeVarType(tvdef) - - return cast(CallableType, expand_type(callee, tvmap)).copy_modified(variables=tvdefs) +"""Expression type checker. This file is conceptually part of TypeChecker.""" + +from typing import cast, Dict, List, Tuple, Callable, Union, Optional + +from mypy.types import ( + Type, AnyType, CallableType, Overloaded, NoneTyp, Void, TypeVarDef, + TupleType, Instance, TypeVarId, TypeVarType, ErasedType, UnionType, + PartialType, DeletedType, UnboundType, UninhabitedType, TypeType +) +from mypy.nodes import ( + NameExpr, RefExpr, Var, FuncDef, OverloadedFuncDef, TypeInfo, CallExpr, + Node, MemberExpr, IntExpr, StrExpr, BytesExpr, UnicodeExpr, FloatExpr, + OpExpr, UnaryExpr, IndexExpr, CastExpr, RevealTypeExpr, TypeApplication, ListExpr, + TupleExpr, DictExpr, FuncExpr, SuperExpr, SliceExpr, Context, + ListComprehension, GeneratorExpr, SetExpr, MypyFile, Decorator, + ConditionalExpr, ComparisonExpr, TempNode, SetComprehension, + DictionaryComprehension, ComplexExpr, EllipsisExpr, + TypeAliasExpr, BackquoteExpr, ARG_POS, ARG_NAMED, ARG_STAR2 +) +from mypy.nodes import function_type +from mypy import nodes +import mypy.checker +from mypy import types +from mypy.sametypes import is_same_type +from mypy.erasetype import replace_meta_vars +from mypy.messages import MessageBuilder +from mypy import messages +from mypy.infer import infer_type_arguments, infer_function_type_arguments +from mypy import join +from mypy.subtypes import is_subtype, is_equivalent +from mypy import applytype +from mypy import erasetype +from mypy.checkmember import analyze_member_access, type_object_type +from mypy.semanal import self_type +from mypy.constraints import get_actual_type +from mypy.checkstrformat import StringFormatterChecker +from mypy.expandtype import expand_type +import mypy.checkexpr + +from mypy import experiments + +# Type of callback user for checking individual function arguments. See +# check_args() below for details. +ArgChecker = Callable[[Type, Type, int, Type, int, int, CallableType, Context, MessageBuilder], + None] + + +class Finished(Exception): + """Raised if we can terminate overload argument check early (no match).""" + + +class ExpressionChecker: + """Expression type checker. + + This class works closely together with checker.TypeChecker. + """ + + # Some services are provided by a TypeChecker instance. + chk = None # type: mypy.checker.TypeChecker + # This is shared with TypeChecker, but stored also here for convenience. + msg = None # type: MessageBuilder + + strfrm_checker = None # type: mypy.checkstrformat.StringFormatterChecker + + def __init__(self, + chk: 'mypy.checker.TypeChecker', + msg: MessageBuilder) -> None: + """Construct an expression type checker.""" + self.chk = chk + self.msg = msg + self.strfrm_checker = mypy.checkexpr.StringFormatterChecker(self, self.chk, self.msg) + + def visit_name_expr(self, e: NameExpr) -> Type: + """Type check a name expression. + + It can be of any kind: local, member or global. + """ + result = self.analyze_ref_expr(e) + return self.chk.narrow_type_from_binder(e, result) + + def analyze_ref_expr(self, e: RefExpr, lvalue: bool = False) -> Type: + result = None # type: Type + node = e.node + if isinstance(node, Var): + # Variable reference. + result = self.analyze_var_ref(node, e) + if isinstance(result, PartialType): + if result.type is None: + # 'None' partial type. It has a well-defined type. In an lvalue context + # we want to preserve the knowledge of it being a partial type. + if not lvalue: + result = NoneTyp() + else: + partial_types = self.chk.find_partial_types(node) + if partial_types is not None and not self.chk.current_node_deferred: + context = partial_types[node] + self.msg.fail(messages.NEED_ANNOTATION_FOR_VAR, context) + result = AnyType() + elif isinstance(node, FuncDef): + # Reference to a global function. + result = function_type(node, self.named_type('builtins.function')) + elif isinstance(node, OverloadedFuncDef): + result = node.type + elif isinstance(node, TypeInfo): + # Reference to a type object. + result = type_object_type(node, self.named_type) + elif isinstance(node, MypyFile): + # Reference to a module object. + result = self.named_type('builtins.module') + elif isinstance(node, Decorator): + result = self.analyze_var_ref(node.var, e) + else: + # Unknown reference; use any type implicitly to avoid + # generating extra type errors. + result = AnyType() + return result + + def analyze_var_ref(self, var: Var, context: Context) -> Type: + if not var.type: + if not var.is_ready and self.chk.typing_mode_full(): + self.chk.handle_cannot_determine_type(var.name(), context) + # Implicit 'Any' type. + return AnyType() + else: + # Look up local type of variable with type (inferred or explicit). + val = self.chk.binder.get(var) + if val is None: + return var.type + else: + return val + + def visit_call_expr(self, e: CallExpr) -> Type: + """Type check a call expression.""" + if e.analyzed: + # It's really a special form that only looks like a call. + return self.accept(e.analyzed, self.chk.type_context[-1]) + self.try_infer_partial_type(e) + callee_type = self.accept(e.callee) + if (self.chk.options.disallow_untyped_calls and + self.chk.typing_mode_full() and + isinstance(callee_type, CallableType) + and callee_type.implicit): + return self.msg.untyped_function_call(callee_type, e) + return self.check_call_expr_with_callee_type(callee_type, e) + + # Types and methods that can be used to infer partial types. + item_args = {'builtins.list': ['append'], + 'builtins.set': ['add', 'discard'], + } + container_args = {'builtins.list': {'extend': ['builtins.list']}, + 'builtins.dict': {'update': ['builtins.dict']}, + 'builtins.set': {'update': ['builtins.set', 'builtins.list']}, + } + + def try_infer_partial_type(self, e: CallExpr) -> None: + if isinstance(e.callee, MemberExpr) and isinstance(e.callee.expr, RefExpr): + var = cast(Var, e.callee.expr.node) + partial_types = self.chk.find_partial_types(var) + if partial_types is not None and not self.chk.current_node_deferred: + partial_type = cast(PartialType, var.type) + if partial_type is None or partial_type.type is None: + # A partial None type -> can't infer anything. + return + typename = partial_type.type.fullname() + methodname = e.callee.name + # Sometimes we can infer a full type for a partial List, Dict or Set type. + # TODO: Don't infer argument expression twice. + if (typename in self.item_args and methodname in self.item_args[typename] + and e.arg_kinds == [ARG_POS]): + item_type = self.accept(e.args[0]) + full_item_type = UnionType.make_simplified_union( + [item_type, partial_type.inner_types[0]]) + if mypy.checker.is_valid_inferred_type(full_item_type): + var.type = self.chk.named_generic_type(typename, [full_item_type]) + del partial_types[var] + elif (typename in self.container_args + and methodname in self.container_args[typename] + and e.arg_kinds == [ARG_POS]): + arg_type = self.accept(e.args[0]) + if isinstance(arg_type, Instance): + arg_typename = arg_type.type.fullname() + if arg_typename in self.container_args[typename][methodname]: + full_item_types = [ + UnionType.make_simplified_union([item_type, prev_type]) + for item_type, prev_type + in zip(arg_type.args, partial_type.inner_types) + ] + if all(mypy.checker.is_valid_inferred_type(item_type) + for item_type in full_item_types): + var.type = self.chk.named_generic_type(typename, + list(full_item_types)) + del partial_types[var] + + def check_call_expr_with_callee_type(self, callee_type: Type, + e: CallExpr) -> Type: + """Type check call expression. + + The given callee type overrides the type of the callee + expression. + """ + return self.check_call(callee_type, e.args, e.arg_kinds, e, + e.arg_names, callable_node=e.callee)[0] + + def check_call(self, callee: Type, args: List[Node], + arg_kinds: List[int], context: Context, + arg_names: List[str] = None, + callable_node: Node = None, + arg_messages: MessageBuilder = None) -> Tuple[Type, Type]: + """Type check a call. + + Also infer type arguments if the callee is a generic function. + + Return (result type, inferred callee type). + + Arguments: + callee: type of the called value + args: actual argument expressions + arg_kinds: contains nodes.ARG_* constant for each argument in args + describing whether the argument is positional, *arg, etc. + arg_names: names of arguments (optional) + callable_node: associate the inferred callable type to this node, + if specified + arg_messages: TODO + """ + arg_messages = arg_messages or self.msg + if isinstance(callee, CallableType): + if callee.is_concrete_type_obj() and callee.type_object().is_abstract: + type = callee.type_object() + self.msg.cannot_instantiate_abstract_class( + callee.type_object().name(), type.abstract_attributes, + context) + + formal_to_actual = map_actuals_to_formals( + arg_kinds, arg_names, + callee.arg_kinds, callee.arg_names, + lambda i: self.accept(args[i])) + + if callee.is_generic(): + callee = freshen_generic_callable(callee) + callee = self.infer_function_type_arguments_using_context( + callee, context) + callee = self.infer_function_type_arguments( + callee, args, arg_kinds, formal_to_actual, context) + + arg_types = self.infer_arg_types_in_context2( + callee, args, arg_kinds, formal_to_actual) + + self.check_argument_count(callee, arg_types, arg_kinds, + arg_names, formal_to_actual, context, self.msg) + + self.check_argument_types(arg_types, arg_kinds, callee, + formal_to_actual, context, + messages=arg_messages) + + if (callee.is_type_obj() and (len(arg_types) == 1) + and is_equivalent(callee.ret_type, self.named_type('builtins.type'))): + callee = callee.copy_modified(ret_type=TypeType(arg_types[0])) + + if callable_node: + # Store the inferred callable type. + self.chk.store_type(callable_node, callee) + return callee.ret_type, callee + elif isinstance(callee, Overloaded): + # Type check arguments in empty context. They will be checked again + # later in a context derived from the signature; these types are + # only used to pick a signature variant. + self.msg.disable_errors() + arg_types = self.infer_arg_types_in_context(None, args) + self.msg.enable_errors() + + target = self.overload_call_target(arg_types, arg_kinds, arg_names, + callee, context, + messages=arg_messages) + return self.check_call(target, args, arg_kinds, context, arg_names, + arg_messages=arg_messages) + elif isinstance(callee, AnyType) or self.chk.typing_mode_none(): + self.infer_arg_types_in_context(None, args) + return AnyType(), AnyType() + elif isinstance(callee, UnionType): + self.msg.disable_type_names += 1 + results = [self.check_call(subtype, args, arg_kinds, context, arg_names, + arg_messages=arg_messages) + for subtype in callee.items] + self.msg.disable_type_names -= 1 + return (UnionType.make_simplified_union([res[0] for res in results]), + callee) + elif isinstance(callee, Instance): + call_function = analyze_member_access('__call__', callee, context, + False, False, self.named_type, self.not_ready_callback, + self.msg) + return self.check_call(call_function, args, arg_kinds, context, arg_names, + callable_node, arg_messages) + elif isinstance(callee, TypeVarType): + return self.check_call(callee.upper_bound, args, arg_kinds, context, arg_names, + callable_node, arg_messages) + elif isinstance(callee, TypeType): + # Pass the original Type[] as context since that's where errors should go. + item = self.analyze_type_type_callee(callee.item, callee) + return self.check_call(item, args, arg_kinds, context, arg_names, + callable_node, arg_messages) + else: + return self.msg.not_callable(callee, context), AnyType() + + def analyze_type_type_callee(self, item: Type, context: Context) -> Type: + """Analyze the callee X in X(...) where X is Type[item]. + + Return a Y that we can pass to check_call(Y, ...). + """ + if isinstance(item, AnyType): + return AnyType() + if isinstance(item, Instance): + return type_object_type(item.type, self.named_type) + if isinstance(item, UnionType): + return UnionType([self.analyze_type_type_callee(item, context) + for item in item.items], item.line) + if isinstance(item, TypeVarType): + # Pretend we're calling the typevar's upper bound, + # i.e. its constructor (a poor approximation for reality, + # but better than AnyType...), but replace the return type + # with typevar. + callee = self.analyze_type_type_callee(item.upper_bound, context) + if isinstance(callee, CallableType): + if callee.is_generic(): + callee = None + else: + callee = callee.copy_modified(ret_type=item) + elif isinstance(callee, Overloaded): + if callee.items()[0].is_generic(): + callee = None + else: + callee = Overloaded([c.copy_modified(ret_type=item) + for c in callee.items()]) + if callee: + return callee + + self.msg.unsupported_type_type(item, context) + return AnyType() + + def infer_arg_types_in_context(self, callee: Optional[CallableType], + args: List[Node]) -> List[Type]: + """Infer argument expression types using a callable type as context. + + For example, if callee argument 2 has type List[int], infer the + argument expression with List[int] type context. + """ + # TODO Always called with callee as None, i.e. empty context. + res = [] # type: List[Type] + + fixed = len(args) + if callee: + fixed = min(fixed, callee.max_fixed_args()) + + arg_type = None # type: Type + ctx = None # type: Type + for i, arg in enumerate(args): + if i < fixed: + if callee and i < len(callee.arg_types): + ctx = callee.arg_types[i] + arg_type = self.accept(arg, ctx) + else: + if callee and callee.is_var_arg: + arg_type = self.accept(arg, callee.arg_types[-1]) + else: + arg_type = self.accept(arg) + if has_erased_component(arg_type): + res.append(NoneTyp()) + else: + res.append(arg_type) + return res + + def infer_arg_types_in_context2( + self, callee: CallableType, args: List[Node], arg_kinds: List[int], + formal_to_actual: List[List[int]]) -> List[Type]: + """Infer argument expression types using a callable type as context. + + For example, if callee argument 2 has type List[int], infer the + argument expression with List[int] type context. + + Returns the inferred types of *actual arguments*. + """ + res = [None] * len(args) # type: List[Type] + + for i, actuals in enumerate(formal_to_actual): + for ai in actuals: + if arg_kinds[ai] not in (nodes.ARG_STAR, nodes.ARG_STAR2): + res[ai] = self.accept(args[ai], callee.arg_types[i]) + + # Fill in the rest of the argument types. + for i, t in enumerate(res): + if not t: + res[i] = self.accept(args[i]) + return res + + def infer_function_type_arguments_using_context( + self, callable: CallableType, error_context: Context) -> CallableType: + """Unify callable return type to type context to infer type vars. + + For example, if the return type is set[t] where 't' is a type variable + of callable, and if the context is set[int], return callable modified + by substituting 't' with 'int'. + """ + ctx = self.chk.type_context[-1] + if not ctx: + return callable + # The return type may have references to type metavariables that + # we are inferring right now. We must consider them as indeterminate + # and they are not potential results; thus we replace them with the + # special ErasedType type. On the other hand, class type variables are + # valid results. + erased_ctx = replace_meta_vars(ctx, ErasedType()) + ret_type = callable.ret_type + if isinstance(ret_type, TypeVarType): + if ret_type.values or (not isinstance(ctx, Instance) or + not cast(Instance, ctx).args): + # The return type is a type variable. If it has values, we can't easily restrict + # type inference to conform to the valid values. If it's unrestricted, we could + # infer a too general type for the type variable if we use context, and this could + # result in confusing and spurious type errors elsewhere. + # + # Give up and just use function arguments for type inference. As an exception, + # if the context is a generic instance type, actually use it as context, as + # this *seems* to usually be the reasonable thing to do. + # + # See also github issues #462 and #360. + ret_type = NoneTyp() + args = infer_type_arguments(callable.type_var_ids(), ret_type, erased_ctx) + # Only substitute non-None and non-erased types. + new_args = [] # type: List[Type] + for arg in args: + if isinstance(arg, (NoneTyp, UninhabitedType)) or has_erased_component(arg): + new_args.append(None) + else: + new_args.append(arg) + return cast(CallableType, self.apply_generic_arguments(callable, new_args, + error_context)) + + def infer_function_type_arguments(self, callee_type: CallableType, + args: List[Node], + arg_kinds: List[int], + formal_to_actual: List[List[int]], + context: Context) -> CallableType: + """Infer the type arguments for a generic callee type. + + Infer based on the types of arguments. + + Return a derived callable type that has the arguments applied. + """ + if not self.chk.typing_mode_none(): + # Disable type errors during type inference. There may be errors + # due to partial available context information at this time, but + # these errors can be safely ignored as the arguments will be + # inferred again later. + self.msg.disable_errors() + + arg_types = self.infer_arg_types_in_context2( + callee_type, args, arg_kinds, formal_to_actual) + + self.msg.enable_errors() + + arg_pass_nums = self.get_arg_infer_passes( + callee_type.arg_types, formal_to_actual, len(args)) + + pass1_args = [] # type: List[Optional[Type]] + for i, arg in enumerate(arg_types): + if arg_pass_nums[i] > 1: + pass1_args.append(None) + else: + pass1_args.append(arg) + + inferred_args = infer_function_type_arguments( + callee_type, pass1_args, arg_kinds, formal_to_actual, + strict=self.chk.typing_mode_full()) # type: List[Type] + + if 2 in arg_pass_nums: + # Second pass of type inference. + (callee_type, + inferred_args) = self.infer_function_type_arguments_pass2( + callee_type, args, arg_kinds, formal_to_actual, + inferred_args, context) + + if callee_type.special_sig == 'dict' and len(inferred_args) == 2 and ( + ARG_NAMED in arg_kinds or ARG_STAR2 in arg_kinds): + # HACK: Infer str key type for dict(...) with keyword args. The type system + # can't represent this so we special case it, as this is a pretty common + # thing. This doesn't quite work with all possible subclasses of dict + # if they shuffle type variables around, as we assume that there is a 1-1 + # correspondence with dict type variables. This is a marginal issue and + # a little tricky to fix so it's left unfixed for now. + if isinstance(inferred_args[0], (NoneTyp, UninhabitedType)): + inferred_args[0] = self.named_type('builtins.str') + elif not is_subtype(self.named_type('builtins.str'), inferred_args[0]): + self.msg.fail(messages.KEYWORD_ARGUMENT_REQUIRES_STR_KEY_TYPE, + context) + else: + # In dynamically typed functions use implicit 'Any' types for + # type variables. + inferred_args = [AnyType()] * len(callee_type.variables) + return self.apply_inferred_arguments(callee_type, inferred_args, + context) + + def infer_function_type_arguments_pass2( + self, callee_type: CallableType, + args: List[Node], + arg_kinds: List[int], + formal_to_actual: List[List[int]], + inferred_args: List[Type], + context: Context) -> Tuple[CallableType, List[Type]]: + """Perform second pass of generic function type argument inference. + + The second pass is needed for arguments with types such as Callable[[T], S], + where both T and S are type variables, when the actual argument is a + lambda with inferred types. The idea is to infer the type variable T + in the first pass (based on the types of other arguments). This lets + us infer the argument and return type of the lambda expression and + thus also the type variable S in this second pass. + + Return (the callee with type vars applied, inferred actual arg types). + """ + # None or erased types in inferred types mean that there was not enough + # information to infer the argument. Replace them with None values so + # that they are not applied yet below. + for i, arg in enumerate(inferred_args): + if isinstance(arg, (NoneTyp, UninhabitedType)) or has_erased_component(arg): + inferred_args[i] = None + + callee_type = cast(CallableType, self.apply_generic_arguments( + callee_type, inferred_args, context)) + arg_types = self.infer_arg_types_in_context2( + callee_type, args, arg_kinds, formal_to_actual) + + inferred_args = infer_function_type_arguments( + callee_type, arg_types, arg_kinds, formal_to_actual) + + return callee_type, inferred_args + + def get_arg_infer_passes(self, arg_types: List[Type], + formal_to_actual: List[List[int]], + num_actuals: int) -> List[int]: + """Return pass numbers for args for two-pass argument type inference. + + For each actual, the pass number is either 1 (first pass) or 2 (second + pass). + + Two-pass argument type inference primarily lets us infer types of + lambdas more effectively. + """ + res = [1] * num_actuals + for i, arg in enumerate(arg_types): + if arg.accept(ArgInferSecondPassQuery()): + for j in formal_to_actual[i]: + res[j] = 2 + return res + + def apply_inferred_arguments(self, callee_type: CallableType, + inferred_args: List[Type], + context: Context) -> CallableType: + """Apply inferred values of type arguments to a generic function. + + Inferred_args contains the values of function type arguments. + """ + # Report error if some of the variables could not be solved. In that + # case assume that all variables have type Any to avoid extra + # bogus error messages. + for i, inferred_type in enumerate(inferred_args): + if not inferred_type or has_erased_component(inferred_type): + # Could not infer a non-trivial type for a type variable. + self.msg.could_not_infer_type_arguments( + callee_type, i + 1, context) + inferred_args = [AnyType()] * len(inferred_args) + # Apply the inferred types to the function type. In this case the + # return type must be CallableType, since we give the right number of type + # arguments. + return cast(CallableType, self.apply_generic_arguments(callee_type, + inferred_args, context)) + + def check_argument_count(self, callee: CallableType, actual_types: List[Type], + actual_kinds: List[int], actual_names: List[str], + formal_to_actual: List[List[int]], + context: Context, + messages: Optional[MessageBuilder]) -> bool: + """Check that there is a value for all required arguments to a function. + + Also check that there are no duplicate values for arguments. Report found errors + using 'messages' if it's not None. + + Return False if there were any errors. Otherwise return True + """ + # TODO(jukka): We could return as soon as we find an error if messages is None. + formal_kinds = callee.arg_kinds + + # Collect list of all actual arguments matched to formal arguments. + all_actuals = [] # type: List[int] + for actuals in formal_to_actual: + all_actuals.extend(actuals) + + is_unexpected_arg_error = False # Keep track of errors to avoid duplicate errors. + ok = True # False if we've found any error. + for i, kind in enumerate(actual_kinds): + if i not in all_actuals and ( + kind != nodes.ARG_STAR or + not is_empty_tuple(actual_types[i])): + # Extra actual: not matched by a formal argument. + ok = False + if kind != nodes.ARG_NAMED: + if messages: + messages.too_many_arguments(callee, context) + else: + if messages: + messages.unexpected_keyword_argument( + callee, actual_names[i], context) + is_unexpected_arg_error = True + elif kind == nodes.ARG_STAR and ( + nodes.ARG_STAR not in formal_kinds): + actual_type = actual_types[i] + if isinstance(actual_type, TupleType): + if all_actuals.count(i) < len(actual_type.items): + # Too many tuple items as some did not match. + if messages: + messages.too_many_arguments(callee, context) + ok = False + # *args can be applied even if the function takes a fixed + # number of positional arguments. This may succeed at runtime. + + for i, kind in enumerate(formal_kinds): + if kind == nodes.ARG_POS and (not formal_to_actual[i] and + not is_unexpected_arg_error): + # No actual for a mandatory positional formal. + if messages: + messages.too_few_arguments(callee, context, actual_names) + ok = False + elif kind in [nodes.ARG_POS, nodes.ARG_OPT, + nodes.ARG_NAMED] and is_duplicate_mapping( + formal_to_actual[i], actual_kinds): + if (self.chk.typing_mode_full() or + isinstance(actual_types[formal_to_actual[i][0]], TupleType)): + if messages: + messages.duplicate_argument_value(callee, i, context) + ok = False + elif (kind == nodes.ARG_NAMED and formal_to_actual[i] and + actual_kinds[formal_to_actual[i][0]] != nodes.ARG_NAMED): + # Positional argument when expecting a keyword argument. + if messages: + messages.too_many_positional_arguments(callee, context) + ok = False + return ok + + def check_argument_types(self, arg_types: List[Type], arg_kinds: List[int], + callee: CallableType, + formal_to_actual: List[List[int]], + context: Context, + messages: MessageBuilder = None, + check_arg: ArgChecker = None) -> None: + """Check argument types against a callable type. + + Report errors if the argument types are not compatible. + """ + messages = messages or self.msg + check_arg = check_arg or self.check_arg + # Keep track of consumed tuple *arg items. + tuple_counter = [0] + for i, actuals in enumerate(formal_to_actual): + for actual in actuals: + arg_type = arg_types[actual] + # Check that a *arg is valid as varargs. + if (arg_kinds[actual] == nodes.ARG_STAR and + not self.is_valid_var_arg(arg_type)): + messages.invalid_var_arg(arg_type, context) + if (arg_kinds[actual] == nodes.ARG_STAR2 and + not self.is_valid_keyword_var_arg(arg_type)): + messages.invalid_keyword_var_arg(arg_type, context) + # Get the type of an individual actual argument (for *args + # and **args this is the item type, not the collection type). + actual_type = get_actual_type(arg_type, arg_kinds[actual], + tuple_counter) + check_arg(actual_type, arg_type, arg_kinds[actual], + callee.arg_types[i], + actual + 1, i + 1, callee, context, messages) + + # There may be some remaining tuple varargs items that haven't + # been checked yet. Handle them. + if (callee.arg_kinds[i] == nodes.ARG_STAR and + arg_kinds[actual] == nodes.ARG_STAR and + isinstance(arg_types[actual], TupleType)): + tuplet = cast(TupleType, arg_types[actual]) + while tuple_counter[0] < len(tuplet.items): + actual_type = get_actual_type(arg_type, + arg_kinds[actual], + tuple_counter) + check_arg(actual_type, arg_type, arg_kinds[actual], + callee.arg_types[i], + actual + 1, i + 1, callee, context, messages) + + def check_arg(self, caller_type: Type, original_caller_type: Type, + caller_kind: int, + callee_type: Type, n: int, m: int, callee: CallableType, + context: Context, messages: MessageBuilder) -> None: + """Check the type of a single argument in a call.""" + if isinstance(caller_type, Void): + messages.does_not_return_value(caller_type, context) + elif isinstance(caller_type, DeletedType): + messages.deleted_as_rvalue(caller_type, context) + elif not is_subtype(caller_type, callee_type): + messages.incompatible_argument(n, m, callee, original_caller_type, + caller_kind, context) + + def overload_call_target(self, arg_types: List[Type], arg_kinds: List[int], + arg_names: List[str], + overload: Overloaded, context: Context, + messages: MessageBuilder = None) -> Type: + """Infer the correct overload item to call with given argument types. + + The return value may be CallableType or AnyType (if an unique item + could not be determined). + """ + messages = messages or self.msg + # TODO: For overlapping signatures we should try to get a more precise + # result than 'Any'. + match = [] # type: List[CallableType] + best_match = 0 + for typ in overload.items(): + similarity = self.erased_signature_similarity(arg_types, arg_kinds, arg_names, + typ, context=context) + if similarity > 0 and similarity >= best_match: + if (match and not is_same_type(match[-1].ret_type, + typ.ret_type) and + not mypy.checker.is_more_precise_signature( + match[-1], typ)): + # Ambiguous return type. Either the function overload is + # overlapping (which we don't handle very well here) or the + # caller has provided some Any argument types; in either + # case we'll fall back to Any. It's okay to use Any types + # in calls. + # + # Overlapping overload items are generally fine if the + # overlapping is only possible when there is multiple + # inheritance, as this is rare. See docstring of + # mypy.meet.is_overlapping_types for more about this. + # + # Note that there is no ambiguity if the items are + # covariant in both argument types and return types with + # respect to type precision. We'll pick the best/closest + # match. + # + # TODO: Consider returning a union type instead if the + # overlapping is NOT due to Any types? + return AnyType() + else: + match.append(typ) + best_match = max(best_match, similarity) + if not match: + messages.no_variant_matches_arguments(overload, arg_types, context) + return AnyType() + else: + if len(match) == 1: + return match[0] + else: + # More than one signature matches. Pick the first *non-erased* + # matching signature, or default to the first one if none + # match. + for m in match: + if self.match_signature_types(arg_types, arg_kinds, arg_names, m, + context=context): + return m + return match[0] + + def erased_signature_similarity(self, arg_types: List[Type], arg_kinds: List[int], + arg_names: List[str], callee: CallableType, + context: Context) -> int: + """Determine whether arguments could match the signature at runtime. + + Return similarity level (0 = no match, 1 = can match, 2 = non-promotion match). See + overload_arg_similarity for a discussion of similarity levels. + """ + formal_to_actual = map_actuals_to_formals(arg_kinds, + arg_names, + callee.arg_kinds, + callee.arg_names, + lambda i: arg_types[i]) + + if not self.check_argument_count(callee, arg_types, arg_kinds, arg_names, + formal_to_actual, None, None): + # Too few or many arguments -> no match. + return 0 + + similarity = 2 + + def check_arg(caller_type: Type, original_caller_type: Type, caller_kind: int, + callee_type: Type, n: int, m: int, callee: CallableType, + context: Context, messages: MessageBuilder) -> None: + nonlocal similarity + similarity = min(similarity, + overload_arg_similarity(caller_type, callee_type)) + if similarity == 0: + # No match -- exit early since none of the remaining work can change + # the result. + raise Finished + + try: + self.check_argument_types(arg_types, arg_kinds, callee, formal_to_actual, + context=context, check_arg=check_arg) + except Finished: + pass + + return similarity + + def match_signature_types(self, arg_types: List[Type], arg_kinds: List[int], + arg_names: List[str], callee: CallableType, + context: Context) -> bool: + """Determine whether arguments types match the signature. + + Assume that argument counts are compatible. + + Return True if arguments match. + """ + formal_to_actual = map_actuals_to_formals(arg_kinds, + arg_names, + callee.arg_kinds, + callee.arg_names, + lambda i: arg_types[i]) + ok = True + + def check_arg(caller_type: Type, original_caller_type: Type, caller_kind: int, + callee_type: Type, n: int, m: int, callee: CallableType, + context: Context, messages: MessageBuilder) -> None: + nonlocal ok + if not is_subtype(caller_type, callee_type): + ok = False + + self.check_argument_types(arg_types, arg_kinds, callee, formal_to_actual, + context=context, check_arg=check_arg) + return ok + + def apply_generic_arguments(self, callable: CallableType, types: List[Type], + context: Context) -> Type: + """Simple wrapper around mypy.applytype.apply_generic_arguments.""" + return applytype.apply_generic_arguments(callable, types, self.msg, context) + + def apply_generic_arguments2(self, overload: Overloaded, types: List[Type], + context: Context) -> Type: + items = [] # type: List[CallableType] + for item in overload.items(): + applied = self.apply_generic_arguments(item, types, context) + if isinstance(applied, CallableType): + items.append(applied) + else: + # There was an error. + return AnyType() + return Overloaded(items) + + def visit_member_expr(self, e: MemberExpr) -> Type: + """Visit member expression (of form e.id).""" + result = self.analyze_ordinary_member_access(e, False) + return self.chk.narrow_type_from_binder(e, result) + + def analyze_ordinary_member_access(self, e: MemberExpr, + is_lvalue: bool) -> Type: + """Analyse member expression or member lvalue.""" + if e.kind is not None: + # This is a reference to a module attribute. + return self.analyze_ref_expr(e) + else: + # This is a reference to a non-module attribute. + return analyze_member_access(e.name, self.accept(e.expr), e, + is_lvalue, False, + self.named_type, self.not_ready_callback, self.msg) + + def analyze_external_member_access(self, member: str, base_type: Type, + context: Context) -> Type: + """Analyse member access that is external, i.e. it cannot + refer to private definitions. Return the result type. + """ + # TODO remove; no private definitions in mypy + return analyze_member_access(member, base_type, context, False, False, + self.named_type, self.not_ready_callback, self.msg) + + def visit_int_expr(self, e: IntExpr) -> Type: + """Type check an integer literal (trivial).""" + return self.named_type('builtins.int') + + def visit_str_expr(self, e: StrExpr) -> Type: + """Type check a string literal (trivial).""" + return self.named_type('builtins.str') + + def visit_bytes_expr(self, e: BytesExpr) -> Type: + """Type check a bytes literal (trivial).""" + return self.named_type('builtins.bytes') + + def visit_unicode_expr(self, e: UnicodeExpr) -> Type: + """Type check a unicode literal (trivial).""" + return self.named_type('builtins.unicode') + + def visit_float_expr(self, e: FloatExpr) -> Type: + """Type check a float literal (trivial).""" + return self.named_type('builtins.float') + + def visit_complex_expr(self, e: ComplexExpr) -> Type: + """Type check a complex literal.""" + return self.named_type('builtins.complex') + + def visit_ellipsis(self, e: EllipsisExpr) -> Type: + """Type check '...'.""" + if self.chk.options.python_version[0] >= 3: + return self.named_type('builtins.ellipsis') + else: + # '...' is not valid in normal Python 2 code, but it can + # be used in stubs. The parser makes sure that we only + # get this far if we are in a stub, and we can safely + # return 'object' as ellipsis is special cased elsewhere. + # The builtins.ellipsis type does not exist in Python 2. + return self.named_type('builtins.object') + + def visit_op_expr(self, e: OpExpr) -> Type: + """Type check a binary operator expression.""" + if e.op == 'and' or e.op == 'or': + return self.check_boolean_op(e, e) + if e.op == '*' and isinstance(e.left, ListExpr): + # Expressions of form [...] * e get special type inference. + return self.check_list_multiply(e) + if e.op == '%' and isinstance(e.left, (StrExpr, BytesExpr)): + return self.strfrm_checker.check_str_interpolation(cast(StrExpr, e.left), e.right) + left_type = self.accept(e.left) + + if e.op in nodes.op_methods: + method = self.get_operator_method(e.op) + result, method_type = self.check_op(method, left_type, e.right, e, + allow_reverse=True) + e.method_type = method_type + return result + else: + raise RuntimeError('Unknown operator {}'.format(e.op)) + + def visit_comparison_expr(self, e: ComparisonExpr) -> Type: + """Type check a comparison expression. + + Comparison expressions are type checked consecutive-pair-wise + That is, 'a < b > c == d' is check as 'a < b and b > c and c == d' + """ + result = None # type: mypy.types.Type + + # Check each consecutive operand pair and their operator + for left, right, operator in zip(e.operands, e.operands[1:], e.operators): + left_type = self.accept(left) + + method_type = None # type: mypy.types.Type + + if operator == 'in' or operator == 'not in': + right_type = self.accept(right) # TODO only evaluate if needed + + # Keep track of whether we get type check errors (these won't be reported, they + # are just to verify whether something is valid typing wise). + local_errors = self.msg.copy() + local_errors.disable_count = 0 + sub_result, method_type = self.check_op_local('__contains__', right_type, + left, e, local_errors) + if isinstance(right_type, PartialType): + # We don't really know if this is an error or not, so just shut up. + pass + elif (local_errors.is_errors() and + # is_valid_var_arg is True for any Iterable + self.is_valid_var_arg(right_type)): + itertype = self.chk.analyze_iterable_item_type(right) + method_type = CallableType( + [left_type], + [nodes.ARG_POS], + [None], + self.chk.bool_type(), + self.named_type('builtins.function')) + sub_result = self.chk.bool_type() + if not is_subtype(left_type, itertype): + self.msg.unsupported_operand_types('in', left_type, right_type, e) + else: + self.msg.add_errors(local_errors) + if operator == 'not in': + sub_result = self.chk.bool_type() + elif operator in nodes.op_methods: + method = self.get_operator_method(operator) + sub_result, method_type = self.check_op(method, left_type, right, e, + allow_reverse=True) + + elif operator == 'is' or operator == 'is not': + sub_result = self.chk.bool_type() + method_type = None + else: + raise RuntimeError('Unknown comparison operator {}'.format(operator)) + + e.method_types.append(method_type) + + # Determine type of boolean-and of result and sub_result + if result is None: + result = sub_result + else: + # TODO: check on void needed? + self.check_not_void(sub_result, e) + result = join.join_types(result, sub_result) + + return result + + def get_operator_method(self, op: str) -> str: + if op == '/' and self.chk.options.python_version[0] == 2: + # TODO also check for "from __future__ import division" + return '__div__' + else: + return nodes.op_methods[op] + + def check_op_local(self, method: str, base_type: Type, arg: Node, + context: Context, local_errors: MessageBuilder) -> Tuple[Type, Type]: + """Type check a binary operation which maps to a method call. + + Return tuple (result type, inferred operator method type). + """ + method_type = analyze_member_access(method, base_type, context, False, False, + self.named_type, self.not_ready_callback, local_errors) + return self.check_call(method_type, [arg], [nodes.ARG_POS], + context, arg_messages=local_errors) + + def check_op(self, method: str, base_type: Type, arg: Node, + context: Context, + allow_reverse: bool = False) -> Tuple[Type, Type]: + """Type check a binary operation which maps to a method call. + + Return tuple (result type, inferred operator method type). + """ + # Use a local error storage for errors related to invalid argument + # type (but NOT other errors). This error may need to be suppressed + # for operators which support __rX methods. + local_errors = self.msg.copy() + local_errors.disable_count = 0 + if not allow_reverse or self.has_member(base_type, method): + result = self.check_op_local(method, base_type, arg, context, + local_errors) + if allow_reverse: + arg_type = self.chk.type_map[arg] + if isinstance(arg_type, AnyType): + # If the right operand has type Any, we can't make any + # conjectures about the type of the result, since the + # operand could have a __r method that returns anything. + + # However, in weak mode, we do make conjectures. + if not self.chk.typing_mode_weak(): + result = AnyType(), result[1] + success = not local_errors.is_errors() + else: + result = AnyType(), AnyType() + success = False + if success or not allow_reverse or isinstance(base_type, AnyType): + # We were able to call the normal variant of the operator method, + # or there was some problem not related to argument type + # validity, or the operator has no __rX method. In any case, we + # don't need to consider the __rX method. + self.msg.add_errors(local_errors) + return result + else: + # Calling the operator method was unsuccessful. Try the __rX + # method of the other operand instead. + rmethod = self.get_reverse_op_method(method) + arg_type = self.accept(arg) + if self.has_member(arg_type, rmethod): + method_type = self.analyze_external_member_access( + rmethod, arg_type, context) + temp = TempNode(base_type) + return self.check_call(method_type, [temp], [nodes.ARG_POS], + context) + else: + # No __rX method either. Do deferred type checking to produce + # error message that we may have missed previously. + # TODO Fix type checking an expression more than once. + return self.check_op_local(method, base_type, arg, context, + self.msg) + + def get_reverse_op_method(self, method: str) -> str: + if method == '__div__' and self.chk.options.python_version[0] == 2: + return '__rdiv__' + else: + return nodes.reverse_op_methods[method] + + def check_boolean_op(self, e: OpExpr, context: Context) -> Type: + """Type check a boolean operation ('and' or 'or').""" + + # A boolean operation can evaluate to either of the operands. + + # We use the current type context to guide the type inference of of + # the left operand. We also use the left operand type to guide the type + # inference of the right operand so that expressions such as + # '[1] or []' are inferred correctly. + ctx = self.chk.type_context[-1] + left_type = self.accept(e.left, ctx) + + if e.op == 'and': + right_map, _ = \ + mypy.checker.find_isinstance_check(e.left, self.chk.type_map, + self.chk.typing_mode_weak()) + elif e.op == 'or': + _, right_map = \ + mypy.checker.find_isinstance_check(e.left, self.chk.type_map, + self.chk.typing_mode_weak()) + else: + right_map = None + + with self.chk.binder.frame_context(): + if right_map: + for var, type in right_map.items(): + self.chk.binder.push(var, type) + + right_type = self.accept(e.right, left_type) + + self.check_not_void(left_type, context) + self.check_not_void(right_type, context) + return UnionType.make_simplified_union([left_type, right_type]) + + def check_list_multiply(self, e: OpExpr) -> Type: + """Type check an expression of form '[...] * e'. + + Type inference is special-cased for this common construct. + """ + right_type = self.accept(e.right) + if is_subtype(right_type, self.named_type('builtins.int')): + # Special case: [...] * . Use the type context of the + # OpExpr, since the multiplication does not affect the type. + left_type = self.accept(e.left, context=self.chk.type_context[-1]) + else: + left_type = self.accept(e.left) + result, method_type = self.check_op('__mul__', left_type, e.right, e) + e.method_type = method_type + return result + + def visit_unary_expr(self, e: UnaryExpr) -> Type: + """Type check an unary operation ('not', '-', '+' or '~').""" + operand_type = self.accept(e.expr) + op = e.op + if op == 'not': + self.check_not_void(operand_type, e) + result = self.chk.bool_type() # type: Type + elif op == '-': + method_type = self.analyze_external_member_access('__neg__', + operand_type, e) + result, method_type = self.check_call(method_type, [], [], e) + e.method_type = method_type + elif op == '+': + method_type = self.analyze_external_member_access('__pos__', + operand_type, e) + result, method_type = self.check_call(method_type, [], [], e) + e.method_type = method_type + else: + assert op == '~', "unhandled unary operator" + method_type = self.analyze_external_member_access('__invert__', + operand_type, e) + result, method_type = self.check_call(method_type, [], [], e) + e.method_type = method_type + return result + + def visit_index_expr(self, e: IndexExpr) -> Type: + """Type check an index expression (base[index]). + + It may also represent type application. + """ + result = self.visit_index_expr_helper(e) + return self.chk.narrow_type_from_binder(e, result) + + def visit_index_expr_helper(self, e: IndexExpr) -> Type: + if e.analyzed: + # It's actually a type application. + return self.accept(e.analyzed) + left_type = self.accept(e.base) + if isinstance(left_type, TupleType) and self.chk.typing_mode_full(): + # Special case for tuples. They support indexing only by integer + # literals. (Except in weak type checking mode.) + index = e.index + if isinstance(index, SliceExpr): + return self.visit_tuple_slice_helper(left_type, index) + + ok = False + if isinstance(index, IntExpr): + n = index.value + ok = True + elif isinstance(index, UnaryExpr): + if index.op == '-': + operand = index.expr + if isinstance(operand, IntExpr): + n = len(left_type.items) - operand.value + ok = True + if ok: + if n >= 0 and n < len(left_type.items): + return left_type.items[n] + else: + self.chk.fail(messages.TUPLE_INDEX_OUT_OF_RANGE, e) + return AnyType() + else: + self.chk.fail(messages.TUPLE_INDEX_MUST_BE_AN_INT_LITERAL, e) + return AnyType() + else: + result, method_type = self.check_op('__getitem__', left_type, e.index, e) + e.method_type = method_type + return result + + def visit_tuple_slice_helper(self, left_type: TupleType, slic: SliceExpr): + begin = None # type: int + end = None # type: int + stride = None # type:int + + if slic.begin_index: + begin = self._get_value(slic.begin_index) + if begin is None: + self.chk.fail( + messages.TUPLE_SLICE_MUST_BE_AN_INT_LITERAL, + slic.begin_index) + return AnyType() + + if slic.end_index: + end = self._get_value(slic.end_index) + if end is None: + self.chk.fail( + messages.TUPLE_SLICE_MUST_BE_AN_INT_LITERAL, + slic.end_index) + return AnyType() + + if slic.stride: + stride = self._get_value(slic.stride) + if stride is None: + self.chk.fail( + messages.TUPLE_SLICE_MUST_BE_AN_INT_LITERAL, + slic.stride) + return AnyType() + + return left_type.slice(begin, stride, end) + + def _get_value(self, index: Node) -> Optional[int]: + if isinstance(index, IntExpr): + return index.value + elif isinstance(index, UnaryExpr): + if index.op == '-': + operand = index.expr + if isinstance(operand, IntExpr): + return -1 * operand.value + return None + + def visit_cast_expr(self, expr: CastExpr) -> Type: + """Type check a cast expression.""" + source_type = self.accept(expr.expr, context=AnyType()) + target_type = expr.type + if self.chk.options.warn_redundant_casts and is_same_type(source_type, target_type): + self.msg.redundant_cast(target_type, expr) + if not self.is_valid_cast(source_type, target_type): + self.msg.invalid_cast(target_type, source_type, expr) + return target_type + + def is_valid_cast(self, source_type: Type, target_type: Type) -> bool: + """Is a cast from source_type to target_type meaningful?""" + return (isinstance(target_type, AnyType) or + (not isinstance(source_type, Void) and + not isinstance(target_type, Void))) + + def visit_reveal_type_expr(self, expr: RevealTypeExpr) -> Type: + """Type check a reveal_type expression.""" + revealed_type = self.accept(expr.expr) + self.msg.reveal_type(revealed_type, expr) + return revealed_type + + def visit_type_application(self, tapp: TypeApplication) -> Type: + """Type check a type application (expr[type, ...]).""" + self.chk.fail(messages.GENERIC_TYPE_NOT_VALID_AS_EXPRESSION, tapp) + return AnyType() + + def visit_type_alias_expr(self, alias: TypeAliasExpr) -> Type: + return AnyType() + + def visit_list_expr(self, e: ListExpr) -> Type: + """Type check a list expression [...].""" + return self.check_list_or_set_expr(e.items, 'builtins.list', '', + e) + + def visit_set_expr(self, e: SetExpr) -> Type: + return self.check_list_or_set_expr(e.items, 'builtins.set', '', e) + + def check_list_or_set_expr(self, items: List[Node], fullname: str, + tag: str, context: Context) -> Type: + # Translate into type checking a generic function call. + tvdef = TypeVarDef('T', -1, [], self.chk.object_type()) + tv = TypeVarType(tvdef) + constructor = CallableType( + [tv], + [nodes.ARG_STAR], + [None], + self.chk.named_generic_type(fullname, [tv]), + self.named_type('builtins.function'), + name=tag, + variables=[tvdef]) + return self.check_call(constructor, + items, + [nodes.ARG_POS] * len(items), context)[0] + + def visit_tuple_expr(self, e: TupleExpr) -> Type: + """Type check a tuple expression.""" + ctx = None # type: TupleType + # Try to determine type context for type inference. + if isinstance(self.chk.type_context[-1], TupleType): + t = self.chk.type_context[-1] + if len(t.items) == len(e.items): + ctx = t + # Infer item types. + items = [] # type: List[Type] + for i in range(len(e.items)): + item = e.items[i] + tt = None # type: Type + if not ctx: + tt = self.accept(item) + else: + tt = self.accept(item, ctx.items[i]) + self.check_not_void(tt, e) + items.append(tt) + fallback_item = join.join_type_list(items) + return TupleType(items, self.chk.named_generic_type('builtins.tuple', [fallback_item])) + + def visit_dict_expr(self, e: DictExpr) -> Type: + # Translate into type checking a generic function call. + ktdef = TypeVarDef('KT', -1, [], self.chk.object_type()) + vtdef = TypeVarDef('VT', -2, [], self.chk.object_type()) + kt = TypeVarType(ktdef) + vt = TypeVarType(vtdef) + # The callable type represents a function like this: + # + # def (*v: Tuple[kt, vt]) -> Dict[kt, vt]: ... + constructor = CallableType( + [TupleType([kt, vt], self.named_type('builtins.tuple'))], + [nodes.ARG_STAR], + [None], + self.chk.named_generic_type('builtins.dict', [kt, vt]), + self.named_type('builtins.function'), + name='', + variables=[ktdef, vtdef]) + # Synthesize function arguments. + args = [] # type: List[Node] + for key, value in e.items: + args.append(TupleExpr([key, value])) + return self.check_call(constructor, + args, + [nodes.ARG_POS] * len(args), e)[0] + + def visit_func_expr(self, e: FuncExpr) -> Type: + """Type check lambda expression.""" + inferred_type = self.infer_lambda_type_using_context(e) + if not inferred_type: + # No useful type context. + ret_type = e.expr().accept(self.chk) + if not e.arguments: + # Form 'lambda: e'; just use the inferred return type. + return CallableType([], [], [], ret_type, self.named_type('builtins.function')) + else: + # TODO: Consider reporting an error. However, this is fine if + # we are just doing the first pass in contextual type + # inference. + return AnyType() + else: + # Type context available. + self.chk.check_func_item(e, type_override=inferred_type) + if e.expr() not in self.chk.type_map: + self.accept(e.expr()) + ret_type = self.chk.type_map[e.expr()] + if isinstance(ret_type, NoneTyp): + # For "lambda ...: None", just use type from the context. + # Important when the context is Callable[..., None] which + # really means Void. See #1425. + return inferred_type + return replace_callable_return_type(inferred_type, ret_type) + + def infer_lambda_type_using_context(self, e: FuncExpr) -> CallableType: + """Try to infer lambda expression type using context. + + Return None if could not infer type. + """ + # TODO also accept 'Any' context + ctx = self.chk.type_context[-1] + if not ctx or not isinstance(ctx, CallableType): + return None + + # The context may have function type variables in it. We replace them + # since these are the type variables we are ultimately trying to infer; + # they must be considered as indeterminate. We use ErasedType since it + # does not affect type inference results (it is for purposes like this + # only). + ctx = replace_meta_vars(ctx, ErasedType()) + + callable_ctx = cast(CallableType, ctx) + + arg_kinds = [arg.kind for arg in e.arguments] + + if callable_ctx.is_ellipsis_args: + # Fill in Any arguments to match the arguments of the lambda. + callable_ctx = callable_ctx.copy_modified( + is_ellipsis_args=False, + arg_types=[AnyType()] * len(arg_kinds), + arg_kinds=arg_kinds + ) + + if callable_ctx.arg_kinds != arg_kinds: + # Incompatible context; cannot use it to infer types. + self.chk.fail(messages.CANNOT_INFER_LAMBDA_TYPE, e) + return None + + return callable_ctx + + def visit_super_expr(self, e: SuperExpr) -> Type: + """Type check a super expression (non-lvalue).""" + t = self.analyze_super(e, False) + return t + + def analyze_super(self, e: SuperExpr, is_lvalue: bool) -> Type: + """Type check a super expression.""" + if e.info and e.info.bases: + # TODO fix multiple inheritance etc + if len(e.info.mro) < 2: + self.chk.fail('Internal error: unexpected mro for {}: {}'.format( + e.info.name(), e.info.mro), e) + return AnyType() + for base in e.info.mro[1:]: + if e.name in base.names or base == e.info.mro[-1]: + if e.info.fallback_to_any and base == e.info.mro[-1]: + # There's an undefined base class, and we're + # at the end of the chain. That's not an error. + return AnyType() + if not self.chk.typing_mode_full(): + return AnyType() + return analyze_member_access(e.name, self_type(e.info), e, + is_lvalue, True, + self.named_type, self.not_ready_callback, + self.msg, base) + else: + # Invalid super. This has been reported by the semantic analyzer. + return AnyType() + + def visit_slice_expr(self, e: SliceExpr) -> Type: + for index in [e.begin_index, e.end_index, e.stride]: + if index: + t = self.accept(index) + self.chk.check_subtype(t, self.named_type('builtins.int'), + index, messages.INVALID_SLICE_INDEX) + return self.named_type('builtins.slice') + + def visit_list_comprehension(self, e: ListComprehension) -> Type: + return self.check_generator_or_comprehension( + e.generator, 'builtins.list', '') + + def visit_set_comprehension(self, e: SetComprehension) -> Type: + return self.check_generator_or_comprehension( + e.generator, 'builtins.set', '') + + def visit_generator_expr(self, e: GeneratorExpr) -> Type: + return self.check_generator_or_comprehension(e, 'typing.Iterator', + '') + + def check_generator_or_comprehension(self, gen: GeneratorExpr, + type_name: str, + id_for_messages: str) -> Type: + """Type check a generator expression or a list comprehension.""" + self.check_for_comp(gen) + + # Infer the type of the list comprehension by using a synthetic generic + # callable type. + tvdef = TypeVarDef('T', -1, [], self.chk.object_type()) + tv = TypeVarType(tvdef) + constructor = CallableType( + [tv], + [nodes.ARG_POS], + [None], + self.chk.named_generic_type(type_name, [tv]), + self.chk.named_type('builtins.function'), + name=id_for_messages, + variables=[tvdef]) + return self.check_call(constructor, + [gen.left_expr], [nodes.ARG_POS], gen)[0] + + def visit_dictionary_comprehension(self, e: DictionaryComprehension): + """Type check a dictionary comprehension.""" + self.check_for_comp(e) + + # Infer the type of the list comprehension by using a synthetic generic + # callable type. + ktdef = TypeVarDef('KT', -1, [], self.chk.object_type()) + vtdef = TypeVarDef('VT', -2, [], self.chk.object_type()) + kt = TypeVarType(ktdef) + vt = TypeVarType(vtdef) + constructor = CallableType( + [kt, vt], + [nodes.ARG_POS, nodes.ARG_POS], + [None, None], + self.chk.named_generic_type('builtins.dict', [kt, vt]), + self.chk.named_type('builtins.function'), + name='', + variables=[ktdef, vtdef]) + return self.check_call(constructor, + [e.key, e.value], [nodes.ARG_POS, nodes.ARG_POS], e)[0] + + def check_for_comp(self, e: Union[GeneratorExpr, DictionaryComprehension]) -> None: + """Check the for_comp part of comprehensions. That is the part from 'for': + ... for x in y if z + """ + with self.chk.binder.frame_context(): + for index, sequence, conditions in zip(e.indices, e.sequences, + e.condlists): + sequence_type = self.chk.analyze_iterable_item_type(sequence) + self.chk.analyze_index_variables(index, sequence_type, e) + for condition in conditions: + self.accept(condition) + + def visit_conditional_expr(self, e: ConditionalExpr) -> Type: + cond_type = self.accept(e.cond) + self.check_not_void(cond_type, e) + ctx = self.chk.type_context[-1] + + # Gain type information from isinstance if it is there + # but only for the current expression + if_map, else_map = mypy.checker.find_isinstance_check( + e.cond, + self.chk.type_map, + self.chk.typing_mode_weak()) + + if_type = self.analyze_cond_branch(if_map, e.if_expr, context=ctx) + + if not mypy.checker.is_valid_inferred_type(if_type): + # Analyze the right branch disregarding the left branch. + else_type = self.analyze_cond_branch(else_map, e.else_expr, context=ctx) + + # If it would make a difference, re-analyze the left + # branch using the right branch's type as context. + if ctx is None or not is_equivalent(else_type, ctx): + # TODO: If it's possible that the previous analysis of + # the left branch produced errors that are avoided + # using this context, suppress those errors. + if_type = self.analyze_cond_branch(if_map, e.if_expr, context=else_type) + + else: + # Analyze the right branch in the context of the left + # branch's type. + else_type = self.analyze_cond_branch(else_map, e.else_expr, context=if_type) + + res = join.join_types(if_type, else_type) + + return res + + def analyze_cond_branch(self, map: Optional[Dict[Node, Type]], + node: Node, context: Optional[Type]) -> Type: + with self.chk.binder.frame_context(): + if map: + for var, type in map.items(): + self.chk.binder.push(var, type) + return self.accept(node, context=context) + + def visit_backquote_expr(self, e: BackquoteExpr) -> Type: + self.accept(e.expr) + return self.named_type('builtins.str') + + # + # Helpers + # + + def accept(self, node: Node, context: Type = None) -> Type: + """Type check a node. Alias for TypeChecker.accept.""" + return self.chk.accept(node, context) + + def check_not_void(self, typ: Type, context: Context) -> None: + """Generate an error if type is Void.""" + self.chk.check_not_void(typ, context) + + def is_boolean(self, typ: Type) -> bool: + """Is type compatible with bool?""" + return is_subtype(typ, self.chk.bool_type()) + + def named_type(self, name: str) -> Instance: + """Return an instance type with type given by the name and no type + arguments. Alias for TypeChecker.named_type. + """ + return self.chk.named_type(name) + + def is_valid_var_arg(self, typ: Type) -> bool: + """Is a type valid as a *args argument?""" + return (isinstance(typ, TupleType) or + is_subtype(typ, self.chk.named_generic_type('typing.Iterable', + [AnyType()])) or + isinstance(typ, AnyType)) + + def is_valid_keyword_var_arg(self, typ: Type) -> bool: + """Is a type valid as a **kwargs argument?""" + return is_subtype(typ, self.chk.named_generic_type( + 'builtins.dict', [self.named_type('builtins.str'), AnyType()])) + + def has_non_method(self, typ: Type, member: str) -> bool: + """Does type have a member variable / property with the given name?""" + if isinstance(typ, Instance): + return (not typ.type.has_method(member) and + typ.type.has_readable_member(member)) + else: + return False + + def has_member(self, typ: Type, member: str) -> bool: + """Does type have member with the given name?""" + # TODO TupleType => also consider tuple attributes + if isinstance(typ, Instance): + return typ.type.has_readable_member(member) + elif isinstance(typ, AnyType): + return True + elif isinstance(typ, UnionType): + result = all(self.has_member(x, member) for x in typ.items) + return result + elif isinstance(typ, TupleType): + return self.has_member(typ.fallback, member) + else: + return False + + def not_ready_callback(self, name: str, context: Context) -> None: + """Called when we can't infer the type of a variable because it's not ready yet. + + Either defer type checking of the enclosing function to the next + pass or report an error. + """ + self.chk.handle_cannot_determine_type(name, context) + + +def map_actuals_to_formals(caller_kinds: List[int], + caller_names: List[str], + callee_kinds: List[int], + callee_names: List[str], + caller_arg_type: Callable[[int], + Type]) -> List[List[int]]: + """Calculate mapping between actual (caller) args and formals. + + The result contains a list of caller argument indexes mapping to each + callee argument index, indexed by callee index. + + The caller_arg_type argument should evaluate to the type of the actual + argument type with the given index. + """ + ncallee = len(callee_kinds) + map = [None] * ncallee # type: List[List[int]] + for i in range(ncallee): + map[i] = [] + j = 0 + for i, kind in enumerate(caller_kinds): + if kind == nodes.ARG_POS: + if j < ncallee: + if callee_kinds[j] in [nodes.ARG_POS, nodes.ARG_OPT, + nodes.ARG_NAMED]: + map[j].append(i) + j += 1 + elif callee_kinds[j] == nodes.ARG_STAR: + map[j].append(i) + elif kind == nodes.ARG_STAR: + # We need to know the actual type to map varargs. + argt = caller_arg_type(i) + if isinstance(argt, TupleType): + # A tuple actual maps to a fixed number of formals. + for _ in range(len(argt.items)): + if j < ncallee: + if callee_kinds[j] != nodes.ARG_STAR2: + map[j].append(i) + else: + raise NotImplementedError() + j += 1 + else: + # Assume that it is an iterable (if it isn't, there will be + # an error later). + while j < ncallee: + if callee_kinds[j] in (nodes.ARG_NAMED, nodes.ARG_STAR2): + break + else: + map[j].append(i) + j += 1 + elif kind == nodes.ARG_NAMED: + name = caller_names[i] + if name in callee_names: + map[callee_names.index(name)].append(i) + elif nodes.ARG_STAR2 in callee_kinds: + map[callee_kinds.index(nodes.ARG_STAR2)].append(i) + else: + assert kind == nodes.ARG_STAR2 + for j in range(ncallee): + # TODO tuple varargs complicate this + no_certain_match = ( + not map[j] or caller_kinds[map[j][0]] == nodes.ARG_STAR) + if ((callee_names[j] and no_certain_match) + or callee_kinds[j] == nodes.ARG_STAR2): + map[j].append(i) + return map + + +def is_empty_tuple(t: Type) -> bool: + return isinstance(t, TupleType) and not t.items + + +def is_duplicate_mapping(mapping: List[int], actual_kinds: List[int]) -> bool: + # Multiple actuals can map to the same formal only if they both come from + # varargs (*args and **kwargs); in this case at runtime it is possible that + # there are no duplicates. We need to allow this, as the convention + # f(..., *args, **kwargs) is common enough. + return len(mapping) > 1 and not ( + len(mapping) == 2 and + actual_kinds[mapping[0]] == nodes.ARG_STAR and + actual_kinds[mapping[1]] == nodes.ARG_STAR2) + + +def replace_callable_return_type(c: CallableType, new_ret_type: Type) -> CallableType: + """Return a copy of a callable type with a different return type.""" + return c.copy_modified(ret_type=new_ret_type) + + +class ArgInferSecondPassQuery(types.TypeQuery): + """Query whether an argument type should be inferred in the second pass. + + The result is True if the type has a type variable in a callable return + type anywhere. For example, the result for Callable[[], T] is True if t is + a type variable. + """ + def __init__(self) -> None: + super().__init__(False, types.ANY_TYPE_STRATEGY) + + def visit_callable_type(self, t: CallableType) -> bool: + return self.query_types(t.arg_types) or t.accept(HasTypeVarQuery()) + + +class HasTypeVarQuery(types.TypeQuery): + """Visitor for querying whether a type has a type variable component.""" + def __init__(self) -> None: + super().__init__(False, types.ANY_TYPE_STRATEGY) + + def visit_type_var(self, t: TypeVarType) -> bool: + return True + + +def has_erased_component(t: Type) -> bool: + return t is not None and t.accept(HasErasedComponentsQuery()) + + +class HasErasedComponentsQuery(types.TypeQuery): + """Visitor for querying whether a type has an erased component.""" + def __init__(self) -> None: + super().__init__(False, types.ANY_TYPE_STRATEGY) + + def visit_erased_type(self, t: ErasedType) -> bool: + return True + + +def overload_arg_similarity(actual: Type, formal: Type) -> int: + """Return if caller argument (actual) is compatible with overloaded signature arg (formal). + + Return a similarity level: + 0: no match + 1: actual is compatible, but only using type promotions (e.g. int vs float) + 2: actual is compatible without type promotions (e.g. int vs object) + + The distinction is important in cases where multiple overload items match. We want + give priority to higher similarity matches. + """ + # Replace type variables with their upper bounds. Overloading + # resolution is based on runtime behavior which erases type + # parameters, so no need to handle type variables occurring within + # a type. + if isinstance(actual, TypeVarType): + actual = actual.erase_to_union_or_bound() + if isinstance(formal, TypeVarType): + formal = formal.erase_to_union_or_bound() + if (isinstance(actual, UninhabitedType) or isinstance(actual, AnyType) or + isinstance(formal, AnyType) or isinstance(formal, CallableType) or + (isinstance(actual, Instance) and actual.type.fallback_to_any)): + # These could match anything at runtime. + return 2 + if not experiments.STRICT_OPTIONAL and isinstance(actual, NoneTyp): + # NoneTyp matches anything if we're not doing strict Optional checking + return 2 + if isinstance(actual, UnionType): + return max(overload_arg_similarity(item, formal) + for item in actual.items) + if isinstance(formal, UnionType): + return max(overload_arg_similarity(actual, item) + for item in formal.items) + if isinstance(formal, TypeType): + if isinstance(actual, TypeType): + # Since Type[T] is covariant, check if actual = Type[A] is + # a subtype of formal = Type[F]. + return overload_arg_similarity(actual.item, formal.item) + elif isinstance(actual, CallableType) and actual.is_type_obj(): + # Check if the actual is a constructor of some sort. + # Note that this is this unsound, since we don't check the __init__ signature. + return overload_arg_similarity(actual.ret_type, formal.item) + else: + return 0 + if isinstance(formal, Instance): + if isinstance(actual, CallableType): + actual = actual.fallback + if isinstance(actual, Overloaded): + actual = actual.items()[0].fallback + if isinstance(actual, TupleType): + actual = actual.fallback + if isinstance(actual, Instance): + # First perform a quick check (as an optimization) and fall back to generic + # subtyping algorithm if type promotions are possible (e.g., int vs. float). + if formal.type in actual.type.mro: + return 2 + elif actual.type._promote and is_subtype(actual, formal): + return 1 + else: + return 0 + elif isinstance(actual, TypeType): + if formal.type.fullname() in {"builtins.object", "builtins.type"}: + return 2 + else: + return 0 + else: + return 0 + if isinstance(actual, UnboundType) or isinstance(formal, UnboundType): + # Either actual or formal is the result of an error; shut up. + return 2 + # Fall back to a conservative equality check for the remaining kinds of type. + return 2 if is_same_type(erasetype.erase_type(actual), erasetype.erase_type(formal)) else 0 + + +def freshen_generic_callable(callee: CallableType) -> CallableType: + tvdefs = [] + tvmap = {} # type: Dict[TypeVarId, Type] + for v in callee.variables: + tvdef = TypeVarDef.new_unification_variable(v) + tvdefs.append(tvdef) + tvmap[v.id] = TypeVarType(tvdef) + + return cast(CallableType, expand_type(callee, tvmap)).copy_modified(variables=tvdefs) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index b74c29f83cd9d..ce6d5190fac6a 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -1,112 +1,112 @@ -from typing import Dict, Tuple, List, cast - -from mypy.types import ( - Type, Instance, CallableType, TypeVisitor, UnboundType, ErrorType, AnyType, - Void, NoneTyp, TypeVarType, Overloaded, TupleType, UnionType, ErasedType, TypeList, - PartialType, DeletedType, UninhabitedType, TypeType, TypeVarId -) - - -def expand_type(typ: Type, env: Dict[TypeVarId, Type]) -> Type: - """Substitute any type variable references in a type given by a type - environment. - """ - - return typ.accept(ExpandTypeVisitor(env)) - - -def expand_type_by_instance(typ: Type, instance: Instance) -> Type: - """Substitute type variables in type using values from an Instance. - Type variables are considered to be bound by the class declaration.""" - - if instance.args == []: - return typ - else: - variables = {} # type: Dict[TypeVarId, Type] - for binder, arg in zip(instance.type.defn.type_vars, instance.args): - variables[binder.id] = arg - return expand_type(typ, variables) - - -class ExpandTypeVisitor(TypeVisitor[Type]): - """Visitor that substitutes type variables with values.""" - - variables = None # type: Dict[TypeVarId, Type] # TypeVar id -> TypeVar value - - def __init__(self, variables: Dict[TypeVarId, Type]) -> None: - self.variables = variables - - def visit_unbound_type(self, t: UnboundType) -> Type: - return t - - def visit_error_type(self, t: ErrorType) -> Type: - return t - - def visit_type_list(self, t: TypeList) -> Type: - assert False, 'Not supported' - - def visit_any(self, t: AnyType) -> Type: - return t - - def visit_void(self, t: Void) -> Type: - return t - - def visit_none_type(self, t: NoneTyp) -> Type: - return t - - def visit_uninhabited_type(self, t: UninhabitedType) -> Type: - return t - - def visit_deleted_type(self, t: DeletedType) -> Type: - return t - - def visit_erased_type(self, t: ErasedType) -> Type: - # Should not get here. - raise RuntimeError() - - def visit_instance(self, t: Instance) -> Type: - args = self.expand_types(t.args) - return Instance(t.type, args, t.line) - - def visit_type_var(self, t: TypeVarType) -> Type: - repl = self.variables.get(t.id, t) - if isinstance(repl, Instance): - inst = repl - # Return copy of instance with type erasure flag on. - return Instance(inst.type, inst.args, inst.line, True) - else: - return repl - - def visit_callable_type(self, t: CallableType) -> Type: - return t.copy_modified(arg_types=self.expand_types(t.arg_types), - ret_type=t.ret_type.accept(self)) - - def visit_overloaded(self, t: Overloaded) -> Type: - items = [] # type: List[CallableType] - for item in t.items(): - items.append(cast(CallableType, item.accept(self))) - return Overloaded(items) - - def visit_tuple_type(self, t: TupleType) -> Type: - return t.with_types(self.expand_types(t.items)) - - def visit_union_type(self, t: UnionType) -> Type: - # After substituting for type variables in t.items, - # some of the resulting types might be subtypes of others. - return UnionType.make_simplified_union(self.expand_types(t.items), t.line) - - def visit_partial_type(self, t: PartialType) -> Type: - return t - - def visit_type_type(self, t: TypeType) -> Type: - # TODO: Verify that the new item type is valid (instance or - # union of instances or Any). Sadly we can't report errors - # here yet. - item = t.item.accept(self) - return TypeType(item) - - def expand_types(self, types: List[Type]) -> List[Type]: - a = [] # type: List[Type] - for t in types: - a.append(t.accept(self)) - return a +from typing import Dict, Tuple, List, cast + +from mypy.types import ( + Type, Instance, CallableType, TypeVisitor, UnboundType, ErrorType, AnyType, + Void, NoneTyp, TypeVarType, Overloaded, TupleType, UnionType, ErasedType, TypeList, + PartialType, DeletedType, UninhabitedType, TypeType, TypeVarId +) + + +def expand_type(typ: Type, env: Dict[TypeVarId, Type]) -> Type: + """Substitute any type variable references in a type given by a type + environment. + """ + + return typ.accept(ExpandTypeVisitor(env)) + + +def expand_type_by_instance(typ: Type, instance: Instance) -> Type: + """Substitute type variables in type using values from an Instance. + Type variables are considered to be bound by the class declaration.""" + + if instance.args == []: + return typ + else: + variables = {} # type: Dict[TypeVarId, Type] + for binder, arg in zip(instance.type.defn.type_vars, instance.args): + variables[binder.id] = arg + return expand_type(typ, variables) + + +class ExpandTypeVisitor(TypeVisitor[Type]): + """Visitor that substitutes type variables with values.""" + + variables = None # type: Dict[TypeVarId, Type] # TypeVar id -> TypeVar value + + def __init__(self, variables: Dict[TypeVarId, Type]) -> None: + self.variables = variables + + def visit_unbound_type(self, t: UnboundType) -> Type: + return t + + def visit_error_type(self, t: ErrorType) -> Type: + return t + + def visit_type_list(self, t: TypeList) -> Type: + assert False, 'Not supported' + + def visit_any(self, t: AnyType) -> Type: + return t + + def visit_void(self, t: Void) -> Type: + return t + + def visit_none_type(self, t: NoneTyp) -> Type: + return t + + def visit_uninhabited_type(self, t: UninhabitedType) -> Type: + return t + + def visit_deleted_type(self, t: DeletedType) -> Type: + return t + + def visit_erased_type(self, t: ErasedType) -> Type: + # Should not get here. + raise RuntimeError() + + def visit_instance(self, t: Instance) -> Type: + args = self.expand_types(t.args) + return Instance(t.type, args, t.line) + + def visit_type_var(self, t: TypeVarType) -> Type: + repl = self.variables.get(t.id, t) + if isinstance(repl, Instance): + inst = repl + # Return copy of instance with type erasure flag on. + return Instance(inst.type, inst.args, inst.line, True) + else: + return repl + + def visit_callable_type(self, t: CallableType) -> Type: + return t.copy_modified(arg_types=self.expand_types(t.arg_types), + ret_type=t.ret_type.accept(self)) + + def visit_overloaded(self, t: Overloaded) -> Type: + items = [] # type: List[CallableType] + for item in t.items(): + items.append(cast(CallableType, item.accept(self))) + return Overloaded(items) + + def visit_tuple_type(self, t: TupleType) -> Type: + return t.with_types(self.expand_types(t.items)) + + def visit_union_type(self, t: UnionType) -> Type: + # After substituting for type variables in t.items, + # some of the resulting types might be subtypes of others. + return UnionType.make_simplified_union(self.expand_types(t.items), t.line) + + def visit_partial_type(self, t: PartialType) -> Type: + return t + + def visit_type_type(self, t: TypeType) -> Type: + # TODO: Verify that the new item type is valid (instance or + # union of instances or Any). Sadly we can't report errors + # here yet. + item = t.item.accept(self) + return TypeType(item) + + def expand_types(self, types: List[Type]) -> List[Type]: + a = [] # type: List[Type] + for t in types: + a.append(t.accept(self)) + return a diff --git a/mypy/join.py b/mypy/join.py index e0709c2717911..684d62cfeaebd 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -1,367 +1,367 @@ -"""Calculation of the least upper bound types (joins).""" - -from typing import List - -from mypy.types import ( - Type, AnyType, NoneTyp, Void, TypeVisitor, Instance, UnboundType, - ErrorType, TypeVarType, CallableType, TupleType, ErasedType, TypeList, - UnionType, FunctionLike, Overloaded, PartialType, DeletedType, - UninhabitedType, TypeType, NamedTupleType -) -from mypy.maptype import map_instance_to_supertype -from mypy.subtypes import is_subtype, is_equivalent, is_subtype_ignoring_tvars - -from mypy import experiments - - -def join_simple(declaration: Type, s: Type, t: Type) -> Type: - """Return a simple least upper bound given the declared type.""" - - if isinstance(s, AnyType): - return s - - if isinstance(s, ErasedType): - return t - - if is_subtype(s, t): - return t - - if is_subtype(t, s): - return s - - if isinstance(declaration, UnionType): - return UnionType.make_simplified_union([s, t]) - - if isinstance(s, NoneTyp) and not isinstance(t, NoneTyp): - s, t = t, s - - if isinstance(s, UninhabitedType) and not isinstance(t, UninhabitedType): - s, t = t, s - - value = t.accept(TypeJoinVisitor(s)) - - if value is None: - # XXX this code path probably should be avoided. - # It seems to happen when a line (x = y) is a type error, and - # it's not clear that assuming that x is arbitrary afterward - # is a good idea. - return declaration - - if declaration is None or is_subtype(value, declaration): - return value - - return declaration - - -def join_types(s: Type, t: Type) -> Type: - """Return the least upper bound of s and t. - - For example, the join of 'int' and 'object' is 'object'. - - If the join does not exist, return an ErrorType instance. - """ - if isinstance(s, AnyType): - return s - - if isinstance(s, ErasedType): - return t - - if isinstance(s, NoneTyp) and not isinstance(t, NoneTyp): - s, t = t, s - - # Use a visitor to handle non-trivial cases. - return t.accept(TypeJoinVisitor(s)) - - -class TypeJoinVisitor(TypeVisitor[Type]): - """Implementation of the least upper bound algorithm. - - Attributes: - s: The other (left) type operand. - """ - - def __init__(self, s: Type) -> None: - self.s = s - - def visit_unbound_type(self, t: UnboundType) -> Type: - if isinstance(self.s, Void) or isinstance(self.s, ErrorType): - return ErrorType() - else: - return AnyType() - - def visit_union_type(self, t: UnionType) -> Type: - if is_subtype(self.s, t): - return t - else: - return UnionType.make_simplified_union([self.s, t]) - - def visit_error_type(self, t: ErrorType) -> Type: - return t - - def visit_type_list(self, t: TypeList) -> Type: - assert False, 'Not supported' - - def visit_any(self, t: AnyType) -> Type: - return t - - def visit_void(self, t: Void) -> Type: - if isinstance(self.s, Void): - return t - else: - return ErrorType() - - def visit_none_type(self, t: NoneTyp) -> Type: - if experiments.STRICT_OPTIONAL: - if isinstance(self.s, (NoneTyp, UninhabitedType)): - return t - else: - return self.default(self.s) - else: - if not isinstance(self.s, Void): - return self.s - else: - return self.default(self.s) - - def visit_uninhabited_type(self, t: UninhabitedType) -> Type: - if not isinstance(self.s, Void): - return self.s - else: - return self.default(self.s) - - def visit_deleted_type(self, t: DeletedType) -> Type: - if not isinstance(self.s, Void): - return self.s - else: - return self.default(self.s) - - def visit_erased_type(self, t: ErasedType) -> Type: - return self.s - - def visit_type_var(self, t: TypeVarType) -> Type: - if isinstance(self.s, TypeVarType) and self.s.id == t.id: - return self.s - else: - return self.default(self.s) - - def visit_instance(self, t: Instance) -> Type: - if isinstance(self.s, Instance): - return join_instances(t, self.s) - elif isinstance(self.s, FunctionLike): - return join_types(t, self.s.fallback) - elif isinstance(self.s, TypeType): - return join_types(t, self.s) - else: - return self.default(self.s) - - def visit_callable_type(self, t: CallableType) -> Type: - # TODO: Consider subtyping instead of just similarity. - if isinstance(self.s, CallableType) and is_similar_callables(t, self.s): - return combine_similar_callables(t, self.s) - elif isinstance(self.s, Overloaded): - # Switch the order of arguments to that we'll get to visit_overloaded. - return join_types(t, self.s) - else: - return join_types(t.fallback, self.s) - - def visit_overloaded(self, t: Overloaded) -> Type: - # This is more complex than most other cases. Here are some - # examples that illustrate how this works. - # - # First let's define a concise notation: - # - Cn are callable types (for n in 1, 2, ...) - # - Ov(C1, C2, ...) is an overloaded type with items C1, C2, ... - # - Callable[[T, ...], S] is written as [T, ...] -> S. - # - # We want some basic properties to hold (assume Cn are all - # unrelated via Any-similarity): - # - # join(Ov(C1, C2), C1) == C1 - # join(Ov(C1, C2), Ov(C1, C2)) == Ov(C1, C2) - # join(Ov(C1, C2), Ov(C1, C3)) == C1 - # join(Ov(C2, C2), C3) == join of fallback types - # - # The presence of Any types makes things more interesting. The join is the - # most general type we can get with respect to Any: - # - # join(Ov([int] -> int, [str] -> str), [Any] -> str) == Any -> str - # - # We could use a simplification step that removes redundancies, but that's not - # implemented right now. Consider this example, where we get a redundancy: - # - # join(Ov([int, Any] -> Any, [str, Any] -> Any), [Any, int] -> Any) == - # Ov([Any, int] -> Any, [Any, int] -> Any) - # - # TODO: Use callable subtyping instead of just similarity. - result = [] # type: List[CallableType] - s = self.s - if isinstance(s, FunctionLike): - # The interesting case where both types are function types. - for t_item in t.items(): - for s_item in s.items(): - if is_similar_callables(t_item, s_item): - result.append(combine_similar_callables(t_item, s_item)) - if result: - # TODO: Simplify redundancies from the result. - if len(result) == 1: - return result[0] - else: - return Overloaded(result) - return join_types(t.fallback, s.fallback) - return join_types(t.fallback, s) - - def visit_tuple_type(self, t: TupleType) -> Type: - if self.s == t: - return self.s - if isinstance(self.s, TupleType) and self.s.length() == t.length(): - items = [] # type: List[Type] - for i in range(t.length()): - items.append(self.join(t.items[i], self.s.items[i])) - if (not (isinstance(t, NamedTupleType) and isinstance(self.s, NamedTupleType)) - or self.s.attrs == t.attrs): - return self.s.with_types(items) - # TODO: What if the fallback types are different? - return TupleType(items, t.fallback) - else: - return self.default(self.s) - - def visit_partial_type(self, t: PartialType) -> Type: - # We only have partial information so we can't decide the join result. We should - # never get here. - assert False, "Internal error" - - def visit_type_type(self, t: TypeType) -> Type: - if isinstance(self.s, TypeType): - return TypeType(self.join(t.item, self.s.item), line=t.line) - elif isinstance(self.s, Instance) and self.s.type.fullname() == 'builtins.type': - return self.s - else: - return self.default(self.s) - - def join(self, s: Type, t: Type) -> Type: - return join_types(s, t) - - def default(self, typ: Type) -> Type: - if isinstance(typ, Instance): - return object_from_instance(typ) - elif isinstance(typ, UnboundType): - return AnyType() - elif isinstance(typ, Void) or isinstance(typ, ErrorType): - return ErrorType() - elif isinstance(typ, TupleType): - return self.default(typ.fallback) - elif isinstance(typ, FunctionLike): - return self.default(typ.fallback) - elif isinstance(typ, TypeVarType): - return self.default(typ.upper_bound) - else: - return AnyType() - - -def join_instances(t: Instance, s: Instance) -> Type: - """Calculate the join of two instance types. - - If allow_interfaces is True, also consider interface-type results for - non-interface types. - - Return ErrorType if the result is ambiguous. - """ - if t.type == s.type: - # Simplest case: join two types with the same base type (but - # potentially different arguments). - if is_subtype(t, s) or is_subtype(s, t): - # Compatible; combine type arguments. - args = [] # type: List[Type] - for i in range(len(t.args)): - args.append(join_types(t.args[i], s.args[i])) - return Instance(t.type, args) - else: - # Incompatible; return trivial result object. - return object_from_instance(t) - elif t.type.bases and is_subtype_ignoring_tvars(t, s): - return join_instances_via_supertype(t, s) - else: - # Now t is not a subtype of s, and t != s. Now s could be a subtype - # of t; alternatively, we need to find a common supertype. This works - # in of the both cases. - return join_instances_via_supertype(s, t) - - -def join_instances_via_supertype(t: Instance, s: Instance) -> Type: - # Give preference to joins via duck typing relationship, so that - # join(int, float) == float, for example. - if t.type._promote and is_subtype(t.type._promote, s): - return join_types(t.type._promote, s) - elif s.type._promote and is_subtype(s.type._promote, t): - return join_types(t, s.type._promote) - # Compute the "best" supertype of t when joined with s. - # The definition of "best" may evolve; for now it is the one with - # the longest MRO. Ties are broken by using the earlier base. - best = None # type: Type - for base in t.type.bases: - mapped = map_instance_to_supertype(t, base.type) - res = join_instances(mapped, s) - if best is None or is_better(res, best): - best = res - assert best is not None - return best - - -def is_better(t: Type, s: Type) -> bool: - # Given two possible results from join_instances_via_supertype(), - # indicate whether t is the better one. - if isinstance(t, Instance): - if not isinstance(s, Instance): - return True - # Use len(mro) as a proxy for the better choice. - if len(t.type.mro) > len(s.type.mro): - return True - return False - - -def is_similar_callables(t: CallableType, s: CallableType) -> bool: - """Return True if t and s are equivalent and have identical numbers of - arguments, default arguments and varargs. - """ - - return (len(t.arg_types) == len(s.arg_types) and t.min_args == s.min_args - and t.is_var_arg == s.is_var_arg and is_equivalent(t, s)) - - -def combine_similar_callables(t: CallableType, s: CallableType) -> CallableType: - arg_types = [] # type: List[Type] - for i in range(len(t.arg_types)): - arg_types.append(join_types(t.arg_types[i], s.arg_types[i])) - # TODO kinds and argument names - # The fallback type can be either 'function' or 'type'. The result should have 'type' as - # fallback only if both operands have it as 'type'. - if t.fallback.type.fullname() != 'builtins.type': - fallback = t.fallback - else: - fallback = s.fallback - return t.copy_modified(arg_types=arg_types, - ret_type=join_types(t.ret_type, s.ret_type), - fallback=fallback, - name=None) - - -def object_from_instance(instance: Instance) -> Instance: - """Construct the type 'builtins.object' from an instance type.""" - # Use the fact that 'object' is always the last class in the mro. - res = Instance(instance.type.mro[-1], []) - return res - - -def join_type_list(types: List[Type]) -> Type: - if not types: - # This is a little arbitrary but reasonable. Any empty tuple should be compatible - # with all variable length tuples, and this makes it possible. A better approach - # would be to use a special bottom type, which we do when strict Optional - # checking is enabled. - if experiments.STRICT_OPTIONAL: - return UninhabitedType() - else: - return NoneTyp() - joined = types[0] - for t in types[1:]: - joined = join_types(joined, t) - return joined +"""Calculation of the least upper bound types (joins).""" + +from typing import List + +from mypy.types import ( + Type, AnyType, NoneTyp, Void, TypeVisitor, Instance, UnboundType, + ErrorType, TypeVarType, CallableType, TupleType, ErasedType, TypeList, + UnionType, FunctionLike, Overloaded, PartialType, DeletedType, + UninhabitedType, TypeType, NamedTupleType +) +from mypy.maptype import map_instance_to_supertype +from mypy.subtypes import is_subtype, is_equivalent, is_subtype_ignoring_tvars + +from mypy import experiments + + +def join_simple(declaration: Type, s: Type, t: Type) -> Type: + """Return a simple least upper bound given the declared type.""" + + if isinstance(s, AnyType): + return s + + if isinstance(s, ErasedType): + return t + + if is_subtype(s, t): + return t + + if is_subtype(t, s): + return s + + if isinstance(declaration, UnionType): + return UnionType.make_simplified_union([s, t]) + + if isinstance(s, NoneTyp) and not isinstance(t, NoneTyp): + s, t = t, s + + if isinstance(s, UninhabitedType) and not isinstance(t, UninhabitedType): + s, t = t, s + + value = t.accept(TypeJoinVisitor(s)) + + if value is None: + # XXX this code path probably should be avoided. + # It seems to happen when a line (x = y) is a type error, and + # it's not clear that assuming that x is arbitrary afterward + # is a good idea. + return declaration + + if declaration is None or is_subtype(value, declaration): + return value + + return declaration + + +def join_types(s: Type, t: Type) -> Type: + """Return the least upper bound of s and t. + + For example, the join of 'int' and 'object' is 'object'. + + If the join does not exist, return an ErrorType instance. + """ + if isinstance(s, AnyType): + return s + + if isinstance(s, ErasedType): + return t + + if isinstance(s, NoneTyp) and not isinstance(t, NoneTyp): + s, t = t, s + + # Use a visitor to handle non-trivial cases. + return t.accept(TypeJoinVisitor(s)) + + +class TypeJoinVisitor(TypeVisitor[Type]): + """Implementation of the least upper bound algorithm. + + Attributes: + s: The other (left) type operand. + """ + + def __init__(self, s: Type) -> None: + self.s = s + + def visit_unbound_type(self, t: UnboundType) -> Type: + if isinstance(self.s, Void) or isinstance(self.s, ErrorType): + return ErrorType() + else: + return AnyType() + + def visit_union_type(self, t: UnionType) -> Type: + if is_subtype(self.s, t): + return t + else: + return UnionType.make_simplified_union([self.s, t]) + + def visit_error_type(self, t: ErrorType) -> Type: + return t + + def visit_type_list(self, t: TypeList) -> Type: + assert False, 'Not supported' + + def visit_any(self, t: AnyType) -> Type: + return t + + def visit_void(self, t: Void) -> Type: + if isinstance(self.s, Void): + return t + else: + return ErrorType() + + def visit_none_type(self, t: NoneTyp) -> Type: + if experiments.STRICT_OPTIONAL: + if isinstance(self.s, (NoneTyp, UninhabitedType)): + return t + else: + return self.default(self.s) + else: + if not isinstance(self.s, Void): + return self.s + else: + return self.default(self.s) + + def visit_uninhabited_type(self, t: UninhabitedType) -> Type: + if not isinstance(self.s, Void): + return self.s + else: + return self.default(self.s) + + def visit_deleted_type(self, t: DeletedType) -> Type: + if not isinstance(self.s, Void): + return self.s + else: + return self.default(self.s) + + def visit_erased_type(self, t: ErasedType) -> Type: + return self.s + + def visit_type_var(self, t: TypeVarType) -> Type: + if isinstance(self.s, TypeVarType) and self.s.id == t.id: + return self.s + else: + return self.default(self.s) + + def visit_instance(self, t: Instance) -> Type: + if isinstance(self.s, Instance): + return join_instances(t, self.s) + elif isinstance(self.s, FunctionLike): + return join_types(t, self.s.fallback) + elif isinstance(self.s, TypeType): + return join_types(t, self.s) + else: + return self.default(self.s) + + def visit_callable_type(self, t: CallableType) -> Type: + # TODO: Consider subtyping instead of just similarity. + if isinstance(self.s, CallableType) and is_similar_callables(t, self.s): + return combine_similar_callables(t, self.s) + elif isinstance(self.s, Overloaded): + # Switch the order of arguments to that we'll get to visit_overloaded. + return join_types(t, self.s) + else: + return join_types(t.fallback, self.s) + + def visit_overloaded(self, t: Overloaded) -> Type: + # This is more complex than most other cases. Here are some + # examples that illustrate how this works. + # + # First let's define a concise notation: + # - Cn are callable types (for n in 1, 2, ...) + # - Ov(C1, C2, ...) is an overloaded type with items C1, C2, ... + # - Callable[[T, ...], S] is written as [T, ...] -> S. + # + # We want some basic properties to hold (assume Cn are all + # unrelated via Any-similarity): + # + # join(Ov(C1, C2), C1) == C1 + # join(Ov(C1, C2), Ov(C1, C2)) == Ov(C1, C2) + # join(Ov(C1, C2), Ov(C1, C3)) == C1 + # join(Ov(C2, C2), C3) == join of fallback types + # + # The presence of Any types makes things more interesting. The join is the + # most general type we can get with respect to Any: + # + # join(Ov([int] -> int, [str] -> str), [Any] -> str) == Any -> str + # + # We could use a simplification step that removes redundancies, but that's not + # implemented right now. Consider this example, where we get a redundancy: + # + # join(Ov([int, Any] -> Any, [str, Any] -> Any), [Any, int] -> Any) == + # Ov([Any, int] -> Any, [Any, int] -> Any) + # + # TODO: Use callable subtyping instead of just similarity. + result = [] # type: List[CallableType] + s = self.s + if isinstance(s, FunctionLike): + # The interesting case where both types are function types. + for t_item in t.items(): + for s_item in s.items(): + if is_similar_callables(t_item, s_item): + result.append(combine_similar_callables(t_item, s_item)) + if result: + # TODO: Simplify redundancies from the result. + if len(result) == 1: + return result[0] + else: + return Overloaded(result) + return join_types(t.fallback, s.fallback) + return join_types(t.fallback, s) + + def visit_tuple_type(self, t: TupleType) -> Type: + if self.s == t: + return self.s + if isinstance(self.s, TupleType) and self.s.length() == t.length(): + items = [] # type: List[Type] + for i in range(t.length()): + items.append(self.join(t.items[i], self.s.items[i])) + if (not (isinstance(t, NamedTupleType) and isinstance(self.s, NamedTupleType)) + or self.s.attrs == t.attrs): + return self.s.with_types(items) + # TODO: What if the fallback types are different? + return TupleType(items, t.fallback) + else: + return self.default(self.s) + + def visit_partial_type(self, t: PartialType) -> Type: + # We only have partial information so we can't decide the join result. We should + # never get here. + assert False, "Internal error" + + def visit_type_type(self, t: TypeType) -> Type: + if isinstance(self.s, TypeType): + return TypeType(self.join(t.item, self.s.item), line=t.line) + elif isinstance(self.s, Instance) and self.s.type.fullname() == 'builtins.type': + return self.s + else: + return self.default(self.s) + + def join(self, s: Type, t: Type) -> Type: + return join_types(s, t) + + def default(self, typ: Type) -> Type: + if isinstance(typ, Instance): + return object_from_instance(typ) + elif isinstance(typ, UnboundType): + return AnyType() + elif isinstance(typ, Void) or isinstance(typ, ErrorType): + return ErrorType() + elif isinstance(typ, TupleType): + return self.default(typ.fallback) + elif isinstance(typ, FunctionLike): + return self.default(typ.fallback) + elif isinstance(typ, TypeVarType): + return self.default(typ.upper_bound) + else: + return AnyType() + + +def join_instances(t: Instance, s: Instance) -> Type: + """Calculate the join of two instance types. + + If allow_interfaces is True, also consider interface-type results for + non-interface types. + + Return ErrorType if the result is ambiguous. + """ + if t.type == s.type: + # Simplest case: join two types with the same base type (but + # potentially different arguments). + if is_subtype(t, s) or is_subtype(s, t): + # Compatible; combine type arguments. + args = [] # type: List[Type] + for i in range(len(t.args)): + args.append(join_types(t.args[i], s.args[i])) + return Instance(t.type, args) + else: + # Incompatible; return trivial result object. + return object_from_instance(t) + elif t.type.bases and is_subtype_ignoring_tvars(t, s): + return join_instances_via_supertype(t, s) + else: + # Now t is not a subtype of s, and t != s. Now s could be a subtype + # of t; alternatively, we need to find a common supertype. This works + # in of the both cases. + return join_instances_via_supertype(s, t) + + +def join_instances_via_supertype(t: Instance, s: Instance) -> Type: + # Give preference to joins via duck typing relationship, so that + # join(int, float) == float, for example. + if t.type._promote and is_subtype(t.type._promote, s): + return join_types(t.type._promote, s) + elif s.type._promote and is_subtype(s.type._promote, t): + return join_types(t, s.type._promote) + # Compute the "best" supertype of t when joined with s. + # The definition of "best" may evolve; for now it is the one with + # the longest MRO. Ties are broken by using the earlier base. + best = None # type: Type + for base in t.type.bases: + mapped = map_instance_to_supertype(t, base.type) + res = join_instances(mapped, s) + if best is None or is_better(res, best): + best = res + assert best is not None + return best + + +def is_better(t: Type, s: Type) -> bool: + # Given two possible results from join_instances_via_supertype(), + # indicate whether t is the better one. + if isinstance(t, Instance): + if not isinstance(s, Instance): + return True + # Use len(mro) as a proxy for the better choice. + if len(t.type.mro) > len(s.type.mro): + return True + return False + + +def is_similar_callables(t: CallableType, s: CallableType) -> bool: + """Return True if t and s are equivalent and have identical numbers of + arguments, default arguments and varargs. + """ + + return (len(t.arg_types) == len(s.arg_types) and t.min_args == s.min_args + and t.is_var_arg == s.is_var_arg and is_equivalent(t, s)) + + +def combine_similar_callables(t: CallableType, s: CallableType) -> CallableType: + arg_types = [] # type: List[Type] + for i in range(len(t.arg_types)): + arg_types.append(join_types(t.arg_types[i], s.arg_types[i])) + # TODO kinds and argument names + # The fallback type can be either 'function' or 'type'. The result should have 'type' as + # fallback only if both operands have it as 'type'. + if t.fallback.type.fullname() != 'builtins.type': + fallback = t.fallback + else: + fallback = s.fallback + return t.copy_modified(arg_types=arg_types, + ret_type=join_types(t.ret_type, s.ret_type), + fallback=fallback, + name=None) + + +def object_from_instance(instance: Instance) -> Instance: + """Construct the type 'builtins.object' from an instance type.""" + # Use the fact that 'object' is always the last class in the mro. + res = Instance(instance.type.mro[-1], []) + return res + + +def join_type_list(types: List[Type]) -> Type: + if not types: + # This is a little arbitrary but reasonable. Any empty tuple should be compatible + # with all variable length tuples, and this makes it possible. A better approach + # would be to use a special bottom type, which we do when strict Optional + # checking is enabled. + if experiments.STRICT_OPTIONAL: + return UninhabitedType() + else: + return NoneTyp() + joined = types[0] + for t in types[1:]: + joined = join_types(joined, t) + return joined diff --git a/mypy/meet.py b/mypy/meet.py index c4e3831e6f5f7..5c98385c7f1f2 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -1,285 +1,285 @@ -from typing import cast, List - -from mypy.join import is_similar_callables, combine_similar_callables -from mypy.types import ( - Type, AnyType, TypeVisitor, UnboundType, Void, ErrorType, NoneTyp, TypeVarType, - Instance, CallableType, TupleType, ErasedType, TypeList, UnionType, PartialType, - DeletedType, UninhabitedType, TypeType -) -from mypy.subtypes import is_subtype -from mypy.nodes import TypeInfo - -from mypy import experiments - -# TODO Describe this module. - - -def meet_types(s: Type, t: Type) -> Type: - """Return the greatest lower bound of two types.""" - if isinstance(s, ErasedType): - return s - if isinstance(s, AnyType): - return t - if isinstance(s, UnionType) and not isinstance(t, UnionType): - s, t = t, s - return t.accept(TypeMeetVisitor(s)) - - -def meet_simple(s: Type, t: Type, default_right: bool = True) -> Type: - if s == t: - return s - if isinstance(s, UnionType): - return UnionType.make_simplified_union([meet_types(x, t) for x in s.items]) - elif not is_overlapping_types(s, t, use_promotions=True): - if experiments.STRICT_OPTIONAL: - return UninhabitedType() - else: - return NoneTyp() - else: - if default_right: - return t - else: - return s - - -def is_overlapping_types(t: Type, s: Type, use_promotions: bool = False) -> bool: - """Can a value of type t be a value of type s, or vice versa? - - Note that this effectively checks against erased types, since type - variables are erased at runtime and the overlapping check is based - on runtime behavior. - - If use_promotions is True, also consider type promotions (int and - float would only be overlapping if it's True). - - This does not consider multiple inheritance. For example, A and B in - the following example are not considered overlapping, even though - via C they can be overlapping: - - class A: ... - class B: ... - class C(A, B): ... - - The rationale is that this case is usually very unlikely as multiple - inheritance is rare. Also, we can't reliably determine whether - multiple inheritance actually occurs somewhere in a program, due to - stub files hiding implementation details, dynamic loading etc. - - TODO: Don't consider tuples always overlapping. - TODO: Don't consider callables always overlapping. - TODO: Don't consider type variables with values always overlapping. - """ - # Since we are effectively working with the erased types, we only - # need to handle occurrences of TypeVarType at the top level. - if isinstance(t, TypeVarType): - t = t.erase_to_union_or_bound() - if isinstance(s, TypeVarType): - s = s.erase_to_union_or_bound() - if isinstance(t, Instance): - if isinstance(s, Instance): - # Consider two classes non-disjoint if one is included in the mro - # of another. - if use_promotions: - # Consider cases like int vs float to be overlapping where - # there is only a type promotion relationship but not proper - # subclassing. - if t.type._promote and is_overlapping_types(t.type._promote, s): - return True - if s.type._promote and is_overlapping_types(s.type._promote, t): - return True - return t.type in s.type.mro or s.type in t.type.mro - if isinstance(t, UnionType): - return any(is_overlapping_types(item, s) - for item in t.items) - if isinstance(s, UnionType): - return any(is_overlapping_types(t, item) - for item in s.items) - if isinstance(t, TypeType) and isinstance(s, TypeType): - # If both types are TypeType, compare their inner types. - return is_overlapping_types(t.item, s.item, use_promotions) - elif isinstance(t, TypeType) or isinstance(s, TypeType): - # If exactly only one of t or s is a TypeType, check if one of them - # is an `object` or a `type` and otherwise assume no overlap. - other = s if isinstance(t, TypeType) else t - if isinstance(other, Instance): - return other.type.fullname() in {'builtins.object', 'builtins.type'} - else: - return False - if experiments.STRICT_OPTIONAL: - if isinstance(t, NoneTyp) != isinstance(s, NoneTyp): - # NoneTyp does not overlap with other non-Union types under strict Optional checking - return False - # We conservatively assume that non-instance, non-union, and non-TypeType types can overlap - # any other types. - return True - - -def nearest_builtin_ancestor(type: TypeInfo) -> TypeInfo: - for base in type.mro: - if base.defn.is_builtinclass: - return base - else: - return None - - -class TypeMeetVisitor(TypeVisitor[Type]): - def __init__(self, s: Type) -> None: - self.s = s - - def visit_unbound_type(self, t: UnboundType) -> Type: - if isinstance(self.s, Void) or isinstance(self.s, ErrorType): - return ErrorType() - elif isinstance(self.s, NoneTyp): - if experiments.STRICT_OPTIONAL: - return AnyType() - else: - return self.s - elif isinstance(self.s, UninhabitedType): - return self.s - else: - return AnyType() - - def visit_error_type(self, t: ErrorType) -> Type: - return t - - def visit_type_list(self, t: TypeList) -> Type: - assert False, 'Not supported' - - def visit_any(self, t: AnyType) -> Type: - return self.s - - def visit_union_type(self, t: UnionType) -> Type: - if isinstance(self.s, UnionType): - meets = [] # type: List[Type] - for x in t.items: - for y in self.s.items: - meets.append(meet_types(x, y)) - else: - meets = [meet_types(x, self.s) - for x in t.items] - return UnionType.make_simplified_union(meets) - - def visit_void(self, t: Void) -> Type: - if isinstance(self.s, Void): - return t - else: - return ErrorType() - - def visit_none_type(self, t: NoneTyp) -> Type: - if experiments.STRICT_OPTIONAL: - if isinstance(self.s, NoneTyp) or (isinstance(self.s, Instance) and - self.s.type.fullname() == 'builtins.object'): - return t - else: - return UninhabitedType() - else: - if not isinstance(self.s, Void) and not isinstance(self.s, ErrorType): - return t - else: - return ErrorType() - - def visit_uninhabited_type(self, t: UninhabitedType) -> Type: - if not isinstance(self.s, Void) and not isinstance(self.s, ErrorType): - return t - else: - return ErrorType() - - def visit_deleted_type(self, t: DeletedType) -> Type: - if not isinstance(self.s, Void) and not isinstance(self.s, ErrorType): - if isinstance(self.s, NoneTyp): - if experiments.STRICT_OPTIONAL: - return t - else: - return self.s - elif isinstance(self.s, UninhabitedType): - return self.s - else: - return t - else: - return ErrorType() - - def visit_erased_type(self, t: ErasedType) -> Type: - return self.s - - def visit_type_var(self, t: TypeVarType) -> Type: - if isinstance(self.s, TypeVarType) and self.s.id == t.id: - return self.s - else: - return self.default(self.s) - - def visit_instance(self, t: Instance) -> Type: - if isinstance(self.s, Instance): - si = self.s - if t.type == si.type: - if is_subtype(t, self.s) or is_subtype(self.s, t): - # Combine type arguments. We could have used join below - # equivalently. - args = [] # type: List[Type] - for i in range(len(t.args)): - args.append(self.meet(t.args[i], si.args[i])) - return Instance(t.type, args) - else: - if experiments.STRICT_OPTIONAL: - return UninhabitedType() - else: - return NoneTyp() - else: - if is_subtype(t, self.s): - return t - elif is_subtype(self.s, t): - # See also above comment. - return self.s - else: - if experiments.STRICT_OPTIONAL: - return UninhabitedType() - else: - return NoneTyp() - elif isinstance(self.s, TypeType): - return meet_types(t, self.s) - else: - return self.default(self.s) - - def visit_callable_type(self, t: CallableType) -> Type: - if isinstance(self.s, CallableType) and is_similar_callables(t, self.s): - return combine_similar_callables(t, self.s) - else: - return self.default(self.s) - - def visit_tuple_type(self, t: TupleType) -> Type: - if isinstance(self.s, TupleType) and self.s.length() == t.length(): - items = [] # type: List[Type] - for i in range(t.length()): - items.append(self.meet(t.items[i], self.s.items[i])) - # TODO: What if the fallbacks are different? - return TupleType(items, t.fallback) - else: - return self.default(self.s) - - def visit_partial_type(self, t: PartialType) -> Type: - # We can't determine the meet of partial types. We should never get here. - assert False, 'Internal error' - - def visit_type_type(self, t: TypeType) -> Type: - if isinstance(self.s, TypeType): - typ = self.meet(t.item, self.s.item) - if not isinstance(typ, NoneTyp): - typ = TypeType(typ, line=t.line) - return typ - elif isinstance(self.s, Instance) and self.s.type.fullname() == 'builtins.type': - return t - else: - return self.default(self.s) - - def meet(self, s, t): - return meet_types(s, t) - - def default(self, typ): - if isinstance(typ, UnboundType): - return AnyType() - elif isinstance(typ, Void) or isinstance(typ, ErrorType): - return ErrorType() - else: - if experiments.STRICT_OPTIONAL: - return UninhabitedType() - else: - return NoneTyp() +from typing import cast, List + +from mypy.join import is_similar_callables, combine_similar_callables +from mypy.types import ( + Type, AnyType, TypeVisitor, UnboundType, Void, ErrorType, NoneTyp, TypeVarType, + Instance, CallableType, TupleType, ErasedType, TypeList, UnionType, PartialType, + DeletedType, UninhabitedType, TypeType +) +from mypy.subtypes import is_subtype +from mypy.nodes import TypeInfo + +from mypy import experiments + +# TODO Describe this module. + + +def meet_types(s: Type, t: Type) -> Type: + """Return the greatest lower bound of two types.""" + if isinstance(s, ErasedType): + return s + if isinstance(s, AnyType): + return t + if isinstance(s, UnionType) and not isinstance(t, UnionType): + s, t = t, s + return t.accept(TypeMeetVisitor(s)) + + +def meet_simple(s: Type, t: Type, default_right: bool = True) -> Type: + if s == t: + return s + if isinstance(s, UnionType): + return UnionType.make_simplified_union([meet_types(x, t) for x in s.items]) + elif not is_overlapping_types(s, t, use_promotions=True): + if experiments.STRICT_OPTIONAL: + return UninhabitedType() + else: + return NoneTyp() + else: + if default_right: + return t + else: + return s + + +def is_overlapping_types(t: Type, s: Type, use_promotions: bool = False) -> bool: + """Can a value of type t be a value of type s, or vice versa? + + Note that this effectively checks against erased types, since type + variables are erased at runtime and the overlapping check is based + on runtime behavior. + + If use_promotions is True, also consider type promotions (int and + float would only be overlapping if it's True). + + This does not consider multiple inheritance. For example, A and B in + the following example are not considered overlapping, even though + via C they can be overlapping: + + class A: ... + class B: ... + class C(A, B): ... + + The rationale is that this case is usually very unlikely as multiple + inheritance is rare. Also, we can't reliably determine whether + multiple inheritance actually occurs somewhere in a program, due to + stub files hiding implementation details, dynamic loading etc. + + TODO: Don't consider tuples always overlapping. + TODO: Don't consider callables always overlapping. + TODO: Don't consider type variables with values always overlapping. + """ + # Since we are effectively working with the erased types, we only + # need to handle occurrences of TypeVarType at the top level. + if isinstance(t, TypeVarType): + t = t.erase_to_union_or_bound() + if isinstance(s, TypeVarType): + s = s.erase_to_union_or_bound() + if isinstance(t, Instance): + if isinstance(s, Instance): + # Consider two classes non-disjoint if one is included in the mro + # of another. + if use_promotions: + # Consider cases like int vs float to be overlapping where + # there is only a type promotion relationship but not proper + # subclassing. + if t.type._promote and is_overlapping_types(t.type._promote, s): + return True + if s.type._promote and is_overlapping_types(s.type._promote, t): + return True + return t.type in s.type.mro or s.type in t.type.mro + if isinstance(t, UnionType): + return any(is_overlapping_types(item, s) + for item in t.items) + if isinstance(s, UnionType): + return any(is_overlapping_types(t, item) + for item in s.items) + if isinstance(t, TypeType) and isinstance(s, TypeType): + # If both types are TypeType, compare their inner types. + return is_overlapping_types(t.item, s.item, use_promotions) + elif isinstance(t, TypeType) or isinstance(s, TypeType): + # If exactly only one of t or s is a TypeType, check if one of them + # is an `object` or a `type` and otherwise assume no overlap. + other = s if isinstance(t, TypeType) else t + if isinstance(other, Instance): + return other.type.fullname() in {'builtins.object', 'builtins.type'} + else: + return False + if experiments.STRICT_OPTIONAL: + if isinstance(t, NoneTyp) != isinstance(s, NoneTyp): + # NoneTyp does not overlap with other non-Union types under strict Optional checking + return False + # We conservatively assume that non-instance, non-union, and non-TypeType types can overlap + # any other types. + return True + + +def nearest_builtin_ancestor(type: TypeInfo) -> TypeInfo: + for base in type.mro: + if base.defn.is_builtinclass: + return base + else: + return None + + +class TypeMeetVisitor(TypeVisitor[Type]): + def __init__(self, s: Type) -> None: + self.s = s + + def visit_unbound_type(self, t: UnboundType) -> Type: + if isinstance(self.s, Void) or isinstance(self.s, ErrorType): + return ErrorType() + elif isinstance(self.s, NoneTyp): + if experiments.STRICT_OPTIONAL: + return AnyType() + else: + return self.s + elif isinstance(self.s, UninhabitedType): + return self.s + else: + return AnyType() + + def visit_error_type(self, t: ErrorType) -> Type: + return t + + def visit_type_list(self, t: TypeList) -> Type: + assert False, 'Not supported' + + def visit_any(self, t: AnyType) -> Type: + return self.s + + def visit_union_type(self, t: UnionType) -> Type: + if isinstance(self.s, UnionType): + meets = [] # type: List[Type] + for x in t.items: + for y in self.s.items: + meets.append(meet_types(x, y)) + else: + meets = [meet_types(x, self.s) + for x in t.items] + return UnionType.make_simplified_union(meets) + + def visit_void(self, t: Void) -> Type: + if isinstance(self.s, Void): + return t + else: + return ErrorType() + + def visit_none_type(self, t: NoneTyp) -> Type: + if experiments.STRICT_OPTIONAL: + if isinstance(self.s, NoneTyp) or (isinstance(self.s, Instance) and + self.s.type.fullname() == 'builtins.object'): + return t + else: + return UninhabitedType() + else: + if not isinstance(self.s, Void) and not isinstance(self.s, ErrorType): + return t + else: + return ErrorType() + + def visit_uninhabited_type(self, t: UninhabitedType) -> Type: + if not isinstance(self.s, Void) and not isinstance(self.s, ErrorType): + return t + else: + return ErrorType() + + def visit_deleted_type(self, t: DeletedType) -> Type: + if not isinstance(self.s, Void) and not isinstance(self.s, ErrorType): + if isinstance(self.s, NoneTyp): + if experiments.STRICT_OPTIONAL: + return t + else: + return self.s + elif isinstance(self.s, UninhabitedType): + return self.s + else: + return t + else: + return ErrorType() + + def visit_erased_type(self, t: ErasedType) -> Type: + return self.s + + def visit_type_var(self, t: TypeVarType) -> Type: + if isinstance(self.s, TypeVarType) and self.s.id == t.id: + return self.s + else: + return self.default(self.s) + + def visit_instance(self, t: Instance) -> Type: + if isinstance(self.s, Instance): + si = self.s + if t.type == si.type: + if is_subtype(t, self.s) or is_subtype(self.s, t): + # Combine type arguments. We could have used join below + # equivalently. + args = [] # type: List[Type] + for i in range(len(t.args)): + args.append(self.meet(t.args[i], si.args[i])) + return Instance(t.type, args) + else: + if experiments.STRICT_OPTIONAL: + return UninhabitedType() + else: + return NoneTyp() + else: + if is_subtype(t, self.s): + return t + elif is_subtype(self.s, t): + # See also above comment. + return self.s + else: + if experiments.STRICT_OPTIONAL: + return UninhabitedType() + else: + return NoneTyp() + elif isinstance(self.s, TypeType): + return meet_types(t, self.s) + else: + return self.default(self.s) + + def visit_callable_type(self, t: CallableType) -> Type: + if isinstance(self.s, CallableType) and is_similar_callables(t, self.s): + return combine_similar_callables(t, self.s) + else: + return self.default(self.s) + + def visit_tuple_type(self, t: TupleType) -> Type: + if isinstance(self.s, TupleType) and self.s.length() == t.length(): + items = [] # type: List[Type] + for i in range(t.length()): + items.append(self.meet(t.items[i], self.s.items[i])) + # TODO: What if the fallbacks are different? + return TupleType(items, t.fallback) + else: + return self.default(self.s) + + def visit_partial_type(self, t: PartialType) -> Type: + # We can't determine the meet of partial types. We should never get here. + assert False, 'Internal error' + + def visit_type_type(self, t: TypeType) -> Type: + if isinstance(self.s, TypeType): + typ = self.meet(t.item, self.s.item) + if not isinstance(typ, NoneTyp): + typ = TypeType(typ, line=t.line) + return typ + elif isinstance(self.s, Instance) and self.s.type.fullname() == 'builtins.type': + return t + else: + return self.default(self.s) + + def meet(self, s, t): + return meet_types(s, t) + + def default(self, typ): + if isinstance(typ, UnboundType): + return AnyType() + elif isinstance(typ, Void) or isinstance(typ, ErrorType): + return ErrorType() + else: + if experiments.STRICT_OPTIONAL: + return UninhabitedType() + else: + return NoneTyp() diff --git a/mypy/messages.py b/mypy/messages.py index cc9b4761b703a..5d82c4845782a 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -1,920 +1,920 @@ -"""Facilities and constants for generating error messages during type checking. - -The type checker itself does not deal with message string literals to -improve code clarity and to simplify localization (in the future).""" - -import re -import difflib - -from typing import cast, List, Dict, Any, Sequence, Iterable, Tuple - -from mypy.errors import Errors -from mypy.types import ( - Type, CallableType, Instance, TypeVarType, TupleType, UnionType, Void, NoneTyp, AnyType, - Overloaded, FunctionLike, DeletedType, TypeType -) -from mypy.nodes import ( - TypeInfo, Context, MypyFile, op_methods, FuncDef, reverse_type_aliases, - ARG_STAR, ARG_STAR2 -) - - -# Constants that represent simple type checker error message, i.e. messages -# that do not have any parameters. - -NO_RETURN_VALUE_EXPECTED = 'No return value expected' -INCOMPATIBLE_RETURN_VALUE_TYPE = 'Incompatible return value type' -RETURN_VALUE_EXPECTED = 'Return value expected' -BOOLEAN_VALUE_EXPECTED = 'Boolean value expected' -BOOLEAN_EXPECTED_FOR_IF = 'Boolean value expected for if condition' -BOOLEAN_EXPECTED_FOR_WHILE = 'Boolean value expected for while condition' -BOOLEAN_EXPECTED_FOR_UNTIL = 'Boolean value expected for until condition' -BOOLEAN_EXPECTED_FOR_NOT = 'Boolean value expected for not operand' -INVALID_EXCEPTION = 'Exception must be derived from BaseException' -INVALID_EXCEPTION_TYPE = 'Exception type must be derived from BaseException' -INVALID_RETURN_TYPE_FOR_GENERATOR = \ - 'The return type of a generator function should be "Generator" or one of its supertypes' -INVALID_GENERATOR_RETURN_ITEM_TYPE = \ - 'The return type of a generator function must be None in its third type parameter in Python 2' -YIELD_VALUE_EXPECTED = 'Yield value expected' -INCOMPATIBLE_TYPES = 'Incompatible types' -INCOMPATIBLE_TYPES_IN_ASSIGNMENT = 'Incompatible types in assignment' -INCOMPATIBLE_REDEFINITION = 'Incompatible redefinition' -INCOMPATIBLE_TYPES_IN_YIELD = 'Incompatible types in yield' -INCOMPATIBLE_TYPES_IN_YIELD_FROM = 'Incompatible types in "yield from"' -INCOMPATIBLE_TYPES_IN_STR_INTERPOLATION = 'Incompatible types in string interpolation' -INIT_MUST_HAVE_NONE_RETURN_TYPE = 'The return type of "__init__" must be None' -GETTER_TYPE_INCOMPATIBLE_WITH_SETTER = \ - 'Type of getter incompatible with setter' -TUPLE_INDEX_MUST_BE_AN_INT_LITERAL = 'Tuple index must be an integer literal' -TUPLE_SLICE_MUST_BE_AN_INT_LITERAL = 'Tuple slice must be an integer literal' -TUPLE_INDEX_OUT_OF_RANGE = 'Tuple index out of range' -TYPE_CONSTANT_EXPECTED = 'Type "Constant" or initializer expected' -INCOMPATIBLE_PAIR_ITEM_TYPE = 'Incompatible Pair item type' -INVALID_TYPE_APPLICATION_TARGET_TYPE = 'Invalid type application target type' -INCOMPATIBLE_TUPLE_ITEM_TYPE = 'Incompatible tuple item type' -INCOMPATIBLE_KEY_TYPE = 'Incompatible dictionary key type' -INCOMPATIBLE_VALUE_TYPE = 'Incompatible dictionary value type' -NEED_ANNOTATION_FOR_VAR = 'Need type annotation for variable' -ITERABLE_EXPECTED = 'Iterable expected' -INCOMPATIBLE_TYPES_IN_FOR = 'Incompatible types in for statement' -INCOMPATIBLE_ARRAY_VAR_ARGS = 'Incompatible variable arguments in call' -INVALID_SLICE_INDEX = 'Slice index must be an integer or None' -CANNOT_INFER_LAMBDA_TYPE = 'Cannot infer type of lambda' -CANNOT_INFER_ITEM_TYPE = 'Cannot infer iterable item type' -CANNOT_ACCESS_INIT = 'Cannot access "__init__" directly' -CANNOT_ASSIGN_TO_METHOD = 'Cannot assign to a method' -CANNOT_ASSIGN_TO_TYPE = 'Cannot assign to a type' -INCONSISTENT_ABSTRACT_OVERLOAD = \ - 'Overloaded method has both abstract and non-abstract variants' -READ_ONLY_PROPERTY_OVERRIDES_READ_WRITE = \ - 'Read-only property cannot override read-write property' -INSTANCE_LAYOUT_CONFLICT = 'Instance layout conflict in multiple inheritance' -FORMAT_REQUIRES_MAPPING = 'Format requires a mapping' -GENERIC_TYPE_NOT_VALID_AS_EXPRESSION = \ - "Generic type is prohibited as a runtime expression (use a type alias or '# type:' comment)" -RETURN_TYPE_CANNOT_BE_CONTRAVARIANT = "Cannot use a contravariant type variable as return type" -FUNCTION_PARAMETER_CANNOT_BE_COVARIANT = "Cannot use a covariant type variable as a parameter" -INCOMPATIBLE_IMPORT_OF = "Incompatible import of" -FUNCTION_TYPE_EXPECTED = "Function is missing a type annotation" -RETURN_TYPE_EXPECTED = "Function is missing a return type annotation" -ARGUMENT_TYPE_EXPECTED = "Function is missing a type annotation for one or more arguments" -KEYWORD_ARGUMENT_REQUIRES_STR_KEY_TYPE = \ - 'Keyword argument only valid with "str" key type in call to "dict"' -ALL_MUST_BE_SEQ_STR = 'Type of __all__ must be {}, not {}' - - -class MessageBuilder: - """Helper class for reporting type checker error messages with parameters. - - The methods of this class need to be provided with the context within a - file; the errors member manages the wider context. - - IDEA: Support a 'verbose mode' that includes full information about types - in error messages and that may otherwise produce more detailed error - messages. - """ - - # Report errors using this instance. It knows about the current file and - # import context. - errors = None # type: Errors - - modules = None # type: Dict[str, MypyFile] - - # Number of times errors have been disabled. - disable_count = 0 - - # Hack to deduplicate error messages from union types - disable_type_names = 0 - - def __init__(self, errors: Errors, modules: Dict[str, MypyFile]) -> None: - self.errors = errors - self.modules = modules - self.disable_count = 0 - self.disable_type_names = 0 - - # - # Helpers - # - - def copy(self) -> 'MessageBuilder': - new = MessageBuilder(self.errors.copy(), self.modules) - new.disable_count = self.disable_count - new.disable_type_names = self.disable_type_names - return new - - def add_errors(self, messages: 'MessageBuilder') -> None: - """Add errors in messages to this builder.""" - if self.disable_count <= 0: - for info in messages.errors.error_info: - self.errors.add_error_info(info) - - def disable_errors(self) -> None: - self.disable_count += 1 - - def enable_errors(self) -> None: - self.disable_count -= 1 - - def is_errors(self) -> bool: - return self.errors.is_errors() - - def report(self, msg: str, context: Context, severity: str, file: str = None) -> None: - """Report an error or note (unless disabled).""" - if self.disable_count <= 0: - self.errors.report(context.get_line() if context else -1, - msg.strip(), severity=severity, file=file) - - def fail(self, msg: str, context: Context, file: str = None) -> None: - """Report an error message (unless disabled).""" - # import pdb; pdb.set_trace() - self.report(msg, context, 'error', file=file) - - def note(self, msg: str, context: Context, file: str = None) -> None: - """Report an error message (unless disabled).""" - self.report(msg, context, 'note', file=file) - - def format(self, typ: Type, verbosity: int = 0) -> str: - """Convert a type to a relatively short string that is suitable for error messages. - - Mostly behave like format_simple below, but never return an empty string. - """ - s = self.format_simple(typ, verbosity) - if s != '': - # If format_simple returns a non-trivial result, use that. - return s - elif isinstance(typ, FunctionLike): - func = typ - if func.is_type_obj(): - # The type of a type object type can be derived from the - # return type (this always works). - itype = cast(Instance, func.items()[0].ret_type) - result = self.format(itype) - if verbosity >= 1: - # In some contexts we want to be explicit about the distinction - # between type X and the type of type object X. - result += ' (type object)' - return result - elif isinstance(func, CallableType): - return_type = strip_quotes(self.format(func.ret_type)) - if func.is_ellipsis_args: - return 'Callable[..., {}]'.format(return_type) - arg_types = [strip_quotes(self.format(t)) for t in func.arg_types] - return 'Callable[[{}], {}]'.format(", ".join(arg_types), return_type) - else: - # Use a simple representation for function types; proper - # function types may result in long and difficult-to-read - # error messages. - return 'overloaded function' - else: - # Default case; we simply have to return something meaningful here. - return 'object' - - def format_simple(self, typ: Type, verbosity: int = 0) -> str: - """Convert simple types to string that is suitable for error messages. - - Return "" for complex types. Try to keep the length of the result - relatively short to avoid overly long error messages. - - Examples: - builtins.int -> 'int' - Any type -> 'Any' - None -> None - callable type -> "" (empty string) - """ - if isinstance(typ, Instance): - itype = typ - # Get the short name of the type. - if verbosity >= 2: - base_str = itype.type.fullname() - else: - base_str = itype.type.name() - if itype.args == []: - # No type arguments. Place the type name in quotes to avoid - # potential for confusion: otherwise, the type name could be - # interpreted as a normal word. - return '"{}"'.format(base_str) - elif itype.type.fullname() == 'builtins.tuple': - item_type = strip_quotes(self.format(itype.args[0])) - return 'Tuple[{}, ...]'.format(item_type) - elif itype.type.fullname() in reverse_type_aliases: - alias = reverse_type_aliases[itype.type.fullname()] - alias = alias.split('.')[-1] - items = [strip_quotes(self.format(arg)) for arg in itype.args] - return '{}[{}]'.format(alias, ', '.join(items)) - else: - # There are type arguments. Convert the arguments to strings - # (using format() instead of format_simple() to avoid empty - # strings). If the result is too long, replace arguments - # with [...]. - a = [] # type: List[str] - for arg in itype.args: - a.append(strip_quotes(self.format(arg))) - s = ', '.join(a) - if len((base_str + s)) < 25: - return '{}[{}]'.format(base_str, s) - else: - return '{}[...]'.format(base_str) - elif isinstance(typ, TypeVarType): - # This is similar to non-generic instance types. - return '"{}"'.format(typ.name) - elif isinstance(typ, TupleType): - # Prefer the name of the fallback class (if not tuple), as it's more informative. - if typ.fallback.type.fullname() != 'builtins.tuple': - return self.format_simple(typ.fallback) - items = [] - for t in typ.items: - items.append(strip_quotes(self.format(t))) - s = '"Tuple[{}]"'.format(', '.join(items)) - if len(s) < 40: - return s - else: - return 'tuple(length {})'.format(len(items)) - elif isinstance(typ, UnionType): - # Only print Unions as Optionals if the Optional wouldn't have to contain another Union - print_as_optional = (len(typ.items) - - sum(isinstance(t, NoneTyp) for t in typ.items) == 1) - if print_as_optional: - rest = [t for t in typ.items if not isinstance(t, NoneTyp)] - return '"Optional[{}]"'.format(strip_quotes(self.format(rest[0]))) - else: - items = [] - for t in typ.items: - items.append(strip_quotes(self.format(t))) - s = '"Union[{}]"'.format(', '.join(items)) - if len(s) < 40: - return s - else: - return 'union type ({} items)'.format(len(items)) - elif isinstance(typ, Void): - return 'None' - elif isinstance(typ, NoneTyp): - return 'None' - elif isinstance(typ, AnyType): - return '"Any"' - elif isinstance(typ, DeletedType): - return '' - elif isinstance(typ, TypeType): - return 'Type[{}]'.format( - strip_quotes(self.format_simple(typ.item, verbosity))) - elif typ is None: - raise RuntimeError('Type is None') - else: - # No simple representation for this type that would convey very - # useful information. No need to mention the type explicitly in a - # message. - return '' - - def format_distinctly(self, type1: Type, type2: Type) -> Tuple[str, str]: - """Jointly format a pair of types to distinct strings. - - Increase the verbosity of the type strings until they become distinct. - """ - verbosity = 0 - for verbosity in range(3): - str1 = self.format(type1, verbosity=verbosity) - str2 = self.format(type2, verbosity=verbosity) - if str1 != str2: - return (str1, str2) - return (str1, str2) - - # - # Specific operations - # - - # The following operations are for generating specific error messages. They - # get some information as arguments, and they build an error message based - # on them. - - def has_no_attr(self, typ: Type, member: str, context: Context) -> Type: - """Report a missing or non-accessible member. - - The type argument is the base type. If member corresponds to - an operator, use the corresponding operator name in the - messages. Return type Any. - """ - if (isinstance(typ, Instance) and - typ.type.has_readable_member(member)): - self.fail('Member "{}" is not assignable'.format(member), context) - elif isinstance(typ, Void): - self.check_void(typ, context) - elif member == '__contains__': - self.fail('Unsupported right operand type for in ({})'.format( - self.format(typ)), context) - elif member in op_methods.values(): - # Access to a binary operator member (e.g. _add). This case does - # not handle indexing operations. - for op, method in op_methods.items(): - if method == member: - self.unsupported_left_operand(op, typ, context) - break - elif member == '__neg__': - self.fail('Unsupported operand type for unary - ({})'.format( - self.format(typ)), context) - elif member == '__pos__': - self.fail('Unsupported operand type for unary + ({})'.format( - self.format(typ)), context) - elif member == '__invert__': - self.fail('Unsupported operand type for ~ ({})'.format( - self.format(typ)), context) - elif member == '__getitem__': - # Indexed get. - self.fail('Value of type {} is not indexable'.format( - self.format(typ)), context) - elif member == '__setitem__': - # Indexed set. - self.fail('Unsupported target for indexed assignment', context) - elif member == '__call__': - self.fail('{} not callable'.format(self.format(typ)), context) - else: - # The non-special case: a missing ordinary attribute. - if not self.disable_type_names: - failed = False - if isinstance(typ, Instance) and typ.type.names: - alternatives = set(typ.type.names.keys()) - matches = [m for m in COMMON_MISTAKES.get(member, []) if m in alternatives] - matches.extend(best_matches(member, alternatives)[:3]) - if matches: - self.fail('{} has no attribute "{}"; maybe {}?'.format( - self.format(typ), member, pretty_or(matches)), context) - failed = True - if not failed: - self.fail('{} has no attribute "{}"'.format(self.format(typ), - member), context) - else: - self.fail('Some element of union has no attribute "{}"'.format( - member), context) - return AnyType() - - def unsupported_operand_types(self, op: str, left_type: Any, - right_type: Any, context: Context) -> None: - """Report unsupported operand types for a binary operation. - - Types can be Type objects or strings. - """ - if isinstance(left_type, Void) or isinstance(right_type, Void): - self.check_void(left_type, context) - self.check_void(right_type, context) - return - left_str = '' - if isinstance(left_type, str): - left_str = left_type - else: - left_str = self.format(left_type) - - right_str = '' - if isinstance(right_type, str): - right_str = right_type - else: - right_str = self.format(right_type) - - if self.disable_type_names: - msg = 'Unsupported operand types for {} (likely involving Union)'.format(op) - else: - msg = 'Unsupported operand types for {} ({} and {})'.format( - op, left_str, right_str) - self.fail(msg, context) - - def unsupported_left_operand(self, op: str, typ: Type, - context: Context) -> None: - if not self.check_void(typ, context): - if self.disable_type_names: - msg = 'Unsupported left operand type for {} (some union)'.format(op) - else: - msg = 'Unsupported left operand type for {} ({})'.format( - op, self.format(typ)) - self.fail(msg, context) - - def type_expected_as_right_operand_of_is(self, context: Context) -> None: - self.fail('Type expected as right operand of "is"', context) - - def not_callable(self, typ: Type, context: Context) -> Type: - self.fail('{} not callable'.format(self.format(typ)), context) - return AnyType() - - def untyped_function_call(self, callee: CallableType, context: Context) -> Type: - name = callee.name if callee.name is not None else '(unknown)' - self.fail('call to untyped function {} in typed context'.format(name), context) - return AnyType() - - def incompatible_argument(self, n: int, m: int, callee: CallableType, arg_type: Type, - arg_kind: int, context: Context) -> None: - """Report an error about an incompatible argument type. - - The argument type is arg_type, argument number is n and the - callee type is 'callee'. If the callee represents a method - that corresponds to an operator, use the corresponding - operator name in the messages. - """ - target = '' - if callee.name: - name = callee.name - base = extract_type(name) - - for op, method in op_methods.items(): - for variant in method, '__r' + method[2:]: - if name.startswith('"{}" of'.format(variant)): - if op == 'in' or variant != method: - # Reversed order of base/argument. - self.unsupported_operand_types(op, arg_type, base, - context) - else: - self.unsupported_operand_types(op, base, arg_type, - context) - return - - if name.startswith('"__getitem__" of'): - self.invalid_index_type(arg_type, base, context) - return - - if name.startswith('"__setitem__" of'): - if n == 1: - self.invalid_index_type(arg_type, base, context) - else: - msg = '{} (expression has type {}, target has type {})' - arg_type_str, callee_type_str = self.format_distinctly(arg_type, - callee.arg_types[n - 1]) - self.fail(msg.format(INCOMPATIBLE_TYPES_IN_ASSIGNMENT, - arg_type_str, callee_type_str), - context) - return - - target = 'to {} '.format(name) - - msg = '' - if callee.name == '': - name = callee.name[1:-1] - n -= 1 - msg = '{} item {} has incompatible type {}'.format( - name[0].upper() + name[1:], n, self.format_simple(arg_type)) - elif callee.name == '': - msg = 'List comprehension has incompatible type List[{}]'.format( - strip_quotes(self.format(arg_type))) - elif callee.name == '': - msg = 'Set comprehension has incompatible type Set[{}]'.format( - strip_quotes(self.format(arg_type))) - elif callee.name == '': - msg = ('{} expression in dictionary comprehension has incompatible type {}; ' - 'expected type {}').format( - 'Key' if n == 1 else 'Value', - self.format(arg_type), - self.format(callee.arg_types[n - 1])) - elif callee.name == '': - msg = 'Generator has incompatible item type {}'.format( - self.format_simple(arg_type)) - else: - try: - expected_type = callee.arg_types[m - 1] - except IndexError: # Varargs callees - expected_type = callee.arg_types[-1] - arg_type_str, expected_type_str = self.format_distinctly(arg_type, expected_type) - #import pdb; pdb.set_trace() - if arg_kind == ARG_STAR: - arg_type_str = '*' + arg_type_str - elif arg_kind == ARG_STAR2: - arg_type_str = '**' + arg_type_str - msg = 'Argument {} {}has incompatible type {}; expected {}'.format( - n, target, arg_type_str, expected_type_str) - self.fail(msg, context) - - def invalid_index_type(self, index_type: Type, base_str: str, - context: Context) -> None: - self.fail('Invalid index type {} for {}'.format( - self.format(index_type), base_str), context) - - def too_few_arguments(self, callee: CallableType, context: Context, - argument_names: List[str]) -> None: - if (argument_names is not None and not all(k is None for k in argument_names) - and len(argument_names) >= 1): - diff = [k for k in callee.arg_names if k not in argument_names] - if len(diff) == 1: - msg = 'Missing positional argument' - else: - msg = 'Missing positional arguments' - if callee.name and diff: - msg += ' "{}" in call to {}'.format('", "'.join(diff), callee.name) - else: - msg = 'Too few arguments' - if callee.name: - msg += ' for {}'.format(callee.name) - self.fail(msg, context) - - def too_many_arguments(self, callee: CallableType, context: Context) -> None: - msg = 'Too many arguments' - if callee.name: - msg += ' for {}'.format(callee.name) - self.fail(msg, context) - - def too_many_positional_arguments(self, callee: CallableType, - context: Context) -> None: - msg = 'Too many positional arguments' - if callee.name: - msg += ' for {}'.format(callee.name) - self.fail(msg, context) - - def unexpected_keyword_argument(self, callee: CallableType, name: str, - context: Context) -> None: - msg = 'Unexpected keyword argument "{}"'.format(name) - if callee.name: - msg += ' for {}'.format(callee.name) - self.fail(msg, context) - if callee.definition: - fullname = callee.definition.fullname() - if fullname is not None and '.' in fullname: - module_name = fullname.rsplit('.', 1)[0] - path = self.modules[module_name].path - self.note('{} defined here'.format(callee.name), callee.definition, file=path) - - def duplicate_argument_value(self, callee: CallableType, index: int, - context: Context) -> None: - self.fail('{} gets multiple values for keyword argument "{}"'. - format(capitalize(callable_name(callee)), - callee.arg_names[index]), context) - - def does_not_return_value(self, void_type: Type, context: Context) -> None: - """Report an error about a void type in a non-void context. - - The first argument must be a void type. If the void type has a - source in it, report it in the error message. This allows - giving messages such as 'Foo does not return a value'. - """ - if (cast(Void, void_type)).source is None: - self.fail('Function does not return a value', context) - else: - self.fail('{} does not return a value'.format( - capitalize((cast(Void, void_type)).source)), context) - - def deleted_as_rvalue(self, typ: DeletedType, context: Context) -> None: - """Report an error about using an deleted type as an rvalue.""" - if typ.source is None: - s = "" - else: - s = " '{}'".format(typ.source) - self.fail('Trying to read deleted variable{}'.format(s), context) - - def deleted_as_lvalue(self, typ: DeletedType, context: Context) -> None: - """Report an error about using an deleted type as an lvalue. - - Currently, this only occurs when trying to assign to an - exception variable outside the local except: blocks. - """ - if typ.source is None: - s = "" - else: - s = " '{}'".format(typ.source) - self.fail('Assignment to variable{} outside except: block'.format(s), context) - - def no_variant_matches_arguments(self, overload: Overloaded, arg_types: List[Type], - context: Context) -> None: - if overload.name(): - self.fail('No overload variant of {} matches argument types {}' - .format(overload.name(), arg_types), context) - else: - self.fail('No overload variant matches argument types {}'.format(arg_types), context) - - def function_variants_overlap(self, n1: int, n2: int, - context: Context) -> None: - self.fail('Function signature variants {} and {} overlap'.format( - n1 + 1, n2 + 1), context) - - def invalid_cast(self, target_type: Type, source_type: Type, - context: Context) -> None: - if not self.check_void(source_type, context): - self.fail('Cannot cast from {} to {}'.format( - self.format(source_type), self.format(target_type)), context) - - def wrong_number_values_to_unpack(self, provided: int, expected: int, - context: Context) -> None: - if provided < expected: - if provided == 1: - self.fail('Need more than 1 value to unpack ({} expected)'.format(expected), - context) - else: - self.fail('Need more than {} values to unpack ({} expected)'.format( - provided, expected), context) - elif provided > expected: - self.fail('Too many values to unpack ({} expected, {} provided)'.format( - expected, provided), context) - - def type_not_iterable(self, type: Type, context: Context) -> None: - self.fail('\'{}\' object is not iterable'.format(type), context) - - def incompatible_operator_assignment(self, op: str, - context: Context) -> None: - self.fail('Result type of {} incompatible in assignment'.format(op), - context) - - def incompatible_value_count_in_assignment(self, lvalue_count: int, - rvalue_count: int, - context: Context) -> None: - if rvalue_count < lvalue_count: - self.fail('Need {} values to assign'.format(lvalue_count), context) - elif rvalue_count > lvalue_count: - self.fail('Too many values to assign', context) - - def type_incompatible_with_supertype(self, name: str, supertype: TypeInfo, - context: Context) -> None: - self.fail('Type of "{}" incompatible with supertype "{}"'.format( - name, supertype.name), context) - - def signature_incompatible_with_supertype( - self, name: str, name_in_super: str, supertype: str, - context: Context) -> None: - target = self.override_target(name, name_in_super, supertype) - self.fail('Signature of "{}" incompatible with {}'.format( - name, target), context) - - def argument_incompatible_with_supertype( - self, arg_num: int, name: str, name_in_supertype: str, - supertype: str, context: Context) -> None: - target = self.override_target(name, name_in_supertype, supertype) - self.fail('Argument {} of "{}" incompatible with {}' - .format(arg_num, name, target), context) - - def return_type_incompatible_with_supertype( - self, name: str, name_in_supertype: str, supertype: str, - context: Context) -> None: - target = self.override_target(name, name_in_supertype, supertype) - self.fail('Return type of "{}" incompatible with {}' - .format(name, target), context) - - def override_target(self, name: str, name_in_super: str, - supertype: str) -> str: - target = 'supertype "{}"'.format(supertype) - if name_in_super != name: - target = '"{}" of {}'.format(name_in_super, target) - return target - - def boolean_return_value_expected(self, method: str, - context: Context) -> None: - self.fail('Boolean return value expected for method "{}"'.format( - method), context) - - def incompatible_type_application(self, expected_arg_count: int, - actual_arg_count: int, - context: Context) -> None: - if expected_arg_count == 0: - self.fail('Type application targets a non-generic function', - context) - elif actual_arg_count > expected_arg_count: - self.fail('Type application has too many types ({} expected)' - .format(expected_arg_count), context) - else: - self.fail('Type application has too few types ({} expected)' - .format(expected_arg_count), context) - - def incompatible_array_item_type(self, typ: Type, index: int, - context: Context) -> None: - self.fail('Array item {} has incompatible type {}'.format( - index, self.format(typ)), context) - - def could_not_infer_type_arguments(self, callee_type: CallableType, n: int, - context: Context) -> None: - if callee_type.name and n > 0: - self.fail('Cannot infer type argument {} of {}'.format( - n, callee_type.name), context) - else: - self.fail('Cannot infer function type argument', context) - - def invalid_var_arg(self, typ: Type, context: Context) -> None: - self.fail('List or tuple expected as variable arguments', context) - - def invalid_keyword_var_arg(self, typ: Type, context: Context) -> None: - if isinstance(typ, Instance) and (typ.type.fullname() == 'builtins.dict'): - self.fail('Keywords must be strings', context) - else: - self.fail('Argument after ** must be a dictionary', - context) - - def incomplete_type_var_match(self, member: str, context: Context) -> None: - self.fail('"{}" has incomplete match to supertype type variable' - .format(member), context) - - def not_implemented(self, msg: str, context: Context) -> Type: - self.fail('Feature not implemented yet ({})'.format(msg), context) - return AnyType() - - def undefined_in_superclass(self, member: str, context: Context) -> None: - self.fail('"{}" undefined in superclass'.format(member), context) - - def check_void(self, typ: Type, context: Context) -> bool: - """If type is void, report an error such as '.. does not - return a value' and return True. Otherwise, return False. - """ - if isinstance(typ, Void): - self.does_not_return_value(typ, context) - return True - else: - return False - - def too_few_string_formatting_arguments(self, context: Context) -> None: - self.fail('Not enough arguments for format string', context) - - def too_many_string_formatting_arguments(self, context: Context) -> None: - self.fail('Not all arguments converted during string formatting', context) - - def incomplete_conversion_specifier_format(self, context: Context) -> None: - self.fail('Incomplete format', context) - - def unsupported_placeholder(self, placeholder: str, context: Context) -> None: - self.fail('Unsupported format character \'%s\'' % placeholder, context) - - def string_interpolation_with_star_and_key(self, context: Context) -> None: - self.fail('String interpolation contains both stars and mapping keys', context) - - def requires_int_or_char(self, context: Context) -> None: - self.fail('%c requires int or char', context) - - def key_not_in_mapping(self, key: str, context: Context) -> None: - self.fail('Key \'%s\' not found in mapping' % key, context) - - def string_interpolation_mixing_key_and_non_keys(self, context: Context) -> None: - self.fail('String interpolation mixes specifier with and without mapping keys', context) - - def cannot_determine_type(self, name: str, context: Context) -> None: - self.fail("Cannot determine type of '%s'" % name, context) - - def cannot_determine_type_in_base(self, name: str, base: str, context: Context) -> None: - self.fail("Cannot determine type of '%s' in base class '%s'" % (name, base), context) - - def invalid_method_type(self, sig: CallableType, context: Context) -> None: - self.fail('Invalid method type', context) - - def invalid_class_method_type(self, sig: CallableType, context: Context) -> None: - self.fail('Invalid class method type', context) - - def incompatible_conditional_function_def(self, defn: FuncDef) -> None: - self.fail('All conditional function variants must have identical ' - 'signatures', defn) - - def cannot_instantiate_abstract_class(self, class_name: str, - abstract_attributes: List[str], - context: Context) -> None: - attrs = format_string_list("'%s'" % a for a in abstract_attributes[:5]) - self.fail("Cannot instantiate abstract class '%s' with abstract " - "attribute%s %s" % (class_name, plural_s(abstract_attributes), - attrs), - context) - - def base_class_definitions_incompatible(self, name: str, base1: TypeInfo, - base2: TypeInfo, - context: Context) -> None: - self.fail('Definition of "{}" in base class "{}" is incompatible ' - 'with definition in base class "{}"'.format( - name, base1.name(), base2.name()), context) - - def cant_assign_to_method(self, context: Context) -> None: - self.fail(CANNOT_ASSIGN_TO_METHOD, context) - - def read_only_property(self, name: str, type: TypeInfo, - context: Context) -> None: - self.fail('Property "{}" defined in "{}" is read-only'.format( - name, type.name()), context) - - def incompatible_typevar_value(self, callee: CallableType, index: int, - type: Type, context: Context) -> None: - self.fail('Type argument {} of {} has incompatible value {}'.format( - index, callable_name(callee), self.format(type)), context) - - def overloaded_signatures_overlap(self, index1: int, index2: int, - context: Context) -> None: - self.fail('Overloaded function signatures {} and {} overlap with ' - 'incompatible return types'.format(index1, index2), context) - - def invalid_reverse_operator_signature(self, reverse: str, other: str, - context: Context) -> None: - self.fail('"Any" return type expected since argument to {} ' - 'does not support {}'.format(reverse, other), context) - - def reverse_operator_method_with_any_arg_must_return_any( - self, method: str, context: Context) -> None: - self.fail('"Any" return type expected since argument to {} ' - 'has type "Any"'.format(method), context) - - def operator_method_signatures_overlap( - self, reverse_class: str, reverse_method: str, forward_class: str, - forward_method: str, context: Context) -> None: - self.fail('Signatures of "{}" of "{}" and "{}" of "{}" ' - 'are unsafely overlapping'.format( - reverse_method, reverse_class, - forward_method, forward_class), - context) - - def forward_operator_not_callable( - self, forward_method: str, context: Context) -> None: - self.fail('Forward operator "{}" is not callable'.format( - forward_method), context) - - def signatures_incompatible(self, method: str, other_method: str, - context: Context) -> None: - self.fail('Signatures of "{}" and "{}" are incompatible'.format( - method, other_method), context) - - def yield_from_invalid_operand_type(self, expr: Type, context: Context) -> Type: - text = self.format(expr) if self.format(expr) != 'object' else expr - self.fail('"yield from" can\'t be applied to {}'.format(text), context) - return AnyType() - - def invalid_signature(self, func_type: Type, context: Context) -> None: - self.fail('Invalid signature "{}"'.format(func_type), context) - - def reveal_type(self, typ: Type, context: Context) -> None: - self.fail('Revealed type is \'{}\''.format(typ), context) - - def unsupported_type_type(self, item: Type, context: Context) -> None: - self.fail('Unsupported type Type[{}]'.format(self.format(item)), context) - - def redundant_cast(self, typ: Type, context: Context) -> None: - self.note('Redundant cast to {}'.format(self.format(typ)), context) - - -def capitalize(s: str) -> str: - """Capitalize the first character of a string.""" - if s == '': - return '' - else: - return s[0].upper() + s[1:] - - -def extract_type(name: str) -> str: - """If the argument is the name of a method (of form C.m), return - the type portion in quotes (e.g. "y"). Otherwise, return the string - unmodified. - """ - name = re.sub('^"[a-zA-Z0-9_]+" of ', '', name) - return name - - -def strip_quotes(s: str) -> str: - """Strip a double quote at the beginning and end of the string, if any.""" - s = re.sub('^"', '', s) - s = re.sub('"$', '', s) - return s - - -def plural_s(s: Sequence[Any]) -> str: - if len(s) > 1: - return 's' - else: - return '' - - -def format_string_list(s: Iterable[str]) -> str: - l = list(s) - assert len(l) > 0 - if len(l) == 1: - return l[0] - else: - return '%s and %s' % (', '.join(l[:-1]), l[-1]) - - -def callable_name(type: CallableType) -> str: - if type.name: - return type.name - else: - return 'function' - - -def temp_message_builder() -> MessageBuilder: - """Return a message builder usable for throwaway errors (which may not format properly).""" - return MessageBuilder(Errors(), {}) - - -# For hard-coding suggested missing member alternatives. -COMMON_MISTAKES = { - 'add': ('append', 'extend'), -} # type: Dict[str, Sequence[str]] - - -def best_matches(current: str, options: Iterable[str]) -> List[str]: - ratios = {v: difflib.SequenceMatcher(a=current, b=v).ratio() for v in options} - return sorted((o for o in options if ratios[o] > 0.75), - reverse=True, key=lambda v: (ratios[v], v)) - - -def pretty_or(args: List[str]) -> str: - quoted = ['"' + a + '"' for a in args] - if len(quoted) == 1: - return quoted[0] - if len(quoted) == 2: - return "{} or {}".format(quoted[0], quoted[1]) - return ", ".join(quoted[:-1]) + ", or " + quoted[-1] +"""Facilities and constants for generating error messages during type checking. + +The type checker itself does not deal with message string literals to +improve code clarity and to simplify localization (in the future).""" + +import re +import difflib + +from typing import cast, List, Dict, Any, Sequence, Iterable, Tuple + +from mypy.errors import Errors +from mypy.types import ( + Type, CallableType, Instance, TypeVarType, TupleType, UnionType, Void, NoneTyp, AnyType, + Overloaded, FunctionLike, DeletedType, TypeType +) +from mypy.nodes import ( + TypeInfo, Context, MypyFile, op_methods, FuncDef, reverse_type_aliases, + ARG_STAR, ARG_STAR2 +) + + +# Constants that represent simple type checker error message, i.e. messages +# that do not have any parameters. + +NO_RETURN_VALUE_EXPECTED = 'No return value expected' +INCOMPATIBLE_RETURN_VALUE_TYPE = 'Incompatible return value type' +RETURN_VALUE_EXPECTED = 'Return value expected' +BOOLEAN_VALUE_EXPECTED = 'Boolean value expected' +BOOLEAN_EXPECTED_FOR_IF = 'Boolean value expected for if condition' +BOOLEAN_EXPECTED_FOR_WHILE = 'Boolean value expected for while condition' +BOOLEAN_EXPECTED_FOR_UNTIL = 'Boolean value expected for until condition' +BOOLEAN_EXPECTED_FOR_NOT = 'Boolean value expected for not operand' +INVALID_EXCEPTION = 'Exception must be derived from BaseException' +INVALID_EXCEPTION_TYPE = 'Exception type must be derived from BaseException' +INVALID_RETURN_TYPE_FOR_GENERATOR = \ + 'The return type of a generator function should be "Generator" or one of its supertypes' +INVALID_GENERATOR_RETURN_ITEM_TYPE = \ + 'The return type of a generator function must be None in its third type parameter in Python 2' +YIELD_VALUE_EXPECTED = 'Yield value expected' +INCOMPATIBLE_TYPES = 'Incompatible types' +INCOMPATIBLE_TYPES_IN_ASSIGNMENT = 'Incompatible types in assignment' +INCOMPATIBLE_REDEFINITION = 'Incompatible redefinition' +INCOMPATIBLE_TYPES_IN_YIELD = 'Incompatible types in yield' +INCOMPATIBLE_TYPES_IN_YIELD_FROM = 'Incompatible types in "yield from"' +INCOMPATIBLE_TYPES_IN_STR_INTERPOLATION = 'Incompatible types in string interpolation' +INIT_MUST_HAVE_NONE_RETURN_TYPE = 'The return type of "__init__" must be None' +GETTER_TYPE_INCOMPATIBLE_WITH_SETTER = \ + 'Type of getter incompatible with setter' +TUPLE_INDEX_MUST_BE_AN_INT_LITERAL = 'Tuple index must be an integer literal' +TUPLE_SLICE_MUST_BE_AN_INT_LITERAL = 'Tuple slice must be an integer literal' +TUPLE_INDEX_OUT_OF_RANGE = 'Tuple index out of range' +TYPE_CONSTANT_EXPECTED = 'Type "Constant" or initializer expected' +INCOMPATIBLE_PAIR_ITEM_TYPE = 'Incompatible Pair item type' +INVALID_TYPE_APPLICATION_TARGET_TYPE = 'Invalid type application target type' +INCOMPATIBLE_TUPLE_ITEM_TYPE = 'Incompatible tuple item type' +INCOMPATIBLE_KEY_TYPE = 'Incompatible dictionary key type' +INCOMPATIBLE_VALUE_TYPE = 'Incompatible dictionary value type' +NEED_ANNOTATION_FOR_VAR = 'Need type annotation for variable' +ITERABLE_EXPECTED = 'Iterable expected' +INCOMPATIBLE_TYPES_IN_FOR = 'Incompatible types in for statement' +INCOMPATIBLE_ARRAY_VAR_ARGS = 'Incompatible variable arguments in call' +INVALID_SLICE_INDEX = 'Slice index must be an integer or None' +CANNOT_INFER_LAMBDA_TYPE = 'Cannot infer type of lambda' +CANNOT_INFER_ITEM_TYPE = 'Cannot infer iterable item type' +CANNOT_ACCESS_INIT = 'Cannot access "__init__" directly' +CANNOT_ASSIGN_TO_METHOD = 'Cannot assign to a method' +CANNOT_ASSIGN_TO_TYPE = 'Cannot assign to a type' +INCONSISTENT_ABSTRACT_OVERLOAD = \ + 'Overloaded method has both abstract and non-abstract variants' +READ_ONLY_PROPERTY_OVERRIDES_READ_WRITE = \ + 'Read-only property cannot override read-write property' +INSTANCE_LAYOUT_CONFLICT = 'Instance layout conflict in multiple inheritance' +FORMAT_REQUIRES_MAPPING = 'Format requires a mapping' +GENERIC_TYPE_NOT_VALID_AS_EXPRESSION = \ + "Generic type is prohibited as a runtime expression (use a type alias or '# type:' comment)" +RETURN_TYPE_CANNOT_BE_CONTRAVARIANT = "Cannot use a contravariant type variable as return type" +FUNCTION_PARAMETER_CANNOT_BE_COVARIANT = "Cannot use a covariant type variable as a parameter" +INCOMPATIBLE_IMPORT_OF = "Incompatible import of" +FUNCTION_TYPE_EXPECTED = "Function is missing a type annotation" +RETURN_TYPE_EXPECTED = "Function is missing a return type annotation" +ARGUMENT_TYPE_EXPECTED = "Function is missing a type annotation for one or more arguments" +KEYWORD_ARGUMENT_REQUIRES_STR_KEY_TYPE = \ + 'Keyword argument only valid with "str" key type in call to "dict"' +ALL_MUST_BE_SEQ_STR = 'Type of __all__ must be {}, not {}' + + +class MessageBuilder: + """Helper class for reporting type checker error messages with parameters. + + The methods of this class need to be provided with the context within a + file; the errors member manages the wider context. + + IDEA: Support a 'verbose mode' that includes full information about types + in error messages and that may otherwise produce more detailed error + messages. + """ + + # Report errors using this instance. It knows about the current file and + # import context. + errors = None # type: Errors + + modules = None # type: Dict[str, MypyFile] + + # Number of times errors have been disabled. + disable_count = 0 + + # Hack to deduplicate error messages from union types + disable_type_names = 0 + + def __init__(self, errors: Errors, modules: Dict[str, MypyFile]) -> None: + self.errors = errors + self.modules = modules + self.disable_count = 0 + self.disable_type_names = 0 + + # + # Helpers + # + + def copy(self) -> 'MessageBuilder': + new = MessageBuilder(self.errors.copy(), self.modules) + new.disable_count = self.disable_count + new.disable_type_names = self.disable_type_names + return new + + def add_errors(self, messages: 'MessageBuilder') -> None: + """Add errors in messages to this builder.""" + if self.disable_count <= 0: + for info in messages.errors.error_info: + self.errors.add_error_info(info) + + def disable_errors(self) -> None: + self.disable_count += 1 + + def enable_errors(self) -> None: + self.disable_count -= 1 + + def is_errors(self) -> bool: + return self.errors.is_errors() + + def report(self, msg: str, context: Context, severity: str, file: str = None) -> None: + """Report an error or note (unless disabled).""" + if self.disable_count <= 0: + self.errors.report(context.get_line() if context else -1, + msg.strip(), severity=severity, file=file) + + def fail(self, msg: str, context: Context, file: str = None) -> None: + """Report an error message (unless disabled).""" + # import pdb; pdb.set_trace() + self.report(msg, context, 'error', file=file) + + def note(self, msg: str, context: Context, file: str = None) -> None: + """Report an error message (unless disabled).""" + self.report(msg, context, 'note', file=file) + + def format(self, typ: Type, verbosity: int = 0) -> str: + """Convert a type to a relatively short string that is suitable for error messages. + + Mostly behave like format_simple below, but never return an empty string. + """ + s = self.format_simple(typ, verbosity) + if s != '': + # If format_simple returns a non-trivial result, use that. + return s + elif isinstance(typ, FunctionLike): + func = typ + if func.is_type_obj(): + # The type of a type object type can be derived from the + # return type (this always works). + itype = cast(Instance, func.items()[0].ret_type) + result = self.format(itype) + if verbosity >= 1: + # In some contexts we want to be explicit about the distinction + # between type X and the type of type object X. + result += ' (type object)' + return result + elif isinstance(func, CallableType): + return_type = strip_quotes(self.format(func.ret_type)) + if func.is_ellipsis_args: + return 'Callable[..., {}]'.format(return_type) + arg_types = [strip_quotes(self.format(t)) for t in func.arg_types] + return 'Callable[[{}], {}]'.format(", ".join(arg_types), return_type) + else: + # Use a simple representation for function types; proper + # function types may result in long and difficult-to-read + # error messages. + return 'overloaded function' + else: + # Default case; we simply have to return something meaningful here. + return 'object' + + def format_simple(self, typ: Type, verbosity: int = 0) -> str: + """Convert simple types to string that is suitable for error messages. + + Return "" for complex types. Try to keep the length of the result + relatively short to avoid overly long error messages. + + Examples: + builtins.int -> 'int' + Any type -> 'Any' + None -> None + callable type -> "" (empty string) + """ + if isinstance(typ, Instance): + itype = typ + # Get the short name of the type. + if verbosity >= 2: + base_str = itype.type.fullname() + else: + base_str = itype.type.name() + if itype.args == []: + # No type arguments. Place the type name in quotes to avoid + # potential for confusion: otherwise, the type name could be + # interpreted as a normal word. + return '"{}"'.format(base_str) + elif itype.type.fullname() == 'builtins.tuple': + item_type = strip_quotes(self.format(itype.args[0])) + return 'Tuple[{}, ...]'.format(item_type) + elif itype.type.fullname() in reverse_type_aliases: + alias = reverse_type_aliases[itype.type.fullname()] + alias = alias.split('.')[-1] + items = [strip_quotes(self.format(arg)) for arg in itype.args] + return '{}[{}]'.format(alias, ', '.join(items)) + else: + # There are type arguments. Convert the arguments to strings + # (using format() instead of format_simple() to avoid empty + # strings). If the result is too long, replace arguments + # with [...]. + a = [] # type: List[str] + for arg in itype.args: + a.append(strip_quotes(self.format(arg))) + s = ', '.join(a) + if len((base_str + s)) < 25: + return '{}[{}]'.format(base_str, s) + else: + return '{}[...]'.format(base_str) + elif isinstance(typ, TypeVarType): + # This is similar to non-generic instance types. + return '"{}"'.format(typ.name) + elif isinstance(typ, TupleType): + # Prefer the name of the fallback class (if not tuple), as it's more informative. + if typ.fallback.type.fullname() != 'builtins.tuple': + return self.format_simple(typ.fallback) + items = [] + for t in typ.items: + items.append(strip_quotes(self.format(t))) + s = '"Tuple[{}]"'.format(', '.join(items)) + if len(s) < 40: + return s + else: + return 'tuple(length {})'.format(len(items)) + elif isinstance(typ, UnionType): + # Only print Unions as Optionals if the Optional wouldn't have to contain another Union + print_as_optional = (len(typ.items) - + sum(isinstance(t, NoneTyp) for t in typ.items) == 1) + if print_as_optional: + rest = [t for t in typ.items if not isinstance(t, NoneTyp)] + return '"Optional[{}]"'.format(strip_quotes(self.format(rest[0]))) + else: + items = [] + for t in typ.items: + items.append(strip_quotes(self.format(t))) + s = '"Union[{}]"'.format(', '.join(items)) + if len(s) < 40: + return s + else: + return 'union type ({} items)'.format(len(items)) + elif isinstance(typ, Void): + return 'None' + elif isinstance(typ, NoneTyp): + return 'None' + elif isinstance(typ, AnyType): + return '"Any"' + elif isinstance(typ, DeletedType): + return '' + elif isinstance(typ, TypeType): + return 'Type[{}]'.format( + strip_quotes(self.format_simple(typ.item, verbosity))) + elif typ is None: + raise RuntimeError('Type is None') + else: + # No simple representation for this type that would convey very + # useful information. No need to mention the type explicitly in a + # message. + return '' + + def format_distinctly(self, type1: Type, type2: Type) -> Tuple[str, str]: + """Jointly format a pair of types to distinct strings. + + Increase the verbosity of the type strings until they become distinct. + """ + verbosity = 0 + for verbosity in range(3): + str1 = self.format(type1, verbosity=verbosity) + str2 = self.format(type2, verbosity=verbosity) + if str1 != str2: + return (str1, str2) + return (str1, str2) + + # + # Specific operations + # + + # The following operations are for generating specific error messages. They + # get some information as arguments, and they build an error message based + # on them. + + def has_no_attr(self, typ: Type, member: str, context: Context) -> Type: + """Report a missing or non-accessible member. + + The type argument is the base type. If member corresponds to + an operator, use the corresponding operator name in the + messages. Return type Any. + """ + if (isinstance(typ, Instance) and + typ.type.has_readable_member(member)): + self.fail('Member "{}" is not assignable'.format(member), context) + elif isinstance(typ, Void): + self.check_void(typ, context) + elif member == '__contains__': + self.fail('Unsupported right operand type for in ({})'.format( + self.format(typ)), context) + elif member in op_methods.values(): + # Access to a binary operator member (e.g. _add). This case does + # not handle indexing operations. + for op, method in op_methods.items(): + if method == member: + self.unsupported_left_operand(op, typ, context) + break + elif member == '__neg__': + self.fail('Unsupported operand type for unary - ({})'.format( + self.format(typ)), context) + elif member == '__pos__': + self.fail('Unsupported operand type for unary + ({})'.format( + self.format(typ)), context) + elif member == '__invert__': + self.fail('Unsupported operand type for ~ ({})'.format( + self.format(typ)), context) + elif member == '__getitem__': + # Indexed get. + self.fail('Value of type {} is not indexable'.format( + self.format(typ)), context) + elif member == '__setitem__': + # Indexed set. + self.fail('Unsupported target for indexed assignment', context) + elif member == '__call__': + self.fail('{} not callable'.format(self.format(typ)), context) + else: + # The non-special case: a missing ordinary attribute. + if not self.disable_type_names: + failed = False + if isinstance(typ, Instance) and typ.type.names: + alternatives = set(typ.type.names.keys()) + matches = [m for m in COMMON_MISTAKES.get(member, []) if m in alternatives] + matches.extend(best_matches(member, alternatives)[:3]) + if matches: + self.fail('{} has no attribute "{}"; maybe {}?'.format( + self.format(typ), member, pretty_or(matches)), context) + failed = True + if not failed: + self.fail('{} has no attribute "{}"'.format(self.format(typ), + member), context) + else: + self.fail('Some element of union has no attribute "{}"'.format( + member), context) + return AnyType() + + def unsupported_operand_types(self, op: str, left_type: Any, + right_type: Any, context: Context) -> None: + """Report unsupported operand types for a binary operation. + + Types can be Type objects or strings. + """ + if isinstance(left_type, Void) or isinstance(right_type, Void): + self.check_void(left_type, context) + self.check_void(right_type, context) + return + left_str = '' + if isinstance(left_type, str): + left_str = left_type + else: + left_str = self.format(left_type) + + right_str = '' + if isinstance(right_type, str): + right_str = right_type + else: + right_str = self.format(right_type) + + if self.disable_type_names: + msg = 'Unsupported operand types for {} (likely involving Union)'.format(op) + else: + msg = 'Unsupported operand types for {} ({} and {})'.format( + op, left_str, right_str) + self.fail(msg, context) + + def unsupported_left_operand(self, op: str, typ: Type, + context: Context) -> None: + if not self.check_void(typ, context): + if self.disable_type_names: + msg = 'Unsupported left operand type for {} (some union)'.format(op) + else: + msg = 'Unsupported left operand type for {} ({})'.format( + op, self.format(typ)) + self.fail(msg, context) + + def type_expected_as_right_operand_of_is(self, context: Context) -> None: + self.fail('Type expected as right operand of "is"', context) + + def not_callable(self, typ: Type, context: Context) -> Type: + self.fail('{} not callable'.format(self.format(typ)), context) + return AnyType() + + def untyped_function_call(self, callee: CallableType, context: Context) -> Type: + name = callee.name if callee.name is not None else '(unknown)' + self.fail('call to untyped function {} in typed context'.format(name), context) + return AnyType() + + def incompatible_argument(self, n: int, m: int, callee: CallableType, arg_type: Type, + arg_kind: int, context: Context) -> None: + """Report an error about an incompatible argument type. + + The argument type is arg_type, argument number is n and the + callee type is 'callee'. If the callee represents a method + that corresponds to an operator, use the corresponding + operator name in the messages. + """ + target = '' + if callee.name: + name = callee.name + base = extract_type(name) + + for op, method in op_methods.items(): + for variant in method, '__r' + method[2:]: + if name.startswith('"{}" of'.format(variant)): + if op == 'in' or variant != method: + # Reversed order of base/argument. + self.unsupported_operand_types(op, arg_type, base, + context) + else: + self.unsupported_operand_types(op, base, arg_type, + context) + return + + if name.startswith('"__getitem__" of'): + self.invalid_index_type(arg_type, base, context) + return + + if name.startswith('"__setitem__" of'): + if n == 1: + self.invalid_index_type(arg_type, base, context) + else: + msg = '{} (expression has type {}, target has type {})' + arg_type_str, callee_type_str = self.format_distinctly(arg_type, + callee.arg_types[n - 1]) + self.fail(msg.format(INCOMPATIBLE_TYPES_IN_ASSIGNMENT, + arg_type_str, callee_type_str), + context) + return + + target = 'to {} '.format(name) + + msg = '' + if callee.name == '': + name = callee.name[1:-1] + n -= 1 + msg = '{} item {} has incompatible type {}'.format( + name[0].upper() + name[1:], n, self.format_simple(arg_type)) + elif callee.name == '': + msg = 'List comprehension has incompatible type List[{}]'.format( + strip_quotes(self.format(arg_type))) + elif callee.name == '': + msg = 'Set comprehension has incompatible type Set[{}]'.format( + strip_quotes(self.format(arg_type))) + elif callee.name == '': + msg = ('{} expression in dictionary comprehension has incompatible type {}; ' + 'expected type {}').format( + 'Key' if n == 1 else 'Value', + self.format(arg_type), + self.format(callee.arg_types[n - 1])) + elif callee.name == '': + msg = 'Generator has incompatible item type {}'.format( + self.format_simple(arg_type)) + else: + try: + expected_type = callee.arg_types[m - 1] + except IndexError: # Varargs callees + expected_type = callee.arg_types[-1] + arg_type_str, expected_type_str = self.format_distinctly(arg_type, expected_type) + #import pdb; pdb.set_trace() + if arg_kind == ARG_STAR: + arg_type_str = '*' + arg_type_str + elif arg_kind == ARG_STAR2: + arg_type_str = '**' + arg_type_str + msg = 'Argument {} {}has incompatible type {}; expected {}'.format( + n, target, arg_type_str, expected_type_str) + self.fail(msg, context) + + def invalid_index_type(self, index_type: Type, base_str: str, + context: Context) -> None: + self.fail('Invalid index type {} for {}'.format( + self.format(index_type), base_str), context) + + def too_few_arguments(self, callee: CallableType, context: Context, + argument_names: List[str]) -> None: + if (argument_names is not None and not all(k is None for k in argument_names) + and len(argument_names) >= 1): + diff = [k for k in callee.arg_names if k not in argument_names] + if len(diff) == 1: + msg = 'Missing positional argument' + else: + msg = 'Missing positional arguments' + if callee.name and diff: + msg += ' "{}" in call to {}'.format('", "'.join(diff), callee.name) + else: + msg = 'Too few arguments' + if callee.name: + msg += ' for {}'.format(callee.name) + self.fail(msg, context) + + def too_many_arguments(self, callee: CallableType, context: Context) -> None: + msg = 'Too many arguments' + if callee.name: + msg += ' for {}'.format(callee.name) + self.fail(msg, context) + + def too_many_positional_arguments(self, callee: CallableType, + context: Context) -> None: + msg = 'Too many positional arguments' + if callee.name: + msg += ' for {}'.format(callee.name) + self.fail(msg, context) + + def unexpected_keyword_argument(self, callee: CallableType, name: str, + context: Context) -> None: + msg = 'Unexpected keyword argument "{}"'.format(name) + if callee.name: + msg += ' for {}'.format(callee.name) + self.fail(msg, context) + if callee.definition: + fullname = callee.definition.fullname() + if fullname is not None and '.' in fullname: + module_name = fullname.rsplit('.', 1)[0] + path = self.modules[module_name].path + self.note('{} defined here'.format(callee.name), callee.definition, file=path) + + def duplicate_argument_value(self, callee: CallableType, index: int, + context: Context) -> None: + self.fail('{} gets multiple values for keyword argument "{}"'. + format(capitalize(callable_name(callee)), + callee.arg_names[index]), context) + + def does_not_return_value(self, void_type: Type, context: Context) -> None: + """Report an error about a void type in a non-void context. + + The first argument must be a void type. If the void type has a + source in it, report it in the error message. This allows + giving messages such as 'Foo does not return a value'. + """ + if (cast(Void, void_type)).source is None: + self.fail('Function does not return a value', context) + else: + self.fail('{} does not return a value'.format( + capitalize((cast(Void, void_type)).source)), context) + + def deleted_as_rvalue(self, typ: DeletedType, context: Context) -> None: + """Report an error about using an deleted type as an rvalue.""" + if typ.source is None: + s = "" + else: + s = " '{}'".format(typ.source) + self.fail('Trying to read deleted variable{}'.format(s), context) + + def deleted_as_lvalue(self, typ: DeletedType, context: Context) -> None: + """Report an error about using an deleted type as an lvalue. + + Currently, this only occurs when trying to assign to an + exception variable outside the local except: blocks. + """ + if typ.source is None: + s = "" + else: + s = " '{}'".format(typ.source) + self.fail('Assignment to variable{} outside except: block'.format(s), context) + + def no_variant_matches_arguments(self, overload: Overloaded, arg_types: List[Type], + context: Context) -> None: + if overload.name(): + self.fail('No overload variant of {} matches argument types {}' + .format(overload.name(), arg_types), context) + else: + self.fail('No overload variant matches argument types {}'.format(arg_types), context) + + def function_variants_overlap(self, n1: int, n2: int, + context: Context) -> None: + self.fail('Function signature variants {} and {} overlap'.format( + n1 + 1, n2 + 1), context) + + def invalid_cast(self, target_type: Type, source_type: Type, + context: Context) -> None: + if not self.check_void(source_type, context): + self.fail('Cannot cast from {} to {}'.format( + self.format(source_type), self.format(target_type)), context) + + def wrong_number_values_to_unpack(self, provided: int, expected: int, + context: Context) -> None: + if provided < expected: + if provided == 1: + self.fail('Need more than 1 value to unpack ({} expected)'.format(expected), + context) + else: + self.fail('Need more than {} values to unpack ({} expected)'.format( + provided, expected), context) + elif provided > expected: + self.fail('Too many values to unpack ({} expected, {} provided)'.format( + expected, provided), context) + + def type_not_iterable(self, type: Type, context: Context) -> None: + self.fail('\'{}\' object is not iterable'.format(type), context) + + def incompatible_operator_assignment(self, op: str, + context: Context) -> None: + self.fail('Result type of {} incompatible in assignment'.format(op), + context) + + def incompatible_value_count_in_assignment(self, lvalue_count: int, + rvalue_count: int, + context: Context) -> None: + if rvalue_count < lvalue_count: + self.fail('Need {} values to assign'.format(lvalue_count), context) + elif rvalue_count > lvalue_count: + self.fail('Too many values to assign', context) + + def type_incompatible_with_supertype(self, name: str, supertype: TypeInfo, + context: Context) -> None: + self.fail('Type of "{}" incompatible with supertype "{}"'.format( + name, supertype.name), context) + + def signature_incompatible_with_supertype( + self, name: str, name_in_super: str, supertype: str, + context: Context) -> None: + target = self.override_target(name, name_in_super, supertype) + self.fail('Signature of "{}" incompatible with {}'.format( + name, target), context) + + def argument_incompatible_with_supertype( + self, arg_num: int, name: str, name_in_supertype: str, + supertype: str, context: Context) -> None: + target = self.override_target(name, name_in_supertype, supertype) + self.fail('Argument {} of "{}" incompatible with {}' + .format(arg_num, name, target), context) + + def return_type_incompatible_with_supertype( + self, name: str, name_in_supertype: str, supertype: str, + context: Context) -> None: + target = self.override_target(name, name_in_supertype, supertype) + self.fail('Return type of "{}" incompatible with {}' + .format(name, target), context) + + def override_target(self, name: str, name_in_super: str, + supertype: str) -> str: + target = 'supertype "{}"'.format(supertype) + if name_in_super != name: + target = '"{}" of {}'.format(name_in_super, target) + return target + + def boolean_return_value_expected(self, method: str, + context: Context) -> None: + self.fail('Boolean return value expected for method "{}"'.format( + method), context) + + def incompatible_type_application(self, expected_arg_count: int, + actual_arg_count: int, + context: Context) -> None: + if expected_arg_count == 0: + self.fail('Type application targets a non-generic function', + context) + elif actual_arg_count > expected_arg_count: + self.fail('Type application has too many types ({} expected)' + .format(expected_arg_count), context) + else: + self.fail('Type application has too few types ({} expected)' + .format(expected_arg_count), context) + + def incompatible_array_item_type(self, typ: Type, index: int, + context: Context) -> None: + self.fail('Array item {} has incompatible type {}'.format( + index, self.format(typ)), context) + + def could_not_infer_type_arguments(self, callee_type: CallableType, n: int, + context: Context) -> None: + if callee_type.name and n > 0: + self.fail('Cannot infer type argument {} of {}'.format( + n, callee_type.name), context) + else: + self.fail('Cannot infer function type argument', context) + + def invalid_var_arg(self, typ: Type, context: Context) -> None: + self.fail('List or tuple expected as variable arguments', context) + + def invalid_keyword_var_arg(self, typ: Type, context: Context) -> None: + if isinstance(typ, Instance) and (typ.type.fullname() == 'builtins.dict'): + self.fail('Keywords must be strings', context) + else: + self.fail('Argument after ** must be a dictionary', + context) + + def incomplete_type_var_match(self, member: str, context: Context) -> None: + self.fail('"{}" has incomplete match to supertype type variable' + .format(member), context) + + def not_implemented(self, msg: str, context: Context) -> Type: + self.fail('Feature not implemented yet ({})'.format(msg), context) + return AnyType() + + def undefined_in_superclass(self, member: str, context: Context) -> None: + self.fail('"{}" undefined in superclass'.format(member), context) + + def check_void(self, typ: Type, context: Context) -> bool: + """If type is void, report an error such as '.. does not + return a value' and return True. Otherwise, return False. + """ + if isinstance(typ, Void): + self.does_not_return_value(typ, context) + return True + else: + return False + + def too_few_string_formatting_arguments(self, context: Context) -> None: + self.fail('Not enough arguments for format string', context) + + def too_many_string_formatting_arguments(self, context: Context) -> None: + self.fail('Not all arguments converted during string formatting', context) + + def incomplete_conversion_specifier_format(self, context: Context) -> None: + self.fail('Incomplete format', context) + + def unsupported_placeholder(self, placeholder: str, context: Context) -> None: + self.fail('Unsupported format character \'%s\'' % placeholder, context) + + def string_interpolation_with_star_and_key(self, context: Context) -> None: + self.fail('String interpolation contains both stars and mapping keys', context) + + def requires_int_or_char(self, context: Context) -> None: + self.fail('%c requires int or char', context) + + def key_not_in_mapping(self, key: str, context: Context) -> None: + self.fail('Key \'%s\' not found in mapping' % key, context) + + def string_interpolation_mixing_key_and_non_keys(self, context: Context) -> None: + self.fail('String interpolation mixes specifier with and without mapping keys', context) + + def cannot_determine_type(self, name: str, context: Context) -> None: + self.fail("Cannot determine type of '%s'" % name, context) + + def cannot_determine_type_in_base(self, name: str, base: str, context: Context) -> None: + self.fail("Cannot determine type of '%s' in base class '%s'" % (name, base), context) + + def invalid_method_type(self, sig: CallableType, context: Context) -> None: + self.fail('Invalid method type', context) + + def invalid_class_method_type(self, sig: CallableType, context: Context) -> None: + self.fail('Invalid class method type', context) + + def incompatible_conditional_function_def(self, defn: FuncDef) -> None: + self.fail('All conditional function variants must have identical ' + 'signatures', defn) + + def cannot_instantiate_abstract_class(self, class_name: str, + abstract_attributes: List[str], + context: Context) -> None: + attrs = format_string_list("'%s'" % a for a in abstract_attributes[:5]) + self.fail("Cannot instantiate abstract class '%s' with abstract " + "attribute%s %s" % (class_name, plural_s(abstract_attributes), + attrs), + context) + + def base_class_definitions_incompatible(self, name: str, base1: TypeInfo, + base2: TypeInfo, + context: Context) -> None: + self.fail('Definition of "{}" in base class "{}" is incompatible ' + 'with definition in base class "{}"'.format( + name, base1.name(), base2.name()), context) + + def cant_assign_to_method(self, context: Context) -> None: + self.fail(CANNOT_ASSIGN_TO_METHOD, context) + + def read_only_property(self, name: str, type: TypeInfo, + context: Context) -> None: + self.fail('Property "{}" defined in "{}" is read-only'.format( + name, type.name()), context) + + def incompatible_typevar_value(self, callee: CallableType, index: int, + type: Type, context: Context) -> None: + self.fail('Type argument {} of {} has incompatible value {}'.format( + index, callable_name(callee), self.format(type)), context) + + def overloaded_signatures_overlap(self, index1: int, index2: int, + context: Context) -> None: + self.fail('Overloaded function signatures {} and {} overlap with ' + 'incompatible return types'.format(index1, index2), context) + + def invalid_reverse_operator_signature(self, reverse: str, other: str, + context: Context) -> None: + self.fail('"Any" return type expected since argument to {} ' + 'does not support {}'.format(reverse, other), context) + + def reverse_operator_method_with_any_arg_must_return_any( + self, method: str, context: Context) -> None: + self.fail('"Any" return type expected since argument to {} ' + 'has type "Any"'.format(method), context) + + def operator_method_signatures_overlap( + self, reverse_class: str, reverse_method: str, forward_class: str, + forward_method: str, context: Context) -> None: + self.fail('Signatures of "{}" of "{}" and "{}" of "{}" ' + 'are unsafely overlapping'.format( + reverse_method, reverse_class, + forward_method, forward_class), + context) + + def forward_operator_not_callable( + self, forward_method: str, context: Context) -> None: + self.fail('Forward operator "{}" is not callable'.format( + forward_method), context) + + def signatures_incompatible(self, method: str, other_method: str, + context: Context) -> None: + self.fail('Signatures of "{}" and "{}" are incompatible'.format( + method, other_method), context) + + def yield_from_invalid_operand_type(self, expr: Type, context: Context) -> Type: + text = self.format(expr) if self.format(expr) != 'object' else expr + self.fail('"yield from" can\'t be applied to {}'.format(text), context) + return AnyType() + + def invalid_signature(self, func_type: Type, context: Context) -> None: + self.fail('Invalid signature "{}"'.format(func_type), context) + + def reveal_type(self, typ: Type, context: Context) -> None: + self.fail('Revealed type is \'{}\''.format(typ), context) + + def unsupported_type_type(self, item: Type, context: Context) -> None: + self.fail('Unsupported type Type[{}]'.format(self.format(item)), context) + + def redundant_cast(self, typ: Type, context: Context) -> None: + self.note('Redundant cast to {}'.format(self.format(typ)), context) + + +def capitalize(s: str) -> str: + """Capitalize the first character of a string.""" + if s == '': + return '' + else: + return s[0].upper() + s[1:] + + +def extract_type(name: str) -> str: + """If the argument is the name of a method (of form C.m), return + the type portion in quotes (e.g. "y"). Otherwise, return the string + unmodified. + """ + name = re.sub('^"[a-zA-Z0-9_]+" of ', '', name) + return name + + +def strip_quotes(s: str) -> str: + """Strip a double quote at the beginning and end of the string, if any.""" + s = re.sub('^"', '', s) + s = re.sub('"$', '', s) + return s + + +def plural_s(s: Sequence[Any]) -> str: + if len(s) > 1: + return 's' + else: + return '' + + +def format_string_list(s: Iterable[str]) -> str: + l = list(s) + assert len(l) > 0 + if len(l) == 1: + return l[0] + else: + return '%s and %s' % (', '.join(l[:-1]), l[-1]) + + +def callable_name(type: CallableType) -> str: + if type.name: + return type.name + else: + return 'function' + + +def temp_message_builder() -> MessageBuilder: + """Return a message builder usable for throwaway errors (which may not format properly).""" + return MessageBuilder(Errors(), {}) + + +# For hard-coding suggested missing member alternatives. +COMMON_MISTAKES = { + 'add': ('append', 'extend'), +} # type: Dict[str, Sequence[str]] + + +def best_matches(current: str, options: Iterable[str]) -> List[str]: + ratios = {v: difflib.SequenceMatcher(a=current, b=v).ratio() for v in options} + return sorted((o for o in options if ratios[o] > 0.75), + reverse=True, key=lambda v: (ratios[v], v)) + + +def pretty_or(args: List[str]) -> str: + quoted = ['"' + a + '"' for a in args] + if len(quoted) == 1: + return quoted[0] + if len(quoted) == 2: + return "{} or {}".format(quoted[0], quoted[1]) + return ", ".join(quoted[:-1]) + ", or " + quoted[-1] diff --git a/mypy/nodes.py b/mypy/nodes.py index 02f86b1537b93..60d7b5d18944f 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1,2242 +1,2242 @@ -"""Abstract syntax tree node classes (i.e. parse tree).""" - -import os -import re -from abc import abstractmethod, ABCMeta - -from typing import ( - Any, TypeVar, List, Tuple, cast, Set, Dict, Union, Optional -) - -from mypy.lex import Token -import mypy.strconv -from mypy.visitor import NodeVisitor -from mypy.util import dump_tagged, short_type - - -class Context: - """Base type for objects that are valid as error message locations.""" - @abstractmethod - def get_line(self) -> int: pass - - -if False: - # break import cycle only needed for mypy - import mypy.types - - -T = TypeVar('T') - -JsonDict = Dict[str, Any] - - -# Symbol table node kinds -# -# TODO rename to use more descriptive names - -LDEF = 0 # type: int -GDEF = 1 # type: int -MDEF = 2 # type: int -MODULE_REF = 3 # type: int -# Type variable declared using TypeVar(...) has kind UNBOUND_TVAR. It's not -# valid as a type. A type variable is valid as a type (kind BOUND_TVAR) within -# (1) a generic class that uses the type variable as a type argument or -# (2) a generic function that refers to the type variable in its signature. -UNBOUND_TVAR = 4 # type: int -BOUND_TVAR = 5 # type: int -TYPE_ALIAS = 6 # type: int -# Placeholder for a name imported via 'from ... import'. Second phase of -# semantic will replace this the actual imported reference. This is -# needed so that we can detect whether a name has been imported during -# XXX what? -UNBOUND_IMPORTED = 7 # type: int - - -LITERAL_YES = 2 -LITERAL_TYPE = 1 -LITERAL_NO = 0 - -# Hard coded name of Enum baseclass. -ENUM_BASECLASS = "enum.Enum" - -node_kinds = { - LDEF: 'Ldef', - GDEF: 'Gdef', - MDEF: 'Mdef', - MODULE_REF: 'ModuleRef', - UNBOUND_TVAR: 'UnboundTvar', - BOUND_TVAR: 'Tvar', - TYPE_ALIAS: 'TypeAlias', - UNBOUND_IMPORTED: 'UnboundImported', -} -inverse_node_kinds = {_kind: _name for _name, _kind in node_kinds.items()} - - -implicit_module_attrs = {'__name__': '__builtins__.str', - '__doc__': '__builtins__.str', - '__file__': '__builtins__.str', - '__package__': '__builtins__.str'} - - -type_aliases = { - 'typing.List': '__builtins__.list', - 'typing.Dict': '__builtins__.dict', - 'typing.Set': '__builtins__.set', -} - -reverse_type_aliases = dict((name.replace('__builtins__', 'builtins'), alias) - for alias, name in type_aliases.items()) # type: Dict[str, str] - - -class Node(Context): - """Common base class for all non-type parse tree nodes.""" - - line = -1 - - literal = LITERAL_NO - literal_hash = None # type: Any - - def __str__(self) -> str: - ans = self.accept(mypy.strconv.StrConv()) - if ans is None: - return repr(self) - return ans - - def set_line(self, target: Union[Token, 'Node', int]) -> 'Node': - if isinstance(target, int): - self.line = target - else: - self.line = target.line - return self - - def get_line(self) -> int: - # TODO this should be just 'line' - return self.line - - def accept(self, visitor: NodeVisitor[T]) -> T: - raise RuntimeError('Not implemented') - - -# These are placeholders for a future refactoring; see #1783. -# For now they serve as (unchecked) documentation of what various -# fields of Node subtypes are expected to contain. -Statement = Node -Expression = Node - - -class SymbolNode(Node): - # Nodes that can be stored in a symbol table. - - # TODO do not use methods for these - - @abstractmethod - def name(self) -> str: pass - - @abstractmethod - def fullname(self) -> str: pass - - # NOTE: Can't use @abstractmethod, since many subclasses of Node - # don't implement serialize(). - def serialize(self) -> Any: - raise NotImplementedError('Cannot serialize {} instance'.format(self.__class__.__name__)) - - @classmethod - def deserialize(cls, data: JsonDict) -> 'SymbolNode': - classname = data['.class'] - glo = globals() - if classname in glo: - cl = glo[classname] - if issubclass(cl, cls) and 'deserialize' in cl.__dict__: - return cl.deserialize(data) - raise NotImplementedError('unexpected .class {}'.format(classname)) - - -class MypyFile(SymbolNode, Statement): - """The abstract syntax tree of a single source file.""" - - # Module name ('__main__' for initial file) - _name = None # type: str - # Fully qualified module name - _fullname = None # type: str - # Path to the file (None if not known) - path = '' - # Top-level definitions and statements - defs = None # type: List[Statement] - # Is there a UTF-8 BOM at the start? - is_bom = False - names = None # type: SymbolTable - # All import nodes within the file (also ones within functions etc.) - imports = None # type: List[ImportBase] - # Lines to ignore when checking - ignored_lines = None # type: Set[int] - # Is this file represented by a stub file (.pyi)? - is_stub = False - # Do weak typing globally in the file? - weak_opts = None # type: Set[str] - - def __init__(self, - defs: List[Statement], - imports: List['ImportBase'], - is_bom: bool = False, - ignored_lines: Set[int] = None, - weak_opts: Set[str] = None) -> None: - self.defs = defs - self.line = 1 # Dummy line number - self.imports = imports - self.is_bom = is_bom - self.weak_opts = weak_opts - if ignored_lines: - self.ignored_lines = ignored_lines - else: - self.ignored_lines = set() - - def name(self) -> str: - return self._name - - def fullname(self) -> str: - return self._fullname - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_mypy_file(self) - - def is_package_init_file(self) -> bool: - return not (self.path is None) and len(self.path) != 0 \ - and os.path.basename(self.path).startswith('__init__.') - - def serialize(self) -> JsonDict: - return {'.class': 'MypyFile', - '_name': self._name, - '_fullname': self._fullname, - 'names': self.names.serialize(self._fullname), - 'is_stub': self.is_stub, - 'path': self.path, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'MypyFile': - assert data['.class'] == 'MypyFile', data - tree = MypyFile([], []) - tree._name = data['_name'] - tree._fullname = data['_fullname'] - tree.names = SymbolTable.deserialize(data['names']) - tree.is_stub = data['is_stub'] - tree.path = data['path'] - return tree - - -class ImportBase(Statement): - """Base class for all import statements.""" - is_unreachable = False - is_top_level = False # Set by semanal.FirstPass - # If an import replaces existing definitions, we construct dummy assignment - # statements that assign the imported names to the names in the current scope, - # for type checking purposes. Example: - # - # x = 1 - # from m import x <-- add assignment representing "x = m.x" - assignments = None # type: List[AssignmentStmt] - - def __init__(self) -> None: - self.assignments = [] - - -class Import(ImportBase): - """import m [as n]""" - - ids = None # type: List[Tuple[str, Optional[str]]] # (module id, as id) - - def __init__(self, ids: List[Tuple[str, Optional[str]]]) -> None: - super().__init__() - self.ids = ids - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_import(self) - - -class ImportFrom(ImportBase): - """from m import x [as y], ...""" - - names = None # type: List[Tuple[str, Optional[str]]] # Tuples (name, as name) - - def __init__(self, id: str, relative: int, names: List[Tuple[str, Optional[str]]]) -> None: - super().__init__() - self.id = id - self.names = names - self.relative = relative - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_import_from(self) - - -class ImportAll(ImportBase): - """from m import *""" - - def __init__(self, id: str, relative: int) -> None: - super().__init__() - self.id = id - self.relative = relative - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_import_all(self) - - -class FuncBase(SymbolNode): - """Abstract base class for function-like nodes""" - - # Type signature. This is usually CallableType or Overloaded, but it can be something else for - # decorated functions/ - type = None # type: mypy.types.Type - # If method, reference to TypeInfo - info = None # type: TypeInfo - is_property = False - _fullname = None # type: str # Name with module prefix - - @abstractmethod - def name(self) -> str: pass - - def fullname(self) -> str: - return self._fullname - - def is_method(self) -> bool: - return bool(self.info) - - -class OverloadedFuncDef(FuncBase, Statement): - """A logical node representing all the variants of an overloaded function. - - This node has no explicit representation in the source program. - Overloaded variants must be consecutive in the source file. - """ - - items = None # type: List[Decorator] - - def __init__(self, items: List['Decorator']) -> None: - self.items = items - self.set_line(items[0].line) - - def name(self) -> str: - return self.items[0].func.name() - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_overloaded_func_def(self) - - def serialize(self) -> JsonDict: - return {'.class': 'OverloadedFuncDef', - 'items': [i.serialize() for i in self.items], - 'type': None if self.type is None else self.type.serialize(), - 'fullname': self._fullname, - 'is_property': self.is_property, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'OverloadedFuncDef': - assert data['.class'] == 'OverloadedFuncDef' - res = OverloadedFuncDef([Decorator.deserialize(d) for d in data['items']]) - if data.get('type') is not None: - res.type = mypy.types.Type.deserialize(data['type']) - res._fullname = data['fullname'] - res.is_property = data['is_property'] - # NOTE: res.info will be set in the fixup phase. - return res - - -class Argument(Node): - """A single argument in a FuncItem.""" - - def __init__(self, variable: 'Var', type_annotation: 'Optional[mypy.types.Type]', - initializer: Optional[Expression], kind: int, - initialization_statement: Optional['AssignmentStmt'] = None) -> None: - self.variable = variable - - self.type_annotation = type_annotation - self.initializer = initializer - - self.initialization_statement = initialization_statement - if not self.initialization_statement: - self.initialization_statement = self._initialization_statement() - - self.kind = kind - - def _initialization_statement(self) -> Optional['AssignmentStmt']: - """Convert the initializer into an assignment statement. - """ - if not self.initializer: - return None - - rvalue = self.initializer - lvalue = NameExpr(self.variable.name()) - assign = AssignmentStmt([lvalue], rvalue) - return assign - - def set_line(self, target: Union[Token, Node, int]) -> Node: - super().set_line(target) - - if self.initializer: - self.initializer.set_line(self.line) - - self.variable.set_line(self.line) - - if self.initialization_statement: - self.initialization_statement.set_line(self.line) - self.initialization_statement.lvalues[0].set_line(self.line) - - def serialize(self) -> JsonDict: - data = {'.class': 'Argument', - 'kind': self.kind, - 'variable': self.variable.serialize(), - 'type_annotation': (None if self.type_annotation is None - else self.type_annotation.serialize()), - } # type: JsonDict - # TODO: initializer? - return data - - @classmethod - def deserialize(cls, data: JsonDict) -> 'Argument': - assert data['.class'] == 'Argument' - return Argument(Var.deserialize(data['variable']), - (None if data.get('type_annotation') is None - else mypy.types.Type.deserialize(data['type_annotation'])), - None, # TODO: initializer? - kind=data['kind']) - - -class FuncItem(FuncBase): - arguments = [] # type: List[Argument] - # Minimum number of arguments - min_args = 0 - # Maximum number of positional arguments, -1 if no explicit limit (*args not included) - max_pos = 0 - body = None # type: Block - # Is this an overload variant of function with more than one overload variant? - is_overload = False - is_generator = False # Contains a yield statement? - is_static = False # Uses @staticmethod? - is_class = False # Uses @classmethod? - # Variants of function with type variables with values expanded - expanded = None # type: List[FuncItem] - - def __init__(self, arguments: List[Argument], body: 'Block', - typ: 'mypy.types.FunctionLike' = None) -> None: - self.arguments = arguments - arg_kinds = [arg.kind for arg in self.arguments] - self.max_pos = arg_kinds.count(ARG_POS) + arg_kinds.count(ARG_OPT) - self.body = body - self.type = typ - self.expanded = [] - - self.min_args = 0 - for i in range(len(self.arguments)): - if self.arguments[i] is None and i < self.max_fixed_argc(): - self.min_args = i + 1 - - def max_fixed_argc(self) -> int: - return self.max_pos - - def set_line(self, target: Union[Token, Node, int]) -> Node: - super().set_line(target) - for arg in self.arguments: - arg.set_line(self.line) - return self - - def is_dynamic(self): - return self.type is None - - -class FuncDef(FuncItem, Statement): - """Function definition. - - This is a non-lambda function defined using 'def'. - """ - - is_decorated = False - is_conditional = False # Defined conditionally (within block)? - is_abstract = False - is_property = False - original_def = None # type: Union[None, FuncDef, Var] # Original conditional definition - - def __init__(self, - name: str, # Function name - arguments: List[Argument], - body: 'Block', - typ: 'mypy.types.FunctionLike' = None) -> None: - super().__init__(arguments, body, typ) - self._name = name - - def name(self) -> str: - return self._name - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_func_def(self) - - def is_constructor(self) -> bool: - return self.info is not None and self._name == '__init__' - - def serialize(self) -> JsonDict: - return {'.class': 'FuncDef', - 'name': self._name, - 'fullname': self._fullname, - 'arguments': [a.serialize() for a in self.arguments], - 'type': None if self.type is None else self.type.serialize(), - 'is_property': self.is_property, - 'is_overload': self.is_overload, - 'is_generator': self.is_generator, - 'is_static': self.is_static, - 'is_class': self.is_class, - 'is_decorated': self.is_decorated, - 'is_conditional': self.is_conditional, - 'is_abstract': self.is_abstract, - # TODO: Do we need expanded, original_def? - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'FuncDef': - assert data['.class'] == 'FuncDef' - body = Block([]) - ret = FuncDef(data['name'], - [Argument.deserialize(a) for a in data['arguments']], - body, - (None if data['type'] is None - else mypy.types.FunctionLike.deserialize(data['type']))) - ret._fullname = data['fullname'] - ret.is_property = data['is_property'] - ret.is_overload = data['is_overload'] - ret.is_generator = data['is_generator'] - ret.is_static = data['is_static'] - ret.is_class = data['is_class'] - ret.is_decorated = data['is_decorated'] - ret.is_conditional = data['is_conditional'] - ret.is_abstract = data['is_abstract'] - # NOTE: ret.info is set in the fixup phase. - return ret - - -class Decorator(SymbolNode, Statement): - """A decorated function. - - A single Decorator object can include any number of function decorators. - """ - - func = None # type: FuncDef # Decorated function - decorators = None # type: List[Expression] # Decorators, at least one # XXX Not true - var = None # type: Var # Represents the decorated function obj - is_overload = False - - def __init__(self, func: FuncDef, decorators: List[Expression], - var: 'Var') -> None: - self.func = func - self.decorators = decorators - self.var = var - self.is_overload = False - - def name(self) -> str: - return self.func.name() - - def fullname(self) -> str: - return self.func.fullname() - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_decorator(self) - - def serialize(self) -> JsonDict: - return {'.class': 'Decorator', - 'func': self.func.serialize(), - 'var': self.var.serialize(), - 'is_overload': self.is_overload, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'Decorator': - assert data['.class'] == 'Decorator' - dec = Decorator(FuncDef.deserialize(data['func']), - [], - Var.deserialize(data['var'])) - dec.is_overload = data['is_overload'] - return dec - - -class Var(SymbolNode, Statement): - """A variable. - - It can refer to global/local variable or a data attribute. - """ - - _name = None # type: str # Name without module prefix - _fullname = None # type: str # Name with module prefix - info = None # type: TypeInfo # Defining class (for member variables) - type = None # type: mypy.types.Type # Declared or inferred type, or None - # Is this the first argument to an ordinary method (usually "self")? - is_self = False - is_ready = False # If inferred, is the inferred type available? - # Is this initialized explicitly to a non-None value in class body? - is_initialized_in_class = False - is_staticmethod = False - is_classmethod = False - is_property = False - is_settable_property = False - - def __init__(self, name: str, type: 'mypy.types.Type' = None) -> None: - self._name = name - self.type = type - self.is_self = False - self.is_ready = True - self.is_initialized_in_class = False - - def name(self) -> str: - return self._name - - def fullname(self) -> str: - return self._fullname - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_var(self) - - def serialize(self) -> JsonDict: - # TODO: Leave default values out? - # NOTE: Sometimes self.is_ready is False here, but we don't care. - data = {'.class': 'Var', - 'name': self._name, - 'fullname': self._fullname, - 'type': None if self.type is None else self.type.serialize(), - 'is_self': self.is_self, - 'is_initialized_in_class': self.is_initialized_in_class, - 'is_staticmethod': self.is_staticmethod, - 'is_classmethod': self.is_classmethod, - 'is_property': self.is_property, - 'is_settable_property': self.is_settable_property, - } # type: JsonDict - return data - - @classmethod - def deserialize(cls, data: JsonDict) -> 'Var': - assert data['.class'] == 'Var' - name = data['name'] - type = None if data['type'] is None else mypy.types.Type.deserialize(data['type']) - v = Var(name, type) - v._fullname = data['fullname'] - v.is_self = data['is_self'] - v.is_initialized_in_class = data['is_initialized_in_class'] - v.is_staticmethod = data['is_staticmethod'] - v.is_classmethod = data['is_classmethod'] - v.is_property = data['is_property'] - v.is_settable_property = data['is_settable_property'] - return v - - -class ClassDef(Statement): - """Class definition""" - - name = None # type: str # Name of the class without module prefix - fullname = None # type: str # Fully qualified name of the class - defs = None # type: Block - type_vars = None # type: List[mypy.types.TypeVarDef] - # Base class expressions (not semantically analyzed -- can be arbitrary expressions) - base_type_exprs = None # type: List[Expression] - info = None # type: TypeInfo # Related TypeInfo - metaclass = '' - decorators = None # type: List[Expression] - # Built-in/extension class? (single implementation inheritance only) - is_builtinclass = False - - def __init__(self, - name: str, - defs: 'Block', - type_vars: List['mypy.types.TypeVarDef'] = None, - base_type_exprs: List[Expression] = None, - metaclass: str = None) -> None: - self.name = name - self.defs = defs - self.type_vars = type_vars or [] - self.base_type_exprs = base_type_exprs or [] - self.metaclass = metaclass - self.decorators = [] - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_class_def(self) - - def is_generic(self) -> bool: - return self.info.is_generic() - - def serialize(self) -> JsonDict: - # Not serialized: defs, base_type_exprs, decorators - return {'.class': 'ClassDef', - 'name': self.name, - 'fullname': self.fullname, - 'type_vars': [v.serialize() for v in self.type_vars], - 'metaclass': self.metaclass, - 'is_builtinclass': self.is_builtinclass, - } - - @classmethod - def deserialize(self, data: JsonDict) -> 'ClassDef': - assert data['.class'] == 'ClassDef' - res = ClassDef(data['name'], - Block([]), - [mypy.types.TypeVarDef.deserialize(v) for v in data['type_vars']], - metaclass=data['metaclass'], - ) - res.fullname = data['fullname'] - res.is_builtinclass = data['is_builtinclass'] - return res - - -class GlobalDecl(Statement): - """Declaration global x, y, ...""" - - names = None # type: List[str] - - def __init__(self, names: List[str]) -> None: - self.names = names - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_global_decl(self) - - -class NonlocalDecl(Statement): - """Declaration nonlocal x, y, ...""" - - names = None # type: List[str] - - def __init__(self, names: List[str]) -> None: - self.names = names - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_nonlocal_decl(self) - - -class Block(Statement): - body = None # type: List[Statement] - # True if we can determine that this block is not executed. For example, - # this applies to blocks that are protected by something like "if PY3:" - # when using Python 2. - is_unreachable = False - - def __init__(self, body: List[Statement]) -> None: - self.body = body - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_block(self) - - -# Statements - - -class ExpressionStmt(Statement): - """An expression as a statement, such as print(s).""" - expr = None # type: Expression - - def __init__(self, expr: Expression) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_expression_stmt(self) - - -class AssignmentStmt(Statement): - """Assignment statement - The same node class is used for single assignment, multiple assignment - (e.g. x, y = z) and chained assignment (e.g. x = y = z), assignments - that define new names, and assignments with explicit types (# type). - - An lvalue can be NameExpr, TupleExpr, ListExpr, MemberExpr, IndexExpr. - """ - - lvalues = None # type: List[Expression] - rvalue = None # type: Expression - # Declared type in a comment, may be None. - type = None # type: mypy.types.Type - - def __init__(self, lvalues: List[Expression], rvalue: Expression, - type: 'mypy.types.Type' = None) -> None: - self.lvalues = lvalues - self.rvalue = rvalue - self.type = type - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_assignment_stmt(self) - - -class OperatorAssignmentStmt(Statement): - """Operator assignment statement such as x += 1""" - - op = '' - lvalue = None # type: Expression - rvalue = None # type: Expression - - def __init__(self, op: str, lvalue: Expression, rvalue: Expression) -> None: - self.op = op - self.lvalue = lvalue - self.rvalue = rvalue - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_operator_assignment_stmt(self) - - -class WhileStmt(Statement): - expr = None # type: Expression - body = None # type: Block - else_body = None # type: Block - - def __init__(self, expr: Expression, body: Block, else_body: Block) -> None: - self.expr = expr - self.body = body - self.else_body = else_body - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_while_stmt(self) - - -class ForStmt(Statement): - # Index variables - index = None # type: Expression - # Expression to iterate - expr = None # type: Expression - body = None # type: Block - else_body = None # type: Block - - def __init__(self, index: Expression, expr: Expression, body: Block, - else_body: Block) -> None: - self.index = index - self.expr = expr - self.body = body - self.else_body = else_body - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_for_stmt(self) - - -class ReturnStmt(Statement): - expr = None # type: Optional[Expression] - - def __init__(self, expr: Optional[Expression]) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_return_stmt(self) - - -class AssertStmt(Statement): - expr = None # type: Expression - - def __init__(self, expr: Expression) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_assert_stmt(self) - - -class DelStmt(Statement): - expr = None # type: Expression - - def __init__(self, expr: Expression) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_del_stmt(self) - - -class BreakStmt(Statement): - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_break_stmt(self) - - -class ContinueStmt(Statement): - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_continue_stmt(self) - - -class PassStmt(Statement): - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_pass_stmt(self) - - -class IfStmt(Statement): - expr = None # type: List[Expression] - body = None # type: List[Block] - else_body = None # type: Block - - def __init__(self, expr: List[Expression], body: List[Block], - else_body: Block) -> None: - self.expr = expr - self.body = body - self.else_body = else_body - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_if_stmt(self) - - -class RaiseStmt(Statement): - expr = None # type: Expression - from_expr = None # type: Expression - - def __init__(self, expr: Expression, from_expr: Expression = None) -> None: - self.expr = expr - self.from_expr = from_expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_raise_stmt(self) - - -class TryStmt(Statement): - body = None # type: Block # Try body - types = None # type: List[Expression] # Except type expressions - vars = None # type: List[NameExpr] # Except variable names - handlers = None # type: List[Block] # Except bodies - else_body = None # type: Block - finally_body = None # type: Block - - def __init__(self, body: Block, vars: List['NameExpr'], types: List[Expression], - handlers: List[Block], else_body: Block, - finally_body: Block) -> None: - self.body = body - self.vars = vars - self.types = types - self.handlers = handlers - self.else_body = else_body - self.finally_body = finally_body - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_try_stmt(self) - - -class WithStmt(Statement): - expr = None # type: List[Expression] - target = None # type: List[Expression] - body = None # type: Block - - def __init__(self, expr: List[Expression], target: List[Expression], - body: Block) -> None: - self.expr = expr - self.target = target - self.body = body - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_with_stmt(self) - - -class PrintStmt(Statement): - """Python 2 print statement""" - - args = None # type: List[Expression] - newline = False - # The file-like target object (given using >>). - target = None # type: Optional[Expression] - - def __init__(self, args: List[Expression], newline: bool, target: Expression = None) -> None: - self.args = args - self.newline = newline - self.target = target - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_print_stmt(self) - - -class ExecStmt(Statement): - """Python 2 exec statement""" - - expr = None # type: Expression - variables1 = None # type: Optional[Expression] - variables2 = None # type: Optional[Expression] - - def __init__(self, expr: Expression, - variables1: Optional[Expression], - variables2: Optional[Expression]) -> None: - self.expr = expr - self.variables1 = variables1 - self.variables2 = variables2 - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_exec_stmt(self) - - -# Expressions - - -class IntExpr(Expression): - """Integer literal""" - - value = 0 - literal = LITERAL_YES - - def __init__(self, value: int) -> None: - self.value = value - self.literal_hash = value - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_int_expr(self) - - -# How mypy uses StrExpr, BytesExpr, and UnicodeExpr: -# In Python 2 mode: -# b'x', 'x' -> StrExpr -# u'x' -> UnicodeExpr -# BytesExpr is unused -# -# In Python 3 mode: -# b'x' -> BytesExpr -# 'x', u'x' -> StrExpr -# UnicodeExpr is unused - -class StrExpr(Expression): - """String literal""" - - value = '' - literal = LITERAL_YES - - def __init__(self, value: str) -> None: - self.value = value - self.literal_hash = value - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_str_expr(self) - - -class BytesExpr(Expression): - """Bytes literal""" - - value = '' # TODO use bytes - literal = LITERAL_YES - - def __init__(self, value: str) -> None: - self.value = value - self.literal_hash = value - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_bytes_expr(self) - - -class UnicodeExpr(Expression): - """Unicode literal (Python 2.x)""" - - value = '' # TODO use bytes - literal = LITERAL_YES - - def __init__(self, value: str) -> None: - self.value = value - self.literal_hash = value - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_unicode_expr(self) - - -class FloatExpr(Expression): - """Float literal""" - - value = 0.0 - literal = LITERAL_YES - - def __init__(self, value: float) -> None: - self.value = value - self.literal_hash = value - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_float_expr(self) - - -class ComplexExpr(Expression): - """Complex literal""" - - value = 0.0j - literal = LITERAL_YES - - def __init__(self, value: complex) -> None: - self.value = value - self.literal_hash = value - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_complex_expr(self) - - -class EllipsisExpr(Expression): - """Ellipsis (...)""" - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_ellipsis(self) - - -class StarExpr(Expression): - """Star expression""" - - expr = None # type: Expression - - def __init__(self, expr: Expression) -> None: - self.expr = expr - self.literal = self.expr.literal - self.literal_hash = ('Star', expr.literal_hash,) - - # Whether this starred expression is used in a tuple/list and as lvalue - self.valid = False - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_star_expr(self) - - -class RefExpr(Expression): - """Abstract base class for name-like constructs""" - - kind = None # type: int # LDEF/GDEF/MDEF/... (None if not available) - node = None # type: SymbolNode # Var, FuncDef or TypeInfo that describes this - fullname = None # type: str # Fully qualified name (or name if not global) - - # Does this define a new name with inferred type? - # - # For members, after semantic analysis, this does not take base - # classes into consideration at all; the type checker deals with these. - is_def = False - - -class NameExpr(RefExpr): - """Name expression - - This refers to a local name, global name or a module. - """ - - name = None # type: str # Name referred to (may be qualified) - # TypeInfo of class surrounding expression (may be None) - info = None # type: TypeInfo - - literal = LITERAL_TYPE - - def __init__(self, name: str) -> None: - self.name = name - self.literal_hash = ('Var', name,) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_name_expr(self) - - def serialize(self) -> JsonDict: - # TODO: Find out where and why NameExpr is being serialized (if at all). - assert False, "Serializing NameExpr: %s" % (self,) - return {'.class': 'NameExpr', - 'kind': self.kind, - 'node': None if self.node is None else self.node.serialize(), - 'fullname': self.fullname, - 'is_def': self.is_def, - 'name': self.name, - 'literal': self.literal, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'NameExpr': - assert data['.class'] == 'NameExpr' - ret = NameExpr(data['name']) - ret.kind = data['kind'] - ret.node = None if data['node'] is None else SymbolNode.deserialize(data['node']) - ret.fullname = data['fullname'] - ret.is_def = data['is_def'] - ret.literal = data['literal'] - return ret - - -class MemberExpr(RefExpr): - """Member access expression x.y""" - - expr = None # type: Expression - name = None # type: str - # The variable node related to a definition. - def_var = None # type: Var - - def __init__(self, expr: Expression, name: str) -> None: - self.expr = expr - self.name = name - self.literal = self.expr.literal - self.literal_hash = ('Member', expr.literal_hash, name) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_member_expr(self) - - -# Kinds of arguments - -# Positional argument -ARG_POS = 0 # type: int -# Positional, optional argument (functions only, not calls) -ARG_OPT = 1 # type: int -# *arg argument -ARG_STAR = 2 # type: int -# Keyword argument x=y in call, or keyword-only function arg -ARG_NAMED = 3 # type: int -# **arg argument -ARG_STAR2 = 4 # type: int - - -class CallExpr(Expression): - """Call expression. - - This can also represent several special forms that are syntactically calls - such as cast(...) and None # type: .... - """ - - callee = None # type: Expression - args = None # type: List[Expression] - arg_kinds = None # type: List[int] # ARG_ constants - # Each name can be None if not a keyword argument. - arg_names = None # type: List[str] - # If not None, the node that represents the meaning of the CallExpr. For - # cast(...) this is a CastExpr. - analyzed = None # type: Optional[Expression] - - def __init__(self, callee: Expression, args: List[Expression], arg_kinds: List[int], - arg_names: List[str] = None, analyzed: Expression = None) -> None: - if not arg_names: - arg_names = [None] * len(args) - - self.callee = callee - self.args = args - self.arg_kinds = arg_kinds - self.arg_names = arg_names - self.analyzed = analyzed - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_call_expr(self) - - -class YieldFromExpr(Expression): - expr = None # type: Expression - - def __init__(self, expr: Expression) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_yield_from_expr(self) - - -class YieldExpr(Expression): - expr = None # type: Optional[Expression] - - def __init__(self, expr: Optional[Expression]) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_yield_expr(self) - - -class IndexExpr(Expression): - """Index expression x[y]. - - Also wraps type application such as List[int] as a special form. - """ - - base = None # type: Expression - index = None # type: Expression - # Inferred __getitem__ method type - method_type = None # type: mypy.types.Type - # If not None, this is actually semantically a type application - # Class[type, ...] or a type alias initializer. - analyzed = None # type: Union[TypeApplication, TypeAliasExpr] - - def __init__(self, base: Expression, index: Expression) -> None: - self.base = base - self.index = index - self.analyzed = None - if self.index.literal == LITERAL_YES: - self.literal = self.base.literal - self.literal_hash = ('Member', base.literal_hash, - index.literal_hash) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_index_expr(self) - - -class UnaryExpr(Expression): - """Unary operation""" - - op = '' - expr = None # type: Expression - # Inferred operator method type - method_type = None # type: mypy.types.Type - - def __init__(self, op: str, expr: Expression) -> None: - self.op = op - self.expr = expr - self.literal = self.expr.literal - self.literal_hash = ('Unary', op, expr.literal_hash) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_unary_expr(self) - - -# Map from binary operator id to related method name (in Python 3). -op_methods = { - '+': '__add__', - '-': '__sub__', - '*': '__mul__', - '/': '__truediv__', - '%': '__mod__', - '//': '__floordiv__', - '**': '__pow__', - '&': '__and__', - '|': '__or__', - '^': '__xor__', - '<<': '__lshift__', - '>>': '__rshift__', - '==': '__eq__', - '!=': '__ne__', - '<': '__lt__', - '>=': '__ge__', - '>': '__gt__', - '<=': '__le__', - 'in': '__contains__', -} # type: Dict[str, str] - -ops_with_inplace_method = { - '+', '-', '*', '/', '%', '//', '**', '&', '|', '^', '<<', '>>'} - -inplace_operator_methods = set( - '__i' + op_methods[op][2:] for op in ops_with_inplace_method) - -reverse_op_methods = { - '__add__': '__radd__', - '__sub__': '__rsub__', - '__mul__': '__rmul__', - '__truediv__': '__rtruediv__', - '__mod__': '__rmod__', - '__floordiv__': '__rfloordiv__', - '__pow__': '__rpow__', - '__and__': '__rand__', - '__or__': '__ror__', - '__xor__': '__rxor__', - '__lshift__': '__rlshift__', - '__rshift__': '__rrshift__', - '__eq__': '__eq__', - '__ne__': '__ne__', - '__lt__': '__gt__', - '__ge__': '__le__', - '__gt__': '__lt__', - '__le__': '__ge__', -} - -normal_from_reverse_op = dict((m, n) for n, m in reverse_op_methods.items()) -reverse_op_method_set = set(reverse_op_methods.values()) - - -class OpExpr(Expression): - """Binary operation (other than . or [] or comparison operators, - which have specific nodes).""" - - op = '' - left = None # type: Expression - right = None # type: Expression - # Inferred type for the operator method type (when relevant). - method_type = None # type: mypy.types.Type - - def __init__(self, op: str, left: Expression, right: Expression) -> None: - self.op = op - self.left = left - self.right = right - self.literal = min(self.left.literal, self.right.literal) - self.literal_hash = ('Binary', op, left.literal_hash, right.literal_hash) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_op_expr(self) - - -class ComparisonExpr(Expression): - """Comparison expression (e.g. a < b > c < d).""" - - operators = None # type: List[str] - operands = None # type: List[Expression] - # Inferred type for the operator methods (when relevant; None for 'is'). - method_types = None # type: List[mypy.types.Type] - - def __init__(self, operators: List[str], operands: List[Expression]) -> None: - self.operators = operators - self.operands = operands - self.method_types = [] - self.literal = min(o.literal for o in self.operands) - self.literal_hash = (('Comparison',) + tuple(operators) + - tuple(o.literal_hash for o in operands)) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_comparison_expr(self) - - -class SliceExpr(Expression): - """Slice expression (e.g. 'x:y', 'x:', '::2' or ':'). - - This is only valid as index in index expressions. - """ - - begin_index = None # type: Optional[Expression] - end_index = None # type: Optional[Expression] - stride = None # type: Optional[Expression] - - def __init__(self, begin_index: Optional[Expression], - end_index: Optional[Expression], - stride: Optional[Expression]) -> None: - self.begin_index = begin_index - self.end_index = end_index - self.stride = stride - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_slice_expr(self) - - -class CastExpr(Expression): - """Cast expression cast(type, expr).""" - - expr = None # type: Expression - type = None # type: mypy.types.Type - - def __init__(self, expr: Expression, typ: 'mypy.types.Type') -> None: - self.expr = expr - self.type = typ - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_cast_expr(self) - - -class RevealTypeExpr(Expression): - """Reveal type expression reveal_type(expr).""" - - expr = None # type: Expression - - def __init__(self, expr: Expression) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_reveal_type_expr(self) - - -class SuperExpr(Expression): - """Expression super().name""" - - name = '' - info = None # type: TypeInfo # Type that contains this super expression - - def __init__(self, name: str) -> None: - self.name = name - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_super_expr(self) - - -class FuncExpr(FuncItem, Expression): - """Lambda expression""" - - def name(self) -> str: - return '' - - def expr(self) -> Expression: - """Return the expression (the body) of the lambda.""" - ret = cast(ReturnStmt, self.body.body[-1]) - return ret.expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_func_expr(self) - - -class ListExpr(Expression): - """List literal expression [...].""" - - items = None # type: List[Expression] - - def __init__(self, items: List[Expression]) -> None: - self.items = items - if all(x.literal == LITERAL_YES for x in items): - self.literal = LITERAL_YES - self.literal_hash = ('List',) + tuple(x.literal_hash for x in items) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_list_expr(self) - - -class DictExpr(Expression): - """Dictionary literal expression {key: value, ...}.""" - - items = None # type: List[Tuple[Expression, Expression]] - - def __init__(self, items: List[Tuple[Expression, Expression]]) -> None: - self.items = items - if all(x[0].literal == LITERAL_YES and x[1].literal == LITERAL_YES - for x in items): - self.literal = LITERAL_YES - self.literal_hash = ('Dict',) + tuple( - (x[0].literal_hash, x[1].literal_hash) for x in items) # type: ignore - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_dict_expr(self) - - -class TupleExpr(Expression): - """Tuple literal expression (..., ...)""" - - items = None # type: List[Expression] - - def __init__(self, items: List[Expression]) -> None: - self.items = items - if all(x.literal == LITERAL_YES for x in items): - self.literal = LITERAL_YES - self.literal_hash = ('Tuple',) + tuple(x.literal_hash for x in items) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_tuple_expr(self) - - -class SetExpr(Expression): - """Set literal expression {value, ...}.""" - - items = None # type: List[Expression] - - def __init__(self, items: List[Expression]) -> None: - self.items = items - if all(x.literal == LITERAL_YES for x in items): - self.literal = LITERAL_YES - self.literal_hash = ('Set',) + tuple(x.literal_hash for x in items) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_set_expr(self) - - -class GeneratorExpr(Expression): - """Generator expression ... for ... in ... [ for ... in ... ] [ if ... ].""" - - left_expr = None # type: Expression - sequences_expr = None # type: List[Expression] - condlists = None # type: List[List[Expression]] - indices = None # type: List[Expression] - - def __init__(self, left_expr: Expression, indices: List[Expression], - sequences: List[Expression], condlists: List[List[Expression]]) -> None: - self.left_expr = left_expr - self.sequences = sequences - self.condlists = condlists - self.indices = indices - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_generator_expr(self) - - -class ListComprehension(Expression): - """List comprehension (e.g. [x + 1 for x in a])""" - - generator = None # type: GeneratorExpr - - def __init__(self, generator: GeneratorExpr) -> None: - self.generator = generator - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_list_comprehension(self) - - -class SetComprehension(Expression): - """Set comprehension (e.g. {x + 1 for x in a})""" - - generator = None # type: GeneratorExpr - - def __init__(self, generator: GeneratorExpr) -> None: - self.generator = generator - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_set_comprehension(self) - - -class DictionaryComprehension(Expression): - """Dictionary comprehension (e.g. {k: v for k, v in a}""" - - key = None # type: Expression - value = None # type: Expression - sequences_expr = None # type: List[Expression] - condlists = None # type: List[List[Expression]] - indices = None # type: List[Expression] - - def __init__(self, key: Expression, value: Expression, indices: List[Expression], - sequences: List[Expression], condlists: List[List[Expression]]) -> None: - self.key = key - self.value = value - self.sequences = sequences - self.condlists = condlists - self.indices = indices - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_dictionary_comprehension(self) - - -class ConditionalExpr(Expression): - """Conditional expression (e.g. x if y else z)""" - - cond = None # type: Expression - if_expr = None # type: Expression - else_expr = None # type: Expression - - def __init__(self, cond: Expression, if_expr: Expression, else_expr: Expression) -> None: - self.cond = cond - self.if_expr = if_expr - self.else_expr = else_expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_conditional_expr(self) - - -class BackquoteExpr(Expression): - """Python 2 expression `...`.""" - - expr = None # type: Expression - - def __init__(self, expr: Expression) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_backquote_expr(self) - - -class TypeApplication(Expression): - """Type application expr[type, ...]""" - - expr = None # type: Expression - types = None # type: List[mypy.types.Type] - - def __init__(self, expr: Expression, types: List['mypy.types.Type']) -> None: - self.expr = expr - self.types = types - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_type_application(self) - - -# Variance of a type variable. For example, T in the definition of -# List[T] is invariant, so List[int] is not a subtype of List[object], -# and also List[object] is not a subtype of List[int]. -# -# The T in Iterable[T] is covariant, so Iterable[int] is a subtype of -# Iterable[object], but not vice versa. -# -# If T is contravariant in Foo[T], Foo[object] is a subtype of -# Foo[int], but not vice versa. -INVARIANT = 0 # type: int -COVARIANT = 1 # type: int -CONTRAVARIANT = 2 # type: int - - -class TypeVarExpr(SymbolNode, Expression): - """Type variable expression TypeVar(...).""" - - _name = '' - _fullname = '' - # Value restriction: only types in the list are valid as values. If the - # list is empty, there is no restriction. - values = None # type: List[mypy.types.Type] - # Upper bound: only subtypes of upper_bound are valid as values. By default - # this is 'object', meaning no restriction. - upper_bound = None # type: mypy.types.Type - # Variance of the type variable. Invariant is the default. - # TypeVar(..., covariant=True) defines a covariant type variable. - # TypeVar(..., contravariant=True) defines a contravariant type - # variable. - variance = INVARIANT - - def __init__(self, name: str, fullname: str, - values: List['mypy.types.Type'], - upper_bound: 'mypy.types.Type', - variance: int=INVARIANT) -> None: - self._name = name - self._fullname = fullname - self.values = values - self.upper_bound = upper_bound - self.variance = variance - - def name(self) -> str: - return self._name - - def fullname(self) -> str: - return self._fullname - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_type_var_expr(self) - - def serialize(self) -> JsonDict: - return {'.class': 'TypeVarExpr', - 'name': self._name, - 'fullname': self._fullname, - 'values': [t.serialize() for t in self.values], - 'upper_bound': self.upper_bound.serialize(), - 'variance': self.variance, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'TypeVarExpr': - assert data['.class'] == 'TypeVarExpr' - return TypeVarExpr(data['name'], - data['fullname'], - [mypy.types.Type.deserialize(v) for v in data['values']], - mypy.types.Type.deserialize(data['upper_bound']), - data['variance']) - - -class TypeAliasExpr(Expression): - """Type alias expression (rvalue).""" - - type = None # type: mypy.types.Type - - def __init__(self, type: 'mypy.types.Type') -> None: - self.type = type - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_type_alias_expr(self) - - -class NamedTupleExpr(Expression): - """Named tuple expression namedtuple(...).""" - - # The class representation of this named tuple (its tuple_type attribute contains - # the tuple item types) - info = None # type: TypeInfo - - def __init__(self, info: 'TypeInfo') -> None: - self.info = info - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_namedtuple_expr(self) - - -class PromoteExpr(Expression): - """Ducktype class decorator expression _promote(...).""" - - type = None # type: mypy.types.Type - - def __init__(self, type: 'mypy.types.Type') -> None: - self.type = type - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit__promote_expr(self) - - -# Constants - - -class TempNode(Expression): - """Temporary dummy node used during type checking. - - This node is not present in the original program; it is just an artifact - of the type checker implementation. It only represents an opaque node with - some fixed type. - """ - - type = None # type: mypy.types.Type - - def __init__(self, typ: 'mypy.types.Type') -> None: - self.type = typ - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_temp_node(self) - - -class TypeInfo(SymbolNode): - """The type structure of a single class. - - Each TypeInfo corresponds one-to-one to a ClassDef, which - represents the AST of the class. - - In type-theory terms, this is a "type constructor", and if the - class is generic then it will be a type constructor of higher kind. - Where the class is used in an actual type, it's in the form of an - Instance, which amounts to a type application of the tycon to - the appropriate number of arguments. - """ - - _fullname = None # type: str # Fully qualified name - defn = None # type: ClassDef # Corresponding ClassDef - # Method Resolution Order: the order of looking up attributes. The first - # value always to refers to this class. - mro = None # type: List[TypeInfo] - subtypes = None # type: Set[TypeInfo] # Direct subclasses encountered so far - names = None # type: SymbolTable # Names defined directly in this type - is_abstract = False # Does the class have any abstract attributes? - abstract_attributes = None # type: List[str] - # Classes inheriting from Enum shadow their true members with a __getattr__, so we - # have to treat them as a special case. - is_enum = False - # If true, any unknown attributes should have type 'Any' instead - # of generating a type error. This would be true if there is a - # base class with type 'Any', but other use cases may be - # possible. This is similar to having __getattr__ that returns Any - # (and __setattr__), but without the __getattr__ method. - fallback_to_any = False - - # Information related to type annotations. - - # Generic type variable names - type_vars = None # type: List[str] - - # Direct base classes. - bases = None # type: List[mypy.types.Instance] - - # Another type which this type will be treated as a subtype of, - # even though it's not a subclass in Python. The non-standard - # `@_promote` decorator introduces this, and there are also - # several builtin examples, in particular `int` -> `float`. - _promote = None # type: mypy.types.Type - - # Representation of a Tuple[...] base class, if the class has any - # (e.g., for named tuples). If this is not None, the actual Type - # object used for this class is not an Instance but a TupleType; - # the corresponding Instance is set as the fallback type of the - # tuple type. - tuple_type = None # type: mypy.types.TupleType - - # Is this a named tuple type? - is_named_tuple = False - - # Is this a dummy from deserialization? - is_dummy = False - - # Alternative to fullname() for 'anonymous' classes. - alt_fullname = None # type: Optional[str] - - def __init__(self, names: 'SymbolTable', defn: ClassDef) -> None: - """Initialize a TypeInfo.""" - self.names = names - self.defn = defn - self.subtypes = set() - self.type_vars = [] - self.bases = [] - # Leave self.mro uninitialized until we compute it for real, - # so we don't accidentally try to use it prematurely. - self._fullname = defn.fullname - self.is_abstract = False - self.abstract_attributes = [] - if defn.type_vars: - for vd in defn.type_vars: - self.type_vars.append(vd.name) - - def name(self) -> str: - """Short name.""" - return self.defn.name - - def fullname(self) -> str: - return self._fullname - - def is_generic(self) -> bool: - """Is the type generic (i.e. does it have type variables)?""" - return len(self.type_vars) > 0 - - def get(self, name: str) -> 'SymbolTableNode': - for cls in self.mro: - n = cls.names.get(name) - if n: - return n - return None - - def __getitem__(self, name: str) -> 'SymbolTableNode': - n = self.get(name) - if n: - return n - else: - raise KeyError(name) - - def __repr__(self) -> str: - return '' % self.fullname() - - # IDEA: Refactor the has* methods to be more consistent and document - # them. - - def has_readable_member(self, name: str) -> bool: - return self.get(name) is not None - - def has_writable_member(self, name: str) -> bool: - return self.has_var(name) - - def has_var(self, name: str) -> bool: - return self.get_var(name) is not None - - def has_method(self, name: str) -> bool: - return self.get_method(name) is not None - - def get_var(self, name: str) -> Var: - for cls in self.mro: - if name in cls.names: - node = cls.names[name].node - if isinstance(node, Var): - return node - else: - return None - return None - - def get_var_or_getter(self, name: str) -> SymbolNode: - # TODO getter - return self.get_var(name) - - def get_var_or_setter(self, name: str) -> SymbolNode: - # TODO setter - return self.get_var(name) - - def get_method(self, name: str) -> FuncBase: - if self.mro is None: # Might be because of a previous error. - return None - for cls in self.mro: - if name in cls.names: - node = cls.names[name].node - if isinstance(node, FuncBase): - return node - else: - return None - return None - - def calculate_mro(self) -> None: - """Calculate and set mro (method resolution order). - - Raise MroError if cannot determine mro. - """ - mro = linearize_hierarchy(self) - assert mro, "Could not produce a MRO at all for %s" % (self,) - self.mro = mro - self.is_enum = self._calculate_is_enum() - - def _calculate_is_enum(self) -> bool: - """ - If this is "enum.Enum" itself, then yes, it's an enum. - If the flag .is_enum has been set on anything in the MRO, it's an enum. - """ - if self.fullname() == ENUM_BASECLASS: - return True - if self.mro: - return any(type_info.is_enum for type_info in self.mro) - return False - - def has_base(self, fullname: str) -> bool: - """Return True if type has a base type with the specified name. - - This can be either via extension or via implementation. - """ - if self.mro: - for cls in self.mro: - if cls.fullname() == fullname: - return True - return False - - def all_subtypes(self) -> 'Set[TypeInfo]': - """Return TypeInfos of all subtypes, including this type, as a set.""" - subtypes = set([self]) - for subt in self.subtypes: - for t in subt.all_subtypes(): - subtypes.add(t) - return subtypes - - def all_base_classes(self) -> 'List[TypeInfo]': - """Return a list of base classes, including indirect bases.""" - assert False - - def direct_base_classes(self) -> 'List[TypeInfo]': - """Return a direct base classes. - - Omit base classes of other base classes. - """ - return [base.type for base in self.bases] - - def __str__(self) -> str: - """Return a string representation of the type. - - This includes the most important information about the type. - """ - base = None # type: str - if self.bases: - base = 'Bases({})'.format(', '.join(str(base) - for base in self.bases)) - return dump_tagged(['Name({})'.format(self.fullname()), - base, - ('Names', sorted(self.names.keys()))], - 'TypeInfo') - - def serialize(self) -> Union[str, JsonDict]: - # NOTE: This is where all ClassDefs originate, so there shouldn't be duplicates. - data = {'.class': 'TypeInfo', - 'fullname': self.fullname(), - 'alt_fullname': self.alt_fullname, - 'names': self.names.serialize(self.alt_fullname or self.fullname()), - 'defn': self.defn.serialize(), - 'is_abstract': self.is_abstract, - 'abstract_attributes': self.abstract_attributes, - 'is_enum': self.is_enum, - 'fallback_to_any': self.fallback_to_any, - 'type_vars': self.type_vars, - 'bases': [b.serialize() for b in self.bases], - '_promote': None if self._promote is None else self._promote.serialize(), - 'tuple_type': None if self.tuple_type is None else self.tuple_type.serialize(), - 'is_named_tuple': self.is_named_tuple, - } - return data - - @classmethod - def deserialize(cls, data: JsonDict) -> 'TypeInfo': - names = SymbolTable.deserialize(data['names']) - defn = ClassDef.deserialize(data['defn']) - ti = TypeInfo(names, defn) - ti._fullname = data['fullname'] - ti.alt_fullname = data['alt_fullname'] - # TODO: Is there a reason to reconstruct ti.subtypes? - ti.is_abstract = data['is_abstract'] - ti.abstract_attributes = data['abstract_attributes'] - ti.is_enum = data['is_enum'] - ti.fallback_to_any = data['fallback_to_any'] - ti.type_vars = data['type_vars'] - ti.bases = [mypy.types.Instance.deserialize(b) for b in data['bases']] - ti._promote = (None if data['_promote'] is None - else mypy.types.Type.deserialize(data['_promote'])) - ti.tuple_type = (None if data['tuple_type'] is None - else mypy.types.TupleType.deserialize(data['tuple_type'])) - ti.is_named_tuple = data['is_named_tuple'] - return ti - - -def NamedTupleTypeInfo(tup: 'mypy.types.NamedTupleType', *args) -> TypeInfo: - info = TypeInfo(*args) - info.tuple_type = tup - info.bases = [tup.fallback] - info.is_named_tuple = True - info.mro = [info] + tup.fallback.type.mro - return info - -class SymbolTableNode: - # Kind of node. Possible values: - # - LDEF: local definition (of any kind) - # - GDEF: global (module-level) definition - # - MDEF: class member definition - # - UNBOUND_TVAR: TypeVar(...) definition, not bound - # - TVAR: type variable in a bound scope (generic function / generic clas) - # - MODULE_REF: reference to a module - # - TYPE_ALIAS: type alias - # - UNBOUND_IMPORTED: temporary kind for imported names - kind = None # type: int - # AST node of definition (FuncDef/Var/TypeInfo/Decorator/TypeVarExpr, - # or None for a bound type variable). - node = None # type: Optional[SymbolNode] - # Type variable definition (for bound type variables only) - tvar_def = None # type: Optional[mypy.types.TypeVarDef] - # Module id (e.g. "foo.bar") or None - mod_id = '' - # If this not None, override the type of the 'node' attribute. - type_override = None # type: Optional[mypy.types.Type] - # If False, this name won't be imported via 'from import *'. - # This has no effect on names within classes. - module_public = True - # For deserialized MODULE_REF nodes, the referenced module name; - # for other nodes, optionally the name of the referenced object. - cross_ref = None # type: Optional[str] - - def __init__(self, kind: int, node: Optional[SymbolNode], mod_id: str = None, - typ: 'mypy.types.Type' = None, - tvar_def: 'mypy.types.TypeVarDef' = None, - module_public: bool = True) -> None: - self.kind = kind - self.node = node - self.type_override = typ - self.mod_id = mod_id - self.tvar_def = tvar_def - self.module_public = module_public - - @property - def fullname(self) -> str: - if self.node is not None: - return self.node.fullname() - else: - return None - - @property - def type(self) -> 'mypy.types.Type': - # IDEA: Get rid of the Any type. - node = self.node # type: Any - if self.type_override is not None: - return self.type_override - elif ((isinstance(node, Var) or isinstance(node, FuncDef)) - and node.type is not None): - return node.type - elif isinstance(node, Decorator): - return node.var.type - else: - return None - - def __str__(self) -> str: - s = '{}/{}'.format(node_kinds[self.kind], short_type(self.node)) - if self.mod_id is not None: - s += ' ({})'.format(self.mod_id) - # Include declared type of variables and functions. - if self.type is not None: - s += ' : {}'.format(self.type) - return s - - def serialize(self, prefix: str, name: str) -> JsonDict: - """Serialize a SymbolTableNode. - - Args: - prefix: full name of the containing module or class; or None - name: name of this object relative to the containing object - """ - data = {'.class': 'SymbolTableNode', - 'kind': node_kinds[self.kind], - } # type: JsonDict - if self.tvar_def: - data['tvar_def'] = self.tvar_def.serialize() - if not self.module_public: - data['module_public'] = False - if self.kind == MODULE_REF: - assert self.node is not None, "Missing module cross ref in %s for %s" % (prefix, name) - data['cross_ref'] = self.node.fullname() - else: - if self.node is not None: - if prefix is not None: - # Check whether this is an alias for another object. - # If the object's canonical full name differs from - # the full name computed from prefix and name, - # it's an alias, and we serialize it as a cross ref. - if isinstance(self.node, TypeInfo): - fullname = self.node.alt_fullname or self.node.fullname() - else: - fullname = self.node.fullname() - if (fullname is not None and '.' in fullname and - fullname != prefix + '.' + name): - data['cross_ref'] = fullname - return data - data['node'] = self.node.serialize() - if self.type_override is not None: - data['type_override'] = self.type_override.serialize() - return data - - @classmethod - def deserialize(cls, data: JsonDict) -> 'SymbolTableNode': - assert data['.class'] == 'SymbolTableNode' - kind = inverse_node_kinds[data['kind']] - if 'cross_ref' in data: - # This will be fixed up later. - stnode = SymbolTableNode(kind, None) - stnode.cross_ref = data['cross_ref'] - else: - node = None - if 'node' in data: - node = SymbolNode.deserialize(data['node']) - typ = None - if 'type_override' in data: - typ = mypy.types.Type.deserialize(data['type_override']) - stnode = SymbolTableNode(kind, node, typ=typ) - if 'tvar_def' in data: - stnode.tvar_def = mypy.types.TypeVarDef.deserialize(data['tvar_def']) - if 'module_public' in data: - stnode.module_public = data['module_public'] - return stnode - - -class SymbolTable(Dict[str, SymbolTableNode]): - def __str__(self) -> str: - a = [] # type: List[str] - for key, value in self.items(): - # Filter out the implicit import of builtins. - if isinstance(value, SymbolTableNode): - if (value.fullname != 'builtins' and - (value.fullname or '').split('.')[-1] not in - implicit_module_attrs): - a.append(' ' + str(key) + ' : ' + str(value)) - else: - a.append(' ') - a = sorted(a) - a.insert(0, 'SymbolTable(') - a[-1] += ')' - return '\n'.join(a) - - def serialize(self, fullname: str) -> JsonDict: - data = {'.class': 'SymbolTable'} # type: JsonDict - for key, value in self.items(): - # Skip __builtins__: it's a reference to the builtins - # module that gets added to every module by - # SemanticAnalyzer.visit_file(), but it shouldn't be - # accessed by users of the module. - if key == '__builtins__': - continue - data[key] = value.serialize(fullname, key) - return data - - @classmethod - def deserialize(cls, data: JsonDict) -> 'SymbolTable': - assert data['.class'] == 'SymbolTable' - st = SymbolTable() - for key, value in data.items(): - if key != '.class': - st[key] = SymbolTableNode.deserialize(value) - return st - - -def function_type(func: FuncBase, fallback: 'mypy.types.Instance') -> 'mypy.types.FunctionLike': - if func.type: - assert isinstance(func.type, mypy.types.FunctionLike) - return func.type - else: - # Implicit type signature with dynamic types. - # Overloaded functions always have a signature, so func must be an ordinary function. - fdef = cast(FuncDef, func) - name = func.name() - if name: - name = '"{}"'.format(name) - names = [] # type: List[str] - for arg in fdef.arguments: - names.append(arg.variable.name()) - - return mypy.types.CallableType( - [mypy.types.AnyType()] * len(fdef.arguments), - [arg.kind for arg in fdef.arguments], - names, - mypy.types.AnyType(), - fallback, - name, - implicit=True, - ) - - -def method_type_with_fallback(func: FuncBase, - fallback: 'mypy.types.Instance') -> 'mypy.types.FunctionLike': - """Return the signature of a method (omit self).""" - return method_type(function_type(func, fallback)) - - -def method_type(sig: 'mypy.types.FunctionLike') -> 'mypy.types.FunctionLike': - if isinstance(sig, mypy.types.CallableType): - return method_callable(sig) - else: - sig = cast(mypy.types.Overloaded, sig) - items = [] # type: List[mypy.types.CallableType] - for c in sig.items(): - items.append(method_callable(c)) - return mypy.types.Overloaded(items) - - -def method_callable(c: 'mypy.types.CallableType') -> 'mypy.types.CallableType': - if c.arg_kinds and c.arg_kinds[0] == ARG_STAR: - # The signature is of the form 'def foo(*args, ...)'. - # In this case we shouldn't drop the first arg, - # since self will be absorbed by the *args. - return c - return c.copy_modified(arg_types=c.arg_types[1:], - arg_kinds=c.arg_kinds[1:], - arg_names=c.arg_names[1:]) - - -class MroError(Exception): - """Raised if a consistent mro cannot be determined for a class.""" - - -def linearize_hierarchy(info: TypeInfo) -> List[TypeInfo]: - # TODO describe - if info.mro: - return info.mro - bases = info.direct_base_classes() - lin_bases = [] - for base in bases: - assert base is not None, "Cannot linearize bases for %s %s" % (info.fullname(), bases) - lin_bases.append(linearize_hierarchy(base)) - lin_bases.append(bases) - return [info] + merge(lin_bases) - - -def merge(seqs: List[List[TypeInfo]]) -> List[TypeInfo]: - seqs = [s[:] for s in seqs] - result = [] # type: List[TypeInfo] - while True: - seqs = [s for s in seqs if s] - if not seqs: - return result - for seq in seqs: - head = seq[0] - if not [s for s in seqs if head in s[1:]]: - break - else: - raise MroError() - result.append(head) - for s in seqs: - if s[0] is head: - del s[0] +"""Abstract syntax tree node classes (i.e. parse tree).""" + +import os +import re +from abc import abstractmethod, ABCMeta + +from typing import ( + Any, TypeVar, List, Tuple, cast, Set, Dict, Union, Optional +) + +from mypy.lex import Token +import mypy.strconv +from mypy.visitor import NodeVisitor +from mypy.util import dump_tagged, short_type + + +class Context: + """Base type for objects that are valid as error message locations.""" + @abstractmethod + def get_line(self) -> int: pass + + +if False: + # break import cycle only needed for mypy + import mypy.types + + +T = TypeVar('T') + +JsonDict = Dict[str, Any] + + +# Symbol table node kinds +# +# TODO rename to use more descriptive names + +LDEF = 0 # type: int +GDEF = 1 # type: int +MDEF = 2 # type: int +MODULE_REF = 3 # type: int +# Type variable declared using TypeVar(...) has kind UNBOUND_TVAR. It's not +# valid as a type. A type variable is valid as a type (kind BOUND_TVAR) within +# (1) a generic class that uses the type variable as a type argument or +# (2) a generic function that refers to the type variable in its signature. +UNBOUND_TVAR = 4 # type: int +BOUND_TVAR = 5 # type: int +TYPE_ALIAS = 6 # type: int +# Placeholder for a name imported via 'from ... import'. Second phase of +# semantic will replace this the actual imported reference. This is +# needed so that we can detect whether a name has been imported during +# XXX what? +UNBOUND_IMPORTED = 7 # type: int + + +LITERAL_YES = 2 +LITERAL_TYPE = 1 +LITERAL_NO = 0 + +# Hard coded name of Enum baseclass. +ENUM_BASECLASS = "enum.Enum" + +node_kinds = { + LDEF: 'Ldef', + GDEF: 'Gdef', + MDEF: 'Mdef', + MODULE_REF: 'ModuleRef', + UNBOUND_TVAR: 'UnboundTvar', + BOUND_TVAR: 'Tvar', + TYPE_ALIAS: 'TypeAlias', + UNBOUND_IMPORTED: 'UnboundImported', +} +inverse_node_kinds = {_kind: _name for _name, _kind in node_kinds.items()} + + +implicit_module_attrs = {'__name__': '__builtins__.str', + '__doc__': '__builtins__.str', + '__file__': '__builtins__.str', + '__package__': '__builtins__.str'} + + +type_aliases = { + 'typing.List': '__builtins__.list', + 'typing.Dict': '__builtins__.dict', + 'typing.Set': '__builtins__.set', +} + +reverse_type_aliases = dict((name.replace('__builtins__', 'builtins'), alias) + for alias, name in type_aliases.items()) # type: Dict[str, str] + + +class Node(Context): + """Common base class for all non-type parse tree nodes.""" + + line = -1 + + literal = LITERAL_NO + literal_hash = None # type: Any + + def __str__(self) -> str: + ans = self.accept(mypy.strconv.StrConv()) + if ans is None: + return repr(self) + return ans + + def set_line(self, target: Union[Token, 'Node', int]) -> 'Node': + if isinstance(target, int): + self.line = target + else: + self.line = target.line + return self + + def get_line(self) -> int: + # TODO this should be just 'line' + return self.line + + def accept(self, visitor: NodeVisitor[T]) -> T: + raise RuntimeError('Not implemented') + + +# These are placeholders for a future refactoring; see #1783. +# For now they serve as (unchecked) documentation of what various +# fields of Node subtypes are expected to contain. +Statement = Node +Expression = Node + + +class SymbolNode(Node): + # Nodes that can be stored in a symbol table. + + # TODO do not use methods for these + + @abstractmethod + def name(self) -> str: pass + + @abstractmethod + def fullname(self) -> str: pass + + # NOTE: Can't use @abstractmethod, since many subclasses of Node + # don't implement serialize(). + def serialize(self) -> Any: + raise NotImplementedError('Cannot serialize {} instance'.format(self.__class__.__name__)) + + @classmethod + def deserialize(cls, data: JsonDict) -> 'SymbolNode': + classname = data['.class'] + glo = globals() + if classname in glo: + cl = glo[classname] + if issubclass(cl, cls) and 'deserialize' in cl.__dict__: + return cl.deserialize(data) + raise NotImplementedError('unexpected .class {}'.format(classname)) + + +class MypyFile(SymbolNode, Statement): + """The abstract syntax tree of a single source file.""" + + # Module name ('__main__' for initial file) + _name = None # type: str + # Fully qualified module name + _fullname = None # type: str + # Path to the file (None if not known) + path = '' + # Top-level definitions and statements + defs = None # type: List[Statement] + # Is there a UTF-8 BOM at the start? + is_bom = False + names = None # type: SymbolTable + # All import nodes within the file (also ones within functions etc.) + imports = None # type: List[ImportBase] + # Lines to ignore when checking + ignored_lines = None # type: Set[int] + # Is this file represented by a stub file (.pyi)? + is_stub = False + # Do weak typing globally in the file? + weak_opts = None # type: Set[str] + + def __init__(self, + defs: List[Statement], + imports: List['ImportBase'], + is_bom: bool = False, + ignored_lines: Set[int] = None, + weak_opts: Set[str] = None) -> None: + self.defs = defs + self.line = 1 # Dummy line number + self.imports = imports + self.is_bom = is_bom + self.weak_opts = weak_opts + if ignored_lines: + self.ignored_lines = ignored_lines + else: + self.ignored_lines = set() + + def name(self) -> str: + return self._name + + def fullname(self) -> str: + return self._fullname + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_mypy_file(self) + + def is_package_init_file(self) -> bool: + return not (self.path is None) and len(self.path) != 0 \ + and os.path.basename(self.path).startswith('__init__.') + + def serialize(self) -> JsonDict: + return {'.class': 'MypyFile', + '_name': self._name, + '_fullname': self._fullname, + 'names': self.names.serialize(self._fullname), + 'is_stub': self.is_stub, + 'path': self.path, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'MypyFile': + assert data['.class'] == 'MypyFile', data + tree = MypyFile([], []) + tree._name = data['_name'] + tree._fullname = data['_fullname'] + tree.names = SymbolTable.deserialize(data['names']) + tree.is_stub = data['is_stub'] + tree.path = data['path'] + return tree + + +class ImportBase(Statement): + """Base class for all import statements.""" + is_unreachable = False + is_top_level = False # Set by semanal.FirstPass + # If an import replaces existing definitions, we construct dummy assignment + # statements that assign the imported names to the names in the current scope, + # for type checking purposes. Example: + # + # x = 1 + # from m import x <-- add assignment representing "x = m.x" + assignments = None # type: List[AssignmentStmt] + + def __init__(self) -> None: + self.assignments = [] + + +class Import(ImportBase): + """import m [as n]""" + + ids = None # type: List[Tuple[str, Optional[str]]] # (module id, as id) + + def __init__(self, ids: List[Tuple[str, Optional[str]]]) -> None: + super().__init__() + self.ids = ids + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_import(self) + + +class ImportFrom(ImportBase): + """from m import x [as y], ...""" + + names = None # type: List[Tuple[str, Optional[str]]] # Tuples (name, as name) + + def __init__(self, id: str, relative: int, names: List[Tuple[str, Optional[str]]]) -> None: + super().__init__() + self.id = id + self.names = names + self.relative = relative + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_import_from(self) + + +class ImportAll(ImportBase): + """from m import *""" + + def __init__(self, id: str, relative: int) -> None: + super().__init__() + self.id = id + self.relative = relative + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_import_all(self) + + +class FuncBase(SymbolNode): + """Abstract base class for function-like nodes""" + + # Type signature. This is usually CallableType or Overloaded, but it can be something else for + # decorated functions/ + type = None # type: mypy.types.Type + # If method, reference to TypeInfo + info = None # type: TypeInfo + is_property = False + _fullname = None # type: str # Name with module prefix + + @abstractmethod + def name(self) -> str: pass + + def fullname(self) -> str: + return self._fullname + + def is_method(self) -> bool: + return bool(self.info) + + +class OverloadedFuncDef(FuncBase, Statement): + """A logical node representing all the variants of an overloaded function. + + This node has no explicit representation in the source program. + Overloaded variants must be consecutive in the source file. + """ + + items = None # type: List[Decorator] + + def __init__(self, items: List['Decorator']) -> None: + self.items = items + self.set_line(items[0].line) + + def name(self) -> str: + return self.items[0].func.name() + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_overloaded_func_def(self) + + def serialize(self) -> JsonDict: + return {'.class': 'OverloadedFuncDef', + 'items': [i.serialize() for i in self.items], + 'type': None if self.type is None else self.type.serialize(), + 'fullname': self._fullname, + 'is_property': self.is_property, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'OverloadedFuncDef': + assert data['.class'] == 'OverloadedFuncDef' + res = OverloadedFuncDef([Decorator.deserialize(d) for d in data['items']]) + if data.get('type') is not None: + res.type = mypy.types.Type.deserialize(data['type']) + res._fullname = data['fullname'] + res.is_property = data['is_property'] + # NOTE: res.info will be set in the fixup phase. + return res + + +class Argument(Node): + """A single argument in a FuncItem.""" + + def __init__(self, variable: 'Var', type_annotation: 'Optional[mypy.types.Type]', + initializer: Optional[Expression], kind: int, + initialization_statement: Optional['AssignmentStmt'] = None) -> None: + self.variable = variable + + self.type_annotation = type_annotation + self.initializer = initializer + + self.initialization_statement = initialization_statement + if not self.initialization_statement: + self.initialization_statement = self._initialization_statement() + + self.kind = kind + + def _initialization_statement(self) -> Optional['AssignmentStmt']: + """Convert the initializer into an assignment statement. + """ + if not self.initializer: + return None + + rvalue = self.initializer + lvalue = NameExpr(self.variable.name()) + assign = AssignmentStmt([lvalue], rvalue) + return assign + + def set_line(self, target: Union[Token, Node, int]) -> Node: + super().set_line(target) + + if self.initializer: + self.initializer.set_line(self.line) + + self.variable.set_line(self.line) + + if self.initialization_statement: + self.initialization_statement.set_line(self.line) + self.initialization_statement.lvalues[0].set_line(self.line) + + def serialize(self) -> JsonDict: + data = {'.class': 'Argument', + 'kind': self.kind, + 'variable': self.variable.serialize(), + 'type_annotation': (None if self.type_annotation is None + else self.type_annotation.serialize()), + } # type: JsonDict + # TODO: initializer? + return data + + @classmethod + def deserialize(cls, data: JsonDict) -> 'Argument': + assert data['.class'] == 'Argument' + return Argument(Var.deserialize(data['variable']), + (None if data.get('type_annotation') is None + else mypy.types.Type.deserialize(data['type_annotation'])), + None, # TODO: initializer? + kind=data['kind']) + + +class FuncItem(FuncBase): + arguments = [] # type: List[Argument] + # Minimum number of arguments + min_args = 0 + # Maximum number of positional arguments, -1 if no explicit limit (*args not included) + max_pos = 0 + body = None # type: Block + # Is this an overload variant of function with more than one overload variant? + is_overload = False + is_generator = False # Contains a yield statement? + is_static = False # Uses @staticmethod? + is_class = False # Uses @classmethod? + # Variants of function with type variables with values expanded + expanded = None # type: List[FuncItem] + + def __init__(self, arguments: List[Argument], body: 'Block', + typ: 'mypy.types.FunctionLike' = None) -> None: + self.arguments = arguments + arg_kinds = [arg.kind for arg in self.arguments] + self.max_pos = arg_kinds.count(ARG_POS) + arg_kinds.count(ARG_OPT) + self.body = body + self.type = typ + self.expanded = [] + + self.min_args = 0 + for i in range(len(self.arguments)): + if self.arguments[i] is None and i < self.max_fixed_argc(): + self.min_args = i + 1 + + def max_fixed_argc(self) -> int: + return self.max_pos + + def set_line(self, target: Union[Token, Node, int]) -> Node: + super().set_line(target) + for arg in self.arguments: + arg.set_line(self.line) + return self + + def is_dynamic(self): + return self.type is None + + +class FuncDef(FuncItem, Statement): + """Function definition. + + This is a non-lambda function defined using 'def'. + """ + + is_decorated = False + is_conditional = False # Defined conditionally (within block)? + is_abstract = False + is_property = False + original_def = None # type: Union[None, FuncDef, Var] # Original conditional definition + + def __init__(self, + name: str, # Function name + arguments: List[Argument], + body: 'Block', + typ: 'mypy.types.FunctionLike' = None) -> None: + super().__init__(arguments, body, typ) + self._name = name + + def name(self) -> str: + return self._name + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_func_def(self) + + def is_constructor(self) -> bool: + return self.info is not None and self._name == '__init__' + + def serialize(self) -> JsonDict: + return {'.class': 'FuncDef', + 'name': self._name, + 'fullname': self._fullname, + 'arguments': [a.serialize() for a in self.arguments], + 'type': None if self.type is None else self.type.serialize(), + 'is_property': self.is_property, + 'is_overload': self.is_overload, + 'is_generator': self.is_generator, + 'is_static': self.is_static, + 'is_class': self.is_class, + 'is_decorated': self.is_decorated, + 'is_conditional': self.is_conditional, + 'is_abstract': self.is_abstract, + # TODO: Do we need expanded, original_def? + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'FuncDef': + assert data['.class'] == 'FuncDef' + body = Block([]) + ret = FuncDef(data['name'], + [Argument.deserialize(a) for a in data['arguments']], + body, + (None if data['type'] is None + else mypy.types.FunctionLike.deserialize(data['type']))) + ret._fullname = data['fullname'] + ret.is_property = data['is_property'] + ret.is_overload = data['is_overload'] + ret.is_generator = data['is_generator'] + ret.is_static = data['is_static'] + ret.is_class = data['is_class'] + ret.is_decorated = data['is_decorated'] + ret.is_conditional = data['is_conditional'] + ret.is_abstract = data['is_abstract'] + # NOTE: ret.info is set in the fixup phase. + return ret + + +class Decorator(SymbolNode, Statement): + """A decorated function. + + A single Decorator object can include any number of function decorators. + """ + + func = None # type: FuncDef # Decorated function + decorators = None # type: List[Expression] # Decorators, at least one # XXX Not true + var = None # type: Var # Represents the decorated function obj + is_overload = False + + def __init__(self, func: FuncDef, decorators: List[Expression], + var: 'Var') -> None: + self.func = func + self.decorators = decorators + self.var = var + self.is_overload = False + + def name(self) -> str: + return self.func.name() + + def fullname(self) -> str: + return self.func.fullname() + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_decorator(self) + + def serialize(self) -> JsonDict: + return {'.class': 'Decorator', + 'func': self.func.serialize(), + 'var': self.var.serialize(), + 'is_overload': self.is_overload, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'Decorator': + assert data['.class'] == 'Decorator' + dec = Decorator(FuncDef.deserialize(data['func']), + [], + Var.deserialize(data['var'])) + dec.is_overload = data['is_overload'] + return dec + + +class Var(SymbolNode, Statement): + """A variable. + + It can refer to global/local variable or a data attribute. + """ + + _name = None # type: str # Name without module prefix + _fullname = None # type: str # Name with module prefix + info = None # type: TypeInfo # Defining class (for member variables) + type = None # type: mypy.types.Type # Declared or inferred type, or None + # Is this the first argument to an ordinary method (usually "self")? + is_self = False + is_ready = False # If inferred, is the inferred type available? + # Is this initialized explicitly to a non-None value in class body? + is_initialized_in_class = False + is_staticmethod = False + is_classmethod = False + is_property = False + is_settable_property = False + + def __init__(self, name: str, type: 'mypy.types.Type' = None) -> None: + self._name = name + self.type = type + self.is_self = False + self.is_ready = True + self.is_initialized_in_class = False + + def name(self) -> str: + return self._name + + def fullname(self) -> str: + return self._fullname + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_var(self) + + def serialize(self) -> JsonDict: + # TODO: Leave default values out? + # NOTE: Sometimes self.is_ready is False here, but we don't care. + data = {'.class': 'Var', + 'name': self._name, + 'fullname': self._fullname, + 'type': None if self.type is None else self.type.serialize(), + 'is_self': self.is_self, + 'is_initialized_in_class': self.is_initialized_in_class, + 'is_staticmethod': self.is_staticmethod, + 'is_classmethod': self.is_classmethod, + 'is_property': self.is_property, + 'is_settable_property': self.is_settable_property, + } # type: JsonDict + return data + + @classmethod + def deserialize(cls, data: JsonDict) -> 'Var': + assert data['.class'] == 'Var' + name = data['name'] + type = None if data['type'] is None else mypy.types.Type.deserialize(data['type']) + v = Var(name, type) + v._fullname = data['fullname'] + v.is_self = data['is_self'] + v.is_initialized_in_class = data['is_initialized_in_class'] + v.is_staticmethod = data['is_staticmethod'] + v.is_classmethod = data['is_classmethod'] + v.is_property = data['is_property'] + v.is_settable_property = data['is_settable_property'] + return v + + +class ClassDef(Statement): + """Class definition""" + + name = None # type: str # Name of the class without module prefix + fullname = None # type: str # Fully qualified name of the class + defs = None # type: Block + type_vars = None # type: List[mypy.types.TypeVarDef] + # Base class expressions (not semantically analyzed -- can be arbitrary expressions) + base_type_exprs = None # type: List[Expression] + info = None # type: TypeInfo # Related TypeInfo + metaclass = '' + decorators = None # type: List[Expression] + # Built-in/extension class? (single implementation inheritance only) + is_builtinclass = False + + def __init__(self, + name: str, + defs: 'Block', + type_vars: List['mypy.types.TypeVarDef'] = None, + base_type_exprs: List[Expression] = None, + metaclass: str = None) -> None: + self.name = name + self.defs = defs + self.type_vars = type_vars or [] + self.base_type_exprs = base_type_exprs or [] + self.metaclass = metaclass + self.decorators = [] + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_class_def(self) + + def is_generic(self) -> bool: + return self.info.is_generic() + + def serialize(self) -> JsonDict: + # Not serialized: defs, base_type_exprs, decorators + return {'.class': 'ClassDef', + 'name': self.name, + 'fullname': self.fullname, + 'type_vars': [v.serialize() for v in self.type_vars], + 'metaclass': self.metaclass, + 'is_builtinclass': self.is_builtinclass, + } + + @classmethod + def deserialize(self, data: JsonDict) -> 'ClassDef': + assert data['.class'] == 'ClassDef' + res = ClassDef(data['name'], + Block([]), + [mypy.types.TypeVarDef.deserialize(v) for v in data['type_vars']], + metaclass=data['metaclass'], + ) + res.fullname = data['fullname'] + res.is_builtinclass = data['is_builtinclass'] + return res + + +class GlobalDecl(Statement): + """Declaration global x, y, ...""" + + names = None # type: List[str] + + def __init__(self, names: List[str]) -> None: + self.names = names + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_global_decl(self) + + +class NonlocalDecl(Statement): + """Declaration nonlocal x, y, ...""" + + names = None # type: List[str] + + def __init__(self, names: List[str]) -> None: + self.names = names + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_nonlocal_decl(self) + + +class Block(Statement): + body = None # type: List[Statement] + # True if we can determine that this block is not executed. For example, + # this applies to blocks that are protected by something like "if PY3:" + # when using Python 2. + is_unreachable = False + + def __init__(self, body: List[Statement]) -> None: + self.body = body + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_block(self) + + +# Statements + + +class ExpressionStmt(Statement): + """An expression as a statement, such as print(s).""" + expr = None # type: Expression + + def __init__(self, expr: Expression) -> None: + self.expr = expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_expression_stmt(self) + + +class AssignmentStmt(Statement): + """Assignment statement + The same node class is used for single assignment, multiple assignment + (e.g. x, y = z) and chained assignment (e.g. x = y = z), assignments + that define new names, and assignments with explicit types (# type). + + An lvalue can be NameExpr, TupleExpr, ListExpr, MemberExpr, IndexExpr. + """ + + lvalues = None # type: List[Expression] + rvalue = None # type: Expression + # Declared type in a comment, may be None. + type = None # type: mypy.types.Type + + def __init__(self, lvalues: List[Expression], rvalue: Expression, + type: 'mypy.types.Type' = None) -> None: + self.lvalues = lvalues + self.rvalue = rvalue + self.type = type + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_assignment_stmt(self) + + +class OperatorAssignmentStmt(Statement): + """Operator assignment statement such as x += 1""" + + op = '' + lvalue = None # type: Expression + rvalue = None # type: Expression + + def __init__(self, op: str, lvalue: Expression, rvalue: Expression) -> None: + self.op = op + self.lvalue = lvalue + self.rvalue = rvalue + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_operator_assignment_stmt(self) + + +class WhileStmt(Statement): + expr = None # type: Expression + body = None # type: Block + else_body = None # type: Block + + def __init__(self, expr: Expression, body: Block, else_body: Block) -> None: + self.expr = expr + self.body = body + self.else_body = else_body + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_while_stmt(self) + + +class ForStmt(Statement): + # Index variables + index = None # type: Expression + # Expression to iterate + expr = None # type: Expression + body = None # type: Block + else_body = None # type: Block + + def __init__(self, index: Expression, expr: Expression, body: Block, + else_body: Block) -> None: + self.index = index + self.expr = expr + self.body = body + self.else_body = else_body + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_for_stmt(self) + + +class ReturnStmt(Statement): + expr = None # type: Optional[Expression] + + def __init__(self, expr: Optional[Expression]) -> None: + self.expr = expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_return_stmt(self) + + +class AssertStmt(Statement): + expr = None # type: Expression + + def __init__(self, expr: Expression) -> None: + self.expr = expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_assert_stmt(self) + + +class DelStmt(Statement): + expr = None # type: Expression + + def __init__(self, expr: Expression) -> None: + self.expr = expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_del_stmt(self) + + +class BreakStmt(Statement): + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_break_stmt(self) + + +class ContinueStmt(Statement): + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_continue_stmt(self) + + +class PassStmt(Statement): + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_pass_stmt(self) + + +class IfStmt(Statement): + expr = None # type: List[Expression] + body = None # type: List[Block] + else_body = None # type: Block + + def __init__(self, expr: List[Expression], body: List[Block], + else_body: Block) -> None: + self.expr = expr + self.body = body + self.else_body = else_body + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_if_stmt(self) + + +class RaiseStmt(Statement): + expr = None # type: Expression + from_expr = None # type: Expression + + def __init__(self, expr: Expression, from_expr: Expression = None) -> None: + self.expr = expr + self.from_expr = from_expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_raise_stmt(self) + + +class TryStmt(Statement): + body = None # type: Block # Try body + types = None # type: List[Expression] # Except type expressions + vars = None # type: List[NameExpr] # Except variable names + handlers = None # type: List[Block] # Except bodies + else_body = None # type: Block + finally_body = None # type: Block + + def __init__(self, body: Block, vars: List['NameExpr'], types: List[Expression], + handlers: List[Block], else_body: Block, + finally_body: Block) -> None: + self.body = body + self.vars = vars + self.types = types + self.handlers = handlers + self.else_body = else_body + self.finally_body = finally_body + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_try_stmt(self) + + +class WithStmt(Statement): + expr = None # type: List[Expression] + target = None # type: List[Expression] + body = None # type: Block + + def __init__(self, expr: List[Expression], target: List[Expression], + body: Block) -> None: + self.expr = expr + self.target = target + self.body = body + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_with_stmt(self) + + +class PrintStmt(Statement): + """Python 2 print statement""" + + args = None # type: List[Expression] + newline = False + # The file-like target object (given using >>). + target = None # type: Optional[Expression] + + def __init__(self, args: List[Expression], newline: bool, target: Expression = None) -> None: + self.args = args + self.newline = newline + self.target = target + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_print_stmt(self) + + +class ExecStmt(Statement): + """Python 2 exec statement""" + + expr = None # type: Expression + variables1 = None # type: Optional[Expression] + variables2 = None # type: Optional[Expression] + + def __init__(self, expr: Expression, + variables1: Optional[Expression], + variables2: Optional[Expression]) -> None: + self.expr = expr + self.variables1 = variables1 + self.variables2 = variables2 + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_exec_stmt(self) + + +# Expressions + + +class IntExpr(Expression): + """Integer literal""" + + value = 0 + literal = LITERAL_YES + + def __init__(self, value: int) -> None: + self.value = value + self.literal_hash = value + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_int_expr(self) + + +# How mypy uses StrExpr, BytesExpr, and UnicodeExpr: +# In Python 2 mode: +# b'x', 'x' -> StrExpr +# u'x' -> UnicodeExpr +# BytesExpr is unused +# +# In Python 3 mode: +# b'x' -> BytesExpr +# 'x', u'x' -> StrExpr +# UnicodeExpr is unused + +class StrExpr(Expression): + """String literal""" + + value = '' + literal = LITERAL_YES + + def __init__(self, value: str) -> None: + self.value = value + self.literal_hash = value + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_str_expr(self) + + +class BytesExpr(Expression): + """Bytes literal""" + + value = '' # TODO use bytes + literal = LITERAL_YES + + def __init__(self, value: str) -> None: + self.value = value + self.literal_hash = value + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_bytes_expr(self) + + +class UnicodeExpr(Expression): + """Unicode literal (Python 2.x)""" + + value = '' # TODO use bytes + literal = LITERAL_YES + + def __init__(self, value: str) -> None: + self.value = value + self.literal_hash = value + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_unicode_expr(self) + + +class FloatExpr(Expression): + """Float literal""" + + value = 0.0 + literal = LITERAL_YES + + def __init__(self, value: float) -> None: + self.value = value + self.literal_hash = value + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_float_expr(self) + + +class ComplexExpr(Expression): + """Complex literal""" + + value = 0.0j + literal = LITERAL_YES + + def __init__(self, value: complex) -> None: + self.value = value + self.literal_hash = value + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_complex_expr(self) + + +class EllipsisExpr(Expression): + """Ellipsis (...)""" + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_ellipsis(self) + + +class StarExpr(Expression): + """Star expression""" + + expr = None # type: Expression + + def __init__(self, expr: Expression) -> None: + self.expr = expr + self.literal = self.expr.literal + self.literal_hash = ('Star', expr.literal_hash,) + + # Whether this starred expression is used in a tuple/list and as lvalue + self.valid = False + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_star_expr(self) + + +class RefExpr(Expression): + """Abstract base class for name-like constructs""" + + kind = None # type: int # LDEF/GDEF/MDEF/... (None if not available) + node = None # type: SymbolNode # Var, FuncDef or TypeInfo that describes this + fullname = None # type: str # Fully qualified name (or name if not global) + + # Does this define a new name with inferred type? + # + # For members, after semantic analysis, this does not take base + # classes into consideration at all; the type checker deals with these. + is_def = False + + +class NameExpr(RefExpr): + """Name expression + + This refers to a local name, global name or a module. + """ + + name = None # type: str # Name referred to (may be qualified) + # TypeInfo of class surrounding expression (may be None) + info = None # type: TypeInfo + + literal = LITERAL_TYPE + + def __init__(self, name: str) -> None: + self.name = name + self.literal_hash = ('Var', name,) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_name_expr(self) + + def serialize(self) -> JsonDict: + # TODO: Find out where and why NameExpr is being serialized (if at all). + assert False, "Serializing NameExpr: %s" % (self,) + return {'.class': 'NameExpr', + 'kind': self.kind, + 'node': None if self.node is None else self.node.serialize(), + 'fullname': self.fullname, + 'is_def': self.is_def, + 'name': self.name, + 'literal': self.literal, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'NameExpr': + assert data['.class'] == 'NameExpr' + ret = NameExpr(data['name']) + ret.kind = data['kind'] + ret.node = None if data['node'] is None else SymbolNode.deserialize(data['node']) + ret.fullname = data['fullname'] + ret.is_def = data['is_def'] + ret.literal = data['literal'] + return ret + + +class MemberExpr(RefExpr): + """Member access expression x.y""" + + expr = None # type: Expression + name = None # type: str + # The variable node related to a definition. + def_var = None # type: Var + + def __init__(self, expr: Expression, name: str) -> None: + self.expr = expr + self.name = name + self.literal = self.expr.literal + self.literal_hash = ('Member', expr.literal_hash, name) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_member_expr(self) + + +# Kinds of arguments + +# Positional argument +ARG_POS = 0 # type: int +# Positional, optional argument (functions only, not calls) +ARG_OPT = 1 # type: int +# *arg argument +ARG_STAR = 2 # type: int +# Keyword argument x=y in call, or keyword-only function arg +ARG_NAMED = 3 # type: int +# **arg argument +ARG_STAR2 = 4 # type: int + + +class CallExpr(Expression): + """Call expression. + + This can also represent several special forms that are syntactically calls + such as cast(...) and None # type: .... + """ + + callee = None # type: Expression + args = None # type: List[Expression] + arg_kinds = None # type: List[int] # ARG_ constants + # Each name can be None if not a keyword argument. + arg_names = None # type: List[str] + # If not None, the node that represents the meaning of the CallExpr. For + # cast(...) this is a CastExpr. + analyzed = None # type: Optional[Expression] + + def __init__(self, callee: Expression, args: List[Expression], arg_kinds: List[int], + arg_names: List[str] = None, analyzed: Expression = None) -> None: + if not arg_names: + arg_names = [None] * len(args) + + self.callee = callee + self.args = args + self.arg_kinds = arg_kinds + self.arg_names = arg_names + self.analyzed = analyzed + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_call_expr(self) + + +class YieldFromExpr(Expression): + expr = None # type: Expression + + def __init__(self, expr: Expression) -> None: + self.expr = expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_yield_from_expr(self) + + +class YieldExpr(Expression): + expr = None # type: Optional[Expression] + + def __init__(self, expr: Optional[Expression]) -> None: + self.expr = expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_yield_expr(self) + + +class IndexExpr(Expression): + """Index expression x[y]. + + Also wraps type application such as List[int] as a special form. + """ + + base = None # type: Expression + index = None # type: Expression + # Inferred __getitem__ method type + method_type = None # type: mypy.types.Type + # If not None, this is actually semantically a type application + # Class[type, ...] or a type alias initializer. + analyzed = None # type: Union[TypeApplication, TypeAliasExpr] + + def __init__(self, base: Expression, index: Expression) -> None: + self.base = base + self.index = index + self.analyzed = None + if self.index.literal == LITERAL_YES: + self.literal = self.base.literal + self.literal_hash = ('Member', base.literal_hash, + index.literal_hash) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_index_expr(self) + + +class UnaryExpr(Expression): + """Unary operation""" + + op = '' + expr = None # type: Expression + # Inferred operator method type + method_type = None # type: mypy.types.Type + + def __init__(self, op: str, expr: Expression) -> None: + self.op = op + self.expr = expr + self.literal = self.expr.literal + self.literal_hash = ('Unary', op, expr.literal_hash) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_unary_expr(self) + + +# Map from binary operator id to related method name (in Python 3). +op_methods = { + '+': '__add__', + '-': '__sub__', + '*': '__mul__', + '/': '__truediv__', + '%': '__mod__', + '//': '__floordiv__', + '**': '__pow__', + '&': '__and__', + '|': '__or__', + '^': '__xor__', + '<<': '__lshift__', + '>>': '__rshift__', + '==': '__eq__', + '!=': '__ne__', + '<': '__lt__', + '>=': '__ge__', + '>': '__gt__', + '<=': '__le__', + 'in': '__contains__', +} # type: Dict[str, str] + +ops_with_inplace_method = { + '+', '-', '*', '/', '%', '//', '**', '&', '|', '^', '<<', '>>'} + +inplace_operator_methods = set( + '__i' + op_methods[op][2:] for op in ops_with_inplace_method) + +reverse_op_methods = { + '__add__': '__radd__', + '__sub__': '__rsub__', + '__mul__': '__rmul__', + '__truediv__': '__rtruediv__', + '__mod__': '__rmod__', + '__floordiv__': '__rfloordiv__', + '__pow__': '__rpow__', + '__and__': '__rand__', + '__or__': '__ror__', + '__xor__': '__rxor__', + '__lshift__': '__rlshift__', + '__rshift__': '__rrshift__', + '__eq__': '__eq__', + '__ne__': '__ne__', + '__lt__': '__gt__', + '__ge__': '__le__', + '__gt__': '__lt__', + '__le__': '__ge__', +} + +normal_from_reverse_op = dict((m, n) for n, m in reverse_op_methods.items()) +reverse_op_method_set = set(reverse_op_methods.values()) + + +class OpExpr(Expression): + """Binary operation (other than . or [] or comparison operators, + which have specific nodes).""" + + op = '' + left = None # type: Expression + right = None # type: Expression + # Inferred type for the operator method type (when relevant). + method_type = None # type: mypy.types.Type + + def __init__(self, op: str, left: Expression, right: Expression) -> None: + self.op = op + self.left = left + self.right = right + self.literal = min(self.left.literal, self.right.literal) + self.literal_hash = ('Binary', op, left.literal_hash, right.literal_hash) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_op_expr(self) + + +class ComparisonExpr(Expression): + """Comparison expression (e.g. a < b > c < d).""" + + operators = None # type: List[str] + operands = None # type: List[Expression] + # Inferred type for the operator methods (when relevant; None for 'is'). + method_types = None # type: List[mypy.types.Type] + + def __init__(self, operators: List[str], operands: List[Expression]) -> None: + self.operators = operators + self.operands = operands + self.method_types = [] + self.literal = min(o.literal for o in self.operands) + self.literal_hash = (('Comparison',) + tuple(operators) + + tuple(o.literal_hash for o in operands)) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_comparison_expr(self) + + +class SliceExpr(Expression): + """Slice expression (e.g. 'x:y', 'x:', '::2' or ':'). + + This is only valid as index in index expressions. + """ + + begin_index = None # type: Optional[Expression] + end_index = None # type: Optional[Expression] + stride = None # type: Optional[Expression] + + def __init__(self, begin_index: Optional[Expression], + end_index: Optional[Expression], + stride: Optional[Expression]) -> None: + self.begin_index = begin_index + self.end_index = end_index + self.stride = stride + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_slice_expr(self) + + +class CastExpr(Expression): + """Cast expression cast(type, expr).""" + + expr = None # type: Expression + type = None # type: mypy.types.Type + + def __init__(self, expr: Expression, typ: 'mypy.types.Type') -> None: + self.expr = expr + self.type = typ + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_cast_expr(self) + + +class RevealTypeExpr(Expression): + """Reveal type expression reveal_type(expr).""" + + expr = None # type: Expression + + def __init__(self, expr: Expression) -> None: + self.expr = expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_reveal_type_expr(self) + + +class SuperExpr(Expression): + """Expression super().name""" + + name = '' + info = None # type: TypeInfo # Type that contains this super expression + + def __init__(self, name: str) -> None: + self.name = name + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_super_expr(self) + + +class FuncExpr(FuncItem, Expression): + """Lambda expression""" + + def name(self) -> str: + return '' + + def expr(self) -> Expression: + """Return the expression (the body) of the lambda.""" + ret = cast(ReturnStmt, self.body.body[-1]) + return ret.expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_func_expr(self) + + +class ListExpr(Expression): + """List literal expression [...].""" + + items = None # type: List[Expression] + + def __init__(self, items: List[Expression]) -> None: + self.items = items + if all(x.literal == LITERAL_YES for x in items): + self.literal = LITERAL_YES + self.literal_hash = ('List',) + tuple(x.literal_hash for x in items) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_list_expr(self) + + +class DictExpr(Expression): + """Dictionary literal expression {key: value, ...}.""" + + items = None # type: List[Tuple[Expression, Expression]] + + def __init__(self, items: List[Tuple[Expression, Expression]]) -> None: + self.items = items + if all(x[0].literal == LITERAL_YES and x[1].literal == LITERAL_YES + for x in items): + self.literal = LITERAL_YES + self.literal_hash = ('Dict',) + tuple( + (x[0].literal_hash, x[1].literal_hash) for x in items) # type: ignore + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_dict_expr(self) + + +class TupleExpr(Expression): + """Tuple literal expression (..., ...)""" + + items = None # type: List[Expression] + + def __init__(self, items: List[Expression]) -> None: + self.items = items + if all(x.literal == LITERAL_YES for x in items): + self.literal = LITERAL_YES + self.literal_hash = ('Tuple',) + tuple(x.literal_hash for x in items) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_tuple_expr(self) + + +class SetExpr(Expression): + """Set literal expression {value, ...}.""" + + items = None # type: List[Expression] + + def __init__(self, items: List[Expression]) -> None: + self.items = items + if all(x.literal == LITERAL_YES for x in items): + self.literal = LITERAL_YES + self.literal_hash = ('Set',) + tuple(x.literal_hash for x in items) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_set_expr(self) + + +class GeneratorExpr(Expression): + """Generator expression ... for ... in ... [ for ... in ... ] [ if ... ].""" + + left_expr = None # type: Expression + sequences_expr = None # type: List[Expression] + condlists = None # type: List[List[Expression]] + indices = None # type: List[Expression] + + def __init__(self, left_expr: Expression, indices: List[Expression], + sequences: List[Expression], condlists: List[List[Expression]]) -> None: + self.left_expr = left_expr + self.sequences = sequences + self.condlists = condlists + self.indices = indices + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_generator_expr(self) + + +class ListComprehension(Expression): + """List comprehension (e.g. [x + 1 for x in a])""" + + generator = None # type: GeneratorExpr + + def __init__(self, generator: GeneratorExpr) -> None: + self.generator = generator + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_list_comprehension(self) + + +class SetComprehension(Expression): + """Set comprehension (e.g. {x + 1 for x in a})""" + + generator = None # type: GeneratorExpr + + def __init__(self, generator: GeneratorExpr) -> None: + self.generator = generator + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_set_comprehension(self) + + +class DictionaryComprehension(Expression): + """Dictionary comprehension (e.g. {k: v for k, v in a}""" + + key = None # type: Expression + value = None # type: Expression + sequences_expr = None # type: List[Expression] + condlists = None # type: List[List[Expression]] + indices = None # type: List[Expression] + + def __init__(self, key: Expression, value: Expression, indices: List[Expression], + sequences: List[Expression], condlists: List[List[Expression]]) -> None: + self.key = key + self.value = value + self.sequences = sequences + self.condlists = condlists + self.indices = indices + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_dictionary_comprehension(self) + + +class ConditionalExpr(Expression): + """Conditional expression (e.g. x if y else z)""" + + cond = None # type: Expression + if_expr = None # type: Expression + else_expr = None # type: Expression + + def __init__(self, cond: Expression, if_expr: Expression, else_expr: Expression) -> None: + self.cond = cond + self.if_expr = if_expr + self.else_expr = else_expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_conditional_expr(self) + + +class BackquoteExpr(Expression): + """Python 2 expression `...`.""" + + expr = None # type: Expression + + def __init__(self, expr: Expression) -> None: + self.expr = expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_backquote_expr(self) + + +class TypeApplication(Expression): + """Type application expr[type, ...]""" + + expr = None # type: Expression + types = None # type: List[mypy.types.Type] + + def __init__(self, expr: Expression, types: List['mypy.types.Type']) -> None: + self.expr = expr + self.types = types + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_type_application(self) + + +# Variance of a type variable. For example, T in the definition of +# List[T] is invariant, so List[int] is not a subtype of List[object], +# and also List[object] is not a subtype of List[int]. +# +# The T in Iterable[T] is covariant, so Iterable[int] is a subtype of +# Iterable[object], but not vice versa. +# +# If T is contravariant in Foo[T], Foo[object] is a subtype of +# Foo[int], but not vice versa. +INVARIANT = 0 # type: int +COVARIANT = 1 # type: int +CONTRAVARIANT = 2 # type: int + + +class TypeVarExpr(SymbolNode, Expression): + """Type variable expression TypeVar(...).""" + + _name = '' + _fullname = '' + # Value restriction: only types in the list are valid as values. If the + # list is empty, there is no restriction. + values = None # type: List[mypy.types.Type] + # Upper bound: only subtypes of upper_bound are valid as values. By default + # this is 'object', meaning no restriction. + upper_bound = None # type: mypy.types.Type + # Variance of the type variable. Invariant is the default. + # TypeVar(..., covariant=True) defines a covariant type variable. + # TypeVar(..., contravariant=True) defines a contravariant type + # variable. + variance = INVARIANT + + def __init__(self, name: str, fullname: str, + values: List['mypy.types.Type'], + upper_bound: 'mypy.types.Type', + variance: int=INVARIANT) -> None: + self._name = name + self._fullname = fullname + self.values = values + self.upper_bound = upper_bound + self.variance = variance + + def name(self) -> str: + return self._name + + def fullname(self) -> str: + return self._fullname + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_type_var_expr(self) + + def serialize(self) -> JsonDict: + return {'.class': 'TypeVarExpr', + 'name': self._name, + 'fullname': self._fullname, + 'values': [t.serialize() for t in self.values], + 'upper_bound': self.upper_bound.serialize(), + 'variance': self.variance, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'TypeVarExpr': + assert data['.class'] == 'TypeVarExpr' + return TypeVarExpr(data['name'], + data['fullname'], + [mypy.types.Type.deserialize(v) for v in data['values']], + mypy.types.Type.deserialize(data['upper_bound']), + data['variance']) + + +class TypeAliasExpr(Expression): + """Type alias expression (rvalue).""" + + type = None # type: mypy.types.Type + + def __init__(self, type: 'mypy.types.Type') -> None: + self.type = type + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_type_alias_expr(self) + + +class NamedTupleExpr(Expression): + """Named tuple expression namedtuple(...).""" + + # The class representation of this named tuple (its tuple_type attribute contains + # the tuple item types) + info = None # type: TypeInfo + + def __init__(self, info: 'TypeInfo') -> None: + self.info = info + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_namedtuple_expr(self) + + +class PromoteExpr(Expression): + """Ducktype class decorator expression _promote(...).""" + + type = None # type: mypy.types.Type + + def __init__(self, type: 'mypy.types.Type') -> None: + self.type = type + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit__promote_expr(self) + + +# Constants + + +class TempNode(Expression): + """Temporary dummy node used during type checking. + + This node is not present in the original program; it is just an artifact + of the type checker implementation. It only represents an opaque node with + some fixed type. + """ + + type = None # type: mypy.types.Type + + def __init__(self, typ: 'mypy.types.Type') -> None: + self.type = typ + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_temp_node(self) + + +class TypeInfo(SymbolNode): + """The type structure of a single class. + + Each TypeInfo corresponds one-to-one to a ClassDef, which + represents the AST of the class. + + In type-theory terms, this is a "type constructor", and if the + class is generic then it will be a type constructor of higher kind. + Where the class is used in an actual type, it's in the form of an + Instance, which amounts to a type application of the tycon to + the appropriate number of arguments. + """ + + _fullname = None # type: str # Fully qualified name + defn = None # type: ClassDef # Corresponding ClassDef + # Method Resolution Order: the order of looking up attributes. The first + # value always to refers to this class. + mro = None # type: List[TypeInfo] + subtypes = None # type: Set[TypeInfo] # Direct subclasses encountered so far + names = None # type: SymbolTable # Names defined directly in this type + is_abstract = False # Does the class have any abstract attributes? + abstract_attributes = None # type: List[str] + # Classes inheriting from Enum shadow their true members with a __getattr__, so we + # have to treat them as a special case. + is_enum = False + # If true, any unknown attributes should have type 'Any' instead + # of generating a type error. This would be true if there is a + # base class with type 'Any', but other use cases may be + # possible. This is similar to having __getattr__ that returns Any + # (and __setattr__), but without the __getattr__ method. + fallback_to_any = False + + # Information related to type annotations. + + # Generic type variable names + type_vars = None # type: List[str] + + # Direct base classes. + bases = None # type: List[mypy.types.Instance] + + # Another type which this type will be treated as a subtype of, + # even though it's not a subclass in Python. The non-standard + # `@_promote` decorator introduces this, and there are also + # several builtin examples, in particular `int` -> `float`. + _promote = None # type: mypy.types.Type + + # Representation of a Tuple[...] base class, if the class has any + # (e.g., for named tuples). If this is not None, the actual Type + # object used for this class is not an Instance but a TupleType; + # the corresponding Instance is set as the fallback type of the + # tuple type. + tuple_type = None # type: mypy.types.TupleType + + # Is this a named tuple type? + is_named_tuple = False + + # Is this a dummy from deserialization? + is_dummy = False + + # Alternative to fullname() for 'anonymous' classes. + alt_fullname = None # type: Optional[str] + + def __init__(self, names: 'SymbolTable', defn: ClassDef) -> None: + """Initialize a TypeInfo.""" + self.names = names + self.defn = defn + self.subtypes = set() + self.type_vars = [] + self.bases = [] + # Leave self.mro uninitialized until we compute it for real, + # so we don't accidentally try to use it prematurely. + self._fullname = defn.fullname + self.is_abstract = False + self.abstract_attributes = [] + if defn.type_vars: + for vd in defn.type_vars: + self.type_vars.append(vd.name) + + def name(self) -> str: + """Short name.""" + return self.defn.name + + def fullname(self) -> str: + return self._fullname + + def is_generic(self) -> bool: + """Is the type generic (i.e. does it have type variables)?""" + return len(self.type_vars) > 0 + + def get(self, name: str) -> 'SymbolTableNode': + for cls in self.mro: + n = cls.names.get(name) + if n: + return n + return None + + def __getitem__(self, name: str) -> 'SymbolTableNode': + n = self.get(name) + if n: + return n + else: + raise KeyError(name) + + def __repr__(self) -> str: + return '' % self.fullname() + + # IDEA: Refactor the has* methods to be more consistent and document + # them. + + def has_readable_member(self, name: str) -> bool: + return self.get(name) is not None + + def has_writable_member(self, name: str) -> bool: + return self.has_var(name) + + def has_var(self, name: str) -> bool: + return self.get_var(name) is not None + + def has_method(self, name: str) -> bool: + return self.get_method(name) is not None + + def get_var(self, name: str) -> Var: + for cls in self.mro: + if name in cls.names: + node = cls.names[name].node + if isinstance(node, Var): + return node + else: + return None + return None + + def get_var_or_getter(self, name: str) -> SymbolNode: + # TODO getter + return self.get_var(name) + + def get_var_or_setter(self, name: str) -> SymbolNode: + # TODO setter + return self.get_var(name) + + def get_method(self, name: str) -> FuncBase: + if self.mro is None: # Might be because of a previous error. + return None + for cls in self.mro: + if name in cls.names: + node = cls.names[name].node + if isinstance(node, FuncBase): + return node + else: + return None + return None + + def calculate_mro(self) -> None: + """Calculate and set mro (method resolution order). + + Raise MroError if cannot determine mro. + """ + mro = linearize_hierarchy(self) + assert mro, "Could not produce a MRO at all for %s" % (self,) + self.mro = mro + self.is_enum = self._calculate_is_enum() + + def _calculate_is_enum(self) -> bool: + """ + If this is "enum.Enum" itself, then yes, it's an enum. + If the flag .is_enum has been set on anything in the MRO, it's an enum. + """ + if self.fullname() == ENUM_BASECLASS: + return True + if self.mro: + return any(type_info.is_enum for type_info in self.mro) + return False + + def has_base(self, fullname: str) -> bool: + """Return True if type has a base type with the specified name. + + This can be either via extension or via implementation. + """ + if self.mro: + for cls in self.mro: + if cls.fullname() == fullname: + return True + return False + + def all_subtypes(self) -> 'Set[TypeInfo]': + """Return TypeInfos of all subtypes, including this type, as a set.""" + subtypes = set([self]) + for subt in self.subtypes: + for t in subt.all_subtypes(): + subtypes.add(t) + return subtypes + + def all_base_classes(self) -> 'List[TypeInfo]': + """Return a list of base classes, including indirect bases.""" + assert False + + def direct_base_classes(self) -> 'List[TypeInfo]': + """Return a direct base classes. + + Omit base classes of other base classes. + """ + return [base.type for base in self.bases] + + def __str__(self) -> str: + """Return a string representation of the type. + + This includes the most important information about the type. + """ + base = None # type: str + if self.bases: + base = 'Bases({})'.format(', '.join(str(base) + for base in self.bases)) + return dump_tagged(['Name({})'.format(self.fullname()), + base, + ('Names', sorted(self.names.keys()))], + 'TypeInfo') + + def serialize(self) -> Union[str, JsonDict]: + # NOTE: This is where all ClassDefs originate, so there shouldn't be duplicates. + data = {'.class': 'TypeInfo', + 'fullname': self.fullname(), + 'alt_fullname': self.alt_fullname, + 'names': self.names.serialize(self.alt_fullname or self.fullname()), + 'defn': self.defn.serialize(), + 'is_abstract': self.is_abstract, + 'abstract_attributes': self.abstract_attributes, + 'is_enum': self.is_enum, + 'fallback_to_any': self.fallback_to_any, + 'type_vars': self.type_vars, + 'bases': [b.serialize() for b in self.bases], + '_promote': None if self._promote is None else self._promote.serialize(), + 'tuple_type': None if self.tuple_type is None else self.tuple_type.serialize(), + 'is_named_tuple': self.is_named_tuple, + } + return data + + @classmethod + def deserialize(cls, data: JsonDict) -> 'TypeInfo': + names = SymbolTable.deserialize(data['names']) + defn = ClassDef.deserialize(data['defn']) + ti = TypeInfo(names, defn) + ti._fullname = data['fullname'] + ti.alt_fullname = data['alt_fullname'] + # TODO: Is there a reason to reconstruct ti.subtypes? + ti.is_abstract = data['is_abstract'] + ti.abstract_attributes = data['abstract_attributes'] + ti.is_enum = data['is_enum'] + ti.fallback_to_any = data['fallback_to_any'] + ti.type_vars = data['type_vars'] + ti.bases = [mypy.types.Instance.deserialize(b) for b in data['bases']] + ti._promote = (None if data['_promote'] is None + else mypy.types.Type.deserialize(data['_promote'])) + ti.tuple_type = (None if data['tuple_type'] is None + else mypy.types.TupleType.deserialize(data['tuple_type'])) + ti.is_named_tuple = data['is_named_tuple'] + return ti + + +def NamedTupleTypeInfo(tup: 'mypy.types.NamedTupleType', *args) -> TypeInfo: + info = TypeInfo(*args) + info.tuple_type = tup + info.bases = [tup.fallback] + info.is_named_tuple = True + info.mro = [info] + tup.fallback.type.mro + return info + +class SymbolTableNode: + # Kind of node. Possible values: + # - LDEF: local definition (of any kind) + # - GDEF: global (module-level) definition + # - MDEF: class member definition + # - UNBOUND_TVAR: TypeVar(...) definition, not bound + # - TVAR: type variable in a bound scope (generic function / generic clas) + # - MODULE_REF: reference to a module + # - TYPE_ALIAS: type alias + # - UNBOUND_IMPORTED: temporary kind for imported names + kind = None # type: int + # AST node of definition (FuncDef/Var/TypeInfo/Decorator/TypeVarExpr, + # or None for a bound type variable). + node = None # type: Optional[SymbolNode] + # Type variable definition (for bound type variables only) + tvar_def = None # type: Optional[mypy.types.TypeVarDef] + # Module id (e.g. "foo.bar") or None + mod_id = '' + # If this not None, override the type of the 'node' attribute. + type_override = None # type: Optional[mypy.types.Type] + # If False, this name won't be imported via 'from import *'. + # This has no effect on names within classes. + module_public = True + # For deserialized MODULE_REF nodes, the referenced module name; + # for other nodes, optionally the name of the referenced object. + cross_ref = None # type: Optional[str] + + def __init__(self, kind: int, node: Optional[SymbolNode], mod_id: str = None, + typ: 'mypy.types.Type' = None, + tvar_def: 'mypy.types.TypeVarDef' = None, + module_public: bool = True) -> None: + self.kind = kind + self.node = node + self.type_override = typ + self.mod_id = mod_id + self.tvar_def = tvar_def + self.module_public = module_public + + @property + def fullname(self) -> str: + if self.node is not None: + return self.node.fullname() + else: + return None + + @property + def type(self) -> 'mypy.types.Type': + # IDEA: Get rid of the Any type. + node = self.node # type: Any + if self.type_override is not None: + return self.type_override + elif ((isinstance(node, Var) or isinstance(node, FuncDef)) + and node.type is not None): + return node.type + elif isinstance(node, Decorator): + return node.var.type + else: + return None + + def __str__(self) -> str: + s = '{}/{}'.format(node_kinds[self.kind], short_type(self.node)) + if self.mod_id is not None: + s += ' ({})'.format(self.mod_id) + # Include declared type of variables and functions. + if self.type is not None: + s += ' : {}'.format(self.type) + return s + + def serialize(self, prefix: str, name: str) -> JsonDict: + """Serialize a SymbolTableNode. + + Args: + prefix: full name of the containing module or class; or None + name: name of this object relative to the containing object + """ + data = {'.class': 'SymbolTableNode', + 'kind': node_kinds[self.kind], + } # type: JsonDict + if self.tvar_def: + data['tvar_def'] = self.tvar_def.serialize() + if not self.module_public: + data['module_public'] = False + if self.kind == MODULE_REF: + assert self.node is not None, "Missing module cross ref in %s for %s" % (prefix, name) + data['cross_ref'] = self.node.fullname() + else: + if self.node is not None: + if prefix is not None: + # Check whether this is an alias for another object. + # If the object's canonical full name differs from + # the full name computed from prefix and name, + # it's an alias, and we serialize it as a cross ref. + if isinstance(self.node, TypeInfo): + fullname = self.node.alt_fullname or self.node.fullname() + else: + fullname = self.node.fullname() + if (fullname is not None and '.' in fullname and + fullname != prefix + '.' + name): + data['cross_ref'] = fullname + return data + data['node'] = self.node.serialize() + if self.type_override is not None: + data['type_override'] = self.type_override.serialize() + return data + + @classmethod + def deserialize(cls, data: JsonDict) -> 'SymbolTableNode': + assert data['.class'] == 'SymbolTableNode' + kind = inverse_node_kinds[data['kind']] + if 'cross_ref' in data: + # This will be fixed up later. + stnode = SymbolTableNode(kind, None) + stnode.cross_ref = data['cross_ref'] + else: + node = None + if 'node' in data: + node = SymbolNode.deserialize(data['node']) + typ = None + if 'type_override' in data: + typ = mypy.types.Type.deserialize(data['type_override']) + stnode = SymbolTableNode(kind, node, typ=typ) + if 'tvar_def' in data: + stnode.tvar_def = mypy.types.TypeVarDef.deserialize(data['tvar_def']) + if 'module_public' in data: + stnode.module_public = data['module_public'] + return stnode + + +class SymbolTable(Dict[str, SymbolTableNode]): + def __str__(self) -> str: + a = [] # type: List[str] + for key, value in self.items(): + # Filter out the implicit import of builtins. + if isinstance(value, SymbolTableNode): + if (value.fullname != 'builtins' and + (value.fullname or '').split('.')[-1] not in + implicit_module_attrs): + a.append(' ' + str(key) + ' : ' + str(value)) + else: + a.append(' ') + a = sorted(a) + a.insert(0, 'SymbolTable(') + a[-1] += ')' + return '\n'.join(a) + + def serialize(self, fullname: str) -> JsonDict: + data = {'.class': 'SymbolTable'} # type: JsonDict + for key, value in self.items(): + # Skip __builtins__: it's a reference to the builtins + # module that gets added to every module by + # SemanticAnalyzer.visit_file(), but it shouldn't be + # accessed by users of the module. + if key == '__builtins__': + continue + data[key] = value.serialize(fullname, key) + return data + + @classmethod + def deserialize(cls, data: JsonDict) -> 'SymbolTable': + assert data['.class'] == 'SymbolTable' + st = SymbolTable() + for key, value in data.items(): + if key != '.class': + st[key] = SymbolTableNode.deserialize(value) + return st + + +def function_type(func: FuncBase, fallback: 'mypy.types.Instance') -> 'mypy.types.FunctionLike': + if func.type: + assert isinstance(func.type, mypy.types.FunctionLike) + return func.type + else: + # Implicit type signature with dynamic types. + # Overloaded functions always have a signature, so func must be an ordinary function. + fdef = cast(FuncDef, func) + name = func.name() + if name: + name = '"{}"'.format(name) + names = [] # type: List[str] + for arg in fdef.arguments: + names.append(arg.variable.name()) + + return mypy.types.CallableType( + [mypy.types.AnyType()] * len(fdef.arguments), + [arg.kind for arg in fdef.arguments], + names, + mypy.types.AnyType(), + fallback, + name, + implicit=True, + ) + + +def method_type_with_fallback(func: FuncBase, + fallback: 'mypy.types.Instance') -> 'mypy.types.FunctionLike': + """Return the signature of a method (omit self).""" + return method_type(function_type(func, fallback)) + + +def method_type(sig: 'mypy.types.FunctionLike') -> 'mypy.types.FunctionLike': + if isinstance(sig, mypy.types.CallableType): + return method_callable(sig) + else: + sig = cast(mypy.types.Overloaded, sig) + items = [] # type: List[mypy.types.CallableType] + for c in sig.items(): + items.append(method_callable(c)) + return mypy.types.Overloaded(items) + + +def method_callable(c: 'mypy.types.CallableType') -> 'mypy.types.CallableType': + if c.arg_kinds and c.arg_kinds[0] == ARG_STAR: + # The signature is of the form 'def foo(*args, ...)'. + # In this case we shouldn't drop the first arg, + # since self will be absorbed by the *args. + return c + return c.copy_modified(arg_types=c.arg_types[1:], + arg_kinds=c.arg_kinds[1:], + arg_names=c.arg_names[1:]) + + +class MroError(Exception): + """Raised if a consistent mro cannot be determined for a class.""" + + +def linearize_hierarchy(info: TypeInfo) -> List[TypeInfo]: + # TODO describe + if info.mro: + return info.mro + bases = info.direct_base_classes() + lin_bases = [] + for base in bases: + assert base is not None, "Cannot linearize bases for %s %s" % (info.fullname(), bases) + lin_bases.append(linearize_hierarchy(base)) + lin_bases.append(bases) + return [info] + merge(lin_bases) + + +def merge(seqs: List[List[TypeInfo]]) -> List[TypeInfo]: + seqs = [s[:] for s in seqs] + result = [] # type: List[TypeInfo] + while True: + seqs = [s for s in seqs if s] + if not seqs: + return result + for seq in seqs: + head = seq[0] + if not [s for s in seqs if head in s[1:]]: + break + else: + raise MroError() + result.append(head) + for s in seqs: + if s[0] is head: + del s[0] diff --git a/mypy/parse.py b/mypy/parse.py index 579f03c57525f..a496f32bebcd3 100644 --- a/mypy/parse.py +++ b/mypy/parse.py @@ -1,2034 +1,2034 @@ -"""Mypy parser. - -Constructs a parse tree (abstract syntax tree) based on a string -representing a source file. Performs only minimal semantic checks. -""" - -import re - -from typing import List, Tuple, Any, Set, cast, Union, Optional - -from mypy import lex, docstring -from mypy.lex import ( - Token, Eof, Bom, Break, Name, Colon, Dedent, IntLit, StrLit, BytesLit, - UnicodeLit, FloatLit, Op, Indent, Keyword, Punct, LexError, ComplexLit, - EllipsisToken -) -import mypy.types -from mypy.nodes import ( - MypyFile, Import, Node, ImportAll, ImportFrom, FuncDef, OverloadedFuncDef, - ClassDef, Decorator, Block, Var, OperatorAssignmentStmt, - ExpressionStmt, AssignmentStmt, ReturnStmt, RaiseStmt, AssertStmt, - DelStmt, BreakStmt, ContinueStmt, PassStmt, GlobalDecl, - WhileStmt, ForStmt, IfStmt, TryStmt, WithStmt, - TupleExpr, GeneratorExpr, ListComprehension, ListExpr, ConditionalExpr, - DictExpr, SetExpr, NameExpr, IntExpr, StrExpr, BytesExpr, UnicodeExpr, - FloatExpr, CallExpr, SuperExpr, MemberExpr, IndexExpr, SliceExpr, OpExpr, - UnaryExpr, FuncExpr, PrintStmt, ImportBase, ComparisonExpr, - StarExpr, YieldFromExpr, NonlocalDecl, DictionaryComprehension, - SetComprehension, ComplexExpr, EllipsisExpr, YieldExpr, ExecStmt, Argument, - BackquoteExpr -) -from mypy import defaults -from mypy import nodes -from mypy.errors import Errors, CompileError -from mypy.types import Type, CallableType, AnyType, UnboundType -from mypy.parsetype import ( - parse_type, parse_types, parse_signature, TypeParseError, parse_str_as_signature -) -from mypy.options import Options - -from mypy import experiments - - -precedence = { - '**': 16, - '-u': 15, '+u': 15, '~': 15, # unary operators (-, + and ~) - '': 14, - '*': 13, '/': 13, '//': 13, '%': 13, - '+': 12, '-': 12, - '>>': 11, '<<': 11, - '&': 10, - '^': 9, - '|': 8, - '==': 7, '!=': 7, '<': 7, '>': 7, '<=': 7, '>=': 7, 'is': 7, 'in': 7, - '*u': 7, # unary * for star expressions - 'not': 6, - 'and': 5, - 'or': 4, - '': 3, # conditional expression - '': 2, # list comprehension - ',': 1} - - -op_assign = set([ - '+=', '-=', '*=', '/=', '//=', '%=', '**=', '|=', '&=', '^=', '>>=', - '<<=']) - -op_comp = set([ - '>', '<', '==', '>=', '<=', '<>', '!=', 'is', 'is', 'in', 'not']) - -none = Token('') # Empty token - - -def parse(source: Union[str, bytes], - fnam: str, - errors: Errors, - options: Options) -> MypyFile: - """Parse a source file, without doing any semantic analysis. - - Return the parse tree. If errors is not provided, raise ParseError - on failure. Otherwise, use the errors object to report parse errors. - - The python_version (major, minor) option determines the Python syntax variant. - """ - if options.fast_parser: - import mypy.fastparse - return mypy.fastparse.parse(source, - fnam=fnam, - errors=errors, - pyversion=options.python_version, - custom_typing_module=options.custom_typing_module) - - is_stub_file = bool(fnam) and fnam.endswith('.pyi') - parser = Parser(fnam, - errors, - options.python_version, - options.custom_typing_module, - is_stub_file=is_stub_file) - tree = parser.parse(source) - tree.path = fnam - tree.is_stub = is_stub_file - return tree - - -class Parser: - """Mypy parser that parses a string into an AST. - - Parses type annotations in addition to basic Python syntax. It supports both Python 2 and 3 - (though Python 2 support is incomplete). - - The AST classes are defined in mypy.nodes and mypy.types. - """ - - tok = None # type: List[Token] - ind = 0 - errors = None # type: Errors - # If True, raise an exception on any parse error. Otherwise, errors are reported via 'errors'. - raise_on_error = False - - # Are we currently parsing the body of a class definition? - is_class_body = False - # All import nodes encountered so far in this parse unit. - imports = None # type: List[ImportBase] - # Names imported from __future__. - future_options = None # type: List[str] - # Lines to ignore (using # type: ignore). - ignored_lines = None # type: Set[int] - - def __init__(self, fnam: str, errors: Errors, pyversion: Tuple[int, int], - custom_typing_module: str = None, is_stub_file: bool = False) -> None: - self.raise_on_error = errors is None - self.pyversion = pyversion - self.custom_typing_module = custom_typing_module - self.is_stub_file = is_stub_file - if errors is not None: - self.errors = errors - else: - self.errors = Errors() - if fnam is not None: - self.errors.set_file(fnam) - else: - self.errors.set_file('') - - def parse(self, s: Union[str, bytes]) -> MypyFile: - self.tok, self.ignored_lines = lex.lex(s, pyversion=self.pyversion, - is_stub_file=self.is_stub_file) - self.ind = 0 - self.imports = [] - self.future_options = [] - file = self.parse_file() - if self.raise_on_error and self.errors.is_errors(): - self.errors.raise_error() - return file - - def weak_opts(self) -> Set[str]: - """Do weak typing if any of the first ten tokens is a comment saying so. - - The comment can be one of: - # mypy: weak=global - # mypy: weak=local - # mypy: weak <- defaults to local - """ - regexp = re.compile(r'^[\s]*# *mypy: *weak(=?)([^\s]*)', re.M) - for t in self.tok[:10]: - for s in [t.string, t.pre]: - m = regexp.search(s) - if m: - opts = set(x for x in m.group(2).split(',') if x) - if not opts: - opts.add('local') - return opts - return set() - - def parse_file(self) -> MypyFile: - """Parse a mypy source file.""" - is_bom = self.parse_bom() - defs = self.parse_defs() - weak_opts = self.weak_opts() - self.expect_type(Eof) - node = MypyFile(defs, self.imports, is_bom, self.ignored_lines, - weak_opts=weak_opts) - return node - - # Parse the initial part - - def parse_bom(self) -> bool: - """Parse the optional byte order mark at the beginning of a file.""" - if isinstance(self.current(), Bom): - self.expect_type(Bom) - if isinstance(self.current(), Break): - self.expect_break() - return True - else: - return False - - def parse_import(self) -> Import: - self.expect('import') - ids = [] - while True: - id = self.parse_qualified_name() - translated = self.translate_module_id(id) - as_id = None - if self.current_str() == 'as': - self.expect('as') - name_tok = self.expect_type(Name) - as_id = name_tok.string - elif translated != id: - as_id = id - ids.append((translated, as_id)) - if self.current_str() != ',': - break - self.expect(',') - node = Import(ids) - self.imports.append(node) - return node - - def translate_module_id(self, id: str) -> str: - """Return the actual, internal module id for a source text id. - - For example, translate '__builtin__' in Python 2 to 'builtins'. - """ - if id == self.custom_typing_module: - return 'typing' - elif id == '__builtin__' and self.pyversion[0] == 2: - # HACK: __builtin__ in Python 2 is aliases to builtins. However, the implementation - # is named __builtin__.py (there is another layer of translation elsewhere). - return 'builtins' - return id - - def parse_import_from(self) -> Node: - self.expect('from') - - # Build the list of beginning relative tokens. - relative = 0 - while self.current_str() in (".", "..."): - relative += len(self.current_str()) - self.skip() - - # Parse qualified name to actually import from. - if self.current_str() == "import": - # Empty/default values. - name = "" - else: - name = self.parse_qualified_name() - - name = self.translate_module_id(name) - - # Parse import list - self.expect('import') - node = None # type: ImportBase - if self.current_str() == '*': - if name == '__future__': - self.parse_error() - # An import all from a module node: - self.skip() - node = ImportAll(name, relative) - else: - is_paren = self.current_str() == '(' - if is_paren: - self.expect('(') - targets = [] # type: List[Tuple[str, str]] - while True: - id, as_id = self.parse_import_name() - if '%s.%s' % (name, id) == self.custom_typing_module: - if targets or self.current_str() == ',': - self.fail('You cannot import any other modules when you ' - 'import a custom typing module', - self.current().line) - node = Import([('typing', as_id)]) - self.skip_until_break() - break - targets.append((id, as_id)) - if self.current_str() != ',': - break - self.expect(',') - if is_paren and self.current_str() == ')': - break - if is_paren: - self.expect(')') - if node is None: - node = ImportFrom(name, relative, targets) - self.imports.append(node) - if name == '__future__': - self.future_options.extend(target[0] for target in targets) - return node - - def parse_import_name(self) -> Tuple[str, Optional[str]]: - tok = self.expect_type(Name) - name = tok.string - if self.current_str() == 'as': - self.skip() - as_name = self.expect_type(Name) - return name, as_name.string - else: - return name, None - - def parse_qualified_name(self) -> str: - """Parse a name with an optional module qualifier. - - Return a tuple with the name as a string and a token array - containing all the components of the name. - """ - tok = self.expect_type(Name) - n = tok.string - while self.current_str() == '.': - self.expect('.') - tok = self.expect_type(Name) - n += '.' + tok.string - return n - - # Parsing global definitions - - def parse_defs(self) -> List[Node]: - defs = [] # type: List[Node] - while not self.eof(): - try: - defn, is_simple = self.parse_statement() - if is_simple: - self.expect_break() - if defn is not None: - if not self.try_combine_overloads(defn, defs): - defs.append(defn) - except ParseError: - pass - return defs - - def parse_class_def(self) -> ClassDef: - old_is_class_body = self.is_class_body - self.is_class_body = True - - self.expect('class') - metaclass = None - - try: - base_types = [] # type: List[Node] - try: - name_tok = self.expect_type(Name) - name = name_tok.string - - self.errors.push_type(name) - - if self.current_str() == '(': - self.skip() - while True: - if self.current_str() == ')': - break - if self.peek().string == '=': - metaclass = self.parse_class_keywords() - break - base_types.append(self.parse_super_type()) - if self.current_str() != ',': - break - self.skip() - self.expect(')') - except ParseError: - pass - - defs, _ = self.parse_block() - - node = ClassDef(name, defs, None, base_types, metaclass=metaclass) - return node - finally: - self.errors.pop_type() - self.is_class_body = old_is_class_body - - def parse_class_keywords(self) -> Optional[str]: - """Parse the class keyword arguments, keeping the metaclass but - ignoring all others. Returns None if the metaclass isn't found. - """ - metaclass = None - while True: - key = self.expect_type(Name) - self.expect('=') - if key.string == 'metaclass': - metaclass = self.parse_qualified_name() - else: - # skip the class value - self.parse_expression(precedence[',']) - if self.current_str() != ',': - break - self.skip() - if self.current_str() == ')': - break - return metaclass - - def parse_super_type(self) -> Node: - return self.parse_expression(precedence[',']) - - def parse_decorated_function_or_class(self) -> Node: - decorators = [] - no_type_checks = False - while self.current_str() == '@': - self.expect('@') - d_exp = self.parse_expression() - if self.is_no_type_check_decorator(d_exp): - no_type_checks = True - decorators.append(d_exp) - self.expect_break() - if self.current_str() != 'class': - func = self.parse_function(no_type_checks) - func.is_decorated = True - var = Var(func.name()) - # Types of decorated functions must always be inferred. - var.is_ready = False - var.set_line(decorators[0].line) - node = Decorator(func, decorators, var) - return node - else: - cls = self.parse_class_def() - cls.decorators = decorators - return cls - - def is_no_type_check_decorator(self, expr: Node) -> bool: - if isinstance(expr, NameExpr): - return expr.name == 'no_type_check' - elif isinstance(expr, MemberExpr): - if isinstance(expr.expr, NameExpr): - return expr.expr.name == 'typing' and expr.name == 'no_type_check' - else: - return False - - def parse_function(self, no_type_checks: bool=False) -> FuncDef: - def_tok = self.expect('def') - is_method = self.is_class_body - self.is_class_body = False - try: - (name, args, typ, is_error, extra_stmts) = self.parse_function_header(no_type_checks) - - arg_kinds = [arg.kind for arg in args] - arg_names = [arg.variable.name() for arg in args] - - body, comment_type = self.parse_block(allow_type=True) - # Potentially insert extra assignment statements to the beginning of the - # body, used to decompose Python 2 tuple arguments. - body.body[:0] = extra_stmts - if comment_type: - # The function has a # type: ... signature. - if typ: - self.errors.report( - def_tok.line, 'Function has duplicate type signatures') - sig = cast(CallableType, comment_type) - if sig.is_ellipsis_args: - # When we encounter an ellipsis, fill in the arg_types with - # a bunch of AnyTypes, emulating Callable[..., T] - arg_types = [AnyType()] * len(arg_kinds) # type: List[Type] - typ = CallableType( - arg_types, - arg_kinds, - arg_names, - sig.ret_type, - None, - line=def_tok.line) - elif is_method and len(sig.arg_kinds) < len(arg_kinds): - self.check_argument_kinds(arg_kinds, - [nodes.ARG_POS] + sig.arg_kinds, - def_tok.line) - # Add implicit 'self' argument to signature. - first_arg = [AnyType()] # type: List[Type] - typ = CallableType( - first_arg + sig.arg_types, - arg_kinds, - arg_names, - sig.ret_type, - None, - line=def_tok.line) - else: - self.check_argument_kinds(arg_kinds, sig.arg_kinds, - def_tok.line) - typ = CallableType( - sig.arg_types, - arg_kinds, - arg_names, - sig.ret_type, - None, - line=def_tok.line) - - # If there was a serious error, we really cannot build a parse tree - # node. - if is_error: - return None - - if typ and isinstance(typ.ret_type, UnboundType): - typ.ret_type.is_ret_type = True - - node = FuncDef(name, args, body, typ) - node.set_line(def_tok) - if typ is not None: - typ.definition = node - return node - finally: - self.errors.pop_function() - self.is_class_body = is_method - - def check_argument_kinds(self, funckinds: List[int], sigkinds: List[int], - line: int) -> None: - """Check that arguments are consistent. - - This verifies that they have the same number and the kinds correspond. - - Arguments: - funckinds: kinds of arguments in function definition - sigkinds: kinds of arguments in signature (after # type:) - """ - if len(funckinds) != len(sigkinds): - if len(funckinds) > len(sigkinds): - self.fail("Type signature has too few arguments", line) - else: - self.fail("Type signature has too many arguments", line) - return - for kind, token in [(nodes.ARG_STAR, '*'), - (nodes.ARG_STAR2, '**')]: - if ((funckinds.count(kind) != sigkinds.count(kind)) or - (kind in funckinds and sigkinds.index(kind) != funckinds.index(kind))): - self.fail( - "Inconsistent use of '{}' in function " - "signature".format(token), line) - - def parse_function_header( - self, no_type_checks: bool=False) -> Tuple[str, - List[Argument], - CallableType, - bool, - List[AssignmentStmt]]: - """Parse function header (a name followed by arguments) - - Return a 5-tuple with the following items: - name - arguments - signature (annotation) - error flag (True if error) - extra statements needed to decompose arguments (usually empty) - - See parse_arg_list for an explanation of the final tuple item. - """ - name = '' - - try: - name_tok = self.expect_type(Name) - name = name_tok.string - - self.errors.push_function(name) - - args, typ, extra_stmts = self.parse_args(no_type_checks) - except ParseError: - if not isinstance(self.current(), Break): - self.ind -= 1 # Kludge: go back to the Break token - # Resynchronise parsing by going back over :, if present. - if isinstance(self.tok[self.ind - 1], Colon): - self.ind -= 1 - return (name, [], None, True, []) - - return (name, args, typ, False, extra_stmts) - - def parse_args(self, no_type_checks: bool=False) -> Tuple[List[Argument], - CallableType, - List[AssignmentStmt]]: - """Parse a function signature (...) [-> t]. - - See parse_arg_list for an explanation of the final tuple item. - """ - lparen = self.expect('(') - - # Parse the argument list (everything within '(' and ')'). - args, extra_stmts = self.parse_arg_list(no_type_checks=no_type_checks) - - self.expect(')') - - if self.current_str() == '->': - self.skip() - if no_type_checks: - self.parse_expression() - ret_type = None - else: - ret_type = self.parse_type() - else: - ret_type = None - - arg_kinds = [arg.kind for arg in args] - self.verify_argument_kinds(arg_kinds, lparen.line) - - annotation = self.build_func_annotation( - ret_type, args, lparen.line) - - return args, annotation, extra_stmts - - def build_func_annotation(self, ret_type: Type, args: List[Argument], - line: int, is_default_ret: bool = False) -> CallableType: - arg_types = [arg.type_annotation for arg in args] - # Are there any type annotations? - if ((ret_type and not is_default_ret) - or arg_types != [None] * len(arg_types)): - # Yes. Construct a type for the function signature. - return self.construct_function_type(args, ret_type, line) - else: - return None - - def parse_arg_list(self, allow_signature: bool = True, - no_type_checks: bool=False) -> Tuple[List[Argument], - List[AssignmentStmt]]: - """Parse function definition argument list. - - This includes everything between '(' and ')' (but not the - parentheses). - - Return tuple (arguments, - extra statements for decomposing arguments). - - The final argument is only used for Python 2 argument lists with - tuples; they contain destructuring assignment statements used to - decompose tuple arguments. For example, consider a header like this: - - . def f((x, y)) - - The actual (sole) argument will be __tuple_arg_1 (a generated - name), whereas the extra statement list will contain a single - assignment statement corresponding to this assignment: - - x, y = __tuple_arg_1 - """ - args = [] # type: List[Argument] - extra_stmts = [] - # This is for checking duplicate argument names. - arg_names = [] # type: List[str] - has_tuple_arg = False - - require_named = False - bare_asterisk_before = -1 - - if self.current_str() != ')' and self.current_str() != ':': - while self.current_str() != ')': - if self.current_str() == '*' and self.peek().string == ',': - self.expect('*') - require_named = True - bare_asterisk_before = len(args) - elif self.current_str() in ['*', '**']: - if bare_asterisk_before == len(args): - # named arguments must follow bare * - self.parse_error() - - arg = self.parse_asterisk_arg( - allow_signature, - no_type_checks, - ) - args.append(arg) - require_named = True - elif self.current_str() == '(': - arg, extra_stmt, names = self.parse_tuple_arg(len(args)) - args.append(arg) - if extra_stmt is not None: - extra_stmts.append(extra_stmt) - has_tuple_arg = True - arg_names.extend(names) - else: - arg, require_named = self.parse_normal_arg( - require_named, - allow_signature, - no_type_checks, - ) - args.append(arg) - arg_names.append(arg.variable.name()) - - if self.current().string != ',': - break - - self.expect(',') - - # Non-tuple argument dupes will be checked elsewhere. Avoid - # generating duplicate errors. - if has_tuple_arg: - self.check_duplicate_argument_names(arg_names) - - return args, extra_stmts - - def check_duplicate_argument_names(self, names: List[str]) -> None: - found = set() # type: Set[str] - for name in names: - if name in found: - self.fail('Duplicate argument name "{}"'.format(name), - self.current().line) - found.add(name) - - def parse_asterisk_arg(self, - allow_signature: bool, - no_type_checks: bool) -> Argument: - asterisk = self.skip() - name = self.expect_type(Name) - variable = Var(name.string) - if asterisk.string == '*': - kind = nodes.ARG_STAR - else: - kind = nodes.ARG_STAR2 - - type = None - if no_type_checks: - self.parse_parameter_annotation() - else: - type = self.parse_arg_type(allow_signature) - - return Argument(variable, type, None, kind) - - def parse_tuple_arg(self, index: int) -> Tuple[Argument, AssignmentStmt, List[str]]: - """Parse a single Python 2 tuple argument. - - Example: def f(x, (y, z)): ... - - The tuple arguments gets transformed into an assignment in the - function body (the second return value). - - Return tuple (argument, decomposing assignment, list of names defined). - - Special case: if the argument is just (x) then it's not a tuple; - we indicate this by returning (argument, None, ['x']). - However, if the argument is (x,) then it *is* a (singleton) tuple. - """ - line = self.current().line - # Generate a new argument name that is very unlikely to clash with anything. - arg_name = '__tuple_arg_{}'.format(index + 1) - if self.pyversion[0] >= 3: - self.fail('Tuples in argument lists only supported in Python 2 mode', line) - paren_arg = self.parse_parentheses() - self.verify_tuple_arg(paren_arg) - if isinstance(paren_arg, NameExpr): - # This isn't a tuple. Revert to a normal argument. - arg_name = paren_arg.name - decompose = None - else: - rvalue = NameExpr(arg_name) - rvalue.set_line(line) - decompose = AssignmentStmt([paren_arg], rvalue) - decompose.set_line(line) - kind = nodes.ARG_POS - initializer = None - if self.current_str() == '=': - self.expect('=') - initializer = self.parse_expression(precedence[',']) - kind = nodes.ARG_OPT - var = Var(arg_name) - arg_names = self.find_tuple_arg_argument_names(paren_arg) - return Argument(var, None, initializer, kind), decompose, arg_names - - def verify_tuple_arg(self, paren_arg: Node) -> None: - if isinstance(paren_arg, TupleExpr): - if not paren_arg.items: - self.fail('Empty tuple not valid as an argument', paren_arg.line) - for item in paren_arg.items: - self.verify_tuple_arg(item) - elif not isinstance(paren_arg, NameExpr): - self.fail('Invalid item in tuple argument', paren_arg.line) - - def find_tuple_arg_argument_names(self, node: Node) -> List[str]: - result = [] # type: List[str] - if isinstance(node, TupleExpr): - for item in node.items: - result.extend(self.find_tuple_arg_argument_names(item)) - elif isinstance(node, NameExpr): - result.append(node.name) - return result - - def parse_normal_arg(self, require_named: bool, - allow_signature: bool, - no_type_checks: bool) -> Tuple[Argument, bool]: - name = self.expect_type(Name) - variable = Var(name.string) - - type = None - if no_type_checks: - self.parse_parameter_annotation() - else: - type = self.parse_arg_type(allow_signature) - - initializer = None # type: Node - if self.current_str() == '=': - self.expect('=') - initializer = self.parse_expression(precedence[',']) - if require_named: - kind = nodes.ARG_NAMED - else: - kind = nodes.ARG_OPT - else: - if require_named: - kind = nodes.ARG_NAMED - else: - kind = nodes.ARG_POS - - self.set_type_optional(type, initializer) - - return Argument(variable, type, initializer, kind), require_named - - def set_type_optional(self, type: Type, initializer: Node) -> None: - if not experiments.STRICT_OPTIONAL: - return - # Indicate that type should be wrapped in an Optional if arg is initialized to None. - optional = isinstance(initializer, NameExpr) and initializer.name == 'None' - if isinstance(type, UnboundType): - type.optional = optional - - def parse_parameter_annotation(self) -> Node: - if self.current_str() == ':': - self.skip() - return self.parse_expression(precedence[',']) - - def parse_arg_type(self, allow_signature: bool) -> Type: - if self.current_str() == ':' and allow_signature: - self.skip() - return self.parse_type() - else: - return None - - def verify_argument_kinds(self, kinds: List[int], line: int) -> None: - found = set() # type: Set[int] - for i, kind in enumerate(kinds): - if kind == nodes.ARG_POS and found & set([nodes.ARG_OPT, - nodes.ARG_STAR, - nodes.ARG_STAR2]): - self.fail('Invalid argument list', line) - elif kind == nodes.ARG_STAR and nodes.ARG_STAR in found: - self.fail('Invalid argument list', line) - elif kind == nodes.ARG_STAR2 and i != len(kinds) - 1: - self.fail('Invalid argument list', line) - found.add(kind) - - def construct_function_type(self, args: List[Argument], ret_type: Type, - line: int) -> CallableType: - # Complete the type annotation by replacing omitted types with 'Any'. - arg_types = [arg.type_annotation for arg in args] - for i in range(len(arg_types)): - if arg_types[i] is None: - arg_types[i] = AnyType(implicit=True) - if ret_type is None: - ret_type = AnyType(implicit=True) - arg_kinds = [arg.kind for arg in args] - arg_names = [arg.variable.name() for arg in args] - return CallableType(arg_types, arg_kinds, arg_names, ret_type, None, name=None, - variables=None, line=line) - - # Parsing statements - - def parse_block(self, allow_type: bool = False) -> Tuple[Block, Type]: - colon = self.expect(':') - if not isinstance(self.current(), Break): - # Block immediately after ':'. - nodes = [] - while True: - ind = self.ind - stmt, is_simple = self.parse_statement() - if not is_simple: - self.parse_error_at(self.tok[ind]) - break - nodes.append(stmt) - brk = self.expect_break() - if brk.string != ';': - break - node = Block(nodes) - node.set_line(colon) - return node, None - else: - # Indented block. - brk = self.expect_break() - type = self.parse_type_comment(brk, signature=True) - self.expect_indent() - stmt_list = [] # type: List[Node] - if allow_type: - cur = self.current() - if type is None and isinstance(cur, StrLit): - ds = docstring.parse_docstring(cur.parsed()) - if ds and False: # TODO: Enable when this is working. - try: - type = parse_str_as_signature(ds.as_type_str(), cur.line) - except TypeParseError: - # We don't require docstrings to be actually correct. - # TODO: Report something here. - type = None - while (not isinstance(self.current(), Dedent) and - not isinstance(self.current(), Eof)): - try: - stmt, is_simple = self.parse_statement() - if is_simple: - self.expect_break() - if stmt is not None: - if not self.try_combine_overloads(stmt, stmt_list): - stmt_list.append(stmt) - except ParseError: - pass - if isinstance(self.current(), Dedent): - self.skip() - node = Block(stmt_list) - node.set_line(colon) - return node, type - - def try_combine_overloads(self, s: Node, stmt: List[Node]) -> bool: - if isinstance(s, Decorator) and stmt: - fdef = s - n = fdef.func.name() - if isinstance(stmt[-1], Decorator) and stmt[-1].func.name() == n: - stmt[-1] = OverloadedFuncDef([stmt[-1], fdef]) - return True - elif isinstance(stmt[-1], OverloadedFuncDef) and stmt[-1].name() == n: - stmt[-1].items.append(fdef) - return True - return False - - def parse_statement(self) -> Tuple[Node, bool]: - stmt = None # type: Node - t = self.current() - ts = self.current_str() - is_simple = True # Is this a non-block statement? - if ts == 'if': - stmt = self.parse_if_stmt() - is_simple = False - elif ts == 'def': - stmt = self.parse_function() - is_simple = False - elif ts == 'while': - stmt = self.parse_while_stmt() - is_simple = False - elif ts == 'return': - stmt = self.parse_return_stmt() - elif ts == 'for': - stmt = self.parse_for_stmt() - is_simple = False - elif ts == 'try': - stmt = self.parse_try_stmt() - is_simple = False - elif ts == 'break': - stmt = self.parse_break_stmt() - elif ts == 'continue': - stmt = self.parse_continue_stmt() - elif ts == 'pass': - stmt = self.parse_pass_stmt() - elif ts == 'raise': - stmt = self.parse_raise_stmt() - elif ts == 'import': - stmt = self.parse_import() - elif ts == 'from': - stmt = self.parse_import_from() - elif ts == 'class': - stmt = self.parse_class_def() - is_simple = False - elif ts == 'global': - stmt = self.parse_global_decl() - elif ts == 'nonlocal' and self.pyversion[0] >= 3: - stmt = self.parse_nonlocal_decl() - elif ts == 'assert': - stmt = self.parse_assert_stmt() - elif ts == 'del': - stmt = self.parse_del_stmt() - elif ts == 'with': - stmt = self.parse_with_stmt() - is_simple = False - elif ts == '@': - stmt = self.parse_decorated_function_or_class() - is_simple = False - elif ts == 'print' and (self.pyversion[0] == 2 and - 'print_function' not in self.future_options): - stmt = self.parse_print_stmt() - elif ts == 'exec' and self.pyversion[0] == 2: - stmt = self.parse_exec_stmt() - else: - stmt = self.parse_expression_or_assignment() - if stmt is not None: - stmt.set_line(t) - return stmt, is_simple - - def parse_expression_or_assignment(self) -> Node: - expr = self.parse_expression(star_expr_allowed=True) - if self.current_str() == '=': - return self.parse_assignment(expr) - elif self.current_str() in op_assign: - # Operator assignment statement. - op = self.current_str()[:-1] - self.skip() - rvalue = self.parse_expression() - return OperatorAssignmentStmt(op, expr, rvalue) - else: - # Expression statement. - return ExpressionStmt(expr) - - def parse_assignment(self, lvalue: Any) -> Node: - """Parse an assignment statement. - - Assume that lvalue has been parsed already, and the current token is '='. - Also parse an optional '# type:' comment. - """ - self.expect('=') - lvalues = [lvalue] - expr = self.parse_expression(star_expr_allowed=True) - while self.current_str() == '=': - self.skip() - lvalues.append(expr) - expr = self.parse_expression(star_expr_allowed=True) - cur = self.current() - if isinstance(cur, Break): - type = self.parse_type_comment(cur, signature=False) - else: - type = None - return AssignmentStmt(lvalues, expr, type) - - def parse_return_stmt(self) -> ReturnStmt: - self.expect('return') - expr = None - current = self.current() - if current.string == 'yield': - self.parse_error() - if not isinstance(current, Break): - expr = self.parse_expression() - node = ReturnStmt(expr) - return node - - def parse_raise_stmt(self) -> RaiseStmt: - self.expect('raise') - expr = None - from_expr = None - if not isinstance(self.current(), Break): - expr = self.parse_expression() - if self.current_str() == 'from': - self.expect('from') - from_expr = self.parse_expression() - node = RaiseStmt(expr, from_expr) - return node - - def parse_assert_stmt(self) -> AssertStmt: - self.expect('assert') - expr = self.parse_expression() - node = AssertStmt(expr) - return node - - def parse_yield_or_yield_from_expr(self) -> Union[YieldFromExpr, YieldExpr]: - self.expect("yield") - expr = None - node = YieldExpr(expr) # type: Union[YieldFromExpr, YieldExpr] - if not isinstance(self.current(), Break): - if self.current_str() == "from": - self.expect("from") - expr = self.parse_expression() # when yield from is assigned to a variable - node = YieldFromExpr(expr) - else: - if self.current_str() == ')': - node = YieldExpr(None) - else: - expr = self.parse_expression() - node = YieldExpr(expr) - return node - - def parse_ellipsis(self) -> EllipsisExpr: - self.expect('...') - node = EllipsisExpr() - return node - - def parse_del_stmt(self) -> DelStmt: - self.expect('del') - expr = self.parse_expression() - node = DelStmt(expr) - return node - - def parse_break_stmt(self) -> BreakStmt: - self.expect('break') - node = BreakStmt() - return node - - def parse_continue_stmt(self) -> ContinueStmt: - self.expect('continue') - node = ContinueStmt() - return node - - def parse_pass_stmt(self) -> PassStmt: - self.expect('pass') - node = PassStmt() - return node - - def parse_global_decl(self) -> GlobalDecl: - self.expect('global') - names = self.parse_identifier_list() - node = GlobalDecl(names) - return node - - def parse_nonlocal_decl(self) -> NonlocalDecl: - self.expect('nonlocal') - names = self.parse_identifier_list() - node = NonlocalDecl(names) - return node - - def parse_identifier_list(self) -> List[str]: - names = [] - while True: - n = self.expect_type(Name) - names.append(n.string) - if self.current_str() != ',': - break - self.skip() - return names - - def parse_while_stmt(self) -> WhileStmt: - is_error = False - self.expect('while') - try: - expr = self.parse_expression() - except ParseError: - is_error = True - body, _ = self.parse_block() - if self.current_str() == 'else': - self.expect('else') - else_body, _ = self.parse_block() - else: - else_body = None - if is_error is not None: - node = WhileStmt(expr, body, else_body) - return node - else: - return None - - def parse_for_stmt(self) -> ForStmt: - self.expect('for') - index = self.parse_for_index_variables() - self.expect('in') - expr = self.parse_expression() - - body, _ = self.parse_block() - - if self.current_str() == 'else': - self.expect('else') - else_body, _ = self.parse_block() - else: - else_body = None - - node = ForStmt(index, expr, body, else_body) - return node - - def parse_for_index_variables(self) -> Node: - # Parse index variables of a 'for' statement. - index_items = [] - force_tuple = False - - while True: - v = self.parse_expression(precedence['in'], - star_expr_allowed=True) # Prevent parsing of for stmt 'in' - index_items.append(v) - if self.current_str() != ',': - break - self.skip() - if self.current_str() == 'in': - force_tuple = True - break - - if len(index_items) == 1 and not force_tuple: - index = index_items[0] - else: - index = TupleExpr(index_items) - index.set_line(index_items[0].get_line()) - - return index - - def parse_if_stmt(self) -> IfStmt: - is_error = False - - self.expect('if') - expr = [] - try: - expr.append(self.parse_expression()) - except ParseError: - is_error = True - - body = [self.parse_block()[0]] - - while self.current_str() == 'elif': - self.expect('elif') - try: - expr.append(self.parse_expression()) - except ParseError: - is_error = True - body.append(self.parse_block()[0]) - - if self.current_str() == 'else': - self.expect('else') - else_body, _ = self.parse_block() - else: - else_body = None - - if not is_error: - node = IfStmt(expr, body, else_body) - return node - else: - return None - - def parse_try_stmt(self) -> Node: - self.expect('try') - body, _ = self.parse_block() - is_error = False - vars = [] # type: List[NameExpr] - types = [] # type: List[Node] - handlers = [] # type: List[Block] - while self.current_str() == 'except': - self.expect('except') - if not isinstance(self.current(), Colon): - try: - t = self.current() - types.append(self.parse_expression(precedence[',']).set_line(t)) - if self.current_str() == 'as': - self.expect('as') - vars.append(self.parse_name_expr()) - elif self.pyversion[0] == 2 and self.current_str() == ',': - self.expect(',') - vars.append(self.parse_name_expr()) - else: - vars.append(None) - except ParseError: - is_error = True - else: - types.append(None) - vars.append(None) - handlers.append(self.parse_block()[0]) - if not is_error: - if self.current_str() == 'else': - self.skip() - else_body, _ = self.parse_block() - else: - else_body = None - if self.current_str() == 'finally': - self.expect('finally') - finally_body, _ = self.parse_block() - else: - finally_body = None - node = TryStmt(body, vars, types, handlers, else_body, - finally_body) - return node - else: - return None - - def parse_with_stmt(self) -> WithStmt: - self.expect('with') - exprs = [] - targets = [] - while True: - expr = self.parse_expression(precedence[',']) - if self.current_str() == 'as': - self.expect('as') - target = self.parse_expression(precedence[',']) - else: - target = None - exprs.append(expr) - targets.append(target) - if self.current_str() != ',': - break - self.expect(',') - body, _ = self.parse_block() - return WithStmt(exprs, targets, body) - - def parse_print_stmt(self) -> PrintStmt: - self.expect('print') - args = [] - target = None - if self.current_str() == '>>': - self.skip() - target = self.parse_expression(precedence[',']) - if self.current_str() == ',': - self.skip() - if isinstance(self.current(), Break): - self.parse_error() - else: - if not isinstance(self.current(), Break): - self.parse_error() - comma = False - while not isinstance(self.current(), Break): - args.append(self.parse_expression(precedence[','])) - if self.current_str() == ',': - comma = True - self.skip() - else: - comma = False - break - return PrintStmt(args, newline=not comma, target=target) - - def parse_exec_stmt(self) -> ExecStmt: - self.expect('exec') - expr = self.parse_expression(precedence['in']) - variables1 = None - variables2 = None - if self.current_str() == 'in': - self.skip() - variables1 = self.parse_expression(precedence[',']) - if self.current_str() == ',': - self.skip() - variables2 = self.parse_expression(precedence[',']) - return ExecStmt(expr, variables1, variables2) - - # Parsing expressions - - def parse_expression(self, prec: int = 0, star_expr_allowed: bool = False) -> Node: - """Parse a subexpression within a specific precedence context.""" - expr = None # type: Node - current = self.current() # Remember token for setting the line number. - - # Parse a "value" expression or unary operator expression and store - # that in expr. - s = self.current_str() - if s == '(': - # Parenthesised expression or cast. - expr = self.parse_parentheses() - elif s == '[': - expr = self.parse_list_expr() - elif s in ['-', '+', 'not', '~']: - # Unary operation. - expr = self.parse_unary_expr() - elif s == 'lambda': - expr = self.parse_lambda_expr() - elif s == '{': - expr = self.parse_dict_or_set_expr() - elif s == '*' and star_expr_allowed: - expr = self.parse_star_expr() - elif s == '`' and self.pyversion[0] == 2: - expr = self.parse_backquote_expr() - else: - if isinstance(current, Name): - # Name expression. - expr = self.parse_name_expr() - elif isinstance(current, IntLit): - expr = self.parse_int_expr() - elif isinstance(current, StrLit): - expr = self.parse_str_expr() - elif isinstance(current, BytesLit): - expr = self.parse_bytes_literal() - elif isinstance(current, UnicodeLit): - expr = self.parse_unicode_literal() - elif isinstance(current, FloatLit): - expr = self.parse_float_expr() - elif isinstance(current, ComplexLit): - expr = self.parse_complex_expr() - elif isinstance(current, Keyword) and s == "yield": - # The expression yield from and yield to assign - expr = self.parse_yield_or_yield_from_expr() - elif isinstance(current, EllipsisToken) and (self.pyversion[0] >= 3 - or self.is_stub_file): - expr = self.parse_ellipsis() - else: - # Invalid expression. - self.parse_error() - - # Set the line of the expression node, if not specified. This - # simplifies recording the line number as not every node type needs to - # deal with it separately. - if expr.line < 0: - expr.set_line(current) - - # Parse operations that require a left argument (stored in expr). - while True: - current = self.current() - s = self.current_str() - if s == '(': - # Call expression. - expr = self.parse_call_expr(expr) - elif s == '.': - # Member access expression. - expr = self.parse_member_expr(expr) - elif s == '[': - # Indexing expression. - expr = self.parse_index_expr(expr) - elif s == ',': - # The comma operator is used to build tuples. Comma also - # separates array items and function arguments, but in this - # case the precedence is too low to build a tuple. - if precedence[','] > prec: - expr = self.parse_tuple_expr(expr) - else: - break - elif s == 'for': - if precedence[''] > prec: - # List comprehension or generator expression. Parse as - # generator expression; it will be converted to list - # comprehension if needed elsewhere. - expr = self.parse_generator_expr(expr) - else: - break - elif s == 'if': - # Conditional expression. - if precedence[''] > prec: - expr = self.parse_conditional_expr(expr) - else: - break - else: - # Binary operation or a special case. - if isinstance(current, Op): - op = self.current_str() - op_prec = precedence[op] - if op == 'not': - # Either "not in" or an error. - op_prec = precedence['in'] - if op_prec > prec: - if op in op_comp: - expr = self.parse_comparison_expr(expr, op_prec) - else: - expr = self.parse_bin_op_expr(expr, op_prec) - else: - # The operation cannot be associated with the - # current left operand due to the precedence - # context; let the caller handle it. - break - else: - # Not an operation that accepts a left argument; let the - # caller handle the rest. - break - - # Set the line of the expression node, if not specified. This - # simplifies recording the line number as not every node type - # needs to deal with it separately. - if expr.line < 0: - expr.set_line(current) - - return expr - - def parse_parentheses(self) -> Node: - self.skip() - if self.current_str() == ')': - # Empty tuple (). - expr = self.parse_empty_tuple_expr() # type: Node - else: - # Parenthesised expression. - expr = self.parse_expression(0, star_expr_allowed=True) - self.expect(')') - return expr - - def parse_star_expr(self) -> Node: - star = self.expect('*') - expr = self.parse_expression(precedence['*u']) - expr = StarExpr(expr) - if expr.line < 0: - expr.set_line(star) - return expr - - def parse_empty_tuple_expr(self) -> TupleExpr: - self.expect(')') - node = TupleExpr([]) - return node - - def parse_list_expr(self) -> Node: - """Parse list literal or list comprehension.""" - items = [] - self.expect('[') - while self.current_str() != ']' and not self.eol(): - items.append(self.parse_expression(precedence[''], star_expr_allowed=True)) - if self.current_str() != ',': - break - self.expect(',') - if self.current_str() == 'for' and len(items) == 1: - items[0] = self.parse_generator_expr(items[0]) - self.expect(']') - if len(items) == 1 and isinstance(items[0], GeneratorExpr): - return ListComprehension(items[0]) - else: - expr = ListExpr(items) - return expr - - def parse_generator_expr(self, left_expr: Node) -> GeneratorExpr: - tok = self.current() - indices, sequences, condlists = self.parse_comp_for() - - gen = GeneratorExpr(left_expr, indices, sequences, condlists) - gen.set_line(tok) - return gen - - def parse_comp_for(self) -> Tuple[List[Node], List[Node], List[List[Node]]]: - indices = [] - sequences = [] - condlists = [] # type: List[List[Node]] - while self.current_str() == 'for': - conds = [] - self.expect('for') - index = self.parse_for_index_variables() - indices.append(index) - self.expect('in') - if self.pyversion[0] >= 3: - sequence = self.parse_expression(precedence['']) - else: - sequence = self.parse_expression_list() - sequences.append(sequence) - while self.current_str() == 'if': - self.skip() - conds.append(self.parse_expression(precedence[''])) - condlists.append(conds) - - return indices, sequences, condlists - - def parse_expression_list(self) -> Node: - prec = precedence[''] - expr = self.parse_expression(prec) - if self.current_str() != ',': - return expr - else: - t = self.current() - return self.parse_tuple_expr(expr, prec).set_line(t) - - def parse_conditional_expr(self, left_expr: Node) -> ConditionalExpr: - self.expect('if') - cond = self.parse_expression(precedence['']) - self.expect('else') - else_expr = self.parse_expression(precedence['']) - return ConditionalExpr(cond, left_expr, else_expr) - - def parse_dict_or_set_expr(self) -> Node: - items = [] # type: List[Tuple[Node, Node]] - self.expect('{') - while self.current_str() != '}' and not self.eol(): - key = self.parse_expression(precedence['']) - if self.current_str() in [',', '}'] and items == []: - return self.parse_set_expr(key) - elif self.current_str() == 'for' and items == []: - return self.parse_set_comprehension(key) - elif self.current_str() != ':': - self.parse_error() - colon = self.expect(':') - value = self.parse_expression(precedence['']) - if self.current_str() == 'for' and items == []: - return self.parse_dict_comprehension(key, value, colon) - items.append((key, value)) - if self.current_str() != ',': - break - self.expect(',') - self.expect('}') - node = DictExpr(items) - return node - - def parse_set_expr(self, first: Node) -> SetExpr: - items = [first] - while self.current_str() != '}' and not self.eol(): - self.expect(',') - if self.current_str() == '}': - break - items.append(self.parse_expression(precedence[','])) - self.expect('}') - expr = SetExpr(items) - return expr - - def parse_set_comprehension(self, expr: Node): - gen = self.parse_generator_expr(expr) - self.expect('}') - set_comp = SetComprehension(gen) - return set_comp - - def parse_dict_comprehension(self, key: Node, value: Node, - colon: Token) -> DictionaryComprehension: - indices, sequences, condlists = self.parse_comp_for() - dic = DictionaryComprehension(key, value, indices, sequences, condlists) - dic.set_line(colon) - self.expect('}') - return dic - - def parse_tuple_expr(self, expr: Node, - prec: int = precedence[',']) -> TupleExpr: - items = [expr] - while True: - self.expect(',') - if (self.current_str() in [')', ']', '=', ':'] or - isinstance(self.current(), Break)): - break - items.append(self.parse_expression(prec, star_expr_allowed=True)) - if self.current_str() != ',': break - node = TupleExpr(items) - return node - - def parse_backquote_expr(self) -> BackquoteExpr: - self.expect('`') - expr = self.parse_expression() - self.expect('`') - return BackquoteExpr(expr) - - def parse_name_expr(self) -> NameExpr: - tok = self.expect_type(Name) - node = NameExpr(tok.string) - node.set_line(tok) - return node - - octal_int = re.compile('0+[1-9]') - - def parse_int_expr(self) -> IntExpr: - tok = self.expect_type(IntLit) - string = tok.string.rstrip('lL') # Strip L prefix (Python 2 long literals) - if self.octal_int.match(string): - value = int(string, 8) - else: - value = int(string, 0) - node = IntExpr(value) - return node - - def parse_str_expr(self) -> Node: - # XXX \uxxxx literals - token = self.expect_type(StrLit) - value = cast(StrLit, token).parsed() - is_unicode = False - while isinstance(self.current(), (StrLit, UnicodeLit)): - token = self.skip() - if isinstance(token, StrLit): - value += token.parsed() - elif isinstance(token, UnicodeLit): - value += token.parsed() - is_unicode = True - if is_unicode or (self.pyversion[0] == 2 and 'unicode_literals' in self.future_options): - node = UnicodeExpr(value) # type: Node - else: - node = StrExpr(value) - return node - - def parse_bytes_literal(self) -> Node: - # XXX \uxxxx literals - tok = [self.expect_type(BytesLit)] - value = (cast(BytesLit, tok[0])).parsed() - while isinstance(self.current(), BytesLit): - t = cast(BytesLit, self.skip()) - value += t.parsed() - if self.pyversion[0] >= 3: - node = BytesExpr(value) # type: Node - else: - node = StrExpr(value) - return node - - def parse_unicode_literal(self) -> Node: - # XXX \uxxxx literals - token = self.expect_type(UnicodeLit) - value = cast(UnicodeLit, token).parsed() - while isinstance(self.current(), (UnicodeLit, StrLit)): - token = cast(Union[UnicodeLit, StrLit], self.skip()) - value += token.parsed() - if self.pyversion[0] >= 3: - # Python 3.3 supports u'...' as an alias of '...'. - node = StrExpr(value) # type: Node - else: - node = UnicodeExpr(value) - return node - - def parse_float_expr(self) -> FloatExpr: - tok = self.expect_type(FloatLit) - node = FloatExpr(float(tok.string)) - return node - - def parse_complex_expr(self) -> ComplexExpr: - tok = self.expect_type(ComplexLit) - node = ComplexExpr(complex(tok.string)) - return node - - def parse_call_expr(self, callee: Any) -> CallExpr: - self.expect('(') - args, kinds, names = self.parse_arg_expr() - self.expect(')') - node = CallExpr(callee, args, kinds, names) - return node - - def parse_arg_expr(self) -> Tuple[List[Node], List[int], List[str]]: - """Parse arguments in a call expression (within '(' and ')'). - - Return a tuple with these items: - argument expressions - argument kinds - argument names (for named arguments; None for ordinary args) - """ - args = [] # type: List[Node] - kinds = [] # type: List[int] - names = [] # type: List[str] - var_arg = False - dict_arg = False - named_args = False - while self.current_str() != ')' and not self.eol() and not dict_arg: - if isinstance(self.current(), Name) and self.peek().string == '=': - # Named argument - name = self.expect_type(Name) - self.expect('=') - kinds.append(nodes.ARG_NAMED) - names.append(name.string) - named_args = True - elif (self.current_str() == '*' and not var_arg and not dict_arg): - # *args - var_arg = True - self.expect('*') - kinds.append(nodes.ARG_STAR) - names.append(None) - elif self.current_str() == '**': - # **kwargs - self.expect('**') - dict_arg = True - kinds.append(nodes.ARG_STAR2) - names.append(None) - elif not var_arg and not named_args: - # Ordinary argument - kinds.append(nodes.ARG_POS) - names.append(None) - else: - self.parse_error() - args.append(self.parse_expression(precedence[','])) - if self.current_str() != ',': - break - self.expect(',') - return args, kinds, names - - def parse_member_expr(self, expr: Any) -> Node: - self.expect('.') - name = self.expect_type(Name) - if (isinstance(expr, CallExpr) and isinstance(expr.callee, NameExpr) - and expr.callee.name == 'super'): - # super() expression - node = SuperExpr(name.string) # type: Node - else: - node = MemberExpr(expr, name.string) - return node - - def parse_index_expr(self, base: Any) -> IndexExpr: - self.expect('[') - index = self.parse_slice_item() - if self.current_str() == ',': - # Extended slicing such as x[1:, :2]. - items = [index] - while self.current_str() == ',': - self.skip() - if self.current_str() == ']' or isinstance(self.current(), Break): - break - items.append(self.parse_slice_item()) - index = TupleExpr(items) - index.set_line(items[0].line) - self.expect(']') - node = IndexExpr(base, index) - return node - - def parse_slice_item(self) -> Node: - if self.current_str() != ':': - if self.current_str() == '...': - # Ellipsis is valid here even in Python 2 (but not elsewhere). - ellipsis = EllipsisExpr() - token = self.skip() - ellipsis.set_line(token) - return ellipsis - else: - item = self.parse_expression(precedence[',']) - else: - item = None - if self.current_str() == ':': - # Slice. - index = item - colon = self.expect(':') - if self.current_str() not in (']', ':', ','): - end_index = self.parse_expression(precedence[',']) - else: - end_index = None - stride = None - if self.current_str() == ':': - self.expect(':') - if self.current_str() not in (']', ','): - stride = self.parse_expression(precedence[',']) - item = SliceExpr(index, end_index, stride).set_line(colon.line) - return item - - def parse_bin_op_expr(self, left: Node, prec: int) -> OpExpr: - op = self.expect_type(Op) - op_str = op.string - if op_str == '~': - self.ind -= 1 - self.parse_error() - right = self.parse_expression(prec) - node = OpExpr(op_str, left, right) - return node - - def parse_comparison_expr(self, left: Node, prec: int) -> ComparisonExpr: - operators_str = [] - operands = [left] - - while True: - op = self.expect_type(Op) - op_str = op.string - if op_str == 'not': - if self.current_str() == 'in': - op_str = 'not in' - self.skip() - else: - self.parse_error() - elif op_str == 'is' and self.current_str() == 'not': - op_str = 'is not' - self.skip() - - operators_str.append(op_str) - operand = self.parse_expression(prec) - operands.append(operand) - - # Continue if next token is a comparison operator - self.current() - s = self.current_str() - if s not in op_comp: - break - - node = ComparisonExpr(operators_str, operands) - return node - - def parse_unary_expr(self) -> UnaryExpr: - op_tok = self.skip() - op = op_tok.string - if op == '-' or op == '+': - prec = precedence['-u'] - else: - prec = precedence[op] - expr = self.parse_expression(prec) - node = UnaryExpr(op, expr) - return node - - def parse_lambda_expr(self) -> FuncExpr: - lambda_tok = self.expect('lambda') - - args, extra_stmts = self.parse_arg_list(allow_signature=False) - - # Use 'object' as the placeholder return type; it will be inferred - # later. We can't use 'Any' since it could make type inference results - # less precise. - ret_type = UnboundType('__builtins__.object') - typ = self.build_func_annotation(ret_type, args, - lambda_tok.line, is_default_ret=True) - - colon = self.expect(':') - - expr = self.parse_expression(precedence[',']) - - nodes = [ReturnStmt(expr).set_line(lambda_tok)] - # Potentially insert extra assignment statements to the beginning of the - # body, used to decompose Python 2 tuple arguments. - nodes[:0] = extra_stmts - body = Block(nodes) - body.set_line(colon) - - return FuncExpr(args, body, typ) - - # Helper methods - - def skip(self) -> Token: - self.ind += 1 - return self.tok[self.ind - 1] - - def expect(self, string: str) -> Token: - if self.current_str() == string: - self.ind += 1 - return self.tok[self.ind - 1] - else: - self.parse_error() - - def expect_indent(self) -> Token: - if isinstance(self.current(), Indent): - return self.expect_type(Indent) - else: - self.fail('Expected an indented block', self.current().line) - return none - - def fail(self, msg: str, line: int) -> None: - self.errors.report(line, msg) - - def expect_type(self, typ: type) -> Token: - current = self.current() - if isinstance(current, typ): - self.ind += 1 - return current - else: - self.parse_error() - - def expect_colon_and_break(self) -> Tuple[Token, Token]: - return self.expect_type(Colon), self.expect_type(Break) - - def expect_break(self) -> Token: - return self.expect_type(Break) - - def current(self) -> Token: - return self.tok[self.ind] - - def current_str(self) -> str: - return self.current().string - - def peek(self) -> Token: - return self.tok[self.ind + 1] - - def parse_error(self) -> None: - self.parse_error_at(self.current()) - raise ParseError() - - def parse_error_at(self, tok: Token, skip: bool = True, reason: Optional[str] = None) -> None: - msg = '' - if isinstance(tok, LexError): - msg = token_repr(tok) - msg = msg[0].upper() + msg[1:] - elif isinstance(tok, Indent) or isinstance(tok, Dedent): - msg = 'Inconsistent indentation' - else: - formatted_reason = ": {}".format(reason) if reason else "" - msg = 'Parse error before {}{}'.format(token_repr(tok), formatted_reason) - - self.errors.report(tok.line, msg) - - if skip: - self.skip_until_next_line() - - def skip_until_break(self) -> None: - n = 0 - while (not isinstance(self.current(), Break) - and not isinstance(self.current(), Eof)): - self.skip() - n += 1 - if isinstance(self.tok[self.ind - 1], Colon) and n > 1: - self.ind -= 1 - - def skip_until_next_line(self) -> None: - self.skip_until_break() - if isinstance(self.current(), Break): - self.skip() - - def eol(self) -> bool: - return isinstance(self.current(), Break) or self.eof() - - def eof(self) -> bool: - return isinstance(self.current(), Eof) - - # Type annotation related functionality - - def parse_type(self) -> Type: - try: - typ, self.ind = parse_type(self.tok, self.ind) - except TypeParseError as e: - self.parse_error_at(e.token, reason=e.message) - raise ParseError() - return typ - - annotation_prefix_re = re.compile(r'#\s*type:') - ignore_prefix_re = re.compile(r'ignore\b') - - def parse_type_comment(self, token: Token, signature: bool) -> Type: - """Parse a '# type: ...' annotation. - - Return None if no annotation found. If signature is True, expect - a type signature of form (...) -> t. - """ - whitespace_or_comments = token.rep().strip() - if self.annotation_prefix_re.match(whitespace_or_comments): - type_as_str = whitespace_or_comments.split(':', 1)[1].strip() - if self.ignore_prefix_re.match(type_as_str): - # Actually a "# type: ignore" annotation -> not a type. - return None - tokens = lex.lex(type_as_str, token.line)[0] - if len(tokens) < 2: - # Empty annotation (only Eof token) - self.errors.report(token.line, 'Empty type annotation') - return None - try: - if not signature: - type, index = parse_types(tokens, 0) - else: - type, index = parse_signature(tokens) - except TypeParseError as e: - self.parse_error_at(e.token, skip=False, reason=e.message) - return None - if index < len(tokens) - 2: - self.parse_error_at(tokens[index], skip=False) - return None - return type - else: - return None - - -class ParseError(Exception): pass - - -def token_repr(tok: Token) -> str: - """Return a representation of a token for use in parse error messages.""" - if isinstance(tok, Break): - return 'end of line' - elif isinstance(tok, Eof): - return 'end of file' - elif isinstance(tok, Keyword) or isinstance(tok, Name): - return '"{}"'.format(tok.string) - elif isinstance(tok, IntLit) or isinstance(tok, FloatLit) or isinstance(tok, ComplexLit): - return 'numeric literal' - elif isinstance(tok, StrLit) or isinstance(tok, UnicodeLit): - return 'string literal' - elif (isinstance(tok, Punct) or isinstance(tok, Op) - or isinstance(tok, Colon)): - return tok.string - elif isinstance(tok, Bom): - return 'byte order mark' - elif isinstance(tok, Indent): - return 'indent' - elif isinstance(tok, Dedent): - return 'dedent' - elif isinstance(tok, EllipsisToken): - return '...' - else: - if isinstance(tok, LexError): - t = tok.type - if t == lex.NUMERIC_LITERAL_ERROR: - return 'invalid numeric literal' - elif t == lex.UNTERMINATED_STRING_LITERAL: - return 'unterminated string literal' - elif t == lex.INVALID_CHARACTER: - msg = 'unrecognized character' - if ord(tok.string) in range(33, 127): - msg += ' ' + tok.string - return msg - elif t == lex.INVALID_DEDENT: - return 'inconsistent indentation' - elif t == lex.DECODE_ERROR: - return tok.message - raise ValueError('Unknown token {}'.format(repr(tok))) - - -if __name__ == '__main__': - # Parse a file and dump the AST (or display errors). - import sys - - def usage(): - print('Usage: parse.py [--py2] [--quiet] FILE [...]', file=sys.stderr) - sys.exit(2) - - args = sys.argv[1:] - pyversion = defaults.PYTHON3_VERSION - quiet = False - while args and args[0].startswith('--'): - if args[0] == '--py2': - pyversion = defaults.PYTHON2_VERSION - elif args[0] == '--quiet': - quiet = True - else: - usage() - args = args[1:] - if len(args) < 1: - usage() - status = 0 - for fnam in args: - s = open(fnam, 'rb').read() - errors = Errors() - try: - options = Options() - options.python_version = pyversion - tree = parse(s, fnam, None, options=options) - if not quiet: - print(tree) - except CompileError as e: - for msg in e.messages: - sys.stderr.write('%s\n' % msg) - status = 1 - sys.exit(status) +"""Mypy parser. + +Constructs a parse tree (abstract syntax tree) based on a string +representing a source file. Performs only minimal semantic checks. +""" + +import re + +from typing import List, Tuple, Any, Set, cast, Union, Optional + +from mypy import lex, docstring +from mypy.lex import ( + Token, Eof, Bom, Break, Name, Colon, Dedent, IntLit, StrLit, BytesLit, + UnicodeLit, FloatLit, Op, Indent, Keyword, Punct, LexError, ComplexLit, + EllipsisToken +) +import mypy.types +from mypy.nodes import ( + MypyFile, Import, Node, ImportAll, ImportFrom, FuncDef, OverloadedFuncDef, + ClassDef, Decorator, Block, Var, OperatorAssignmentStmt, + ExpressionStmt, AssignmentStmt, ReturnStmt, RaiseStmt, AssertStmt, + DelStmt, BreakStmt, ContinueStmt, PassStmt, GlobalDecl, + WhileStmt, ForStmt, IfStmt, TryStmt, WithStmt, + TupleExpr, GeneratorExpr, ListComprehension, ListExpr, ConditionalExpr, + DictExpr, SetExpr, NameExpr, IntExpr, StrExpr, BytesExpr, UnicodeExpr, + FloatExpr, CallExpr, SuperExpr, MemberExpr, IndexExpr, SliceExpr, OpExpr, + UnaryExpr, FuncExpr, PrintStmt, ImportBase, ComparisonExpr, + StarExpr, YieldFromExpr, NonlocalDecl, DictionaryComprehension, + SetComprehension, ComplexExpr, EllipsisExpr, YieldExpr, ExecStmt, Argument, + BackquoteExpr +) +from mypy import defaults +from mypy import nodes +from mypy.errors import Errors, CompileError +from mypy.types import Type, CallableType, AnyType, UnboundType +from mypy.parsetype import ( + parse_type, parse_types, parse_signature, TypeParseError, parse_str_as_signature +) +from mypy.options import Options + +from mypy import experiments + + +precedence = { + '**': 16, + '-u': 15, '+u': 15, '~': 15, # unary operators (-, + and ~) + '': 14, + '*': 13, '/': 13, '//': 13, '%': 13, + '+': 12, '-': 12, + '>>': 11, '<<': 11, + '&': 10, + '^': 9, + '|': 8, + '==': 7, '!=': 7, '<': 7, '>': 7, '<=': 7, '>=': 7, 'is': 7, 'in': 7, + '*u': 7, # unary * for star expressions + 'not': 6, + 'and': 5, + 'or': 4, + '': 3, # conditional expression + '': 2, # list comprehension + ',': 1} + + +op_assign = set([ + '+=', '-=', '*=', '/=', '//=', '%=', '**=', '|=', '&=', '^=', '>>=', + '<<=']) + +op_comp = set([ + '>', '<', '==', '>=', '<=', '<>', '!=', 'is', 'is', 'in', 'not']) + +none = Token('') # Empty token + + +def parse(source: Union[str, bytes], + fnam: str, + errors: Errors, + options: Options) -> MypyFile: + """Parse a source file, without doing any semantic analysis. + + Return the parse tree. If errors is not provided, raise ParseError + on failure. Otherwise, use the errors object to report parse errors. + + The python_version (major, minor) option determines the Python syntax variant. + """ + if options.fast_parser: + import mypy.fastparse + return mypy.fastparse.parse(source, + fnam=fnam, + errors=errors, + pyversion=options.python_version, + custom_typing_module=options.custom_typing_module) + + is_stub_file = bool(fnam) and fnam.endswith('.pyi') + parser = Parser(fnam, + errors, + options.python_version, + options.custom_typing_module, + is_stub_file=is_stub_file) + tree = parser.parse(source) + tree.path = fnam + tree.is_stub = is_stub_file + return tree + + +class Parser: + """Mypy parser that parses a string into an AST. + + Parses type annotations in addition to basic Python syntax. It supports both Python 2 and 3 + (though Python 2 support is incomplete). + + The AST classes are defined in mypy.nodes and mypy.types. + """ + + tok = None # type: List[Token] + ind = 0 + errors = None # type: Errors + # If True, raise an exception on any parse error. Otherwise, errors are reported via 'errors'. + raise_on_error = False + + # Are we currently parsing the body of a class definition? + is_class_body = False + # All import nodes encountered so far in this parse unit. + imports = None # type: List[ImportBase] + # Names imported from __future__. + future_options = None # type: List[str] + # Lines to ignore (using # type: ignore). + ignored_lines = None # type: Set[int] + + def __init__(self, fnam: str, errors: Errors, pyversion: Tuple[int, int], + custom_typing_module: str = None, is_stub_file: bool = False) -> None: + self.raise_on_error = errors is None + self.pyversion = pyversion + self.custom_typing_module = custom_typing_module + self.is_stub_file = is_stub_file + if errors is not None: + self.errors = errors + else: + self.errors = Errors() + if fnam is not None: + self.errors.set_file(fnam) + else: + self.errors.set_file('') + + def parse(self, s: Union[str, bytes]) -> MypyFile: + self.tok, self.ignored_lines = lex.lex(s, pyversion=self.pyversion, + is_stub_file=self.is_stub_file) + self.ind = 0 + self.imports = [] + self.future_options = [] + file = self.parse_file() + if self.raise_on_error and self.errors.is_errors(): + self.errors.raise_error() + return file + + def weak_opts(self) -> Set[str]: + """Do weak typing if any of the first ten tokens is a comment saying so. + + The comment can be one of: + # mypy: weak=global + # mypy: weak=local + # mypy: weak <- defaults to local + """ + regexp = re.compile(r'^[\s]*# *mypy: *weak(=?)([^\s]*)', re.M) + for t in self.tok[:10]: + for s in [t.string, t.pre]: + m = regexp.search(s) + if m: + opts = set(x for x in m.group(2).split(',') if x) + if not opts: + opts.add('local') + return opts + return set() + + def parse_file(self) -> MypyFile: + """Parse a mypy source file.""" + is_bom = self.parse_bom() + defs = self.parse_defs() + weak_opts = self.weak_opts() + self.expect_type(Eof) + node = MypyFile(defs, self.imports, is_bom, self.ignored_lines, + weak_opts=weak_opts) + return node + + # Parse the initial part + + def parse_bom(self) -> bool: + """Parse the optional byte order mark at the beginning of a file.""" + if isinstance(self.current(), Bom): + self.expect_type(Bom) + if isinstance(self.current(), Break): + self.expect_break() + return True + else: + return False + + def parse_import(self) -> Import: + self.expect('import') + ids = [] + while True: + id = self.parse_qualified_name() + translated = self.translate_module_id(id) + as_id = None + if self.current_str() == 'as': + self.expect('as') + name_tok = self.expect_type(Name) + as_id = name_tok.string + elif translated != id: + as_id = id + ids.append((translated, as_id)) + if self.current_str() != ',': + break + self.expect(',') + node = Import(ids) + self.imports.append(node) + return node + + def translate_module_id(self, id: str) -> str: + """Return the actual, internal module id for a source text id. + + For example, translate '__builtin__' in Python 2 to 'builtins'. + """ + if id == self.custom_typing_module: + return 'typing' + elif id == '__builtin__' and self.pyversion[0] == 2: + # HACK: __builtin__ in Python 2 is aliases to builtins. However, the implementation + # is named __builtin__.py (there is another layer of translation elsewhere). + return 'builtins' + return id + + def parse_import_from(self) -> Node: + self.expect('from') + + # Build the list of beginning relative tokens. + relative = 0 + while self.current_str() in (".", "..."): + relative += len(self.current_str()) + self.skip() + + # Parse qualified name to actually import from. + if self.current_str() == "import": + # Empty/default values. + name = "" + else: + name = self.parse_qualified_name() + + name = self.translate_module_id(name) + + # Parse import list + self.expect('import') + node = None # type: ImportBase + if self.current_str() == '*': + if name == '__future__': + self.parse_error() + # An import all from a module node: + self.skip() + node = ImportAll(name, relative) + else: + is_paren = self.current_str() == '(' + if is_paren: + self.expect('(') + targets = [] # type: List[Tuple[str, str]] + while True: + id, as_id = self.parse_import_name() + if '%s.%s' % (name, id) == self.custom_typing_module: + if targets or self.current_str() == ',': + self.fail('You cannot import any other modules when you ' + 'import a custom typing module', + self.current().line) + node = Import([('typing', as_id)]) + self.skip_until_break() + break + targets.append((id, as_id)) + if self.current_str() != ',': + break + self.expect(',') + if is_paren and self.current_str() == ')': + break + if is_paren: + self.expect(')') + if node is None: + node = ImportFrom(name, relative, targets) + self.imports.append(node) + if name == '__future__': + self.future_options.extend(target[0] for target in targets) + return node + + def parse_import_name(self) -> Tuple[str, Optional[str]]: + tok = self.expect_type(Name) + name = tok.string + if self.current_str() == 'as': + self.skip() + as_name = self.expect_type(Name) + return name, as_name.string + else: + return name, None + + def parse_qualified_name(self) -> str: + """Parse a name with an optional module qualifier. + + Return a tuple with the name as a string and a token array + containing all the components of the name. + """ + tok = self.expect_type(Name) + n = tok.string + while self.current_str() == '.': + self.expect('.') + tok = self.expect_type(Name) + n += '.' + tok.string + return n + + # Parsing global definitions + + def parse_defs(self) -> List[Node]: + defs = [] # type: List[Node] + while not self.eof(): + try: + defn, is_simple = self.parse_statement() + if is_simple: + self.expect_break() + if defn is not None: + if not self.try_combine_overloads(defn, defs): + defs.append(defn) + except ParseError: + pass + return defs + + def parse_class_def(self) -> ClassDef: + old_is_class_body = self.is_class_body + self.is_class_body = True + + self.expect('class') + metaclass = None + + try: + base_types = [] # type: List[Node] + try: + name_tok = self.expect_type(Name) + name = name_tok.string + + self.errors.push_type(name) + + if self.current_str() == '(': + self.skip() + while True: + if self.current_str() == ')': + break + if self.peek().string == '=': + metaclass = self.parse_class_keywords() + break + base_types.append(self.parse_super_type()) + if self.current_str() != ',': + break + self.skip() + self.expect(')') + except ParseError: + pass + + defs, _ = self.parse_block() + + node = ClassDef(name, defs, None, base_types, metaclass=metaclass) + return node + finally: + self.errors.pop_type() + self.is_class_body = old_is_class_body + + def parse_class_keywords(self) -> Optional[str]: + """Parse the class keyword arguments, keeping the metaclass but + ignoring all others. Returns None if the metaclass isn't found. + """ + metaclass = None + while True: + key = self.expect_type(Name) + self.expect('=') + if key.string == 'metaclass': + metaclass = self.parse_qualified_name() + else: + # skip the class value + self.parse_expression(precedence[',']) + if self.current_str() != ',': + break + self.skip() + if self.current_str() == ')': + break + return metaclass + + def parse_super_type(self) -> Node: + return self.parse_expression(precedence[',']) + + def parse_decorated_function_or_class(self) -> Node: + decorators = [] + no_type_checks = False + while self.current_str() == '@': + self.expect('@') + d_exp = self.parse_expression() + if self.is_no_type_check_decorator(d_exp): + no_type_checks = True + decorators.append(d_exp) + self.expect_break() + if self.current_str() != 'class': + func = self.parse_function(no_type_checks) + func.is_decorated = True + var = Var(func.name()) + # Types of decorated functions must always be inferred. + var.is_ready = False + var.set_line(decorators[0].line) + node = Decorator(func, decorators, var) + return node + else: + cls = self.parse_class_def() + cls.decorators = decorators + return cls + + def is_no_type_check_decorator(self, expr: Node) -> bool: + if isinstance(expr, NameExpr): + return expr.name == 'no_type_check' + elif isinstance(expr, MemberExpr): + if isinstance(expr.expr, NameExpr): + return expr.expr.name == 'typing' and expr.name == 'no_type_check' + else: + return False + + def parse_function(self, no_type_checks: bool=False) -> FuncDef: + def_tok = self.expect('def') + is_method = self.is_class_body + self.is_class_body = False + try: + (name, args, typ, is_error, extra_stmts) = self.parse_function_header(no_type_checks) + + arg_kinds = [arg.kind for arg in args] + arg_names = [arg.variable.name() for arg in args] + + body, comment_type = self.parse_block(allow_type=True) + # Potentially insert extra assignment statements to the beginning of the + # body, used to decompose Python 2 tuple arguments. + body.body[:0] = extra_stmts + if comment_type: + # The function has a # type: ... signature. + if typ: + self.errors.report( + def_tok.line, 'Function has duplicate type signatures') + sig = cast(CallableType, comment_type) + if sig.is_ellipsis_args: + # When we encounter an ellipsis, fill in the arg_types with + # a bunch of AnyTypes, emulating Callable[..., T] + arg_types = [AnyType()] * len(arg_kinds) # type: List[Type] + typ = CallableType( + arg_types, + arg_kinds, + arg_names, + sig.ret_type, + None, + line=def_tok.line) + elif is_method and len(sig.arg_kinds) < len(arg_kinds): + self.check_argument_kinds(arg_kinds, + [nodes.ARG_POS] + sig.arg_kinds, + def_tok.line) + # Add implicit 'self' argument to signature. + first_arg = [AnyType()] # type: List[Type] + typ = CallableType( + first_arg + sig.arg_types, + arg_kinds, + arg_names, + sig.ret_type, + None, + line=def_tok.line) + else: + self.check_argument_kinds(arg_kinds, sig.arg_kinds, + def_tok.line) + typ = CallableType( + sig.arg_types, + arg_kinds, + arg_names, + sig.ret_type, + None, + line=def_tok.line) + + # If there was a serious error, we really cannot build a parse tree + # node. + if is_error: + return None + + if typ and isinstance(typ.ret_type, UnboundType): + typ.ret_type.is_ret_type = True + + node = FuncDef(name, args, body, typ) + node.set_line(def_tok) + if typ is not None: + typ.definition = node + return node + finally: + self.errors.pop_function() + self.is_class_body = is_method + + def check_argument_kinds(self, funckinds: List[int], sigkinds: List[int], + line: int) -> None: + """Check that arguments are consistent. + + This verifies that they have the same number and the kinds correspond. + + Arguments: + funckinds: kinds of arguments in function definition + sigkinds: kinds of arguments in signature (after # type:) + """ + if len(funckinds) != len(sigkinds): + if len(funckinds) > len(sigkinds): + self.fail("Type signature has too few arguments", line) + else: + self.fail("Type signature has too many arguments", line) + return + for kind, token in [(nodes.ARG_STAR, '*'), + (nodes.ARG_STAR2, '**')]: + if ((funckinds.count(kind) != sigkinds.count(kind)) or + (kind in funckinds and sigkinds.index(kind) != funckinds.index(kind))): + self.fail( + "Inconsistent use of '{}' in function " + "signature".format(token), line) + + def parse_function_header( + self, no_type_checks: bool=False) -> Tuple[str, + List[Argument], + CallableType, + bool, + List[AssignmentStmt]]: + """Parse function header (a name followed by arguments) + + Return a 5-tuple with the following items: + name + arguments + signature (annotation) + error flag (True if error) + extra statements needed to decompose arguments (usually empty) + + See parse_arg_list for an explanation of the final tuple item. + """ + name = '' + + try: + name_tok = self.expect_type(Name) + name = name_tok.string + + self.errors.push_function(name) + + args, typ, extra_stmts = self.parse_args(no_type_checks) + except ParseError: + if not isinstance(self.current(), Break): + self.ind -= 1 # Kludge: go back to the Break token + # Resynchronise parsing by going back over :, if present. + if isinstance(self.tok[self.ind - 1], Colon): + self.ind -= 1 + return (name, [], None, True, []) + + return (name, args, typ, False, extra_stmts) + + def parse_args(self, no_type_checks: bool=False) -> Tuple[List[Argument], + CallableType, + List[AssignmentStmt]]: + """Parse a function signature (...) [-> t]. + + See parse_arg_list for an explanation of the final tuple item. + """ + lparen = self.expect('(') + + # Parse the argument list (everything within '(' and ')'). + args, extra_stmts = self.parse_arg_list(no_type_checks=no_type_checks) + + self.expect(')') + + if self.current_str() == '->': + self.skip() + if no_type_checks: + self.parse_expression() + ret_type = None + else: + ret_type = self.parse_type() + else: + ret_type = None + + arg_kinds = [arg.kind for arg in args] + self.verify_argument_kinds(arg_kinds, lparen.line) + + annotation = self.build_func_annotation( + ret_type, args, lparen.line) + + return args, annotation, extra_stmts + + def build_func_annotation(self, ret_type: Type, args: List[Argument], + line: int, is_default_ret: bool = False) -> CallableType: + arg_types = [arg.type_annotation for arg in args] + # Are there any type annotations? + if ((ret_type and not is_default_ret) + or arg_types != [None] * len(arg_types)): + # Yes. Construct a type for the function signature. + return self.construct_function_type(args, ret_type, line) + else: + return None + + def parse_arg_list(self, allow_signature: bool = True, + no_type_checks: bool=False) -> Tuple[List[Argument], + List[AssignmentStmt]]: + """Parse function definition argument list. + + This includes everything between '(' and ')' (but not the + parentheses). + + Return tuple (arguments, + extra statements for decomposing arguments). + + The final argument is only used for Python 2 argument lists with + tuples; they contain destructuring assignment statements used to + decompose tuple arguments. For example, consider a header like this: + + . def f((x, y)) + + The actual (sole) argument will be __tuple_arg_1 (a generated + name), whereas the extra statement list will contain a single + assignment statement corresponding to this assignment: + + x, y = __tuple_arg_1 + """ + args = [] # type: List[Argument] + extra_stmts = [] + # This is for checking duplicate argument names. + arg_names = [] # type: List[str] + has_tuple_arg = False + + require_named = False + bare_asterisk_before = -1 + + if self.current_str() != ')' and self.current_str() != ':': + while self.current_str() != ')': + if self.current_str() == '*' and self.peek().string == ',': + self.expect('*') + require_named = True + bare_asterisk_before = len(args) + elif self.current_str() in ['*', '**']: + if bare_asterisk_before == len(args): + # named arguments must follow bare * + self.parse_error() + + arg = self.parse_asterisk_arg( + allow_signature, + no_type_checks, + ) + args.append(arg) + require_named = True + elif self.current_str() == '(': + arg, extra_stmt, names = self.parse_tuple_arg(len(args)) + args.append(arg) + if extra_stmt is not None: + extra_stmts.append(extra_stmt) + has_tuple_arg = True + arg_names.extend(names) + else: + arg, require_named = self.parse_normal_arg( + require_named, + allow_signature, + no_type_checks, + ) + args.append(arg) + arg_names.append(arg.variable.name()) + + if self.current().string != ',': + break + + self.expect(',') + + # Non-tuple argument dupes will be checked elsewhere. Avoid + # generating duplicate errors. + if has_tuple_arg: + self.check_duplicate_argument_names(arg_names) + + return args, extra_stmts + + def check_duplicate_argument_names(self, names: List[str]) -> None: + found = set() # type: Set[str] + for name in names: + if name in found: + self.fail('Duplicate argument name "{}"'.format(name), + self.current().line) + found.add(name) + + def parse_asterisk_arg(self, + allow_signature: bool, + no_type_checks: bool) -> Argument: + asterisk = self.skip() + name = self.expect_type(Name) + variable = Var(name.string) + if asterisk.string == '*': + kind = nodes.ARG_STAR + else: + kind = nodes.ARG_STAR2 + + type = None + if no_type_checks: + self.parse_parameter_annotation() + else: + type = self.parse_arg_type(allow_signature) + + return Argument(variable, type, None, kind) + + def parse_tuple_arg(self, index: int) -> Tuple[Argument, AssignmentStmt, List[str]]: + """Parse a single Python 2 tuple argument. + + Example: def f(x, (y, z)): ... + + The tuple arguments gets transformed into an assignment in the + function body (the second return value). + + Return tuple (argument, decomposing assignment, list of names defined). + + Special case: if the argument is just (x) then it's not a tuple; + we indicate this by returning (argument, None, ['x']). + However, if the argument is (x,) then it *is* a (singleton) tuple. + """ + line = self.current().line + # Generate a new argument name that is very unlikely to clash with anything. + arg_name = '__tuple_arg_{}'.format(index + 1) + if self.pyversion[0] >= 3: + self.fail('Tuples in argument lists only supported in Python 2 mode', line) + paren_arg = self.parse_parentheses() + self.verify_tuple_arg(paren_arg) + if isinstance(paren_arg, NameExpr): + # This isn't a tuple. Revert to a normal argument. + arg_name = paren_arg.name + decompose = None + else: + rvalue = NameExpr(arg_name) + rvalue.set_line(line) + decompose = AssignmentStmt([paren_arg], rvalue) + decompose.set_line(line) + kind = nodes.ARG_POS + initializer = None + if self.current_str() == '=': + self.expect('=') + initializer = self.parse_expression(precedence[',']) + kind = nodes.ARG_OPT + var = Var(arg_name) + arg_names = self.find_tuple_arg_argument_names(paren_arg) + return Argument(var, None, initializer, kind), decompose, arg_names + + def verify_tuple_arg(self, paren_arg: Node) -> None: + if isinstance(paren_arg, TupleExpr): + if not paren_arg.items: + self.fail('Empty tuple not valid as an argument', paren_arg.line) + for item in paren_arg.items: + self.verify_tuple_arg(item) + elif not isinstance(paren_arg, NameExpr): + self.fail('Invalid item in tuple argument', paren_arg.line) + + def find_tuple_arg_argument_names(self, node: Node) -> List[str]: + result = [] # type: List[str] + if isinstance(node, TupleExpr): + for item in node.items: + result.extend(self.find_tuple_arg_argument_names(item)) + elif isinstance(node, NameExpr): + result.append(node.name) + return result + + def parse_normal_arg(self, require_named: bool, + allow_signature: bool, + no_type_checks: bool) -> Tuple[Argument, bool]: + name = self.expect_type(Name) + variable = Var(name.string) + + type = None + if no_type_checks: + self.parse_parameter_annotation() + else: + type = self.parse_arg_type(allow_signature) + + initializer = None # type: Node + if self.current_str() == '=': + self.expect('=') + initializer = self.parse_expression(precedence[',']) + if require_named: + kind = nodes.ARG_NAMED + else: + kind = nodes.ARG_OPT + else: + if require_named: + kind = nodes.ARG_NAMED + else: + kind = nodes.ARG_POS + + self.set_type_optional(type, initializer) + + return Argument(variable, type, initializer, kind), require_named + + def set_type_optional(self, type: Type, initializer: Node) -> None: + if not experiments.STRICT_OPTIONAL: + return + # Indicate that type should be wrapped in an Optional if arg is initialized to None. + optional = isinstance(initializer, NameExpr) and initializer.name == 'None' + if isinstance(type, UnboundType): + type.optional = optional + + def parse_parameter_annotation(self) -> Node: + if self.current_str() == ':': + self.skip() + return self.parse_expression(precedence[',']) + + def parse_arg_type(self, allow_signature: bool) -> Type: + if self.current_str() == ':' and allow_signature: + self.skip() + return self.parse_type() + else: + return None + + def verify_argument_kinds(self, kinds: List[int], line: int) -> None: + found = set() # type: Set[int] + for i, kind in enumerate(kinds): + if kind == nodes.ARG_POS and found & set([nodes.ARG_OPT, + nodes.ARG_STAR, + nodes.ARG_STAR2]): + self.fail('Invalid argument list', line) + elif kind == nodes.ARG_STAR and nodes.ARG_STAR in found: + self.fail('Invalid argument list', line) + elif kind == nodes.ARG_STAR2 and i != len(kinds) - 1: + self.fail('Invalid argument list', line) + found.add(kind) + + def construct_function_type(self, args: List[Argument], ret_type: Type, + line: int) -> CallableType: + # Complete the type annotation by replacing omitted types with 'Any'. + arg_types = [arg.type_annotation for arg in args] + for i in range(len(arg_types)): + if arg_types[i] is None: + arg_types[i] = AnyType(implicit=True) + if ret_type is None: + ret_type = AnyType(implicit=True) + arg_kinds = [arg.kind for arg in args] + arg_names = [arg.variable.name() for arg in args] + return CallableType(arg_types, arg_kinds, arg_names, ret_type, None, name=None, + variables=None, line=line) + + # Parsing statements + + def parse_block(self, allow_type: bool = False) -> Tuple[Block, Type]: + colon = self.expect(':') + if not isinstance(self.current(), Break): + # Block immediately after ':'. + nodes = [] + while True: + ind = self.ind + stmt, is_simple = self.parse_statement() + if not is_simple: + self.parse_error_at(self.tok[ind]) + break + nodes.append(stmt) + brk = self.expect_break() + if brk.string != ';': + break + node = Block(nodes) + node.set_line(colon) + return node, None + else: + # Indented block. + brk = self.expect_break() + type = self.parse_type_comment(brk, signature=True) + self.expect_indent() + stmt_list = [] # type: List[Node] + if allow_type: + cur = self.current() + if type is None and isinstance(cur, StrLit): + ds = docstring.parse_docstring(cur.parsed()) + if ds and False: # TODO: Enable when this is working. + try: + type = parse_str_as_signature(ds.as_type_str(), cur.line) + except TypeParseError: + # We don't require docstrings to be actually correct. + # TODO: Report something here. + type = None + while (not isinstance(self.current(), Dedent) and + not isinstance(self.current(), Eof)): + try: + stmt, is_simple = self.parse_statement() + if is_simple: + self.expect_break() + if stmt is not None: + if not self.try_combine_overloads(stmt, stmt_list): + stmt_list.append(stmt) + except ParseError: + pass + if isinstance(self.current(), Dedent): + self.skip() + node = Block(stmt_list) + node.set_line(colon) + return node, type + + def try_combine_overloads(self, s: Node, stmt: List[Node]) -> bool: + if isinstance(s, Decorator) and stmt: + fdef = s + n = fdef.func.name() + if isinstance(stmt[-1], Decorator) and stmt[-1].func.name() == n: + stmt[-1] = OverloadedFuncDef([stmt[-1], fdef]) + return True + elif isinstance(stmt[-1], OverloadedFuncDef) and stmt[-1].name() == n: + stmt[-1].items.append(fdef) + return True + return False + + def parse_statement(self) -> Tuple[Node, bool]: + stmt = None # type: Node + t = self.current() + ts = self.current_str() + is_simple = True # Is this a non-block statement? + if ts == 'if': + stmt = self.parse_if_stmt() + is_simple = False + elif ts == 'def': + stmt = self.parse_function() + is_simple = False + elif ts == 'while': + stmt = self.parse_while_stmt() + is_simple = False + elif ts == 'return': + stmt = self.parse_return_stmt() + elif ts == 'for': + stmt = self.parse_for_stmt() + is_simple = False + elif ts == 'try': + stmt = self.parse_try_stmt() + is_simple = False + elif ts == 'break': + stmt = self.parse_break_stmt() + elif ts == 'continue': + stmt = self.parse_continue_stmt() + elif ts == 'pass': + stmt = self.parse_pass_stmt() + elif ts == 'raise': + stmt = self.parse_raise_stmt() + elif ts == 'import': + stmt = self.parse_import() + elif ts == 'from': + stmt = self.parse_import_from() + elif ts == 'class': + stmt = self.parse_class_def() + is_simple = False + elif ts == 'global': + stmt = self.parse_global_decl() + elif ts == 'nonlocal' and self.pyversion[0] >= 3: + stmt = self.parse_nonlocal_decl() + elif ts == 'assert': + stmt = self.parse_assert_stmt() + elif ts == 'del': + stmt = self.parse_del_stmt() + elif ts == 'with': + stmt = self.parse_with_stmt() + is_simple = False + elif ts == '@': + stmt = self.parse_decorated_function_or_class() + is_simple = False + elif ts == 'print' and (self.pyversion[0] == 2 and + 'print_function' not in self.future_options): + stmt = self.parse_print_stmt() + elif ts == 'exec' and self.pyversion[0] == 2: + stmt = self.parse_exec_stmt() + else: + stmt = self.parse_expression_or_assignment() + if stmt is not None: + stmt.set_line(t) + return stmt, is_simple + + def parse_expression_or_assignment(self) -> Node: + expr = self.parse_expression(star_expr_allowed=True) + if self.current_str() == '=': + return self.parse_assignment(expr) + elif self.current_str() in op_assign: + # Operator assignment statement. + op = self.current_str()[:-1] + self.skip() + rvalue = self.parse_expression() + return OperatorAssignmentStmt(op, expr, rvalue) + else: + # Expression statement. + return ExpressionStmt(expr) + + def parse_assignment(self, lvalue: Any) -> Node: + """Parse an assignment statement. + + Assume that lvalue has been parsed already, and the current token is '='. + Also parse an optional '# type:' comment. + """ + self.expect('=') + lvalues = [lvalue] + expr = self.parse_expression(star_expr_allowed=True) + while self.current_str() == '=': + self.skip() + lvalues.append(expr) + expr = self.parse_expression(star_expr_allowed=True) + cur = self.current() + if isinstance(cur, Break): + type = self.parse_type_comment(cur, signature=False) + else: + type = None + return AssignmentStmt(lvalues, expr, type) + + def parse_return_stmt(self) -> ReturnStmt: + self.expect('return') + expr = None + current = self.current() + if current.string == 'yield': + self.parse_error() + if not isinstance(current, Break): + expr = self.parse_expression() + node = ReturnStmt(expr) + return node + + def parse_raise_stmt(self) -> RaiseStmt: + self.expect('raise') + expr = None + from_expr = None + if not isinstance(self.current(), Break): + expr = self.parse_expression() + if self.current_str() == 'from': + self.expect('from') + from_expr = self.parse_expression() + node = RaiseStmt(expr, from_expr) + return node + + def parse_assert_stmt(self) -> AssertStmt: + self.expect('assert') + expr = self.parse_expression() + node = AssertStmt(expr) + return node + + def parse_yield_or_yield_from_expr(self) -> Union[YieldFromExpr, YieldExpr]: + self.expect("yield") + expr = None + node = YieldExpr(expr) # type: Union[YieldFromExpr, YieldExpr] + if not isinstance(self.current(), Break): + if self.current_str() == "from": + self.expect("from") + expr = self.parse_expression() # when yield from is assigned to a variable + node = YieldFromExpr(expr) + else: + if self.current_str() == ')': + node = YieldExpr(None) + else: + expr = self.parse_expression() + node = YieldExpr(expr) + return node + + def parse_ellipsis(self) -> EllipsisExpr: + self.expect('...') + node = EllipsisExpr() + return node + + def parse_del_stmt(self) -> DelStmt: + self.expect('del') + expr = self.parse_expression() + node = DelStmt(expr) + return node + + def parse_break_stmt(self) -> BreakStmt: + self.expect('break') + node = BreakStmt() + return node + + def parse_continue_stmt(self) -> ContinueStmt: + self.expect('continue') + node = ContinueStmt() + return node + + def parse_pass_stmt(self) -> PassStmt: + self.expect('pass') + node = PassStmt() + return node + + def parse_global_decl(self) -> GlobalDecl: + self.expect('global') + names = self.parse_identifier_list() + node = GlobalDecl(names) + return node + + def parse_nonlocal_decl(self) -> NonlocalDecl: + self.expect('nonlocal') + names = self.parse_identifier_list() + node = NonlocalDecl(names) + return node + + def parse_identifier_list(self) -> List[str]: + names = [] + while True: + n = self.expect_type(Name) + names.append(n.string) + if self.current_str() != ',': + break + self.skip() + return names + + def parse_while_stmt(self) -> WhileStmt: + is_error = False + self.expect('while') + try: + expr = self.parse_expression() + except ParseError: + is_error = True + body, _ = self.parse_block() + if self.current_str() == 'else': + self.expect('else') + else_body, _ = self.parse_block() + else: + else_body = None + if is_error is not None: + node = WhileStmt(expr, body, else_body) + return node + else: + return None + + def parse_for_stmt(self) -> ForStmt: + self.expect('for') + index = self.parse_for_index_variables() + self.expect('in') + expr = self.parse_expression() + + body, _ = self.parse_block() + + if self.current_str() == 'else': + self.expect('else') + else_body, _ = self.parse_block() + else: + else_body = None + + node = ForStmt(index, expr, body, else_body) + return node + + def parse_for_index_variables(self) -> Node: + # Parse index variables of a 'for' statement. + index_items = [] + force_tuple = False + + while True: + v = self.parse_expression(precedence['in'], + star_expr_allowed=True) # Prevent parsing of for stmt 'in' + index_items.append(v) + if self.current_str() != ',': + break + self.skip() + if self.current_str() == 'in': + force_tuple = True + break + + if len(index_items) == 1 and not force_tuple: + index = index_items[0] + else: + index = TupleExpr(index_items) + index.set_line(index_items[0].get_line()) + + return index + + def parse_if_stmt(self) -> IfStmt: + is_error = False + + self.expect('if') + expr = [] + try: + expr.append(self.parse_expression()) + except ParseError: + is_error = True + + body = [self.parse_block()[0]] + + while self.current_str() == 'elif': + self.expect('elif') + try: + expr.append(self.parse_expression()) + except ParseError: + is_error = True + body.append(self.parse_block()[0]) + + if self.current_str() == 'else': + self.expect('else') + else_body, _ = self.parse_block() + else: + else_body = None + + if not is_error: + node = IfStmt(expr, body, else_body) + return node + else: + return None + + def parse_try_stmt(self) -> Node: + self.expect('try') + body, _ = self.parse_block() + is_error = False + vars = [] # type: List[NameExpr] + types = [] # type: List[Node] + handlers = [] # type: List[Block] + while self.current_str() == 'except': + self.expect('except') + if not isinstance(self.current(), Colon): + try: + t = self.current() + types.append(self.parse_expression(precedence[',']).set_line(t)) + if self.current_str() == 'as': + self.expect('as') + vars.append(self.parse_name_expr()) + elif self.pyversion[0] == 2 and self.current_str() == ',': + self.expect(',') + vars.append(self.parse_name_expr()) + else: + vars.append(None) + except ParseError: + is_error = True + else: + types.append(None) + vars.append(None) + handlers.append(self.parse_block()[0]) + if not is_error: + if self.current_str() == 'else': + self.skip() + else_body, _ = self.parse_block() + else: + else_body = None + if self.current_str() == 'finally': + self.expect('finally') + finally_body, _ = self.parse_block() + else: + finally_body = None + node = TryStmt(body, vars, types, handlers, else_body, + finally_body) + return node + else: + return None + + def parse_with_stmt(self) -> WithStmt: + self.expect('with') + exprs = [] + targets = [] + while True: + expr = self.parse_expression(precedence[',']) + if self.current_str() == 'as': + self.expect('as') + target = self.parse_expression(precedence[',']) + else: + target = None + exprs.append(expr) + targets.append(target) + if self.current_str() != ',': + break + self.expect(',') + body, _ = self.parse_block() + return WithStmt(exprs, targets, body) + + def parse_print_stmt(self) -> PrintStmt: + self.expect('print') + args = [] + target = None + if self.current_str() == '>>': + self.skip() + target = self.parse_expression(precedence[',']) + if self.current_str() == ',': + self.skip() + if isinstance(self.current(), Break): + self.parse_error() + else: + if not isinstance(self.current(), Break): + self.parse_error() + comma = False + while not isinstance(self.current(), Break): + args.append(self.parse_expression(precedence[','])) + if self.current_str() == ',': + comma = True + self.skip() + else: + comma = False + break + return PrintStmt(args, newline=not comma, target=target) + + def parse_exec_stmt(self) -> ExecStmt: + self.expect('exec') + expr = self.parse_expression(precedence['in']) + variables1 = None + variables2 = None + if self.current_str() == 'in': + self.skip() + variables1 = self.parse_expression(precedence[',']) + if self.current_str() == ',': + self.skip() + variables2 = self.parse_expression(precedence[',']) + return ExecStmt(expr, variables1, variables2) + + # Parsing expressions + + def parse_expression(self, prec: int = 0, star_expr_allowed: bool = False) -> Node: + """Parse a subexpression within a specific precedence context.""" + expr = None # type: Node + current = self.current() # Remember token for setting the line number. + + # Parse a "value" expression or unary operator expression and store + # that in expr. + s = self.current_str() + if s == '(': + # Parenthesised expression or cast. + expr = self.parse_parentheses() + elif s == '[': + expr = self.parse_list_expr() + elif s in ['-', '+', 'not', '~']: + # Unary operation. + expr = self.parse_unary_expr() + elif s == 'lambda': + expr = self.parse_lambda_expr() + elif s == '{': + expr = self.parse_dict_or_set_expr() + elif s == '*' and star_expr_allowed: + expr = self.parse_star_expr() + elif s == '`' and self.pyversion[0] == 2: + expr = self.parse_backquote_expr() + else: + if isinstance(current, Name): + # Name expression. + expr = self.parse_name_expr() + elif isinstance(current, IntLit): + expr = self.parse_int_expr() + elif isinstance(current, StrLit): + expr = self.parse_str_expr() + elif isinstance(current, BytesLit): + expr = self.parse_bytes_literal() + elif isinstance(current, UnicodeLit): + expr = self.parse_unicode_literal() + elif isinstance(current, FloatLit): + expr = self.parse_float_expr() + elif isinstance(current, ComplexLit): + expr = self.parse_complex_expr() + elif isinstance(current, Keyword) and s == "yield": + # The expression yield from and yield to assign + expr = self.parse_yield_or_yield_from_expr() + elif isinstance(current, EllipsisToken) and (self.pyversion[0] >= 3 + or self.is_stub_file): + expr = self.parse_ellipsis() + else: + # Invalid expression. + self.parse_error() + + # Set the line of the expression node, if not specified. This + # simplifies recording the line number as not every node type needs to + # deal with it separately. + if expr.line < 0: + expr.set_line(current) + + # Parse operations that require a left argument (stored in expr). + while True: + current = self.current() + s = self.current_str() + if s == '(': + # Call expression. + expr = self.parse_call_expr(expr) + elif s == '.': + # Member access expression. + expr = self.parse_member_expr(expr) + elif s == '[': + # Indexing expression. + expr = self.parse_index_expr(expr) + elif s == ',': + # The comma operator is used to build tuples. Comma also + # separates array items and function arguments, but in this + # case the precedence is too low to build a tuple. + if precedence[','] > prec: + expr = self.parse_tuple_expr(expr) + else: + break + elif s == 'for': + if precedence[''] > prec: + # List comprehension or generator expression. Parse as + # generator expression; it will be converted to list + # comprehension if needed elsewhere. + expr = self.parse_generator_expr(expr) + else: + break + elif s == 'if': + # Conditional expression. + if precedence[''] > prec: + expr = self.parse_conditional_expr(expr) + else: + break + else: + # Binary operation or a special case. + if isinstance(current, Op): + op = self.current_str() + op_prec = precedence[op] + if op == 'not': + # Either "not in" or an error. + op_prec = precedence['in'] + if op_prec > prec: + if op in op_comp: + expr = self.parse_comparison_expr(expr, op_prec) + else: + expr = self.parse_bin_op_expr(expr, op_prec) + else: + # The operation cannot be associated with the + # current left operand due to the precedence + # context; let the caller handle it. + break + else: + # Not an operation that accepts a left argument; let the + # caller handle the rest. + break + + # Set the line of the expression node, if not specified. This + # simplifies recording the line number as not every node type + # needs to deal with it separately. + if expr.line < 0: + expr.set_line(current) + + return expr + + def parse_parentheses(self) -> Node: + self.skip() + if self.current_str() == ')': + # Empty tuple (). + expr = self.parse_empty_tuple_expr() # type: Node + else: + # Parenthesised expression. + expr = self.parse_expression(0, star_expr_allowed=True) + self.expect(')') + return expr + + def parse_star_expr(self) -> Node: + star = self.expect('*') + expr = self.parse_expression(precedence['*u']) + expr = StarExpr(expr) + if expr.line < 0: + expr.set_line(star) + return expr + + def parse_empty_tuple_expr(self) -> TupleExpr: + self.expect(')') + node = TupleExpr([]) + return node + + def parse_list_expr(self) -> Node: + """Parse list literal or list comprehension.""" + items = [] + self.expect('[') + while self.current_str() != ']' and not self.eol(): + items.append(self.parse_expression(precedence[''], star_expr_allowed=True)) + if self.current_str() != ',': + break + self.expect(',') + if self.current_str() == 'for' and len(items) == 1: + items[0] = self.parse_generator_expr(items[0]) + self.expect(']') + if len(items) == 1 and isinstance(items[0], GeneratorExpr): + return ListComprehension(items[0]) + else: + expr = ListExpr(items) + return expr + + def parse_generator_expr(self, left_expr: Node) -> GeneratorExpr: + tok = self.current() + indices, sequences, condlists = self.parse_comp_for() + + gen = GeneratorExpr(left_expr, indices, sequences, condlists) + gen.set_line(tok) + return gen + + def parse_comp_for(self) -> Tuple[List[Node], List[Node], List[List[Node]]]: + indices = [] + sequences = [] + condlists = [] # type: List[List[Node]] + while self.current_str() == 'for': + conds = [] + self.expect('for') + index = self.parse_for_index_variables() + indices.append(index) + self.expect('in') + if self.pyversion[0] >= 3: + sequence = self.parse_expression(precedence['']) + else: + sequence = self.parse_expression_list() + sequences.append(sequence) + while self.current_str() == 'if': + self.skip() + conds.append(self.parse_expression(precedence[''])) + condlists.append(conds) + + return indices, sequences, condlists + + def parse_expression_list(self) -> Node: + prec = precedence[''] + expr = self.parse_expression(prec) + if self.current_str() != ',': + return expr + else: + t = self.current() + return self.parse_tuple_expr(expr, prec).set_line(t) + + def parse_conditional_expr(self, left_expr: Node) -> ConditionalExpr: + self.expect('if') + cond = self.parse_expression(precedence['']) + self.expect('else') + else_expr = self.parse_expression(precedence['']) + return ConditionalExpr(cond, left_expr, else_expr) + + def parse_dict_or_set_expr(self) -> Node: + items = [] # type: List[Tuple[Node, Node]] + self.expect('{') + while self.current_str() != '}' and not self.eol(): + key = self.parse_expression(precedence['']) + if self.current_str() in [',', '}'] and items == []: + return self.parse_set_expr(key) + elif self.current_str() == 'for' and items == []: + return self.parse_set_comprehension(key) + elif self.current_str() != ':': + self.parse_error() + colon = self.expect(':') + value = self.parse_expression(precedence['']) + if self.current_str() == 'for' and items == []: + return self.parse_dict_comprehension(key, value, colon) + items.append((key, value)) + if self.current_str() != ',': + break + self.expect(',') + self.expect('}') + node = DictExpr(items) + return node + + def parse_set_expr(self, first: Node) -> SetExpr: + items = [first] + while self.current_str() != '}' and not self.eol(): + self.expect(',') + if self.current_str() == '}': + break + items.append(self.parse_expression(precedence[','])) + self.expect('}') + expr = SetExpr(items) + return expr + + def parse_set_comprehension(self, expr: Node): + gen = self.parse_generator_expr(expr) + self.expect('}') + set_comp = SetComprehension(gen) + return set_comp + + def parse_dict_comprehension(self, key: Node, value: Node, + colon: Token) -> DictionaryComprehension: + indices, sequences, condlists = self.parse_comp_for() + dic = DictionaryComprehension(key, value, indices, sequences, condlists) + dic.set_line(colon) + self.expect('}') + return dic + + def parse_tuple_expr(self, expr: Node, + prec: int = precedence[',']) -> TupleExpr: + items = [expr] + while True: + self.expect(',') + if (self.current_str() in [')', ']', '=', ':'] or + isinstance(self.current(), Break)): + break + items.append(self.parse_expression(prec, star_expr_allowed=True)) + if self.current_str() != ',': break + node = TupleExpr(items) + return node + + def parse_backquote_expr(self) -> BackquoteExpr: + self.expect('`') + expr = self.parse_expression() + self.expect('`') + return BackquoteExpr(expr) + + def parse_name_expr(self) -> NameExpr: + tok = self.expect_type(Name) + node = NameExpr(tok.string) + node.set_line(tok) + return node + + octal_int = re.compile('0+[1-9]') + + def parse_int_expr(self) -> IntExpr: + tok = self.expect_type(IntLit) + string = tok.string.rstrip('lL') # Strip L prefix (Python 2 long literals) + if self.octal_int.match(string): + value = int(string, 8) + else: + value = int(string, 0) + node = IntExpr(value) + return node + + def parse_str_expr(self) -> Node: + # XXX \uxxxx literals + token = self.expect_type(StrLit) + value = cast(StrLit, token).parsed() + is_unicode = False + while isinstance(self.current(), (StrLit, UnicodeLit)): + token = self.skip() + if isinstance(token, StrLit): + value += token.parsed() + elif isinstance(token, UnicodeLit): + value += token.parsed() + is_unicode = True + if is_unicode or (self.pyversion[0] == 2 and 'unicode_literals' in self.future_options): + node = UnicodeExpr(value) # type: Node + else: + node = StrExpr(value) + return node + + def parse_bytes_literal(self) -> Node: + # XXX \uxxxx literals + tok = [self.expect_type(BytesLit)] + value = (cast(BytesLit, tok[0])).parsed() + while isinstance(self.current(), BytesLit): + t = cast(BytesLit, self.skip()) + value += t.parsed() + if self.pyversion[0] >= 3: + node = BytesExpr(value) # type: Node + else: + node = StrExpr(value) + return node + + def parse_unicode_literal(self) -> Node: + # XXX \uxxxx literals + token = self.expect_type(UnicodeLit) + value = cast(UnicodeLit, token).parsed() + while isinstance(self.current(), (UnicodeLit, StrLit)): + token = cast(Union[UnicodeLit, StrLit], self.skip()) + value += token.parsed() + if self.pyversion[0] >= 3: + # Python 3.3 supports u'...' as an alias of '...'. + node = StrExpr(value) # type: Node + else: + node = UnicodeExpr(value) + return node + + def parse_float_expr(self) -> FloatExpr: + tok = self.expect_type(FloatLit) + node = FloatExpr(float(tok.string)) + return node + + def parse_complex_expr(self) -> ComplexExpr: + tok = self.expect_type(ComplexLit) + node = ComplexExpr(complex(tok.string)) + return node + + def parse_call_expr(self, callee: Any) -> CallExpr: + self.expect('(') + args, kinds, names = self.parse_arg_expr() + self.expect(')') + node = CallExpr(callee, args, kinds, names) + return node + + def parse_arg_expr(self) -> Tuple[List[Node], List[int], List[str]]: + """Parse arguments in a call expression (within '(' and ')'). + + Return a tuple with these items: + argument expressions + argument kinds + argument names (for named arguments; None for ordinary args) + """ + args = [] # type: List[Node] + kinds = [] # type: List[int] + names = [] # type: List[str] + var_arg = False + dict_arg = False + named_args = False + while self.current_str() != ')' and not self.eol() and not dict_arg: + if isinstance(self.current(), Name) and self.peek().string == '=': + # Named argument + name = self.expect_type(Name) + self.expect('=') + kinds.append(nodes.ARG_NAMED) + names.append(name.string) + named_args = True + elif (self.current_str() == '*' and not var_arg and not dict_arg): + # *args + var_arg = True + self.expect('*') + kinds.append(nodes.ARG_STAR) + names.append(None) + elif self.current_str() == '**': + # **kwargs + self.expect('**') + dict_arg = True + kinds.append(nodes.ARG_STAR2) + names.append(None) + elif not var_arg and not named_args: + # Ordinary argument + kinds.append(nodes.ARG_POS) + names.append(None) + else: + self.parse_error() + args.append(self.parse_expression(precedence[','])) + if self.current_str() != ',': + break + self.expect(',') + return args, kinds, names + + def parse_member_expr(self, expr: Any) -> Node: + self.expect('.') + name = self.expect_type(Name) + if (isinstance(expr, CallExpr) and isinstance(expr.callee, NameExpr) + and expr.callee.name == 'super'): + # super() expression + node = SuperExpr(name.string) # type: Node + else: + node = MemberExpr(expr, name.string) + return node + + def parse_index_expr(self, base: Any) -> IndexExpr: + self.expect('[') + index = self.parse_slice_item() + if self.current_str() == ',': + # Extended slicing such as x[1:, :2]. + items = [index] + while self.current_str() == ',': + self.skip() + if self.current_str() == ']' or isinstance(self.current(), Break): + break + items.append(self.parse_slice_item()) + index = TupleExpr(items) + index.set_line(items[0].line) + self.expect(']') + node = IndexExpr(base, index) + return node + + def parse_slice_item(self) -> Node: + if self.current_str() != ':': + if self.current_str() == '...': + # Ellipsis is valid here even in Python 2 (but not elsewhere). + ellipsis = EllipsisExpr() + token = self.skip() + ellipsis.set_line(token) + return ellipsis + else: + item = self.parse_expression(precedence[',']) + else: + item = None + if self.current_str() == ':': + # Slice. + index = item + colon = self.expect(':') + if self.current_str() not in (']', ':', ','): + end_index = self.parse_expression(precedence[',']) + else: + end_index = None + stride = None + if self.current_str() == ':': + self.expect(':') + if self.current_str() not in (']', ','): + stride = self.parse_expression(precedence[',']) + item = SliceExpr(index, end_index, stride).set_line(colon.line) + return item + + def parse_bin_op_expr(self, left: Node, prec: int) -> OpExpr: + op = self.expect_type(Op) + op_str = op.string + if op_str == '~': + self.ind -= 1 + self.parse_error() + right = self.parse_expression(prec) + node = OpExpr(op_str, left, right) + return node + + def parse_comparison_expr(self, left: Node, prec: int) -> ComparisonExpr: + operators_str = [] + operands = [left] + + while True: + op = self.expect_type(Op) + op_str = op.string + if op_str == 'not': + if self.current_str() == 'in': + op_str = 'not in' + self.skip() + else: + self.parse_error() + elif op_str == 'is' and self.current_str() == 'not': + op_str = 'is not' + self.skip() + + operators_str.append(op_str) + operand = self.parse_expression(prec) + operands.append(operand) + + # Continue if next token is a comparison operator + self.current() + s = self.current_str() + if s not in op_comp: + break + + node = ComparisonExpr(operators_str, operands) + return node + + def parse_unary_expr(self) -> UnaryExpr: + op_tok = self.skip() + op = op_tok.string + if op == '-' or op == '+': + prec = precedence['-u'] + else: + prec = precedence[op] + expr = self.parse_expression(prec) + node = UnaryExpr(op, expr) + return node + + def parse_lambda_expr(self) -> FuncExpr: + lambda_tok = self.expect('lambda') + + args, extra_stmts = self.parse_arg_list(allow_signature=False) + + # Use 'object' as the placeholder return type; it will be inferred + # later. We can't use 'Any' since it could make type inference results + # less precise. + ret_type = UnboundType('__builtins__.object') + typ = self.build_func_annotation(ret_type, args, + lambda_tok.line, is_default_ret=True) + + colon = self.expect(':') + + expr = self.parse_expression(precedence[',']) + + nodes = [ReturnStmt(expr).set_line(lambda_tok)] + # Potentially insert extra assignment statements to the beginning of the + # body, used to decompose Python 2 tuple arguments. + nodes[:0] = extra_stmts + body = Block(nodes) + body.set_line(colon) + + return FuncExpr(args, body, typ) + + # Helper methods + + def skip(self) -> Token: + self.ind += 1 + return self.tok[self.ind - 1] + + def expect(self, string: str) -> Token: + if self.current_str() == string: + self.ind += 1 + return self.tok[self.ind - 1] + else: + self.parse_error() + + def expect_indent(self) -> Token: + if isinstance(self.current(), Indent): + return self.expect_type(Indent) + else: + self.fail('Expected an indented block', self.current().line) + return none + + def fail(self, msg: str, line: int) -> None: + self.errors.report(line, msg) + + def expect_type(self, typ: type) -> Token: + current = self.current() + if isinstance(current, typ): + self.ind += 1 + return current + else: + self.parse_error() + + def expect_colon_and_break(self) -> Tuple[Token, Token]: + return self.expect_type(Colon), self.expect_type(Break) + + def expect_break(self) -> Token: + return self.expect_type(Break) + + def current(self) -> Token: + return self.tok[self.ind] + + def current_str(self) -> str: + return self.current().string + + def peek(self) -> Token: + return self.tok[self.ind + 1] + + def parse_error(self) -> None: + self.parse_error_at(self.current()) + raise ParseError() + + def parse_error_at(self, tok: Token, skip: bool = True, reason: Optional[str] = None) -> None: + msg = '' + if isinstance(tok, LexError): + msg = token_repr(tok) + msg = msg[0].upper() + msg[1:] + elif isinstance(tok, Indent) or isinstance(tok, Dedent): + msg = 'Inconsistent indentation' + else: + formatted_reason = ": {}".format(reason) if reason else "" + msg = 'Parse error before {}{}'.format(token_repr(tok), formatted_reason) + + self.errors.report(tok.line, msg) + + if skip: + self.skip_until_next_line() + + def skip_until_break(self) -> None: + n = 0 + while (not isinstance(self.current(), Break) + and not isinstance(self.current(), Eof)): + self.skip() + n += 1 + if isinstance(self.tok[self.ind - 1], Colon) and n > 1: + self.ind -= 1 + + def skip_until_next_line(self) -> None: + self.skip_until_break() + if isinstance(self.current(), Break): + self.skip() + + def eol(self) -> bool: + return isinstance(self.current(), Break) or self.eof() + + def eof(self) -> bool: + return isinstance(self.current(), Eof) + + # Type annotation related functionality + + def parse_type(self) -> Type: + try: + typ, self.ind = parse_type(self.tok, self.ind) + except TypeParseError as e: + self.parse_error_at(e.token, reason=e.message) + raise ParseError() + return typ + + annotation_prefix_re = re.compile(r'#\s*type:') + ignore_prefix_re = re.compile(r'ignore\b') + + def parse_type_comment(self, token: Token, signature: bool) -> Type: + """Parse a '# type: ...' annotation. + + Return None if no annotation found. If signature is True, expect + a type signature of form (...) -> t. + """ + whitespace_or_comments = token.rep().strip() + if self.annotation_prefix_re.match(whitespace_or_comments): + type_as_str = whitespace_or_comments.split(':', 1)[1].strip() + if self.ignore_prefix_re.match(type_as_str): + # Actually a "# type: ignore" annotation -> not a type. + return None + tokens = lex.lex(type_as_str, token.line)[0] + if len(tokens) < 2: + # Empty annotation (only Eof token) + self.errors.report(token.line, 'Empty type annotation') + return None + try: + if not signature: + type, index = parse_types(tokens, 0) + else: + type, index = parse_signature(tokens) + except TypeParseError as e: + self.parse_error_at(e.token, skip=False, reason=e.message) + return None + if index < len(tokens) - 2: + self.parse_error_at(tokens[index], skip=False) + return None + return type + else: + return None + + +class ParseError(Exception): pass + + +def token_repr(tok: Token) -> str: + """Return a representation of a token for use in parse error messages.""" + if isinstance(tok, Break): + return 'end of line' + elif isinstance(tok, Eof): + return 'end of file' + elif isinstance(tok, Keyword) or isinstance(tok, Name): + return '"{}"'.format(tok.string) + elif isinstance(tok, IntLit) or isinstance(tok, FloatLit) or isinstance(tok, ComplexLit): + return 'numeric literal' + elif isinstance(tok, StrLit) or isinstance(tok, UnicodeLit): + return 'string literal' + elif (isinstance(tok, Punct) or isinstance(tok, Op) + or isinstance(tok, Colon)): + return tok.string + elif isinstance(tok, Bom): + return 'byte order mark' + elif isinstance(tok, Indent): + return 'indent' + elif isinstance(tok, Dedent): + return 'dedent' + elif isinstance(tok, EllipsisToken): + return '...' + else: + if isinstance(tok, LexError): + t = tok.type + if t == lex.NUMERIC_LITERAL_ERROR: + return 'invalid numeric literal' + elif t == lex.UNTERMINATED_STRING_LITERAL: + return 'unterminated string literal' + elif t == lex.INVALID_CHARACTER: + msg = 'unrecognized character' + if ord(tok.string) in range(33, 127): + msg += ' ' + tok.string + return msg + elif t == lex.INVALID_DEDENT: + return 'inconsistent indentation' + elif t == lex.DECODE_ERROR: + return tok.message + raise ValueError('Unknown token {}'.format(repr(tok))) + + +if __name__ == '__main__': + # Parse a file and dump the AST (or display errors). + import sys + + def usage(): + print('Usage: parse.py [--py2] [--quiet] FILE [...]', file=sys.stderr) + sys.exit(2) + + args = sys.argv[1:] + pyversion = defaults.PYTHON3_VERSION + quiet = False + while args and args[0].startswith('--'): + if args[0] == '--py2': + pyversion = defaults.PYTHON2_VERSION + elif args[0] == '--quiet': + quiet = True + else: + usage() + args = args[1:] + if len(args) < 1: + usage() + status = 0 + for fnam in args: + s = open(fnam, 'rb').read() + errors = Errors() + try: + options = Options() + options.python_version = pyversion + tree = parse(s, fnam, None, options=options) + if not quiet: + print(tree) + except CompileError as e: + for msg in e.messages: + sys.stderr.write('%s\n' % msg) + status = 1 + sys.exit(status) diff --git a/mypy/semanal.py b/mypy/semanal.py index 8ab29f560a4e7..0f4331499d7cc 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1,2839 +1,2839 @@ -"""The semantic analyzer. - -Bind names to definitions and do various other simple consistency -checks. For example, consider this program: - - x = 1 - y = x - -Here semantic analysis would detect that the assignment 'x = 1' -defines a new variable, the type of which is to be inferred (in a -later pass; type inference or type checking is not part of semantic -analysis). Also, it would bind both references to 'x' to the same -module-level variable node. The second assignment would also be -analyzed, and the type of 'y' marked as being inferred. - -Semantic analysis is the first analysis pass after parsing, and it is -subdivided into three passes: - - * FirstPass looks up externally visible names defined in a module but - ignores imports and local definitions. It helps enable (some) - cyclic references between modules, such as module 'a' that imports - module 'b' and used names defined in b *and* vice versa. The first - pass can be performed before dependent modules have been processed. - - * SemanticAnalyzer is the second pass. It does the bulk of the work. - It assumes that dependent modules have been semantically analyzed, - up to the second pass, unless there is a import cycle. - - * ThirdPass checks that type argument counts are valid; for example, - it will reject Dict[int]. We don't do this in the second pass, - since we infer the type argument counts of classes during this - pass, and it is possible to refer to classes defined later in a - file, which would not have the type argument count set yet. This - pass also recomputes the method resolution order of each class, in - case one of its bases belongs to a module involved in an import - loop. - -Semantic analysis of types is implemented in module mypy.typeanal. - -TODO: Check if the third pass slows down type checking significantly. - We could probably get rid of it -- for example, we could collect all - analyzed types in a collection and check them without having to - traverse the entire AST. -""" - -from typing import ( - List, Dict, Set, Tuple, cast, Any, overload, TypeVar, Union, Optional, Callable -) - -from mypy.nodes import ( - MypyFile, TypeInfo, Node, AssignmentStmt, FuncDef, OverloadedFuncDef, - ClassDef, Var, GDEF, MODULE_REF, FuncItem, Import, - ImportFrom, ImportAll, Block, LDEF, NameExpr, MemberExpr, - IndexExpr, TupleExpr, ListExpr, ExpressionStmt, ReturnStmt, - RaiseStmt, AssertStmt, OperatorAssignmentStmt, WhileStmt, - ForStmt, BreakStmt, ContinueStmt, IfStmt, TryStmt, WithStmt, DelStmt, - GlobalDecl, SuperExpr, DictExpr, CallExpr, RefExpr, OpExpr, UnaryExpr, - SliceExpr, CastExpr, RevealTypeExpr, TypeApplication, Context, SymbolTable, - SymbolTableNode, BOUND_TVAR, UNBOUND_TVAR, ListComprehension, GeneratorExpr, - FuncExpr, MDEF, FuncBase, Decorator, SetExpr, TypeVarExpr, - StrExpr, BytesExpr, PrintStmt, ConditionalExpr, PromoteExpr, - ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, MroError, type_aliases, - YieldFromExpr, NamedTupleExpr, NonlocalDecl, - SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr, - YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, COVARIANT, CONTRAVARIANT, - IntExpr, FloatExpr, UnicodeExpr, - INVARIANT, UNBOUND_IMPORTED, Expression, EllipsisExpr, NamedTupleTypeInfo -) -from mypy.visitor import NodeVisitor -from mypy.traverser import TraverserVisitor -from mypy.errors import Errors, report_internal_error -from mypy.types import ( - NoneTyp, CallableType, Overloaded, Instance, Type, TypeVarType, AnyType, - FunctionLike, UnboundType, TypeList, ErrorType, TypeVarDef, - replace_leading_arg_type, TupleType, UnionType, StarType, EllipsisType, - NamedTupleType -) -from mypy.nodes import function_type, implicit_module_attrs -from mypy.typeanal import TypeAnalyser, TypeAnalyserPass3, analyze_type_alias -from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError -from mypy.lex import lex -from mypy.parsetype import parse_type -from mypy.sametypes import is_same_type -from mypy.erasetype import erase_typevars -from mypy import defaults -from mypy.options import Options - - -T = TypeVar('T') - - -# Inferred value of an expression. -ALWAYS_TRUE = 0 -ALWAYS_FALSE = 1 -TRUTH_VALUE_UNKNOWN = 2 - -# Map from obsolete name to the current spelling. -obsolete_name_mapping = { - 'typing.Function': 'typing.Callable', - 'typing.typevar': 'typing.TypeVar', -} - -# Hard coded type promotions (shared between all Python versions). -# These add extra ad-hoc edges to the subtyping relation. For example, -# int is considered a subtype of float, even though there is no -# subclass relationship. -TYPE_PROMOTIONS = { - 'builtins.int': 'builtins.float', - 'builtins.float': 'builtins.complex', -} - -# Hard coded type promotions for Python 3. -# -# Note that the bytearray -> bytes promotion is a little unsafe -# as some functions only accept bytes objects. Here convenience -# trumps safety. -TYPE_PROMOTIONS_PYTHON3 = TYPE_PROMOTIONS.copy() -TYPE_PROMOTIONS_PYTHON3.update({ - 'builtins.bytearray': 'builtins.bytes', -}) - -# Hard coded type promotions for Python 2. -# -# These promotions are unsafe, but we are doing them anyway -# for convenience and also for Python 3 compatibility -# (bytearray -> str). -TYPE_PROMOTIONS_PYTHON2 = TYPE_PROMOTIONS.copy() -TYPE_PROMOTIONS_PYTHON2.update({ - 'builtins.str': 'builtins.unicode', - 'builtins.bytearray': 'builtins.str', -}) - -# When analyzing a function, should we analyze the whole function in one go, or -# should we only perform one phase of the analysis? The latter is used for -# nested functions. In the first phase we add the function to the symbol table -# but don't process body. In the second phase we process function body. This -# way we can have mutually recursive nested functions. -FUNCTION_BOTH_PHASES = 0 # Everthing in one go -FUNCTION_FIRST_PHASE_POSTPONE_SECOND = 1 # Add to symbol table but postpone body -FUNCTION_SECOND_PHASE = 2 # Only analyze body - - -class SemanticAnalyzer(NodeVisitor): - """Semantically analyze parsed mypy files. - - The analyzer binds names and does various consistency checks for a - parse tree. Note that type checking is performed as a separate - pass. - - This is the second phase of semantic analysis. - """ - - # Library search paths - lib_path = None # type: List[str] - # Module name space - modules = None # type: Dict[str, MypyFile] - # Global name space for current module - globals = None # type: SymbolTable - # Names declared using "global" (separate set for each scope) - global_decls = None # type: List[Set[str]] - # Names declated using "nonlocal" (separate set for each scope) - nonlocal_decls = None # type: List[Set[str]] - # Local names of function scopes; None for non-function scopes. - locals = None # type: List[SymbolTable] - # Nested block depths of scopes - block_depth = None # type: List[int] - # TypeInfo of directly enclosing class (or None) - type = None # type: TypeInfo - # Stack of outer classes (the second tuple item contains tvars). - type_stack = None # type: List[TypeInfo] - # Type variables that are bound by the directly enclosing class - bound_tvars = None # type: List[SymbolTableNode] - # Stack of type variables that were bound by outer classess - tvar_stack = None # type: List[List[SymbolTableNode]] - # Do weak type checking in this file - weak_opts = set() # type: Set[str] - - # Stack of functions being analyzed - function_stack = None # type: List[FuncItem] - # Stack of next available function type variable ids - next_function_tvar_id_stack = None # type: List[int] - - # Status of postponing analysis of nested function bodies. By using this we - # can have mutually recursive nested functions. Values are FUNCTION_x - # constants. Note that separate phasea are not used for methods. - postpone_nested_functions_stack = None # type: List[int] - # Postponed functions collected if - # postpone_nested_functions_stack[-1] == FUNCTION_FIRST_PHASE_POSTPONE_SECOND. - postponed_functions_stack = None # type: List[List[Node]] - - loop_depth = 0 # Depth of breakable loops - cur_mod_id = '' # Current module id (or None) (phase 2) - is_stub_file = False # Are we analyzing a stub file? - imports = None # type: Set[str] # Imported modules (during phase 2 analysis) - errors = None # type: Errors # Keeps track of generated errors - - def __init__(self, - lib_path: List[str], - errors: Errors, - options: Options) -> None: - """Construct semantic analyzer. - - Use lib_path to search for modules, and report analysis errors - using the Errors instance. - """ - self.locals = [None] - self.imports = set() - self.type = None - self.type_stack = [] - self.bound_tvars = None - self.tvar_stack = [] - self.function_stack = [] - self.next_function_tvar_id_stack = [-1] - self.block_depth = [0] - self.loop_depth = 0 - self.lib_path = lib_path - self.errors = errors - self.modules = {} - self.options = options - self.postpone_nested_functions_stack = [FUNCTION_BOTH_PHASES] - self.postponed_functions_stack = [] - self.all_exports = set() # type: Set[str] - - def visit_file(self, file_node: MypyFile, fnam: str) -> None: - self.errors.set_file(fnam) - self.cur_mod_node = file_node - self.cur_mod_id = file_node.fullname() - self.is_stub_file = fnam.lower().endswith('.pyi') - self.globals = file_node.names - self.weak_opts = file_node.weak_opts - - if 'builtins' in self.modules: - self.globals['__builtins__'] = SymbolTableNode( - MODULE_REF, self.modules['builtins'], self.cur_mod_id) - - for name in implicit_module_attrs: - v = self.globals[name].node - if isinstance(v, Var): - v.type = self.anal_type(v.type) - v.is_ready = True - - defs = file_node.defs - for d in defs: - self.accept(d) - - if self.cur_mod_id == 'builtins': - remove_imported_names_from_symtable(self.globals, 'builtins') - - if '__all__' in self.globals: - for name, g in self.globals.items(): - if name not in self.all_exports: - g.module_public = False - - def visit_func_def(self, defn: FuncDef) -> None: - phase_info = self.postpone_nested_functions_stack[-1] - if phase_info != FUNCTION_SECOND_PHASE: - self.function_stack.append(defn) - # First phase of analysis for function. - self.errors.push_function(defn.name()) - self.update_function_type_variables(defn) - self.errors.pop_function() - self.function_stack.pop() - - defn.is_conditional = self.block_depth[-1] > 0 - - # TODO(jukka): Figure out how to share the various cases. It doesn't - # make sense to have (almost) duplicate code (here and elsewhere) for - # 3 cases: module-level, class-level and local names. Maybe implement - # a common stack of namespaces. As the 3 kinds of namespaces have - # different semantics, this wouldn't always work, but it might still - # be a win. - if self.is_class_scope(): - # Method definition - defn.info = self.type - if not defn.is_decorated and not defn.is_overload: - if defn.name() in self.type.names: - # Redefinition. Conditional redefinition is okay. - n = self.type.names[defn.name()].node - if self.is_conditional_func(n, defn): - defn.original_def = cast(FuncDef, n) - else: - self.name_already_defined(defn.name(), defn) - self.type.names[defn.name()] = SymbolTableNode(MDEF, defn) - self.prepare_method_signature(defn) - elif self.is_func_scope(): - # Nested function - if not defn.is_decorated and not defn.is_overload: - if defn.name() in self.locals[-1]: - # Redefinition. Conditional redefinition is okay. - n = self.locals[-1][defn.name()].node - if self.is_conditional_func(n, defn): - defn.original_def = cast(FuncDef, n) - else: - self.name_already_defined(defn.name(), defn) - else: - self.add_local(defn, defn) - else: - # Top-level function - if not defn.is_decorated and not defn.is_overload: - symbol = self.globals.get(defn.name()) - if isinstance(symbol.node, FuncDef) and symbol.node != defn: - # This is redefinition. Conditional redefinition is okay. - original_def = symbol.node - if self.is_conditional_func(original_def, defn): - # Conditional function definition -- multiple defs are ok. - defn.original_def = original_def - else: - # Report error. - self.check_no_global(defn.name(), defn, True) - if phase_info == FUNCTION_FIRST_PHASE_POSTPONE_SECOND: - # Postpone this function (for the second phase). - self.postponed_functions_stack[-1].append(defn) - return - if phase_info != FUNCTION_FIRST_PHASE_POSTPONE_SECOND: - # Second phase of analysis for function. - self.errors.push_function(defn.name()) - self.analyze_function(defn) - self.errors.pop_function() - - def prepare_method_signature(self, func: FuncDef) -> None: - """Check basic signature validity and tweak annotation of self/cls argument.""" - # Only non-static methods are special. - if not func.is_static: - if not func.arguments: - self.fail('Method must have at least one argument', func) - elif func.type: - sig = cast(FunctionLike, func.type) - if func.is_class: - leading_type = self.class_type(self.type) - else: - leading_type = self_type(self.type) - func.type = replace_implicit_first_type(sig, leading_type) - - def is_conditional_func(self, previous: Node, new: FuncDef) -> bool: - """Does 'new' conditionally redefine 'previous'? - - We reject straight redefinitions of functions, as they are usually - a programming error. For example: - - . def f(): ... - . def f(): ... # Error: 'f' redefined - """ - return isinstance(previous, (FuncDef, Var)) and new.is_conditional - - def update_function_type_variables(self, defn: FuncDef) -> None: - """Make any type variables in the signature of defn explicit. - - Update the signature of defn to contain type variable definitions - if defn is generic. - """ - if defn.type: - functype = cast(CallableType, defn.type) - typevars = self.infer_type_variables(functype) - # Do not define a new type variable if already defined in scope. - typevars = [(name, tvar) for name, tvar in typevars - if not self.is_defined_type_var(name, defn)] - if typevars: - next_tvar_id = self.next_function_tvar_id() - defs = [TypeVarDef(tvar[0], next_tvar_id - i, - tvar[1].values, tvar[1].upper_bound, - tvar[1].variance) - for i, tvar in enumerate(typevars)] - functype.variables = defs - - def infer_type_variables(self, - type: CallableType) -> List[Tuple[str, TypeVarExpr]]: - """Return list of unique type variables referred to in a callable.""" - names = [] # type: List[str] - tvars = [] # type: List[TypeVarExpr] - for arg in type.arg_types + [type.ret_type]: - for name, tvar_expr in self.find_type_variables_in_type(arg): - if name not in names: - names.append(name) - tvars.append(tvar_expr) - return list(zip(names, tvars)) - - def find_type_variables_in_type( - self, type: Type) -> List[Tuple[str, TypeVarExpr]]: - """Return a list of all unique type variable references in type. - - This effectively does partial name binding, results of which are mostly thrown away. - """ - result = [] # type: List[Tuple[str, TypeVarExpr]] - if isinstance(type, UnboundType): - name = type.name - node = self.lookup_qualified(name, type) - if node and node.kind == UNBOUND_TVAR: - result.append((name, cast(TypeVarExpr, node.node))) - for arg in type.args: - result.extend(self.find_type_variables_in_type(arg)) - elif isinstance(type, TypeList): - for item in type.items: - result.extend(self.find_type_variables_in_type(item)) - elif isinstance(type, UnionType): - for item in type.items: - result.extend(self.find_type_variables_in_type(item)) - elif isinstance(type, AnyType): - pass - elif isinstance(type, EllipsisType) or isinstance(type, TupleType): - pass - else: - assert False, 'Unsupported type %s' % type - return result - - def is_defined_type_var(self, tvar: str, context: Node) -> bool: - return self.lookup_qualified(tvar, context).kind == BOUND_TVAR - - def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: - t = [] # type: List[CallableType] - for i, item in enumerate(defn.items): - # TODO support decorated overloaded functions properly - item.is_overload = True - item.func.is_overload = True - item.accept(self) - t.append(cast(CallableType, function_type(item.func, - self.builtin_type('builtins.function')))) - if item.func.is_property and i == 0: - # This defines a property, probably with a setter and/or deleter. - self.analyze_property_with_multi_part_definition(defn) - break - if not [dec for dec in item.decorators - if refers_to_fullname(dec, 'typing.overload')]: - self.fail("'overload' decorator expected", item) - - defn.type = Overloaded(t) - defn.type.line = defn.line - - if self.is_class_scope(): - self.type.names[defn.name()] = SymbolTableNode(MDEF, defn, - typ=defn.type) - defn.info = self.type - elif self.is_func_scope(): - self.add_local(defn, defn) - - def analyze_property_with_multi_part_definition(self, defn: OverloadedFuncDef) -> None: - """Analyze a property defined using multiple methods (e.g., using @x.setter). - - Assume that the first method (@property) has already been analyzed. - """ - defn.is_property = True - items = defn.items - for item in items[1:]: - if len(item.decorators) == 1: - node = item.decorators[0] - if isinstance(node, MemberExpr): - if node.name == 'setter': - # The first item represents the entire property. - defn.items[0].var.is_settable_property = True - # Get abstractness from the original definition. - item.func.is_abstract = items[0].func.is_abstract - else: - self.fail("Decorated property not supported", item) - item.func.accept(self) - - def next_function_tvar_id(self) -> int: - return self.next_function_tvar_id_stack[-1] - - def analyze_function(self, defn: FuncItem) -> None: - is_method = self.is_class_scope() - - tvarnodes = self.add_func_type_variables_to_symbol_table(defn) - next_function_tvar_id = min([self.next_function_tvar_id()] + - [n.tvar_def.id.raw_id - 1 for n in tvarnodes]) - self.next_function_tvar_id_stack.append(next_function_tvar_id) - - if defn.type: - # Signature must be analyzed in the surrounding scope so that - # class-level imported names and type variables are in scope. - defn.type = self.anal_type(defn.type) - self.check_function_signature(defn) - if isinstance(defn, FuncDef): - defn.type = set_callable_name(defn.type, defn) - for arg in defn.arguments: - if arg.initializer: - arg.initializer.accept(self) - self.function_stack.append(defn) - self.enter() - for arg in defn.arguments: - self.add_local(arg.variable, defn) - for arg in defn.arguments: - if arg.initialization_statement: - lvalue = arg.initialization_statement.lvalues[0] - lvalue.accept(self) - - # The first argument of a non-static, non-class method is like 'self' - # (though the name could be different), having the enclosing class's - # instance type. - if is_method and not defn.is_static and not defn.is_class and defn.arguments: - defn.arguments[0].variable.is_self = True - - # First analyze body of the function but ignore nested functions. - self.postpone_nested_functions_stack.append(FUNCTION_FIRST_PHASE_POSTPONE_SECOND) - self.postponed_functions_stack.append([]) - defn.body.accept(self) - - # Analyze nested functions (if any) as a second phase. - self.postpone_nested_functions_stack[-1] = FUNCTION_SECOND_PHASE - for postponed in self.postponed_functions_stack[-1]: - postponed.accept(self) - self.postpone_nested_functions_stack.pop() - self.postponed_functions_stack.pop() - - self.next_function_tvar_id_stack.pop() - disable_typevars(tvarnodes) - - self.leave() - self.function_stack.pop() - - def add_func_type_variables_to_symbol_table( - self, defn: FuncItem) -> List[SymbolTableNode]: - nodes = [] # type: List[SymbolTableNode] - if defn.type: - tt = defn.type - names = self.type_var_names() - items = cast(CallableType, tt).variables - for item in items: - name = item.name - if name in names: - self.name_already_defined(name, defn) - node = self.bind_type_var(name, item, defn) - nodes.append(node) - names.add(name) - return nodes - - def type_var_names(self) -> Set[str]: - if not self.type: - return set() - else: - return set(self.type.type_vars) - - def bind_type_var(self, fullname: str, tvar_def: TypeVarDef, - context: Context) -> SymbolTableNode: - node = self.lookup_qualified(fullname, context) - node.kind = BOUND_TVAR - node.tvar_def = tvar_def - return node - - def check_function_signature(self, fdef: FuncItem) -> None: - sig = cast(CallableType, fdef.type) - if len(sig.arg_types) < len(fdef.arguments): - self.fail('Type signature has too few arguments', fdef) - # Add dummy Any arguments to prevent crashes later. - extra_anys = [AnyType()] * (len(fdef.arguments) - len(sig.arg_types)) - sig.arg_types.extend(extra_anys) - elif len(sig.arg_types) > len(fdef.arguments): - self.fail('Type signature has too many arguments', fdef) - - def visit_class_def(self, defn: ClassDef) -> None: - self.clean_up_bases_and_infer_type_variables(defn) - self.setup_class_def_analysis(defn) - - self.bind_class_type_vars(defn) - - self.analyze_base_classes(defn) - self.analyze_metaclass(defn) - - for decorator in defn.decorators: - self.analyze_class_decorator(defn, decorator) - - self.enter_class(defn) - - self.setup_is_builtinclass(defn) - - # Analyze class body. - defn.defs.accept(self) - - self.calculate_abstract_status(defn.info) - self.setup_type_promotion(defn) - - self.leave_class() - self.unbind_class_type_vars() - - def enter_class(self, defn: ClassDef) -> None: - # Remember previous active class - self.type_stack.append(self.type) - self.locals.append(None) # Add class scope - self.block_depth.append(-1) # The class body increments this to 0 - self.postpone_nested_functions_stack.append(FUNCTION_BOTH_PHASES) - self.type = defn.info - - def leave_class(self) -> None: - """ Restore analyzer state. """ - self.postpone_nested_functions_stack.pop() - self.block_depth.pop() - self.locals.pop() - self.type = self.type_stack.pop() - - def bind_class_type_vars(self, defn: ClassDef) -> None: - """ Unbind type variables of previously active class and bind - the type variables for the active class. - """ - if self.bound_tvars: - disable_typevars(self.bound_tvars) - self.tvar_stack.append(self.bound_tvars) - self.bound_tvars = self.bind_class_type_variables_in_symbol_table(defn.info) - - def unbind_class_type_vars(self) -> None: - """ Unbind the active class' type vars and rebind the - type vars of the previously active class. - """ - disable_typevars(self.bound_tvars) - self.bound_tvars = self.tvar_stack.pop() - if self.bound_tvars: - enable_typevars(self.bound_tvars) - - def analyze_class_decorator(self, defn: ClassDef, decorator: Node) -> None: - decorator.accept(self) - - def setup_is_builtinclass(self, defn: ClassDef): - for decorator in defn.decorators: - if refers_to_fullname(decorator, 'typing.builtinclass'): - defn.is_builtinclass = True - if defn.fullname == 'builtins.object': - # Only 'object' is marked as a built-in class, as otherwise things elsewhere - # would break. We need a better way of dealing with built-in classes. - defn.is_builtinclass = True - - def calculate_abstract_status(self, typ: TypeInfo) -> None: - """Calculate abstract status of a class. - - Set is_abstract of the type to True if the type has an unimplemented - abstract attribute. Also compute a list of abstract attributes. - """ - concrete = set() # type: Set[str] - abstract = [] # type: List[str] - for base in typ.mro: - for name, symnode in base.names.items(): - node = symnode.node - if isinstance(node, OverloadedFuncDef): - # Unwrap an overloaded function definition. We can just - # check arbitrarily the first overload item. If the - # different items have a different abstract status, there - # should be an error reported elsewhere. - func = node.items[0] # type: Node - else: - func = node - if isinstance(func, Decorator): - fdef = func.func - if fdef.is_abstract and name not in concrete: - typ.is_abstract = True - abstract.append(name) - concrete.add(name) - typ.abstract_attributes = sorted(abstract) - - def setup_type_promotion(self, defn: ClassDef) -> None: - """Setup extra, ad-hoc subtyping relationships between classes (promotion). - - This includes things like 'int' being compatible with 'float'. - """ - promote_target = None # type: Type - for decorator in defn.decorators: - if isinstance(decorator, CallExpr): - analyzed = decorator.analyzed - if isinstance(analyzed, PromoteExpr): - # _promote class decorator (undocumented faeture). - promote_target = analyzed.type - if not promote_target: - promotions = (TYPE_PROMOTIONS_PYTHON3 if self.options.python_version[0] >= 3 - else TYPE_PROMOTIONS_PYTHON2) - if defn.fullname in promotions: - promote_target = self.named_type_or_none(promotions[defn.fullname]) - defn.info._promote = promote_target - - def clean_up_bases_and_infer_type_variables(self, defn: ClassDef) -> None: - """Remove extra base classes such as Generic and infer type vars. - - For example, consider this class: - - . class Foo(Bar, Generic[T]): ... - - Now we will remove Generic[T] from bases of Foo and infer that the - type variable 'T' is a type argument of Foo. - - Note that this is performed *before* semantic analysis. - """ - removed = [] # type: List[int] - type_vars = [] # type: List[TypeVarDef] - for i, base_expr in enumerate(defn.base_type_exprs): - try: - base = expr_to_unanalyzed_type(base_expr) - except TypeTranslationError: - # This error will be caught later. - continue - tvars = self.analyze_typevar_declaration(base) - if tvars is not None: - if type_vars: - self.fail('Duplicate Generic in bases', defn) - removed.append(i) - for j, (name, tvar_expr) in enumerate(tvars): - type_vars.append(TypeVarDef(name, j + 1, tvar_expr.values, - tvar_expr.upper_bound, tvar_expr.variance)) - if type_vars: - defn.type_vars = type_vars - if defn.info: - defn.info.type_vars = [tv.name for tv in type_vars] - for i in reversed(removed): - del defn.base_type_exprs[i] - - def analyze_typevar_declaration(self, t: Type) -> Optional[List[Tuple[str, TypeVarExpr]]]: - if not isinstance(t, UnboundType): - return None - unbound = t - sym = self.lookup_qualified(unbound.name, unbound) - if sym is None or sym.node is None: - return None - if sym.node.fullname() == 'typing.Generic': - tvars = [] # type: List[Tuple[str, TypeVarExpr]] - for arg in unbound.args: - tvar = self.analyze_unbound_tvar(arg) - if tvar: - tvars.append(tvar) - else: - self.fail('Free type variable expected in %s[...]' % - sym.node.name(), t) - return tvars - return None - - def analyze_unbound_tvar(self, t: Type) -> Tuple[str, TypeVarExpr]: - if not isinstance(t, UnboundType): - return None - unbound = t - sym = self.lookup_qualified(unbound.name, unbound) - if sym is not None and sym.kind == UNBOUND_TVAR: - return unbound.name, cast(TypeVarExpr, sym.node) - return None - - def setup_class_def_analysis(self, defn: ClassDef) -> None: - """Prepare for the analysis of a class definition.""" - if not defn.info: - defn.info = TypeInfo(SymbolTable(), defn) - defn.info._fullname = defn.info.name() - if self.is_func_scope() or self.type: - kind = MDEF - if self.is_func_scope(): - kind = LDEF - self.add_symbol(defn.name, SymbolTableNode(kind, defn.info), defn) - - def analyze_base_classes(self, defn: ClassDef) -> None: - """Analyze and set up base classes. - - This computes several attributes on the corresponding TypeInfo defn.info - related to the base classes: defn.info.bases, defn.info.mro, and - miscellaneous others (at least tuple_type, fallback_to_any, and is_enum.) - """ - - base_types = [] # type: List[Instance] - info = defn.info - for base_expr in defn.base_type_exprs: - try: - base = self.expr_to_analyzed_type(base_expr) - except TypeTranslationError: - self.fail('Invalid base class', base_expr) - info.fallback_to_any = True - continue - - if isinstance(base, TupleType): - if info.tuple_type: - self.fail("Class has two incompatible bases derived from tuple", defn) - defn.incompatible_baseclass = True - if (not self.is_stub_file - and not info.is_named_tuple - and base.fallback.type.fullname() == 'builtins.tuple'): - self.fail("Tuple[...] not supported as a base class outside a stub file", defn) - info.tuple_type = base - base_types.append(base.fallback) - elif isinstance(base, Instance): - base_types.append(base) - elif isinstance(base, AnyType): - info.fallback_to_any = True - else: - self.fail('Invalid base class', base_expr) - info.fallback_to_any = True - - # Add 'object' as implicit base if there is no other base class. - if (not base_types and defn.fullname != 'builtins.object'): - base_types.append(self.object_type()) - - info.bases = base_types - - # Calculate the MRO. It might be incomplete at this point if - # the bases of defn include classes imported from other - # modules in an import loop. We'll recompute it in ThirdPass. - if not self.verify_base_classes(defn): - info.mro = [] - return - calculate_class_mro(defn, self.fail) - # If there are cyclic imports, we may be missing 'object' in - # the MRO. Fix MRO if needed. - if info.mro and info.mro[-1].fullname() != 'builtins.object': - info.mro.append(self.object_type().type) - - def expr_to_analyzed_type(self, expr: Node) -> Type: - if isinstance(expr, CallExpr): - expr.accept(self) - info = self.check_namedtuple(expr) - if info is None: - # Some form of namedtuple is the only valid type that looks like a call - # expression. This isn't a valid type. - raise TypeTranslationError() - fallback = Instance(info, []) - return TupleType(info.tuple_type.items, fallback=fallback) - typ = expr_to_unanalyzed_type(expr) - return self.anal_type(typ) - - def verify_base_classes(self, defn: ClassDef) -> bool: - info = defn.info - for base in info.bases: - baseinfo = base.type - if self.is_base_class(info, baseinfo): - self.fail('Cycle in inheritance hierarchy', defn) - # Clear bases to forcefully get rid of the cycle. - info.bases = [] - if baseinfo.fullname() == 'builtins.bool': - self.fail("'%s' is not a valid base class" % - baseinfo.name(), defn) - return False - dup = find_duplicate(info.direct_base_classes()) - if dup: - self.fail('Duplicate base class "%s"' % dup.name(), defn) - return False - return True - - def is_base_class(self, t: TypeInfo, s: TypeInfo) -> bool: - """Determine if t is a base class of s (but do not use mro).""" - # Search the base class graph for t, starting from s. - worklist = [s] - visited = {s} - while worklist: - nxt = worklist.pop() - if nxt == t: - return True - for base in nxt.bases: - if base.type not in visited: - worklist.append(base.type) - visited.add(base.type) - return False - - def analyze_metaclass(self, defn: ClassDef) -> None: - if defn.metaclass: - sym = self.lookup_qualified(defn.metaclass, defn) - if sym is not None and not isinstance(sym.node, TypeInfo): - self.fail("Invalid metaclass '%s'" % defn.metaclass, defn) - - def object_type(self) -> Instance: - return self.named_type('__builtins__.object') - - def class_type(self, info: TypeInfo) -> Type: - # Construct a function type whose fallback is cls. - from mypy import checkmember # To avoid import cycle. - leading_type = checkmember.type_object_type(info, self.builtin_type) - if isinstance(leading_type, Overloaded): - # Overloaded __init__ is too complex to handle. Plus it's stubs only. - return AnyType() - else: - return leading_type - - def named_type(self, qualified_name: str, args: List[Type] = None) -> Instance: - sym = self.lookup_qualified(qualified_name, None) - return Instance(cast(TypeInfo, sym.node), args or []) - - def named_type_or_none(self, qualified_name: str) -> Instance: - sym = self.lookup_fully_qualified_or_none(qualified_name) - if not sym: - return None - return Instance(cast(TypeInfo, sym.node), []) - - def is_instance_type(self, t: Type) -> bool: - return isinstance(t, Instance) - - def bind_class_type_variables_in_symbol_table( - self, info: TypeInfo) -> List[SymbolTableNode]: - nodes = [] # type: List[SymbolTableNode] - for var, binder in zip(info.type_vars, info.defn.type_vars): - node = self.bind_type_var(var, binder, info) - nodes.append(node) - return nodes - - def visit_import(self, i: Import) -> None: - for id, as_id in i.ids: - if as_id is not None: - self.add_module_symbol(id, as_id, module_public=True, context=i) - else: - # Modules imported in a stub file without using 'as x' won't get exported when - # doing 'from m import *'. - module_public = not self.is_stub_file - base = id.split('.')[0] - self.add_module_symbol(base, base, module_public=module_public, - context=i) - - def add_module_symbol(self, id: str, as_id: str, module_public: bool, - context: Context) -> None: - if id in self.modules: - m = self.modules[id] - self.add_symbol(as_id, SymbolTableNode(MODULE_REF, m, self.cur_mod_id, - module_public=module_public), context) - else: - self.add_unknown_symbol(as_id, context) - - def visit_import_from(self, imp: ImportFrom) -> None: - import_id = self.correct_relative_import(imp) - if import_id in self.modules: - module = self.modules[import_id] - for id, as_id in imp.names: - node = module.names.get(id) - if node and node.kind != UNBOUND_IMPORTED: - node = self.normalize_type_alias(node, imp) - if not node: - return - imported_id = as_id or id - existing_symbol = self.globals.get(imported_id) - if existing_symbol: - # Import can redefine a variable. They get special treatment. - if self.process_import_over_existing_name( - imported_id, existing_symbol, node, imp): - continue - # 'from m import x as x' exports x in a stub file. - module_public = not self.is_stub_file or as_id is not None - symbol = SymbolTableNode(node.kind, node.node, - self.cur_mod_id, - node.type_override, - module_public=module_public) - self.add_symbol(imported_id, symbol, imp) - else: - message = "Module has no attribute '{}'".format(id) - extra = self.undefined_name_extra_info('{}.{}'.format(import_id, id)) - if extra: - message += " {}".format(extra) - self.fail(message, imp) - else: - # Missing module. - for id, as_id in imp.names: - self.add_unknown_symbol(as_id or id, imp) - - def process_import_over_existing_name(self, - imported_id: str, existing_symbol: SymbolTableNode, - module_symbol: SymbolTableNode, - import_node: ImportBase) -> bool: - if (existing_symbol.kind in (LDEF, GDEF, MDEF) and - isinstance(existing_symbol.node, (Var, FuncDef))): - # This is a valid import over an existing definition in the file. Construct a dummy - # assignment that we'll use to type check the import. - lvalue = NameExpr(imported_id) - lvalue.kind = existing_symbol.kind - lvalue.node = existing_symbol.node - rvalue = NameExpr(imported_id) - rvalue.kind = module_symbol.kind - rvalue.node = module_symbol.node - assignment = AssignmentStmt([lvalue], rvalue) - for node in assignment, lvalue, rvalue: - node.set_line(import_node) - import_node.assignments.append(assignment) - return True - return False - - def normalize_type_alias(self, node: SymbolTableNode, - ctx: Context) -> SymbolTableNode: - if node.fullname in type_aliases: - # Node refers to an aliased type such as typing.List; normalize. - node = self.lookup_qualified(type_aliases[node.fullname], ctx) - if node.fullname == 'typing.DefaultDict': - self.add_module_symbol('collections', '__mypy_collections__', False, ctx) - node = self.lookup_qualified('__mypy_collections__.defaultdict', ctx) - return node - - def correct_relative_import(self, node: Union[ImportFrom, ImportAll]) -> str: - if node.relative == 0: - return node.id - - parts = self.cur_mod_id.split(".") - cur_mod_id = self.cur_mod_id - - rel = node.relative - if self.cur_mod_node.is_package_init_file(): - rel -= 1 - if len(parts) < rel: - self.fail("Relative import climbs too many namespaces", node) - if rel != 0: - cur_mod_id = ".".join(parts[:-rel]) - - return cur_mod_id + (("." + node.id) if node.id else "") - - def visit_import_all(self, i: ImportAll) -> None: - i_id = self.correct_relative_import(i) - if i_id in self.modules: - m = self.modules[i_id] - for name, node in m.names.items(): - node = self.normalize_type_alias(node, i) - if not name.startswith('_') and node.module_public: - existing_symbol = self.globals.get(name) - if existing_symbol: - # Import can redefine a variable. They get special treatment. - if self.process_import_over_existing_name( - name, existing_symbol, node, i): - continue - self.add_symbol(name, SymbolTableNode(node.kind, node.node, - self.cur_mod_id), i) - else: - # Don't add any dummy symbols for 'from x import *' if 'x' is unknown. - pass - - def add_unknown_symbol(self, name: str, context: Context) -> None: - var = Var(name) - var._fullname = self.qualified_name(name) - var.is_ready = True - var.type = AnyType() - self.add_symbol(name, SymbolTableNode(GDEF, var, self.cur_mod_id), context) - - # - # Statements - # - - def visit_block(self, b: Block) -> None: - if b.is_unreachable: - return - self.block_depth[-1] += 1 - for s in b.body: - self.accept(s) - self.block_depth[-1] -= 1 - - def visit_block_maybe(self, b: Block) -> None: - if b: - self.visit_block(b) - - def anal_type(self, t: Type, allow_tuple_literal: bool = False) -> Type: - if t: - if allow_tuple_literal: - # Types such as (t1, t2, ...) only allowed in assignment statements. They'll - # generate errors elsewhere, and Tuple[t1, t2, ...] must be used instead. - if isinstance(t, TupleType): - # Unlike TypeAnalyser, also allow implicit tuple types (without Tuple[...]). - star_count = sum(1 for item in t.items if isinstance(item, StarType)) - if star_count > 1: - self.fail('At most one star type allowed in a tuple', t) - return TupleType([AnyType() for _ in t.items], - self.builtin_type('builtins.tuple'), t.line) - items = [self.anal_type(item, True) - for item in t.items] - return TupleType(items, self.builtin_type('builtins.tuple'), t.line) - a = TypeAnalyser(self.lookup_qualified, - self.lookup_fully_qualified, - self.fail) - return t.accept(a) - else: - return None - - def visit_assignment_stmt(self, s: AssignmentStmt) -> None: - for lval in s.lvalues: - self.analyze_lvalue(lval, explicit_type=s.type is not None) - s.rvalue.accept(self) - if s.type: - allow_tuple_literal = isinstance(s.lvalues[-1], (TupleExpr, ListExpr)) - s.type = self.anal_type(s.type, allow_tuple_literal) - else: - # For simple assignments, allow binding type aliases. - # Also set the type if the rvalue is a simple literal. - if (s.type is None and len(s.lvalues) == 1 and - isinstance(s.lvalues[0], NameExpr)): - if s.lvalues[0].is_def: - s.type = self.analyze_simple_literal_type(s.rvalue) - res = analyze_type_alias(s.rvalue, - self.lookup_qualified, - self.lookup_fully_qualified, - self.fail) - if res and (not isinstance(res, Instance) or cast(Instance, res).args): - # TODO: What if this gets reassigned? - name = s.lvalues[0] - node = self.lookup(name.name, name) - node.kind = TYPE_ALIAS - node.type_override = res - if isinstance(s.rvalue, IndexExpr): - s.rvalue.analyzed = TypeAliasExpr(res) - if s.type: - # Store type into nodes. - for lvalue in s.lvalues: - self.store_declared_types(lvalue, s.type) - self.check_and_set_up_type_alias(s) - self.process_typevar_declaration(s) - self.process_namedtuple_definition(s) - - if (len(s.lvalues) == 1 and isinstance(s.lvalues[0], NameExpr) and - s.lvalues[0].name == '__all__' and s.lvalues[0].kind == GDEF and - isinstance(s.rvalue, (ListExpr, TupleExpr))): - self.add_exports(*s.rvalue.items) - - def analyze_simple_literal_type(self, rvalue: Node) -> Optional[Type]: - """Return builtins.int if rvalue is an int literal, etc.""" - if self.weak_opts or self.options.semantic_analysis_only or self.function_stack: - # Skip this if any weak options are set. - # Also skip if we're only doing the semantic analysis pass. - # This is mostly to avoid breaking unit tests. - # Also skip inside a function; this is to avoid confusing - # the code that handles dead code due to isinstance() - # inside type variables with value restrictions (like - # AnyStr). - return None - if isinstance(rvalue, IntExpr): - return self.named_type_or_none('builtins.int') - if isinstance(rvalue, FloatExpr): - return self.named_type_or_none('builtins.float') - if isinstance(rvalue, StrExpr): - return self.named_type_or_none('builtins.str') - if isinstance(rvalue, BytesExpr): - return self.named_type_or_none('builtins.bytes') - if isinstance(rvalue, UnicodeExpr): - return self.named_type_or_none('builtins.unicode') - return None - - def check_and_set_up_type_alias(self, s: AssignmentStmt) -> None: - """Check if assignment creates a type alias and set it up as needed.""" - # For now, type aliases only work at the top level of a module. - if (len(s.lvalues) == 1 and not self.is_func_scope() and not self.type - and not s.type): - lvalue = s.lvalues[0] - if isinstance(lvalue, NameExpr): - if not lvalue.is_def: - # Only a definition can create a type alias, not regular assignment. - return - rvalue = s.rvalue - if isinstance(rvalue, RefExpr): - node = rvalue.node - if isinstance(node, TypeInfo): - # TODO: We should record the fact that this is a variable - # that refers to a type, rather than making this - # just an alias for the type. - self.globals[lvalue.name].node = node - - def analyze_lvalue(self, lval: Node, nested: bool = False, - add_global: bool = False, - explicit_type: bool = False) -> None: - """Analyze an lvalue or assignment target. - - Only if add_global is True, add name to globals table. If nested - is true, the lvalue is within a tuple or list lvalue expression. - """ - - if isinstance(lval, NameExpr): - # Top-level definitions within some statements (at least while) are - # not handled in the first pass, so they have to be added now. - nested_global = (not self.is_func_scope() and - self.block_depth[-1] > 0 and - not self.type) - if (add_global or nested_global) and lval.name not in self.globals: - # Define new global name. - v = Var(lval.name) - v._fullname = self.qualified_name(lval.name) - v.is_ready = False # Type not inferred yet - lval.node = v - lval.is_def = True - lval.kind = GDEF - lval.fullname = v._fullname - self.globals[lval.name] = SymbolTableNode(GDEF, v, - self.cur_mod_id) - elif isinstance(lval.node, Var) and lval.is_def: - # Since the is_def flag is set, this must have been analyzed - # already in the first pass and added to the symbol table. - assert lval.node.name() in self.globals - elif (self.is_func_scope() and lval.name not in self.locals[-1] and - lval.name not in self.global_decls[-1] and - lval.name not in self.nonlocal_decls[-1]): - # Define new local name. - v = Var(lval.name) - lval.node = v - lval.is_def = True - lval.kind = LDEF - lval.fullname = lval.name - self.add_local(v, lval) - elif not self.is_func_scope() and (self.type and - lval.name not in self.type.names): - # Define a new attribute within class body. - v = Var(lval.name) - v.info = self.type - v.is_initialized_in_class = True - v.set_line(lval) - lval.node = v - lval.is_def = True - lval.kind = MDEF - lval.fullname = lval.name - self.type.names[lval.name] = SymbolTableNode(MDEF, v) - else: - # Bind to an existing name. - if explicit_type: - self.name_already_defined(lval.name, lval) - lval.accept(self) - self.check_lvalue_validity(lval.node, lval) - elif isinstance(lval, MemberExpr): - if not add_global: - self.analyze_member_lvalue(lval) - if explicit_type and not self.is_self_member_ref(lval): - self.fail('Type cannot be declared in assignment to non-self ' - 'attribute', lval) - elif isinstance(lval, IndexExpr): - if explicit_type: - self.fail('Unexpected type declaration', lval) - if not add_global: - lval.accept(self) - elif (isinstance(lval, TupleExpr) or - isinstance(lval, ListExpr)): - items = cast(Any, lval).items - if len(items) == 0 and isinstance(lval, TupleExpr): - self.fail("Can't assign to ()", lval) - self.analyze_tuple_or_list_lvalue(cast(Union[ListExpr, TupleExpr], lval), - add_global, explicit_type) - elif isinstance(lval, StarExpr): - if nested: - self.analyze_lvalue(lval.expr, nested, add_global, explicit_type) - else: - self.fail('Starred assignment target must be in a list or tuple', lval) - else: - self.fail('Invalid assignment target', lval) - - def analyze_tuple_or_list_lvalue(self, lval: Union[ListExpr, TupleExpr], - add_global: bool = False, - explicit_type: bool = False) -> None: - """Analyze an lvalue or assignment target that is a list or tuple.""" - items = lval.items - star_exprs = [cast(StarExpr, item) - for item in items - if isinstance(item, StarExpr)] - - if len(star_exprs) > 1: - self.fail('Two starred expressions in assignment', lval) - else: - if len(star_exprs) == 1: - star_exprs[0].valid = True - for i in items: - self.analyze_lvalue(i, nested=True, add_global=add_global, - explicit_type = explicit_type) - - def analyze_member_lvalue(self, lval: MemberExpr) -> None: - lval.accept(self) - if (self.is_self_member_ref(lval) and - self.type.get(lval.name) is None): - # Implicit attribute definition in __init__. - lval.is_def = True - v = Var(lval.name) - v.info = self.type - v.is_ready = False - lval.def_var = v - lval.node = v - self.type.names[lval.name] = SymbolTableNode(MDEF, v) - self.check_lvalue_validity(lval.node, lval) - - def is_self_member_ref(self, memberexpr: MemberExpr) -> bool: - """Does memberexpr to refer to an attribute of self?""" - if not isinstance(memberexpr.expr, NameExpr): - return False - node = memberexpr.expr.node - return isinstance(node, Var) and node.is_self - - def check_lvalue_validity(self, node: Node, ctx: Context) -> None: - if isinstance(node, (TypeInfo, TypeVarExpr)): - self.fail('Invalid assignment target', ctx) - - def store_declared_types(self, lvalue: Node, typ: Type) -> None: - if isinstance(typ, StarType) and not isinstance(lvalue, StarExpr): - self.fail('Star type only allowed for starred expressions', lvalue) - if isinstance(lvalue, RefExpr): - lvalue.is_def = False - if isinstance(lvalue.node, Var): - var = lvalue.node - var.type = typ - var.is_ready = True - # If node is not a variable, we'll catch it elsewhere. - elif isinstance(lvalue, TupleExpr): - if isinstance(typ, TupleType): - if len(lvalue.items) != len(typ.items): - self.fail('Incompatible number of tuple items', lvalue) - return - for item, itemtype in zip(lvalue.items, typ.items): - self.store_declared_types(item, itemtype) - else: - self.fail('Tuple type expected for multiple variables', - lvalue) - elif isinstance(lvalue, StarExpr): - if isinstance(typ, StarType): - self.store_declared_types(lvalue.expr, typ.type) - else: - self.fail('Star type expected for starred expression', lvalue) - else: - # This has been flagged elsewhere as an error, so just ignore here. - pass - - def process_typevar_declaration(self, s: AssignmentStmt) -> None: - """Check if s declares a TypeVar; it yes, store it in symbol table.""" - call = self.get_typevar_declaration(s) - if not call: - return - - lvalue = cast(NameExpr, s.lvalues[0]) - name = lvalue.name - if not lvalue.is_def: - if s.type: - self.fail("Cannot declare the type of a type variable", s) - else: - self.fail("Cannot redefine '%s' as a type variable" % name, s) - return - - if not self.check_typevar_name(call, name, s): - return - - # Constraining types - n_values = call.arg_kinds[1:].count(ARG_POS) - values = self.analyze_types(call.args[1:1 + n_values]) - - res = self.process_typevar_parameters(call.args[1 + n_values:], - call.arg_names[1 + n_values:], - call.arg_kinds[1 + n_values:], - bool(values), - s) - if res is None: - return - variance, upper_bound = res - - # Yes, it's a valid type variable definition! Add it to the symbol table. - node = self.lookup(name, s) - node.kind = UNBOUND_TVAR - TypeVar = TypeVarExpr(name, node.fullname, values, upper_bound, variance) - TypeVar.line = call.line - call.analyzed = TypeVar - node.node = TypeVar - - def check_typevar_name(self, call: CallExpr, name: str, context: Context) -> bool: - if len(call.args) < 1: - self.fail("Too few arguments for TypeVar()", context) - return False - if not isinstance(call.args[0], (StrExpr, BytesExpr)) or not call.arg_kinds[0] == ARG_POS: - self.fail("TypeVar() expects a string literal as first argument", context) - return False - if cast(StrExpr, call.args[0]).value != name: - self.fail("Unexpected TypeVar() argument value", context) - return False - return True - - def get_typevar_declaration(self, s: AssignmentStmt) -> Optional[CallExpr]: - """Returns the TypeVar() call expression if `s` is a type var declaration - or None otherwise. - """ - if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): - return None - if not isinstance(s.rvalue, CallExpr): - return None - call = s.rvalue - if not isinstance(call.callee, RefExpr): - return None - callee = call.callee - if callee.fullname != 'typing.TypeVar': - return None - return call - - def process_typevar_parameters(self, args: List[Node], - names: List[Optional[str]], - kinds: List[int], - has_values: bool, - context: Context) -> Optional[Tuple[int, Type]]: - covariant = False - contravariant = False - upper_bound = self.object_type() # type: Type - for param_value, param_name, param_kind in zip(args, names, kinds): - if not param_kind == ARG_NAMED: - self.fail("Unexpected argument to TypeVar()", context) - return None - if param_name == 'covariant': - if isinstance(param_value, NameExpr): - if param_value.name == 'True': - covariant = True - else: - self.fail("TypeVar 'covariant' may only be 'True'", context) - return None - else: - self.fail("TypeVar 'covariant' may only be 'True'", context) - return None - elif param_name == 'contravariant': - if isinstance(param_value, NameExpr): - if param_value.name == 'True': - contravariant = True - else: - self.fail("TypeVar 'contravariant' may only be 'True'", context) - return None - else: - self.fail("TypeVar 'contravariant' may only be 'True'", context) - return None - elif param_name == 'bound': - if has_values: - self.fail("TypeVar cannot have both values and an upper bound", context) - return None - try: - upper_bound = self.expr_to_analyzed_type(param_value) - except TypeTranslationError: - self.fail("TypeVar 'bound' must be a type", param_value) - return None - elif param_name == 'values': - # Probably using obsolete syntax with values=(...). Explain the current syntax. - self.fail("TypeVar 'values' argument not supported", context) - self.fail("Use TypeVar('T', t, ...) instead of TypeVar('T', values=(t, ...))", - context) - return None - else: - self.fail("Unexpected argument to TypeVar(): {}".format(param_name), context) - return None - - if covariant and contravariant: - self.fail("TypeVar cannot be both covariant and contravariant", context) - return None - elif covariant: - variance = COVARIANT - elif contravariant: - variance = CONTRAVARIANT - else: - variance = INVARIANT - return (variance, upper_bound) - - def process_namedtuple_definition(self, s: AssignmentStmt) -> None: - """Check if s defines a namedtuple; if yes, store the definition in symbol table.""" - if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): - return - lvalue = cast(NameExpr, s.lvalues[0]) - name = lvalue.name - named_tuple = self.check_namedtuple(s.rvalue, name) - if named_tuple is None: - return - # Yes, it's a valid namedtuple definition. Add it to the symbol table. - node = self.lookup(name, s) - node.kind = GDEF # TODO locally defined namedtuple - # TODO call.analyzed - node.node = named_tuple - - def check_namedtuple(self, node: Node, var_name: str = None) -> TypeInfo: - """Check if a call defines a namedtuple. - - The optional var_name argument is the name of the variable to - which this is assigned, if any. - - If it does, return the corresponding TypeInfo. Return None otherwise. - - If the definition is invalid but looks like a namedtuple, - report errors but return (some) TypeInfo. - """ - if not isinstance(node, CallExpr): - return None - call = node - if not isinstance(call.callee, RefExpr): - return None - callee = call.callee - fullname = callee.fullname - if fullname not in ('collections.namedtuple', 'typing.NamedTuple'): - return None - items, types, ok = self.parse_namedtuple_args(call, fullname) - if not ok: - # Error. Construct dummy return value. - return self.build_namedtuple_typeinfo('namedtuple', [], []) - else: - # Give it a unique name derived from the line number. - name = cast(StrExpr, call.args[0]).value - if name != var_name: - name += '@' + str(call.line) - info = self.build_namedtuple_typeinfo(name, items, types) - # Store it as a global just in case it would remain anonymous. - self.globals[name] = SymbolTableNode(GDEF, info, self.cur_mod_id) - call.analyzed = NamedTupleExpr(info).set_line(call.line) - return info - - def parse_namedtuple_args(self, call: CallExpr, - fullname: str) -> Tuple[List[str], List[Type], bool]: - # TODO Share code with check_argument_count in checkexpr.py? - args = call.args - if len(args) < 2: - return self.fail_namedtuple_arg("Too few arguments for namedtuple()", call) - if len(args) > 2: - return self.fail_namedtuple_arg("Too many arguments for namedtuple()", call) - if call.arg_kinds != [ARG_POS, ARG_POS]: - return self.fail_namedtuple_arg("Unexpected arguments to namedtuple()", call) - if not isinstance(args[0], (StrExpr, BytesExpr)): - return self.fail_namedtuple_arg( - "namedtuple() expects a string literal as the first argument", call) - types = [] # type: List[Type] - ok = True - if not isinstance(args[1], ListExpr): - if fullname == 'collections.namedtuple' and isinstance(args[1], (StrExpr, BytesExpr)): - str_expr = cast(StrExpr, args[1]) - items = str_expr.value.split() - else: - return self.fail_namedtuple_arg( - "List literal expected as the second argument to namedtuple()", call) - else: - listexpr = args[1] - if fullname == 'collections.namedtuple': - # The fields argument contains just names, with implicit Any types. - if any(not isinstance(item, (StrExpr, BytesExpr)) for item in listexpr.items): - return self.fail_namedtuple_arg("String literal expected as namedtuple() item", - call) - items = [cast(StrExpr, item).value for item in listexpr.items] - else: - # The fields argument contains (name, type) tuples. - items, types, ok = self.parse_namedtuple_fields_with_types(listexpr.items, call) - if not types: - types = [AnyType() for _ in items] - return items, types, ok - - def parse_namedtuple_fields_with_types(self, nodes: List[Node], - context: Context) -> Tuple[List[str], List[Type], bool]: - items = [] # type: List[str] - types = [] # type: List[Type] - for item in nodes: - if isinstance(item, TupleExpr): - if len(item.items) != 2: - return self.fail_namedtuple_arg("Invalid NamedTuple field definition", - item) - name, type_node = item.items - if isinstance(name, (StrExpr, BytesExpr)): - items.append(name.value) - else: - return self.fail_namedtuple_arg("Invalid NamedTuple() field name", item) - try: - type = expr_to_unanalyzed_type(type_node) - except TypeTranslationError: - return self.fail_namedtuple_arg('Invalid field type', type_node) - types.append(self.anal_type(type)) - else: - return self.fail_namedtuple_arg("Tuple expected as NamedTuple() field", item) - return items, types, True - - def fail_namedtuple_arg(self, message: str, - context: Context) -> Tuple[List[str], List[Type], bool]: - self.fail(message, context) - return [], [], False - - def build_namedtuple_typeinfo(self, name: str, items: List[str], - types: List[Type]) -> TypeInfo: - symbols = SymbolTable() - tup = NamedTupleType(name, items, types, self.named_type('__builtins__.tuple', types)) - class_def = ClassDef(name, Block([])) - class_def.fullname = self.qualified_name(name) - info = NamedTupleTypeInfo(tup, symbols, class_def) - info.attrs = items - vars = [Var(item, typ) for item, typ in zip(items, types)] - # Add named tuple items as attributes. - # TODO: Make them read-only. - for var in vars: - var.info = info - symbols[var.name()] = SymbolTableNode(MDEF, var) - this_type = self_type(info) - # Add __init__, _replace, _asdict methods and _fields static field - self.add_namedtuple_method('_replace', symbols, info, this_type, this_type, - self.make_factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) - self.add_namedtuple_method('__init__', symbols, info, this_type, NoneTyp(), - self.make_factory_args(vars, ARG_POS), info.name()) - # TODO: refine to OrderedDict[Union[types]] - self.add_namedtuple_method('_asdict', symbols, info, this_type, AnyType(), []) - return info - - def make_factory_args(self, vars: List[Var], kind: int, - initializer: Expression = None) -> List[Argument]: - return [Argument(var, var.type, initializer, kind) for var in vars] - - def add_namedtuple_method(self, funcname: str, symbols: SymbolTable, info: TypeInfo, - this_type: Type, ret: Type, args: List[Argument], name=None) -> None: - args = [Argument(Var('self'), this_type, None, ARG_POS)] + args - types = [arg.type_annotation for arg in args] - items = [arg.variable.name() for arg in args] - arg_kinds = [arg.kind for arg in args] - # TODO: Make sure that the self argument name is not visible? - signature = CallableType(types, arg_kinds, items, ret, - self.named_type('__builtins__.function'), - name=name or info.name() + '.' + funcname) - func = FuncDef(funcname, - args, - Block([]), - typ=signature) - func.info = info - symbols[funcname] = SymbolTableNode(MDEF, func) - - def analyze_types(self, items: List[Node]) -> List[Type]: - result = [] # type: List[Type] - for node in items: - try: - result.append(self.anal_type(expr_to_unanalyzed_type(node))) - except TypeTranslationError: - self.fail('Type expected', node) - result.append(AnyType()) - return result - - def visit_decorator(self, dec: Decorator) -> None: - for d in dec.decorators: - d.accept(self) - removed = [] # type: List[int] - no_type_check = False - for i, d in enumerate(dec.decorators): - # A bunch of decorators are special cased here. - if refers_to_fullname(d, 'abc.abstractmethod'): - removed.append(i) - dec.func.is_abstract = True - self.check_decorated_function_is_method('abstractmethod', dec) - elif refers_to_fullname(d, 'asyncio.tasks.coroutine'): - removed.append(i) - elif refers_to_fullname(d, 'builtins.staticmethod'): - removed.append(i) - dec.func.is_static = True - dec.var.is_staticmethod = True - self.check_decorated_function_is_method('staticmethod', dec) - elif refers_to_fullname(d, 'builtins.classmethod'): - removed.append(i) - dec.func.is_class = True - dec.var.is_classmethod = True - self.check_decorated_function_is_method('classmethod', dec) - elif (refers_to_fullname(d, 'builtins.property') or - refers_to_fullname(d, 'abc.abstractproperty')): - removed.append(i) - dec.func.is_property = True - dec.var.is_property = True - if refers_to_fullname(d, 'abc.abstractproperty'): - dec.func.is_abstract = True - self.check_decorated_function_is_method('property', dec) - if len(dec.func.arguments) > 1: - self.fail('Too many arguments', dec.func) - elif refers_to_fullname(d, 'typing.no_type_check'): - dec.var.type = AnyType() - no_type_check = True - for i in reversed(removed): - del dec.decorators[i] - if not dec.is_overload or dec.var.is_property: - if self.is_func_scope(): - self.add_symbol(dec.var.name(), SymbolTableNode(LDEF, dec), - dec) - elif self.type: - dec.var.info = self.type - dec.var.is_initialized_in_class = True - self.add_symbol(dec.var.name(), SymbolTableNode(MDEF, dec), - dec) - if not no_type_check: - dec.func.accept(self) - if dec.decorators and dec.var.is_property: - self.fail('Decorated property not supported', dec) - - def check_decorated_function_is_method(self, decorator: str, - context: Context) -> None: - if not self.type or self.is_func_scope(): - self.fail("'%s' used with a non-method" % decorator, context) - - def visit_expression_stmt(self, s: ExpressionStmt) -> None: - s.expr.accept(self) - - def visit_return_stmt(self, s: ReturnStmt) -> None: - if not self.is_func_scope(): - self.fail("'return' outside function", s) - if s.expr: - s.expr.accept(self) - - def visit_raise_stmt(self, s: RaiseStmt) -> None: - if s.expr: - s.expr.accept(self) - if s.from_expr: - s.from_expr.accept(self) - - def visit_assert_stmt(self, s: AssertStmt) -> None: - if s.expr: - s.expr.accept(self) - - def visit_operator_assignment_stmt(self, - s: OperatorAssignmentStmt) -> None: - s.lvalue.accept(self) - s.rvalue.accept(self) - if (isinstance(s.lvalue, NameExpr) and s.lvalue.name == '__all__' and - s.lvalue.kind == GDEF and isinstance(s.rvalue, (ListExpr, TupleExpr))): - self.add_exports(*s.rvalue.items) - - def visit_while_stmt(self, s: WhileStmt) -> None: - s.expr.accept(self) - self.loop_depth += 1 - s.body.accept(self) - self.loop_depth -= 1 - self.visit_block_maybe(s.else_body) - - def visit_for_stmt(self, s: ForStmt) -> None: - s.expr.accept(self) - - # Bind index variables and check if they define new names. - self.analyze_lvalue(s.index) - - self.loop_depth += 1 - self.visit_block(s.body) - self.loop_depth -= 1 - - self.visit_block_maybe(s.else_body) - - def visit_break_stmt(self, s: BreakStmt) -> None: - if self.loop_depth == 0: - self.fail("'break' outside loop", s, True, blocker=True) - - def visit_continue_stmt(self, s: ContinueStmt) -> None: - if self.loop_depth == 0: - self.fail("'continue' outside loop", s, True, blocker=True) - - def visit_if_stmt(self, s: IfStmt) -> None: - infer_reachability_of_if_statement(s, pyversion=self.options.python_version) - for i in range(len(s.expr)): - s.expr[i].accept(self) - self.visit_block(s.body[i]) - self.visit_block_maybe(s.else_body) - - def visit_try_stmt(self, s: TryStmt) -> None: - self.analyze_try_stmt(s, self) - - def analyze_try_stmt(self, s: TryStmt, visitor: NodeVisitor, - add_global: bool = False) -> None: - s.body.accept(visitor) - for type, var, handler in zip(s.types, s.vars, s.handlers): - if type: - type.accept(visitor) - if var: - self.analyze_lvalue(var, add_global=add_global) - handler.accept(visitor) - if s.else_body: - s.else_body.accept(visitor) - if s.finally_body: - s.finally_body.accept(visitor) - - def visit_with_stmt(self, s: WithStmt) -> None: - for e, n in zip(s.expr, s.target): - e.accept(self) - if n: - self.analyze_lvalue(n) - self.visit_block(s.body) - - def visit_del_stmt(self, s: DelStmt) -> None: - s.expr.accept(self) - if not self.is_valid_del_target(s.expr): - self.fail('Invalid delete target', s) - - def is_valid_del_target(self, s: Node) -> bool: - if isinstance(s, (IndexExpr, NameExpr, MemberExpr)): - return True - elif isinstance(s, TupleExpr): - return all(self.is_valid_del_target(item) for item in s.items) - - def visit_global_decl(self, g: GlobalDecl) -> None: - for name in g.names: - if name in self.nonlocal_decls[-1]: - self.fail("Name '{}' is nonlocal and global".format(name), g) - self.global_decls[-1].add(name) - - def visit_nonlocal_decl(self, d: NonlocalDecl) -> None: - if not self.is_func_scope(): - self.fail("nonlocal declaration not allowed at module level", d) - else: - for name in d.names: - for table in reversed(self.locals[:-1]): - if table is not None and name in table: - break - else: - self.fail("No binding for nonlocal '{}' found".format(name), d) - - if self.locals[-1] is not None and name in self.locals[-1]: - self.fail("Name '{}' is already defined in local " - "scope before nonlocal declaration".format(name), d) - - if name in self.global_decls[-1]: - self.fail("Name '{}' is nonlocal and global".format(name), d) - self.nonlocal_decls[-1].add(name) - - def visit_print_stmt(self, s: PrintStmt) -> None: - for arg in s.args: - arg.accept(self) - if s.target: - s.target.accept(self) - - def visit_exec_stmt(self, s: ExecStmt) -> None: - s.expr.accept(self) - if s.variables1: - s.variables1.accept(self) - if s.variables2: - s.variables2.accept(self) - - # - # Expressions - # - - def visit_name_expr(self, expr: NameExpr) -> None: - n = self.lookup(expr.name, expr) - if n: - if n.kind == BOUND_TVAR: - self.fail("'{}' is a type variable and only valid in type " - "context".format(expr.name), expr) - else: - expr.kind = n.kind - expr.node = n.node - expr.fullname = n.fullname - - def visit_super_expr(self, expr: SuperExpr) -> None: - if not self.type: - self.fail('"super" used outside class', expr) - return - expr.info = self.type - - def visit_tuple_expr(self, expr: TupleExpr) -> None: - for item in expr.items: - item.accept(self) - - def visit_list_expr(self, expr: ListExpr) -> None: - for item in expr.items: - item.accept(self) - - def visit_set_expr(self, expr: SetExpr) -> None: - for item in expr.items: - item.accept(self) - - def visit_dict_expr(self, expr: DictExpr) -> None: - for key, value in expr.items: - key.accept(self) - value.accept(self) - - def visit_star_expr(self, expr: StarExpr) -> None: - if not expr.valid: - self.fail('Can use starred expression only as assignment target', expr) - else: - expr.expr.accept(self) - - def visit_yield_from_expr(self, e: YieldFromExpr) -> None: - if not self.is_func_scope(): # not sure - self.fail("'yield from' outside function", e, True, blocker=True) - else: - self.function_stack[-1].is_generator = True - if e.expr: - e.expr.accept(self) - - def visit_call_expr(self, expr: CallExpr) -> None: - """Analyze a call expression. - - Some call expressions are recognized as special forms, including - cast(...) and Any(...). - """ - expr.callee.accept(self) - if refers_to_fullname(expr.callee, 'typing.cast'): - # Special form cast(...). - if not self.check_fixed_args(expr, 2, 'cast'): - return - # Translate first argument to an unanalyzed type. - try: - target = expr_to_unanalyzed_type(expr.args[0]) - except TypeTranslationError: - self.fail('Cast target is not a type', expr) - return - # Piggyback CastExpr object to the CallExpr object; it takes - # precedence over the CallExpr semantics. - expr.analyzed = CastExpr(expr.args[1], target) - expr.analyzed.line = expr.line - expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'builtins.reveal_type'): - if not self.check_fixed_args(expr, 1, 'reveal_type'): - return - expr.analyzed = RevealTypeExpr(expr.args[0]) - expr.analyzed.line = expr.line - expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'typing.Any'): - # Special form Any(...). - if not self.check_fixed_args(expr, 1, 'Any'): - return - expr.analyzed = CastExpr(expr.args[0], AnyType()) - expr.analyzed.line = expr.line - expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'typing._promote'): - # Special form _promote(...). - if not self.check_fixed_args(expr, 1, '_promote'): - return - # Translate first argument to an unanalyzed type. - try: - target = expr_to_unanalyzed_type(expr.args[0]) - except TypeTranslationError: - self.fail('Argument 1 to _promote is not a type', expr) - return - expr.analyzed = PromoteExpr(target) - expr.analyzed.line = expr.line - expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'builtins.dict'): - expr.analyzed = self.translate_dict_call(expr) - else: - # Normal call expression. - for a in expr.args: - a.accept(self) - - if (isinstance(expr.callee, MemberExpr) and - isinstance(expr.callee.expr, NameExpr) and - expr.callee.expr.name == '__all__' and - expr.callee.expr.kind == GDEF and - expr.callee.name in ('append', 'extend')): - if expr.callee.name == 'append' and expr.args: - self.add_exports(expr.args[0]) - elif (expr.callee.name == 'extend' and expr.args and - isinstance(expr.args[0], (ListExpr, TupleExpr))): - self.add_exports(*expr.args[0].items) - - def translate_dict_call(self, call: CallExpr) -> Optional[DictExpr]: - """Translate 'dict(x=y, ...)' to {'x': y, ...}. - - For other variants of dict(...), return None. - """ - if not call.args: - return None - if not all(kind == ARG_NAMED for kind in call.arg_kinds): - # Must still accept those args. - for a in call.args: - a.accept(self) - return None - expr = DictExpr([(StrExpr(key), value) - for key, value in zip(call.arg_names, call.args)]) - expr.set_line(call) - expr.accept(self) - return expr - - def check_fixed_args(self, expr: CallExpr, numargs: int, - name: str) -> bool: - """Verify that expr has specified number of positional args. - - Return True if the arguments are valid. - """ - s = 's' - if numargs == 1: - s = '' - if len(expr.args) != numargs: - self.fail("'%s' expects %d argument%s" % (name, numargs, s), - expr) - return False - if expr.arg_kinds != [ARG_POS] * numargs: - self.fail("'%s' must be called with %s positional argument%s" % - (name, numargs, s), expr) - return False - return True - - def visit_member_expr(self, expr: MemberExpr) -> None: - base = expr.expr - base.accept(self) - # Bind references to module attributes. - if isinstance(base, RefExpr) and base.kind == MODULE_REF: - file = cast(MypyFile, base.node) - n = file.names.get(expr.name, None) if file is not None else None - if n: - n = self.normalize_type_alias(n, expr) - if not n: - return - expr.kind = n.kind - expr.fullname = n.fullname - expr.node = n.node - else: - # We only catch some errors here; the rest will be - # catched during type checking. - # - # This way we can report a larger number of errors in - # one type checker run. If we reported errors here, - # the build would terminate after semantic analysis - # and we wouldn't be able to report any type errors. - full_name = '%s.%s' % (file.fullname() if file is not None else None, expr.name) - if full_name in obsolete_name_mapping: - self.fail("Module has no attribute %r (it's now called %r)" % ( - expr.name, obsolete_name_mapping[full_name]), expr) - - def visit_op_expr(self, expr: OpExpr) -> None: - expr.left.accept(self) - expr.right.accept(self) - - def visit_comparison_expr(self, expr: ComparisonExpr) -> None: - for operand in expr.operands: - operand.accept(self) - - def visit_unary_expr(self, expr: UnaryExpr) -> None: - expr.expr.accept(self) - - def visit_index_expr(self, expr: IndexExpr) -> None: - expr.base.accept(self) - if refers_to_class_or_function(expr.base): - # Special form -- type application. - # Translate index to an unanalyzed type. - types = [] # type: List[Type] - if isinstance(expr.index, TupleExpr): - items = expr.index.items - else: - items = [expr.index] - for item in items: - try: - typearg = expr_to_unanalyzed_type(item) - except TypeTranslationError: - self.fail('Type expected within [...]', expr) - return - typearg = self.anal_type(typearg) - types.append(typearg) - expr.analyzed = TypeApplication(expr.base, types) - expr.analyzed.line = expr.line - else: - expr.index.accept(self) - - def visit_slice_expr(self, expr: SliceExpr) -> None: - if expr.begin_index: - expr.begin_index.accept(self) - if expr.end_index: - expr.end_index.accept(self) - if expr.stride: - expr.stride.accept(self) - - def visit_cast_expr(self, expr: CastExpr) -> None: - expr.expr.accept(self) - expr.type = self.anal_type(expr.type) - - def visit_reveal_type_expr(self, expr: RevealTypeExpr) -> None: - expr.expr.accept(self) - - def visit_type_application(self, expr: TypeApplication) -> None: - expr.expr.accept(self) - for i in range(len(expr.types)): - expr.types[i] = self.anal_type(expr.types[i]) - - def visit_list_comprehension(self, expr: ListComprehension) -> None: - expr.generator.accept(self) - - def visit_set_comprehension(self, expr: SetComprehension) -> None: - expr.generator.accept(self) - - def visit_dictionary_comprehension(self, expr: DictionaryComprehension) -> None: - self.enter() - self.analyze_comp_for(expr) - expr.key.accept(self) - expr.value.accept(self) - self.leave() - self.analyze_comp_for_2(expr) - - def visit_generator_expr(self, expr: GeneratorExpr) -> None: - self.enter() - self.analyze_comp_for(expr) - expr.left_expr.accept(self) - self.leave() - self.analyze_comp_for_2(expr) - - def analyze_comp_for(self, expr: Union[GeneratorExpr, - DictionaryComprehension]) -> None: - """Analyses the 'comp_for' part of comprehensions (part 1). - - That is the part after 'for' in (x for x in l if p). This analyzes - variables and conditions which are analyzed in a local scope. - """ - for i, (index, sequence, conditions) in enumerate(zip(expr.indices, - expr.sequences, - expr.condlists)): - if i > 0: - sequence.accept(self) - # Bind index variables. - self.analyze_lvalue(index) - for cond in conditions: - cond.accept(self) - - def analyze_comp_for_2(self, expr: Union[GeneratorExpr, - DictionaryComprehension]) -> None: - """Analyses the 'comp_for' part of comprehensions (part 2). - - That is the part after 'for' in (x for x in l if p). This analyzes - the 'l' part which is analyzed in the surrounding scope. - """ - expr.sequences[0].accept(self) - - def visit_func_expr(self, expr: FuncExpr) -> None: - self.analyze_function(expr) - - def visit_conditional_expr(self, expr: ConditionalExpr) -> None: - expr.if_expr.accept(self) - expr.cond.accept(self) - expr.else_expr.accept(self) - - def visit_backquote_expr(self, expr: BackquoteExpr) -> None: - expr.expr.accept(self) - - def visit__promote_expr(self, expr: PromoteExpr) -> None: - expr.type = self.anal_type(expr.type) - - def visit_yield_expr(self, expr: YieldExpr) -> None: - if not self.is_func_scope(): - self.fail("'yield' outside function", expr, True, blocker=True) - else: - self.function_stack[-1].is_generator = True - if expr.expr: - expr.expr.accept(self) - - # - # Helpers - # - - def lookup(self, name: str, ctx: Context) -> SymbolTableNode: - """Look up an unqualified name in all active namespaces.""" - # 1a. Name declared using 'global x' takes precedence - if name in self.global_decls[-1]: - if name in self.globals: - return self.globals[name] - else: - self.name_not_defined(name, ctx) - return None - # 1b. Name declared using 'nonlocal x' takes precedence - if name in self.nonlocal_decls[-1]: - for table in reversed(self.locals[:-1]): - if table is not None and name in table: - return table[name] - else: - self.name_not_defined(name, ctx) - return None - # 2. Class attributes (if within class definition) - if self.is_class_scope() and name in self.type.names: - return self.type[name] - # 3. Local (function) scopes - for table in reversed(self.locals): - if table is not None and name in table: - return table[name] - # 4. Current file global scope - if name in self.globals: - return self.globals[name] - # 5. Builtins - b = self.globals.get('__builtins__', None) - if b: - table = cast(MypyFile, b.node).names - if name in table: - if name[0] == "_" and name[1] != "_": - self.name_not_defined(name, ctx) - return None - node = table[name] - return node - # Give up. - self.name_not_defined(name, ctx) - self.check_for_obsolete_short_name(name, ctx) - return None - - def check_for_obsolete_short_name(self, name: str, ctx: Context) -> None: - matches = [obsolete_name - for obsolete_name in obsolete_name_mapping - if obsolete_name.rsplit('.', 1)[-1] == name] - if len(matches) == 1: - self.note("(Did you mean '{}'?)".format(obsolete_name_mapping[matches[0]]), ctx) - - def lookup_qualified(self, name: str, ctx: Context) -> SymbolTableNode: - if '.' not in name: - return self.lookup(name, ctx) - else: - parts = name.split('.') - n = self.lookup(parts[0], ctx) # type: SymbolTableNode - if n: - for i in range(1, len(parts)): - if isinstance(n.node, TypeInfo): - if n.node.mro is None: - # We haven't yet analyzed the class `n.node`. Fall back to direct - # lookup in the names declared directly under it, without its base - # classes. This can happen when we have a forward reference to a - # nested class, and the reference is bound before the outer class - # has been fully semantically analyzed. - # - # A better approach would be to introduce a new analysis pass or - # to move things around between passes, but this unblocks a common - # use case even though this is a little limited in case there is - # inheritance involved. - result = n.node.names.get(parts[i]) - else: - result = n.node.get(parts[i]) - n = result - elif isinstance(n.node, MypyFile): - n = n.node.names.get(parts[i], None) - # TODO: What if node is Var or FuncDef? - if not n: - self.name_not_defined(name, ctx) - break - if n: - n = self.normalize_type_alias(n, ctx) - return n - - def builtin_type(self, fully_qualified_name: str) -> Instance: - node = self.lookup_fully_qualified(fully_qualified_name) - info = cast(TypeInfo, node.node) - return Instance(info, []) - - def lookup_fully_qualified(self, name: str) -> SymbolTableNode: - """Lookup a fully qualified name. - - Assume that the name is defined. This happens in the global namespace -- the local - module namespace is ignored. - """ - assert '.' in name - parts = name.split('.') - n = self.modules[parts[0]] - for i in range(1, len(parts) - 1): - n = cast(MypyFile, n.names[parts[i]].node) - return n.names[parts[-1]] - - def lookup_fully_qualified_or_none(self, name: str) -> SymbolTableNode: - """Lookup a fully qualified name. - - Assume that the name is defined. This happens in the global namespace -- the local - module namespace is ignored. - """ - assert '.' in name - parts = name.split('.') - n = self.modules[parts[0]] - for i in range(1, len(parts) - 1): - next_sym = n.names.get(parts[i]) - if not next_sym: - return None - n = cast(MypyFile, next_sym.node) - return n.names.get(parts[-1]) - - def qualified_name(self, n: str) -> str: - return self.cur_mod_id + '.' + n - - def enter(self) -> None: - self.locals.append(SymbolTable()) - self.global_decls.append(set()) - self.nonlocal_decls.append(set()) - # -1 since entering block will increment this to 0. - self.block_depth.append(-1) - - def leave(self) -> None: - self.locals.pop() - self.global_decls.pop() - self.nonlocal_decls.pop() - self.block_depth.pop() - - def is_func_scope(self) -> bool: - return self.locals[-1] is not None - - def is_class_scope(self) -> bool: - return self.type is not None and not self.is_func_scope() - - def add_symbol(self, name: str, node: SymbolTableNode, - context: Context) -> None: - if self.is_func_scope(): - if name in self.locals[-1]: - # Flag redefinition unless this is a reimport of a module. - if not (node.kind == MODULE_REF and - self.locals[-1][name].node == node.node): - self.name_already_defined(name, context) - self.locals[-1][name] = node - elif self.type: - self.type.names[name] = node - else: - existing = self.globals.get(name) - if existing and (not isinstance(node.node, MypyFile) or - existing.node != node.node) and existing.kind != UNBOUND_IMPORTED: - # Modules can be imported multiple times to support import - # of multiple submodules of a package (e.g. a.x and a.y). - ok = False - # Only report an error if the symbol collision provides a different type. - if existing.type and node.type and is_same_type(existing.type, node.type): - ok = True - if not ok: - self.name_already_defined(name, context) - self.globals[name] = node - - def add_var(self, v: Var, ctx: Context) -> None: - if self.is_func_scope(): - self.add_local(v, ctx) - else: - self.globals[v.name()] = SymbolTableNode(GDEF, v, self.cur_mod_id) - v._fullname = self.qualified_name(v.name()) - - def add_local(self, node: Union[Var, FuncBase], ctx: Context) -> None: - name = node.name() - if name in self.locals[-1]: - self.name_already_defined(name, ctx) - node._fullname = name - self.locals[-1][name] = SymbolTableNode(LDEF, node) - - def add_exports(self, *exps: Node) -> None: - for exp in exps: - if isinstance(exp, StrExpr): - self.all_exports.add(exp.value) - - def check_no_global(self, n: str, ctx: Context, - is_overloaded_func: bool = False) -> None: - if n in self.globals: - prev_is_overloaded = isinstance(self.globals[n], OverloadedFuncDef) - if is_overloaded_func and prev_is_overloaded: - self.fail("Nonconsecutive overload {} found".format(n), ctx) - elif prev_is_overloaded: - self.fail("Definition of '{}' missing 'overload'".format(n), ctx) - else: - self.name_already_defined(n, ctx) - - def name_not_defined(self, name: str, ctx: Context) -> None: - message = "Name '{}' is not defined".format(name) - extra = self.undefined_name_extra_info(name) - if extra: - message += ' {}'.format(extra) - self.fail(message, ctx) - - def name_already_defined(self, name: str, ctx: Context) -> None: - self.fail("Name '{}' already defined".format(name), ctx) - - def fail(self, msg: str, ctx: Context, serious: bool = False, *, - blocker: bool = False) -> None: - if (not serious and - not self.options.check_untyped_defs and - self.function_stack and - self.function_stack[-1].is_dynamic()): - return - self.errors.report(ctx.get_line(), msg, blocker=blocker) - - def note(self, msg: str, ctx: Context) -> None: - if (not self.options.check_untyped_defs and - self.function_stack and - self.function_stack[-1].is_dynamic()): - return - self.errors.report(ctx.get_line(), msg, severity='note') - - def undefined_name_extra_info(self, fullname: str) -> Optional[str]: - if fullname in obsolete_name_mapping: - return "(it's now called '{}')".format(obsolete_name_mapping[fullname]) - else: - return None - - def accept(self, node: Node) -> None: - try: - node.accept(self) - except Exception as err: - report_internal_error(err, self.errors.file, node.line) - - -class FirstPass(NodeVisitor): - """First phase of semantic analysis. - - See docstring of 'analyze()' below for a description of what this does. - """ - - def __init__(self, sem: SemanticAnalyzer) -> None: - self.sem = sem - self.pyversion = sem.options.python_version - - def analyze(self, file: MypyFile, fnam: str, mod_id: str) -> None: - """Perform the first analysis pass. - - Populate module global table. Resolve the full names of - definitions not nested within functions and construct type - info structures, but do not resolve inter-definition - references such as base classes. - - Also add implicit definitions such as __name__. - - In this phase we don't resolve imports. For 'from ... import', - we generate dummy symbol table nodes for the imported names, - and these will get resolved in later phases of semantic - analysis. - """ - sem = self.sem - sem.cur_mod_id = mod_id - sem.errors.set_file(fnam) - sem.globals = SymbolTable() - sem.global_decls = [set()] - sem.nonlocal_decls = [set()] - sem.block_depth = [0] - - defs = file.defs - - # Add implicit definitions of module '__name__' etc. - for name, t in implicit_module_attrs.items(): - v = Var(name, UnboundType(t)) - v._fullname = self.sem.qualified_name(name) - self.sem.globals[name] = SymbolTableNode(GDEF, v, self.sem.cur_mod_id) - - for d in defs: - d.accept(self) - - # Add implicit definition of literals/keywords to builtins, as we - # cannot define a variable with them explicitly. - if mod_id == 'builtins': - literal_types = [ - ('None', NoneTyp()), - # reveal_type is a mypy-only function that gives an error with the type of its arg - ('reveal_type', AnyType()), - ] # type: List[Tuple[str, Type]] - - # TODO(ddfisher): This guard is only needed because mypy defines - # fake builtins for its tests which often don't define bool. If - # mypy is fast enough that we no longer need those, this - # conditional check should be removed. - if 'bool' in self.sem.globals: - bool_type = self.sem.named_type('bool') - literal_types.extend([ - ('True', bool_type), - ('False', bool_type), - ('__debug__', bool_type), - ]) - - for name, typ in literal_types: - v = Var(name, typ) - v._fullname = self.sem.qualified_name(name) - self.sem.globals[name] = SymbolTableNode(GDEF, v, self.sem.cur_mod_id) - - def visit_block(self, b: Block) -> None: - if b.is_unreachable: - return - self.sem.block_depth[-1] += 1 - for node in b.body: - node.accept(self) - self.sem.block_depth[-1] -= 1 - - def visit_assignment_stmt(self, s: AssignmentStmt) -> None: - for lval in s.lvalues: - self.analyze_lvalue(lval, explicit_type=s.type is not None) - - def visit_func_def(self, func: FuncDef) -> None: - sem = self.sem - func.is_conditional = sem.block_depth[-1] > 0 - func._fullname = sem.qualified_name(func.name()) - if func.name() in sem.globals: - # Already defined in this module. - original_sym = sem.globals[func.name()] - if original_sym.kind == UNBOUND_IMPORTED: - # Ah this is an imported name. We can't resolve them now, so we'll postpone - # this until the main phase of semantic analysis. - return - original_def = original_sym.node - if sem.is_conditional_func(original_def, func): - # Conditional function definition -- multiple defs are ok. - func.original_def = cast(FuncDef, original_def) - else: - # Report error. - sem.check_no_global(func.name(), func) - else: - sem.globals[func.name()] = SymbolTableNode(GDEF, func, sem.cur_mod_id) - - def visit_overloaded_func_def(self, func: OverloadedFuncDef) -> None: - self.sem.check_no_global(func.name(), func, True) - func._fullname = self.sem.qualified_name(func.name()) - self.sem.globals[func.name()] = SymbolTableNode(GDEF, func, - self.sem.cur_mod_id) - - def visit_class_def(self, cdef: ClassDef) -> None: - self.sem.check_no_global(cdef.name, cdef) - cdef.fullname = self.sem.qualified_name(cdef.name) - info = TypeInfo(SymbolTable(), cdef) - info.set_line(cdef.line) - cdef.info = info - self.sem.globals[cdef.name] = SymbolTableNode(GDEF, info, - self.sem.cur_mod_id) - self.process_nested_classes(cdef) - - def process_nested_classes(self, outer_def: ClassDef) -> None: - for node in outer_def.defs.body: - if isinstance(node, ClassDef): - node.info = TypeInfo(SymbolTable(), node) - if outer_def.fullname: - node.info._fullname = outer_def.fullname + '.' + node.info.name() - else: - node.info._fullname = node.info.name() - symbol = SymbolTableNode(MDEF, node.info) - outer_def.info.names[node.name] = symbol - self.process_nested_classes(node) - - def visit_import_from(self, node: ImportFrom) -> None: - # We can't bind module names during the first pass, as the target module might be - # unprocessed. However, we add dummy unbound imported names to the symbol table so - # that we at least know that the name refers to a module. - node.is_top_level = True - for name, as_name in node.names: - imported_name = as_name or name - if imported_name not in self.sem.globals: - self.sem.add_symbol(imported_name, SymbolTableNode(UNBOUND_IMPORTED, None), node) - - def visit_import(self, node: Import) -> None: - node.is_top_level = True - # This is similar to visit_import_from -- see the comment there. - for id, as_id in node.ids: - imported_id = as_id or id - if imported_id not in self.sem.globals: - self.sem.add_symbol(imported_id, SymbolTableNode(UNBOUND_IMPORTED, None), node) - else: - # If the previous symbol is a variable, this should take precedence. - self.sem.globals[imported_id] = SymbolTableNode(UNBOUND_IMPORTED, None) - - def visit_import_all(self, node: ImportAll) -> None: - node.is_top_level = True - - def visit_while_stmt(self, s: WhileStmt) -> None: - s.body.accept(self) - if s.else_body: - s.else_body.accept(self) - - def visit_for_stmt(self, s: ForStmt) -> None: - self.analyze_lvalue(s.index) - s.body.accept(self) - if s.else_body: - s.else_body.accept(self) - - def visit_with_stmt(self, s: WithStmt) -> None: - for n in s.target: - if n: - self.analyze_lvalue(n) - s.body.accept(self) - - def visit_decorator(self, d: Decorator) -> None: - d.var._fullname = self.sem.qualified_name(d.var.name()) - self.sem.add_symbol(d.var.name(), SymbolTableNode(GDEF, d.var), d) - - def visit_if_stmt(self, s: IfStmt) -> None: - infer_reachability_of_if_statement(s, pyversion=self.pyversion) - for node in s.body: - node.accept(self) - if s.else_body: - s.else_body.accept(self) - - def visit_try_stmt(self, s: TryStmt) -> None: - self.sem.analyze_try_stmt(s, self, add_global=True) - - def analyze_lvalue(self, lvalue: Node, explicit_type: bool = False) -> None: - self.sem.analyze_lvalue(lvalue, add_global=True, explicit_type=explicit_type) - - -class ThirdPass(TraverserVisitor): - """The third and final pass of semantic analysis. - - Check type argument counts and values of generic types, and perform some - straightforward type inference. - """ - - def __init__(self, modules: Dict[str, MypyFile], errors: Errors) -> None: - self.modules = modules - self.errors = errors - - def visit_file(self, file_node: MypyFile, fnam: str) -> None: - self.errors.set_file(fnam) - self.accept(file_node) - - def accept(self, node: Node) -> None: - try: - node.accept(self) - except Exception as err: - report_internal_error(err, self.errors.file, node.line) - - def visit_block(self, b: Block) -> None: - if b.is_unreachable: - return - super().visit_block(b) - - def visit_func_def(self, fdef: FuncDef) -> None: - self.errors.push_function(fdef.name()) - self.analyze(fdef.type) - super().visit_func_def(fdef) - self.errors.pop_function() - - def visit_class_def(self, tdef: ClassDef) -> None: - for type in tdef.info.bases: - self.analyze(type) - # Recompute MRO now that we have analyzed all modules, to pick - # up superclasses of bases imported from other modules in an - # import loop. (Only do so if we succeeded the first time.) - if tdef.info.mro: - tdef.info.mro = [] # Force recomputation - calculate_class_mro(tdef, self.fail) - super().visit_class_def(tdef) - - def visit_decorator(self, dec: Decorator) -> None: - """Try to infer the type of the decorated function. - - This lets us resolve references to decorated functions during - type checking when there are cyclic imports, as otherwise the - type might not be available when we need it. - - This basically uses a simple special-purpose type inference - engine just for decorators. - """ - super().visit_decorator(dec) - if dec.var.is_property: - # Decorators are expected to have a callable type (it's a little odd). - if dec.func.type is None: - dec.var.type = CallableType( - [AnyType()], - [ARG_POS], - [None], - AnyType(), - self.builtin_type('function'), - name=dec.var.name()) - elif isinstance(dec.func.type, CallableType): - dec.var.type = dec.func.type - return - decorator_preserves_type = True - for expr in dec.decorators: - preserve_type = False - if isinstance(expr, RefExpr) and isinstance(expr.node, FuncDef): - if is_identity_signature(expr.node.type): - preserve_type = True - if not preserve_type: - decorator_preserves_type = False - break - if decorator_preserves_type: - # No non-identity decorators left. We can trivially infer the type - # of the function here. - dec.var.type = function_type(dec.func, self.builtin_type('function')) - if dec.decorators: - if returns_any_if_called(dec.decorators[0]): - # The outermost decorator will return Any so we know the type of the - # decorated function. - dec.var.type = AnyType() - sig = find_fixed_callable_return(dec.decorators[0]) - if sig: - # The outermost decorator always returns the same kind of function, - # so we know that this is the type of the decoratored function. - orig_sig = function_type(dec.func, self.builtin_type('function')) - sig.name = orig_sig.items()[0].name - dec.var.type = sig - - def visit_assignment_stmt(self, s: AssignmentStmt) -> None: - self.analyze(s.type) - super().visit_assignment_stmt(s) - - def visit_cast_expr(self, e: CastExpr) -> None: - self.analyze(e.type) - super().visit_cast_expr(e) - - def visit_reveal_type_expr(self, e: RevealTypeExpr) -> None: - super().visit_reveal_type_expr(e) - - def visit_type_application(self, e: TypeApplication) -> None: - for type in e.types: - self.analyze(type) - super().visit_type_application(e) - - # Helpers - - def analyze(self, type: Type) -> None: - if type: - analyzer = TypeAnalyserPass3(self.fail) - type.accept(analyzer) - - def fail(self, msg: str, ctx: Context) -> None: - self.errors.report(ctx.get_line(), msg) - - def builtin_type(self, name: str, args: List[Type] = None) -> Instance: - names = self.modules['builtins'] - sym = names.names[name] - assert isinstance(sym.node, TypeInfo) - return Instance(sym.node, args or []) - - -def self_type(typ: TypeInfo) -> Union[Instance, TupleType]: - """For a non-generic type, return instance type representing the type. - For a generic G type with parameters T1, .., Tn, return G[T1, ..., Tn]. - """ - tv = [] # type: List[Type] - for i in range(len(typ.type_vars)): - tv.append(TypeVarType(typ.defn.type_vars[i])) - inst = Instance(typ, tv) - if typ.tuple_type is None: - return inst - return typ.tuple_type.self_type(inst) - - -def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: - if isinstance(sig, CallableType): - return replace_leading_arg_type(sig, new) - else: - sig = cast(Overloaded, sig) - return Overloaded([cast(CallableType, replace_implicit_first_type(i, new)) - for i in sig.items()]) - - -def set_callable_name(sig: Type, fdef: FuncDef) -> Type: - if isinstance(sig, FunctionLike): - if fdef.info: - return sig.with_name( - '"{}" of "{}"'.format(fdef.name(), fdef.info.name())) - else: - return sig.with_name('"{}"'.format(fdef.name())) - else: - return sig - - -def refers_to_fullname(node: Node, fullname: str) -> bool: - """Is node a name or member expression with the given full name?""" - return isinstance(node, RefExpr) and node.fullname == fullname - - -def refers_to_class_or_function(node: Node) -> bool: - """Does semantically analyzed node refer to a class?""" - return (isinstance(node, RefExpr) and - isinstance(node.node, (TypeInfo, FuncDef, OverloadedFuncDef))) - - -def calculate_class_mro(defn: ClassDef, fail: Callable[[str, Context], None]) -> None: - try: - defn.info.calculate_mro() - except MroError: - fail("Cannot determine consistent method resolution order " - '(MRO) for "%s"' % defn.name, defn) - defn.info.mro = [] - # The property of falling back to Any is inherited. - defn.info.fallback_to_any = any(baseinfo.fallback_to_any for baseinfo in defn.info.mro) - - -def find_duplicate(list: List[T]) -> T: - """If the list has duplicates, return one of the duplicates. - - Otherwise, return None. - """ - for i in range(1, len(list)): - if list[i] in list[:i]: - return list[i] - return None - - -def disable_typevars(nodes: List[SymbolTableNode]) -> None: - for node in nodes: - assert node.kind in (BOUND_TVAR, UNBOUND_TVAR) - node.kind = UNBOUND_TVAR - - -def enable_typevars(nodes: List[SymbolTableNode]) -> None: - for node in nodes: - assert node.kind in (BOUND_TVAR, UNBOUND_TVAR) - node.kind = BOUND_TVAR - - -def remove_imported_names_from_symtable(names: SymbolTable, - module: str) -> None: - """Remove all imported names from the symbol table of a module.""" - removed = [] # type: List[str] - for name, node in names.items(): - if node.node is None: - continue - fullname = node.node.fullname() - prefix = fullname[:fullname.rfind('.')] - if prefix != module: - removed.append(name) - for name in removed: - del names[name] - - -def infer_reachability_of_if_statement(s: IfStmt, - pyversion: Tuple[int, int]) -> None: - for i in range(len(s.expr)): - result = infer_if_condition_value(s.expr[i], pyversion) - if result == ALWAYS_FALSE: - # The condition is always false, so we skip the if/elif body. - mark_block_unreachable(s.body[i]) - elif result == ALWAYS_TRUE: - # This condition is always true, so all of the remaining - # elif/else bodies will never be executed. - for body in s.body[i + 1:]: - mark_block_unreachable(body) - if s.else_body: - mark_block_unreachable(s.else_body) - break - - -def infer_if_condition_value(expr: Node, pyversion: Tuple[int, int]) -> int: - """Infer whether if condition is always true/false. - - Return ALWAYS_TRUE if always true, ALWAYS_FALSE if always false, - and TRUTH_VALUE_UNKNOWN otherwise. - """ - name = '' - negated = False - alias = expr - if isinstance(alias, UnaryExpr): - if alias.op == 'not': - expr = alias.expr - negated = True - if isinstance(expr, NameExpr): - name = expr.name - elif isinstance(expr, MemberExpr): - name = expr.name - result = TRUTH_VALUE_UNKNOWN - if name == 'PY2': - result = ALWAYS_TRUE if pyversion[0] == 2 else ALWAYS_FALSE - elif name == 'PY3': - result = ALWAYS_TRUE if pyversion[0] == 3 else ALWAYS_FALSE - elif name == 'MYPY': - result = ALWAYS_TRUE - if negated: - if result == ALWAYS_TRUE: - result = ALWAYS_FALSE - elif result == ALWAYS_FALSE: - result = ALWAYS_TRUE - return result - - -def mark_block_unreachable(block: Block) -> None: - block.is_unreachable = True - block.accept(MarkImportsUnreachableVisitor()) - - -class MarkImportsUnreachableVisitor(TraverserVisitor): - """Visitor that flags all imports nested within a node as unreachable.""" - - def visit_import(self, node: Import) -> None: - node.is_unreachable = True - - def visit_import_from(self, node: ImportFrom) -> None: - node.is_unreachable = True - - def visit_import_all(self, node: ImportAll) -> None: - node.is_unreachable = True - - -def is_identity_signature(sig: Type) -> bool: - """Is type a callable of form T -> T (where T is a type variable)?""" - if isinstance(sig, CallableType) and sig.arg_kinds == [ARG_POS]: - if isinstance(sig.arg_types[0], TypeVarType) and isinstance(sig.ret_type, TypeVarType): - return sig.arg_types[0].id == sig.ret_type.id - return False - - -def returns_any_if_called(expr: Node) -> bool: - """Return True if we can predict that expr will return Any if called. - - This only uses information available during semantic analysis so this - will sometimes return False because of insufficient information (as - type inference hasn't run yet). - """ - if isinstance(expr, RefExpr): - if isinstance(expr.node, FuncDef): - typ = expr.node.type - if typ is None: - # No signature -> default to Any. - return True - # Explicit Any return? - return isinstance(typ, CallableType) and isinstance(typ.ret_type, AnyType) - elif isinstance(expr.node, Var): - typ = expr.node.type - return typ is None or isinstance(typ, AnyType) - elif isinstance(expr, CallExpr): - return returns_any_if_called(expr.callee) - return False - - -def find_fixed_callable_return(expr: Node) -> Optional[CallableType]: - if isinstance(expr, RefExpr): - if isinstance(expr.node, FuncDef): - typ = expr.node.type - if typ: - if isinstance(typ, CallableType) and has_no_typevars(typ.ret_type): - if isinstance(typ.ret_type, CallableType): - return typ.ret_type - elif isinstance(expr, CallExpr): - t = find_fixed_callable_return(expr.callee) - if t: - if isinstance(t.ret_type, CallableType): - return t.ret_type - return None - - -def has_no_typevars(typ: Type) -> bool: - return is_same_type(typ, erase_typevars(typ)) +"""The semantic analyzer. + +Bind names to definitions and do various other simple consistency +checks. For example, consider this program: + + x = 1 + y = x + +Here semantic analysis would detect that the assignment 'x = 1' +defines a new variable, the type of which is to be inferred (in a +later pass; type inference or type checking is not part of semantic +analysis). Also, it would bind both references to 'x' to the same +module-level variable node. The second assignment would also be +analyzed, and the type of 'y' marked as being inferred. + +Semantic analysis is the first analysis pass after parsing, and it is +subdivided into three passes: + + * FirstPass looks up externally visible names defined in a module but + ignores imports and local definitions. It helps enable (some) + cyclic references between modules, such as module 'a' that imports + module 'b' and used names defined in b *and* vice versa. The first + pass can be performed before dependent modules have been processed. + + * SemanticAnalyzer is the second pass. It does the bulk of the work. + It assumes that dependent modules have been semantically analyzed, + up to the second pass, unless there is a import cycle. + + * ThirdPass checks that type argument counts are valid; for example, + it will reject Dict[int]. We don't do this in the second pass, + since we infer the type argument counts of classes during this + pass, and it is possible to refer to classes defined later in a + file, which would not have the type argument count set yet. This + pass also recomputes the method resolution order of each class, in + case one of its bases belongs to a module involved in an import + loop. + +Semantic analysis of types is implemented in module mypy.typeanal. + +TODO: Check if the third pass slows down type checking significantly. + We could probably get rid of it -- for example, we could collect all + analyzed types in a collection and check them without having to + traverse the entire AST. +""" + +from typing import ( + List, Dict, Set, Tuple, cast, Any, overload, TypeVar, Union, Optional, Callable +) + +from mypy.nodes import ( + MypyFile, TypeInfo, Node, AssignmentStmt, FuncDef, OverloadedFuncDef, + ClassDef, Var, GDEF, MODULE_REF, FuncItem, Import, + ImportFrom, ImportAll, Block, LDEF, NameExpr, MemberExpr, + IndexExpr, TupleExpr, ListExpr, ExpressionStmt, ReturnStmt, + RaiseStmt, AssertStmt, OperatorAssignmentStmt, WhileStmt, + ForStmt, BreakStmt, ContinueStmt, IfStmt, TryStmt, WithStmt, DelStmt, + GlobalDecl, SuperExpr, DictExpr, CallExpr, RefExpr, OpExpr, UnaryExpr, + SliceExpr, CastExpr, RevealTypeExpr, TypeApplication, Context, SymbolTable, + SymbolTableNode, BOUND_TVAR, UNBOUND_TVAR, ListComprehension, GeneratorExpr, + FuncExpr, MDEF, FuncBase, Decorator, SetExpr, TypeVarExpr, + StrExpr, BytesExpr, PrintStmt, ConditionalExpr, PromoteExpr, + ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, MroError, type_aliases, + YieldFromExpr, NamedTupleExpr, NonlocalDecl, + SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr, + YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, COVARIANT, CONTRAVARIANT, + IntExpr, FloatExpr, UnicodeExpr, + INVARIANT, UNBOUND_IMPORTED, Expression, EllipsisExpr, NamedTupleTypeInfo +) +from mypy.visitor import NodeVisitor +from mypy.traverser import TraverserVisitor +from mypy.errors import Errors, report_internal_error +from mypy.types import ( + NoneTyp, CallableType, Overloaded, Instance, Type, TypeVarType, AnyType, + FunctionLike, UnboundType, TypeList, ErrorType, TypeVarDef, + replace_leading_arg_type, TupleType, UnionType, StarType, EllipsisType, + NamedTupleType +) +from mypy.nodes import function_type, implicit_module_attrs +from mypy.typeanal import TypeAnalyser, TypeAnalyserPass3, analyze_type_alias +from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError +from mypy.lex import lex +from mypy.parsetype import parse_type +from mypy.sametypes import is_same_type +from mypy.erasetype import erase_typevars +from mypy import defaults +from mypy.options import Options + + +T = TypeVar('T') + + +# Inferred value of an expression. +ALWAYS_TRUE = 0 +ALWAYS_FALSE = 1 +TRUTH_VALUE_UNKNOWN = 2 + +# Map from obsolete name to the current spelling. +obsolete_name_mapping = { + 'typing.Function': 'typing.Callable', + 'typing.typevar': 'typing.TypeVar', +} + +# Hard coded type promotions (shared between all Python versions). +# These add extra ad-hoc edges to the subtyping relation. For example, +# int is considered a subtype of float, even though there is no +# subclass relationship. +TYPE_PROMOTIONS = { + 'builtins.int': 'builtins.float', + 'builtins.float': 'builtins.complex', +} + +# Hard coded type promotions for Python 3. +# +# Note that the bytearray -> bytes promotion is a little unsafe +# as some functions only accept bytes objects. Here convenience +# trumps safety. +TYPE_PROMOTIONS_PYTHON3 = TYPE_PROMOTIONS.copy() +TYPE_PROMOTIONS_PYTHON3.update({ + 'builtins.bytearray': 'builtins.bytes', +}) + +# Hard coded type promotions for Python 2. +# +# These promotions are unsafe, but we are doing them anyway +# for convenience and also for Python 3 compatibility +# (bytearray -> str). +TYPE_PROMOTIONS_PYTHON2 = TYPE_PROMOTIONS.copy() +TYPE_PROMOTIONS_PYTHON2.update({ + 'builtins.str': 'builtins.unicode', + 'builtins.bytearray': 'builtins.str', +}) + +# When analyzing a function, should we analyze the whole function in one go, or +# should we only perform one phase of the analysis? The latter is used for +# nested functions. In the first phase we add the function to the symbol table +# but don't process body. In the second phase we process function body. This +# way we can have mutually recursive nested functions. +FUNCTION_BOTH_PHASES = 0 # Everthing in one go +FUNCTION_FIRST_PHASE_POSTPONE_SECOND = 1 # Add to symbol table but postpone body +FUNCTION_SECOND_PHASE = 2 # Only analyze body + + +class SemanticAnalyzer(NodeVisitor): + """Semantically analyze parsed mypy files. + + The analyzer binds names and does various consistency checks for a + parse tree. Note that type checking is performed as a separate + pass. + + This is the second phase of semantic analysis. + """ + + # Library search paths + lib_path = None # type: List[str] + # Module name space + modules = None # type: Dict[str, MypyFile] + # Global name space for current module + globals = None # type: SymbolTable + # Names declared using "global" (separate set for each scope) + global_decls = None # type: List[Set[str]] + # Names declated using "nonlocal" (separate set for each scope) + nonlocal_decls = None # type: List[Set[str]] + # Local names of function scopes; None for non-function scopes. + locals = None # type: List[SymbolTable] + # Nested block depths of scopes + block_depth = None # type: List[int] + # TypeInfo of directly enclosing class (or None) + type = None # type: TypeInfo + # Stack of outer classes (the second tuple item contains tvars). + type_stack = None # type: List[TypeInfo] + # Type variables that are bound by the directly enclosing class + bound_tvars = None # type: List[SymbolTableNode] + # Stack of type variables that were bound by outer classess + tvar_stack = None # type: List[List[SymbolTableNode]] + # Do weak type checking in this file + weak_opts = set() # type: Set[str] + + # Stack of functions being analyzed + function_stack = None # type: List[FuncItem] + # Stack of next available function type variable ids + next_function_tvar_id_stack = None # type: List[int] + + # Status of postponing analysis of nested function bodies. By using this we + # can have mutually recursive nested functions. Values are FUNCTION_x + # constants. Note that separate phasea are not used for methods. + postpone_nested_functions_stack = None # type: List[int] + # Postponed functions collected if + # postpone_nested_functions_stack[-1] == FUNCTION_FIRST_PHASE_POSTPONE_SECOND. + postponed_functions_stack = None # type: List[List[Node]] + + loop_depth = 0 # Depth of breakable loops + cur_mod_id = '' # Current module id (or None) (phase 2) + is_stub_file = False # Are we analyzing a stub file? + imports = None # type: Set[str] # Imported modules (during phase 2 analysis) + errors = None # type: Errors # Keeps track of generated errors + + def __init__(self, + lib_path: List[str], + errors: Errors, + options: Options) -> None: + """Construct semantic analyzer. + + Use lib_path to search for modules, and report analysis errors + using the Errors instance. + """ + self.locals = [None] + self.imports = set() + self.type = None + self.type_stack = [] + self.bound_tvars = None + self.tvar_stack = [] + self.function_stack = [] + self.next_function_tvar_id_stack = [-1] + self.block_depth = [0] + self.loop_depth = 0 + self.lib_path = lib_path + self.errors = errors + self.modules = {} + self.options = options + self.postpone_nested_functions_stack = [FUNCTION_BOTH_PHASES] + self.postponed_functions_stack = [] + self.all_exports = set() # type: Set[str] + + def visit_file(self, file_node: MypyFile, fnam: str) -> None: + self.errors.set_file(fnam) + self.cur_mod_node = file_node + self.cur_mod_id = file_node.fullname() + self.is_stub_file = fnam.lower().endswith('.pyi') + self.globals = file_node.names + self.weak_opts = file_node.weak_opts + + if 'builtins' in self.modules: + self.globals['__builtins__'] = SymbolTableNode( + MODULE_REF, self.modules['builtins'], self.cur_mod_id) + + for name in implicit_module_attrs: + v = self.globals[name].node + if isinstance(v, Var): + v.type = self.anal_type(v.type) + v.is_ready = True + + defs = file_node.defs + for d in defs: + self.accept(d) + + if self.cur_mod_id == 'builtins': + remove_imported_names_from_symtable(self.globals, 'builtins') + + if '__all__' in self.globals: + for name, g in self.globals.items(): + if name not in self.all_exports: + g.module_public = False + + def visit_func_def(self, defn: FuncDef) -> None: + phase_info = self.postpone_nested_functions_stack[-1] + if phase_info != FUNCTION_SECOND_PHASE: + self.function_stack.append(defn) + # First phase of analysis for function. + self.errors.push_function(defn.name()) + self.update_function_type_variables(defn) + self.errors.pop_function() + self.function_stack.pop() + + defn.is_conditional = self.block_depth[-1] > 0 + + # TODO(jukka): Figure out how to share the various cases. It doesn't + # make sense to have (almost) duplicate code (here and elsewhere) for + # 3 cases: module-level, class-level and local names. Maybe implement + # a common stack of namespaces. As the 3 kinds of namespaces have + # different semantics, this wouldn't always work, but it might still + # be a win. + if self.is_class_scope(): + # Method definition + defn.info = self.type + if not defn.is_decorated and not defn.is_overload: + if defn.name() in self.type.names: + # Redefinition. Conditional redefinition is okay. + n = self.type.names[defn.name()].node + if self.is_conditional_func(n, defn): + defn.original_def = cast(FuncDef, n) + else: + self.name_already_defined(defn.name(), defn) + self.type.names[defn.name()] = SymbolTableNode(MDEF, defn) + self.prepare_method_signature(defn) + elif self.is_func_scope(): + # Nested function + if not defn.is_decorated and not defn.is_overload: + if defn.name() in self.locals[-1]: + # Redefinition. Conditional redefinition is okay. + n = self.locals[-1][defn.name()].node + if self.is_conditional_func(n, defn): + defn.original_def = cast(FuncDef, n) + else: + self.name_already_defined(defn.name(), defn) + else: + self.add_local(defn, defn) + else: + # Top-level function + if not defn.is_decorated and not defn.is_overload: + symbol = self.globals.get(defn.name()) + if isinstance(symbol.node, FuncDef) and symbol.node != defn: + # This is redefinition. Conditional redefinition is okay. + original_def = symbol.node + if self.is_conditional_func(original_def, defn): + # Conditional function definition -- multiple defs are ok. + defn.original_def = original_def + else: + # Report error. + self.check_no_global(defn.name(), defn, True) + if phase_info == FUNCTION_FIRST_PHASE_POSTPONE_SECOND: + # Postpone this function (for the second phase). + self.postponed_functions_stack[-1].append(defn) + return + if phase_info != FUNCTION_FIRST_PHASE_POSTPONE_SECOND: + # Second phase of analysis for function. + self.errors.push_function(defn.name()) + self.analyze_function(defn) + self.errors.pop_function() + + def prepare_method_signature(self, func: FuncDef) -> None: + """Check basic signature validity and tweak annotation of self/cls argument.""" + # Only non-static methods are special. + if not func.is_static: + if not func.arguments: + self.fail('Method must have at least one argument', func) + elif func.type: + sig = cast(FunctionLike, func.type) + if func.is_class: + leading_type = self.class_type(self.type) + else: + leading_type = self_type(self.type) + func.type = replace_implicit_first_type(sig, leading_type) + + def is_conditional_func(self, previous: Node, new: FuncDef) -> bool: + """Does 'new' conditionally redefine 'previous'? + + We reject straight redefinitions of functions, as they are usually + a programming error. For example: + + . def f(): ... + . def f(): ... # Error: 'f' redefined + """ + return isinstance(previous, (FuncDef, Var)) and new.is_conditional + + def update_function_type_variables(self, defn: FuncDef) -> None: + """Make any type variables in the signature of defn explicit. + + Update the signature of defn to contain type variable definitions + if defn is generic. + """ + if defn.type: + functype = cast(CallableType, defn.type) + typevars = self.infer_type_variables(functype) + # Do not define a new type variable if already defined in scope. + typevars = [(name, tvar) for name, tvar in typevars + if not self.is_defined_type_var(name, defn)] + if typevars: + next_tvar_id = self.next_function_tvar_id() + defs = [TypeVarDef(tvar[0], next_tvar_id - i, + tvar[1].values, tvar[1].upper_bound, + tvar[1].variance) + for i, tvar in enumerate(typevars)] + functype.variables = defs + + def infer_type_variables(self, + type: CallableType) -> List[Tuple[str, TypeVarExpr]]: + """Return list of unique type variables referred to in a callable.""" + names = [] # type: List[str] + tvars = [] # type: List[TypeVarExpr] + for arg in type.arg_types + [type.ret_type]: + for name, tvar_expr in self.find_type_variables_in_type(arg): + if name not in names: + names.append(name) + tvars.append(tvar_expr) + return list(zip(names, tvars)) + + def find_type_variables_in_type( + self, type: Type) -> List[Tuple[str, TypeVarExpr]]: + """Return a list of all unique type variable references in type. + + This effectively does partial name binding, results of which are mostly thrown away. + """ + result = [] # type: List[Tuple[str, TypeVarExpr]] + if isinstance(type, UnboundType): + name = type.name + node = self.lookup_qualified(name, type) + if node and node.kind == UNBOUND_TVAR: + result.append((name, cast(TypeVarExpr, node.node))) + for arg in type.args: + result.extend(self.find_type_variables_in_type(arg)) + elif isinstance(type, TypeList): + for item in type.items: + result.extend(self.find_type_variables_in_type(item)) + elif isinstance(type, UnionType): + for item in type.items: + result.extend(self.find_type_variables_in_type(item)) + elif isinstance(type, AnyType): + pass + elif isinstance(type, EllipsisType) or isinstance(type, TupleType): + pass + else: + assert False, 'Unsupported type %s' % type + return result + + def is_defined_type_var(self, tvar: str, context: Node) -> bool: + return self.lookup_qualified(tvar, context).kind == BOUND_TVAR + + def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: + t = [] # type: List[CallableType] + for i, item in enumerate(defn.items): + # TODO support decorated overloaded functions properly + item.is_overload = True + item.func.is_overload = True + item.accept(self) + t.append(cast(CallableType, function_type(item.func, + self.builtin_type('builtins.function')))) + if item.func.is_property and i == 0: + # This defines a property, probably with a setter and/or deleter. + self.analyze_property_with_multi_part_definition(defn) + break + if not [dec for dec in item.decorators + if refers_to_fullname(dec, 'typing.overload')]: + self.fail("'overload' decorator expected", item) + + defn.type = Overloaded(t) + defn.type.line = defn.line + + if self.is_class_scope(): + self.type.names[defn.name()] = SymbolTableNode(MDEF, defn, + typ=defn.type) + defn.info = self.type + elif self.is_func_scope(): + self.add_local(defn, defn) + + def analyze_property_with_multi_part_definition(self, defn: OverloadedFuncDef) -> None: + """Analyze a property defined using multiple methods (e.g., using @x.setter). + + Assume that the first method (@property) has already been analyzed. + """ + defn.is_property = True + items = defn.items + for item in items[1:]: + if len(item.decorators) == 1: + node = item.decorators[0] + if isinstance(node, MemberExpr): + if node.name == 'setter': + # The first item represents the entire property. + defn.items[0].var.is_settable_property = True + # Get abstractness from the original definition. + item.func.is_abstract = items[0].func.is_abstract + else: + self.fail("Decorated property not supported", item) + item.func.accept(self) + + def next_function_tvar_id(self) -> int: + return self.next_function_tvar_id_stack[-1] + + def analyze_function(self, defn: FuncItem) -> None: + is_method = self.is_class_scope() + + tvarnodes = self.add_func_type_variables_to_symbol_table(defn) + next_function_tvar_id = min([self.next_function_tvar_id()] + + [n.tvar_def.id.raw_id - 1 for n in tvarnodes]) + self.next_function_tvar_id_stack.append(next_function_tvar_id) + + if defn.type: + # Signature must be analyzed in the surrounding scope so that + # class-level imported names and type variables are in scope. + defn.type = self.anal_type(defn.type) + self.check_function_signature(defn) + if isinstance(defn, FuncDef): + defn.type = set_callable_name(defn.type, defn) + for arg in defn.arguments: + if arg.initializer: + arg.initializer.accept(self) + self.function_stack.append(defn) + self.enter() + for arg in defn.arguments: + self.add_local(arg.variable, defn) + for arg in defn.arguments: + if arg.initialization_statement: + lvalue = arg.initialization_statement.lvalues[0] + lvalue.accept(self) + + # The first argument of a non-static, non-class method is like 'self' + # (though the name could be different), having the enclosing class's + # instance type. + if is_method and not defn.is_static and not defn.is_class and defn.arguments: + defn.arguments[0].variable.is_self = True + + # First analyze body of the function but ignore nested functions. + self.postpone_nested_functions_stack.append(FUNCTION_FIRST_PHASE_POSTPONE_SECOND) + self.postponed_functions_stack.append([]) + defn.body.accept(self) + + # Analyze nested functions (if any) as a second phase. + self.postpone_nested_functions_stack[-1] = FUNCTION_SECOND_PHASE + for postponed in self.postponed_functions_stack[-1]: + postponed.accept(self) + self.postpone_nested_functions_stack.pop() + self.postponed_functions_stack.pop() + + self.next_function_tvar_id_stack.pop() + disable_typevars(tvarnodes) + + self.leave() + self.function_stack.pop() + + def add_func_type_variables_to_symbol_table( + self, defn: FuncItem) -> List[SymbolTableNode]: + nodes = [] # type: List[SymbolTableNode] + if defn.type: + tt = defn.type + names = self.type_var_names() + items = cast(CallableType, tt).variables + for item in items: + name = item.name + if name in names: + self.name_already_defined(name, defn) + node = self.bind_type_var(name, item, defn) + nodes.append(node) + names.add(name) + return nodes + + def type_var_names(self) -> Set[str]: + if not self.type: + return set() + else: + return set(self.type.type_vars) + + def bind_type_var(self, fullname: str, tvar_def: TypeVarDef, + context: Context) -> SymbolTableNode: + node = self.lookup_qualified(fullname, context) + node.kind = BOUND_TVAR + node.tvar_def = tvar_def + return node + + def check_function_signature(self, fdef: FuncItem) -> None: + sig = cast(CallableType, fdef.type) + if len(sig.arg_types) < len(fdef.arguments): + self.fail('Type signature has too few arguments', fdef) + # Add dummy Any arguments to prevent crashes later. + extra_anys = [AnyType()] * (len(fdef.arguments) - len(sig.arg_types)) + sig.arg_types.extend(extra_anys) + elif len(sig.arg_types) > len(fdef.arguments): + self.fail('Type signature has too many arguments', fdef) + + def visit_class_def(self, defn: ClassDef) -> None: + self.clean_up_bases_and_infer_type_variables(defn) + self.setup_class_def_analysis(defn) + + self.bind_class_type_vars(defn) + + self.analyze_base_classes(defn) + self.analyze_metaclass(defn) + + for decorator in defn.decorators: + self.analyze_class_decorator(defn, decorator) + + self.enter_class(defn) + + self.setup_is_builtinclass(defn) + + # Analyze class body. + defn.defs.accept(self) + + self.calculate_abstract_status(defn.info) + self.setup_type_promotion(defn) + + self.leave_class() + self.unbind_class_type_vars() + + def enter_class(self, defn: ClassDef) -> None: + # Remember previous active class + self.type_stack.append(self.type) + self.locals.append(None) # Add class scope + self.block_depth.append(-1) # The class body increments this to 0 + self.postpone_nested_functions_stack.append(FUNCTION_BOTH_PHASES) + self.type = defn.info + + def leave_class(self) -> None: + """ Restore analyzer state. """ + self.postpone_nested_functions_stack.pop() + self.block_depth.pop() + self.locals.pop() + self.type = self.type_stack.pop() + + def bind_class_type_vars(self, defn: ClassDef) -> None: + """ Unbind type variables of previously active class and bind + the type variables for the active class. + """ + if self.bound_tvars: + disable_typevars(self.bound_tvars) + self.tvar_stack.append(self.bound_tvars) + self.bound_tvars = self.bind_class_type_variables_in_symbol_table(defn.info) + + def unbind_class_type_vars(self) -> None: + """ Unbind the active class' type vars and rebind the + type vars of the previously active class. + """ + disable_typevars(self.bound_tvars) + self.bound_tvars = self.tvar_stack.pop() + if self.bound_tvars: + enable_typevars(self.bound_tvars) + + def analyze_class_decorator(self, defn: ClassDef, decorator: Node) -> None: + decorator.accept(self) + + def setup_is_builtinclass(self, defn: ClassDef): + for decorator in defn.decorators: + if refers_to_fullname(decorator, 'typing.builtinclass'): + defn.is_builtinclass = True + if defn.fullname == 'builtins.object': + # Only 'object' is marked as a built-in class, as otherwise things elsewhere + # would break. We need a better way of dealing with built-in classes. + defn.is_builtinclass = True + + def calculate_abstract_status(self, typ: TypeInfo) -> None: + """Calculate abstract status of a class. + + Set is_abstract of the type to True if the type has an unimplemented + abstract attribute. Also compute a list of abstract attributes. + """ + concrete = set() # type: Set[str] + abstract = [] # type: List[str] + for base in typ.mro: + for name, symnode in base.names.items(): + node = symnode.node + if isinstance(node, OverloadedFuncDef): + # Unwrap an overloaded function definition. We can just + # check arbitrarily the first overload item. If the + # different items have a different abstract status, there + # should be an error reported elsewhere. + func = node.items[0] # type: Node + else: + func = node + if isinstance(func, Decorator): + fdef = func.func + if fdef.is_abstract and name not in concrete: + typ.is_abstract = True + abstract.append(name) + concrete.add(name) + typ.abstract_attributes = sorted(abstract) + + def setup_type_promotion(self, defn: ClassDef) -> None: + """Setup extra, ad-hoc subtyping relationships between classes (promotion). + + This includes things like 'int' being compatible with 'float'. + """ + promote_target = None # type: Type + for decorator in defn.decorators: + if isinstance(decorator, CallExpr): + analyzed = decorator.analyzed + if isinstance(analyzed, PromoteExpr): + # _promote class decorator (undocumented faeture). + promote_target = analyzed.type + if not promote_target: + promotions = (TYPE_PROMOTIONS_PYTHON3 if self.options.python_version[0] >= 3 + else TYPE_PROMOTIONS_PYTHON2) + if defn.fullname in promotions: + promote_target = self.named_type_or_none(promotions[defn.fullname]) + defn.info._promote = promote_target + + def clean_up_bases_and_infer_type_variables(self, defn: ClassDef) -> None: + """Remove extra base classes such as Generic and infer type vars. + + For example, consider this class: + + . class Foo(Bar, Generic[T]): ... + + Now we will remove Generic[T] from bases of Foo and infer that the + type variable 'T' is a type argument of Foo. + + Note that this is performed *before* semantic analysis. + """ + removed = [] # type: List[int] + type_vars = [] # type: List[TypeVarDef] + for i, base_expr in enumerate(defn.base_type_exprs): + try: + base = expr_to_unanalyzed_type(base_expr) + except TypeTranslationError: + # This error will be caught later. + continue + tvars = self.analyze_typevar_declaration(base) + if tvars is not None: + if type_vars: + self.fail('Duplicate Generic in bases', defn) + removed.append(i) + for j, (name, tvar_expr) in enumerate(tvars): + type_vars.append(TypeVarDef(name, j + 1, tvar_expr.values, + tvar_expr.upper_bound, tvar_expr.variance)) + if type_vars: + defn.type_vars = type_vars + if defn.info: + defn.info.type_vars = [tv.name for tv in type_vars] + for i in reversed(removed): + del defn.base_type_exprs[i] + + def analyze_typevar_declaration(self, t: Type) -> Optional[List[Tuple[str, TypeVarExpr]]]: + if not isinstance(t, UnboundType): + return None + unbound = t + sym = self.lookup_qualified(unbound.name, unbound) + if sym is None or sym.node is None: + return None + if sym.node.fullname() == 'typing.Generic': + tvars = [] # type: List[Tuple[str, TypeVarExpr]] + for arg in unbound.args: + tvar = self.analyze_unbound_tvar(arg) + if tvar: + tvars.append(tvar) + else: + self.fail('Free type variable expected in %s[...]' % + sym.node.name(), t) + return tvars + return None + + def analyze_unbound_tvar(self, t: Type) -> Tuple[str, TypeVarExpr]: + if not isinstance(t, UnboundType): + return None + unbound = t + sym = self.lookup_qualified(unbound.name, unbound) + if sym is not None and sym.kind == UNBOUND_TVAR: + return unbound.name, cast(TypeVarExpr, sym.node) + return None + + def setup_class_def_analysis(self, defn: ClassDef) -> None: + """Prepare for the analysis of a class definition.""" + if not defn.info: + defn.info = TypeInfo(SymbolTable(), defn) + defn.info._fullname = defn.info.name() + if self.is_func_scope() or self.type: + kind = MDEF + if self.is_func_scope(): + kind = LDEF + self.add_symbol(defn.name, SymbolTableNode(kind, defn.info), defn) + + def analyze_base_classes(self, defn: ClassDef) -> None: + """Analyze and set up base classes. + + This computes several attributes on the corresponding TypeInfo defn.info + related to the base classes: defn.info.bases, defn.info.mro, and + miscellaneous others (at least tuple_type, fallback_to_any, and is_enum.) + """ + + base_types = [] # type: List[Instance] + info = defn.info + for base_expr in defn.base_type_exprs: + try: + base = self.expr_to_analyzed_type(base_expr) + except TypeTranslationError: + self.fail('Invalid base class', base_expr) + info.fallback_to_any = True + continue + + if isinstance(base, TupleType): + if info.tuple_type: + self.fail("Class has two incompatible bases derived from tuple", defn) + defn.incompatible_baseclass = True + if (not self.is_stub_file + and not info.is_named_tuple + and base.fallback.type.fullname() == 'builtins.tuple'): + self.fail("Tuple[...] not supported as a base class outside a stub file", defn) + info.tuple_type = base + base_types.append(base.fallback) + elif isinstance(base, Instance): + base_types.append(base) + elif isinstance(base, AnyType): + info.fallback_to_any = True + else: + self.fail('Invalid base class', base_expr) + info.fallback_to_any = True + + # Add 'object' as implicit base if there is no other base class. + if (not base_types and defn.fullname != 'builtins.object'): + base_types.append(self.object_type()) + + info.bases = base_types + + # Calculate the MRO. It might be incomplete at this point if + # the bases of defn include classes imported from other + # modules in an import loop. We'll recompute it in ThirdPass. + if not self.verify_base_classes(defn): + info.mro = [] + return + calculate_class_mro(defn, self.fail) + # If there are cyclic imports, we may be missing 'object' in + # the MRO. Fix MRO if needed. + if info.mro and info.mro[-1].fullname() != 'builtins.object': + info.mro.append(self.object_type().type) + + def expr_to_analyzed_type(self, expr: Node) -> Type: + if isinstance(expr, CallExpr): + expr.accept(self) + info = self.check_namedtuple(expr) + if info is None: + # Some form of namedtuple is the only valid type that looks like a call + # expression. This isn't a valid type. + raise TypeTranslationError() + fallback = Instance(info, []) + return TupleType(info.tuple_type.items, fallback=fallback) + typ = expr_to_unanalyzed_type(expr) + return self.anal_type(typ) + + def verify_base_classes(self, defn: ClassDef) -> bool: + info = defn.info + for base in info.bases: + baseinfo = base.type + if self.is_base_class(info, baseinfo): + self.fail('Cycle in inheritance hierarchy', defn) + # Clear bases to forcefully get rid of the cycle. + info.bases = [] + if baseinfo.fullname() == 'builtins.bool': + self.fail("'%s' is not a valid base class" % + baseinfo.name(), defn) + return False + dup = find_duplicate(info.direct_base_classes()) + if dup: + self.fail('Duplicate base class "%s"' % dup.name(), defn) + return False + return True + + def is_base_class(self, t: TypeInfo, s: TypeInfo) -> bool: + """Determine if t is a base class of s (but do not use mro).""" + # Search the base class graph for t, starting from s. + worklist = [s] + visited = {s} + while worklist: + nxt = worklist.pop() + if nxt == t: + return True + for base in nxt.bases: + if base.type not in visited: + worklist.append(base.type) + visited.add(base.type) + return False + + def analyze_metaclass(self, defn: ClassDef) -> None: + if defn.metaclass: + sym = self.lookup_qualified(defn.metaclass, defn) + if sym is not None and not isinstance(sym.node, TypeInfo): + self.fail("Invalid metaclass '%s'" % defn.metaclass, defn) + + def object_type(self) -> Instance: + return self.named_type('__builtins__.object') + + def class_type(self, info: TypeInfo) -> Type: + # Construct a function type whose fallback is cls. + from mypy import checkmember # To avoid import cycle. + leading_type = checkmember.type_object_type(info, self.builtin_type) + if isinstance(leading_type, Overloaded): + # Overloaded __init__ is too complex to handle. Plus it's stubs only. + return AnyType() + else: + return leading_type + + def named_type(self, qualified_name: str, args: List[Type] = None) -> Instance: + sym = self.lookup_qualified(qualified_name, None) + return Instance(cast(TypeInfo, sym.node), args or []) + + def named_type_or_none(self, qualified_name: str) -> Instance: + sym = self.lookup_fully_qualified_or_none(qualified_name) + if not sym: + return None + return Instance(cast(TypeInfo, sym.node), []) + + def is_instance_type(self, t: Type) -> bool: + return isinstance(t, Instance) + + def bind_class_type_variables_in_symbol_table( + self, info: TypeInfo) -> List[SymbolTableNode]: + nodes = [] # type: List[SymbolTableNode] + for var, binder in zip(info.type_vars, info.defn.type_vars): + node = self.bind_type_var(var, binder, info) + nodes.append(node) + return nodes + + def visit_import(self, i: Import) -> None: + for id, as_id in i.ids: + if as_id is not None: + self.add_module_symbol(id, as_id, module_public=True, context=i) + else: + # Modules imported in a stub file without using 'as x' won't get exported when + # doing 'from m import *'. + module_public = not self.is_stub_file + base = id.split('.')[0] + self.add_module_symbol(base, base, module_public=module_public, + context=i) + + def add_module_symbol(self, id: str, as_id: str, module_public: bool, + context: Context) -> None: + if id in self.modules: + m = self.modules[id] + self.add_symbol(as_id, SymbolTableNode(MODULE_REF, m, self.cur_mod_id, + module_public=module_public), context) + else: + self.add_unknown_symbol(as_id, context) + + def visit_import_from(self, imp: ImportFrom) -> None: + import_id = self.correct_relative_import(imp) + if import_id in self.modules: + module = self.modules[import_id] + for id, as_id in imp.names: + node = module.names.get(id) + if node and node.kind != UNBOUND_IMPORTED: + node = self.normalize_type_alias(node, imp) + if not node: + return + imported_id = as_id or id + existing_symbol = self.globals.get(imported_id) + if existing_symbol: + # Import can redefine a variable. They get special treatment. + if self.process_import_over_existing_name( + imported_id, existing_symbol, node, imp): + continue + # 'from m import x as x' exports x in a stub file. + module_public = not self.is_stub_file or as_id is not None + symbol = SymbolTableNode(node.kind, node.node, + self.cur_mod_id, + node.type_override, + module_public=module_public) + self.add_symbol(imported_id, symbol, imp) + else: + message = "Module has no attribute '{}'".format(id) + extra = self.undefined_name_extra_info('{}.{}'.format(import_id, id)) + if extra: + message += " {}".format(extra) + self.fail(message, imp) + else: + # Missing module. + for id, as_id in imp.names: + self.add_unknown_symbol(as_id or id, imp) + + def process_import_over_existing_name(self, + imported_id: str, existing_symbol: SymbolTableNode, + module_symbol: SymbolTableNode, + import_node: ImportBase) -> bool: + if (existing_symbol.kind in (LDEF, GDEF, MDEF) and + isinstance(existing_symbol.node, (Var, FuncDef))): + # This is a valid import over an existing definition in the file. Construct a dummy + # assignment that we'll use to type check the import. + lvalue = NameExpr(imported_id) + lvalue.kind = existing_symbol.kind + lvalue.node = existing_symbol.node + rvalue = NameExpr(imported_id) + rvalue.kind = module_symbol.kind + rvalue.node = module_symbol.node + assignment = AssignmentStmt([lvalue], rvalue) + for node in assignment, lvalue, rvalue: + node.set_line(import_node) + import_node.assignments.append(assignment) + return True + return False + + def normalize_type_alias(self, node: SymbolTableNode, + ctx: Context) -> SymbolTableNode: + if node.fullname in type_aliases: + # Node refers to an aliased type such as typing.List; normalize. + node = self.lookup_qualified(type_aliases[node.fullname], ctx) + if node.fullname == 'typing.DefaultDict': + self.add_module_symbol('collections', '__mypy_collections__', False, ctx) + node = self.lookup_qualified('__mypy_collections__.defaultdict', ctx) + return node + + def correct_relative_import(self, node: Union[ImportFrom, ImportAll]) -> str: + if node.relative == 0: + return node.id + + parts = self.cur_mod_id.split(".") + cur_mod_id = self.cur_mod_id + + rel = node.relative + if self.cur_mod_node.is_package_init_file(): + rel -= 1 + if len(parts) < rel: + self.fail("Relative import climbs too many namespaces", node) + if rel != 0: + cur_mod_id = ".".join(parts[:-rel]) + + return cur_mod_id + (("." + node.id) if node.id else "") + + def visit_import_all(self, i: ImportAll) -> None: + i_id = self.correct_relative_import(i) + if i_id in self.modules: + m = self.modules[i_id] + for name, node in m.names.items(): + node = self.normalize_type_alias(node, i) + if not name.startswith('_') and node.module_public: + existing_symbol = self.globals.get(name) + if existing_symbol: + # Import can redefine a variable. They get special treatment. + if self.process_import_over_existing_name( + name, existing_symbol, node, i): + continue + self.add_symbol(name, SymbolTableNode(node.kind, node.node, + self.cur_mod_id), i) + else: + # Don't add any dummy symbols for 'from x import *' if 'x' is unknown. + pass + + def add_unknown_symbol(self, name: str, context: Context) -> None: + var = Var(name) + var._fullname = self.qualified_name(name) + var.is_ready = True + var.type = AnyType() + self.add_symbol(name, SymbolTableNode(GDEF, var, self.cur_mod_id), context) + + # + # Statements + # + + def visit_block(self, b: Block) -> None: + if b.is_unreachable: + return + self.block_depth[-1] += 1 + for s in b.body: + self.accept(s) + self.block_depth[-1] -= 1 + + def visit_block_maybe(self, b: Block) -> None: + if b: + self.visit_block(b) + + def anal_type(self, t: Type, allow_tuple_literal: bool = False) -> Type: + if t: + if allow_tuple_literal: + # Types such as (t1, t2, ...) only allowed in assignment statements. They'll + # generate errors elsewhere, and Tuple[t1, t2, ...] must be used instead. + if isinstance(t, TupleType): + # Unlike TypeAnalyser, also allow implicit tuple types (without Tuple[...]). + star_count = sum(1 for item in t.items if isinstance(item, StarType)) + if star_count > 1: + self.fail('At most one star type allowed in a tuple', t) + return TupleType([AnyType() for _ in t.items], + self.builtin_type('builtins.tuple'), t.line) + items = [self.anal_type(item, True) + for item in t.items] + return TupleType(items, self.builtin_type('builtins.tuple'), t.line) + a = TypeAnalyser(self.lookup_qualified, + self.lookup_fully_qualified, + self.fail) + return t.accept(a) + else: + return None + + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + for lval in s.lvalues: + self.analyze_lvalue(lval, explicit_type=s.type is not None) + s.rvalue.accept(self) + if s.type: + allow_tuple_literal = isinstance(s.lvalues[-1], (TupleExpr, ListExpr)) + s.type = self.anal_type(s.type, allow_tuple_literal) + else: + # For simple assignments, allow binding type aliases. + # Also set the type if the rvalue is a simple literal. + if (s.type is None and len(s.lvalues) == 1 and + isinstance(s.lvalues[0], NameExpr)): + if s.lvalues[0].is_def: + s.type = self.analyze_simple_literal_type(s.rvalue) + res = analyze_type_alias(s.rvalue, + self.lookup_qualified, + self.lookup_fully_qualified, + self.fail) + if res and (not isinstance(res, Instance) or cast(Instance, res).args): + # TODO: What if this gets reassigned? + name = s.lvalues[0] + node = self.lookup(name.name, name) + node.kind = TYPE_ALIAS + node.type_override = res + if isinstance(s.rvalue, IndexExpr): + s.rvalue.analyzed = TypeAliasExpr(res) + if s.type: + # Store type into nodes. + for lvalue in s.lvalues: + self.store_declared_types(lvalue, s.type) + self.check_and_set_up_type_alias(s) + self.process_typevar_declaration(s) + self.process_namedtuple_definition(s) + + if (len(s.lvalues) == 1 and isinstance(s.lvalues[0], NameExpr) and + s.lvalues[0].name == '__all__' and s.lvalues[0].kind == GDEF and + isinstance(s.rvalue, (ListExpr, TupleExpr))): + self.add_exports(*s.rvalue.items) + + def analyze_simple_literal_type(self, rvalue: Node) -> Optional[Type]: + """Return builtins.int if rvalue is an int literal, etc.""" + if self.weak_opts or self.options.semantic_analysis_only or self.function_stack: + # Skip this if any weak options are set. + # Also skip if we're only doing the semantic analysis pass. + # This is mostly to avoid breaking unit tests. + # Also skip inside a function; this is to avoid confusing + # the code that handles dead code due to isinstance() + # inside type variables with value restrictions (like + # AnyStr). + return None + if isinstance(rvalue, IntExpr): + return self.named_type_or_none('builtins.int') + if isinstance(rvalue, FloatExpr): + return self.named_type_or_none('builtins.float') + if isinstance(rvalue, StrExpr): + return self.named_type_or_none('builtins.str') + if isinstance(rvalue, BytesExpr): + return self.named_type_or_none('builtins.bytes') + if isinstance(rvalue, UnicodeExpr): + return self.named_type_or_none('builtins.unicode') + return None + + def check_and_set_up_type_alias(self, s: AssignmentStmt) -> None: + """Check if assignment creates a type alias and set it up as needed.""" + # For now, type aliases only work at the top level of a module. + if (len(s.lvalues) == 1 and not self.is_func_scope() and not self.type + and not s.type): + lvalue = s.lvalues[0] + if isinstance(lvalue, NameExpr): + if not lvalue.is_def: + # Only a definition can create a type alias, not regular assignment. + return + rvalue = s.rvalue + if isinstance(rvalue, RefExpr): + node = rvalue.node + if isinstance(node, TypeInfo): + # TODO: We should record the fact that this is a variable + # that refers to a type, rather than making this + # just an alias for the type. + self.globals[lvalue.name].node = node + + def analyze_lvalue(self, lval: Node, nested: bool = False, + add_global: bool = False, + explicit_type: bool = False) -> None: + """Analyze an lvalue or assignment target. + + Only if add_global is True, add name to globals table. If nested + is true, the lvalue is within a tuple or list lvalue expression. + """ + + if isinstance(lval, NameExpr): + # Top-level definitions within some statements (at least while) are + # not handled in the first pass, so they have to be added now. + nested_global = (not self.is_func_scope() and + self.block_depth[-1] > 0 and + not self.type) + if (add_global or nested_global) and lval.name not in self.globals: + # Define new global name. + v = Var(lval.name) + v._fullname = self.qualified_name(lval.name) + v.is_ready = False # Type not inferred yet + lval.node = v + lval.is_def = True + lval.kind = GDEF + lval.fullname = v._fullname + self.globals[lval.name] = SymbolTableNode(GDEF, v, + self.cur_mod_id) + elif isinstance(lval.node, Var) and lval.is_def: + # Since the is_def flag is set, this must have been analyzed + # already in the first pass and added to the symbol table. + assert lval.node.name() in self.globals + elif (self.is_func_scope() and lval.name not in self.locals[-1] and + lval.name not in self.global_decls[-1] and + lval.name not in self.nonlocal_decls[-1]): + # Define new local name. + v = Var(lval.name) + lval.node = v + lval.is_def = True + lval.kind = LDEF + lval.fullname = lval.name + self.add_local(v, lval) + elif not self.is_func_scope() and (self.type and + lval.name not in self.type.names): + # Define a new attribute within class body. + v = Var(lval.name) + v.info = self.type + v.is_initialized_in_class = True + v.set_line(lval) + lval.node = v + lval.is_def = True + lval.kind = MDEF + lval.fullname = lval.name + self.type.names[lval.name] = SymbolTableNode(MDEF, v) + else: + # Bind to an existing name. + if explicit_type: + self.name_already_defined(lval.name, lval) + lval.accept(self) + self.check_lvalue_validity(lval.node, lval) + elif isinstance(lval, MemberExpr): + if not add_global: + self.analyze_member_lvalue(lval) + if explicit_type and not self.is_self_member_ref(lval): + self.fail('Type cannot be declared in assignment to non-self ' + 'attribute', lval) + elif isinstance(lval, IndexExpr): + if explicit_type: + self.fail('Unexpected type declaration', lval) + if not add_global: + lval.accept(self) + elif (isinstance(lval, TupleExpr) or + isinstance(lval, ListExpr)): + items = cast(Any, lval).items + if len(items) == 0 and isinstance(lval, TupleExpr): + self.fail("Can't assign to ()", lval) + self.analyze_tuple_or_list_lvalue(cast(Union[ListExpr, TupleExpr], lval), + add_global, explicit_type) + elif isinstance(lval, StarExpr): + if nested: + self.analyze_lvalue(lval.expr, nested, add_global, explicit_type) + else: + self.fail('Starred assignment target must be in a list or tuple', lval) + else: + self.fail('Invalid assignment target', lval) + + def analyze_tuple_or_list_lvalue(self, lval: Union[ListExpr, TupleExpr], + add_global: bool = False, + explicit_type: bool = False) -> None: + """Analyze an lvalue or assignment target that is a list or tuple.""" + items = lval.items + star_exprs = [cast(StarExpr, item) + for item in items + if isinstance(item, StarExpr)] + + if len(star_exprs) > 1: + self.fail('Two starred expressions in assignment', lval) + else: + if len(star_exprs) == 1: + star_exprs[0].valid = True + for i in items: + self.analyze_lvalue(i, nested=True, add_global=add_global, + explicit_type = explicit_type) + + def analyze_member_lvalue(self, lval: MemberExpr) -> None: + lval.accept(self) + if (self.is_self_member_ref(lval) and + self.type.get(lval.name) is None): + # Implicit attribute definition in __init__. + lval.is_def = True + v = Var(lval.name) + v.info = self.type + v.is_ready = False + lval.def_var = v + lval.node = v + self.type.names[lval.name] = SymbolTableNode(MDEF, v) + self.check_lvalue_validity(lval.node, lval) + + def is_self_member_ref(self, memberexpr: MemberExpr) -> bool: + """Does memberexpr to refer to an attribute of self?""" + if not isinstance(memberexpr.expr, NameExpr): + return False + node = memberexpr.expr.node + return isinstance(node, Var) and node.is_self + + def check_lvalue_validity(self, node: Node, ctx: Context) -> None: + if isinstance(node, (TypeInfo, TypeVarExpr)): + self.fail('Invalid assignment target', ctx) + + def store_declared_types(self, lvalue: Node, typ: Type) -> None: + if isinstance(typ, StarType) and not isinstance(lvalue, StarExpr): + self.fail('Star type only allowed for starred expressions', lvalue) + if isinstance(lvalue, RefExpr): + lvalue.is_def = False + if isinstance(lvalue.node, Var): + var = lvalue.node + var.type = typ + var.is_ready = True + # If node is not a variable, we'll catch it elsewhere. + elif isinstance(lvalue, TupleExpr): + if isinstance(typ, TupleType): + if len(lvalue.items) != len(typ.items): + self.fail('Incompatible number of tuple items', lvalue) + return + for item, itemtype in zip(lvalue.items, typ.items): + self.store_declared_types(item, itemtype) + else: + self.fail('Tuple type expected for multiple variables', + lvalue) + elif isinstance(lvalue, StarExpr): + if isinstance(typ, StarType): + self.store_declared_types(lvalue.expr, typ.type) + else: + self.fail('Star type expected for starred expression', lvalue) + else: + # This has been flagged elsewhere as an error, so just ignore here. + pass + + def process_typevar_declaration(self, s: AssignmentStmt) -> None: + """Check if s declares a TypeVar; it yes, store it in symbol table.""" + call = self.get_typevar_declaration(s) + if not call: + return + + lvalue = cast(NameExpr, s.lvalues[0]) + name = lvalue.name + if not lvalue.is_def: + if s.type: + self.fail("Cannot declare the type of a type variable", s) + else: + self.fail("Cannot redefine '%s' as a type variable" % name, s) + return + + if not self.check_typevar_name(call, name, s): + return + + # Constraining types + n_values = call.arg_kinds[1:].count(ARG_POS) + values = self.analyze_types(call.args[1:1 + n_values]) + + res = self.process_typevar_parameters(call.args[1 + n_values:], + call.arg_names[1 + n_values:], + call.arg_kinds[1 + n_values:], + bool(values), + s) + if res is None: + return + variance, upper_bound = res + + # Yes, it's a valid type variable definition! Add it to the symbol table. + node = self.lookup(name, s) + node.kind = UNBOUND_TVAR + TypeVar = TypeVarExpr(name, node.fullname, values, upper_bound, variance) + TypeVar.line = call.line + call.analyzed = TypeVar + node.node = TypeVar + + def check_typevar_name(self, call: CallExpr, name: str, context: Context) -> bool: + if len(call.args) < 1: + self.fail("Too few arguments for TypeVar()", context) + return False + if not isinstance(call.args[0], (StrExpr, BytesExpr)) or not call.arg_kinds[0] == ARG_POS: + self.fail("TypeVar() expects a string literal as first argument", context) + return False + if cast(StrExpr, call.args[0]).value != name: + self.fail("Unexpected TypeVar() argument value", context) + return False + return True + + def get_typevar_declaration(self, s: AssignmentStmt) -> Optional[CallExpr]: + """Returns the TypeVar() call expression if `s` is a type var declaration + or None otherwise. + """ + if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): + return None + if not isinstance(s.rvalue, CallExpr): + return None + call = s.rvalue + if not isinstance(call.callee, RefExpr): + return None + callee = call.callee + if callee.fullname != 'typing.TypeVar': + return None + return call + + def process_typevar_parameters(self, args: List[Node], + names: List[Optional[str]], + kinds: List[int], + has_values: bool, + context: Context) -> Optional[Tuple[int, Type]]: + covariant = False + contravariant = False + upper_bound = self.object_type() # type: Type + for param_value, param_name, param_kind in zip(args, names, kinds): + if not param_kind == ARG_NAMED: + self.fail("Unexpected argument to TypeVar()", context) + return None + if param_name == 'covariant': + if isinstance(param_value, NameExpr): + if param_value.name == 'True': + covariant = True + else: + self.fail("TypeVar 'covariant' may only be 'True'", context) + return None + else: + self.fail("TypeVar 'covariant' may only be 'True'", context) + return None + elif param_name == 'contravariant': + if isinstance(param_value, NameExpr): + if param_value.name == 'True': + contravariant = True + else: + self.fail("TypeVar 'contravariant' may only be 'True'", context) + return None + else: + self.fail("TypeVar 'contravariant' may only be 'True'", context) + return None + elif param_name == 'bound': + if has_values: + self.fail("TypeVar cannot have both values and an upper bound", context) + return None + try: + upper_bound = self.expr_to_analyzed_type(param_value) + except TypeTranslationError: + self.fail("TypeVar 'bound' must be a type", param_value) + return None + elif param_name == 'values': + # Probably using obsolete syntax with values=(...). Explain the current syntax. + self.fail("TypeVar 'values' argument not supported", context) + self.fail("Use TypeVar('T', t, ...) instead of TypeVar('T', values=(t, ...))", + context) + return None + else: + self.fail("Unexpected argument to TypeVar(): {}".format(param_name), context) + return None + + if covariant and contravariant: + self.fail("TypeVar cannot be both covariant and contravariant", context) + return None + elif covariant: + variance = COVARIANT + elif contravariant: + variance = CONTRAVARIANT + else: + variance = INVARIANT + return (variance, upper_bound) + + def process_namedtuple_definition(self, s: AssignmentStmt) -> None: + """Check if s defines a namedtuple; if yes, store the definition in symbol table.""" + if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): + return + lvalue = cast(NameExpr, s.lvalues[0]) + name = lvalue.name + named_tuple = self.check_namedtuple(s.rvalue, name) + if named_tuple is None: + return + # Yes, it's a valid namedtuple definition. Add it to the symbol table. + node = self.lookup(name, s) + node.kind = GDEF # TODO locally defined namedtuple + # TODO call.analyzed + node.node = named_tuple + + def check_namedtuple(self, node: Node, var_name: str = None) -> TypeInfo: + """Check if a call defines a namedtuple. + + The optional var_name argument is the name of the variable to + which this is assigned, if any. + + If it does, return the corresponding TypeInfo. Return None otherwise. + + If the definition is invalid but looks like a namedtuple, + report errors but return (some) TypeInfo. + """ + if not isinstance(node, CallExpr): + return None + call = node + if not isinstance(call.callee, RefExpr): + return None + callee = call.callee + fullname = callee.fullname + if fullname not in ('collections.namedtuple', 'typing.NamedTuple'): + return None + items, types, ok = self.parse_namedtuple_args(call, fullname) + if not ok: + # Error. Construct dummy return value. + return self.build_namedtuple_typeinfo('namedtuple', [], []) + else: + # Give it a unique name derived from the line number. + name = cast(StrExpr, call.args[0]).value + if name != var_name: + name += '@' + str(call.line) + info = self.build_namedtuple_typeinfo(name, items, types) + # Store it as a global just in case it would remain anonymous. + self.globals[name] = SymbolTableNode(GDEF, info, self.cur_mod_id) + call.analyzed = NamedTupleExpr(info).set_line(call.line) + return info + + def parse_namedtuple_args(self, call: CallExpr, + fullname: str) -> Tuple[List[str], List[Type], bool]: + # TODO Share code with check_argument_count in checkexpr.py? + args = call.args + if len(args) < 2: + return self.fail_namedtuple_arg("Too few arguments for namedtuple()", call) + if len(args) > 2: + return self.fail_namedtuple_arg("Too many arguments for namedtuple()", call) + if call.arg_kinds != [ARG_POS, ARG_POS]: + return self.fail_namedtuple_arg("Unexpected arguments to namedtuple()", call) + if not isinstance(args[0], (StrExpr, BytesExpr)): + return self.fail_namedtuple_arg( + "namedtuple() expects a string literal as the first argument", call) + types = [] # type: List[Type] + ok = True + if not isinstance(args[1], ListExpr): + if fullname == 'collections.namedtuple' and isinstance(args[1], (StrExpr, BytesExpr)): + str_expr = cast(StrExpr, args[1]) + items = str_expr.value.split() + else: + return self.fail_namedtuple_arg( + "List literal expected as the second argument to namedtuple()", call) + else: + listexpr = args[1] + if fullname == 'collections.namedtuple': + # The fields argument contains just names, with implicit Any types. + if any(not isinstance(item, (StrExpr, BytesExpr)) for item in listexpr.items): + return self.fail_namedtuple_arg("String literal expected as namedtuple() item", + call) + items = [cast(StrExpr, item).value for item in listexpr.items] + else: + # The fields argument contains (name, type) tuples. + items, types, ok = self.parse_namedtuple_fields_with_types(listexpr.items, call) + if not types: + types = [AnyType() for _ in items] + return items, types, ok + + def parse_namedtuple_fields_with_types(self, nodes: List[Node], + context: Context) -> Tuple[List[str], List[Type], bool]: + items = [] # type: List[str] + types = [] # type: List[Type] + for item in nodes: + if isinstance(item, TupleExpr): + if len(item.items) != 2: + return self.fail_namedtuple_arg("Invalid NamedTuple field definition", + item) + name, type_node = item.items + if isinstance(name, (StrExpr, BytesExpr)): + items.append(name.value) + else: + return self.fail_namedtuple_arg("Invalid NamedTuple() field name", item) + try: + type = expr_to_unanalyzed_type(type_node) + except TypeTranslationError: + return self.fail_namedtuple_arg('Invalid field type', type_node) + types.append(self.anal_type(type)) + else: + return self.fail_namedtuple_arg("Tuple expected as NamedTuple() field", item) + return items, types, True + + def fail_namedtuple_arg(self, message: str, + context: Context) -> Tuple[List[str], List[Type], bool]: + self.fail(message, context) + return [], [], False + + def build_namedtuple_typeinfo(self, name: str, items: List[str], + types: List[Type]) -> TypeInfo: + symbols = SymbolTable() + tup = NamedTupleType(name, items, types, self.named_type('__builtins__.tuple', types)) + class_def = ClassDef(name, Block([])) + class_def.fullname = self.qualified_name(name) + info = NamedTupleTypeInfo(tup, symbols, class_def) + info.attrs = items + vars = [Var(item, typ) for item, typ in zip(items, types)] + # Add named tuple items as attributes. + # TODO: Make them read-only. + for var in vars: + var.info = info + symbols[var.name()] = SymbolTableNode(MDEF, var) + this_type = self_type(info) + # Add __init__, _replace, _asdict methods and _fields static field + self.add_namedtuple_method('_replace', symbols, info, this_type, this_type, + self.make_factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) + self.add_namedtuple_method('__init__', symbols, info, this_type, NoneTyp(), + self.make_factory_args(vars, ARG_POS), info.name()) + # TODO: refine to OrderedDict[Union[types]] + self.add_namedtuple_method('_asdict', symbols, info, this_type, AnyType(), []) + return info + + def make_factory_args(self, vars: List[Var], kind: int, + initializer: Expression = None) -> List[Argument]: + return [Argument(var, var.type, initializer, kind) for var in vars] + + def add_namedtuple_method(self, funcname: str, symbols: SymbolTable, info: TypeInfo, + this_type: Type, ret: Type, args: List[Argument], name=None) -> None: + args = [Argument(Var('self'), this_type, None, ARG_POS)] + args + types = [arg.type_annotation for arg in args] + items = [arg.variable.name() for arg in args] + arg_kinds = [arg.kind for arg in args] + # TODO: Make sure that the self argument name is not visible? + signature = CallableType(types, arg_kinds, items, ret, + self.named_type('__builtins__.function'), + name=name or info.name() + '.' + funcname) + func = FuncDef(funcname, + args, + Block([]), + typ=signature) + func.info = info + symbols[funcname] = SymbolTableNode(MDEF, func) + + def analyze_types(self, items: List[Node]) -> List[Type]: + result = [] # type: List[Type] + for node in items: + try: + result.append(self.anal_type(expr_to_unanalyzed_type(node))) + except TypeTranslationError: + self.fail('Type expected', node) + result.append(AnyType()) + return result + + def visit_decorator(self, dec: Decorator) -> None: + for d in dec.decorators: + d.accept(self) + removed = [] # type: List[int] + no_type_check = False + for i, d in enumerate(dec.decorators): + # A bunch of decorators are special cased here. + if refers_to_fullname(d, 'abc.abstractmethod'): + removed.append(i) + dec.func.is_abstract = True + self.check_decorated_function_is_method('abstractmethod', dec) + elif refers_to_fullname(d, 'asyncio.tasks.coroutine'): + removed.append(i) + elif refers_to_fullname(d, 'builtins.staticmethod'): + removed.append(i) + dec.func.is_static = True + dec.var.is_staticmethod = True + self.check_decorated_function_is_method('staticmethod', dec) + elif refers_to_fullname(d, 'builtins.classmethod'): + removed.append(i) + dec.func.is_class = True + dec.var.is_classmethod = True + self.check_decorated_function_is_method('classmethod', dec) + elif (refers_to_fullname(d, 'builtins.property') or + refers_to_fullname(d, 'abc.abstractproperty')): + removed.append(i) + dec.func.is_property = True + dec.var.is_property = True + if refers_to_fullname(d, 'abc.abstractproperty'): + dec.func.is_abstract = True + self.check_decorated_function_is_method('property', dec) + if len(dec.func.arguments) > 1: + self.fail('Too many arguments', dec.func) + elif refers_to_fullname(d, 'typing.no_type_check'): + dec.var.type = AnyType() + no_type_check = True + for i in reversed(removed): + del dec.decorators[i] + if not dec.is_overload or dec.var.is_property: + if self.is_func_scope(): + self.add_symbol(dec.var.name(), SymbolTableNode(LDEF, dec), + dec) + elif self.type: + dec.var.info = self.type + dec.var.is_initialized_in_class = True + self.add_symbol(dec.var.name(), SymbolTableNode(MDEF, dec), + dec) + if not no_type_check: + dec.func.accept(self) + if dec.decorators and dec.var.is_property: + self.fail('Decorated property not supported', dec) + + def check_decorated_function_is_method(self, decorator: str, + context: Context) -> None: + if not self.type or self.is_func_scope(): + self.fail("'%s' used with a non-method" % decorator, context) + + def visit_expression_stmt(self, s: ExpressionStmt) -> None: + s.expr.accept(self) + + def visit_return_stmt(self, s: ReturnStmt) -> None: + if not self.is_func_scope(): + self.fail("'return' outside function", s) + if s.expr: + s.expr.accept(self) + + def visit_raise_stmt(self, s: RaiseStmt) -> None: + if s.expr: + s.expr.accept(self) + if s.from_expr: + s.from_expr.accept(self) + + def visit_assert_stmt(self, s: AssertStmt) -> None: + if s.expr: + s.expr.accept(self) + + def visit_operator_assignment_stmt(self, + s: OperatorAssignmentStmt) -> None: + s.lvalue.accept(self) + s.rvalue.accept(self) + if (isinstance(s.lvalue, NameExpr) and s.lvalue.name == '__all__' and + s.lvalue.kind == GDEF and isinstance(s.rvalue, (ListExpr, TupleExpr))): + self.add_exports(*s.rvalue.items) + + def visit_while_stmt(self, s: WhileStmt) -> None: + s.expr.accept(self) + self.loop_depth += 1 + s.body.accept(self) + self.loop_depth -= 1 + self.visit_block_maybe(s.else_body) + + def visit_for_stmt(self, s: ForStmt) -> None: + s.expr.accept(self) + + # Bind index variables and check if they define new names. + self.analyze_lvalue(s.index) + + self.loop_depth += 1 + self.visit_block(s.body) + self.loop_depth -= 1 + + self.visit_block_maybe(s.else_body) + + def visit_break_stmt(self, s: BreakStmt) -> None: + if self.loop_depth == 0: + self.fail("'break' outside loop", s, True, blocker=True) + + def visit_continue_stmt(self, s: ContinueStmt) -> None: + if self.loop_depth == 0: + self.fail("'continue' outside loop", s, True, blocker=True) + + def visit_if_stmt(self, s: IfStmt) -> None: + infer_reachability_of_if_statement(s, pyversion=self.options.python_version) + for i in range(len(s.expr)): + s.expr[i].accept(self) + self.visit_block(s.body[i]) + self.visit_block_maybe(s.else_body) + + def visit_try_stmt(self, s: TryStmt) -> None: + self.analyze_try_stmt(s, self) + + def analyze_try_stmt(self, s: TryStmt, visitor: NodeVisitor, + add_global: bool = False) -> None: + s.body.accept(visitor) + for type, var, handler in zip(s.types, s.vars, s.handlers): + if type: + type.accept(visitor) + if var: + self.analyze_lvalue(var, add_global=add_global) + handler.accept(visitor) + if s.else_body: + s.else_body.accept(visitor) + if s.finally_body: + s.finally_body.accept(visitor) + + def visit_with_stmt(self, s: WithStmt) -> None: + for e, n in zip(s.expr, s.target): + e.accept(self) + if n: + self.analyze_lvalue(n) + self.visit_block(s.body) + + def visit_del_stmt(self, s: DelStmt) -> None: + s.expr.accept(self) + if not self.is_valid_del_target(s.expr): + self.fail('Invalid delete target', s) + + def is_valid_del_target(self, s: Node) -> bool: + if isinstance(s, (IndexExpr, NameExpr, MemberExpr)): + return True + elif isinstance(s, TupleExpr): + return all(self.is_valid_del_target(item) for item in s.items) + + def visit_global_decl(self, g: GlobalDecl) -> None: + for name in g.names: + if name in self.nonlocal_decls[-1]: + self.fail("Name '{}' is nonlocal and global".format(name), g) + self.global_decls[-1].add(name) + + def visit_nonlocal_decl(self, d: NonlocalDecl) -> None: + if not self.is_func_scope(): + self.fail("nonlocal declaration not allowed at module level", d) + else: + for name in d.names: + for table in reversed(self.locals[:-1]): + if table is not None and name in table: + break + else: + self.fail("No binding for nonlocal '{}' found".format(name), d) + + if self.locals[-1] is not None and name in self.locals[-1]: + self.fail("Name '{}' is already defined in local " + "scope before nonlocal declaration".format(name), d) + + if name in self.global_decls[-1]: + self.fail("Name '{}' is nonlocal and global".format(name), d) + self.nonlocal_decls[-1].add(name) + + def visit_print_stmt(self, s: PrintStmt) -> None: + for arg in s.args: + arg.accept(self) + if s.target: + s.target.accept(self) + + def visit_exec_stmt(self, s: ExecStmt) -> None: + s.expr.accept(self) + if s.variables1: + s.variables1.accept(self) + if s.variables2: + s.variables2.accept(self) + + # + # Expressions + # + + def visit_name_expr(self, expr: NameExpr) -> None: + n = self.lookup(expr.name, expr) + if n: + if n.kind == BOUND_TVAR: + self.fail("'{}' is a type variable and only valid in type " + "context".format(expr.name), expr) + else: + expr.kind = n.kind + expr.node = n.node + expr.fullname = n.fullname + + def visit_super_expr(self, expr: SuperExpr) -> None: + if not self.type: + self.fail('"super" used outside class', expr) + return + expr.info = self.type + + def visit_tuple_expr(self, expr: TupleExpr) -> None: + for item in expr.items: + item.accept(self) + + def visit_list_expr(self, expr: ListExpr) -> None: + for item in expr.items: + item.accept(self) + + def visit_set_expr(self, expr: SetExpr) -> None: + for item in expr.items: + item.accept(self) + + def visit_dict_expr(self, expr: DictExpr) -> None: + for key, value in expr.items: + key.accept(self) + value.accept(self) + + def visit_star_expr(self, expr: StarExpr) -> None: + if not expr.valid: + self.fail('Can use starred expression only as assignment target', expr) + else: + expr.expr.accept(self) + + def visit_yield_from_expr(self, e: YieldFromExpr) -> None: + if not self.is_func_scope(): # not sure + self.fail("'yield from' outside function", e, True, blocker=True) + else: + self.function_stack[-1].is_generator = True + if e.expr: + e.expr.accept(self) + + def visit_call_expr(self, expr: CallExpr) -> None: + """Analyze a call expression. + + Some call expressions are recognized as special forms, including + cast(...) and Any(...). + """ + expr.callee.accept(self) + if refers_to_fullname(expr.callee, 'typing.cast'): + # Special form cast(...). + if not self.check_fixed_args(expr, 2, 'cast'): + return + # Translate first argument to an unanalyzed type. + try: + target = expr_to_unanalyzed_type(expr.args[0]) + except TypeTranslationError: + self.fail('Cast target is not a type', expr) + return + # Piggyback CastExpr object to the CallExpr object; it takes + # precedence over the CallExpr semantics. + expr.analyzed = CastExpr(expr.args[1], target) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, 'builtins.reveal_type'): + if not self.check_fixed_args(expr, 1, 'reveal_type'): + return + expr.analyzed = RevealTypeExpr(expr.args[0]) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, 'typing.Any'): + # Special form Any(...). + if not self.check_fixed_args(expr, 1, 'Any'): + return + expr.analyzed = CastExpr(expr.args[0], AnyType()) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, 'typing._promote'): + # Special form _promote(...). + if not self.check_fixed_args(expr, 1, '_promote'): + return + # Translate first argument to an unanalyzed type. + try: + target = expr_to_unanalyzed_type(expr.args[0]) + except TypeTranslationError: + self.fail('Argument 1 to _promote is not a type', expr) + return + expr.analyzed = PromoteExpr(target) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, 'builtins.dict'): + expr.analyzed = self.translate_dict_call(expr) + else: + # Normal call expression. + for a in expr.args: + a.accept(self) + + if (isinstance(expr.callee, MemberExpr) and + isinstance(expr.callee.expr, NameExpr) and + expr.callee.expr.name == '__all__' and + expr.callee.expr.kind == GDEF and + expr.callee.name in ('append', 'extend')): + if expr.callee.name == 'append' and expr.args: + self.add_exports(expr.args[0]) + elif (expr.callee.name == 'extend' and expr.args and + isinstance(expr.args[0], (ListExpr, TupleExpr))): + self.add_exports(*expr.args[0].items) + + def translate_dict_call(self, call: CallExpr) -> Optional[DictExpr]: + """Translate 'dict(x=y, ...)' to {'x': y, ...}. + + For other variants of dict(...), return None. + """ + if not call.args: + return None + if not all(kind == ARG_NAMED for kind in call.arg_kinds): + # Must still accept those args. + for a in call.args: + a.accept(self) + return None + expr = DictExpr([(StrExpr(key), value) + for key, value in zip(call.arg_names, call.args)]) + expr.set_line(call) + expr.accept(self) + return expr + + def check_fixed_args(self, expr: CallExpr, numargs: int, + name: str) -> bool: + """Verify that expr has specified number of positional args. + + Return True if the arguments are valid. + """ + s = 's' + if numargs == 1: + s = '' + if len(expr.args) != numargs: + self.fail("'%s' expects %d argument%s" % (name, numargs, s), + expr) + return False + if expr.arg_kinds != [ARG_POS] * numargs: + self.fail("'%s' must be called with %s positional argument%s" % + (name, numargs, s), expr) + return False + return True + + def visit_member_expr(self, expr: MemberExpr) -> None: + base = expr.expr + base.accept(self) + # Bind references to module attributes. + if isinstance(base, RefExpr) and base.kind == MODULE_REF: + file = cast(MypyFile, base.node) + n = file.names.get(expr.name, None) if file is not None else None + if n: + n = self.normalize_type_alias(n, expr) + if not n: + return + expr.kind = n.kind + expr.fullname = n.fullname + expr.node = n.node + else: + # We only catch some errors here; the rest will be + # catched during type checking. + # + # This way we can report a larger number of errors in + # one type checker run. If we reported errors here, + # the build would terminate after semantic analysis + # and we wouldn't be able to report any type errors. + full_name = '%s.%s' % (file.fullname() if file is not None else None, expr.name) + if full_name in obsolete_name_mapping: + self.fail("Module has no attribute %r (it's now called %r)" % ( + expr.name, obsolete_name_mapping[full_name]), expr) + + def visit_op_expr(self, expr: OpExpr) -> None: + expr.left.accept(self) + expr.right.accept(self) + + def visit_comparison_expr(self, expr: ComparisonExpr) -> None: + for operand in expr.operands: + operand.accept(self) + + def visit_unary_expr(self, expr: UnaryExpr) -> None: + expr.expr.accept(self) + + def visit_index_expr(self, expr: IndexExpr) -> None: + expr.base.accept(self) + if refers_to_class_or_function(expr.base): + # Special form -- type application. + # Translate index to an unanalyzed type. + types = [] # type: List[Type] + if isinstance(expr.index, TupleExpr): + items = expr.index.items + else: + items = [expr.index] + for item in items: + try: + typearg = expr_to_unanalyzed_type(item) + except TypeTranslationError: + self.fail('Type expected within [...]', expr) + return + typearg = self.anal_type(typearg) + types.append(typearg) + expr.analyzed = TypeApplication(expr.base, types) + expr.analyzed.line = expr.line + else: + expr.index.accept(self) + + def visit_slice_expr(self, expr: SliceExpr) -> None: + if expr.begin_index: + expr.begin_index.accept(self) + if expr.end_index: + expr.end_index.accept(self) + if expr.stride: + expr.stride.accept(self) + + def visit_cast_expr(self, expr: CastExpr) -> None: + expr.expr.accept(self) + expr.type = self.anal_type(expr.type) + + def visit_reveal_type_expr(self, expr: RevealTypeExpr) -> None: + expr.expr.accept(self) + + def visit_type_application(self, expr: TypeApplication) -> None: + expr.expr.accept(self) + for i in range(len(expr.types)): + expr.types[i] = self.anal_type(expr.types[i]) + + def visit_list_comprehension(self, expr: ListComprehension) -> None: + expr.generator.accept(self) + + def visit_set_comprehension(self, expr: SetComprehension) -> None: + expr.generator.accept(self) + + def visit_dictionary_comprehension(self, expr: DictionaryComprehension) -> None: + self.enter() + self.analyze_comp_for(expr) + expr.key.accept(self) + expr.value.accept(self) + self.leave() + self.analyze_comp_for_2(expr) + + def visit_generator_expr(self, expr: GeneratorExpr) -> None: + self.enter() + self.analyze_comp_for(expr) + expr.left_expr.accept(self) + self.leave() + self.analyze_comp_for_2(expr) + + def analyze_comp_for(self, expr: Union[GeneratorExpr, + DictionaryComprehension]) -> None: + """Analyses the 'comp_for' part of comprehensions (part 1). + + That is the part after 'for' in (x for x in l if p). This analyzes + variables and conditions which are analyzed in a local scope. + """ + for i, (index, sequence, conditions) in enumerate(zip(expr.indices, + expr.sequences, + expr.condlists)): + if i > 0: + sequence.accept(self) + # Bind index variables. + self.analyze_lvalue(index) + for cond in conditions: + cond.accept(self) + + def analyze_comp_for_2(self, expr: Union[GeneratorExpr, + DictionaryComprehension]) -> None: + """Analyses the 'comp_for' part of comprehensions (part 2). + + That is the part after 'for' in (x for x in l if p). This analyzes + the 'l' part which is analyzed in the surrounding scope. + """ + expr.sequences[0].accept(self) + + def visit_func_expr(self, expr: FuncExpr) -> None: + self.analyze_function(expr) + + def visit_conditional_expr(self, expr: ConditionalExpr) -> None: + expr.if_expr.accept(self) + expr.cond.accept(self) + expr.else_expr.accept(self) + + def visit_backquote_expr(self, expr: BackquoteExpr) -> None: + expr.expr.accept(self) + + def visit__promote_expr(self, expr: PromoteExpr) -> None: + expr.type = self.anal_type(expr.type) + + def visit_yield_expr(self, expr: YieldExpr) -> None: + if not self.is_func_scope(): + self.fail("'yield' outside function", expr, True, blocker=True) + else: + self.function_stack[-1].is_generator = True + if expr.expr: + expr.expr.accept(self) + + # + # Helpers + # + + def lookup(self, name: str, ctx: Context) -> SymbolTableNode: + """Look up an unqualified name in all active namespaces.""" + # 1a. Name declared using 'global x' takes precedence + if name in self.global_decls[-1]: + if name in self.globals: + return self.globals[name] + else: + self.name_not_defined(name, ctx) + return None + # 1b. Name declared using 'nonlocal x' takes precedence + if name in self.nonlocal_decls[-1]: + for table in reversed(self.locals[:-1]): + if table is not None and name in table: + return table[name] + else: + self.name_not_defined(name, ctx) + return None + # 2. Class attributes (if within class definition) + if self.is_class_scope() and name in self.type.names: + return self.type[name] + # 3. Local (function) scopes + for table in reversed(self.locals): + if table is not None and name in table: + return table[name] + # 4. Current file global scope + if name in self.globals: + return self.globals[name] + # 5. Builtins + b = self.globals.get('__builtins__', None) + if b: + table = cast(MypyFile, b.node).names + if name in table: + if name[0] == "_" and name[1] != "_": + self.name_not_defined(name, ctx) + return None + node = table[name] + return node + # Give up. + self.name_not_defined(name, ctx) + self.check_for_obsolete_short_name(name, ctx) + return None + + def check_for_obsolete_short_name(self, name: str, ctx: Context) -> None: + matches = [obsolete_name + for obsolete_name in obsolete_name_mapping + if obsolete_name.rsplit('.', 1)[-1] == name] + if len(matches) == 1: + self.note("(Did you mean '{}'?)".format(obsolete_name_mapping[matches[0]]), ctx) + + def lookup_qualified(self, name: str, ctx: Context) -> SymbolTableNode: + if '.' not in name: + return self.lookup(name, ctx) + else: + parts = name.split('.') + n = self.lookup(parts[0], ctx) # type: SymbolTableNode + if n: + for i in range(1, len(parts)): + if isinstance(n.node, TypeInfo): + if n.node.mro is None: + # We haven't yet analyzed the class `n.node`. Fall back to direct + # lookup in the names declared directly under it, without its base + # classes. This can happen when we have a forward reference to a + # nested class, and the reference is bound before the outer class + # has been fully semantically analyzed. + # + # A better approach would be to introduce a new analysis pass or + # to move things around between passes, but this unblocks a common + # use case even though this is a little limited in case there is + # inheritance involved. + result = n.node.names.get(parts[i]) + else: + result = n.node.get(parts[i]) + n = result + elif isinstance(n.node, MypyFile): + n = n.node.names.get(parts[i], None) + # TODO: What if node is Var or FuncDef? + if not n: + self.name_not_defined(name, ctx) + break + if n: + n = self.normalize_type_alias(n, ctx) + return n + + def builtin_type(self, fully_qualified_name: str) -> Instance: + node = self.lookup_fully_qualified(fully_qualified_name) + info = cast(TypeInfo, node.node) + return Instance(info, []) + + def lookup_fully_qualified(self, name: str) -> SymbolTableNode: + """Lookup a fully qualified name. + + Assume that the name is defined. This happens in the global namespace -- the local + module namespace is ignored. + """ + assert '.' in name + parts = name.split('.') + n = self.modules[parts[0]] + for i in range(1, len(parts) - 1): + n = cast(MypyFile, n.names[parts[i]].node) + return n.names[parts[-1]] + + def lookup_fully_qualified_or_none(self, name: str) -> SymbolTableNode: + """Lookup a fully qualified name. + + Assume that the name is defined. This happens in the global namespace -- the local + module namespace is ignored. + """ + assert '.' in name + parts = name.split('.') + n = self.modules[parts[0]] + for i in range(1, len(parts) - 1): + next_sym = n.names.get(parts[i]) + if not next_sym: + return None + n = cast(MypyFile, next_sym.node) + return n.names.get(parts[-1]) + + def qualified_name(self, n: str) -> str: + return self.cur_mod_id + '.' + n + + def enter(self) -> None: + self.locals.append(SymbolTable()) + self.global_decls.append(set()) + self.nonlocal_decls.append(set()) + # -1 since entering block will increment this to 0. + self.block_depth.append(-1) + + def leave(self) -> None: + self.locals.pop() + self.global_decls.pop() + self.nonlocal_decls.pop() + self.block_depth.pop() + + def is_func_scope(self) -> bool: + return self.locals[-1] is not None + + def is_class_scope(self) -> bool: + return self.type is not None and not self.is_func_scope() + + def add_symbol(self, name: str, node: SymbolTableNode, + context: Context) -> None: + if self.is_func_scope(): + if name in self.locals[-1]: + # Flag redefinition unless this is a reimport of a module. + if not (node.kind == MODULE_REF and + self.locals[-1][name].node == node.node): + self.name_already_defined(name, context) + self.locals[-1][name] = node + elif self.type: + self.type.names[name] = node + else: + existing = self.globals.get(name) + if existing and (not isinstance(node.node, MypyFile) or + existing.node != node.node) and existing.kind != UNBOUND_IMPORTED: + # Modules can be imported multiple times to support import + # of multiple submodules of a package (e.g. a.x and a.y). + ok = False + # Only report an error if the symbol collision provides a different type. + if existing.type and node.type and is_same_type(existing.type, node.type): + ok = True + if not ok: + self.name_already_defined(name, context) + self.globals[name] = node + + def add_var(self, v: Var, ctx: Context) -> None: + if self.is_func_scope(): + self.add_local(v, ctx) + else: + self.globals[v.name()] = SymbolTableNode(GDEF, v, self.cur_mod_id) + v._fullname = self.qualified_name(v.name()) + + def add_local(self, node: Union[Var, FuncBase], ctx: Context) -> None: + name = node.name() + if name in self.locals[-1]: + self.name_already_defined(name, ctx) + node._fullname = name + self.locals[-1][name] = SymbolTableNode(LDEF, node) + + def add_exports(self, *exps: Node) -> None: + for exp in exps: + if isinstance(exp, StrExpr): + self.all_exports.add(exp.value) + + def check_no_global(self, n: str, ctx: Context, + is_overloaded_func: bool = False) -> None: + if n in self.globals: + prev_is_overloaded = isinstance(self.globals[n], OverloadedFuncDef) + if is_overloaded_func and prev_is_overloaded: + self.fail("Nonconsecutive overload {} found".format(n), ctx) + elif prev_is_overloaded: + self.fail("Definition of '{}' missing 'overload'".format(n), ctx) + else: + self.name_already_defined(n, ctx) + + def name_not_defined(self, name: str, ctx: Context) -> None: + message = "Name '{}' is not defined".format(name) + extra = self.undefined_name_extra_info(name) + if extra: + message += ' {}'.format(extra) + self.fail(message, ctx) + + def name_already_defined(self, name: str, ctx: Context) -> None: + self.fail("Name '{}' already defined".format(name), ctx) + + def fail(self, msg: str, ctx: Context, serious: bool = False, *, + blocker: bool = False) -> None: + if (not serious and + not self.options.check_untyped_defs and + self.function_stack and + self.function_stack[-1].is_dynamic()): + return + self.errors.report(ctx.get_line(), msg, blocker=blocker) + + def note(self, msg: str, ctx: Context) -> None: + if (not self.options.check_untyped_defs and + self.function_stack and + self.function_stack[-1].is_dynamic()): + return + self.errors.report(ctx.get_line(), msg, severity='note') + + def undefined_name_extra_info(self, fullname: str) -> Optional[str]: + if fullname in obsolete_name_mapping: + return "(it's now called '{}')".format(obsolete_name_mapping[fullname]) + else: + return None + + def accept(self, node: Node) -> None: + try: + node.accept(self) + except Exception as err: + report_internal_error(err, self.errors.file, node.line) + + +class FirstPass(NodeVisitor): + """First phase of semantic analysis. + + See docstring of 'analyze()' below for a description of what this does. + """ + + def __init__(self, sem: SemanticAnalyzer) -> None: + self.sem = sem + self.pyversion = sem.options.python_version + + def analyze(self, file: MypyFile, fnam: str, mod_id: str) -> None: + """Perform the first analysis pass. + + Populate module global table. Resolve the full names of + definitions not nested within functions and construct type + info structures, but do not resolve inter-definition + references such as base classes. + + Also add implicit definitions such as __name__. + + In this phase we don't resolve imports. For 'from ... import', + we generate dummy symbol table nodes for the imported names, + and these will get resolved in later phases of semantic + analysis. + """ + sem = self.sem + sem.cur_mod_id = mod_id + sem.errors.set_file(fnam) + sem.globals = SymbolTable() + sem.global_decls = [set()] + sem.nonlocal_decls = [set()] + sem.block_depth = [0] + + defs = file.defs + + # Add implicit definitions of module '__name__' etc. + for name, t in implicit_module_attrs.items(): + v = Var(name, UnboundType(t)) + v._fullname = self.sem.qualified_name(name) + self.sem.globals[name] = SymbolTableNode(GDEF, v, self.sem.cur_mod_id) + + for d in defs: + d.accept(self) + + # Add implicit definition of literals/keywords to builtins, as we + # cannot define a variable with them explicitly. + if mod_id == 'builtins': + literal_types = [ + ('None', NoneTyp()), + # reveal_type is a mypy-only function that gives an error with the type of its arg + ('reveal_type', AnyType()), + ] # type: List[Tuple[str, Type]] + + # TODO(ddfisher): This guard is only needed because mypy defines + # fake builtins for its tests which often don't define bool. If + # mypy is fast enough that we no longer need those, this + # conditional check should be removed. + if 'bool' in self.sem.globals: + bool_type = self.sem.named_type('bool') + literal_types.extend([ + ('True', bool_type), + ('False', bool_type), + ('__debug__', bool_type), + ]) + + for name, typ in literal_types: + v = Var(name, typ) + v._fullname = self.sem.qualified_name(name) + self.sem.globals[name] = SymbolTableNode(GDEF, v, self.sem.cur_mod_id) + + def visit_block(self, b: Block) -> None: + if b.is_unreachable: + return + self.sem.block_depth[-1] += 1 + for node in b.body: + node.accept(self) + self.sem.block_depth[-1] -= 1 + + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + for lval in s.lvalues: + self.analyze_lvalue(lval, explicit_type=s.type is not None) + + def visit_func_def(self, func: FuncDef) -> None: + sem = self.sem + func.is_conditional = sem.block_depth[-1] > 0 + func._fullname = sem.qualified_name(func.name()) + if func.name() in sem.globals: + # Already defined in this module. + original_sym = sem.globals[func.name()] + if original_sym.kind == UNBOUND_IMPORTED: + # Ah this is an imported name. We can't resolve them now, so we'll postpone + # this until the main phase of semantic analysis. + return + original_def = original_sym.node + if sem.is_conditional_func(original_def, func): + # Conditional function definition -- multiple defs are ok. + func.original_def = cast(FuncDef, original_def) + else: + # Report error. + sem.check_no_global(func.name(), func) + else: + sem.globals[func.name()] = SymbolTableNode(GDEF, func, sem.cur_mod_id) + + def visit_overloaded_func_def(self, func: OverloadedFuncDef) -> None: + self.sem.check_no_global(func.name(), func, True) + func._fullname = self.sem.qualified_name(func.name()) + self.sem.globals[func.name()] = SymbolTableNode(GDEF, func, + self.sem.cur_mod_id) + + def visit_class_def(self, cdef: ClassDef) -> None: + self.sem.check_no_global(cdef.name, cdef) + cdef.fullname = self.sem.qualified_name(cdef.name) + info = TypeInfo(SymbolTable(), cdef) + info.set_line(cdef.line) + cdef.info = info + self.sem.globals[cdef.name] = SymbolTableNode(GDEF, info, + self.sem.cur_mod_id) + self.process_nested_classes(cdef) + + def process_nested_classes(self, outer_def: ClassDef) -> None: + for node in outer_def.defs.body: + if isinstance(node, ClassDef): + node.info = TypeInfo(SymbolTable(), node) + if outer_def.fullname: + node.info._fullname = outer_def.fullname + '.' + node.info.name() + else: + node.info._fullname = node.info.name() + symbol = SymbolTableNode(MDEF, node.info) + outer_def.info.names[node.name] = symbol + self.process_nested_classes(node) + + def visit_import_from(self, node: ImportFrom) -> None: + # We can't bind module names during the first pass, as the target module might be + # unprocessed. However, we add dummy unbound imported names to the symbol table so + # that we at least know that the name refers to a module. + node.is_top_level = True + for name, as_name in node.names: + imported_name = as_name or name + if imported_name not in self.sem.globals: + self.sem.add_symbol(imported_name, SymbolTableNode(UNBOUND_IMPORTED, None), node) + + def visit_import(self, node: Import) -> None: + node.is_top_level = True + # This is similar to visit_import_from -- see the comment there. + for id, as_id in node.ids: + imported_id = as_id or id + if imported_id not in self.sem.globals: + self.sem.add_symbol(imported_id, SymbolTableNode(UNBOUND_IMPORTED, None), node) + else: + # If the previous symbol is a variable, this should take precedence. + self.sem.globals[imported_id] = SymbolTableNode(UNBOUND_IMPORTED, None) + + def visit_import_all(self, node: ImportAll) -> None: + node.is_top_level = True + + def visit_while_stmt(self, s: WhileStmt) -> None: + s.body.accept(self) + if s.else_body: + s.else_body.accept(self) + + def visit_for_stmt(self, s: ForStmt) -> None: + self.analyze_lvalue(s.index) + s.body.accept(self) + if s.else_body: + s.else_body.accept(self) + + def visit_with_stmt(self, s: WithStmt) -> None: + for n in s.target: + if n: + self.analyze_lvalue(n) + s.body.accept(self) + + def visit_decorator(self, d: Decorator) -> None: + d.var._fullname = self.sem.qualified_name(d.var.name()) + self.sem.add_symbol(d.var.name(), SymbolTableNode(GDEF, d.var), d) + + def visit_if_stmt(self, s: IfStmt) -> None: + infer_reachability_of_if_statement(s, pyversion=self.pyversion) + for node in s.body: + node.accept(self) + if s.else_body: + s.else_body.accept(self) + + def visit_try_stmt(self, s: TryStmt) -> None: + self.sem.analyze_try_stmt(s, self, add_global=True) + + def analyze_lvalue(self, lvalue: Node, explicit_type: bool = False) -> None: + self.sem.analyze_lvalue(lvalue, add_global=True, explicit_type=explicit_type) + + +class ThirdPass(TraverserVisitor): + """The third and final pass of semantic analysis. + + Check type argument counts and values of generic types, and perform some + straightforward type inference. + """ + + def __init__(self, modules: Dict[str, MypyFile], errors: Errors) -> None: + self.modules = modules + self.errors = errors + + def visit_file(self, file_node: MypyFile, fnam: str) -> None: + self.errors.set_file(fnam) + self.accept(file_node) + + def accept(self, node: Node) -> None: + try: + node.accept(self) + except Exception as err: + report_internal_error(err, self.errors.file, node.line) + + def visit_block(self, b: Block) -> None: + if b.is_unreachable: + return + super().visit_block(b) + + def visit_func_def(self, fdef: FuncDef) -> None: + self.errors.push_function(fdef.name()) + self.analyze(fdef.type) + super().visit_func_def(fdef) + self.errors.pop_function() + + def visit_class_def(self, tdef: ClassDef) -> None: + for type in tdef.info.bases: + self.analyze(type) + # Recompute MRO now that we have analyzed all modules, to pick + # up superclasses of bases imported from other modules in an + # import loop. (Only do so if we succeeded the first time.) + if tdef.info.mro: + tdef.info.mro = [] # Force recomputation + calculate_class_mro(tdef, self.fail) + super().visit_class_def(tdef) + + def visit_decorator(self, dec: Decorator) -> None: + """Try to infer the type of the decorated function. + + This lets us resolve references to decorated functions during + type checking when there are cyclic imports, as otherwise the + type might not be available when we need it. + + This basically uses a simple special-purpose type inference + engine just for decorators. + """ + super().visit_decorator(dec) + if dec.var.is_property: + # Decorators are expected to have a callable type (it's a little odd). + if dec.func.type is None: + dec.var.type = CallableType( + [AnyType()], + [ARG_POS], + [None], + AnyType(), + self.builtin_type('function'), + name=dec.var.name()) + elif isinstance(dec.func.type, CallableType): + dec.var.type = dec.func.type + return + decorator_preserves_type = True + for expr in dec.decorators: + preserve_type = False + if isinstance(expr, RefExpr) and isinstance(expr.node, FuncDef): + if is_identity_signature(expr.node.type): + preserve_type = True + if not preserve_type: + decorator_preserves_type = False + break + if decorator_preserves_type: + # No non-identity decorators left. We can trivially infer the type + # of the function here. + dec.var.type = function_type(dec.func, self.builtin_type('function')) + if dec.decorators: + if returns_any_if_called(dec.decorators[0]): + # The outermost decorator will return Any so we know the type of the + # decorated function. + dec.var.type = AnyType() + sig = find_fixed_callable_return(dec.decorators[0]) + if sig: + # The outermost decorator always returns the same kind of function, + # so we know that this is the type of the decoratored function. + orig_sig = function_type(dec.func, self.builtin_type('function')) + sig.name = orig_sig.items()[0].name + dec.var.type = sig + + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + self.analyze(s.type) + super().visit_assignment_stmt(s) + + def visit_cast_expr(self, e: CastExpr) -> None: + self.analyze(e.type) + super().visit_cast_expr(e) + + def visit_reveal_type_expr(self, e: RevealTypeExpr) -> None: + super().visit_reveal_type_expr(e) + + def visit_type_application(self, e: TypeApplication) -> None: + for type in e.types: + self.analyze(type) + super().visit_type_application(e) + + # Helpers + + def analyze(self, type: Type) -> None: + if type: + analyzer = TypeAnalyserPass3(self.fail) + type.accept(analyzer) + + def fail(self, msg: str, ctx: Context) -> None: + self.errors.report(ctx.get_line(), msg) + + def builtin_type(self, name: str, args: List[Type] = None) -> Instance: + names = self.modules['builtins'] + sym = names.names[name] + assert isinstance(sym.node, TypeInfo) + return Instance(sym.node, args or []) + + +def self_type(typ: TypeInfo) -> Union[Instance, TupleType]: + """For a non-generic type, return instance type representing the type. + For a generic G type with parameters T1, .., Tn, return G[T1, ..., Tn]. + """ + tv = [] # type: List[Type] + for i in range(len(typ.type_vars)): + tv.append(TypeVarType(typ.defn.type_vars[i])) + inst = Instance(typ, tv) + if typ.tuple_type is None: + return inst + return typ.tuple_type.self_type(inst) + + +def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: + if isinstance(sig, CallableType): + return replace_leading_arg_type(sig, new) + else: + sig = cast(Overloaded, sig) + return Overloaded([cast(CallableType, replace_implicit_first_type(i, new)) + for i in sig.items()]) + + +def set_callable_name(sig: Type, fdef: FuncDef) -> Type: + if isinstance(sig, FunctionLike): + if fdef.info: + return sig.with_name( + '"{}" of "{}"'.format(fdef.name(), fdef.info.name())) + else: + return sig.with_name('"{}"'.format(fdef.name())) + else: + return sig + + +def refers_to_fullname(node: Node, fullname: str) -> bool: + """Is node a name or member expression with the given full name?""" + return isinstance(node, RefExpr) and node.fullname == fullname + + +def refers_to_class_or_function(node: Node) -> bool: + """Does semantically analyzed node refer to a class?""" + return (isinstance(node, RefExpr) and + isinstance(node.node, (TypeInfo, FuncDef, OverloadedFuncDef))) + + +def calculate_class_mro(defn: ClassDef, fail: Callable[[str, Context], None]) -> None: + try: + defn.info.calculate_mro() + except MroError: + fail("Cannot determine consistent method resolution order " + '(MRO) for "%s"' % defn.name, defn) + defn.info.mro = [] + # The property of falling back to Any is inherited. + defn.info.fallback_to_any = any(baseinfo.fallback_to_any for baseinfo in defn.info.mro) + + +def find_duplicate(list: List[T]) -> T: + """If the list has duplicates, return one of the duplicates. + + Otherwise, return None. + """ + for i in range(1, len(list)): + if list[i] in list[:i]: + return list[i] + return None + + +def disable_typevars(nodes: List[SymbolTableNode]) -> None: + for node in nodes: + assert node.kind in (BOUND_TVAR, UNBOUND_TVAR) + node.kind = UNBOUND_TVAR + + +def enable_typevars(nodes: List[SymbolTableNode]) -> None: + for node in nodes: + assert node.kind in (BOUND_TVAR, UNBOUND_TVAR) + node.kind = BOUND_TVAR + + +def remove_imported_names_from_symtable(names: SymbolTable, + module: str) -> None: + """Remove all imported names from the symbol table of a module.""" + removed = [] # type: List[str] + for name, node in names.items(): + if node.node is None: + continue + fullname = node.node.fullname() + prefix = fullname[:fullname.rfind('.')] + if prefix != module: + removed.append(name) + for name in removed: + del names[name] + + +def infer_reachability_of_if_statement(s: IfStmt, + pyversion: Tuple[int, int]) -> None: + for i in range(len(s.expr)): + result = infer_if_condition_value(s.expr[i], pyversion) + if result == ALWAYS_FALSE: + # The condition is always false, so we skip the if/elif body. + mark_block_unreachable(s.body[i]) + elif result == ALWAYS_TRUE: + # This condition is always true, so all of the remaining + # elif/else bodies will never be executed. + for body in s.body[i + 1:]: + mark_block_unreachable(body) + if s.else_body: + mark_block_unreachable(s.else_body) + break + + +def infer_if_condition_value(expr: Node, pyversion: Tuple[int, int]) -> int: + """Infer whether if condition is always true/false. + + Return ALWAYS_TRUE if always true, ALWAYS_FALSE if always false, + and TRUTH_VALUE_UNKNOWN otherwise. + """ + name = '' + negated = False + alias = expr + if isinstance(alias, UnaryExpr): + if alias.op == 'not': + expr = alias.expr + negated = True + if isinstance(expr, NameExpr): + name = expr.name + elif isinstance(expr, MemberExpr): + name = expr.name + result = TRUTH_VALUE_UNKNOWN + if name == 'PY2': + result = ALWAYS_TRUE if pyversion[0] == 2 else ALWAYS_FALSE + elif name == 'PY3': + result = ALWAYS_TRUE if pyversion[0] == 3 else ALWAYS_FALSE + elif name == 'MYPY': + result = ALWAYS_TRUE + if negated: + if result == ALWAYS_TRUE: + result = ALWAYS_FALSE + elif result == ALWAYS_FALSE: + result = ALWAYS_TRUE + return result + + +def mark_block_unreachable(block: Block) -> None: + block.is_unreachable = True + block.accept(MarkImportsUnreachableVisitor()) + + +class MarkImportsUnreachableVisitor(TraverserVisitor): + """Visitor that flags all imports nested within a node as unreachable.""" + + def visit_import(self, node: Import) -> None: + node.is_unreachable = True + + def visit_import_from(self, node: ImportFrom) -> None: + node.is_unreachable = True + + def visit_import_all(self, node: ImportAll) -> None: + node.is_unreachable = True + + +def is_identity_signature(sig: Type) -> bool: + """Is type a callable of form T -> T (where T is a type variable)?""" + if isinstance(sig, CallableType) and sig.arg_kinds == [ARG_POS]: + if isinstance(sig.arg_types[0], TypeVarType) and isinstance(sig.ret_type, TypeVarType): + return sig.arg_types[0].id == sig.ret_type.id + return False + + +def returns_any_if_called(expr: Node) -> bool: + """Return True if we can predict that expr will return Any if called. + + This only uses information available during semantic analysis so this + will sometimes return False because of insufficient information (as + type inference hasn't run yet). + """ + if isinstance(expr, RefExpr): + if isinstance(expr.node, FuncDef): + typ = expr.node.type + if typ is None: + # No signature -> default to Any. + return True + # Explicit Any return? + return isinstance(typ, CallableType) and isinstance(typ.ret_type, AnyType) + elif isinstance(expr.node, Var): + typ = expr.node.type + return typ is None or isinstance(typ, AnyType) + elif isinstance(expr, CallExpr): + return returns_any_if_called(expr.callee) + return False + + +def find_fixed_callable_return(expr: Node) -> Optional[CallableType]: + if isinstance(expr, RefExpr): + if isinstance(expr.node, FuncDef): + typ = expr.node.type + if typ: + if isinstance(typ, CallableType) and has_no_typevars(typ.ret_type): + if isinstance(typ.ret_type, CallableType): + return typ.ret_type + elif isinstance(expr, CallExpr): + t = find_fixed_callable_return(expr.callee) + if t: + if isinstance(t.ret_type, CallableType): + return t.ret_type + return None + + +def has_no_typevars(typ: Type) -> bool: + return is_same_type(typ, erase_typevars(typ)) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 42405695d5aed..6872e297c3a64 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -1,409 +1,409 @@ -from typing import cast, List, Dict, Callable - -from mypy.types import ( - Type, AnyType, UnboundType, TypeVisitor, ErrorType, Void, NoneTyp, - Instance, TypeVarType, CallableType, TupleType, NamedTupleType, UnionType, Overloaded, ErasedType, TypeList, - PartialType, DeletedType, UninhabitedType, TypeType, is_named_instance -) -import mypy.applytype -import mypy.constraints -# Circular import; done in the function instead. -# import mypy.solve -from mypy import messages, sametypes -from mypy.nodes import CONTRAVARIANT, COVARIANT -from mypy.maptype import map_instance_to_supertype - -from mypy import experiments - - -TypeParameterChecker = Callable[[Type, Type, int], bool] - - -def check_type_parameter(lefta: Type, righta: Type, variance: int) -> bool: - if variance == COVARIANT: - return is_subtype(lefta, righta, check_type_parameter) - elif variance == CONTRAVARIANT: - return is_subtype(righta, lefta, check_type_parameter) - else: - return is_equivalent(lefta, righta, check_type_parameter) - - -def is_subtype(left: Type, right: Type, - type_parameter_checker: TypeParameterChecker = check_type_parameter) -> bool: - """Is 'left' subtype of 'right'? - - Also consider Any to be a subtype of any type, and vice versa. This - recursively applies to components of composite types (List[int] is subtype - of List[Any], for example). - - type_parameter_checker is used to check the type parameters (for example, - A with B in is_subtype(C[A], C[B]). The default checks for subtype relation - between the type arguments (e.g., A and B), taking the variance of the - type var into account. - """ - if (isinstance(right, AnyType) or isinstance(right, UnboundType) - or isinstance(right, ErasedType)): - return True - elif isinstance(right, UnionType) and not isinstance(left, UnionType): - return any(is_subtype(left, item, type_parameter_checker) - for item in right.items) - else: - return left.accept(SubtypeVisitor(right, type_parameter_checker)) - - -def is_subtype_ignoring_tvars(left: Type, right: Type) -> bool: - def ignore_tvars(s: Type, t: Type, v: int) -> bool: - return True - return is_subtype(left, right, ignore_tvars) - - -def is_equivalent(a: Type, b: Type, - type_parameter_checker=check_type_parameter) -> bool: - return is_subtype(a, b, type_parameter_checker) and is_subtype(b, a, type_parameter_checker) - - -def satisfies_upper_bound(a: Type, upper_bound: Type) -> bool: - """Is 'a' valid value for a type variable with the given 'upper_bound'? - - Same as is_subtype except that Void is considered to be a subtype of - any upper_bound. This is needed in a case like - - def f(g: Callable[[], T]) -> T: ... - def h() -> None: ... - f(h) - """ - return isinstance(a, Void) or is_subtype(a, upper_bound) - - -class SubtypeVisitor(TypeVisitor[bool]): - - def __init__(self, right: Type, - type_parameter_checker: TypeParameterChecker) -> None: - self.right = right - self.check_type_parameter = type_parameter_checker - - # visit_x(left) means: is left (which is an instance of X) a subtype of - # right? - - def visit_unbound_type(self, left: UnboundType) -> bool: - return True - - def visit_error_type(self, left: ErrorType) -> bool: - return False - - def visit_type_list(self, t: TypeList) -> bool: - assert False, 'Not supported' - - def visit_any(self, left: AnyType) -> bool: - return True - - def visit_void(self, left: Void) -> bool: - return isinstance(self.right, Void) - - def visit_none_type(self, left: NoneTyp) -> bool: - if experiments.STRICT_OPTIONAL: - return (isinstance(self.right, NoneTyp) or - is_named_instance(self.right, 'builtins.object')) - else: - return not isinstance(self.right, Void) - - def visit_uninhabited_type(self, left: UninhabitedType) -> bool: - return not isinstance(self.right, Void) - - def visit_erased_type(self, left: ErasedType) -> bool: - return True - - def visit_deleted_type(self, left: DeletedType) -> bool: - return True - - def visit_instance(self, left: Instance) -> bool: - if left.type.fallback_to_any: - return True - right = self.right - if isinstance(right, Instance): - if left.type._promote and is_subtype(left.type._promote, - self.right, - self.check_type_parameter): - return True - rname = right.type.fullname() - if not left.type.has_base(rname) and rname != 'builtins.object': - return False - - # Map left type to corresponding right instances. - t = map_instance_to_supertype(left, right.type) - - return all(self.check_type_parameter(lefta, righta, tvar.variance) - for lefta, righta, tvar in - zip(t.args, right.args, right.type.defn.type_vars)) - else: - return False - - def visit_type_var(self, left: TypeVarType) -> bool: - right = self.right - if isinstance(right, TypeVarType): - return left.id == right.id - else: - return is_subtype(left.upper_bound, self.right) - - def visit_callable_type(self, left: CallableType) -> bool: - right = self.right - if isinstance(right, CallableType): - return is_callable_subtype(left, right) - elif isinstance(right, Overloaded): - return all(is_subtype(left, item, self.check_type_parameter) - for item in right.items()) - elif isinstance(right, Instance): - return is_subtype(left.fallback, right) - elif isinstance(right, TypeType): - # This is unsound, we don't check the __init__ signature. - return left.is_type_obj() and is_subtype(left.ret_type, right.item) - else: - return False - - def visit_tuple_type(self, left: TupleType) -> bool: - right = self.right - if isinstance(right, Instance): - if is_named_instance(right, 'builtins.object'): - return True - if is_named_instance(right, 'builtins.tuple'): - target_item_type = right.args[0] - return all(is_subtype(item, target_item_type) - for item in left.items) - elif is_named_instance(right, 'typing.Sized'): - return True - elif (is_named_instance(right, 'typing.Iterable') or - is_named_instance(right, 'typing.Container') or - is_named_instance(right, 'typing.Sequence') or - is_named_instance(right, 'typing.Reversible')): - iter_type = right.args[0] - return all(is_subtype(li, iter_type) for li in left.items) - return False - elif isinstance(right, TupleType): - if len(left.items) != len(right.items): - return False - for l, r in zip(left.items, right.items): - if not is_subtype(l, r, self.check_type_parameter): - return False - if not is_subtype(left.fallback, right.fallback, self.check_type_parameter): - return False - return True - else: - return False - - def visit_namedtuple_type(self, left: NamedTupleType) -> bool: - if isinstance(self.right, NamedTupleType): - # left.attrs.issubset(right.attrs) is insufficient - # (a: int, b: str) is not subtype of (b: int, a: str) - if any(l != r for l, r in zip(left.attrs, self.right.attrs)): - return False - return self.visit_tuple_type(left) - - def visit_overloaded(self, left: Overloaded) -> bool: - right = self.right - if isinstance(right, Instance): - return is_subtype(left.fallback, right) - elif isinstance(right, CallableType) or is_named_instance( - right, 'builtins.type'): - for item in left.items(): - if is_subtype(item, right, self.check_type_parameter): - return True - return False - elif isinstance(right, Overloaded): - # TODO: this may be too restrictive - if len(left.items()) != len(right.items()): - return False - for i in range(len(left.items())): - if not is_subtype(left.items()[i], right.items()[i], self.check_type_parameter): - return False - return True - elif isinstance(right, UnboundType): - return True - elif isinstance(right, TypeType): - # All the items must have the same type object status, so - # it's sufficient to query only (any) one of them. - # This is unsound, we don't check the __init__ signature. - return left.is_type_obj() and is_subtype(left.items()[0].ret_type, right.item) - else: - return False - - def visit_union_type(self, left: UnionType) -> bool: - return all(is_subtype(item, self.right, self.check_type_parameter) - for item in left.items) - - def visit_partial_type(self, left: PartialType) -> bool: - # This is indeterminate as we don't really know the complete type yet. - raise RuntimeError - - def visit_type_type(self, left: TypeType) -> bool: - right = self.right - if isinstance(right, TypeType): - return is_subtype(left.item, right.item) - if isinstance(right, CallableType): - # This is unsound, we don't check the __init__ signature. - return right.is_type_obj() and is_subtype(left.item, right.ret_type) - if (isinstance(right, Instance) and - right.type.fullname() in ('builtins.type', 'builtins.object')): - # Treat builtins.type the same as Type[Any]; - # treat builtins.object the same as Any. - return True - return False - - -def is_callable_subtype(left: CallableType, right: CallableType, - ignore_return: bool = False) -> bool: - """Is left a subtype of right?""" - # TODO: Support named arguments, **args, etc. - # Non-type cannot be a subtype of type. - if right.is_type_obj() and not left.is_type_obj(): - return False - - # A callable L is a subtype of a generic callable R if L is a - # subtype of every type obtained from R by substituting types for - # the variables of R. We can check this by simply leaving the - # generic variables of R as type variables, effectively varying - # over all possible values. - - # It's okay even if these variables share ids with generic - # type variables of L, because generating and solving - # constraints for the variables of L to make L a subtype of R - # (below) treats type variables on the two sides as independent. - - if left.variables: - # Apply generic type variables away in left via type inference. - left = unify_generic_callable(left, right, ignore_return=ignore_return) - if left is None: - return False - - # Check return types. - if not ignore_return and not is_subtype(left.ret_type, right.ret_type): - return False - - if right.is_ellipsis_args: - return True - - # Check argument types. - if left.min_args > right.min_args: - return False - if left.is_var_arg: - return is_var_arg_callable_subtype_helper(left, right) - if right.is_var_arg: - return False - if len(left.arg_types) < len(right.arg_types): - return False - for i in range(len(right.arg_types)): - if not is_subtype(right.arg_types[i], left.arg_types[i]): - return False - return True - - -def is_var_arg_callable_subtype_helper(left: CallableType, right: CallableType) -> bool: - """Is left a subtype of right, assuming left has *args? - - See also is_callable_subtype for additional assumptions we can make. - """ - left_fixed = left.max_fixed_args() - right_fixed = right.max_fixed_args() - num_fixed_matching = min(left_fixed, right_fixed) - for i in range(num_fixed_matching): - if not is_subtype(right.arg_types[i], left.arg_types[i]): - return False - if not right.is_var_arg: - for i in range(num_fixed_matching, len(right.arg_types)): - if not is_subtype(right.arg_types[i], left.arg_types[-1]): - return False - return True - else: - for i in range(left_fixed, right_fixed): - if not is_subtype(right.arg_types[i], left.arg_types[-1]): - return False - for i in range(right_fixed, left_fixed): - if not is_subtype(right.arg_types[-1], left.arg_types[i]): - return False - return is_subtype(right.arg_types[-1], left.arg_types[-1]) - - -def unify_generic_callable(type: CallableType, target: CallableType, - ignore_return: bool) -> CallableType: - """Try to unify a generic callable type with another callable type. - - Return unified CallableType if successful; otherwise, return None. - """ - import mypy.solve - constraints = [] # type: List[mypy.constraints.Constraint] - for arg_type, target_arg_type in zip(type.arg_types, target.arg_types): - c = mypy.constraints.infer_constraints( - arg_type, target_arg_type, mypy.constraints.SUPERTYPE_OF) - constraints.extend(c) - if not ignore_return: - c = mypy.constraints.infer_constraints( - type.ret_type, target.ret_type, mypy.constraints.SUBTYPE_OF) - constraints.extend(c) - type_var_ids = [tvar.id for tvar in type.variables] - inferred_vars = mypy.solve.solve_constraints(type_var_ids, constraints) - if None in inferred_vars: - return None - msg = messages.temp_message_builder() - applied = mypy.applytype.apply_generic_arguments(type, inferred_vars, msg, context=target) - if msg.is_errors() or not isinstance(applied, CallableType): - return None - return cast(CallableType, applied) - - -def restrict_subtype_away(t: Type, s: Type) -> Type: - """Return a supertype of (t intersect not s) - - Currently just remove elements of a union type. - """ - if isinstance(t, UnionType): - new_items = [item for item in t.items if not is_subtype(item, s)] - return UnionType.make_union(new_items) - else: - return t - - -def is_proper_subtype(t: Type, s: Type) -> bool: - """Check if t is a proper subtype of s? - - For proper subtypes, there's no need to rely on compatibility due to - Any types. Any instance type t is also a proper subtype of t. - """ - # FIX tuple types - if isinstance(t, Instance): - if isinstance(s, Instance): - if not t.type.has_base(s.type.fullname()): - return False - - def check_argument(left: Type, right: Type, variance: int) -> bool: - if variance == COVARIANT: - return is_proper_subtype(left, right) - elif variance == CONTRAVARIANT: - return is_proper_subtype(right, left) - else: - return sametypes.is_same_type(left, right) - - # Map left type to corresponding right instances. - t = map_instance_to_supertype(t, s.type) - - return all(check_argument(ta, ra, tvar.variance) for ta, ra, tvar in - zip(t.args, s.args, s.type.defn.type_vars)) - return False - else: - return sametypes.is_same_type(t, s) - - -def is_more_precise(t: Type, s: Type) -> bool: - """Check if t is a more precise type than s. - - A t is a proper subtype of s, t is also more precise than s. Also, if - s is Any, t is more precise than s for any t. Finally, if t is the same - type as s, t is more precise than s. - """ - # TODO Should List[int] be more precise than List[Any]? - if isinstance(s, AnyType): - return True - if isinstance(s, Instance): - if isinstance(t, CallableType): - # Fall back to subclass check and ignore other properties of the callable. - return is_proper_subtype(t.fallback, s) - return is_proper_subtype(t, s) - return sametypes.is_same_type(t, s) +from typing import cast, List, Dict, Callable + +from mypy.types import ( + Type, AnyType, UnboundType, TypeVisitor, ErrorType, Void, NoneTyp, + Instance, TypeVarType, CallableType, TupleType, NamedTupleType, UnionType, Overloaded, ErasedType, TypeList, + PartialType, DeletedType, UninhabitedType, TypeType, is_named_instance +) +import mypy.applytype +import mypy.constraints +# Circular import; done in the function instead. +# import mypy.solve +from mypy import messages, sametypes +from mypy.nodes import CONTRAVARIANT, COVARIANT +from mypy.maptype import map_instance_to_supertype + +from mypy import experiments + + +TypeParameterChecker = Callable[[Type, Type, int], bool] + + +def check_type_parameter(lefta: Type, righta: Type, variance: int) -> bool: + if variance == COVARIANT: + return is_subtype(lefta, righta, check_type_parameter) + elif variance == CONTRAVARIANT: + return is_subtype(righta, lefta, check_type_parameter) + else: + return is_equivalent(lefta, righta, check_type_parameter) + + +def is_subtype(left: Type, right: Type, + type_parameter_checker: TypeParameterChecker = check_type_parameter) -> bool: + """Is 'left' subtype of 'right'? + + Also consider Any to be a subtype of any type, and vice versa. This + recursively applies to components of composite types (List[int] is subtype + of List[Any], for example). + + type_parameter_checker is used to check the type parameters (for example, + A with B in is_subtype(C[A], C[B]). The default checks for subtype relation + between the type arguments (e.g., A and B), taking the variance of the + type var into account. + """ + if (isinstance(right, AnyType) or isinstance(right, UnboundType) + or isinstance(right, ErasedType)): + return True + elif isinstance(right, UnionType) and not isinstance(left, UnionType): + return any(is_subtype(left, item, type_parameter_checker) + for item in right.items) + else: + return left.accept(SubtypeVisitor(right, type_parameter_checker)) + + +def is_subtype_ignoring_tvars(left: Type, right: Type) -> bool: + def ignore_tvars(s: Type, t: Type, v: int) -> bool: + return True + return is_subtype(left, right, ignore_tvars) + + +def is_equivalent(a: Type, b: Type, + type_parameter_checker=check_type_parameter) -> bool: + return is_subtype(a, b, type_parameter_checker) and is_subtype(b, a, type_parameter_checker) + + +def satisfies_upper_bound(a: Type, upper_bound: Type) -> bool: + """Is 'a' valid value for a type variable with the given 'upper_bound'? + + Same as is_subtype except that Void is considered to be a subtype of + any upper_bound. This is needed in a case like + + def f(g: Callable[[], T]) -> T: ... + def h() -> None: ... + f(h) + """ + return isinstance(a, Void) or is_subtype(a, upper_bound) + + +class SubtypeVisitor(TypeVisitor[bool]): + + def __init__(self, right: Type, + type_parameter_checker: TypeParameterChecker) -> None: + self.right = right + self.check_type_parameter = type_parameter_checker + + # visit_x(left) means: is left (which is an instance of X) a subtype of + # right? + + def visit_unbound_type(self, left: UnboundType) -> bool: + return True + + def visit_error_type(self, left: ErrorType) -> bool: + return False + + def visit_type_list(self, t: TypeList) -> bool: + assert False, 'Not supported' + + def visit_any(self, left: AnyType) -> bool: + return True + + def visit_void(self, left: Void) -> bool: + return isinstance(self.right, Void) + + def visit_none_type(self, left: NoneTyp) -> bool: + if experiments.STRICT_OPTIONAL: + return (isinstance(self.right, NoneTyp) or + is_named_instance(self.right, 'builtins.object')) + else: + return not isinstance(self.right, Void) + + def visit_uninhabited_type(self, left: UninhabitedType) -> bool: + return not isinstance(self.right, Void) + + def visit_erased_type(self, left: ErasedType) -> bool: + return True + + def visit_deleted_type(self, left: DeletedType) -> bool: + return True + + def visit_instance(self, left: Instance) -> bool: + if left.type.fallback_to_any: + return True + right = self.right + if isinstance(right, Instance): + if left.type._promote and is_subtype(left.type._promote, + self.right, + self.check_type_parameter): + return True + rname = right.type.fullname() + if not left.type.has_base(rname) and rname != 'builtins.object': + return False + + # Map left type to corresponding right instances. + t = map_instance_to_supertype(left, right.type) + + return all(self.check_type_parameter(lefta, righta, tvar.variance) + for lefta, righta, tvar in + zip(t.args, right.args, right.type.defn.type_vars)) + else: + return False + + def visit_type_var(self, left: TypeVarType) -> bool: + right = self.right + if isinstance(right, TypeVarType): + return left.id == right.id + else: + return is_subtype(left.upper_bound, self.right) + + def visit_callable_type(self, left: CallableType) -> bool: + right = self.right + if isinstance(right, CallableType): + return is_callable_subtype(left, right) + elif isinstance(right, Overloaded): + return all(is_subtype(left, item, self.check_type_parameter) + for item in right.items()) + elif isinstance(right, Instance): + return is_subtype(left.fallback, right) + elif isinstance(right, TypeType): + # This is unsound, we don't check the __init__ signature. + return left.is_type_obj() and is_subtype(left.ret_type, right.item) + else: + return False + + def visit_tuple_type(self, left: TupleType) -> bool: + right = self.right + if isinstance(right, Instance): + if is_named_instance(right, 'builtins.object'): + return True + if is_named_instance(right, 'builtins.tuple'): + target_item_type = right.args[0] + return all(is_subtype(item, target_item_type) + for item in left.items) + elif is_named_instance(right, 'typing.Sized'): + return True + elif (is_named_instance(right, 'typing.Iterable') or + is_named_instance(right, 'typing.Container') or + is_named_instance(right, 'typing.Sequence') or + is_named_instance(right, 'typing.Reversible')): + iter_type = right.args[0] + return all(is_subtype(li, iter_type) for li in left.items) + return False + elif isinstance(right, TupleType): + if len(left.items) != len(right.items): + return False + for l, r in zip(left.items, right.items): + if not is_subtype(l, r, self.check_type_parameter): + return False + if not is_subtype(left.fallback, right.fallback, self.check_type_parameter): + return False + return True + else: + return False + + def visit_namedtuple_type(self, left: NamedTupleType) -> bool: + if isinstance(self.right, NamedTupleType): + # left.attrs.issubset(right.attrs) is insufficient + # (a: int, b: str) is not subtype of (b: int, a: str) + if any(l != r for l, r in zip(left.attrs, self.right.attrs)): + return False + return self.visit_tuple_type(left) + + def visit_overloaded(self, left: Overloaded) -> bool: + right = self.right + if isinstance(right, Instance): + return is_subtype(left.fallback, right) + elif isinstance(right, CallableType) or is_named_instance( + right, 'builtins.type'): + for item in left.items(): + if is_subtype(item, right, self.check_type_parameter): + return True + return False + elif isinstance(right, Overloaded): + # TODO: this may be too restrictive + if len(left.items()) != len(right.items()): + return False + for i in range(len(left.items())): + if not is_subtype(left.items()[i], right.items()[i], self.check_type_parameter): + return False + return True + elif isinstance(right, UnboundType): + return True + elif isinstance(right, TypeType): + # All the items must have the same type object status, so + # it's sufficient to query only (any) one of them. + # This is unsound, we don't check the __init__ signature. + return left.is_type_obj() and is_subtype(left.items()[0].ret_type, right.item) + else: + return False + + def visit_union_type(self, left: UnionType) -> bool: + return all(is_subtype(item, self.right, self.check_type_parameter) + for item in left.items) + + def visit_partial_type(self, left: PartialType) -> bool: + # This is indeterminate as we don't really know the complete type yet. + raise RuntimeError + + def visit_type_type(self, left: TypeType) -> bool: + right = self.right + if isinstance(right, TypeType): + return is_subtype(left.item, right.item) + if isinstance(right, CallableType): + # This is unsound, we don't check the __init__ signature. + return right.is_type_obj() and is_subtype(left.item, right.ret_type) + if (isinstance(right, Instance) and + right.type.fullname() in ('builtins.type', 'builtins.object')): + # Treat builtins.type the same as Type[Any]; + # treat builtins.object the same as Any. + return True + return False + + +def is_callable_subtype(left: CallableType, right: CallableType, + ignore_return: bool = False) -> bool: + """Is left a subtype of right?""" + # TODO: Support named arguments, **args, etc. + # Non-type cannot be a subtype of type. + if right.is_type_obj() and not left.is_type_obj(): + return False + + # A callable L is a subtype of a generic callable R if L is a + # subtype of every type obtained from R by substituting types for + # the variables of R. We can check this by simply leaving the + # generic variables of R as type variables, effectively varying + # over all possible values. + + # It's okay even if these variables share ids with generic + # type variables of L, because generating and solving + # constraints for the variables of L to make L a subtype of R + # (below) treats type variables on the two sides as independent. + + if left.variables: + # Apply generic type variables away in left via type inference. + left = unify_generic_callable(left, right, ignore_return=ignore_return) + if left is None: + return False + + # Check return types. + if not ignore_return and not is_subtype(left.ret_type, right.ret_type): + return False + + if right.is_ellipsis_args: + return True + + # Check argument types. + if left.min_args > right.min_args: + return False + if left.is_var_arg: + return is_var_arg_callable_subtype_helper(left, right) + if right.is_var_arg: + return False + if len(left.arg_types) < len(right.arg_types): + return False + for i in range(len(right.arg_types)): + if not is_subtype(right.arg_types[i], left.arg_types[i]): + return False + return True + + +def is_var_arg_callable_subtype_helper(left: CallableType, right: CallableType) -> bool: + """Is left a subtype of right, assuming left has *args? + + See also is_callable_subtype for additional assumptions we can make. + """ + left_fixed = left.max_fixed_args() + right_fixed = right.max_fixed_args() + num_fixed_matching = min(left_fixed, right_fixed) + for i in range(num_fixed_matching): + if not is_subtype(right.arg_types[i], left.arg_types[i]): + return False + if not right.is_var_arg: + for i in range(num_fixed_matching, len(right.arg_types)): + if not is_subtype(right.arg_types[i], left.arg_types[-1]): + return False + return True + else: + for i in range(left_fixed, right_fixed): + if not is_subtype(right.arg_types[i], left.arg_types[-1]): + return False + for i in range(right_fixed, left_fixed): + if not is_subtype(right.arg_types[-1], left.arg_types[i]): + return False + return is_subtype(right.arg_types[-1], left.arg_types[-1]) + + +def unify_generic_callable(type: CallableType, target: CallableType, + ignore_return: bool) -> CallableType: + """Try to unify a generic callable type with another callable type. + + Return unified CallableType if successful; otherwise, return None. + """ + import mypy.solve + constraints = [] # type: List[mypy.constraints.Constraint] + for arg_type, target_arg_type in zip(type.arg_types, target.arg_types): + c = mypy.constraints.infer_constraints( + arg_type, target_arg_type, mypy.constraints.SUPERTYPE_OF) + constraints.extend(c) + if not ignore_return: + c = mypy.constraints.infer_constraints( + type.ret_type, target.ret_type, mypy.constraints.SUBTYPE_OF) + constraints.extend(c) + type_var_ids = [tvar.id for tvar in type.variables] + inferred_vars = mypy.solve.solve_constraints(type_var_ids, constraints) + if None in inferred_vars: + return None + msg = messages.temp_message_builder() + applied = mypy.applytype.apply_generic_arguments(type, inferred_vars, msg, context=target) + if msg.is_errors() or not isinstance(applied, CallableType): + return None + return cast(CallableType, applied) + + +def restrict_subtype_away(t: Type, s: Type) -> Type: + """Return a supertype of (t intersect not s) + + Currently just remove elements of a union type. + """ + if isinstance(t, UnionType): + new_items = [item for item in t.items if not is_subtype(item, s)] + return UnionType.make_union(new_items) + else: + return t + + +def is_proper_subtype(t: Type, s: Type) -> bool: + """Check if t is a proper subtype of s? + + For proper subtypes, there's no need to rely on compatibility due to + Any types. Any instance type t is also a proper subtype of t. + """ + # FIX tuple types + if isinstance(t, Instance): + if isinstance(s, Instance): + if not t.type.has_base(s.type.fullname()): + return False + + def check_argument(left: Type, right: Type, variance: int) -> bool: + if variance == COVARIANT: + return is_proper_subtype(left, right) + elif variance == CONTRAVARIANT: + return is_proper_subtype(right, left) + else: + return sametypes.is_same_type(left, right) + + # Map left type to corresponding right instances. + t = map_instance_to_supertype(t, s.type) + + return all(check_argument(ta, ra, tvar.variance) for ta, ra, tvar in + zip(t.args, s.args, s.type.defn.type_vars)) + return False + else: + return sametypes.is_same_type(t, s) + + +def is_more_precise(t: Type, s: Type) -> bool: + """Check if t is a more precise type than s. + + A t is a proper subtype of s, t is also more precise than s. Also, if + s is Any, t is more precise than s for any t. Finally, if t is the same + type as s, t is more precise than s. + """ + # TODO Should List[int] be more precise than List[Any]? + if isinstance(s, AnyType): + return True + if isinstance(s, Instance): + if isinstance(t, CallableType): + # Fall back to subclass check and ignore other properties of the callable. + return is_proper_subtype(t.fallback, s) + return is_proper_subtype(t, s) + return sametypes.is_same_type(t, s) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index f10048c4f0fe4..1841f283a19e1 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -1,430 +1,430 @@ -"""Semantic analysis of types""" - -from typing import Callable, cast, List, Tuple - -from mypy.types import ( - Type, UnboundType, TypeVarType, TupleType, NamedTupleType, UnionType, Instance, AnyType, CallableType, - Void, NoneTyp, DeletedType, TypeList, TypeVarDef, TypeVisitor, StarType, PartialType, - EllipsisType, UninhabitedType, TypeType -) -from mypy.nodes import ( - BOUND_TVAR, TYPE_ALIAS, UNBOUND_IMPORTED, - TypeInfo, Context, SymbolTableNode, TypeVarExpr, Var, Node, - IndexExpr, RefExpr -) -from mypy.sametypes import is_same_type -from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError -from mypy.subtypes import satisfies_upper_bound -from mypy import nodes -from mypy import experiments - - -type_constructors = {'typing.Tuple', 'typing.Union', 'typing.Callable', 'typing.Type'} - - -def analyze_type_alias(node: Node, - lookup_func: Callable[[str, Context], SymbolTableNode], - lookup_fqn_func: Callable[[str], SymbolTableNode], - fail_func: Callable[[str, Context], None]) -> Type: - """Return type if node is valid as a type alias rvalue. - - Return None otherwise. 'node' must have been semantically analyzed. - """ - # Quickly return None if the expression doesn't look like a type. Note - # that we don't support straight string literals as type aliases - # (only string literals within index expressions). - if isinstance(node, RefExpr): - if not (isinstance(node.node, TypeInfo) or - node.fullname == 'typing.Any' or - node.kind == TYPE_ALIAS): - return None - elif isinstance(node, IndexExpr): - base = node.base - if isinstance(base, RefExpr): - if not (isinstance(base.node, TypeInfo) or - base.fullname in type_constructors): - return None - else: - return None - else: - return None - - # It's a type alias (though it may be an invalid one). - try: - type = expr_to_unanalyzed_type(node) - except TypeTranslationError: - fail_func('Invalid type alias', node) - return None - analyzer = TypeAnalyser(lookup_func, lookup_fqn_func, fail_func) - return type.accept(analyzer) - - -class TypeAnalyser(TypeVisitor[Type]): - """Semantic analyzer for types (semantic analysis pass 2). - - Converts unbound types into bound types. - """ - - def __init__(self, - lookup_func: Callable[[str, Context], SymbolTableNode], - lookup_fqn_func: Callable[[str], SymbolTableNode], - fail_func: Callable[[str, Context], None]) -> None: - self.lookup = lookup_func - self.lookup_fqn_func = lookup_fqn_func - self.fail = fail_func - - def visit_unbound_type(self, t: UnboundType) -> Type: - if t.optional: - t.optional = False - # We don't need to worry about double-wrapping Optionals or - # wrapping Anys: Union simplification will take care of that. - return UnionType.make_simplified_union([self.visit_unbound_type(t), NoneTyp()]) - sym = self.lookup(t.name, t) - if sym is not None: - if sym.node is None: - # UNBOUND_IMPORTED can happen if an unknown name was imported. - if sym.kind != UNBOUND_IMPORTED: - self.fail('Internal error (node is None, kind={})'.format(sym.kind), t) - return AnyType() - fullname = sym.node.fullname() - if sym.kind == BOUND_TVAR: - if len(t.args) > 0: - self.fail('Type variable "{}" used with arguments'.format( - t.name), t) - assert sym.tvar_def is not None - return TypeVarType(sym.tvar_def, t.line) - elif fullname == 'builtins.None': - if experiments.STRICT_OPTIONAL: - if t.is_ret_type: - return Void() - else: - return NoneTyp() - else: - return Void() - elif fullname == 'typing.Any': - return AnyType() - elif fullname == 'typing.Tuple': - if len(t.args) == 2 and isinstance(t.args[1], EllipsisType): - # Tuple[T, ...] (uniform, variable-length tuple) - node = self.lookup_fqn_func('builtins.tuple') - tuple_info = cast(TypeInfo, node.node) - return Instance(tuple_info, [t.args[0].accept(self)], t.line) - return self.tuple_type(self.anal_array(t.args)) - elif fullname == 'typing.Union': - items = self.anal_array(t.args) - items = [item for item in items if not isinstance(item, Void)] - return UnionType.make_union(items) - elif fullname == 'typing.Optional': - if len(t.args) != 1: - self.fail('Optional[...] must have exactly one type argument', t) - return AnyType() - items = self.anal_array(t.args) - if experiments.STRICT_OPTIONAL: - return UnionType.make_simplified_union([items[0], NoneTyp()]) - else: - # Without strict Optional checking Optional[t] is just an alias for t. - return items[0] - elif fullname == 'typing.Callable': - return self.analyze_callable_type(t) - elif fullname == 'typing.Type': - if len(t.args) == 0: - return TypeType(AnyType(), line=t.line) - if len(t.args) != 1: - self.fail('Type[...] must have exactly one type argument', t) - items = self.anal_array(t.args) - item = items[0] - return TypeType(item, line=t.line) - elif sym.kind == TYPE_ALIAS: - # TODO: Generic type aliases. - return sym.type_override - elif not isinstance(sym.node, TypeInfo): - name = sym.fullname - if name is None: - name = sym.node.name() - if isinstance(sym.node, Var) and isinstance(sym.node.type, AnyType): - # Something with an Any type -- make it an alias for Any in a type - # context. This is slightly problematic as it allows using the type 'Any' - # as a base class -- however, this will fail soon at runtime so the problem - # is pretty minor. - return AnyType() - self.fail('Invalid type "{}"'.format(name), t) - return t - info = sym.node # type: TypeInfo - if len(t.args) > 0 and info.fullname() == 'builtins.tuple': - return TupleType(self.anal_array(t.args), - Instance(info, [AnyType()], t.line), - t.line) - else: - # Analyze arguments and construct Instance type. The - # number of type arguments and their values are - # checked only later, since we do not always know the - # valid count at this point. Thus we may construct an - # Instance with an invalid number of type arguments. - instance = Instance(info, self.anal_array(t.args), t.line) - if info.tuple_type is None: - return instance - else: - # The class has a Tuple[...] base class so it will be - # represented as a tuple type. - if t.args: - self.fail('Generic tuple types not supported', t) - return AnyType() - # import pdb; pdb.set_trace() - if info.is_named_tuple: - return NamedTupleType(info.name(), info.attrs, - self.anal_array(info.tuple_type.items), - fallback=instance, - line=t.line) - return TupleType(self.anal_array(info.tuple_type.items), - fallback=instance, - line=t.line) - else: - return AnyType() - - def visit_any(self, t: AnyType) -> Type: - return t - - def visit_void(self, t: Void) -> Type: - return t - - def visit_none_type(self, t: NoneTyp) -> Type: - return t - - def visit_uninhabited_type(self, t: UninhabitedType) -> Type: - return t - - def visit_deleted_type(self, t: DeletedType) -> Type: - return t - - def visit_type_list(self, t: TypeList) -> Type: - self.fail('Invalid type', t) - return AnyType() - - def visit_instance(self, t: Instance) -> Type: - return t - - def visit_type_var(self, t: TypeVarType) -> Type: - raise RuntimeError('TypeVarType is already analyzed') - - def visit_callable_type(self, t: CallableType) -> Type: - return t.copy_modified(arg_types=self.anal_array(t.arg_types), - ret_type=t.ret_type.accept(self), - fallback=t.fallback or self.builtin_type('builtins.function'), - variables=self.anal_var_defs(t.variables)) - - def visit_tuple_type(self, t: TupleType) -> Type: - if t.implicit: - self.fail('Invalid tuple literal type', t) - return AnyType() - star_count = sum(1 for item in t.items if isinstance(item, StarType)) - if star_count > 1: - self.fail('At most one star type allowed in a tuple', t) - return AnyType() - fallback = t.fallback if t.fallback else self.builtin_type('builtins.tuple', [AnyType()]) - return TupleType(self.anal_array(t.items), fallback, t.line) - - def visit_namedtuple_type(self, t: NamedTupleType) -> Type: - return NamedTupleType(t.name, t.attrs, - self.anal_array(t.items), - fallback=t.fallback, - line=t.line) - - def visit_star_type(self, t: StarType) -> Type: - return StarType(t.type.accept(self), t.line) - - def visit_union_type(self, t: UnionType) -> Type: - return UnionType(self.anal_array(t.items), t.line) - - def visit_partial_type(self, t: PartialType) -> Type: - assert False, "Internal error: Unexpected partial type" - - def visit_ellipsis_type(self, t: EllipsisType) -> Type: - self.fail("Unexpected '...'", t) - return AnyType() - - def visit_type_type(self, t: TypeType) -> Type: - return TypeType(t.item.accept(self), line=t.line) - - def analyze_callable_type(self, t: UnboundType) -> Type: - fallback = self.builtin_type('builtins.function') - if len(t.args) == 0: - # Callable (bare). Treat as Callable[..., Any]. - return CallableType([AnyType(), AnyType()], - [nodes.ARG_STAR, nodes.ARG_STAR2], - [None, None], - ret_type=AnyType(), - fallback=fallback, - is_ellipsis_args=True) - elif len(t.args) == 2: - ret_type = t.args[1].accept(self) - if isinstance(t.args[0], TypeList): - # Callable[[ARG, ...], RET] (ordinary callable type) - args = t.args[0].items - return CallableType(self.anal_array(args), - [nodes.ARG_POS] * len(args), - [None] * len(args), - ret_type=ret_type, - fallback=fallback) - elif isinstance(t.args[0], EllipsisType): - # Callable[..., RET] (with literal ellipsis; accept arbitrary arguments) - return CallableType([AnyType(), AnyType()], - [nodes.ARG_STAR, nodes.ARG_STAR2], - [None, None], - ret_type=ret_type, - fallback=fallback, - is_ellipsis_args=True) - else: - self.fail('The first argument to Callable must be a list of types or "..."', t) - return AnyType() - - self.fail('Invalid function type', t) - return AnyType() - - def anal_array(self, a: List[Type]) -> List[Type]: - res = [] # type: List[Type] - for t in a: - res.append(t.accept(self)) - return res - - def anal_var_defs(self, var_defs: List[TypeVarDef]) -> List[TypeVarDef]: - a = [] # type: List[TypeVarDef] - for vd in var_defs: - a.append(TypeVarDef(vd.name, vd.id.raw_id, self.anal_array(vd.values), - vd.upper_bound.accept(self), - vd.variance, - vd.line)) - return a - - def builtin_type(self, fully_qualified_name: str, args: List[Type] = None) -> Instance: - node = self.lookup_fqn_func(fully_qualified_name) - info = cast(TypeInfo, node.node) - return Instance(info, args or []) - - def tuple_type(self, items: List[Type]) -> TupleType: - return TupleType(items, fallback=self.builtin_type('builtins.tuple', [AnyType()])) - - -class TypeAnalyserPass3(TypeVisitor[None]): - """Analyze type argument counts and values of generic types. - - This is semantic analysis pass 3 for types. - - Perform these operations: - - * Report error for invalid type argument counts, such as List[x, y]. - * Make implicit Any type arguments explicit my modifying types - in-place. For example, modify Foo into Foo[Any] if Foo expects a single - type argument. - * If a type variable has a value restriction, ensure that the value is - valid. For example, reject IO[int] if the type argument must be str - or bytes. - - We can't do this earlier than the third pass, since type argument counts - are only determined in pass 2, and we have to support forward references - to types. - """ - - def __init__(self, fail_func: Callable[[str, Context], None]) -> None: - self.fail = fail_func - - def visit_instance(self, t: Instance) -> None: - info = t.type - # Check type argument count. - if len(t.args) != len(info.type_vars): - if len(t.args) == 0: - # Insert implicit 'Any' type arguments. - t.args = [AnyType()] * len(info.type_vars) - return - # Invalid number of type parameters. - n = len(info.type_vars) - s = '{} type arguments'.format(n) - if n == 0: - s = 'no type arguments' - elif n == 1: - s = '1 type argument' - act = str(len(t.args)) - if act == '0': - act = 'none' - self.fail('"{}" expects {}, but {} given'.format( - info.name(), s, act), t) - # Construct the correct number of type arguments, as - # otherwise the type checker may crash as it expects - # things to be right. - t.args = [AnyType() for _ in info.type_vars] - elif info.defn.type_vars: - # Check type argument values. - for arg, TypeVar in zip(t.args, info.defn.type_vars): - if TypeVar.values: - if isinstance(arg, TypeVarType): - arg_values = arg.values - if not arg_values: - self.fail('Type variable "{}" not valid as type ' - 'argument value for "{}"'.format( - arg.name, info.name()), t) - continue - else: - arg_values = [arg] - self.check_type_var_values(info, arg_values, - TypeVar.values, t) - if not satisfies_upper_bound(arg, TypeVar.upper_bound): - self.fail('Type argument "{}" of "{}" must be ' - 'a subtype of "{}"'.format( - arg, info.name(), TypeVar.upper_bound), t) - for arg in t.args: - arg.accept(self) - - def check_type_var_values(self, type: TypeInfo, actuals: List[Type], - valids: List[Type], context: Context) -> None: - for actual in actuals: - if (not isinstance(actual, AnyType) and - not any(is_same_type(actual, value) for value in valids)): - self.fail('Invalid type argument value for "{}"'.format( - type.name()), context) - - def visit_callable_type(self, t: CallableType) -> None: - t.ret_type.accept(self) - for arg_type in t.arg_types: - arg_type.accept(self) - - def visit_tuple_type(self, t: TupleType) -> None: - for item in t.items: - item.accept(self) - - def visit_union_type(self, t: UnionType) -> None: - for item in t.items: - item.accept(self) - - def visit_star_type(self, t: StarType) -> None: - t.type.accept(self) - - # Other kinds of type are trivial, since they are atomic (or invalid). - - def visit_unbound_type(self, t: UnboundType) -> None: - pass - - def visit_any(self, t: AnyType) -> None: - pass - - def visit_void(self, t: Void) -> None: - pass - - def visit_none_type(self, t: NoneTyp) -> None: - pass - - def visit_uninhabited_type(self, t: UninhabitedType) -> None: - pass - - def visit_deleted_type(self, t: DeletedType) -> None: - pass - - def visit_type_list(self, t: TypeList) -> None: - self.fail('Invalid type', t) - - def visit_type_var(self, t: TypeVarType) -> None: - pass - - def visit_partial_type(self, t: PartialType) -> None: - pass - - def visit_type_type(self, t: TypeType) -> None: - pass +"""Semantic analysis of types""" + +from typing import Callable, cast, List, Tuple + +from mypy.types import ( + Type, UnboundType, TypeVarType, TupleType, NamedTupleType, UnionType, Instance, AnyType, CallableType, + Void, NoneTyp, DeletedType, TypeList, TypeVarDef, TypeVisitor, StarType, PartialType, + EllipsisType, UninhabitedType, TypeType +) +from mypy.nodes import ( + BOUND_TVAR, TYPE_ALIAS, UNBOUND_IMPORTED, + TypeInfo, Context, SymbolTableNode, TypeVarExpr, Var, Node, + IndexExpr, RefExpr +) +from mypy.sametypes import is_same_type +from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError +from mypy.subtypes import satisfies_upper_bound +from mypy import nodes +from mypy import experiments + + +type_constructors = {'typing.Tuple', 'typing.Union', 'typing.Callable', 'typing.Type'} + + +def analyze_type_alias(node: Node, + lookup_func: Callable[[str, Context], SymbolTableNode], + lookup_fqn_func: Callable[[str], SymbolTableNode], + fail_func: Callable[[str, Context], None]) -> Type: + """Return type if node is valid as a type alias rvalue. + + Return None otherwise. 'node' must have been semantically analyzed. + """ + # Quickly return None if the expression doesn't look like a type. Note + # that we don't support straight string literals as type aliases + # (only string literals within index expressions). + if isinstance(node, RefExpr): + if not (isinstance(node.node, TypeInfo) or + node.fullname == 'typing.Any' or + node.kind == TYPE_ALIAS): + return None + elif isinstance(node, IndexExpr): + base = node.base + if isinstance(base, RefExpr): + if not (isinstance(base.node, TypeInfo) or + base.fullname in type_constructors): + return None + else: + return None + else: + return None + + # It's a type alias (though it may be an invalid one). + try: + type = expr_to_unanalyzed_type(node) + except TypeTranslationError: + fail_func('Invalid type alias', node) + return None + analyzer = TypeAnalyser(lookup_func, lookup_fqn_func, fail_func) + return type.accept(analyzer) + + +class TypeAnalyser(TypeVisitor[Type]): + """Semantic analyzer for types (semantic analysis pass 2). + + Converts unbound types into bound types. + """ + + def __init__(self, + lookup_func: Callable[[str, Context], SymbolTableNode], + lookup_fqn_func: Callable[[str], SymbolTableNode], + fail_func: Callable[[str, Context], None]) -> None: + self.lookup = lookup_func + self.lookup_fqn_func = lookup_fqn_func + self.fail = fail_func + + def visit_unbound_type(self, t: UnboundType) -> Type: + if t.optional: + t.optional = False + # We don't need to worry about double-wrapping Optionals or + # wrapping Anys: Union simplification will take care of that. + return UnionType.make_simplified_union([self.visit_unbound_type(t), NoneTyp()]) + sym = self.lookup(t.name, t) + if sym is not None: + if sym.node is None: + # UNBOUND_IMPORTED can happen if an unknown name was imported. + if sym.kind != UNBOUND_IMPORTED: + self.fail('Internal error (node is None, kind={})'.format(sym.kind), t) + return AnyType() + fullname = sym.node.fullname() + if sym.kind == BOUND_TVAR: + if len(t.args) > 0: + self.fail('Type variable "{}" used with arguments'.format( + t.name), t) + assert sym.tvar_def is not None + return TypeVarType(sym.tvar_def, t.line) + elif fullname == 'builtins.None': + if experiments.STRICT_OPTIONAL: + if t.is_ret_type: + return Void() + else: + return NoneTyp() + else: + return Void() + elif fullname == 'typing.Any': + return AnyType() + elif fullname == 'typing.Tuple': + if len(t.args) == 2 and isinstance(t.args[1], EllipsisType): + # Tuple[T, ...] (uniform, variable-length tuple) + node = self.lookup_fqn_func('builtins.tuple') + tuple_info = cast(TypeInfo, node.node) + return Instance(tuple_info, [t.args[0].accept(self)], t.line) + return self.tuple_type(self.anal_array(t.args)) + elif fullname == 'typing.Union': + items = self.anal_array(t.args) + items = [item for item in items if not isinstance(item, Void)] + return UnionType.make_union(items) + elif fullname == 'typing.Optional': + if len(t.args) != 1: + self.fail('Optional[...] must have exactly one type argument', t) + return AnyType() + items = self.anal_array(t.args) + if experiments.STRICT_OPTIONAL: + return UnionType.make_simplified_union([items[0], NoneTyp()]) + else: + # Without strict Optional checking Optional[t] is just an alias for t. + return items[0] + elif fullname == 'typing.Callable': + return self.analyze_callable_type(t) + elif fullname == 'typing.Type': + if len(t.args) == 0: + return TypeType(AnyType(), line=t.line) + if len(t.args) != 1: + self.fail('Type[...] must have exactly one type argument', t) + items = self.anal_array(t.args) + item = items[0] + return TypeType(item, line=t.line) + elif sym.kind == TYPE_ALIAS: + # TODO: Generic type aliases. + return sym.type_override + elif not isinstance(sym.node, TypeInfo): + name = sym.fullname + if name is None: + name = sym.node.name() + if isinstance(sym.node, Var) and isinstance(sym.node.type, AnyType): + # Something with an Any type -- make it an alias for Any in a type + # context. This is slightly problematic as it allows using the type 'Any' + # as a base class -- however, this will fail soon at runtime so the problem + # is pretty minor. + return AnyType() + self.fail('Invalid type "{}"'.format(name), t) + return t + info = sym.node # type: TypeInfo + if len(t.args) > 0 and info.fullname() == 'builtins.tuple': + return TupleType(self.anal_array(t.args), + Instance(info, [AnyType()], t.line), + t.line) + else: + # Analyze arguments and construct Instance type. The + # number of type arguments and their values are + # checked only later, since we do not always know the + # valid count at this point. Thus we may construct an + # Instance with an invalid number of type arguments. + instance = Instance(info, self.anal_array(t.args), t.line) + if info.tuple_type is None: + return instance + else: + # The class has a Tuple[...] base class so it will be + # represented as a tuple type. + if t.args: + self.fail('Generic tuple types not supported', t) + return AnyType() + # import pdb; pdb.set_trace() + if info.is_named_tuple: + return NamedTupleType(info.name(), info.attrs, + self.anal_array(info.tuple_type.items), + fallback=instance, + line=t.line) + return TupleType(self.anal_array(info.tuple_type.items), + fallback=instance, + line=t.line) + else: + return AnyType() + + def visit_any(self, t: AnyType) -> Type: + return t + + def visit_void(self, t: Void) -> Type: + return t + + def visit_none_type(self, t: NoneTyp) -> Type: + return t + + def visit_uninhabited_type(self, t: UninhabitedType) -> Type: + return t + + def visit_deleted_type(self, t: DeletedType) -> Type: + return t + + def visit_type_list(self, t: TypeList) -> Type: + self.fail('Invalid type', t) + return AnyType() + + def visit_instance(self, t: Instance) -> Type: + return t + + def visit_type_var(self, t: TypeVarType) -> Type: + raise RuntimeError('TypeVarType is already analyzed') + + def visit_callable_type(self, t: CallableType) -> Type: + return t.copy_modified(arg_types=self.anal_array(t.arg_types), + ret_type=t.ret_type.accept(self), + fallback=t.fallback or self.builtin_type('builtins.function'), + variables=self.anal_var_defs(t.variables)) + + def visit_tuple_type(self, t: TupleType) -> Type: + if t.implicit: + self.fail('Invalid tuple literal type', t) + return AnyType() + star_count = sum(1 for item in t.items if isinstance(item, StarType)) + if star_count > 1: + self.fail('At most one star type allowed in a tuple', t) + return AnyType() + fallback = t.fallback if t.fallback else self.builtin_type('builtins.tuple', [AnyType()]) + return TupleType(self.anal_array(t.items), fallback, t.line) + + def visit_namedtuple_type(self, t: NamedTupleType) -> Type: + return NamedTupleType(t.name, t.attrs, + self.anal_array(t.items), + fallback=t.fallback, + line=t.line) + + def visit_star_type(self, t: StarType) -> Type: + return StarType(t.type.accept(self), t.line) + + def visit_union_type(self, t: UnionType) -> Type: + return UnionType(self.anal_array(t.items), t.line) + + def visit_partial_type(self, t: PartialType) -> Type: + assert False, "Internal error: Unexpected partial type" + + def visit_ellipsis_type(self, t: EllipsisType) -> Type: + self.fail("Unexpected '...'", t) + return AnyType() + + def visit_type_type(self, t: TypeType) -> Type: + return TypeType(t.item.accept(self), line=t.line) + + def analyze_callable_type(self, t: UnboundType) -> Type: + fallback = self.builtin_type('builtins.function') + if len(t.args) == 0: + # Callable (bare). Treat as Callable[..., Any]. + return CallableType([AnyType(), AnyType()], + [nodes.ARG_STAR, nodes.ARG_STAR2], + [None, None], + ret_type=AnyType(), + fallback=fallback, + is_ellipsis_args=True) + elif len(t.args) == 2: + ret_type = t.args[1].accept(self) + if isinstance(t.args[0], TypeList): + # Callable[[ARG, ...], RET] (ordinary callable type) + args = t.args[0].items + return CallableType(self.anal_array(args), + [nodes.ARG_POS] * len(args), + [None] * len(args), + ret_type=ret_type, + fallback=fallback) + elif isinstance(t.args[0], EllipsisType): + # Callable[..., RET] (with literal ellipsis; accept arbitrary arguments) + return CallableType([AnyType(), AnyType()], + [nodes.ARG_STAR, nodes.ARG_STAR2], + [None, None], + ret_type=ret_type, + fallback=fallback, + is_ellipsis_args=True) + else: + self.fail('The first argument to Callable must be a list of types or "..."', t) + return AnyType() + + self.fail('Invalid function type', t) + return AnyType() + + def anal_array(self, a: List[Type]) -> List[Type]: + res = [] # type: List[Type] + for t in a: + res.append(t.accept(self)) + return res + + def anal_var_defs(self, var_defs: List[TypeVarDef]) -> List[TypeVarDef]: + a = [] # type: List[TypeVarDef] + for vd in var_defs: + a.append(TypeVarDef(vd.name, vd.id.raw_id, self.anal_array(vd.values), + vd.upper_bound.accept(self), + vd.variance, + vd.line)) + return a + + def builtin_type(self, fully_qualified_name: str, args: List[Type] = None) -> Instance: + node = self.lookup_fqn_func(fully_qualified_name) + info = cast(TypeInfo, node.node) + return Instance(info, args or []) + + def tuple_type(self, items: List[Type]) -> TupleType: + return TupleType(items, fallback=self.builtin_type('builtins.tuple', [AnyType()])) + + +class TypeAnalyserPass3(TypeVisitor[None]): + """Analyze type argument counts and values of generic types. + + This is semantic analysis pass 3 for types. + + Perform these operations: + + * Report error for invalid type argument counts, such as List[x, y]. + * Make implicit Any type arguments explicit my modifying types + in-place. For example, modify Foo into Foo[Any] if Foo expects a single + type argument. + * If a type variable has a value restriction, ensure that the value is + valid. For example, reject IO[int] if the type argument must be str + or bytes. + + We can't do this earlier than the third pass, since type argument counts + are only determined in pass 2, and we have to support forward references + to types. + """ + + def __init__(self, fail_func: Callable[[str, Context], None]) -> None: + self.fail = fail_func + + def visit_instance(self, t: Instance) -> None: + info = t.type + # Check type argument count. + if len(t.args) != len(info.type_vars): + if len(t.args) == 0: + # Insert implicit 'Any' type arguments. + t.args = [AnyType()] * len(info.type_vars) + return + # Invalid number of type parameters. + n = len(info.type_vars) + s = '{} type arguments'.format(n) + if n == 0: + s = 'no type arguments' + elif n == 1: + s = '1 type argument' + act = str(len(t.args)) + if act == '0': + act = 'none' + self.fail('"{}" expects {}, but {} given'.format( + info.name(), s, act), t) + # Construct the correct number of type arguments, as + # otherwise the type checker may crash as it expects + # things to be right. + t.args = [AnyType() for _ in info.type_vars] + elif info.defn.type_vars: + # Check type argument values. + for arg, TypeVar in zip(t.args, info.defn.type_vars): + if TypeVar.values: + if isinstance(arg, TypeVarType): + arg_values = arg.values + if not arg_values: + self.fail('Type variable "{}" not valid as type ' + 'argument value for "{}"'.format( + arg.name, info.name()), t) + continue + else: + arg_values = [arg] + self.check_type_var_values(info, arg_values, + TypeVar.values, t) + if not satisfies_upper_bound(arg, TypeVar.upper_bound): + self.fail('Type argument "{}" of "{}" must be ' + 'a subtype of "{}"'.format( + arg, info.name(), TypeVar.upper_bound), t) + for arg in t.args: + arg.accept(self) + + def check_type_var_values(self, type: TypeInfo, actuals: List[Type], + valids: List[Type], context: Context) -> None: + for actual in actuals: + if (not isinstance(actual, AnyType) and + not any(is_same_type(actual, value) for value in valids)): + self.fail('Invalid type argument value for "{}"'.format( + type.name()), context) + + def visit_callable_type(self, t: CallableType) -> None: + t.ret_type.accept(self) + for arg_type in t.arg_types: + arg_type.accept(self) + + def visit_tuple_type(self, t: TupleType) -> None: + for item in t.items: + item.accept(self) + + def visit_union_type(self, t: UnionType) -> None: + for item in t.items: + item.accept(self) + + def visit_star_type(self, t: StarType) -> None: + t.type.accept(self) + + # Other kinds of type are trivial, since they are atomic (or invalid). + + def visit_unbound_type(self, t: UnboundType) -> None: + pass + + def visit_any(self, t: AnyType) -> None: + pass + + def visit_void(self, t: Void) -> None: + pass + + def visit_none_type(self, t: NoneTyp) -> None: + pass + + def visit_uninhabited_type(self, t: UninhabitedType) -> None: + pass + + def visit_deleted_type(self, t: DeletedType) -> None: + pass + + def visit_type_list(self, t: TypeList) -> None: + self.fail('Invalid type', t) + + def visit_type_var(self, t: TypeVarType) -> None: + pass + + def visit_partial_type(self, t: PartialType) -> None: + pass + + def visit_type_type(self, t: TypeType) -> None: + pass diff --git a/mypy/types.py b/mypy/types.py index 57bc7bd6b26f7..7d8dcb1c1a603 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1,1451 +1,1451 @@ -"""Classes for representing mypy types.""" - -from abc import abstractmethod -from typing import ( - Any, TypeVar, Dict, List, Tuple, cast, Generic, Set, Sequence, Optional, Union -) - -import mypy.nodes -from mypy.nodes import INVARIANT, SymbolNode - -from mypy import experiments - - -T = TypeVar('T') - -JsonDict = Dict[str, Any] - - -class Type(mypy.nodes.Context): - """Abstract base class for all types.""" - - line = 0 - - def __init__(self, line: int = -1) -> None: - self.line = line - - def get_line(self) -> int: - return self.line - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - raise RuntimeError('Not implemented') - - def __repr__(self) -> str: - return self.accept(TypeStrVisitor()) - - def serialize(self) -> JsonDict: - raise NotImplementedError('Cannot serialize {} instance'.format(self.__class__.__name__)) - - @classmethod - def deserialize(cls, data: JsonDict) -> 'Type': - classname = data['.class'] - glo = globals() - if classname in glo: - cl = glo[classname] - if 'deserialize' in cl.__dict__: - return cl.deserialize(data) - raise NotImplementedError('unexpected .class {}'.format(classname)) - - -class TypeVarId: - # A type variable is uniquely identified by its raw id and meta level. - - # For plain variables (type parameters of generic classes and - # functions) raw ids are allocated by semantic analysis, using - # positive ids 1, 2, ... for generic class parameters and negative - # ids -1, ... for generic function type arguments. This convention - # is only used to keep type variable ids distinct when allocating - # them; the type checker makes no distinction between class and - # function type variables. - - # Metavariables are allocated unique ids starting from 1. - raw_id = 0 # type: int - - # Level of the variable in type inference. Currently either 0 for - # declared types, or 1 for type inference metavariables. - meta_level = 0 # type: int - - # Class variable used for allocating fresh ids for metavariables. - next_raw_id = 1 # type: int - - def __init__(self, raw_id: int, meta_level: int = 0) -> None: - self.raw_id = raw_id - self.meta_level = meta_level - - @staticmethod - def new(meta_level: int) -> 'TypeVarId': - raw_id = TypeVarId.next_raw_id - TypeVarId.next_raw_id += 1 - return TypeVarId(raw_id, meta_level) - - def __repr__(self) -> str: - return self.raw_id.__repr__() - - def __eq__(self, other: object) -> bool: - if isinstance(other, TypeVarId): - return (self.raw_id == other.raw_id and - self.meta_level == other.meta_level) - else: - return False - - def __ne__(self, other: object) -> bool: - return not (self == other) - - def __hash__(self) -> int: - return hash((self.raw_id, self.meta_level)) - - def is_meta_var(self) -> bool: - return self.meta_level > 0 - - -class TypeVarDef(mypy.nodes.Context): - """Definition of a single type variable.""" - - name = '' - id = None # type: TypeVarId - values = None # type: List[Type] # Value restriction, empty list if no restriction - upper_bound = None # type: Type - variance = INVARIANT # type: int - line = 0 - - def __init__(self, name: str, id: Union[TypeVarId, int], values: Optional[List[Type]], - upper_bound: Type, variance: int = INVARIANT, line: int = -1) -> None: - self.name = name - if isinstance(id, int): - id = TypeVarId(id) - self.id = id - self.values = values - self.upper_bound = upper_bound - self.variance = variance - self.line = line - - @staticmethod - def new_unification_variable(old: 'TypeVarDef') -> 'TypeVarDef': - new_id = TypeVarId.new(meta_level=1) - return TypeVarDef(old.name, new_id, old.values, - old.upper_bound, old.variance, old.line) - - def get_line(self) -> int: - return self.line - - def __repr__(self) -> str: - if self.values: - return '{} in {}'.format(self.name, tuple(self.values)) - elif not is_named_instance(self.upper_bound, 'builtins.object'): - return '{} <: {}'.format(self.name, self.upper_bound) - else: - return self.name - - def serialize(self) -> JsonDict: - assert not self.id.is_meta_var() - return {'.class': 'TypeVarDef', - 'name': self.name, - 'id': self.id.raw_id, - 'values': None if self.values is None else [v.serialize() for v in self.values], - 'upper_bound': self.upper_bound.serialize(), - 'variance': self.variance, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'TypeVarDef': - assert data['.class'] == 'TypeVarDef' - return TypeVarDef(data['name'], - data['id'], - None if data['values'] is None - else [Type.deserialize(v) for v in data['values']], - Type.deserialize(data['upper_bound']), - data['variance'], - ) - - -class UnboundType(Type): - """Instance type that has not been bound during semantic analysis.""" - - name = '' - args = None # type: List[Type] - # should this type be wrapped in an Optional? - optional = False - # is this type a return type? - is_ret_type = False - - def __init__(self, - name: str, - args: List[Type] = None, - line: int = -1, - optional: bool = False, - is_ret_type: bool = False) -> None: - if not args: - args = [] - self.name = name - self.args = args - self.optional = optional - self.is_ret_type = is_ret_type - super().__init__(line) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_unbound_type(self) - - def serialize(self) -> JsonDict: - return {'.class': 'UnboundType', - 'name': self.name, - 'args': [a.serialize() for a in self.args], - } - - @classmethod - def deserialize(self, data: JsonDict) -> 'UnboundType': - assert data['.class'] == 'UnboundType' - return UnboundType(data['name'], - [Type.deserialize(a) for a in data['args']]) - - -class ErrorType(Type): - """The error type is used as the result of failed type operations.""" - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_error_type(self) - - -class TypeList(Type): - """A list of types [...]. - - This is only used for the arguments of a Callable type, i.e. for - [arg, ...] in Callable[[arg, ...], ret]. This is not a real type - but a syntactic AST construct. - """ - - items = None # type: List[Type] - - def __init__(self, items: List[Type], line: int = -1) -> None: - super().__init__(line) - self.items = items - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_type_list(self) - - def serialize(self) -> JsonDict: - return {'.class': 'TypeList', - 'items': [t.serialize() for t in self.items], - } - - @classmethod - def deserialize(self, data: JsonDict) -> 'TypeList': - assert data['.class'] == 'TypeList' - return TypeList([Type.deserialize(t) for t in data['items']]) - - -class AnyType(Type): - """The type 'Any'.""" - - def __init__(self, implicit=False, line: int = -1) -> None: - super().__init__(line) - self.implicit = implicit - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_any(self) - - def serialize(self) -> JsonDict: - return {'.class': 'AnyType'} - - @classmethod - def deserialize(cls, data: JsonDict) -> 'AnyType': - assert data['.class'] == 'AnyType' - return AnyType() - - -class Void(Type): - """The return type 'None'. - - This can only be used as the return type in a callable type and as - the result type of calling such callable. - """ - - source = '' # May be None; function that generated this value - - def __init__(self, source: str = None, line: int = -1) -> None: - self.source = source - super().__init__(line) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_void(self) - - def with_source(self, source: str) -> 'Void': - return Void(source, self.line) - - def serialize(self) -> JsonDict: - return {'.class': 'Void'} - - @classmethod - def deserialize(cls, data: JsonDict) -> 'Void': - assert data['.class'] == 'Void' - return Void() - - -class UninhabitedType(Type): - """This type has no members. - - This type is almost the bottom type, except it is not a subtype of Void. - With strict Optional checking, it is the only common subtype between all - other types, which allows `meet` to be well defined. Without strict - Optional checking, NoneTyp fills this role. - - In general, for any type T that isn't Void: - join(UninhabitedType, T) = T - meet(UninhabitedType, T) = UninhabitedType - is_subtype(UninhabitedType, T) = True - """ - - def __init__(self, line: int = -1) -> None: - super().__init__(line) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_uninhabited_type(self) - - def serialize(self) -> JsonDict: - return {'.class': 'UninhabitedType'} - - @classmethod - def deserialize(cls, data: JsonDict) -> 'UninhabitedType': - assert data['.class'] == 'UninhabitedType' - return UninhabitedType() - - -class NoneTyp(Type): - """The type of 'None'. - - Without strict Optional checking: - This is only used internally during type inference. Programs - cannot declare a variable of this type, and the type checker - refuses to infer this type for a variable. However, subexpressions - often have this type. Note that this is not used as the result - type when calling a function with a void type, even though - semantically such a function returns a None value; the void type - is used instead so that we can report an error if the caller tries - to do anything with the return value. - - With strict Optional checking: - This type can be written by users as 'None', except as the return value - of a function, where 'None' means Void. - """ - - def __init__(self, line: int = -1) -> None: - super().__init__(line) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_none_type(self) - - def serialize(self) -> JsonDict: - return {'.class': 'NoneTyp'} - - @classmethod - def deserialize(self, data: JsonDict) -> 'NoneTyp': - assert data['.class'] == 'NoneTyp' - return NoneTyp() - - -class ErasedType(Type): - """Placeholder for an erased type. - - This is used during type inference. This has the special property that - it is ignored during type inference. - """ - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_erased_type(self) - - -class DeletedType(Type): - """Type of deleted variables. - - These can be used as lvalues but not rvalues. - """ - - source = '' # May be None; name that generated this value - - def __init__(self, source: str = None, line: int = -1) -> None: - self.source = source - super().__init__(line) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_deleted_type(self) - - def serialize(self) -> JsonDict: - return {'.class': 'DeletedType', - 'source': self.source} - - @classmethod - def deserialize(self, data: JsonDict) -> 'DeletedType': - assert data['.class'] == 'DeletedType' - return DeletedType(data['source']) - - -class Instance(Type): - """An instance type of form C[T1, ..., Tn]. - - The list of type variables may be empty. - """ - - type = None # type: mypy.nodes.TypeInfo - args = None # type: List[Type] - erased = False # True if result of type variable substitution - - def __init__(self, typ: mypy.nodes.TypeInfo, args: List[Type], - line: int = -1, erased: bool = False) -> None: - self.type = typ - self.args = args - self.erased = erased - super().__init__(line) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_instance(self) - - type_ref = None # type: str - - def serialize(self) -> JsonDict: - data = {'.class': 'Instance', - } # type: JsonDict - assert self.type is not None - data['type_ref'] = self.type.alt_fullname or self.type.fullname() - if self.args: - data['args'] = [arg.serialize() for arg in self.args] - return data - - @classmethod - def deserialize(cls, data: JsonDict) -> 'Instance': - assert data['.class'] == 'Instance' - args = [] # type: List[Type] - if 'args' in data: - args_list = data['args'] - assert isinstance(args_list, list) - args = [Type.deserialize(arg) for arg in args_list] - inst = Instance(None, args) - inst.type_ref = data['type_ref'] # Will be fixed up by fixup.py later. - return inst - - -class TypeVarType(Type): - """A type variable type. - - This refers to either a class type variable (id > 0) or a function - type variable (id < 0). - """ - - name = '' # Name of the type variable (for messages and debugging) - id = None # type: TypeVarId - values = None # type: List[Type] # Value restriction, empty list if no restriction - upper_bound = None # type: Type # Upper bound for values - # See comments in TypeVarDef for more about variance. - variance = INVARIANT # type: int - - def __init__(self, binder: TypeVarDef, line: int = -1) -> None: - self.name = binder.name - self.id = binder.id - self.values = binder.values - self.upper_bound = binder.upper_bound - self.variance = binder.variance - super().__init__(line) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_type_var(self) - - def erase_to_union_or_bound(self) -> Type: - if self.values: - return UnionType.make_simplified_union(self.values) - else: - return self.upper_bound - - def serialize(self) -> JsonDict: - assert not self.id.is_meta_var() - return {'.class': 'TypeVarType', - 'name': self.name, - 'id': self.id.raw_id, - 'values': [v.serialize() for v in self.values], - 'upper_bound': self.upper_bound.serialize(), - 'variance': self.variance, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'TypeVarType': - assert data['.class'] == 'TypeVarType' - tvdef = TypeVarDef(data['name'], - data['id'], - [Type.deserialize(v) for v in data['values']], - Type.deserialize(data['upper_bound']), - data['variance']) - return TypeVarType(tvdef) - - -class FunctionLike(Type): - """Abstract base class for function types.""" - - @abstractmethod - def is_type_obj(self) -> bool: pass - - def is_concrete_type_obj(self) -> bool: - return self.is_type_obj() - - @abstractmethod - def type_object(self) -> mypy.nodes.TypeInfo: pass - - @abstractmethod - def items(self) -> List['CallableType']: pass - - @abstractmethod - def with_name(self, name: str) -> 'FunctionLike': pass - - # Corresponding instance type (e.g. builtins.type) - fallback = None # type: Instance - - @classmethod - def deserialize(cls, data: JsonDict) -> 'FunctionLike': - return cast(FunctionLike, super().deserialize(data)) - - -_dummy = object() # type: Any - - -class CallableType(FunctionLike): - """Type of a non-overloaded callable object (function).""" - - arg_types = None # type: List[Type] # Types of function arguments - arg_kinds = None # type: List[int] # mypy.nodes.ARG_ constants - arg_names = None # type: List[str] # None if not a keyword argument - min_args = 0 # Minimum number of arguments; derived from arg_kinds - is_var_arg = False # Is it a varargs function? derived from arg_kinds - ret_type = None # type: Type # Return value type - name = '' # Name (may be None; for error messages) - definition = None # type: SymbolNode # For error messages. May be None. - # Type variables for a generic function - variables = None # type: List[TypeVarDef] - - # Is this Callable[..., t] (with literal '...')? - is_ellipsis_args = False - # Is this callable constructed for the benefit of a classmethod's 'cls' argument? - is_classmethod_class = False - # Was this type implicitly generated instead of explicitly specified by the user? - implicit = False - # Defined for signatures that require special handling (currently only value is 'dict' - # for a signature similar to 'dict') - special_sig = None # type: Optional[str] - - def __init__(self, - arg_types: List[Type], - arg_kinds: List[int], - arg_names: List[str], - ret_type: Type, - fallback: Instance, - name: str = None, - definition: SymbolNode = None, - variables: List[TypeVarDef] = None, - line: int = -1, - is_ellipsis_args: bool = False, - implicit=False, - is_classmethod_class=False, - special_sig=None, - ) -> None: - if variables is None: - variables = [] - self.arg_types = arg_types - self.arg_kinds = arg_kinds - self.arg_names = arg_names - self.min_args = arg_kinds.count(mypy.nodes.ARG_POS) - self.is_var_arg = mypy.nodes.ARG_STAR in arg_kinds - self.ret_type = ret_type - self.fallback = fallback - assert not name or ' 'CallableType': - return CallableType( - arg_types=arg_types if arg_types is not _dummy else self.arg_types, - arg_kinds=arg_kinds if arg_kinds is not _dummy else self.arg_kinds, - arg_names=arg_names if arg_names is not _dummy else self.arg_names, - ret_type=ret_type if ret_type is not _dummy else self.ret_type, - fallback=fallback if fallback is not _dummy else self.fallback, - name=name if name is not _dummy else self.name, - definition=definition if definition is not _dummy else self.definition, - variables=variables if variables is not _dummy else self.variables, - line=line if line is not _dummy else self.line, - is_ellipsis_args=( - is_ellipsis_args if is_ellipsis_args is not _dummy else self.is_ellipsis_args), - implicit=self.implicit, - is_classmethod_class=self.is_classmethod_class, - special_sig=special_sig if special_sig is not _dummy else self.special_sig, - ) - - def is_type_obj(self) -> bool: - return self.fallback.type is not None and self.fallback.type.fullname() == 'builtins.type' - - def is_concrete_type_obj(self) -> bool: - return self.is_type_obj() and self.is_classmethod_class - - def type_object(self) -> mypy.nodes.TypeInfo: - assert self.is_type_obj() - ret = self.ret_type - if isinstance(ret, TupleType): - ret = ret.fallback - return cast(Instance, ret).type - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_callable_type(self) - - def with_name(self, name: str) -> 'CallableType': - """Return a copy of this type with the specified name.""" - ret = self.ret_type - if isinstance(ret, Void): - ret = ret.with_source(name) - return self.copy_modified(ret_type=ret, name=name) - - def max_fixed_args(self) -> int: - n = len(self.arg_types) - if self.is_var_arg: - n -= 1 - return n - - def items(self) -> List['CallableType']: - return [self] - - def is_generic(self) -> bool: - return bool(self.variables) - - def type_var_ids(self) -> List[TypeVarId]: - a = [] # type: List[TypeVarId] - for tv in self.variables: - a.append(tv.id) - return a - - def serialize(self) -> JsonDict: - # TODO: As an optimization, leave out everything related to - # generic functions for non-generic functions. - return {'.class': 'CallableType', - 'arg_types': [(None if t is None else t.serialize()) - for t in self.arg_types], - 'arg_kinds': self.arg_kinds, - 'arg_names': self.arg_names, - 'ret_type': self.ret_type.serialize(), - 'fallback': self.fallback.serialize(), - 'name': self.name, - # We don't serialize the definition (only used for error messages). - 'variables': [v.serialize() for v in self.variables], - 'is_ellipsis_args': self.is_ellipsis_args, - 'implicit': self.implicit, - 'is_classmethod_class': self.is_classmethod_class, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'CallableType': - assert data['.class'] == 'CallableType' - # TODO: Set definition to the containing SymbolNode? - return CallableType([(None if t is None else Type.deserialize(t)) - for t in data['arg_types']], - data['arg_kinds'], - data['arg_names'], - Type.deserialize(data['ret_type']), - Instance.deserialize(data['fallback']), - name=data['name'], - variables=[TypeVarDef.deserialize(v) for v in data['variables']], - is_ellipsis_args=data['is_ellipsis_args'], - implicit=data['implicit'], - is_classmethod_class=data['is_classmethod_class'], - ) - - -class Overloaded(FunctionLike): - """Overloaded function type T1, ... Tn, where each Ti is CallableType. - - The variant to call is chosen based on static argument - types. Overloaded function types can only be defined in stub - files, and thus there is no explicit runtime dispatch - implementation. - """ - - _items = None # type: List[CallableType] # Must not be empty - - def __init__(self, items: List[CallableType]) -> None: - self._items = items - self.fallback = items[0].fallback - super().__init__(items[0].line) - - def items(self) -> List[CallableType]: - return self._items - - def name(self) -> str: - return self._items[0].name - - def is_type_obj(self) -> bool: - # All the items must have the same type object status, so it's - # sufficient to query only (any) one of them. - return self._items[0].is_type_obj() - - def type_object(self) -> mypy.nodes.TypeInfo: - # All the items must have the same type object, so it's sufficient to - # query only (any) one of them. - return self._items[0].type_object() - - def with_name(self, name: str) -> 'Overloaded': - ni = [] # type: List[CallableType] - for it in self._items: - ni.append(it.with_name(name)) - return Overloaded(ni) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_overloaded(self) - - def serialize(self) -> JsonDict: - return {'.class': 'Overloaded', - 'items': [t.serialize() for t in self.items()], - } - - @classmethod - def deserialize(self, data: JsonDict) -> 'Overloaded': - assert data['.class'] == 'Overloaded' - return Overloaded([CallableType.deserialize(t) for t in data['items']]) - - -class TupleType(Type): - """The tuple type Tuple[T1, ..., Tn] (at least one type argument). - - Instance variables: - items: tuple item types - fallback: the underlying instance type that is used for non-tuple methods - (this is currently always builtins.tuple, but it could be different for named - tuples, for example) - implicit: if True, derived from a tuple expression (t,....) instead of Tuple[t, ...] - """ - - items = None # type: List[Type] - fallback = None # type: Instance - implicit = False - - def __init__(self, items: List[Type], fallback: Instance, line: int = -1, - implicit: bool = False) -> None: - #import pdb; pdb.set_trace() - self.items = items - self.fallback = fallback - self.implicit = implicit - super().__init__(line) - - def length(self) -> int: - return len(self.items) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_tuple_type(self) - - def serialize(self) -> JsonDict: - return {'.class': 'TupleType', - 'items': [t.serialize() for t in self.items], - 'fallback': self.fallback.serialize(), - 'implicit': self.implicit, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'TupleType': - assert data['.class'] == 'TupleType' - return TupleType([Type.deserialize(t) for t in data['items']], - Instance.deserialize(data['fallback']), - implicit=data['implicit']) - - def self_type(self, inst: Instance) -> 'TupleType': - return TupleType(self.items, inst, self.line, self.implicit) - - def with_types(self, items: List[Type]) -> 'TupleType': - return TupleType(items, self.fallback, self.line, self.implicit) - - def slice(self, begin: int, stride: int, end: int) -> 'TupleType': - return TupleType(self.items[begin:end:stride], self.fallback, - self.line, self.implicit) - - -class NamedTupleType(TupleType): - attrs = None # type: List[str] - name = None # type: str - - def __init__(self, name: str, attrs: List[str], *args, **kwargs) -> None: - self.attrs = attrs - self.name = name - super().__init__(*args, **kwargs) - - def self_type(self, inst: Instance) -> 'NamedTupleType': - return NamedTupleType(self.name, self.attrs, self.items, inst, self.line) - - def with_types(self, items: List[Type]) -> 'NamedTupleType': - return NamedTupleType(self.name, self.attrs, items, - self.fallback, self.line, self.implicit) - - def __eq__(self, other) -> bool: - return self.serialize() == other.serialize() - - def serialize(self) -> JsonDict: - d = super().serialize() - d.update({'.class': 'NamedTupleType', - 'name': self.name, - 'attrs': self.attrs}) - return d - - @classmethod - def deserialize(cls, data: JsonDict) -> 'NamedTupleType': - assert data['.class'] == 'NamedTupleType' - data['.class'] = 'TupleType' - tup = TupleType.deserialize(data) - return NamedTupleType(data['name'], - data['attrs'], - tup.items, - tup.fallback, - implicit=tup.implicit) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - try: - return visitor.visit_namedtuple_type(self) - except: - return super().accept(visitor) - - -class StarType(Type): - """The star type *type_parameter. - - This is not a real type but a syntactic AST construct. - """ - - type = None # type: Type - - def __init__(self, type: Type, line: int = -1) -> None: - self.type = type - super().__init__(line) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_star_type(self) - - -class UnionType(Type): - """The union type Union[T1, ..., Tn] (at least one type argument).""" - - items = None # type: List[Type] - - def __init__(self, items: List[Type], line: int = -1) -> None: - self.items = items - super().__init__(line) - - @staticmethod - def make_union(items: List[Type], line: int = -1) -> Type: - if len(items) > 1: - return UnionType(items, line) - elif len(items) == 1: - return items[0] - else: - if experiments.STRICT_OPTIONAL: - return UninhabitedType() - else: - return Void() - - @staticmethod - def make_simplified_union(items: List[Type], line: int = -1) -> Type: - while any(isinstance(typ, UnionType) for typ in items): - all_items = [] # type: List[Type] - for typ in items: - if isinstance(typ, UnionType): - all_items.extend(typ.items) - else: - all_items.append(typ) - items = all_items - - if any(isinstance(typ, AnyType) for typ in items): - return AnyType() - - from mypy.subtypes import is_subtype - removed = set() # type: Set[int] - for i in range(len(items)): - if any(is_subtype(items[i], items[j]) for j in range(len(items)) - if j not in removed and j != i): - removed.add(i) - - simplified_set = [items[i] for i in range(len(items)) if i not in removed] - return UnionType.make_union(simplified_set) - - def length(self) -> int: - return len(self.items) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_union_type(self) - - def has_readable_member(self, name: str) -> bool: - """For a tree of unions of instances, check whether all instances have a given member. - - TODO: Deal with attributes of TupleType etc. - TODO: This should probably be refactored to go elsewhere. - """ - return all((isinstance(x, UnionType) and x.has_readable_member(name)) or - (isinstance(x, Instance) and x.type.has_readable_member(name)) - for x in self.items) - - def serialize(self) -> JsonDict: - return {'.class': 'UnionType', - 'items': [t.serialize() for t in self.items], - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'UnionType': - assert data['.class'] == 'UnionType' - return UnionType([Type.deserialize(t) for t in data['items']]) - - -class PartialType(Type): - """Type such as List[?] where type arguments are unknown, or partial None type. - - These are used for inferring types in multiphase initialization such as this: - - x = [] # x gets a partial type List[?], as item type is unknown - x.append(1) # partial type gets replaced with normal type List[int] - - Or with None: - - x = None # x gets a partial type None - if c: - x = 1 # Infer actual type int for x - """ - - # None for the 'None' partial type; otherwise a generic class - type = None # type: Optional[mypy.nodes.TypeInfo] - var = None # type: mypy.nodes.Var - inner_types = None # type: List[Type] - - def __init__(self, - type: Optional['mypy.nodes.TypeInfo'], - var: 'mypy.nodes.Var', - inner_types: List[Type]) -> None: - self.type = type - self.var = var - self.inner_types = inner_types - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_partial_type(self) - - -class EllipsisType(Type): - """The type ... (ellipsis). - - This is not a real type but a syntactic AST construct, used in Callable[..., T], for example. - - A semantically analyzed type will never have ellipsis types. - """ - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_ellipsis_type(self) - - def serialize(self) -> JsonDict: - return {'.class': 'EllipsisType'} - - @classmethod - def deserialize(self, data: JsonDict) -> 'EllipsisType': - assert data['.class'] == 'EllipsisType' - return EllipsisType() - - -class TypeType(Type): - """For types like Type[User]. - - This annotates variables that are class objects, constrained by - the type argument. See PEP 484 for more details. - - We may encounter expressions whose values are specific classes; - those are represented as callables (possibly overloaded) - corresponding to the class's constructor's signature and returning - an instance of that class. The difference with Type[C] is that - those callables always represent the exact class given as the - return type; Type[C] represents any class that's a subclass of C, - and C may also be a type variable or a union (or Any). - - Many questions around subtype relationships between Type[C1] and - def(...) -> C2 are answered by looking at the subtype - relationships between C1 and C2, since Type[] is considered - covariant. - - There's an unsolved problem with constructor signatures (also - unsolved in PEP 484): calling a variable whose type is Type[C] - assumes the constructor signature for C, even though a subclass of - C might completely change the constructor signature. For now we - just assume that users of Type[C] are careful not to do that (in - the future we might detect when they are violating that - assumption). - """ - - # This can't be everything, but it can be a class reference, - # a generic class instance, a union, Any, a type variable... - item = None # type: Type - - def __init__(self, item: Type, *, line: int = -1) -> None: - super().__init__(line) - self.item = item - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_type_type(self) - - def serialize(self) -> JsonDict: - return {'.class': 'TypeType', 'item': self.item.serialize()} - - def deserialize(self, data: JsonDict) -> 'TypeType': - assert data['.class'] == 'TypeType' - return TypeType(Type.deserialize(data['item'])) - - -# -# Visitor-related classes -# - - -class TypeVisitor(Generic[T]): - """Visitor class for types (Type subclasses). - - The parameter T is the return type of the visit methods. - """ - - def _notimplemented_helper(self, name: str) -> NotImplementedError: - return NotImplementedError("Method {}.visit_{}() not implemented\n" - .format(type(self).__name__, name) - + "This is a known bug, track development in " - + "'https://github.com/JukkaL/mypy/issues/730'") - - @abstractmethod - def visit_unbound_type(self, t: UnboundType) -> T: - pass - - def visit_type_list(self, t: TypeList) -> T: - raise self._notimplemented_helper('type_list') - - def visit_error_type(self, t: ErrorType) -> T: - raise self._notimplemented_helper('error_type') - - @abstractmethod - def visit_any(self, t: AnyType) -> T: - pass - - @abstractmethod - def visit_void(self, t: Void) -> T: - pass - - @abstractmethod - def visit_none_type(self, t: NoneTyp) -> T: - pass - - @abstractmethod - def visit_uninhabited_type(self, t: UninhabitedType) -> T: - pass - - def visit_erased_type(self, t: ErasedType) -> T: - raise self._notimplemented_helper('erased_type') - - @abstractmethod - def visit_deleted_type(self, t: DeletedType) -> T: - pass - - @abstractmethod - def visit_type_var(self, t: TypeVarType) -> T: - pass - - @abstractmethod - def visit_instance(self, t: Instance) -> T: - pass - - @abstractmethod - def visit_callable_type(self, t: CallableType) -> T: - pass - - def visit_overloaded(self, t: Overloaded) -> T: - raise self._notimplemented_helper('overloaded') - - @abstractmethod - def visit_tuple_type(self, t: TupleType) -> T: - pass - - def visit_star_type(self, t: StarType) -> T: - raise self._notimplemented_helper('star_type') - - @abstractmethod - def visit_union_type(self, t: UnionType) -> T: - pass - - @abstractmethod - def visit_partial_type(self, t: PartialType) -> T: - pass - - def visit_ellipsis_type(self, t: EllipsisType) -> T: - raise self._notimplemented_helper('ellipsis_type') - - @abstractmethod - def visit_type_type(self, t: TypeType) -> T: - pass - - -class TypeTranslator(TypeVisitor[Type]): - """Identity type transformation. - - Subclass this and override some methods to implement a non-trivial - transformation. - """ - - def visit_unbound_type(self, t: UnboundType) -> Type: - return t - - def visit_type_list(self, t: TypeList) -> Type: - return t - - def visit_error_type(self, t: ErrorType) -> Type: - return t - - def visit_any(self, t: AnyType) -> Type: - return t - - def visit_void(self, t: Void) -> Type: - return t - - def visit_none_type(self, t: NoneTyp) -> Type: - return t - - def visit_uninhabited_type(self, t: UninhabitedType) -> Type: - return t - - def visit_erased_type(self, t: ErasedType) -> Type: - return t - - def visit_deleted_type(self, t: DeletedType) -> Type: - return t - - def visit_instance(self, t: Instance) -> Type: - return Instance(t.type, self.translate_types(t.args), t.line) - - def visit_type_var(self, t: TypeVarType) -> Type: - return t - - def visit_partial_type(self, t: PartialType) -> Type: - return t - - def visit_callable_type(self, t: CallableType) -> Type: - return t.copy_modified(arg_types=self.translate_types(t.arg_types), - ret_type=t.ret_type.accept(self), - variables=self.translate_variables(t.variables)) - - def visit_tuple_type(self, t: TupleType) -> Type: - return TupleType(self.translate_types(t.items), - cast(Any, t.fallback.accept(self)), - t.line) - - def visit_star_type(self, t: StarType) -> Type: - return StarType(t.type.accept(self), t.line) - - def visit_union_type(self, t: UnionType) -> Type: - return UnionType(self.translate_types(t.items), t.line) - - def visit_ellipsis_type(self, t: EllipsisType) -> Type: - return t - - def translate_types(self, types: List[Type]) -> List[Type]: - return [t.accept(self) for t in types] - - def translate_variables(self, - variables: List[TypeVarDef]) -> List[TypeVarDef]: - return variables - - def visit_overloaded(self, t: Overloaded) -> Type: - items = [] # type: List[CallableType] - for item in t.items(): - new = item.accept(self) - if isinstance(new, CallableType): - items.append(new) - else: - raise RuntimeError('CallableType expectected, but got {}'.format(type(new))) - return Overloaded(items=items) - - def visit_type_type(self, t: TypeType) -> Type: - return TypeType(t.item.accept(self), line=t.line) - - -class TypeStrVisitor(TypeVisitor[str]): - """Visitor for pretty-printing types into strings. - - This is mostly for debugging/testing. - - Do not preserve original formatting. - - Notes: - - Represent unbound types as Foo? or Foo?[...]. - - Represent the NoneTyp type as None. - """ - - def visit_unbound_type(self, t): - s = t.name + '?' - if t.args != []: - s += '[{}]'.format(self.list_str(t.args)) - return s - - def visit_type_list(self, t): - return ''.format(self.list_str(t.items)) - - def visit_error_type(self, t): - return '' - - def visit_any(self, t): - return 'Any' - - def visit_void(self, t): - return 'void' - - def visit_none_type(self, t): - # Fully qualify to make this distinct from the None value. - return "builtins.None" - - def visit_uninhabited_type(self, t): - return "" - - def visit_erased_type(self, t): - return "" - - def visit_deleted_type(self, t): - if t.source is None: - return "" - else: - return "".format(t.source) - - def visit_instance(self, t): - s = t.type.fullname() if t.type is not None else '' - if t.erased: - s += '*' - if t.args != []: - s += '[{}]'.format(self.list_str(t.args)) - return s - - def visit_type_var(self, t): - if t.name is None: - # Anonymous type variable type (only numeric id). - return '`{}'.format(t.id) - else: - # Named type variable type. - return '{}`{}'.format(t.name, t.id) - - def visit_callable_type(self, t): - s = '' - bare_asterisk = False - for i in range(len(t.arg_types)): - if s != '': - s += ', ' - if t.arg_kinds[i] == mypy.nodes.ARG_NAMED and not bare_asterisk: - s += '*, ' - bare_asterisk = True - if t.arg_kinds[i] == mypy.nodes.ARG_STAR: - s += '*' - if t.arg_kinds[i] == mypy.nodes.ARG_STAR2: - s += '**' - if t.arg_names[i]: - s += t.arg_names[i] + ': ' - s += str(t.arg_types[i]) - if t.arg_kinds[i] == mypy.nodes.ARG_OPT: - s += ' =' - - s = '({})'.format(s) - - if not isinstance(t.ret_type, Void): - s += ' -> {}'.format(t.ret_type) - - if t.variables: - s = '{} {}'.format(t.variables, s) - - return 'def {}'.format(s) - - def visit_overloaded(self, t): - a = [] - for i in t.items(): - a.append(i.accept(self)) - return 'Overload({})'.format(', '.join(a)) - - def visit_tuple_type(self, t): - s = self.list_str(t.items) - if t.fallback and t.fallback.type: - fallback_name = t.fallback.type.fullname() - if fallback_name != 'builtins.tuple': - return 'Tuple[{}, fallback={}]'.format(s, t.fallback.accept(self)) - return 'Tuple[{}]'.format(s) - - def visit_namedtuple_type(self, t): - s = self.list_str(zip(t.attrs, t.items)) - return '{}[{}]'.format(t.name, s) - - def visit_star_type(self, t): - s = t.type.accept(self) - return '*{}'.format(s) - - def visit_union_type(self, t): - s = self.list_str(t.items) - return 'Union[{}]'.format(s) - - def visit_partial_type(self, t: PartialType) -> str: - if t.type is None: - return '' - else: - return ''.format(t.type.name(), - ', '.join(['?'] * len(t.type.type_vars))) - - def visit_ellipsis_type(self, t): - return '...' - - def visit_type_type(self, t): - return 'Type[{}]'.format(t.item.accept(self)) - - def list_str(self, a): - """Convert items of an array to strings (pretty-print types) - and join the results with commas. - """ - res = [] - for t in a: - if isinstance(t, Type): - res.append(t.accept(self)) - else: - res.append(str(t)) - return ', '.join(res) - - -# These constants define the method used by TypeQuery to combine multiple -# query results, e.g. for tuple types. The strategy is not used for empty -# result lists; in that case the default value takes precedence. -ANY_TYPE_STRATEGY = 0 # Return True if any of the results are True. -ALL_TYPES_STRATEGY = 1 # Return True if all of the results are True. - - -class TypeQuery(TypeVisitor[bool]): - """Visitor for performing simple boolean queries of types. - - This class allows defining the default value for leafs to simplify the - implementation of many queries. - """ - - default = False # Default result - strategy = 0 # Strategy for combining multiple values (ANY_TYPE_STRATEGY or ALL_TYPES_...). - - def __init__(self, default: bool, strategy: int) -> None: - """Construct a query visitor. - - Use the given default result and strategy for combining - multiple results. The strategy must be either - ANY_TYPE_STRATEGY or ALL_TYPES_STRATEGY. - """ - self.default = default - self.strategy = strategy - - def visit_unbound_type(self, t: UnboundType) -> bool: - return self.default - - def visit_type_list(self, t: TypeList) -> bool: - return self.default - - def visit_error_type(self, t: ErrorType) -> bool: - return self.default - - def visit_any(self, t: AnyType) -> bool: - return self.default - - def visit_void(self, t: Void) -> bool: - return self.default - - def visit_uninhabited_type(self, t: UninhabitedType) -> bool: - return self.default - - def visit_none_type(self, t: NoneTyp) -> bool: - return self.default - - def visit_erased_type(self, t: ErasedType) -> bool: - return self.default - - def visit_deleted_type(self, t: DeletedType) -> bool: - return self.default - - def visit_type_var(self, t: TypeVarType) -> bool: - return self.default - - def visit_partial_type(self, t: PartialType) -> bool: - return self.default - - def visit_instance(self, t: Instance) -> bool: - return self.query_types(t.args) - - def visit_callable_type(self, t: CallableType) -> bool: - # FIX generics - return self.query_types(t.arg_types + [t.ret_type]) - - def visit_tuple_type(self, t: TupleType) -> bool: - return self.query_types(t.items) - - def visit_star_type(self, t: StarType) -> bool: - return t.type.accept(self) - - def visit_union_type(self, t: UnionType) -> bool: - return self.query_types(t.items) - - def visit_overloaded(self, t: Overloaded) -> bool: - return self.query_types(t.items()) - - def visit_type_type(self, t: TypeType) -> bool: - return t.item.accept(self) - - def query_types(self, types: Sequence[Type]) -> bool: - """Perform a query for a list of types. - - Use the strategy constant to combine the results. - """ - if not types: - # Use default result for empty list. - return self.default - if self.strategy == ANY_TYPE_STRATEGY: - # Return True if at least one component is true. - res = False - for t in types: - res = res or t.accept(self) - if res: - break - return res - else: - # Return True if all components are true. - res = True - for t in types: - res = res and t.accept(self) - if not res: - break - return res - - -def strip_type(typ: Type) -> Type: - """Make a copy of type without 'debugging info' (function name).""" - - if isinstance(typ, CallableType): - return typ.copy_modified(name=None) - elif isinstance(typ, Overloaded): - return Overloaded([cast(CallableType, strip_type(item)) - for item in typ.items()]) - else: - return typ - - -def replace_leading_arg_type(t: CallableType, self_type: Type) -> CallableType: - """Return a copy of a callable type with a different self argument type. - - Assume that the callable is the signature of a method. - """ - return t.copy_modified(arg_types=[self_type] + t.arg_types[1:]) - - -def is_named_instance(t: Type, fullname: str) -> bool: - return (isinstance(t, Instance) and - t.type is not None and - t.type.fullname() == fullname) +"""Classes for representing mypy types.""" + +from abc import abstractmethod +from typing import ( + Any, TypeVar, Dict, List, Tuple, cast, Generic, Set, Sequence, Optional, Union +) + +import mypy.nodes +from mypy.nodes import INVARIANT, SymbolNode + +from mypy import experiments + + +T = TypeVar('T') + +JsonDict = Dict[str, Any] + + +class Type(mypy.nodes.Context): + """Abstract base class for all types.""" + + line = 0 + + def __init__(self, line: int = -1) -> None: + self.line = line + + def get_line(self) -> int: + return self.line + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + raise RuntimeError('Not implemented') + + def __repr__(self) -> str: + return self.accept(TypeStrVisitor()) + + def serialize(self) -> JsonDict: + raise NotImplementedError('Cannot serialize {} instance'.format(self.__class__.__name__)) + + @classmethod + def deserialize(cls, data: JsonDict) -> 'Type': + classname = data['.class'] + glo = globals() + if classname in glo: + cl = glo[classname] + if 'deserialize' in cl.__dict__: + return cl.deserialize(data) + raise NotImplementedError('unexpected .class {}'.format(classname)) + + +class TypeVarId: + # A type variable is uniquely identified by its raw id and meta level. + + # For plain variables (type parameters of generic classes and + # functions) raw ids are allocated by semantic analysis, using + # positive ids 1, 2, ... for generic class parameters and negative + # ids -1, ... for generic function type arguments. This convention + # is only used to keep type variable ids distinct when allocating + # them; the type checker makes no distinction between class and + # function type variables. + + # Metavariables are allocated unique ids starting from 1. + raw_id = 0 # type: int + + # Level of the variable in type inference. Currently either 0 for + # declared types, or 1 for type inference metavariables. + meta_level = 0 # type: int + + # Class variable used for allocating fresh ids for metavariables. + next_raw_id = 1 # type: int + + def __init__(self, raw_id: int, meta_level: int = 0) -> None: + self.raw_id = raw_id + self.meta_level = meta_level + + @staticmethod + def new(meta_level: int) -> 'TypeVarId': + raw_id = TypeVarId.next_raw_id + TypeVarId.next_raw_id += 1 + return TypeVarId(raw_id, meta_level) + + def __repr__(self) -> str: + return self.raw_id.__repr__() + + def __eq__(self, other: object) -> bool: + if isinstance(other, TypeVarId): + return (self.raw_id == other.raw_id and + self.meta_level == other.meta_level) + else: + return False + + def __ne__(self, other: object) -> bool: + return not (self == other) + + def __hash__(self) -> int: + return hash((self.raw_id, self.meta_level)) + + def is_meta_var(self) -> bool: + return self.meta_level > 0 + + +class TypeVarDef(mypy.nodes.Context): + """Definition of a single type variable.""" + + name = '' + id = None # type: TypeVarId + values = None # type: List[Type] # Value restriction, empty list if no restriction + upper_bound = None # type: Type + variance = INVARIANT # type: int + line = 0 + + def __init__(self, name: str, id: Union[TypeVarId, int], values: Optional[List[Type]], + upper_bound: Type, variance: int = INVARIANT, line: int = -1) -> None: + self.name = name + if isinstance(id, int): + id = TypeVarId(id) + self.id = id + self.values = values + self.upper_bound = upper_bound + self.variance = variance + self.line = line + + @staticmethod + def new_unification_variable(old: 'TypeVarDef') -> 'TypeVarDef': + new_id = TypeVarId.new(meta_level=1) + return TypeVarDef(old.name, new_id, old.values, + old.upper_bound, old.variance, old.line) + + def get_line(self) -> int: + return self.line + + def __repr__(self) -> str: + if self.values: + return '{} in {}'.format(self.name, tuple(self.values)) + elif not is_named_instance(self.upper_bound, 'builtins.object'): + return '{} <: {}'.format(self.name, self.upper_bound) + else: + return self.name + + def serialize(self) -> JsonDict: + assert not self.id.is_meta_var() + return {'.class': 'TypeVarDef', + 'name': self.name, + 'id': self.id.raw_id, + 'values': None if self.values is None else [v.serialize() for v in self.values], + 'upper_bound': self.upper_bound.serialize(), + 'variance': self.variance, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'TypeVarDef': + assert data['.class'] == 'TypeVarDef' + return TypeVarDef(data['name'], + data['id'], + None if data['values'] is None + else [Type.deserialize(v) for v in data['values']], + Type.deserialize(data['upper_bound']), + data['variance'], + ) + + +class UnboundType(Type): + """Instance type that has not been bound during semantic analysis.""" + + name = '' + args = None # type: List[Type] + # should this type be wrapped in an Optional? + optional = False + # is this type a return type? + is_ret_type = False + + def __init__(self, + name: str, + args: List[Type] = None, + line: int = -1, + optional: bool = False, + is_ret_type: bool = False) -> None: + if not args: + args = [] + self.name = name + self.args = args + self.optional = optional + self.is_ret_type = is_ret_type + super().__init__(line) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_unbound_type(self) + + def serialize(self) -> JsonDict: + return {'.class': 'UnboundType', + 'name': self.name, + 'args': [a.serialize() for a in self.args], + } + + @classmethod + def deserialize(self, data: JsonDict) -> 'UnboundType': + assert data['.class'] == 'UnboundType' + return UnboundType(data['name'], + [Type.deserialize(a) for a in data['args']]) + + +class ErrorType(Type): + """The error type is used as the result of failed type operations.""" + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_error_type(self) + + +class TypeList(Type): + """A list of types [...]. + + This is only used for the arguments of a Callable type, i.e. for + [arg, ...] in Callable[[arg, ...], ret]. This is not a real type + but a syntactic AST construct. + """ + + items = None # type: List[Type] + + def __init__(self, items: List[Type], line: int = -1) -> None: + super().__init__(line) + self.items = items + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_type_list(self) + + def serialize(self) -> JsonDict: + return {'.class': 'TypeList', + 'items': [t.serialize() for t in self.items], + } + + @classmethod + def deserialize(self, data: JsonDict) -> 'TypeList': + assert data['.class'] == 'TypeList' + return TypeList([Type.deserialize(t) for t in data['items']]) + + +class AnyType(Type): + """The type 'Any'.""" + + def __init__(self, implicit=False, line: int = -1) -> None: + super().__init__(line) + self.implicit = implicit + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_any(self) + + def serialize(self) -> JsonDict: + return {'.class': 'AnyType'} + + @classmethod + def deserialize(cls, data: JsonDict) -> 'AnyType': + assert data['.class'] == 'AnyType' + return AnyType() + + +class Void(Type): + """The return type 'None'. + + This can only be used as the return type in a callable type and as + the result type of calling such callable. + """ + + source = '' # May be None; function that generated this value + + def __init__(self, source: str = None, line: int = -1) -> None: + self.source = source + super().__init__(line) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_void(self) + + def with_source(self, source: str) -> 'Void': + return Void(source, self.line) + + def serialize(self) -> JsonDict: + return {'.class': 'Void'} + + @classmethod + def deserialize(cls, data: JsonDict) -> 'Void': + assert data['.class'] == 'Void' + return Void() + + +class UninhabitedType(Type): + """This type has no members. + + This type is almost the bottom type, except it is not a subtype of Void. + With strict Optional checking, it is the only common subtype between all + other types, which allows `meet` to be well defined. Without strict + Optional checking, NoneTyp fills this role. + + In general, for any type T that isn't Void: + join(UninhabitedType, T) = T + meet(UninhabitedType, T) = UninhabitedType + is_subtype(UninhabitedType, T) = True + """ + + def __init__(self, line: int = -1) -> None: + super().__init__(line) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_uninhabited_type(self) + + def serialize(self) -> JsonDict: + return {'.class': 'UninhabitedType'} + + @classmethod + def deserialize(cls, data: JsonDict) -> 'UninhabitedType': + assert data['.class'] == 'UninhabitedType' + return UninhabitedType() + + +class NoneTyp(Type): + """The type of 'None'. + + Without strict Optional checking: + This is only used internally during type inference. Programs + cannot declare a variable of this type, and the type checker + refuses to infer this type for a variable. However, subexpressions + often have this type. Note that this is not used as the result + type when calling a function with a void type, even though + semantically such a function returns a None value; the void type + is used instead so that we can report an error if the caller tries + to do anything with the return value. + + With strict Optional checking: + This type can be written by users as 'None', except as the return value + of a function, where 'None' means Void. + """ + + def __init__(self, line: int = -1) -> None: + super().__init__(line) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_none_type(self) + + def serialize(self) -> JsonDict: + return {'.class': 'NoneTyp'} + + @classmethod + def deserialize(self, data: JsonDict) -> 'NoneTyp': + assert data['.class'] == 'NoneTyp' + return NoneTyp() + + +class ErasedType(Type): + """Placeholder for an erased type. + + This is used during type inference. This has the special property that + it is ignored during type inference. + """ + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_erased_type(self) + + +class DeletedType(Type): + """Type of deleted variables. + + These can be used as lvalues but not rvalues. + """ + + source = '' # May be None; name that generated this value + + def __init__(self, source: str = None, line: int = -1) -> None: + self.source = source + super().__init__(line) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_deleted_type(self) + + def serialize(self) -> JsonDict: + return {'.class': 'DeletedType', + 'source': self.source} + + @classmethod + def deserialize(self, data: JsonDict) -> 'DeletedType': + assert data['.class'] == 'DeletedType' + return DeletedType(data['source']) + + +class Instance(Type): + """An instance type of form C[T1, ..., Tn]. + + The list of type variables may be empty. + """ + + type = None # type: mypy.nodes.TypeInfo + args = None # type: List[Type] + erased = False # True if result of type variable substitution + + def __init__(self, typ: mypy.nodes.TypeInfo, args: List[Type], + line: int = -1, erased: bool = False) -> None: + self.type = typ + self.args = args + self.erased = erased + super().__init__(line) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_instance(self) + + type_ref = None # type: str + + def serialize(self) -> JsonDict: + data = {'.class': 'Instance', + } # type: JsonDict + assert self.type is not None + data['type_ref'] = self.type.alt_fullname or self.type.fullname() + if self.args: + data['args'] = [arg.serialize() for arg in self.args] + return data + + @classmethod + def deserialize(cls, data: JsonDict) -> 'Instance': + assert data['.class'] == 'Instance' + args = [] # type: List[Type] + if 'args' in data: + args_list = data['args'] + assert isinstance(args_list, list) + args = [Type.deserialize(arg) for arg in args_list] + inst = Instance(None, args) + inst.type_ref = data['type_ref'] # Will be fixed up by fixup.py later. + return inst + + +class TypeVarType(Type): + """A type variable type. + + This refers to either a class type variable (id > 0) or a function + type variable (id < 0). + """ + + name = '' # Name of the type variable (for messages and debugging) + id = None # type: TypeVarId + values = None # type: List[Type] # Value restriction, empty list if no restriction + upper_bound = None # type: Type # Upper bound for values + # See comments in TypeVarDef for more about variance. + variance = INVARIANT # type: int + + def __init__(self, binder: TypeVarDef, line: int = -1) -> None: + self.name = binder.name + self.id = binder.id + self.values = binder.values + self.upper_bound = binder.upper_bound + self.variance = binder.variance + super().__init__(line) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_type_var(self) + + def erase_to_union_or_bound(self) -> Type: + if self.values: + return UnionType.make_simplified_union(self.values) + else: + return self.upper_bound + + def serialize(self) -> JsonDict: + assert not self.id.is_meta_var() + return {'.class': 'TypeVarType', + 'name': self.name, + 'id': self.id.raw_id, + 'values': [v.serialize() for v in self.values], + 'upper_bound': self.upper_bound.serialize(), + 'variance': self.variance, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'TypeVarType': + assert data['.class'] == 'TypeVarType' + tvdef = TypeVarDef(data['name'], + data['id'], + [Type.deserialize(v) for v in data['values']], + Type.deserialize(data['upper_bound']), + data['variance']) + return TypeVarType(tvdef) + + +class FunctionLike(Type): + """Abstract base class for function types.""" + + @abstractmethod + def is_type_obj(self) -> bool: pass + + def is_concrete_type_obj(self) -> bool: + return self.is_type_obj() + + @abstractmethod + def type_object(self) -> mypy.nodes.TypeInfo: pass + + @abstractmethod + def items(self) -> List['CallableType']: pass + + @abstractmethod + def with_name(self, name: str) -> 'FunctionLike': pass + + # Corresponding instance type (e.g. builtins.type) + fallback = None # type: Instance + + @classmethod + def deserialize(cls, data: JsonDict) -> 'FunctionLike': + return cast(FunctionLike, super().deserialize(data)) + + +_dummy = object() # type: Any + + +class CallableType(FunctionLike): + """Type of a non-overloaded callable object (function).""" + + arg_types = None # type: List[Type] # Types of function arguments + arg_kinds = None # type: List[int] # mypy.nodes.ARG_ constants + arg_names = None # type: List[str] # None if not a keyword argument + min_args = 0 # Minimum number of arguments; derived from arg_kinds + is_var_arg = False # Is it a varargs function? derived from arg_kinds + ret_type = None # type: Type # Return value type + name = '' # Name (may be None; for error messages) + definition = None # type: SymbolNode # For error messages. May be None. + # Type variables for a generic function + variables = None # type: List[TypeVarDef] + + # Is this Callable[..., t] (with literal '...')? + is_ellipsis_args = False + # Is this callable constructed for the benefit of a classmethod's 'cls' argument? + is_classmethod_class = False + # Was this type implicitly generated instead of explicitly specified by the user? + implicit = False + # Defined for signatures that require special handling (currently only value is 'dict' + # for a signature similar to 'dict') + special_sig = None # type: Optional[str] + + def __init__(self, + arg_types: List[Type], + arg_kinds: List[int], + arg_names: List[str], + ret_type: Type, + fallback: Instance, + name: str = None, + definition: SymbolNode = None, + variables: List[TypeVarDef] = None, + line: int = -1, + is_ellipsis_args: bool = False, + implicit=False, + is_classmethod_class=False, + special_sig=None, + ) -> None: + if variables is None: + variables = [] + self.arg_types = arg_types + self.arg_kinds = arg_kinds + self.arg_names = arg_names + self.min_args = arg_kinds.count(mypy.nodes.ARG_POS) + self.is_var_arg = mypy.nodes.ARG_STAR in arg_kinds + self.ret_type = ret_type + self.fallback = fallback + assert not name or ' 'CallableType': + return CallableType( + arg_types=arg_types if arg_types is not _dummy else self.arg_types, + arg_kinds=arg_kinds if arg_kinds is not _dummy else self.arg_kinds, + arg_names=arg_names if arg_names is not _dummy else self.arg_names, + ret_type=ret_type if ret_type is not _dummy else self.ret_type, + fallback=fallback if fallback is not _dummy else self.fallback, + name=name if name is not _dummy else self.name, + definition=definition if definition is not _dummy else self.definition, + variables=variables if variables is not _dummy else self.variables, + line=line if line is not _dummy else self.line, + is_ellipsis_args=( + is_ellipsis_args if is_ellipsis_args is not _dummy else self.is_ellipsis_args), + implicit=self.implicit, + is_classmethod_class=self.is_classmethod_class, + special_sig=special_sig if special_sig is not _dummy else self.special_sig, + ) + + def is_type_obj(self) -> bool: + return self.fallback.type is not None and self.fallback.type.fullname() == 'builtins.type' + + def is_concrete_type_obj(self) -> bool: + return self.is_type_obj() and self.is_classmethod_class + + def type_object(self) -> mypy.nodes.TypeInfo: + assert self.is_type_obj() + ret = self.ret_type + if isinstance(ret, TupleType): + ret = ret.fallback + return cast(Instance, ret).type + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_callable_type(self) + + def with_name(self, name: str) -> 'CallableType': + """Return a copy of this type with the specified name.""" + ret = self.ret_type + if isinstance(ret, Void): + ret = ret.with_source(name) + return self.copy_modified(ret_type=ret, name=name) + + def max_fixed_args(self) -> int: + n = len(self.arg_types) + if self.is_var_arg: + n -= 1 + return n + + def items(self) -> List['CallableType']: + return [self] + + def is_generic(self) -> bool: + return bool(self.variables) + + def type_var_ids(self) -> List[TypeVarId]: + a = [] # type: List[TypeVarId] + for tv in self.variables: + a.append(tv.id) + return a + + def serialize(self) -> JsonDict: + # TODO: As an optimization, leave out everything related to + # generic functions for non-generic functions. + return {'.class': 'CallableType', + 'arg_types': [(None if t is None else t.serialize()) + for t in self.arg_types], + 'arg_kinds': self.arg_kinds, + 'arg_names': self.arg_names, + 'ret_type': self.ret_type.serialize(), + 'fallback': self.fallback.serialize(), + 'name': self.name, + # We don't serialize the definition (only used for error messages). + 'variables': [v.serialize() for v in self.variables], + 'is_ellipsis_args': self.is_ellipsis_args, + 'implicit': self.implicit, + 'is_classmethod_class': self.is_classmethod_class, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'CallableType': + assert data['.class'] == 'CallableType' + # TODO: Set definition to the containing SymbolNode? + return CallableType([(None if t is None else Type.deserialize(t)) + for t in data['arg_types']], + data['arg_kinds'], + data['arg_names'], + Type.deserialize(data['ret_type']), + Instance.deserialize(data['fallback']), + name=data['name'], + variables=[TypeVarDef.deserialize(v) for v in data['variables']], + is_ellipsis_args=data['is_ellipsis_args'], + implicit=data['implicit'], + is_classmethod_class=data['is_classmethod_class'], + ) + + +class Overloaded(FunctionLike): + """Overloaded function type T1, ... Tn, where each Ti is CallableType. + + The variant to call is chosen based on static argument + types. Overloaded function types can only be defined in stub + files, and thus there is no explicit runtime dispatch + implementation. + """ + + _items = None # type: List[CallableType] # Must not be empty + + def __init__(self, items: List[CallableType]) -> None: + self._items = items + self.fallback = items[0].fallback + super().__init__(items[0].line) + + def items(self) -> List[CallableType]: + return self._items + + def name(self) -> str: + return self._items[0].name + + def is_type_obj(self) -> bool: + # All the items must have the same type object status, so it's + # sufficient to query only (any) one of them. + return self._items[0].is_type_obj() + + def type_object(self) -> mypy.nodes.TypeInfo: + # All the items must have the same type object, so it's sufficient to + # query only (any) one of them. + return self._items[0].type_object() + + def with_name(self, name: str) -> 'Overloaded': + ni = [] # type: List[CallableType] + for it in self._items: + ni.append(it.with_name(name)) + return Overloaded(ni) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_overloaded(self) + + def serialize(self) -> JsonDict: + return {'.class': 'Overloaded', + 'items': [t.serialize() for t in self.items()], + } + + @classmethod + def deserialize(self, data: JsonDict) -> 'Overloaded': + assert data['.class'] == 'Overloaded' + return Overloaded([CallableType.deserialize(t) for t in data['items']]) + + +class TupleType(Type): + """The tuple type Tuple[T1, ..., Tn] (at least one type argument). + + Instance variables: + items: tuple item types + fallback: the underlying instance type that is used for non-tuple methods + (this is currently always builtins.tuple, but it could be different for named + tuples, for example) + implicit: if True, derived from a tuple expression (t,....) instead of Tuple[t, ...] + """ + + items = None # type: List[Type] + fallback = None # type: Instance + implicit = False + + def __init__(self, items: List[Type], fallback: Instance, line: int = -1, + implicit: bool = False) -> None: + #import pdb; pdb.set_trace() + self.items = items + self.fallback = fallback + self.implicit = implicit + super().__init__(line) + + def length(self) -> int: + return len(self.items) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_tuple_type(self) + + def serialize(self) -> JsonDict: + return {'.class': 'TupleType', + 'items': [t.serialize() for t in self.items], + 'fallback': self.fallback.serialize(), + 'implicit': self.implicit, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'TupleType': + assert data['.class'] == 'TupleType' + return TupleType([Type.deserialize(t) for t in data['items']], + Instance.deserialize(data['fallback']), + implicit=data['implicit']) + + def self_type(self, inst: Instance) -> 'TupleType': + return TupleType(self.items, inst, self.line, self.implicit) + + def with_types(self, items: List[Type]) -> 'TupleType': + return TupleType(items, self.fallback, self.line, self.implicit) + + def slice(self, begin: int, stride: int, end: int) -> 'TupleType': + return TupleType(self.items[begin:end:stride], self.fallback, + self.line, self.implicit) + + +class NamedTupleType(TupleType): + attrs = None # type: List[str] + name = None # type: str + + def __init__(self, name: str, attrs: List[str], *args, **kwargs) -> None: + self.attrs = attrs + self.name = name + super().__init__(*args, **kwargs) + + def self_type(self, inst: Instance) -> 'NamedTupleType': + return NamedTupleType(self.name, self.attrs, self.items, inst, self.line) + + def with_types(self, items: List[Type]) -> 'NamedTupleType': + return NamedTupleType(self.name, self.attrs, items, + self.fallback, self.line, self.implicit) + + def __eq__(self, other) -> bool: + return self.serialize() == other.serialize() + + def serialize(self) -> JsonDict: + d = super().serialize() + d.update({'.class': 'NamedTupleType', + 'name': self.name, + 'attrs': self.attrs}) + return d + + @classmethod + def deserialize(cls, data: JsonDict) -> 'NamedTupleType': + assert data['.class'] == 'NamedTupleType' + data['.class'] = 'TupleType' + tup = TupleType.deserialize(data) + return NamedTupleType(data['name'], + data['attrs'], + tup.items, + tup.fallback, + implicit=tup.implicit) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + try: + return visitor.visit_namedtuple_type(self) + except: + return super().accept(visitor) + + +class StarType(Type): + """The star type *type_parameter. + + This is not a real type but a syntactic AST construct. + """ + + type = None # type: Type + + def __init__(self, type: Type, line: int = -1) -> None: + self.type = type + super().__init__(line) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_star_type(self) + + +class UnionType(Type): + """The union type Union[T1, ..., Tn] (at least one type argument).""" + + items = None # type: List[Type] + + def __init__(self, items: List[Type], line: int = -1) -> None: + self.items = items + super().__init__(line) + + @staticmethod + def make_union(items: List[Type], line: int = -1) -> Type: + if len(items) > 1: + return UnionType(items, line) + elif len(items) == 1: + return items[0] + else: + if experiments.STRICT_OPTIONAL: + return UninhabitedType() + else: + return Void() + + @staticmethod + def make_simplified_union(items: List[Type], line: int = -1) -> Type: + while any(isinstance(typ, UnionType) for typ in items): + all_items = [] # type: List[Type] + for typ in items: + if isinstance(typ, UnionType): + all_items.extend(typ.items) + else: + all_items.append(typ) + items = all_items + + if any(isinstance(typ, AnyType) for typ in items): + return AnyType() + + from mypy.subtypes import is_subtype + removed = set() # type: Set[int] + for i in range(len(items)): + if any(is_subtype(items[i], items[j]) for j in range(len(items)) + if j not in removed and j != i): + removed.add(i) + + simplified_set = [items[i] for i in range(len(items)) if i not in removed] + return UnionType.make_union(simplified_set) + + def length(self) -> int: + return len(self.items) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_union_type(self) + + def has_readable_member(self, name: str) -> bool: + """For a tree of unions of instances, check whether all instances have a given member. + + TODO: Deal with attributes of TupleType etc. + TODO: This should probably be refactored to go elsewhere. + """ + return all((isinstance(x, UnionType) and x.has_readable_member(name)) or + (isinstance(x, Instance) and x.type.has_readable_member(name)) + for x in self.items) + + def serialize(self) -> JsonDict: + return {'.class': 'UnionType', + 'items': [t.serialize() for t in self.items], + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'UnionType': + assert data['.class'] == 'UnionType' + return UnionType([Type.deserialize(t) for t in data['items']]) + + +class PartialType(Type): + """Type such as List[?] where type arguments are unknown, or partial None type. + + These are used for inferring types in multiphase initialization such as this: + + x = [] # x gets a partial type List[?], as item type is unknown + x.append(1) # partial type gets replaced with normal type List[int] + + Or with None: + + x = None # x gets a partial type None + if c: + x = 1 # Infer actual type int for x + """ + + # None for the 'None' partial type; otherwise a generic class + type = None # type: Optional[mypy.nodes.TypeInfo] + var = None # type: mypy.nodes.Var + inner_types = None # type: List[Type] + + def __init__(self, + type: Optional['mypy.nodes.TypeInfo'], + var: 'mypy.nodes.Var', + inner_types: List[Type]) -> None: + self.type = type + self.var = var + self.inner_types = inner_types + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_partial_type(self) + + +class EllipsisType(Type): + """The type ... (ellipsis). + + This is not a real type but a syntactic AST construct, used in Callable[..., T], for example. + + A semantically analyzed type will never have ellipsis types. + """ + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_ellipsis_type(self) + + def serialize(self) -> JsonDict: + return {'.class': 'EllipsisType'} + + @classmethod + def deserialize(self, data: JsonDict) -> 'EllipsisType': + assert data['.class'] == 'EllipsisType' + return EllipsisType() + + +class TypeType(Type): + """For types like Type[User]. + + This annotates variables that are class objects, constrained by + the type argument. See PEP 484 for more details. + + We may encounter expressions whose values are specific classes; + those are represented as callables (possibly overloaded) + corresponding to the class's constructor's signature and returning + an instance of that class. The difference with Type[C] is that + those callables always represent the exact class given as the + return type; Type[C] represents any class that's a subclass of C, + and C may also be a type variable or a union (or Any). + + Many questions around subtype relationships between Type[C1] and + def(...) -> C2 are answered by looking at the subtype + relationships between C1 and C2, since Type[] is considered + covariant. + + There's an unsolved problem with constructor signatures (also + unsolved in PEP 484): calling a variable whose type is Type[C] + assumes the constructor signature for C, even though a subclass of + C might completely change the constructor signature. For now we + just assume that users of Type[C] are careful not to do that (in + the future we might detect when they are violating that + assumption). + """ + + # This can't be everything, but it can be a class reference, + # a generic class instance, a union, Any, a type variable... + item = None # type: Type + + def __init__(self, item: Type, *, line: int = -1) -> None: + super().__init__(line) + self.item = item + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_type_type(self) + + def serialize(self) -> JsonDict: + return {'.class': 'TypeType', 'item': self.item.serialize()} + + def deserialize(self, data: JsonDict) -> 'TypeType': + assert data['.class'] == 'TypeType' + return TypeType(Type.deserialize(data['item'])) + + +# +# Visitor-related classes +# + + +class TypeVisitor(Generic[T]): + """Visitor class for types (Type subclasses). + + The parameter T is the return type of the visit methods. + """ + + def _notimplemented_helper(self, name: str) -> NotImplementedError: + return NotImplementedError("Method {}.visit_{}() not implemented\n" + .format(type(self).__name__, name) + + "This is a known bug, track development in " + + "'https://github.com/JukkaL/mypy/issues/730'") + + @abstractmethod + def visit_unbound_type(self, t: UnboundType) -> T: + pass + + def visit_type_list(self, t: TypeList) -> T: + raise self._notimplemented_helper('type_list') + + def visit_error_type(self, t: ErrorType) -> T: + raise self._notimplemented_helper('error_type') + + @abstractmethod + def visit_any(self, t: AnyType) -> T: + pass + + @abstractmethod + def visit_void(self, t: Void) -> T: + pass + + @abstractmethod + def visit_none_type(self, t: NoneTyp) -> T: + pass + + @abstractmethod + def visit_uninhabited_type(self, t: UninhabitedType) -> T: + pass + + def visit_erased_type(self, t: ErasedType) -> T: + raise self._notimplemented_helper('erased_type') + + @abstractmethod + def visit_deleted_type(self, t: DeletedType) -> T: + pass + + @abstractmethod + def visit_type_var(self, t: TypeVarType) -> T: + pass + + @abstractmethod + def visit_instance(self, t: Instance) -> T: + pass + + @abstractmethod + def visit_callable_type(self, t: CallableType) -> T: + pass + + def visit_overloaded(self, t: Overloaded) -> T: + raise self._notimplemented_helper('overloaded') + + @abstractmethod + def visit_tuple_type(self, t: TupleType) -> T: + pass + + def visit_star_type(self, t: StarType) -> T: + raise self._notimplemented_helper('star_type') + + @abstractmethod + def visit_union_type(self, t: UnionType) -> T: + pass + + @abstractmethod + def visit_partial_type(self, t: PartialType) -> T: + pass + + def visit_ellipsis_type(self, t: EllipsisType) -> T: + raise self._notimplemented_helper('ellipsis_type') + + @abstractmethod + def visit_type_type(self, t: TypeType) -> T: + pass + + +class TypeTranslator(TypeVisitor[Type]): + """Identity type transformation. + + Subclass this and override some methods to implement a non-trivial + transformation. + """ + + def visit_unbound_type(self, t: UnboundType) -> Type: + return t + + def visit_type_list(self, t: TypeList) -> Type: + return t + + def visit_error_type(self, t: ErrorType) -> Type: + return t + + def visit_any(self, t: AnyType) -> Type: + return t + + def visit_void(self, t: Void) -> Type: + return t + + def visit_none_type(self, t: NoneTyp) -> Type: + return t + + def visit_uninhabited_type(self, t: UninhabitedType) -> Type: + return t + + def visit_erased_type(self, t: ErasedType) -> Type: + return t + + def visit_deleted_type(self, t: DeletedType) -> Type: + return t + + def visit_instance(self, t: Instance) -> Type: + return Instance(t.type, self.translate_types(t.args), t.line) + + def visit_type_var(self, t: TypeVarType) -> Type: + return t + + def visit_partial_type(self, t: PartialType) -> Type: + return t + + def visit_callable_type(self, t: CallableType) -> Type: + return t.copy_modified(arg_types=self.translate_types(t.arg_types), + ret_type=t.ret_type.accept(self), + variables=self.translate_variables(t.variables)) + + def visit_tuple_type(self, t: TupleType) -> Type: + return TupleType(self.translate_types(t.items), + cast(Any, t.fallback.accept(self)), + t.line) + + def visit_star_type(self, t: StarType) -> Type: + return StarType(t.type.accept(self), t.line) + + def visit_union_type(self, t: UnionType) -> Type: + return UnionType(self.translate_types(t.items), t.line) + + def visit_ellipsis_type(self, t: EllipsisType) -> Type: + return t + + def translate_types(self, types: List[Type]) -> List[Type]: + return [t.accept(self) for t in types] + + def translate_variables(self, + variables: List[TypeVarDef]) -> List[TypeVarDef]: + return variables + + def visit_overloaded(self, t: Overloaded) -> Type: + items = [] # type: List[CallableType] + for item in t.items(): + new = item.accept(self) + if isinstance(new, CallableType): + items.append(new) + else: + raise RuntimeError('CallableType expectected, but got {}'.format(type(new))) + return Overloaded(items=items) + + def visit_type_type(self, t: TypeType) -> Type: + return TypeType(t.item.accept(self), line=t.line) + + +class TypeStrVisitor(TypeVisitor[str]): + """Visitor for pretty-printing types into strings. + + This is mostly for debugging/testing. + + Do not preserve original formatting. + + Notes: + - Represent unbound types as Foo? or Foo?[...]. + - Represent the NoneTyp type as None. + """ + + def visit_unbound_type(self, t): + s = t.name + '?' + if t.args != []: + s += '[{}]'.format(self.list_str(t.args)) + return s + + def visit_type_list(self, t): + return ''.format(self.list_str(t.items)) + + def visit_error_type(self, t): + return '' + + def visit_any(self, t): + return 'Any' + + def visit_void(self, t): + return 'void' + + def visit_none_type(self, t): + # Fully qualify to make this distinct from the None value. + return "builtins.None" + + def visit_uninhabited_type(self, t): + return "" + + def visit_erased_type(self, t): + return "" + + def visit_deleted_type(self, t): + if t.source is None: + return "" + else: + return "".format(t.source) + + def visit_instance(self, t): + s = t.type.fullname() if t.type is not None else '' + if t.erased: + s += '*' + if t.args != []: + s += '[{}]'.format(self.list_str(t.args)) + return s + + def visit_type_var(self, t): + if t.name is None: + # Anonymous type variable type (only numeric id). + return '`{}'.format(t.id) + else: + # Named type variable type. + return '{}`{}'.format(t.name, t.id) + + def visit_callable_type(self, t): + s = '' + bare_asterisk = False + for i in range(len(t.arg_types)): + if s != '': + s += ', ' + if t.arg_kinds[i] == mypy.nodes.ARG_NAMED and not bare_asterisk: + s += '*, ' + bare_asterisk = True + if t.arg_kinds[i] == mypy.nodes.ARG_STAR: + s += '*' + if t.arg_kinds[i] == mypy.nodes.ARG_STAR2: + s += '**' + if t.arg_names[i]: + s += t.arg_names[i] + ': ' + s += str(t.arg_types[i]) + if t.arg_kinds[i] == mypy.nodes.ARG_OPT: + s += ' =' + + s = '({})'.format(s) + + if not isinstance(t.ret_type, Void): + s += ' -> {}'.format(t.ret_type) + + if t.variables: + s = '{} {}'.format(t.variables, s) + + return 'def {}'.format(s) + + def visit_overloaded(self, t): + a = [] + for i in t.items(): + a.append(i.accept(self)) + return 'Overload({})'.format(', '.join(a)) + + def visit_tuple_type(self, t): + s = self.list_str(t.items) + if t.fallback and t.fallback.type: + fallback_name = t.fallback.type.fullname() + if fallback_name != 'builtins.tuple': + return 'Tuple[{}, fallback={}]'.format(s, t.fallback.accept(self)) + return 'Tuple[{}]'.format(s) + + def visit_namedtuple_type(self, t): + s = self.list_str(zip(t.attrs, t.items)) + return '{}[{}]'.format(t.name, s) + + def visit_star_type(self, t): + s = t.type.accept(self) + return '*{}'.format(s) + + def visit_union_type(self, t): + s = self.list_str(t.items) + return 'Union[{}]'.format(s) + + def visit_partial_type(self, t: PartialType) -> str: + if t.type is None: + return '' + else: + return ''.format(t.type.name(), + ', '.join(['?'] * len(t.type.type_vars))) + + def visit_ellipsis_type(self, t): + return '...' + + def visit_type_type(self, t): + return 'Type[{}]'.format(t.item.accept(self)) + + def list_str(self, a): + """Convert items of an array to strings (pretty-print types) + and join the results with commas. + """ + res = [] + for t in a: + if isinstance(t, Type): + res.append(t.accept(self)) + else: + res.append(str(t)) + return ', '.join(res) + + +# These constants define the method used by TypeQuery to combine multiple +# query results, e.g. for tuple types. The strategy is not used for empty +# result lists; in that case the default value takes precedence. +ANY_TYPE_STRATEGY = 0 # Return True if any of the results are True. +ALL_TYPES_STRATEGY = 1 # Return True if all of the results are True. + + +class TypeQuery(TypeVisitor[bool]): + """Visitor for performing simple boolean queries of types. + + This class allows defining the default value for leafs to simplify the + implementation of many queries. + """ + + default = False # Default result + strategy = 0 # Strategy for combining multiple values (ANY_TYPE_STRATEGY or ALL_TYPES_...). + + def __init__(self, default: bool, strategy: int) -> None: + """Construct a query visitor. + + Use the given default result and strategy for combining + multiple results. The strategy must be either + ANY_TYPE_STRATEGY or ALL_TYPES_STRATEGY. + """ + self.default = default + self.strategy = strategy + + def visit_unbound_type(self, t: UnboundType) -> bool: + return self.default + + def visit_type_list(self, t: TypeList) -> bool: + return self.default + + def visit_error_type(self, t: ErrorType) -> bool: + return self.default + + def visit_any(self, t: AnyType) -> bool: + return self.default + + def visit_void(self, t: Void) -> bool: + return self.default + + def visit_uninhabited_type(self, t: UninhabitedType) -> bool: + return self.default + + def visit_none_type(self, t: NoneTyp) -> bool: + return self.default + + def visit_erased_type(self, t: ErasedType) -> bool: + return self.default + + def visit_deleted_type(self, t: DeletedType) -> bool: + return self.default + + def visit_type_var(self, t: TypeVarType) -> bool: + return self.default + + def visit_partial_type(self, t: PartialType) -> bool: + return self.default + + def visit_instance(self, t: Instance) -> bool: + return self.query_types(t.args) + + def visit_callable_type(self, t: CallableType) -> bool: + # FIX generics + return self.query_types(t.arg_types + [t.ret_type]) + + def visit_tuple_type(self, t: TupleType) -> bool: + return self.query_types(t.items) + + def visit_star_type(self, t: StarType) -> bool: + return t.type.accept(self) + + def visit_union_type(self, t: UnionType) -> bool: + return self.query_types(t.items) + + def visit_overloaded(self, t: Overloaded) -> bool: + return self.query_types(t.items()) + + def visit_type_type(self, t: TypeType) -> bool: + return t.item.accept(self) + + def query_types(self, types: Sequence[Type]) -> bool: + """Perform a query for a list of types. + + Use the strategy constant to combine the results. + """ + if not types: + # Use default result for empty list. + return self.default + if self.strategy == ANY_TYPE_STRATEGY: + # Return True if at least one component is true. + res = False + for t in types: + res = res or t.accept(self) + if res: + break + return res + else: + # Return True if all components are true. + res = True + for t in types: + res = res and t.accept(self) + if not res: + break + return res + + +def strip_type(typ: Type) -> Type: + """Make a copy of type without 'debugging info' (function name).""" + + if isinstance(typ, CallableType): + return typ.copy_modified(name=None) + elif isinstance(typ, Overloaded): + return Overloaded([cast(CallableType, strip_type(item)) + for item in typ.items()]) + else: + return typ + + +def replace_leading_arg_type(t: CallableType, self_type: Type) -> CallableType: + """Return a copy of a callable type with a different self argument type. + + Assume that the callable is the signature of a method. + """ + return t.copy_modified(arg_types=[self_type] + t.arg_types[1:]) + + +def is_named_instance(t: Type, fullname: str) -> bool: + return (isinstance(t, Instance) and + t.type is not None and + t.type.fullname() == fullname) diff --git a/test-data/unit/check-lists.test b/test-data/unit/check-lists.test index a6df5ae740c61..9db5fb018abfa 100644 --- a/test-data/unit/check-lists.test +++ b/test-data/unit/check-lists.test @@ -1,63 +1,63 @@ --- Nested list assignment --- ----------------------------- - -[case testNestedListAssignment] -from typing import List -a1, b1, c1 = None, None, None # type: (A, B, C) -a2, b2, c2 = None, None, None # type: (A, B, C) - -a1, [b1, c1] = a2, [b2, c2] -a1, [a1, [b1, c1]] = a2, [a2, [b2, c2]] -a1, [a1, [a1, b1]] = a1, [a1, [a1, c1]] # E: Incompatible types in assignment (expression has type "C", variable has type "B") - -class A: pass -class B: pass -class C: pass -[builtins fixtures/list.py] -[out] - -[case testNestedListAssignmentToTuple] -from typing import List -a, b, c = None, None, None # type: (A, B, C) - -a, b = [a, b] -a, b = [a] # E: Need more than 1 value to unpack (2 expected) -a, b = [a, b, c] # E: Too many values to unpack (2 expected, 3 provided) - -class A: pass -class B: pass -class C: pass -[builtins fixtures/list.py] -[out] - -[case testListAssignmentFromTuple] -from typing import List -a, b, c = None, None, None # type: (A, B, C) -t = a, b - -[a, b], c = t, c -[a, c], c = t, c # E: Incompatible types in assignment (expression has type "B", variable has type "C") -[a, a, a], c = t, c # E: Need more than 2 values to unpack (3 expected) -[a], c = t, c # E: Too many values to unpack (1 expected, 2 provided) - -class A: pass -class B: pass -class C: pass -[builtins fixtures/list.py] -[out] - -[case testListAssignmentUnequalAmountToUnpack] -from typing import List -a, b, c = None, None, None # type: (A, B, C) - -def f() -> None: # needed because test parser tries to parse [a, b] as section header - [a, b] = [a, b] - [a, b] = [a] # E: Need more than 1 value to unpack (2 expected) - [a, b] = [a, b, c] # E: Too many values to unpack (2 expected, 3 provided) - -class A: pass -class B: pass -class C: pass -[builtins fixtures/list.py] -[out] -main: note: In function "f": +-- Nested list assignment +-- ----------------------------- + +[case testNestedListAssignment] +from typing import List +a1, b1, c1 = None, None, None # type: (A, B, C) +a2, b2, c2 = None, None, None # type: (A, B, C) + +a1, [b1, c1] = a2, [b2, c2] +a1, [a1, [b1, c1]] = a2, [a2, [b2, c2]] +a1, [a1, [a1, b1]] = a1, [a1, [a1, c1]] # E: Incompatible types in assignment (expression has type "C", variable has type "B") + +class A: pass +class B: pass +class C: pass +[builtins fixtures/list.py] +[out] + +[case testNestedListAssignmentToTuple] +from typing import List +a, b, c = None, None, None # type: (A, B, C) + +a, b = [a, b] +a, b = [a] # E: Need more than 1 value to unpack (2 expected) +a, b = [a, b, c] # E: Too many values to unpack (2 expected, 3 provided) + +class A: pass +class B: pass +class C: pass +[builtins fixtures/list.py] +[out] + +[case testListAssignmentFromTuple] +from typing import List +a, b, c = None, None, None # type: (A, B, C) +t = a, b + +[a, b], c = t, c +[a, c], c = t, c # E: Incompatible types in assignment (expression has type "B", variable has type "C") +[a, a, a], c = t, c # E: Need more than 2 values to unpack (3 expected) +[a], c = t, c # E: Too many values to unpack (1 expected, 2 provided) + +class A: pass +class B: pass +class C: pass +[builtins fixtures/list.py] +[out] + +[case testListAssignmentUnequalAmountToUnpack] +from typing import List +a, b, c = None, None, None # type: (A, B, C) + +def f() -> None: # needed because test parser tries to parse [a, b] as section header + [a, b] = [a, b] + [a, b] = [a] # E: Need more than 1 value to unpack (2 expected) + [a, b] = [a, b, c] # E: Too many values to unpack (2 expected, 3 provided) + +class A: pass +class B: pass +class C: pass +[builtins fixtures/list.py] +[out] +main: note: In function "f": diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 86c19956ca634..adb07ec522705 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -1,223 +1,223 @@ -[case testNamedTupleDeclaration] -from typing import NamedTuple - -A = NamedTuple('A', [('x', str), ('y', int)]) - -[case testNamedTupleUsedAsTuple] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -a, b = x -b = x[0] -a = x[1] -a, b, c = x # E: Need more than 2 values to unpack (3 expected) -x[2] # E: Tuple index out of range - -[case testAccessingNamedTupleAttributes] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -x.x -x.y -x.z # E: "X" has no attribute "z" - -[case testCreateNamedTupleWithPositionalArguments] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = X(1, 'x') -x.x -x.z # E: "X" has no attribute "z" -x = X(1) # E: Too few arguments for "X" -x = X(1, 2, 3) # E: Too many arguments for "X" - -[case testCreateNamedTupleWithKeywordArguments] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = X(x=1, y='x') -x = X(1, y='x') -x = X(x=1, z=1) # E: Unexpected keyword argument "z" for "X" -x = X(y=1) # E: Missing positional argument "x" in call to "X" - -[case testCreateNamedTupleAndUseAsTuple] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = X(1, 'x') -a, b = x -a, b, c = x # E: Need more than 2 values to unpack (3 expected) - -[case testNamedTupleWithItemTypes] -from typing import NamedTuple -N = NamedTuple('N', [('a', int), - ('b', str)]) -n = N(1, 'x') -s = n.a # type: str # E: Incompatible types in assignment (expression has type "int", \ - variable has type "str") -i = n.b # type: int # E: Incompatible types in assignment (expression has type "str", \ - variable has type "int") -x, y = n -x = y # E: Incompatible types in assignment (expression has type "str", variable has type "int") - -[case testNamedTupleConstructorArgumentTypes] -from typing import NamedTuple -N = NamedTuple('N', [('a', int), - ('b', str)]) -n = N('x', 'x') # E: Argument 1 to "N" has incompatible type "str"; expected "int" -n = N(1, b=2) # E: Argument 2 to "N" has incompatible type "int"; expected "str" -N(1, 'x') -N(b='x', a=1) - -[case testNamedTupleAsBaseClass] -from typing import NamedTuple -N = NamedTuple('N', [('a', int), - ('b', str)]) -class X(N): - pass -x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" -s = '' -i = 0 -s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") -i, s = x -s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") - -[case testNamedTupleAsBaseClass2] -from typing import NamedTuple -class X(NamedTuple('N', [('a', int), - ('b', str)])): - pass -x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" -s = '' -i = 0 -s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") -i, s = x -s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") - -[case testTwoNamedTuplesAsBaseClasses] -from typing import NamedTuple -A = NamedTuple('A', [('a', int)]) -B = NamedTuple('B', [('a', int)]) -class X(A, B): # E: Class has two incompatible bases derived from tuple - pass - -[case testTwoNamedTuplesAsBaseClasses2] -from typing import NamedTuple -A = NamedTuple('A', [('a', int)]) -class X(A, NamedTuple('B', [('a', int)])): # E: Class has two incompatible bases derived from tuple - pass - -[case testSelfTypeWithNamedTupleAsBase] -from typing import NamedTuple -A = NamedTuple('A', [('a', int), ('b', str)]) -class B(A): - def f(self, x: int) -> None: - self.f(self.a) - self.f(self.b) # E: Argument 1 to "f" of "B" has incompatible type "str"; expected "int" - i = 0 - s = '' - i, s = self - i, i = self # E: Incompatible types in assignment (expression has type "str", \ - variable has type "int") -[out] -main: note: In member "f" of class "B": - -[case testTypeReferenceToClassDerivedFromNamedTuple] -from typing import NamedTuple -A = NamedTuple('A', [('a', int), ('b', str)]) -class B(A): - def f(self, x: 'B') -> None: - i = 0 - s = '' - self = x - i, s = x - i, s = x.a, x.b - i, s = x.a, x.a # E: Incompatible types in assignment (expression has type "int", \ - variable has type "str") - i, i = self # E: Incompatible types in assignment (expression has type "str", \ - variable has type "int") -[out] -main: note: In member "f" of class "B": - -[case testNamedTupleSubtyping] -from typing import NamedTuple, Tuple -A = NamedTuple('A', [('a', int), ('b', str)]) -class B(A): pass -a = A(1, '') -b = B(1, '') -t = None # type: Tuple[int, str] -b = a # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "A") -b = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "B") -t = a -t = (1, '') -t = b -a = b - -[case testNamedTupleSimpleTypeInference] -from typing import NamedTuple, Tuple -A = NamedTuple('A', [('a', int)]) -l = [A(1), A(2)] -a = A(1) -a = l[0] -(i,) = l[0] -i, i = l[0] # E: Need more than 1 value to unpack (2 expected) -a = (1,) # E: Incompatible types in assignment (expression has type "Tuple[int]", \ - variable has type "A") -[builtins fixtures/list.py] - -[case testNamedTupleMissingClassAttribute] -import collections -MyNamedTuple = collections.namedtuple('MyNamedTuple', ['spam', 'eggs']) -MyNamedTuple.x # E: "MyNamedTuple" has no attribute "x" - -[case testNamedTupleEmptyItems] -from typing import NamedTuple -A = NamedTuple('A', []) -[builtins fixtures/list.py] - -[case testNamedTupleProperty] -from typing import NamedTuple -A = NamedTuple('A', [('a', int)]) -class B(A): - @property - def b(self) -> int: - return self.a -class C(B): pass -B(1).b -C(2).b -[builtins fixtures/property.py] - -[case testNamedTupleAsDict] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -a = x._asdict() - -[case testNamedTupleReplace] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -x = x._replace(y=5) -x = x._replace(z=5) # E: Unexpected keyword argument "z" for X._replace -x = x._replace(5) # E: Too many positional arguments for X._replace - -[case testNamedTupleReplaceAsClass] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -X._replace(x, x=1, y=2) -X._replace(x=1, y=2) # E: Missing positional argument "self" in call to X._replace - -[case testNamedTupleReplaceTyped] -from typing import NamedTuple - -X = NamedTuple('X', [('x', int), ('y', str)]) -x = None # type: X -x = x._replace(x=5) -x = x._replace(y=5) # E: Argument 1 to X._replace has incompatible type "int"; expected "str" +[case testNamedTupleDeclaration] +from typing import NamedTuple + +A = NamedTuple('A', [('x', str), ('y', int)]) + +[case testNamedTupleUsedAsTuple] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +a, b = x +b = x[0] +a = x[1] +a, b, c = x # E: Need more than 2 values to unpack (3 expected) +x[2] # E: Tuple index out of range + +[case testAccessingNamedTupleAttributes] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +x.x +x.y +x.z # E: "X" has no attribute "z" + +[case testCreateNamedTupleWithPositionalArguments] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = X(1, 'x') +x.x +x.z # E: "X" has no attribute "z" +x = X(1) # E: Too few arguments for "X" +x = X(1, 2, 3) # E: Too many arguments for "X" + +[case testCreateNamedTupleWithKeywordArguments] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = X(x=1, y='x') +x = X(1, y='x') +x = X(x=1, z=1) # E: Unexpected keyword argument "z" for "X" +x = X(y=1) # E: Missing positional argument "x" in call to "X" + +[case testCreateNamedTupleAndUseAsTuple] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = X(1, 'x') +a, b = x +a, b, c = x # E: Need more than 2 values to unpack (3 expected) + +[case testNamedTupleWithItemTypes] +from typing import NamedTuple +N = NamedTuple('N', [('a', int), + ('b', str)]) +n = N(1, 'x') +s = n.a # type: str # E: Incompatible types in assignment (expression has type "int", \ + variable has type "str") +i = n.b # type: int # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +x, y = n +x = y # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testNamedTupleConstructorArgumentTypes] +from typing import NamedTuple +N = NamedTuple('N', [('a', int), + ('b', str)]) +n = N('x', 'x') # E: Argument 1 to "N" has incompatible type "str"; expected "int" +n = N(1, b=2) # E: Argument 2 to "N" has incompatible type "int"; expected "str" +N(1, 'x') +N(b='x', a=1) + +[case testNamedTupleAsBaseClass] +from typing import NamedTuple +N = NamedTuple('N', [('a', int), + ('b', str)]) +class X(N): + pass +x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" +s = '' +i = 0 +s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") +i, s = x +s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testNamedTupleAsBaseClass2] +from typing import NamedTuple +class X(NamedTuple('N', [('a', int), + ('b', str)])): + pass +x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" +s = '' +i = 0 +s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") +i, s = x +s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testTwoNamedTuplesAsBaseClasses] +from typing import NamedTuple +A = NamedTuple('A', [('a', int)]) +B = NamedTuple('B', [('a', int)]) +class X(A, B): # E: Class has two incompatible bases derived from tuple + pass + +[case testTwoNamedTuplesAsBaseClasses2] +from typing import NamedTuple +A = NamedTuple('A', [('a', int)]) +class X(A, NamedTuple('B', [('a', int)])): # E: Class has two incompatible bases derived from tuple + pass + +[case testSelfTypeWithNamedTupleAsBase] +from typing import NamedTuple +A = NamedTuple('A', [('a', int), ('b', str)]) +class B(A): + def f(self, x: int) -> None: + self.f(self.a) + self.f(self.b) # E: Argument 1 to "f" of "B" has incompatible type "str"; expected "int" + i = 0 + s = '' + i, s = self + i, i = self # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +[out] +main: note: In member "f" of class "B": + +[case testTypeReferenceToClassDerivedFromNamedTuple] +from typing import NamedTuple +A = NamedTuple('A', [('a', int), ('b', str)]) +class B(A): + def f(self, x: 'B') -> None: + i = 0 + s = '' + self = x + i, s = x + i, s = x.a, x.b + i, s = x.a, x.a # E: Incompatible types in assignment (expression has type "int", \ + variable has type "str") + i, i = self # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +[out] +main: note: In member "f" of class "B": + +[case testNamedTupleSubtyping] +from typing import NamedTuple, Tuple +A = NamedTuple('A', [('a', int), ('b', str)]) +class B(A): pass +a = A(1, '') +b = B(1, '') +t = None # type: Tuple[int, str] +b = a # E: Incompatible types in assignment (expression has type "A", variable has type "B") +a = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "A") +b = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "B") +t = a +t = (1, '') +t = b +a = b + +[case testNamedTupleSimpleTypeInference] +from typing import NamedTuple, Tuple +A = NamedTuple('A', [('a', int)]) +l = [A(1), A(2)] +a = A(1) +a = l[0] +(i,) = l[0] +i, i = l[0] # E: Need more than 1 value to unpack (2 expected) +a = (1,) # E: Incompatible types in assignment (expression has type "Tuple[int]", \ + variable has type "A") +[builtins fixtures/list.py] + +[case testNamedTupleMissingClassAttribute] +import collections +MyNamedTuple = collections.namedtuple('MyNamedTuple', ['spam', 'eggs']) +MyNamedTuple.x # E: "MyNamedTuple" has no attribute "x" + +[case testNamedTupleEmptyItems] +from typing import NamedTuple +A = NamedTuple('A', []) +[builtins fixtures/list.py] + +[case testNamedTupleProperty] +from typing import NamedTuple +A = NamedTuple('A', [('a', int)]) +class B(A): + @property + def b(self) -> int: + return self.a +class C(B): pass +B(1).b +C(2).b +[builtins fixtures/property.py] + +[case testNamedTupleAsDict] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +a = x._asdict() + +[case testNamedTupleReplace] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +x = x._replace(y=5) +x = x._replace(z=5) # E: Unexpected keyword argument "z" for X._replace +x = x._replace(5) # E: Too many positional arguments for X._replace + +[case testNamedTupleReplaceAsClass] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +X._replace(x, x=1, y=2) +X._replace(x=1, y=2) # E: Missing positional argument "self" in call to X._replace + +[case testNamedTupleReplaceTyped] +from typing import NamedTuple + +X = NamedTuple('X', [('x', int), ('y', str)]) +x = None # type: X +x = x._replace(x=5) +x = x._replace(y=5) # E: Argument 1 to X._replace has incompatible type "int"; expected "str" diff --git a/test-data/unit/check-tuples.test b/test-data/unit/check-tuples.test index f946c451123fd..50d1b3e64598d 100644 --- a/test-data/unit/check-tuples.test +++ b/test-data/unit/check-tuples.test @@ -1,770 +1,770 @@ --- Normal assignment and subtyping --- ------------------------------- - - -[case testTupleAssignmentWithTupleTypes] -from typing import Tuple -t1 = None # type: Tuple[A] -t2 = None # type: Tuple[B] -t3 = None # type: Tuple[A, A] -t4 = None # type: Tuple[A, B] -t5 = None # type: Tuple[B, A] - -t1 = t2 # E: Incompatible types in assignment (expression has type "Tuple[B]", variable has type "Tuple[A]") -t1 = t3 # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A]") -t3 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A]", variable has type "Tuple[A, A]") -t3 = t4 # E: Incompatible types in assignment (expression has type "Tuple[A, B]", variable has type "Tuple[A, A]") -t3 = t5 # E: Incompatible types in assignment (expression has type "Tuple[B, A]", variable has type "Tuple[A, A]") - -# Ok -t1 = t1 -t2 = t2 -t3 = t3 -t4 = t4 -t5 = t5 - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - -[case testTupleSubtyping] -from typing import Tuple -t1 = None # type: Tuple[A, A] -t2 = None # type: Tuple[A, B] -t3 = None # type: Tuple[B, A] - -t2 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A, B]") -t2 = t3 # E: Incompatible types in assignment (expression has type "Tuple[B, A]", variable has type "Tuple[A, B]") -t3 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[B, A]") -t3 = t2 # E: Incompatible types in assignment (expression has type "Tuple[A, B]", variable has type "Tuple[B, A]") - -t1 = t2 -t1 = t3 - -class A: pass -class B(A): pass -[builtins fixtures/tuple.py] - -[case testTupleCompatibilityWithOtherTypes] -from typing import Tuple -a, o = None, None # type: (A, object) -t = None # type: Tuple[A, A] - -a = t # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "A") -t = o # E: Incompatible types in assignment (expression has type "object", variable has type "Tuple[A, A]") -t = a # E: Incompatible types in assignment (expression has type "A", variable has type "Tuple[A, A]") -# TODO: callable types + tuples - -# Ok -o = t -t = None - -class A: pass -[builtins fixtures/tuple.py] - -[case testNestedTupleTypes] -from typing import Tuple -t1 = None # type: Tuple[A, Tuple[A, A]] -t2 = None # type: Tuple[B, Tuple[B, B]] - -t2 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, Tuple[A, A]]", variable has type "Tuple[B, Tuple[B, B]]") -t1 = t2 - -class A: pass -class B(A): pass -[builtins fixtures/tuple.py] - -[case testNestedTupleTypes2] -from typing import Tuple -t1 = None # type: Tuple[A, Tuple[A, A]] -t2 = None # type: Tuple[B, Tuple[B, B]] - -t2 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, Tuple[A, A]]", variable has type "Tuple[B, Tuple[B, B]]") -t1 = t2 - -class A: pass -class B(A): pass -[builtins fixtures/tuple.py] - -[case testSubtypingWithNamedTupleType] -from typing import Tuple -t1 = None # type: Tuple[A, A] -t2 = None # type: tuple - -t1 = t2 # E: Incompatible types in assignment (expression has type Tuple[Any, ...], variable has type "Tuple[A, A]") -t2 = t1 - -class A: pass -[builtins fixtures/tuple.py] - -[case testTupleInitializationWithNone] -from typing import Tuple -t = None # type: Tuple[A, A] -t = None -class A: pass -[builtins fixtures/tuple.py] - - --- Tuple expressions --- ----------------- - - -[case testTupleExpressions] -from typing import Tuple -t1 = None # type: tuple -t2 = None # type: Tuple[A] -t3 = None # type: Tuple[A, B] - -a, b, c = None, None, None # type: (A, B, C) - -t2 = () # E: Incompatible types in assignment (expression has type "Tuple[]", variable has type "Tuple[A]") -t2 = (a, a) # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A]") -t3 = (a, a) # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A, B]") -t3 = (b, b) # E: Incompatible types in assignment (expression has type "Tuple[B, B]", variable has type "Tuple[A, B]") -t3 = (a, b, a) # E: Incompatible types in assignment (expression has type "Tuple[A, B, A]", variable has type "Tuple[A, B]") - -t1 = () -t1 = (a,) -t2 = (a,) -t3 = (a, b) -t3 = (a, c) -t3 = (None, None) - -class A: pass -class B: pass -class C(B): pass -[builtins fixtures/tuple.py] - -[case testVoidValueInTuple] -import typing -(None, f()) # E: "f" does not return a value -(f(), None) # E: "f" does not return a value - -def f() -> None: pass -[builtins fixtures/tuple.py] - - --- Indexing --- -------- - - -[case testIndexingTuples] -from typing import Tuple -t1 = None # type: Tuple[A, B] -t2 = None # type: Tuple[A] -t3 = None # type: Tuple[A, B, C, D, E] -a, b = None, None # type: (A, B) -x = None # type: Tuple[A, B, C] -y = None # type: Tuple[A, C, E] -n = 0 - -a = t1[1] # E: Incompatible types in assignment (expression has type "B", variable has type "A") -b = t1[0] # E: Incompatible types in assignment (expression has type "A", variable has type "B") -t1[2] # E: Tuple index out of range -t1[3] # E: Tuple index out of range -t2[1] # E: Tuple index out of range -t1[n] # E: Tuple index must be an integer literal -t3[n:] # E: Tuple slice must be an integer literal -b = t1[(0)] # E: Incompatible types in assignment (expression has type "A", variable has type "B") - -a = t1[0] -b = t1[1] -b = t1[-1] -a = t1[(0)] -x = t3[0:3] # type (A, B, C) -y = t3[0:5:2] # type (A, C, E) -x = t3[:-2] # type (A, B, C) - -class A: pass -class B: pass -class C: pass -class D: pass -class E: pass -[builtins fixtures/tuple.py] - -[case testIndexingTuplesWithNegativeIntegers] -from typing import Tuple -t1 = None # type: Tuple[A, B] -t2 = None # type: Tuple[A] -a, b = None, None # type: A, B - -a = t1[-1] # E: Incompatible types in assignment (expression has type "B", variable has type "A") -b = t1[-2] # E: Incompatible types in assignment (expression has type "A", variable has type "B") -t1[-3] # E: Tuple index out of range -t1[-4] # E: Tuple index out of range -b = t2[(-1)] # E: Incompatible types in assignment (expression has type "A", variable has type "B") - -a = t1[-2] -b = t1[-1] -a = t2[(-1)] - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - -[case testAssigningToTupleItems] -from typing import Tuple -t = None # type: Tuple[A, B] -n = 0 - -t[0] = A() # E: Unsupported target for indexed assignment -t[2] = A() # E: Unsupported target for indexed assignment -t[n] = A() # E: Unsupported target for indexed assignment - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - - --- Multiple assignment --- ------------------- - - -[case testMultipleAssignmentWithTuples] -from typing import Tuple -t1 = None # type: Tuple[A, B] -t2 = None # type: Tuple[A, B, A] -a, b = None, None # type: (A, B) - -a, a = t1 # E: Incompatible types in assignment (expression has type "B", variable has type "A") -b, b = t1 # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a, b, b = t2 # E: Incompatible types in assignment (expression has type "A", variable has type "B") - -a, b = t1 -a, b, a = t2 - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - -[case testMultipleAssignmentWithInvalidNumberOfValues] -from typing import Tuple -t1 = None # type: Tuple[A, A, A] -a = None # type: A - -a, a = t1 # E: Too many values to unpack (2 expected, 3 provided) -a, a, a, a = t1 # E: Need more than 3 values to unpack (4 expected) - -a, a, a = t1 - -class A: pass -[builtins fixtures/tuple.py] - -[case testMultipleAssignmentWithTupleExpressionRvalue] - -a, b = None, None # type: (A, B) - -a, b = a, a # Fail -a, b = b, a # Fail - -a, b = a, b -a, a = a, a - -class A: pass -class B: pass -[builtins fixtures/tuple.py] -[out] -main:4: error: Incompatible types in assignment (expression has type "A", variable has type "B") -main:5: error: Incompatible types in assignment (expression has type "B", variable has type "A") -main:5: error: Incompatible types in assignment (expression has type "A", variable has type "B") - -[case testSubtypingInMultipleAssignment] - -a, b = None, None # type: (A, B) - -b, b = a, b # E: Incompatible types in assignment (expression has type "A", variable has type "B") -b, b = b, a # E: Incompatible types in assignment (expression has type "A", variable has type "B") - -a, b = b, b -b, a = b, b - -class A: pass -class B(A): pass -[builtins fixtures/tuple.py] - -[case testInitializationWithMultipleValues] - -a, b = None, None # type: (A, B) - -a1, b1 = a, a # type: (A, B) # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a2, b2 = b, b # type: (A, B) # E: Incompatible types in assignment (expression has type "B", variable has type "A") -a3, b3 = a # type: (A, B) # E: '__main__.A' object is not iterable -a4, b4 = None # type: (A, B) # E: 'builtins.None' object is not iterable -a5, b5 = a, b, a # type: (A, B) # E: Too many values to unpack (2 expected, 3 provided) - -ax, bx = a, b # type: (A, B) - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - -[case testMultipleAssignmentWithNonTupleRvalue] - -a, b = None, None # type: (A, B) -def f(): pass - -a, b = None # E: 'builtins.None' object is not iterable -a, b = a # E: '__main__.A' object is not iterable -a, b = f # E: 'def () -> Any' object is not iterable - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - -[case testMultipleAssignmentWithIndexedLvalues] - -a, b = None, None # type: (A, B) -aa, bb = None, None # type: (AA, BB) - -a[a], b[b] = a, bb # E: Incompatible types in assignment (expression has type "A", target has type "AA") -a[a], b[b] = aa, b # E: Incompatible types in assignment (expression has type "B", target has type "BB") -a[aa], b[b] = aa, bb # E: Invalid index type "AA" for "A" -a[a], b[bb] = aa, bb # E: Invalid index type "BB" for "B" -a[a], b[b] = aa, bb - -class A: - def __setitem__(self, x: 'A', y: 'AA') -> None: pass -class B: - def __setitem__(self, x: 'B', y: 'BB') -> None: pass - -class AA: pass -class BB: pass -[builtins fixtures/tuple.py] - -[case testMultipleDeclarationWithParentheses] - -(a, b) = (None, None) # type: int, str -a = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -b = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") -a = 1 -b = '' - -[case testMultipleAssignmentWithExtraParentheses] - -a, b = None, None # type: (A, B) - -(a, b) = (a, a) # E: Incompatible types in assignment (expression has type "A", variable has type "B") -(a, b) = (b, b) # E: Incompatible types in assignment (expression has type "B", variable has type "A") -((a), (b)) = ((a), (a)) # E: Incompatible types in assignment (expression has type "A", variable has type "B") -((a), (b)) = ((b), (b)) # E: Incompatible types in assignment (expression has type "B", variable has type "A") -[a, b] = a, a # E: Incompatible types in assignment (expression has type "A", variable has type "B") -[a, b] = b, b # E: Incompatible types in assignment (expression has type "B", variable has type "A") - -(a, b) = (a, b) -((a), (b)) = ((a), (b)) -[a, b] = a, b - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - -[case testMultipleAssignmentUsingSingleTupleType] -from typing import Tuple -a, b = None, None # type: Tuple[int, str] -a = 1 -b = '' -a = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -b = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") - -[case testMultipleAssignmentWithMixedVariables] -a = b, c = 1, 1 -x, y = p, q = 1, 1 -u, v, w = r, s = 1, 1 # E: Need more than 2 values to unpack (3 expected) -d, e = f, g, h = 1, 1 # E: Need more than 2 values to unpack (3 expected) - - --- Assignment to starred expressions --- --------------------------------- - - -[case testAssignmentToStarMissingAnnotation] -from typing import List -t = 1, 2 -a, b, *c = 1, 2 # E: Need type annotation for variable -aa, bb, *cc = t # E: Need type annotation for variable -[builtins fixtures/list.py] - -[case testAssignmentToStarAnnotation] -from typing import List -li, lo = None, None # type: List[int], List[object] -a, b, *c = 1, 2 # type: int, int, *List[int] -c = lo # E: Incompatible types in assignment (expression has type List[object], variable has type List[int]) -c = li -[builtins fixtures/list.py] - -[case testAssignmentToStarCount1] -from typing import List -ca = None # type: List[int] -c = [1] -a, b, *c = 1, # E: Need more than 1 value to unpack (2 expected) -a, b, *c = 1, 2 -a, b, *c = 1, 2, 3 -a, b, *c = 1, 2, 3, 4 -[builtins fixtures/list.py] - -[case testAssignmentToStarCount2] -from typing import List -ca = None # type: List[int] -t1 = 1, -t2 = 1, 2 -t3 = 1, 2, 3 -t4 = 1, 2, 3, 4 -c = [1] -a, b, *c = t1 # E: Need more than 1 value to unpack (2 expected) -a, b, *c = t2 -a, b, *c = t3 -a, b, *c = t4 -[builtins fixtures/list.py] - -[case testAssignmentToStarFromAny] -from typing import Any -a, c = Any(1), C() -p, *q = a -c = a -c = q - -class C: pass - -[case testAssignmentToComplexStar] -from typing import List -li = None # type: List[int] -a, *(li) = 1, -a, *(b, c) = 1, 2 # E: Need more than 1 value to unpack (2 expected) -a, *(b, c) = 1, 2, 3 -a, *(b, c) = 1, 2, 3, 4 # E: Too many values to unpack (2 expected, 3 provided) -[builtins fixtures/list.py] - -[case testAssignmentToStarFromTupleType] -from typing import List, Tuple -li = None # type: List[int] -la = None # type: List[A] -ta = None # type: Tuple[A, A, A] -a, *la = ta -a, *li = ta # E -a, *na = ta -na = la -na = a # E - -class A: pass -[builtins fixtures/list.py] -[out] -main:6: error: List item 0 has incompatible type "A" -main:6: error: List item 1 has incompatible type "A" -main:9: error: Incompatible types in assignment (expression has type "A", variable has type List[A]) - -[case testAssignmentToStarFromTupleInference] -from typing import List -li = None # type: List[int] -la = None # type: List[A] -a, *l = A(), A() -l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) -l = la - -class A: pass -[builtins fixtures/list.py] -[out] - -[case testAssignmentToStarFromListInference] -from typing import List -li = None # type: List[int] -la = None # type: List[A] -a, *l = [A(), A()] -l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) -l = la - -class A: pass -[builtins fixtures/list.py] -[out] - -[case testAssignmentToStarFromTupleTypeInference] -from typing import List, Tuple -li = None # type: List[int] -la = None # type: List[A] -ta = None # type: Tuple[A, A, A] -a, *l = ta -l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) -l = la - -class A: pass -[builtins fixtures/list.py] -[out] - -[case testAssignmentToStarFromListTypeInference] -from typing import List -li = None # type: List[int] -la = None # type: List[A] -a, *l = la -l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) -l = la - -class A: pass -[builtins fixtures/list.py] -[out] - - --- Nested tuple assignment --- ---------------------------- - - -[case testNestedTupleAssignment1] - -a1, b1, c1 = None, None, None # type: (A, B, C) -a2, b2, c2 = None, None, None # type: (A, B, C) - -a1, (b1, c1) = a2, (b2, c2) -a1, (a1, (b1, c1)) = a2, (a2, (b2, c2)) -a1, (a1, (a1, b1)) = a1, (a1, (a1, c1)) # Fail - -class A: pass -class B: pass -class C: pass -[out] -main:7: error: Incompatible types in assignment (expression has type "C", variable has type "B") - -[case testNestedTupleAssignment2] - -a1, b1, c1 = None, None, None # type: (A, B, C) -a2, b2, c2 = None, None, None # type: (A, B, C) -t = a1, b1 - -a2, b2 = t -(a2, b2), c2 = t, c1 -(a2, c2), c2 = t, c1 # Fail -t, c2 = (a2, b2), c2 -t, c2 = (a2, a2), c2 # Fail -t = a1, a1, a1 # Fail -t = a1 # Fail -a2, a2, a2 = t # Fail -a2, = t # Fail -a2 = t # Fail - -class A: pass -class B: pass -class C: pass -[out] -main:8: error: Incompatible types in assignment (expression has type "B", variable has type "C") -main:10: error: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A, B]") -main:11: error: Incompatible types in assignment (expression has type "Tuple[A, A, A]", variable has type "Tuple[A, B]") -main:12: error: Incompatible types in assignment (expression has type "A", variable has type "Tuple[A, B]") -main:13: error: Need more than 2 values to unpack (3 expected) -main:14: error: Too many values to unpack (1 expected, 2 provided) -main:15: error: Incompatible types in assignment (expression has type "Tuple[A, B]", variable has type "A") - - --- Error messages --- -------------- - - -[case testTupleErrorMessages] - -a = None # type: A - -(a, a) + a # E: Unsupported left operand type for + (Tuple[A, ...]) -a + (a, a) # E: Unsupported operand types for + ("A" and "Tuple[A, A]") -f((a, a)) # E: Argument 1 to "f" has incompatible type "Tuple[A, A]"; expected "A" -(a, a).foo # E: Tuple[A, ...] has no attribute "foo" - -def f(x: 'A') -> None: pass - -class A: - def __add__(self, x: 'A') -> 'A': pass -[builtins fixtures/tuple.py] - -[case testLargeTuplesInErrorMessages] - -a = None # type: LongTypeName -a + (a, a, a, a, a, a, a) # Fail - -class LongTypeName: - def __add__(self, x: 'LongTypeName') -> 'LongTypeName': pass -[builtins fixtures/tuple.py] -[out] -main:3: error: Unsupported operand types for + ("LongTypeName" and tuple(length 7)) - - --- Tuple methods --- ------------- - - -[case testTupleMethods] -from typing import Tuple -t = None # type: Tuple[int, str] -i = 0 -s = '' -b = bool() - -s = t.__len__() # E: Incompatible types in assignment (expression has type "int", variable has type "str") -i = t.__str__() # E: Incompatible types in assignment (expression has type "str", variable has type "int") -i = s in t # E: Incompatible types in assignment (expression has type "bool", variable has type "int") -t.foo # E: Tuple[Any, ...] has no attribute "foo" - -i = t.__len__() -s = t.__str__() -b = s in t - -[file builtins.py] -from typing import TypeVar, Generic -_T = TypeVar('_T') -class object: - def __init__(self) -> None: pass -class tuple(Generic[_T]): - def __len__(self) -> int: pass - def __str__(self) -> str: pass - def __contains__(self, o: object) -> bool: pass -class int: pass -class str: pass -class bool: pass -class type: pass -class function: pass - - --- For loop over tuple --- ------------------- - - -[case testForLoopOverTuple] -import typing -t = 1, 2 -for x in t: - x = 1 - x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -[builtins fixtures/for.py] - -[case testForLoopOverEmptyTuple] -import typing -t = () -for x in t: pass # E: Need type annotation for variable -[builtins fixtures/for.py] - -[case testForLoopOverNoneValuedTuple] -import typing -t = () -for x in None, None: pass # E: Need type annotation for variable -[builtins fixtures/for.py] - -[case testForLoopOverTupleAndSubtyping] -import typing -class A: pass -class B(A): pass -for x in B(), A(): - x = A() - x = B() - x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "A") -[builtins fixtures/for.py] - -[case testTupleIterable] -y = 'a' -x = sum((1,2)) -y = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") -[builtins fixtures/tuple.py] - - --- Tuple as a base type --- -------------------- - - -[case testTupleBaseClass] -import m -[file m.pyi] -from typing import Tuple -class A(Tuple[int, str]): - def f(self, x: int) -> None: - a, b = 1, '' - a, b = self - b, a = self # Error - self.f('') # Error -[builtins fixtures/tuple.py] -[out] -main:1: note: In module imported here: -tmp/m.pyi: note: In member "f" of class "A": -tmp/m.pyi:6: error: Incompatible types in assignment (expression has type "int", variable has type "str") -tmp/m.pyi:6: error: Incompatible types in assignment (expression has type "str", variable has type "int") -tmp/m.pyi:7: error: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" - -[case testInvalidTupleBaseClass] -from typing import Tuple -class A(Tuple[int, str]): pass # E: Tuple[...] not supported as a base class outside a stub file -[builtins fixtures/tuple.py] -[out] - -[case testValidTupleBaseClass] -from typing import Tuple -class A(tuple): pass -[out] - -[case testTupleBaseClass2-skip] -import m -[file m.pyi] -# This doesn't work correctly -- no errors are reported (#867) -from typing import Tuple -a = None # type: A -class A(Tuple[int, str]): pass -x, y = a -x() # Expected: "int" not callable -y() # Expected: "str" not callable -[out] -(should fail) - -[case testGenericClassWithTupleBaseClass] -from typing import TypeVar, Generic, Tuple -T = TypeVar('T') -class Test(Generic[T], Tuple[T]): pass -x = Test() # type: Test[int] -[builtins fixtures/tuple.py] -[out] -main:3: error: Tuple[...] not supported as a base class outside a stub file -main:4: error: Generic tuple types not supported - - --- Variable-length tuples (Tuple[t, ...] with literal '...') --- --------------------------------------------------------- - - -[case testIndexingVariableLengthTuple] -from typing import Tuple -x = () # type: Tuple[str, ...] -n = 5 -x[n]() # E: "str" not callable -x[3]() # E: "str" not callable -[builtins fixtures/tuple.py] - -[case testSubtypingVariableLengthTuple] -from typing import Tuple -class A: pass -class B(A): pass -def fa(t: Tuple[A, ...]) -> None: pass -def fb(t: Tuple[B, ...]) -> None: pass -ta = () # type: Tuple[A, ...] -tb = () # type: Tuple[B, ...] -fa(ta) -fa(tb) -fb(tb) -fb(ta) # E: Argument 1 to "fb" has incompatible type Tuple[A, ...]; expected Tuple[B, ...] -[builtins fixtures/tuple.py] - -[case testSubtypingFixedAndVariableLengthTuples] -from typing import Tuple -class A: pass -class B(A): pass -def fa(t: Tuple[A, ...]) -> None: pass -def fb(t: Tuple[B, ...]) -> None: pass -aa = (A(), A()) -ab = (A(), B()) -bb = (B(), B()) -fa(aa) -fa(ab) -fa(bb) -fb(bb) -fb(ab) # E: Argument 1 to "fb" has incompatible type "Tuple[A, B]"; expected Tuple[B, ...] -fb(aa) # E: Argument 1 to "fb" has incompatible type "Tuple[A, A]"; expected Tuple[B, ...] -[builtins fixtures/tuple.py] - -[case testSubtypingTupleIsContainer] -from typing import Container -a = None # type: Container[str] -a = () - -[case testSubtypingTupleIsSized] -from typing import Sized -a = None # type: Sized -a = () +-- Normal assignment and subtyping +-- ------------------------------- + + +[case testTupleAssignmentWithTupleTypes] +from typing import Tuple +t1 = None # type: Tuple[A] +t2 = None # type: Tuple[B] +t3 = None # type: Tuple[A, A] +t4 = None # type: Tuple[A, B] +t5 = None # type: Tuple[B, A] + +t1 = t2 # E: Incompatible types in assignment (expression has type "Tuple[B]", variable has type "Tuple[A]") +t1 = t3 # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A]") +t3 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A]", variable has type "Tuple[A, A]") +t3 = t4 # E: Incompatible types in assignment (expression has type "Tuple[A, B]", variable has type "Tuple[A, A]") +t3 = t5 # E: Incompatible types in assignment (expression has type "Tuple[B, A]", variable has type "Tuple[A, A]") + +# Ok +t1 = t1 +t2 = t2 +t3 = t3 +t4 = t4 +t5 = t5 + +class A: pass +class B: pass +[builtins fixtures/tuple.py] + +[case testTupleSubtyping] +from typing import Tuple +t1 = None # type: Tuple[A, A] +t2 = None # type: Tuple[A, B] +t3 = None # type: Tuple[B, A] + +t2 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A, B]") +t2 = t3 # E: Incompatible types in assignment (expression has type "Tuple[B, A]", variable has type "Tuple[A, B]") +t3 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[B, A]") +t3 = t2 # E: Incompatible types in assignment (expression has type "Tuple[A, B]", variable has type "Tuple[B, A]") + +t1 = t2 +t1 = t3 + +class A: pass +class B(A): pass +[builtins fixtures/tuple.py] + +[case testTupleCompatibilityWithOtherTypes] +from typing import Tuple +a, o = None, None # type: (A, object) +t = None # type: Tuple[A, A] + +a = t # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "A") +t = o # E: Incompatible types in assignment (expression has type "object", variable has type "Tuple[A, A]") +t = a # E: Incompatible types in assignment (expression has type "A", variable has type "Tuple[A, A]") +# TODO: callable types + tuples + +# Ok +o = t +t = None + +class A: pass +[builtins fixtures/tuple.py] + +[case testNestedTupleTypes] +from typing import Tuple +t1 = None # type: Tuple[A, Tuple[A, A]] +t2 = None # type: Tuple[B, Tuple[B, B]] + +t2 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, Tuple[A, A]]", variable has type "Tuple[B, Tuple[B, B]]") +t1 = t2 + +class A: pass +class B(A): pass +[builtins fixtures/tuple.py] + +[case testNestedTupleTypes2] +from typing import Tuple +t1 = None # type: Tuple[A, Tuple[A, A]] +t2 = None # type: Tuple[B, Tuple[B, B]] + +t2 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, Tuple[A, A]]", variable has type "Tuple[B, Tuple[B, B]]") +t1 = t2 + +class A: pass +class B(A): pass +[builtins fixtures/tuple.py] + +[case testSubtypingWithNamedTupleType] +from typing import Tuple +t1 = None # type: Tuple[A, A] +t2 = None # type: tuple + +t1 = t2 # E: Incompatible types in assignment (expression has type Tuple[Any, ...], variable has type "Tuple[A, A]") +t2 = t1 + +class A: pass +[builtins fixtures/tuple.py] + +[case testTupleInitializationWithNone] +from typing import Tuple +t = None # type: Tuple[A, A] +t = None +class A: pass +[builtins fixtures/tuple.py] + + +-- Tuple expressions +-- ----------------- + + +[case testTupleExpressions] +from typing import Tuple +t1 = None # type: tuple +t2 = None # type: Tuple[A] +t3 = None # type: Tuple[A, B] + +a, b, c = None, None, None # type: (A, B, C) + +t2 = () # E: Incompatible types in assignment (expression has type "Tuple[]", variable has type "Tuple[A]") +t2 = (a, a) # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A]") +t3 = (a, a) # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A, B]") +t3 = (b, b) # E: Incompatible types in assignment (expression has type "Tuple[B, B]", variable has type "Tuple[A, B]") +t3 = (a, b, a) # E: Incompatible types in assignment (expression has type "Tuple[A, B, A]", variable has type "Tuple[A, B]") + +t1 = () +t1 = (a,) +t2 = (a,) +t3 = (a, b) +t3 = (a, c) +t3 = (None, None) + +class A: pass +class B: pass +class C(B): pass +[builtins fixtures/tuple.py] + +[case testVoidValueInTuple] +import typing +(None, f()) # E: "f" does not return a value +(f(), None) # E: "f" does not return a value + +def f() -> None: pass +[builtins fixtures/tuple.py] + + +-- Indexing +-- -------- + + +[case testIndexingTuples] +from typing import Tuple +t1 = None # type: Tuple[A, B] +t2 = None # type: Tuple[A] +t3 = None # type: Tuple[A, B, C, D, E] +a, b = None, None # type: (A, B) +x = None # type: Tuple[A, B, C] +y = None # type: Tuple[A, C, E] +n = 0 + +a = t1[1] # E: Incompatible types in assignment (expression has type "B", variable has type "A") +b = t1[0] # E: Incompatible types in assignment (expression has type "A", variable has type "B") +t1[2] # E: Tuple index out of range +t1[3] # E: Tuple index out of range +t2[1] # E: Tuple index out of range +t1[n] # E: Tuple index must be an integer literal +t3[n:] # E: Tuple slice must be an integer literal +b = t1[(0)] # E: Incompatible types in assignment (expression has type "A", variable has type "B") + +a = t1[0] +b = t1[1] +b = t1[-1] +a = t1[(0)] +x = t3[0:3] # type (A, B, C) +y = t3[0:5:2] # type (A, C, E) +x = t3[:-2] # type (A, B, C) + +class A: pass +class B: pass +class C: pass +class D: pass +class E: pass +[builtins fixtures/tuple.py] + +[case testIndexingTuplesWithNegativeIntegers] +from typing import Tuple +t1 = None # type: Tuple[A, B] +t2 = None # type: Tuple[A] +a, b = None, None # type: A, B + +a = t1[-1] # E: Incompatible types in assignment (expression has type "B", variable has type "A") +b = t1[-2] # E: Incompatible types in assignment (expression has type "A", variable has type "B") +t1[-3] # E: Tuple index out of range +t1[-4] # E: Tuple index out of range +b = t2[(-1)] # E: Incompatible types in assignment (expression has type "A", variable has type "B") + +a = t1[-2] +b = t1[-1] +a = t2[(-1)] + +class A: pass +class B: pass +[builtins fixtures/tuple.py] + +[case testAssigningToTupleItems] +from typing import Tuple +t = None # type: Tuple[A, B] +n = 0 + +t[0] = A() # E: Unsupported target for indexed assignment +t[2] = A() # E: Unsupported target for indexed assignment +t[n] = A() # E: Unsupported target for indexed assignment + +class A: pass +class B: pass +[builtins fixtures/tuple.py] + + +-- Multiple assignment +-- ------------------- + + +[case testMultipleAssignmentWithTuples] +from typing import Tuple +t1 = None # type: Tuple[A, B] +t2 = None # type: Tuple[A, B, A] +a, b = None, None # type: (A, B) + +a, a = t1 # E: Incompatible types in assignment (expression has type "B", variable has type "A") +b, b = t1 # E: Incompatible types in assignment (expression has type "A", variable has type "B") +a, b, b = t2 # E: Incompatible types in assignment (expression has type "A", variable has type "B") + +a, b = t1 +a, b, a = t2 + +class A: pass +class B: pass +[builtins fixtures/tuple.py] + +[case testMultipleAssignmentWithInvalidNumberOfValues] +from typing import Tuple +t1 = None # type: Tuple[A, A, A] +a = None # type: A + +a, a = t1 # E: Too many values to unpack (2 expected, 3 provided) +a, a, a, a = t1 # E: Need more than 3 values to unpack (4 expected) + +a, a, a = t1 + +class A: pass +[builtins fixtures/tuple.py] + +[case testMultipleAssignmentWithTupleExpressionRvalue] + +a, b = None, None # type: (A, B) + +a, b = a, a # Fail +a, b = b, a # Fail + +a, b = a, b +a, a = a, a + +class A: pass +class B: pass +[builtins fixtures/tuple.py] +[out] +main:4: error: Incompatible types in assignment (expression has type "A", variable has type "B") +main:5: error: Incompatible types in assignment (expression has type "B", variable has type "A") +main:5: error: Incompatible types in assignment (expression has type "A", variable has type "B") + +[case testSubtypingInMultipleAssignment] + +a, b = None, None # type: (A, B) + +b, b = a, b # E: Incompatible types in assignment (expression has type "A", variable has type "B") +b, b = b, a # E: Incompatible types in assignment (expression has type "A", variable has type "B") + +a, b = b, b +b, a = b, b + +class A: pass +class B(A): pass +[builtins fixtures/tuple.py] + +[case testInitializationWithMultipleValues] + +a, b = None, None # type: (A, B) + +a1, b1 = a, a # type: (A, B) # E: Incompatible types in assignment (expression has type "A", variable has type "B") +a2, b2 = b, b # type: (A, B) # E: Incompatible types in assignment (expression has type "B", variable has type "A") +a3, b3 = a # type: (A, B) # E: '__main__.A' object is not iterable +a4, b4 = None # type: (A, B) # E: 'builtins.None' object is not iterable +a5, b5 = a, b, a # type: (A, B) # E: Too many values to unpack (2 expected, 3 provided) + +ax, bx = a, b # type: (A, B) + +class A: pass +class B: pass +[builtins fixtures/tuple.py] + +[case testMultipleAssignmentWithNonTupleRvalue] + +a, b = None, None # type: (A, B) +def f(): pass + +a, b = None # E: 'builtins.None' object is not iterable +a, b = a # E: '__main__.A' object is not iterable +a, b = f # E: 'def () -> Any' object is not iterable + +class A: pass +class B: pass +[builtins fixtures/tuple.py] + +[case testMultipleAssignmentWithIndexedLvalues] + +a, b = None, None # type: (A, B) +aa, bb = None, None # type: (AA, BB) + +a[a], b[b] = a, bb # E: Incompatible types in assignment (expression has type "A", target has type "AA") +a[a], b[b] = aa, b # E: Incompatible types in assignment (expression has type "B", target has type "BB") +a[aa], b[b] = aa, bb # E: Invalid index type "AA" for "A" +a[a], b[bb] = aa, bb # E: Invalid index type "BB" for "B" +a[a], b[b] = aa, bb + +class A: + def __setitem__(self, x: 'A', y: 'AA') -> None: pass +class B: + def __setitem__(self, x: 'B', y: 'BB') -> None: pass + +class AA: pass +class BB: pass +[builtins fixtures/tuple.py] + +[case testMultipleDeclarationWithParentheses] + +(a, b) = (None, None) # type: int, str +a = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +b = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") +a = 1 +b = '' + +[case testMultipleAssignmentWithExtraParentheses] + +a, b = None, None # type: (A, B) + +(a, b) = (a, a) # E: Incompatible types in assignment (expression has type "A", variable has type "B") +(a, b) = (b, b) # E: Incompatible types in assignment (expression has type "B", variable has type "A") +((a), (b)) = ((a), (a)) # E: Incompatible types in assignment (expression has type "A", variable has type "B") +((a), (b)) = ((b), (b)) # E: Incompatible types in assignment (expression has type "B", variable has type "A") +[a, b] = a, a # E: Incompatible types in assignment (expression has type "A", variable has type "B") +[a, b] = b, b # E: Incompatible types in assignment (expression has type "B", variable has type "A") + +(a, b) = (a, b) +((a), (b)) = ((a), (b)) +[a, b] = a, b + +class A: pass +class B: pass +[builtins fixtures/tuple.py] + +[case testMultipleAssignmentUsingSingleTupleType] +from typing import Tuple +a, b = None, None # type: Tuple[int, str] +a = 1 +b = '' +a = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +b = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testMultipleAssignmentWithMixedVariables] +a = b, c = 1, 1 +x, y = p, q = 1, 1 +u, v, w = r, s = 1, 1 # E: Need more than 2 values to unpack (3 expected) +d, e = f, g, h = 1, 1 # E: Need more than 2 values to unpack (3 expected) + + +-- Assignment to starred expressions +-- --------------------------------- + + +[case testAssignmentToStarMissingAnnotation] +from typing import List +t = 1, 2 +a, b, *c = 1, 2 # E: Need type annotation for variable +aa, bb, *cc = t # E: Need type annotation for variable +[builtins fixtures/list.py] + +[case testAssignmentToStarAnnotation] +from typing import List +li, lo = None, None # type: List[int], List[object] +a, b, *c = 1, 2 # type: int, int, *List[int] +c = lo # E: Incompatible types in assignment (expression has type List[object], variable has type List[int]) +c = li +[builtins fixtures/list.py] + +[case testAssignmentToStarCount1] +from typing import List +ca = None # type: List[int] +c = [1] +a, b, *c = 1, # E: Need more than 1 value to unpack (2 expected) +a, b, *c = 1, 2 +a, b, *c = 1, 2, 3 +a, b, *c = 1, 2, 3, 4 +[builtins fixtures/list.py] + +[case testAssignmentToStarCount2] +from typing import List +ca = None # type: List[int] +t1 = 1, +t2 = 1, 2 +t3 = 1, 2, 3 +t4 = 1, 2, 3, 4 +c = [1] +a, b, *c = t1 # E: Need more than 1 value to unpack (2 expected) +a, b, *c = t2 +a, b, *c = t3 +a, b, *c = t4 +[builtins fixtures/list.py] + +[case testAssignmentToStarFromAny] +from typing import Any +a, c = Any(1), C() +p, *q = a +c = a +c = q + +class C: pass + +[case testAssignmentToComplexStar] +from typing import List +li = None # type: List[int] +a, *(li) = 1, +a, *(b, c) = 1, 2 # E: Need more than 1 value to unpack (2 expected) +a, *(b, c) = 1, 2, 3 +a, *(b, c) = 1, 2, 3, 4 # E: Too many values to unpack (2 expected, 3 provided) +[builtins fixtures/list.py] + +[case testAssignmentToStarFromTupleType] +from typing import List, Tuple +li = None # type: List[int] +la = None # type: List[A] +ta = None # type: Tuple[A, A, A] +a, *la = ta +a, *li = ta # E +a, *na = ta +na = la +na = a # E + +class A: pass +[builtins fixtures/list.py] +[out] +main:6: error: List item 0 has incompatible type "A" +main:6: error: List item 1 has incompatible type "A" +main:9: error: Incompatible types in assignment (expression has type "A", variable has type List[A]) + +[case testAssignmentToStarFromTupleInference] +from typing import List +li = None # type: List[int] +la = None # type: List[A] +a, *l = A(), A() +l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) +l = la + +class A: pass +[builtins fixtures/list.py] +[out] + +[case testAssignmentToStarFromListInference] +from typing import List +li = None # type: List[int] +la = None # type: List[A] +a, *l = [A(), A()] +l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) +l = la + +class A: pass +[builtins fixtures/list.py] +[out] + +[case testAssignmentToStarFromTupleTypeInference] +from typing import List, Tuple +li = None # type: List[int] +la = None # type: List[A] +ta = None # type: Tuple[A, A, A] +a, *l = ta +l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) +l = la + +class A: pass +[builtins fixtures/list.py] +[out] + +[case testAssignmentToStarFromListTypeInference] +from typing import List +li = None # type: List[int] +la = None # type: List[A] +a, *l = la +l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) +l = la + +class A: pass +[builtins fixtures/list.py] +[out] + + +-- Nested tuple assignment +-- ---------------------------- + + +[case testNestedTupleAssignment1] + +a1, b1, c1 = None, None, None # type: (A, B, C) +a2, b2, c2 = None, None, None # type: (A, B, C) + +a1, (b1, c1) = a2, (b2, c2) +a1, (a1, (b1, c1)) = a2, (a2, (b2, c2)) +a1, (a1, (a1, b1)) = a1, (a1, (a1, c1)) # Fail + +class A: pass +class B: pass +class C: pass +[out] +main:7: error: Incompatible types in assignment (expression has type "C", variable has type "B") + +[case testNestedTupleAssignment2] + +a1, b1, c1 = None, None, None # type: (A, B, C) +a2, b2, c2 = None, None, None # type: (A, B, C) +t = a1, b1 + +a2, b2 = t +(a2, b2), c2 = t, c1 +(a2, c2), c2 = t, c1 # Fail +t, c2 = (a2, b2), c2 +t, c2 = (a2, a2), c2 # Fail +t = a1, a1, a1 # Fail +t = a1 # Fail +a2, a2, a2 = t # Fail +a2, = t # Fail +a2 = t # Fail + +class A: pass +class B: pass +class C: pass +[out] +main:8: error: Incompatible types in assignment (expression has type "B", variable has type "C") +main:10: error: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A, B]") +main:11: error: Incompatible types in assignment (expression has type "Tuple[A, A, A]", variable has type "Tuple[A, B]") +main:12: error: Incompatible types in assignment (expression has type "A", variable has type "Tuple[A, B]") +main:13: error: Need more than 2 values to unpack (3 expected) +main:14: error: Too many values to unpack (1 expected, 2 provided) +main:15: error: Incompatible types in assignment (expression has type "Tuple[A, B]", variable has type "A") + + +-- Error messages +-- -------------- + + +[case testTupleErrorMessages] + +a = None # type: A + +(a, a) + a # E: Unsupported left operand type for + (Tuple[A, ...]) +a + (a, a) # E: Unsupported operand types for + ("A" and "Tuple[A, A]") +f((a, a)) # E: Argument 1 to "f" has incompatible type "Tuple[A, A]"; expected "A" +(a, a).foo # E: Tuple[A, ...] has no attribute "foo" + +def f(x: 'A') -> None: pass + +class A: + def __add__(self, x: 'A') -> 'A': pass +[builtins fixtures/tuple.py] + +[case testLargeTuplesInErrorMessages] + +a = None # type: LongTypeName +a + (a, a, a, a, a, a, a) # Fail + +class LongTypeName: + def __add__(self, x: 'LongTypeName') -> 'LongTypeName': pass +[builtins fixtures/tuple.py] +[out] +main:3: error: Unsupported operand types for + ("LongTypeName" and tuple(length 7)) + + +-- Tuple methods +-- ------------- + + +[case testTupleMethods] +from typing import Tuple +t = None # type: Tuple[int, str] +i = 0 +s = '' +b = bool() + +s = t.__len__() # E: Incompatible types in assignment (expression has type "int", variable has type "str") +i = t.__str__() # E: Incompatible types in assignment (expression has type "str", variable has type "int") +i = s in t # E: Incompatible types in assignment (expression has type "bool", variable has type "int") +t.foo # E: Tuple[Any, ...] has no attribute "foo" + +i = t.__len__() +s = t.__str__() +b = s in t + +[file builtins.py] +from typing import TypeVar, Generic +_T = TypeVar('_T') +class object: + def __init__(self) -> None: pass +class tuple(Generic[_T]): + def __len__(self) -> int: pass + def __str__(self) -> str: pass + def __contains__(self, o: object) -> bool: pass +class int: pass +class str: pass +class bool: pass +class type: pass +class function: pass + + +-- For loop over tuple +-- ------------------- + + +[case testForLoopOverTuple] +import typing +t = 1, 2 +for x in t: + x = 1 + x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +[builtins fixtures/for.py] + +[case testForLoopOverEmptyTuple] +import typing +t = () +for x in t: pass # E: Need type annotation for variable +[builtins fixtures/for.py] + +[case testForLoopOverNoneValuedTuple] +import typing +t = () +for x in None, None: pass # E: Need type annotation for variable +[builtins fixtures/for.py] + +[case testForLoopOverTupleAndSubtyping] +import typing +class A: pass +class B(A): pass +for x in B(), A(): + x = A() + x = B() + x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "A") +[builtins fixtures/for.py] + +[case testTupleIterable] +y = 'a' +x = sum((1,2)) +y = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") +[builtins fixtures/tuple.py] + + +-- Tuple as a base type +-- -------------------- + + +[case testTupleBaseClass] +import m +[file m.pyi] +from typing import Tuple +class A(Tuple[int, str]): + def f(self, x: int) -> None: + a, b = 1, '' + a, b = self + b, a = self # Error + self.f('') # Error +[builtins fixtures/tuple.py] +[out] +main:1: note: In module imported here: +tmp/m.pyi: note: In member "f" of class "A": +tmp/m.pyi:6: error: Incompatible types in assignment (expression has type "int", variable has type "str") +tmp/m.pyi:6: error: Incompatible types in assignment (expression has type "str", variable has type "int") +tmp/m.pyi:7: error: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" + +[case testInvalidTupleBaseClass] +from typing import Tuple +class A(Tuple[int, str]): pass # E: Tuple[...] not supported as a base class outside a stub file +[builtins fixtures/tuple.py] +[out] + +[case testValidTupleBaseClass] +from typing import Tuple +class A(tuple): pass +[out] + +[case testTupleBaseClass2-skip] +import m +[file m.pyi] +# This doesn't work correctly -- no errors are reported (#867) +from typing import Tuple +a = None # type: A +class A(Tuple[int, str]): pass +x, y = a +x() # Expected: "int" not callable +y() # Expected: "str" not callable +[out] +(should fail) + +[case testGenericClassWithTupleBaseClass] +from typing import TypeVar, Generic, Tuple +T = TypeVar('T') +class Test(Generic[T], Tuple[T]): pass +x = Test() # type: Test[int] +[builtins fixtures/tuple.py] +[out] +main:3: error: Tuple[...] not supported as a base class outside a stub file +main:4: error: Generic tuple types not supported + + +-- Variable-length tuples (Tuple[t, ...] with literal '...') +-- --------------------------------------------------------- + + +[case testIndexingVariableLengthTuple] +from typing import Tuple +x = () # type: Tuple[str, ...] +n = 5 +x[n]() # E: "str" not callable +x[3]() # E: "str" not callable +[builtins fixtures/tuple.py] + +[case testSubtypingVariableLengthTuple] +from typing import Tuple +class A: pass +class B(A): pass +def fa(t: Tuple[A, ...]) -> None: pass +def fb(t: Tuple[B, ...]) -> None: pass +ta = () # type: Tuple[A, ...] +tb = () # type: Tuple[B, ...] +fa(ta) +fa(tb) +fb(tb) +fb(ta) # E: Argument 1 to "fb" has incompatible type Tuple[A, ...]; expected Tuple[B, ...] +[builtins fixtures/tuple.py] + +[case testSubtypingFixedAndVariableLengthTuples] +from typing import Tuple +class A: pass +class B(A): pass +def fa(t: Tuple[A, ...]) -> None: pass +def fb(t: Tuple[B, ...]) -> None: pass +aa = (A(), A()) +ab = (A(), B()) +bb = (B(), B()) +fa(aa) +fa(ab) +fa(bb) +fb(bb) +fb(ab) # E: Argument 1 to "fb" has incompatible type "Tuple[A, B]"; expected Tuple[B, ...] +fb(aa) # E: Argument 1 to "fb" has incompatible type "Tuple[A, A]"; expected Tuple[B, ...] +[builtins fixtures/tuple.py] + +[case testSubtypingTupleIsContainer] +from typing import Container +a = None # type: Container[str] +a = () + +[case testSubtypingTupleIsSized] +from typing import Sized +a = None # type: Sized +a = () From 158b0681d38af993f9f006b7608ff3dedf3cb7a9 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Sun, 14 Aug 2016 19:22:01 +0300 Subject: [PATCH 21/74] flake8-ing --- mypy/join.py | 2 +- mypy/messages.py | 1 - mypy/nodes.py | 3 ++- mypy/semanal.py | 8 ++++---- mypy/subtypes.py | 4 ++-- mypy/typeanal.py | 18 ++++++++---------- mypy/types.py | 10 ++++------ 7 files changed, 21 insertions(+), 25 deletions(-) diff --git a/mypy/join.py b/mypy/join.py index 684d62cfeaebd..061052e6d87f8 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -217,7 +217,7 @@ def visit_tuple_type(self, t: TupleType) -> Type: for i in range(t.length()): items.append(self.join(t.items[i], self.s.items[i])) if (not (isinstance(t, NamedTupleType) and isinstance(self.s, NamedTupleType)) - or self.s.attrs == t.attrs): + or self.s.attrs == t.attrs): return self.s.with_types(items) # TODO: What if the fallback types are different? return TupleType(items, t.fallback) diff --git a/mypy/messages.py b/mypy/messages.py index 5d82c4845782a..d8aed9626a49d 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -487,7 +487,6 @@ def incompatible_argument(self, n: int, m: int, callee: CallableType, arg_type: except IndexError: # Varargs callees expected_type = callee.arg_types[-1] arg_type_str, expected_type_str = self.format_distinctly(arg_type, expected_type) - #import pdb; pdb.set_trace() if arg_kind == ARG_STAR: arg_type_str = '*' + arg_type_str elif arg_kind == ARG_STAR2: diff --git a/mypy/nodes.py b/mypy/nodes.py index 60d7b5d18944f..3e4f3f4fda129 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1179,7 +1179,7 @@ def __init__(self, callee: Expression, args: List[Expression], arg_kinds: List[i arg_names: List[str] = None, analyzed: Expression = None) -> None: if not arg_names: arg_names = [None] * len(args) - + self.callee = callee self.args = args self.arg_kinds = arg_kinds @@ -1985,6 +1985,7 @@ def NamedTupleTypeInfo(tup: 'mypy.types.NamedTupleType', *args) -> TypeInfo: info.mro = [info] + tup.fallback.type.mro return info + class SymbolTableNode: # Kind of node. Possible values: # - LDEF: local definition (of any kind) diff --git a/mypy/semanal.py b/mypy/semanal.py index 0f4331499d7cc..d7fe54678b70d 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1541,15 +1541,15 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], this_type = self_type(info) # Add __init__, _replace, _asdict methods and _fields static field self.add_namedtuple_method('_replace', symbols, info, this_type, this_type, - self.make_factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) + self.factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) self.add_namedtuple_method('__init__', symbols, info, this_type, NoneTyp(), - self.make_factory_args(vars, ARG_POS), info.name()) + self.factory_args(vars, ARG_POS), info.name()) # TODO: refine to OrderedDict[Union[types]] self.add_namedtuple_method('_asdict', symbols, info, this_type, AnyType(), []) return info - def make_factory_args(self, vars: List[Var], kind: int, - initializer: Expression = None) -> List[Argument]: + def factory_args(self, vars: List[Var], kind: int, + initializer: Expression = None) -> List[Argument]: return [Argument(var, var.type, initializer, kind) for var in vars] def add_namedtuple_method(self, funcname: str, symbols: SymbolTable, info: TypeInfo, diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 6872e297c3a64..32b44c36e89f4 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -2,8 +2,8 @@ from mypy.types import ( Type, AnyType, UnboundType, TypeVisitor, ErrorType, Void, NoneTyp, - Instance, TypeVarType, CallableType, TupleType, NamedTupleType, UnionType, Overloaded, ErasedType, TypeList, - PartialType, DeletedType, UninhabitedType, TypeType, is_named_instance + Instance, TypeVarType, CallableType, TupleType, NamedTupleType, UnionType, Overloaded, + ErasedType, TypeList, PartialType, DeletedType, UninhabitedType, TypeType, is_named_instance ) import mypy.applytype import mypy.constraints diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 1841f283a19e1..092849567c425 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -1,15 +1,15 @@ """Semantic analysis of types""" -from typing import Callable, cast, List, Tuple +from typing import Callable, cast, List from mypy.types import ( - Type, UnboundType, TypeVarType, TupleType, NamedTupleType, UnionType, Instance, AnyType, CallableType, - Void, NoneTyp, DeletedType, TypeList, TypeVarDef, TypeVisitor, StarType, PartialType, - EllipsisType, UninhabitedType, TypeType + Type, UnboundType, TypeVarType, TupleType, NamedTupleType, UnionType, Instance, + AnyType, CallableType, Void, NoneTyp, DeletedType, TypeList, TypeVarDef, TypeVisitor, + StarType, PartialType, EllipsisType, UninhabitedType, TypeType ) from mypy.nodes import ( BOUND_TVAR, TYPE_ALIAS, UNBOUND_IMPORTED, - TypeInfo, Context, SymbolTableNode, TypeVarExpr, Var, Node, + TypeInfo, Context, SymbolTableNode, Var, Node, IndexExpr, RefExpr ) from mypy.sametypes import is_same_type @@ -171,7 +171,7 @@ def visit_unbound_type(self, t: UnboundType) -> Type: return AnyType() # import pdb; pdb.set_trace() if info.is_named_tuple: - return NamedTupleType(info.name(), info.attrs, + return NamedTupleType(info.name(), info.attrs, self.anal_array(info.tuple_type.items), fallback=instance, line=t.line) @@ -224,10 +224,8 @@ def visit_tuple_type(self, t: TupleType) -> Type: return TupleType(self.anal_array(t.items), fallback, t.line) def visit_namedtuple_type(self, t: NamedTupleType) -> Type: - return NamedTupleType(t.name, t.attrs, - self.anal_array(t.items), - fallback=t.fallback, - line=t.line) + return NamedTupleType(t.name, t.attrs, self.anal_array(t.items), + fallback=t.fallback, line=t.line) def visit_star_type(self, t: StarType) -> Type: return StarType(t.type.accept(self), t.line) diff --git a/mypy/types.py b/mypy/types.py index 7d8dcb1c1a603..b23e84333be5d 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -735,7 +735,6 @@ class TupleType(Type): def __init__(self, items: List[Type], fallback: Instance, line: int = -1, implicit: bool = False) -> None: - #import pdb; pdb.set_trace() self.items = items self.fallback = fallback self.implicit = implicit @@ -790,7 +789,7 @@ def with_types(self, items: List[Type]) -> 'NamedTupleType': def __eq__(self, other) -> bool: return self.serialize() == other.serialize() - + def serialize(self) -> JsonDict: d = super().serialize() d.update({'.class': 'NamedTupleType', @@ -808,12 +807,11 @@ def deserialize(cls, data: JsonDict) -> 'NamedTupleType': tup.items, tup.fallback, implicit=tup.implicit) - + def accept(self, visitor: 'TypeVisitor[T]') -> T: - try: + if hasattr(visitor, "visit_namedtuple_type"): return visitor.visit_namedtuple_type(self) - except: - return super().accept(visitor) + return super().accept(visitor) class StarType(Type): From 90ffb08c0e15b57dd0c64b61e6c918ac20b706bf Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Sun, 14 Aug 2016 19:45:59 +0300 Subject: [PATCH 22/74] s/self_type/with_fallback. Fix visitor type --- mypy/semanal.py | 2 +- mypy/types.py | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index d7fe54678b70d..bc27841d2d327 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -2638,7 +2638,7 @@ def self_type(typ: TypeInfo) -> Union[Instance, TupleType]: inst = Instance(typ, tv) if typ.tuple_type is None: return inst - return typ.tuple_type.self_type(inst) + return typ.tuple_type.with_fallback(inst) def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: diff --git a/mypy/types.py b/mypy/types.py index b23e84333be5d..090f01ed6d001 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -760,7 +760,7 @@ def deserialize(cls, data: JsonDict) -> 'TupleType': Instance.deserialize(data['fallback']), implicit=data['implicit']) - def self_type(self, inst: Instance) -> 'TupleType': + def with_fallback(self, inst: Instance) -> 'TupleType': return TupleType(self.items, inst, self.line, self.implicit) def with_types(self, items: List[Type]) -> 'TupleType': @@ -780,7 +780,7 @@ def __init__(self, name: str, attrs: List[str], *args, **kwargs) -> None: self.name = name super().__init__(*args, **kwargs) - def self_type(self, inst: Instance) -> 'NamedTupleType': + def with_fallback(self, inst: Instance) -> 'NamedTupleType': return NamedTupleType(self.name, self.attrs, self.items, inst, self.line) def with_types(self, items: List[Type]) -> 'NamedTupleType': @@ -809,9 +809,7 @@ def deserialize(cls, data: JsonDict) -> 'NamedTupleType': implicit=tup.implicit) def accept(self, visitor: 'TypeVisitor[T]') -> T: - if hasattr(visitor, "visit_namedtuple_type"): - return visitor.visit_namedtuple_type(self) - return super().accept(visitor) + return visitor.visit_namedtuple_type(self) class StarType(Type): @@ -1070,6 +1068,9 @@ def visit_overloaded(self, t: Overloaded) -> T: def visit_tuple_type(self, t: TupleType) -> T: pass + def visit_namedtuple_type(self, t: NamedTupleType) -> T: + return self.visit_tuple_type(t) + def visit_star_type(self, t: StarType) -> T: raise self._notimplemented_helper('star_type') @@ -1142,6 +1143,10 @@ def visit_tuple_type(self, t: TupleType) -> Type: cast(Any, t.fallback.accept(self)), t.line) + def visit_namedtuple_type(self, t: NamedTupleType) -> Type: + t = t.with_types(self.translate_types(t.items)) + return t.with_fallback(cast(Any, t.fallback.accept(self))) + def visit_star_type(self, t: StarType) -> Type: return StarType(t.type.accept(self), t.line) From 9a6e6cda84c28ddd2c3e7a8a8518598f5a6e155b Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Sun, 14 Aug 2016 20:44:48 +0300 Subject: [PATCH 23/74] uniform copy_with. typechecks. --- mypy/checker.py | 2 +- mypy/expandtype.py | 2 +- mypy/join.py | 2 +- mypy/nodes.py | 1 + mypy/semanal.py | 5 ++--- mypy/typeanal.py | 25 +++++++++---------------- mypy/types.py | 35 ++++++++++++++++++----------------- 7 files changed, 33 insertions(+), 39 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 7063758096dfa..c457ca712ad30 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -801,7 +801,7 @@ def visit_class_def(self, defn: ClassDef) -> Type: with self.binder.frame_context(): self.accept(defn.defs) self.binder = old_binder - if not hasattr(defn, 'incompatible_baseclass'): + if not defn.has_incompatible_baseclass: # Otherwise we've already found errors; more errors are not useful self.check_multiple_inheritance(typ) self.leave_partial_types() diff --git a/mypy/expandtype.py b/mypy/expandtype.py index ce6d5190fac6a..25a34ea06b2a3 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -88,7 +88,7 @@ def visit_overloaded(self, t: Overloaded) -> Type: return Overloaded(items) def visit_tuple_type(self, t: TupleType) -> Type: - return t.with_types(self.expand_types(t.items)) + return t.copy_with(items=self.expand_types(t.items)) def visit_union_type(self, t: UnionType) -> Type: # After substituting for type variables in t.items, diff --git a/mypy/join.py b/mypy/join.py index 061052e6d87f8..f9ae697cfba39 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -218,7 +218,7 @@ def visit_tuple_type(self, t: TupleType) -> Type: items.append(self.join(t.items[i], self.s.items[i])) if (not (isinstance(t, NamedTupleType) and isinstance(self.s, NamedTupleType)) or self.s.attrs == t.attrs): - return self.s.with_types(items) + return self.s.copy_with(items=items) # TODO: What if the fallback types are different? return TupleType(items, t.fallback) else: diff --git a/mypy/nodes.py b/mypy/nodes.py index 3e4f3f4fda129..e0e9b080d5371 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -636,6 +636,7 @@ class ClassDef(Statement): decorators = None # type: List[Expression] # Built-in/extension class? (single implementation inheritance only) is_builtinclass = False + has_incompatible_baseclass = False def __init__(self, name: str, diff --git a/mypy/semanal.py b/mypy/semanal.py index bc27841d2d327..fb7981e1fee88 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -756,7 +756,7 @@ def analyze_base_classes(self, defn: ClassDef) -> None: if isinstance(base, TupleType): if info.tuple_type: self.fail("Class has two incompatible bases derived from tuple", defn) - defn.incompatible_baseclass = True + defn.has_incompatible_baseclass = True if (not self.is_stub_file and not info.is_named_tuple and base.fallback.type.fullname() == 'builtins.tuple'): @@ -1531,7 +1531,6 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], class_def = ClassDef(name, Block([])) class_def.fullname = self.qualified_name(name) info = NamedTupleTypeInfo(tup, symbols, class_def) - info.attrs = items vars = [Var(item, typ) for item, typ in zip(items, types)] # Add named tuple items as attributes. # TODO: Make them read-only. @@ -2638,7 +2637,7 @@ def self_type(typ: TypeInfo) -> Union[Instance, TupleType]: inst = Instance(typ, tv) if typ.tuple_type is None: return inst - return typ.tuple_type.with_fallback(inst) + return typ.tuple_type.copy_with(fallback=inst) def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 092849567c425..241febab24a95 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -161,23 +161,16 @@ def visit_unbound_type(self, t: UnboundType) -> Type: # valid count at this point. Thus we may construct an # Instance with an invalid number of type arguments. instance = Instance(info, self.anal_array(t.args), t.line) - if info.tuple_type is None: + tup = info.tuple_type + if tup is None: return instance - else: - # The class has a Tuple[...] base class so it will be - # represented as a tuple type. - if t.args: - self.fail('Generic tuple types not supported', t) - return AnyType() - # import pdb; pdb.set_trace() - if info.is_named_tuple: - return NamedTupleType(info.name(), info.attrs, - self.anal_array(info.tuple_type.items), - fallback=instance, - line=t.line) - return TupleType(self.anal_array(info.tuple_type.items), - fallback=instance, - line=t.line) + # The class has a Tuple[...] base class so it will be + # represented as a tuple type. + if t.args: + self.fail('Generic tuple types not supported', t) + return AnyType() + return tup.copy_with(items=self.anal_array(tup.items), + fallback=instance) else: return AnyType() diff --git a/mypy/types.py b/mypy/types.py index 090f01ed6d001..bc8eec17e27ff 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -760,11 +760,13 @@ def deserialize(cls, data: JsonDict) -> 'TupleType': Instance.deserialize(data['fallback']), implicit=data['implicit']) - def with_fallback(self, inst: Instance) -> 'TupleType': - return TupleType(self.items, inst, self.line, self.implicit) - - def with_types(self, items: List[Type]) -> 'TupleType': - return TupleType(items, self.fallback, self.line, self.implicit) + def copy_with(self, *, fallback: Instance = None, + items: List[Type] = None) -> 'TupleType': + if fallback is None: + fallback = self.fallback + if items is None: + items = self.items + return TupleType(items, fallback, self.line) def slice(self, begin: int, stride: int, end: int) -> 'TupleType': return TupleType(self.items[begin:end:stride], self.fallback, @@ -776,19 +778,18 @@ class NamedTupleType(TupleType): name = None # type: str def __init__(self, name: str, attrs: List[str], *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + assert len(self.items) == len(attrs) self.attrs = attrs self.name = name - super().__init__(*args, **kwargs) - - def with_fallback(self, inst: Instance) -> 'NamedTupleType': - return NamedTupleType(self.name, self.attrs, self.items, inst, self.line) - - def with_types(self, items: List[Type]) -> 'NamedTupleType': - return NamedTupleType(self.name, self.attrs, items, - self.fallback, self.line, self.implicit) - def __eq__(self, other) -> bool: - return self.serialize() == other.serialize() + def copy_with(self, *, fallback: Instance = None, + items: List[Type] = None) -> 'NamedTupleType': + if fallback is None: + fallback = self.fallback + if items is None: + items = self.items + return NamedTupleType(self.name, self.attrs, items, fallback, self.line) def serialize(self) -> JsonDict: d = super().serialize() @@ -1144,8 +1145,8 @@ def visit_tuple_type(self, t: TupleType) -> Type: t.line) def visit_namedtuple_type(self, t: NamedTupleType) -> Type: - t = t.with_types(self.translate_types(t.items)) - return t.with_fallback(cast(Any, t.fallback.accept(self))) + return t.copy_with(items=self.translate_types(t.items), + fallback=cast(Any, t.fallback.accept(self))) def visit_star_type(self, t: StarType) -> Type: return StarType(t.type.accept(self), t.line) From 76d709e4c2a79f48ea3291386052c86bd55726f3 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Sun, 14 Aug 2016 20:54:08 +0300 Subject: [PATCH 24/74] uniform copy_with. typechecks. --- mypy/semanal.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mypy/semanal.py b/mypy/semanal.py index fb7981e1fee88..28ee0ed69cf62 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1466,6 +1466,7 @@ def parse_namedtuple_args(self, call: CallExpr, if len(args) < 2: return self.fail_namedtuple_arg("Too few arguments for namedtuple()", call) if len(args) > 2: + # FIX incorrect. There are two additional parameters return self.fail_namedtuple_arg("Too many arguments for namedtuple()", call) if call.arg_kinds != [ARG_POS, ARG_POS]: return self.fail_namedtuple_arg("Unexpected arguments to namedtuple()", call) @@ -1536,6 +1537,7 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], # TODO: Make them read-only. for var in vars: var.info = info + var.is_property = True symbols[var.name()] = SymbolTableNode(MDEF, var) this_type = self_type(info) # Add __init__, _replace, _asdict methods and _fields static field From 20bf222edf1fc7d2030e09ee057ee1c82209be6e Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Mon, 15 Aug 2016 22:33:15 +0300 Subject: [PATCH 25/74] rename tests to startwith NamedTuple --- test-data/unit/check-namedtuple.test | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index adb07ec522705..74e5872aed256 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -14,7 +14,7 @@ a = x[1] a, b, c = x # E: Need more than 2 values to unpack (3 expected) x[2] # E: Tuple index out of range -[case testAccessingNamedTupleAttributes] +[case testNamedTupleAccessingAttributes] from collections import namedtuple X = namedtuple('X', ['x', 'y']) @@ -23,7 +23,7 @@ x.x x.y x.z # E: "X" has no attribute "z" -[case testCreateNamedTupleWithPositionalArguments] +[case testNamedTupleCreateWithPositionalArguments] from collections import namedtuple X = namedtuple('X', ['x', 'y']) @@ -42,7 +42,7 @@ x = X(1, y='x') x = X(x=1, z=1) # E: Unexpected keyword argument "z" for "X" x = X(y=1) # E: Missing positional argument "x" in call to "X" -[case testCreateNamedTupleAndUseAsTuple] +[case testNamedTupleCreateAndUseAsTuple] from collections import namedtuple X = namedtuple('X', ['x', 'y']) @@ -96,20 +96,20 @@ s = x.a # E: Incompatible types in assignment (expression has type "int", varia i, s = x s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") -[case testTwoNamedTuplesAsBaseClasses] +[case testNamedTuplesTwoAsBaseClasses] from typing import NamedTuple A = NamedTuple('A', [('a', int)]) B = NamedTuple('B', [('a', int)]) class X(A, B): # E: Class has two incompatible bases derived from tuple pass -[case testTwoNamedTuplesAsBaseClasses2] +[case testNamedTuplesTwoAsBaseClasses2] from typing import NamedTuple A = NamedTuple('A', [('a', int)]) class X(A, NamedTuple('B', [('a', int)])): # E: Class has two incompatible bases derived from tuple pass -[case testSelfTypeWithNamedTupleAsBase] +[case testNamedTupleSelfTypeWithNamedTupleAsBase] from typing import NamedTuple A = NamedTuple('A', [('a', int), ('b', str)]) class B(A): @@ -124,7 +124,7 @@ class B(A): [out] main: note: In member "f" of class "B": -[case testTypeReferenceToClassDerivedFromNamedTuple] +[case testNamedTupleTypeReferenceToClassDerivedFrom] from typing import NamedTuple A = NamedTuple('A', [('a', int), ('b', str)]) class B(A): From a2696c2abc6d20954f2063531bca50ec033ea076 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Mon, 15 Aug 2016 23:26:34 +0300 Subject: [PATCH 26/74] Read-only attributes. Can't set new attributes in subclasses yet --- mypy/checkmember.py | 4 +++- mypy/messages.py | 1 - mypy/semanal.py | 3 +-- test-data/unit/check-namedtuple.test | 22 +++++++++++++++++----- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/mypy/checkmember.py b/mypy/checkmember.py index cc0be2058fbf9..b5f5ae1f2e3cb 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -151,7 +151,6 @@ def analyze_member_var_access(name: str, itype: Instance, info: TypeInfo, if isinstance(vv, Decorator): # The associated Var node of a decorator contains the type. v = vv.var - if isinstance(v, Var): return analyze_var(name, v, itype, info, node, is_lvalue, msg, not_ready_callback) elif isinstance(v, FuncDef): @@ -191,6 +190,9 @@ def analyze_var(name: str, var: Var, itype: Instance, info: TypeInfo, node: Cont if isinstance(typ, PartialType): return handle_partial_attribute_type(typ, is_lvalue, msg, var) t = expand_type_by_instance(typ, itype) + if is_lvalue and var.is_property and not var.is_settable_property: + # TODO allow setting attributes in subclass (although it is probably an error) + msg.read_only_property(name, info, node) if var.is_initialized_in_class and isinstance(t, FunctionLike): if is_lvalue: if var.is_property: diff --git a/mypy/messages.py b/mypy/messages.py index d8aed9626a49d..209010764eca3 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -146,7 +146,6 @@ def report(self, msg: str, context: Context, severity: str, file: str = None) -> def fail(self, msg: str, context: Context, file: str = None) -> None: """Report an error message (unless disabled).""" - # import pdb; pdb.set_trace() self.report(msg, context, 'error', file=file) def note(self, msg: str, context: Context, file: str = None) -> None: diff --git a/mypy/semanal.py b/mypy/semanal.py index 28ee0ed69cf62..27a17b6277a8b 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1545,7 +1545,7 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], self.factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) self.add_namedtuple_method('__init__', symbols, info, this_type, NoneTyp(), self.factory_args(vars, ARG_POS), info.name()) - # TODO: refine to OrderedDict[Union[types]] + # TODO: refine to OrderedDict[str, Union[types]] self.add_namedtuple_method('_asdict', symbols, info, this_type, AnyType(), []) return info @@ -1559,7 +1559,6 @@ def add_namedtuple_method(self, funcname: str, symbols: SymbolTable, info: TypeI types = [arg.type_annotation for arg in args] items = [arg.variable.name() for arg in args] arg_kinds = [arg.kind for arg in args] - # TODO: Make sure that the self argument name is not visible? signature = CallableType(types, arg_kinds, items, ret, self.named_type('__builtins__.function'), name=name or info.name() + '.' + funcname) diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 74e5872aed256..ea12d058b8204 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -1,8 +1,3 @@ -[case testNamedTupleDeclaration] -from typing import NamedTuple - -A = NamedTuple('A', [('x', str), ('y', int)]) - [case testNamedTupleUsedAsTuple] from collections import namedtuple @@ -23,6 +18,21 @@ x.x x.y x.z # E: "X" has no attribute "z" +[case testNamedTupleAttributesAreReadOnly] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +x.x = 5 # E: Property "x" defined in "X" is read-only +x.y = 5 # E: Property "y" defined in "X" is read-only +x.z = 5 # E: "X" has no attribute "z" + +class A(X): pass +a = None # type: A +a.x = 5 # E: Property "x" defined in "A" is read-only +a.y = 5 # E: Property "y" defined in "A" is read-only +-- a.z = 5 # not supported yet + [case testNamedTupleCreateWithPositionalArguments] from collections import namedtuple @@ -164,6 +174,7 @@ a = A(1) a = l[0] (i,) = l[0] i, i = l[0] # E: Need more than 1 value to unpack (2 expected) +l = [A(1)] a = (1,) # E: Incompatible types in assignment (expression has type "Tuple[int]", \ variable has type "A") [builtins fixtures/list.py] @@ -203,6 +214,7 @@ from collections import namedtuple X = namedtuple('X', ['x', 'y']) x = None # type: X x = x._replace(y=5) +x = x._replace(x=3, y=5) x = x._replace(z=5) # E: Unexpected keyword argument "z" for X._replace x = x._replace(5) # E: Too many positional arguments for X._replace From 9e1b866b45113fcbff4430bc30c009fcb0246e25 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Tue, 16 Aug 2016 00:28:24 +0300 Subject: [PATCH 27/74] update semantic namedtuple repr name --- test-data/unit/semanal-namedtuple.test | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test-data/unit/semanal-namedtuple.test b/test-data/unit/semanal-namedtuple.test index 9ff9f6af6e8a1..78a678b974adf 100644 --- a/test-data/unit/semanal-namedtuple.test +++ b/test-data/unit/semanal-namedtuple.test @@ -1,6 +1,6 @@ -- Semantic analysis of named tuples -[case testSimpleNamedtuple] +[case testSimpleNamedTuple] from collections import namedtuple N = namedtuple('N', ['a']) def f() -> N: pass @@ -9,14 +9,14 @@ MypyFile:1( ImportFrom:1(collections, [namedtuple]) AssignmentStmt:2( NameExpr(N* [__main__.N]) - NamedTupleExpr:2(N, Tuple[Any])) + NamedTupleExpr:2(N, N[('a', Any)])) FuncDef:3( f - def () -> Tuple[Any, fallback=__main__.N] + def () -> N[('a', Any)] Block:3( PassStmt:3()))) -[case testTwoItemNamedtuple] +[case testTwoItemNamedTuple] from collections import namedtuple N = namedtuple('N', ['a', 'xyz']) def f() -> N: pass @@ -25,14 +25,14 @@ MypyFile:1( ImportFrom:1(collections, [namedtuple]) AssignmentStmt:2( NameExpr(N* [__main__.N]) - NamedTupleExpr:2(N, Tuple[Any, Any])) + NamedTupleExpr:2(N, N[('a', Any), ('xyz', Any)])) FuncDef:3( f - def () -> Tuple[Any, Any, fallback=__main__.N] + def () -> N[('a', Any), ('xyz', Any)] Block:3( PassStmt:3()))) -[case testTwoItemNamedtupleWithShorthandSyntax] +[case testTwoItemNamedTupleWithShorthandSyntax] from collections import namedtuple N = namedtuple('N', ' a xyz ') def f() -> N: pass @@ -41,10 +41,10 @@ MypyFile:1( ImportFrom:1(collections, [namedtuple]) AssignmentStmt:2( NameExpr(N* [__main__.N]) - NamedTupleExpr:2(N, Tuple[Any, Any])) + NamedTupleExpr:2(N, N[('a', Any), ('xyz', Any)])) FuncDef:3( f - def () -> Tuple[Any, Any, fallback=__main__.N] + def () -> N[('a', Any), ('xyz', Any)] Block:3( PassStmt:3()))) @@ -57,7 +57,7 @@ MypyFile:1( ImportFrom:1(typing, [NamedTuple]) AssignmentStmt:2( NameExpr(N* [__main__.N]) - NamedTupleExpr:2(N, Tuple[builtins.int, builtins.str]))) + NamedTupleExpr:2(N, N[('a', builtins.int), ('b', builtins.str)]))) [case testNamedTupleBaseClass] from collections import namedtuple @@ -68,11 +68,11 @@ MypyFile:1( ImportFrom:1(collections, [namedtuple]) AssignmentStmt:2( NameExpr(N* [__main__.N]) - NamedTupleExpr:2(N, Tuple[Any])) + NamedTupleExpr:2(N, N[('x', Any)])) ClassDef:3( A TupleType( - Tuple[Any, fallback=__main__.N]) + N[('x', Any)]) BaseType( __main__.N) PassStmt:3())) From f7b531fbbd30a9c330a062925b20ad50f77c6348 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Tue, 16 Aug 2016 01:37:56 +0300 Subject: [PATCH 28/74] support _fields --- mypy/semanal.py | 42 +++++++++++++++------------- mypy/types.py | 10 +++++++ test-data/unit/check-namedtuple.test | 12 ++++++++ 3 files changed, 45 insertions(+), 19 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 27a17b6277a8b..6fea9755edc23 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -43,8 +43,10 @@ traverse the entire AST. """ +from functools import partial + from typing import ( - List, Dict, Set, Tuple, cast, Any, overload, TypeVar, Union, Optional, Callable + List, Dict, Set, Tuple, cast, Any, TypeVar, Union, Optional, Callable ) from mypy.nodes import ( @@ -71,18 +73,15 @@ from mypy.errors import Errors, report_internal_error from mypy.types import ( NoneTyp, CallableType, Overloaded, Instance, Type, TypeVarType, AnyType, - FunctionLike, UnboundType, TypeList, ErrorType, TypeVarDef, + FunctionLike, UnboundType, TypeList, TypeVarDef, replace_leading_arg_type, TupleType, UnionType, StarType, EllipsisType, NamedTupleType ) from mypy.nodes import function_type, implicit_module_attrs from mypy.typeanal import TypeAnalyser, TypeAnalyserPass3, analyze_type_alias from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError -from mypy.lex import lex -from mypy.parsetype import parse_type from mypy.sametypes import is_same_type from mypy.erasetype import erase_typevars -from mypy import defaults from mypy.options import Options @@ -1533,28 +1532,36 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], class_def.fullname = self.qualified_name(name) info = NamedTupleTypeInfo(tup, symbols, class_def) vars = [Var(item, typ) for item, typ in zip(items, types)] - # Add named tuple items as attributes. - # TODO: Make them read-only. + # Add namedtuple items as attributes. for var in vars: var.info = info var.is_property = True symbols[var.name()] = SymbolTableNode(MDEF, var) + + typ = TupleType([self.named_type('__builtins__.str') for _ in items], + self.named_type('__builtins__.tuple', [AnyType()])) + _fields = Var('_fields', typ) + _fields.info = info + _fields.is_initialized_in_class = True + symbols[_fields.name()] = SymbolTableNode(MDEF, _fields) + this_type = self_type(info) - # Add __init__, _replace, _asdict methods and _fields static field - self.add_namedtuple_method('_replace', symbols, info, this_type, this_type, - self.factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) - self.add_namedtuple_method('__init__', symbols, info, this_type, NoneTyp(), - self.factory_args(vars, ARG_POS), info.name()) + add_method = partial(self.add_namedtuple_method, symbols, info, this_type) + add_method('_replace', this_type, + self.factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) + add_method('__init__', NoneTyp(), + self.factory_args(vars, ARG_POS), info.name()) # TODO: refine to OrderedDict[str, Union[types]] - self.add_namedtuple_method('_asdict', symbols, info, this_type, AnyType(), []) + add_method('_asdict', AnyType(), []) + # self.add_namedtuple_method('_source', self.named_type_or_none('builtins.str'), []) return info def factory_args(self, vars: List[Var], kind: int, initializer: Expression = None) -> List[Argument]: return [Argument(var, var.type, initializer, kind) for var in vars] - def add_namedtuple_method(self, funcname: str, symbols: SymbolTable, info: TypeInfo, - this_type: Type, ret: Type, args: List[Argument], name=None) -> None: + def add_namedtuple_method(self, symbols: SymbolTable, info: TypeInfo, this_type: Type, + funcname: str, ret: Type, args: List[Argument], name=None) -> None: args = [Argument(Var('self'), this_type, None, ARG_POS)] + args types = [arg.type_annotation for arg in args] items = [arg.variable.name() for arg in args] @@ -1562,10 +1569,7 @@ def add_namedtuple_method(self, funcname: str, symbols: SymbolTable, info: TypeI signature = CallableType(types, arg_kinds, items, ret, self.named_type('__builtins__.function'), name=name or info.name() + '.' + funcname) - func = FuncDef(funcname, - args, - Block([]), - typ=signature) + func = FuncDef(funcname, args, Block([]), typ=signature) func.info = info symbols[funcname] = SymbolTableNode(MDEF, func) diff --git a/mypy/types.py b/mypy/types.py index bc8eec17e27ff..c900ac1644bae 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -774,6 +774,16 @@ def slice(self, begin: int, stride: int, end: int) -> 'TupleType': class NamedTupleType(TupleType): + """The namedtuple type NamedTuple[name, (a1, T1), ..., (an, Tn)]. + Subtype of Tuple[T1, ..., Tn] + + Instance variables: + name: type name + attrs: names of the attributes + + Uses the default TupleType.fallback: builtins.tuple + """ + attrs = None # type: List[str] name = None # type: str diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index ea12d058b8204..ad9e3b95b7a98 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -233,3 +233,15 @@ X = NamedTuple('X', [('x', int), ('y', str)]) x = None # type: X x = x._replace(x=5) x = x._replace(y=5) # E: Argument 1 to X._replace has incompatible type "int"; expected "str" + +[case testNamedTupleFields] +from typing import NamedTuple + +X = NamedTuple('X', [('x', int), ('y', str)]) +a = None # type: str +b = None # type: str +a, b = X._fields +x = None # type: X +a, b = x._fields +c = None # type: int +a, c = X._fields # E: Incompatible types in assignment (expression has type "str", variable has type "int") From 67bdafe8c13fb61572c08d537205edf9ad299b03 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Tue, 16 Aug 2016 01:57:12 +0300 Subject: [PATCH 29/74] support _source. test for unit namedtuple --- mypy/semanal.py | 5 +++++ test-data/unit/check-namedtuple.test | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/mypy/semanal.py b/mypy/semanal.py index 6fea9755edc23..a6a8c9348c821 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1545,6 +1545,11 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], _fields.is_initialized_in_class = True symbols[_fields.name()] = SymbolTableNode(MDEF, _fields) + _source = Var('_source', self.named_type('__builtins__.str')) + _source.info = info + _source.is_initialized_in_class = True + symbols[_source.name()] = SymbolTableNode(MDEF, _source) + this_type = self_type(info) add_method = partial(self.add_namedtuple_method, symbols, info, this_type) add_method('_replace', this_type, diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index ad9e3b95b7a98..daf547ec8f8d5 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -245,3 +245,23 @@ x = None # type: X a, b = x._fields c = None # type: int a, c = X._fields # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testNamedTupleSource] +from typing import NamedTuple + +X = NamedTuple('X', [('x', int), ('y', str)]) +x = None # type: X +a = None # type: str +b = None # type: int +a = X._source +a = x._source +b = X._source # E: Incompatible types in assignment (expression has type "str", variable has type "int") +b = x._source # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testNamedTupleUnit] +from typing import NamedTuple + +X = NamedTuple('X', []) +x = X() # type: X +x = x._replace() +_ = x._fields[0] # E: Tuple index out of range From 39bce742db6da7f61ef7e8eca4397317a9abdd08 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Tue, 16 Aug 2016 02:30:39 +0300 Subject: [PATCH 30/74] support _asdict()->OrderedDict as an UnboundType --- mypy/semanal.py | 40 +++++++++++++------------- mypy/types.py | 2 +- test-data/unit/check-namedtuple.test | 3 +- test-data/unit/lib-stub/collections.py | 3 +- 4 files changed, 25 insertions(+), 23 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index a6a8c9348c821..44964c28afc98 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1531,36 +1531,36 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], class_def = ClassDef(name, Block([])) class_def.fullname = self.qualified_name(name) info = NamedTupleTypeInfo(tup, symbols, class_def) + vars = [Var(item, typ) for item, typ in zip(items, types)] - # Add namedtuple items as attributes. - for var in vars: - var.info = info - var.is_property = True - symbols[var.name()] = SymbolTableNode(MDEF, var) - - typ = TupleType([self.named_type('__builtins__.str') for _ in items], - self.named_type('__builtins__.tuple', [AnyType()])) - _fields = Var('_fields', typ) - _fields.info = info - _fields.is_initialized_in_class = True - symbols[_fields.name()] = SymbolTableNode(MDEF, _fields) - - _source = Var('_source', self.named_type('__builtins__.str')) - _source.info = info - _source.is_initialized_in_class = True - symbols[_source.name()] = SymbolTableNode(MDEF, _source) - this_type = self_type(info) + add_field = partial(self.add_namedtuple_field, symbols, info) add_method = partial(self.add_namedtuple_method, symbols, info, this_type) + strtype = self.named_type('__builtins__.str') + + for var in vars: + add_field(var, is_property=True) + + tuple_of_strins = TupleType([strtype for _ in items], + self.named_type('__builtins__.tuple', [AnyType()])) + add_field(Var('_fields', tuple_of_strins), is_initialized_in_class=True) + add_field(Var('_source', strtype), is_initialized_in_class=True) + add_method('_replace', this_type, self.factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) add_method('__init__', NoneTyp(), self.factory_args(vars, ARG_POS), info.name()) # TODO: refine to OrderedDict[str, Union[types]] - add_method('_asdict', AnyType(), []) - # self.add_namedtuple_method('_source', self.named_type_or_none('builtins.str'), []) + add_method('_asdict', UnboundType('OrderedDict', is_ret_type=True), []) return info + def add_namedtuple_field(self, symbols, info, var, + is_initialized_in_class=False, is_property=False): + var.info = info + var.is_initialized_in_class = is_initialized_in_class + var.is_property = is_property + symbols[var.name()] = SymbolTableNode(MDEF, var) + def factory_args(self, vars: List[Var], kind: int, initializer: Expression = None) -> List[Argument]: return [Argument(var, var.type, initializer, kind) for var in vars] diff --git a/mypy/types.py b/mypy/types.py index c900ac1644bae..373bff4154662 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -780,7 +780,7 @@ class NamedTupleType(TupleType): Instance variables: name: type name attrs: names of the attributes - + Uses the default TupleType.fallback: builtins.tuple """ diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index daf547ec8f8d5..b4dbe7c640393 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -202,10 +202,11 @@ C(2).b [builtins fixtures/property.py] [case testNamedTupleAsDict] -from collections import namedtuple +from collections import namedtuple, OrderedDict X = namedtuple('X', ['x', 'y']) x = None # type: X +a = None # type: OrderedDict a = x._asdict() [case testNamedTupleReplace] diff --git a/test-data/unit/lib-stub/collections.py b/test-data/unit/lib-stub/collections.py index fa6f4ee58bcc6..29e4b8d37f16d 100644 --- a/test-data/unit/lib-stub/collections.py +++ b/test-data/unit/lib-stub/collections.py @@ -1 +1,2 @@ -namedtuple = object() +namedtuple = object() +class OrderedDict: pass \ No newline at end of file From a2db6a0d6ab024fd678cb9c449a3b2afb452e65a Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Tue, 16 Aug 2016 02:40:30 +0300 Subject: [PATCH 31/74] minor cleanup --- mypy/nodes.py | 2 +- mypy/semanal.py | 5 +++-- mypy/typeanal.py | 15 ++++++++------- test-data/unit/lib-stub/collections.py | 6 ++++-- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index e0e9b080d5371..49d7e821f9393 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1978,7 +1978,7 @@ def deserialize(cls, data: JsonDict) -> 'TypeInfo': return ti -def NamedTupleTypeInfo(tup: 'mypy.types.NamedTupleType', *args) -> TypeInfo: +def namedtuple_type_info(tup: 'mypy.types.NamedTupleType', *args) -> TypeInfo: info = TypeInfo(*args) info.tuple_type = tup info.bases = [tup.fallback] diff --git a/mypy/semanal.py b/mypy/semanal.py index 44964c28afc98..d78894512e461 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -66,7 +66,8 @@ SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr, YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, COVARIANT, CONTRAVARIANT, IntExpr, FloatExpr, UnicodeExpr, - INVARIANT, UNBOUND_IMPORTED, Expression, EllipsisExpr, NamedTupleTypeInfo + INVARIANT, UNBOUND_IMPORTED, Expression, EllipsisExpr, + namedtuple_type_info ) from mypy.visitor import NodeVisitor from mypy.traverser import TraverserVisitor @@ -1530,7 +1531,7 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], tup = NamedTupleType(name, items, types, self.named_type('__builtins__.tuple', types)) class_def = ClassDef(name, Block([])) class_def.fullname = self.qualified_name(name) - info = NamedTupleTypeInfo(tup, symbols, class_def) + info = namedtuple_type_info(tup, symbols, class_def) vars = [Var(item, typ) for item, typ in zip(items, types)] this_type = self_type(info) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 241febab24a95..b7f0e84e0a839 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -164,13 +164,14 @@ def visit_unbound_type(self, t: UnboundType) -> Type: tup = info.tuple_type if tup is None: return instance - # The class has a Tuple[...] base class so it will be - # represented as a tuple type. - if t.args: - self.fail('Generic tuple types not supported', t) - return AnyType() - return tup.copy_with(items=self.anal_array(tup.items), - fallback=instance) + else: + # The class has a Tuple[...] base class so it will be + # represented as a tuple type. + if t.args: + self.fail('Generic tuple types not supported', t) + return AnyType() + return tup.copy_with(items=self.anal_array(tup.items), + fallback=instance) else: return AnyType() diff --git a/test-data/unit/lib-stub/collections.py b/test-data/unit/lib-stub/collections.py index 29e4b8d37f16d..fabe356675b4a 100644 --- a/test-data/unit/lib-stub/collections.py +++ b/test-data/unit/lib-stub/collections.py @@ -1,2 +1,4 @@ -namedtuple = object() -class OrderedDict: pass \ No newline at end of file +namedtuple = object() + +class OrderedDict: + pass From 8784ce0d260f75716699038f61009df93dacc990 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Tue, 16 Aug 2016 05:55:59 +0300 Subject: [PATCH 32/74] support _make and _field_types --- mypy/semanal.py | 51 +++++++++++++++++++--------- test-data/unit/check-namedtuple.test | 10 ++++++ 2 files changed, 45 insertions(+), 16 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index d78894512e461..8c02f94c1bd25 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -61,7 +61,7 @@ SymbolTableNode, BOUND_TVAR, UNBOUND_TVAR, ListComprehension, GeneratorExpr, FuncExpr, MDEF, FuncBase, Decorator, SetExpr, TypeVarExpr, StrExpr, BytesExpr, PrintStmt, ConditionalExpr, PromoteExpr, - ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, MroError, type_aliases, + ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, ARG_OPT, MroError, type_aliases, YieldFromExpr, NamedTupleExpr, NonlocalDecl, SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr, YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, COVARIANT, CONTRAVARIANT, @@ -76,7 +76,7 @@ NoneTyp, CallableType, Overloaded, Instance, Type, TypeVarType, AnyType, FunctionLike, UnboundType, TypeList, TypeVarDef, replace_leading_arg_type, TupleType, UnionType, StarType, EllipsisType, - NamedTupleType + NamedTupleType, TypeType ) from mypy.nodes import function_type, implicit_module_attrs from mypy.typeanal import TypeAnalyser, TypeAnalyserPass3, analyze_type_alias @@ -1531,32 +1531,47 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], tup = NamedTupleType(name, items, types, self.named_type('__builtins__.tuple', types)) class_def = ClassDef(name, Block([])) class_def.fullname = self.qualified_name(name) + self.enter_class(class_def) info = namedtuple_type_info(tup, symbols, class_def) vars = [Var(item, typ) for item, typ in zip(items, types)] this_type = self_type(info) add_field = partial(self.add_namedtuple_field, symbols, info) - add_method = partial(self.add_namedtuple_method, symbols, info, this_type) strtype = self.named_type('__builtins__.str') for var in vars: add_field(var, is_property=True) - tuple_of_strins = TupleType([strtype for _ in items], - self.named_type('__builtins__.tuple', [AnyType()])) - add_field(Var('_fields', tuple_of_strins), is_initialized_in_class=True) - add_field(Var('_source', strtype), is_initialized_in_class=True) + tuple_of_strings = TupleType([strtype for _ in items], + self.named_type('__builtins__.tuple', [AnyType()])) + add_field(Var('_fields', tuple_of_strings), + is_initialized_in_class=True) + add_field(Var('_field_types', UnboundType('Dict', [strtype, AnyType()])), + is_initialized_in_class=True) + add_field(Var('_source', strtype), + is_initialized_in_class=True) - add_method('_replace', this_type, - self.factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) - add_method('__init__', NoneTyp(), - self.factory_args(vars, ARG_POS), info.name()) + add_method = partial(self.add_namedtuple_method, symbols, info, this_type) + add_method('_replace', ret=this_type, + args=self.factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) + add_method('__init__', ret=NoneTyp(), + args=self.factory_args(vars, ARG_POS), name=info.name()) # TODO: refine to OrderedDict[str, Union[types]] - add_method('_asdict', UnboundType('OrderedDict', is_ret_type=True), []) + add_method('_asdict', ret=UnboundType('collections.OrderedDict', is_ret_type=True), + args=[]) + + # TODO: refine signature + union = UnboundType('Iterable', [UnionType(types)]) + add_method('_make', ret=this_type, is_classmethod=True, + args=[Argument(Var('iterable', union), union, None, ARG_POS), + Argument(Var('new'), AnyType(), EllipsisExpr(), ARG_NAMED), + Argument(Var('len'), AnyType(), EllipsisExpr(), ARG_NAMED)]) + self.leave_class() return info - def add_namedtuple_field(self, symbols, info, var, - is_initialized_in_class=False, is_property=False): + def add_namedtuple_field(self, symbols: SymbolTable, info: TypeInfo, var: Var, + is_initialized_in_class: bool = False, + is_property: bool = False) -> None: var.info = info var.is_initialized_in_class = is_initialized_in_class var.is_property = is_property @@ -1567,16 +1582,20 @@ def factory_args(self, vars: List[Var], kind: int, return [Argument(var, var.type, initializer, kind) for var in vars] def add_namedtuple_method(self, symbols: SymbolTable, info: TypeInfo, this_type: Type, - funcname: str, ret: Type, args: List[Argument], name=None) -> None: - args = [Argument(Var('self'), this_type, None, ARG_POS)] + args + funcname: str, ret: Type, args: List[Argument], name=None, + is_classmethod=False) -> None: + if not is_classmethod: + args = [Argument(Var('self'), this_type, None, ARG_POS)] + args types = [arg.type_annotation for arg in args] items = [arg.variable.name() for arg in args] arg_kinds = [arg.kind for arg in args] signature = CallableType(types, arg_kinds, items, ret, self.named_type('__builtins__.function'), name=name or info.name() + '.' + funcname) + signature.is_classmethod_class = is_classmethod func = FuncDef(funcname, args, Block([]), typ=signature) func.info = info + func.is_class = is_classmethod symbols[funcname] = SymbolTableNode(MDEF, func) def analyze_types(self, items: List[Node]) -> List[Type]: diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index b4dbe7c640393..f339525cfb392 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -235,6 +235,16 @@ x = None # type: X x = x._replace(x=5) x = x._replace(y=5) # E: Argument 1 to X._replace has incompatible type "int"; expected "str" + +[case testNamedTupleMake] +from typing import NamedTuple + +X = NamedTuple('X', [('x', int), ('y', str)]) +x = None # type: X +x = X._make([5, 'a']) + +[builtins fixtures/list.py] + [case testNamedTupleFields] from typing import NamedTuple From 191de7c6bb1e6069947f397bf36270a55fc7ed08 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Wed, 6 Jul 2016 03:32:07 +0300 Subject: [PATCH 33/74] namedtuple: add (shaky) support for _replace, _asdict + naive test --- mypy/semanal.py | 38 +++++++++++--- mypy/test/testcheck.py | 74 ++++++++++++++-------------- test-data/unit/check-namedtuple.test | 15 ++++++ 3 files changed, 83 insertions(+), 44 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index c0aea9585c2bb..962c37ad44abb 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -59,7 +59,7 @@ SymbolTableNode, BOUND_TVAR, UNBOUND_TVAR, ListComprehension, GeneratorExpr, FuncExpr, MDEF, FuncBase, Decorator, SetExpr, TypeVarExpr, StrExpr, BytesExpr, PrintStmt, ConditionalExpr, PromoteExpr, - ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, MroError, type_aliases, + ComparisonExpr, StarExpr, ARG_POS, ARG_OPT, ARG_STAR, ARG_NAMED, MroError, type_aliases, YieldFromExpr, NamedTupleExpr, NonlocalDecl, SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr, YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, COVARIANT, CONTRAVARIANT, @@ -1534,9 +1534,13 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], var.info = info var.type = typ symbols[item] = SymbolTableNode(MDEF, var) - # Add a __init__ method. - init = self.make_namedtuple_init(info, items, types) + # Add a __init__, _replace, asdict methods. + init = self.make_namedtuple_create('__init__', NoneTyp(), info, items, types, ARG_POS) symbols['__init__'] = SymbolTableNode(MDEF, init) + replace = self.make_namedtuple_create('_replace', AnyType(), info, items, types, ARG_NAMED, EllipsisType) + symbols['_replace'] = SymbolTableNode(MDEF, replace) + asdict = self.make_namedtuple_asdict(info) + symbols['_asdict'] = SymbolTableNode(MDEF, asdict) info.tuple_type = TupleType(types, self.named_type('__builtins__.tuple', [AnyType()])) info.is_named_tuple = True info.mro = [info] + info.tuple_type.fallback.type.mro @@ -1546,19 +1550,39 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], def make_argument(self, name: str, type: Type) -> Argument: return Argument(Var(name), type, None, ARG_POS) - def make_namedtuple_init(self, info: TypeInfo, items: List[str], - types: List[Type]) -> FuncDef: + def make_namedtuple_create(self, funcname, ret: Type, info: TypeInfo, items: List[str], + types: List[Type], kind, initializer=None) -> FuncDef: args = [self.make_argument(item, type) for item, type in zip(items, types)] + for arg in args: + if initializer is not None: + arg.initializer = initializer + arg.kind = kind # TODO: Make sure that the self argument name is not visible? args = [Argument(Var('__self'), NoneTyp(), None, ARG_POS)] + args arg_kinds = [arg.kind for arg in args] signature = CallableType([cast(Type, None)] + types, arg_kinds, ['__self'] + items, - NoneTyp(), + ret, self.named_type('__builtins__.function'), name=info.name()) - func = FuncDef('__init__', + func = FuncDef(funcname, + args, + Block([]), + typ=signature) + func.info = info + return func + + def make_namedtuple_asdict(self, info: TypeInfo) -> FuncDef: + args = [Argument(Var('__self'), NoneTyp(), None, ARG_POS)] + arg_kinds = [arg.kind for arg in args] + signature = CallableType([cast(Type, None)], + arg_kinds, + ['__self'], + AnyType(), + self.named_type('__builtins__.function'), + name=info.name()) + func = FuncDef('_asdict', args, Block([]), typ=signature) diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 0c9ed87cdc185..7b41c0969308b 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -24,44 +24,44 @@ # List of files that contain test case descriptions. files = [ - 'check-basic.test', - 'check-classes.test', - 'check-expressions.test', - 'check-statements.test', - 'check-generics.test', - 'check-tuples.test', - 'check-dynamic-typing.test', - 'check-weak-typing.test', - 'check-functions.test', - 'check-inference.test', - 'check-inference-context.test', - 'check-varargs.test', - 'check-kwargs.test', - 'check-overloading.test', - 'check-type-checks.test', - 'check-abstract.test', - 'check-multiple-inheritance.test', - 'check-super.test', - 'check-modules.test', - 'check-generic-subtyping.test', - 'check-typevar-values.test', - 'check-python2.test', - 'check-unsupported.test', - 'check-unreachable-code.test', - 'check-unions.test', - 'check-isinstance.test', - 'check-lists.test', + # 'check-basic.test', + # 'check-classes.test', + # 'check-expressions.test', + # 'check-statements.test', + # 'check-generics.test', + # 'check-tuples.test', + # 'check-dynamic-typing.test', + # 'check-weak-typing.test', + # 'check-functions.test', + # 'check-inference.test', + # 'check-inference-context.test', + # 'check-varargs.test', + # 'check-kwargs.test', + # 'check-overloading.test', + # 'check-type-checks.test', + # 'check-abstract.test', + # 'check-multiple-inheritance.test', + # 'check-super.test', + # 'check-modules.test', + # 'check-generic-subtyping.test', + # 'check-typevar-values.test', + # 'check-python2.test', + # 'check-unsupported.test', + # 'check-unreachable-code.test', + # 'check-unions.test', + # 'check-isinstance.test', + # 'check-lists.test', 'check-namedtuple.test', - 'check-type-aliases.test', - 'check-ignore.test', - 'check-type-promotion.test', - 'check-semanal-error.test', - 'check-flags.test', - 'check-incremental.test', - 'check-bound.test', - 'check-optional.test', - 'check-fastparse.test', - 'check-warnings.test', + # 'check-type-aliases.test', + # 'check-ignore.test', + # 'check-type-promotion.test', + # 'check-semanal-error.test', + # 'check-flags.test', + # 'check-incremental.test', + # 'check-bound.test', + # 'check-optional.test', + # 'check-fastparse.test', + # 'check-warnings.test', ] diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index d51f078c8fd47..ed635b973bd3e 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -185,3 +185,18 @@ class C(B): pass B(1).b C(2).b [builtins fixtures/property.py] + +[case testNamedTupleAsDict] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +a = x._asdict() + +[case testNamedTupleAsDict] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +a = x._replace(y=5) + From 6d2d6d1bef0ac5c0875ab6b7cf2ded45d099eb03 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Wed, 6 Jul 2016 11:14:19 +0300 Subject: [PATCH 34/74] namedtuple: add tests and refine types --- mypy/semanal.py | 44 +++++++++++----------------- test-data/unit/check-namedtuple.test | 21 +++++++++++-- 2 files changed, 36 insertions(+), 29 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 962c37ad44abb..13dfd21571f4f 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1534,13 +1534,20 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], var.info = info var.type = typ symbols[item] = SymbolTableNode(MDEF, var) - # Add a __init__, _replace, asdict methods. - init = self.make_namedtuple_create('__init__', NoneTyp(), info, items, types, ARG_POS) + # Add a __init__, _replace, _asdict methods. + init = self.make_namedtuple_create('__init__', info, items, types, + NoneTyp(), ARG_POS) symbols['__init__'] = SymbolTableNode(MDEF, init) - replace = self.make_namedtuple_create('_replace', AnyType(), info, items, types, ARG_NAMED, EllipsisType) + replace = self.make_namedtuple_create('_replace', info, items, types, + AnyType(), ARG_NAMED, initializer=EllipsisType) symbols['_replace'] = SymbolTableNode(MDEF, replace) - asdict = self.make_namedtuple_asdict(info) + asdict = self.make_namedtuple_method(info, '_asdict', AnyType()) symbols['_asdict'] = SymbolTableNode(MDEF, asdict) + fields_type = TupleType([self.builtin_type('builtins.str') for _ in items], + self.builtin_type('builtins.tuple')) + fields = self.make_namedtuple_method(info, '_fields', fields_type) + fields.is_static = True + symbols['_fields'] = SymbolTableNode(MDEF, fields) info.tuple_type = TupleType(types, self.named_type('__builtins__.tuple', [AnyType()])) info.is_named_tuple = True info.mro = [info] + info.tuple_type.fallback.type.mro @@ -1550,13 +1557,12 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], def make_argument(self, name: str, type: Type) -> Argument: return Argument(Var(name), type, None, ARG_POS) - def make_namedtuple_create(self, funcname, ret: Type, info: TypeInfo, items: List[str], - types: List[Type], kind, initializer=None) -> FuncDef: - args = [self.make_argument(item, type) for item, type in zip(items, types)] - for arg in args: - if initializer is not None: - arg.initializer = initializer - arg.kind = kind + def make_namedtuple_create(self, funcname, info: TypeInfo, items: List[str], types: List[Type], + ret: Type, kind, initializer=None) -> FuncDef: + args = [Argument(Var(item), type, initializer, kind) for item, type in zip(items, types)] + return self.make_namedtuple_method(info, funcname, ret, args, types, items) + + def make_namedtuple_method(self, info: TypeInfo, funcname: str, ret, args=[], types=[], items=[]) -> FuncDef: # TODO: Make sure that the self argument name is not visible? args = [Argument(Var('__self'), NoneTyp(), None, ARG_POS)] + args arg_kinds = [arg.kind for arg in args] @@ -1572,22 +1578,6 @@ def make_namedtuple_create(self, funcname, ret: Type, info: TypeInfo, items: Lis typ=signature) func.info = info return func - - def make_namedtuple_asdict(self, info: TypeInfo) -> FuncDef: - args = [Argument(Var('__self'), NoneTyp(), None, ARG_POS)] - arg_kinds = [arg.kind for arg in args] - signature = CallableType([cast(Type, None)], - arg_kinds, - ['__self'], - AnyType(), - self.named_type('__builtins__.function'), - name=info.name()) - func = FuncDef('_asdict', - args, - Block([]), - typ=signature) - func.info = info - return func def analyze_types(self, items: List[Node]) -> List[Type]: result = [] # type: List[Type] diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index ed635b973bd3e..efd6ed94f58e5 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -193,10 +193,27 @@ X = namedtuple('X', ['x', 'y']) x = None # type: X a = x._asdict() -[case testNamedTupleAsDict] +[case testNamedTupleReplace] from collections import namedtuple X = namedtuple('X', ['x', 'y']) x = None # type: X -a = x._replace(y=5) +x = x._replace(y=5) + +[case testNamedTupleReplaceTyped] +from typing import NamedTuple + +X = NamedTuple('X', [('x', int), ('y', str)]) +x = None # type: X +x = x._replace(x=5) +x = x._replace(y=5) # E: Argument 1 to X has incompatible type "int"; expected "str" +[case testNamedTupleFields] +from typing import Tuple +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +a = None # type: Tuple[str, str] +a = x._fields() +[builtins fixtures/tuple.py] From 8819af9733f401c0bf3f5ce2114de1d0f497a574 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Wed, 6 Jul 2016 12:38:34 +0300 Subject: [PATCH 35/74] minor cleanup --- mypy/semanal.py | 40 +++++++++++------------ mypy/test/testcheck.py | 74 +++++++++++++++++++++--------------------- 2 files changed, 56 insertions(+), 58 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 13dfd21571f4f..20f7b876a6a33 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -64,7 +64,7 @@ SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr, YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, COVARIANT, CONTRAVARIANT, IntExpr, FloatExpr, UnicodeExpr, - INVARIANT, UNBOUND_IMPORTED + INVARIANT, UNBOUND_IMPORTED, Expression ) from mypy.visitor import NodeVisitor from mypy.traverser import TraverserVisitor @@ -1534,18 +1534,20 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], var.info = info var.type = typ symbols[item] = SymbolTableNode(MDEF, var) - # Add a __init__, _replace, _asdict methods. - init = self.make_namedtuple_create('__init__', info, items, types, - NoneTyp(), ARG_POS) + # Add a __init__, _replace, _asdict methods and _fields static field + init = self.make_namedtuple_method('__init__', info, NoneTyp(), + self.make_factory_args(items, types, ARG_POS)) symbols['__init__'] = SymbolTableNode(MDEF, init) - replace = self.make_namedtuple_create('_replace', info, items, types, - AnyType(), ARG_NAMED, initializer=EllipsisType) + #TODO: refine return type to type(self) + replace = self.make_namedtuple_method('_replace', info, AnyType(), + self.make_factory_args(items, types, ARG_NAMED, initializer=EllipsisType)) symbols['_replace'] = SymbolTableNode(MDEF, replace) - asdict = self.make_namedtuple_method(info, '_asdict', AnyType()) + #TODO: refine to OrderedDict[Union[types]] + asdict = self.make_namedtuple_method('_asdict', info, AnyType()) symbols['_asdict'] = SymbolTableNode(MDEF, asdict) fields_type = TupleType([self.builtin_type('builtins.str') for _ in items], self.builtin_type('builtins.tuple')) - fields = self.make_namedtuple_method(info, '_fields', fields_type) + fields = self.make_namedtuple_method('_fields', info, fields_type) fields.is_static = True symbols['_fields'] = SymbolTableNode(MDEF, fields) info.tuple_type = TupleType(types, self.named_type('__builtins__.tuple', [AnyType()])) @@ -1554,22 +1556,18 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], info.bases = [info.tuple_type.fallback] return info - def make_argument(self, name: str, type: Type) -> Argument: - return Argument(Var(name), type, None, ARG_POS) - - def make_namedtuple_create(self, funcname, info: TypeInfo, items: List[str], types: List[Type], - ret: Type, kind, initializer=None) -> FuncDef: - args = [Argument(Var(item), type, initializer, kind) for item, type in zip(items, types)] - return self.make_namedtuple_method(info, funcname, ret, args, types, items) + def make_factory_args(self, items: List[str], types: List[Type], + kind: int, initializer: Expression = None) -> FuncDef: + return [Argument(Var(item), type, initializer, kind) for item, type in zip(items, types)] - def make_namedtuple_method(self, info: TypeInfo, funcname: str, ret, args=[], types=[], items=[]) -> FuncDef: + def make_namedtuple_method(self, funcname: str, info: TypeInfo, + ret: Type, args: List[Argument] = []) -> FuncDef: + types = [cast(Type, None)] + [arg.type_annotation for arg in args] + items = ['__self'] + [arg.variable.name() for arg in args] + arg_kinds = [ARG_POS] + [arg.kind for arg in args] # TODO: Make sure that the self argument name is not visible? args = [Argument(Var('__self'), NoneTyp(), None, ARG_POS)] + args - arg_kinds = [arg.kind for arg in args] - signature = CallableType([cast(Type, None)] + types, - arg_kinds, - ['__self'] + items, - ret, + signature = CallableType(types, arg_kinds, items, ret, self.named_type('__builtins__.function'), name=info.name()) func = FuncDef(funcname, diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 7b41c0969308b..0c9ed87cdc185 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -24,44 +24,44 @@ # List of files that contain test case descriptions. files = [ - # 'check-basic.test', - # 'check-classes.test', - # 'check-expressions.test', - # 'check-statements.test', - # 'check-generics.test', - # 'check-tuples.test', - # 'check-dynamic-typing.test', - # 'check-weak-typing.test', - # 'check-functions.test', - # 'check-inference.test', - # 'check-inference-context.test', - # 'check-varargs.test', - # 'check-kwargs.test', - # 'check-overloading.test', - # 'check-type-checks.test', - # 'check-abstract.test', - # 'check-multiple-inheritance.test', - # 'check-super.test', - # 'check-modules.test', - # 'check-generic-subtyping.test', - # 'check-typevar-values.test', - # 'check-python2.test', - # 'check-unsupported.test', - # 'check-unreachable-code.test', - # 'check-unions.test', - # 'check-isinstance.test', - # 'check-lists.test', + 'check-basic.test', + 'check-classes.test', + 'check-expressions.test', + 'check-statements.test', + 'check-generics.test', + 'check-tuples.test', + 'check-dynamic-typing.test', + 'check-weak-typing.test', + 'check-functions.test', + 'check-inference.test', + 'check-inference-context.test', + 'check-varargs.test', + 'check-kwargs.test', + 'check-overloading.test', + 'check-type-checks.test', + 'check-abstract.test', + 'check-multiple-inheritance.test', + 'check-super.test', + 'check-modules.test', + 'check-generic-subtyping.test', + 'check-typevar-values.test', + 'check-python2.test', + 'check-unsupported.test', + 'check-unreachable-code.test', + 'check-unions.test', + 'check-isinstance.test', + 'check-lists.test', 'check-namedtuple.test', - # 'check-type-aliases.test', - # 'check-ignore.test', - # 'check-type-promotion.test', - # 'check-semanal-error.test', - # 'check-flags.test', - # 'check-incremental.test', - # 'check-bound.test', - # 'check-optional.test', - # 'check-fastparse.test', - # 'check-warnings.test', + 'check-type-aliases.test', + 'check-ignore.test', + 'check-type-promotion.test', + 'check-semanal-error.test', + 'check-flags.test', + 'check-incremental.test', + 'check-bound.test', + 'check-optional.test', + 'check-fastparse.test', + 'check-warnings.test', ] From 9fb513744678a8a9050b383f0782fbd7d92e423f Mon Sep 17 00:00:00 2001 From: Elazar Date: Wed, 6 Jul 2016 13:44:05 +0300 Subject: [PATCH 36/74] flake8-ing --- mypy/semanal.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 20f7b876a6a33..d65cf37cd69ee 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1538,11 +1538,11 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], init = self.make_namedtuple_method('__init__', info, NoneTyp(), self.make_factory_args(items, types, ARG_POS)) symbols['__init__'] = SymbolTableNode(MDEF, init) - #TODO: refine return type to type(self) + # TODO: refine return type to type(self) replace = self.make_namedtuple_method('_replace', info, AnyType(), self.make_factory_args(items, types, ARG_NAMED, initializer=EllipsisType)) symbols['_replace'] = SymbolTableNode(MDEF, replace) - #TODO: refine to OrderedDict[Union[types]] + # TODO: refine to OrderedDict[Union[types]] asdict = self.make_namedtuple_method('_asdict', info, AnyType()) symbols['_asdict'] = SymbolTableNode(MDEF, asdict) fields_type = TupleType([self.builtin_type('builtins.str') for _ in items], @@ -1557,9 +1557,9 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], return info def make_factory_args(self, items: List[str], types: List[Type], - kind: int, initializer: Expression = None) -> FuncDef: + kind: int, initializer: Expression = None) -> FuncDef: return [Argument(Var(item), type, initializer, kind) for item, type in zip(items, types)] - + def make_namedtuple_method(self, funcname: str, info: TypeInfo, ret: Type, args: List[Argument] = []) -> FuncDef: types = [cast(Type, None)] + [arg.type_annotation for arg in args] From 35c30a22a5c2eb26052757042aeb08933086a2d6 Mon Sep 17 00:00:00 2001 From: Elazar Date: Wed, 6 Jul 2016 13:50:11 +0300 Subject: [PATCH 37/74] fix type errors --- mypy/semanal.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index d65cf37cd69ee..2fe405e05c978 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -64,7 +64,7 @@ SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr, YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, COVARIANT, CONTRAVARIANT, IntExpr, FloatExpr, UnicodeExpr, - INVARIANT, UNBOUND_IMPORTED, Expression + INVARIANT, UNBOUND_IMPORTED, Expression, EllipsisExpr ) from mypy.visitor import NodeVisitor from mypy.traverser import TraverserVisitor @@ -1540,7 +1540,7 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], symbols['__init__'] = SymbolTableNode(MDEF, init) # TODO: refine return type to type(self) replace = self.make_namedtuple_method('_replace', info, AnyType(), - self.make_factory_args(items, types, ARG_NAMED, initializer=EllipsisType)) + self.make_factory_args(items, types, ARG_NAMED, initializer=EllipsisExpr())) symbols['_replace'] = SymbolTableNode(MDEF, replace) # TODO: refine to OrderedDict[Union[types]] asdict = self.make_namedtuple_method('_asdict', info, AnyType()) @@ -1557,7 +1557,7 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], return info def make_factory_args(self, items: List[str], types: List[Type], - kind: int, initializer: Expression = None) -> FuncDef: + kind: int, initializer: Expression = None) -> List[Argument]: return [Argument(Var(item), type, initializer, kind) for item, type in zip(items, types)] def make_namedtuple_method(self, funcname: str, info: TypeInfo, From a642a7ff1594c6c1866d5b56dc567aad910d88f4 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Wed, 6 Jul 2016 18:25:03 +0300 Subject: [PATCH 38/74] _fields will not be supported --- mypy/semanal.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 2fe405e05c978..a4c7ff0429b85 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1545,11 +1545,6 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], # TODO: refine to OrderedDict[Union[types]] asdict = self.make_namedtuple_method('_asdict', info, AnyType()) symbols['_asdict'] = SymbolTableNode(MDEF, asdict) - fields_type = TupleType([self.builtin_type('builtins.str') for _ in items], - self.builtin_type('builtins.tuple')) - fields = self.make_namedtuple_method('_fields', info, fields_type) - fields.is_static = True - symbols['_fields'] = SymbolTableNode(MDEF, fields) info.tuple_type = TupleType(types, self.named_type('__builtins__.tuple', [AnyType()])) info.is_named_tuple = True info.mro = [info] + info.tuple_type.fallback.type.mro From c505cbce8ab18a4c1a897bcb5ba7bd9f62886861 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Wed, 6 Jul 2016 20:15:33 +0300 Subject: [PATCH 39/74] slight refactoring --- mypy/semanal.py | 26 +++-- test-data/unit/check-namedtuple.test | 152 --------------------------- 2 files changed, 12 insertions(+), 166 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index a4c7ff0429b85..0e7013ce7f149 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -59,7 +59,7 @@ SymbolTableNode, BOUND_TVAR, UNBOUND_TVAR, ListComprehension, GeneratorExpr, FuncExpr, MDEF, FuncBase, Decorator, SetExpr, TypeVarExpr, StrExpr, BytesExpr, PrintStmt, ConditionalExpr, PromoteExpr, - ComparisonExpr, StarExpr, ARG_POS, ARG_OPT, ARG_STAR, ARG_NAMED, MroError, type_aliases, + ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, MroError, type_aliases, YieldFromExpr, NamedTupleExpr, NonlocalDecl, SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr, YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, COVARIANT, CONTRAVARIANT, @@ -1534,17 +1534,15 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], var.info = info var.type = typ symbols[item] = SymbolTableNode(MDEF, var) - # Add a __init__, _replace, _asdict methods and _fields static field - init = self.make_namedtuple_method('__init__', info, NoneTyp(), + # Add __init__, _replace, _asdict methods and _fields static field + self.add_namedtuple_method('__init__', symbols, info, NoneTyp(), self.make_factory_args(items, types, ARG_POS)) - symbols['__init__'] = SymbolTableNode(MDEF, init) # TODO: refine return type to type(self) - replace = self.make_namedtuple_method('_replace', info, AnyType(), + self.add_namedtuple_method('_replace', symbols, info, AnyType(), self.make_factory_args(items, types, ARG_NAMED, initializer=EllipsisExpr())) - symbols['_replace'] = SymbolTableNode(MDEF, replace) # TODO: refine to OrderedDict[Union[types]] - asdict = self.make_namedtuple_method('_asdict', info, AnyType()) - symbols['_asdict'] = SymbolTableNode(MDEF, asdict) + self.add_namedtuple_method('_asdict', symbols, info, AnyType()) + info.tuple_type = TupleType(types, self.named_type('__builtins__.tuple', [AnyType()])) info.is_named_tuple = True info.mro = [info] + info.tuple_type.fallback.type.mro @@ -1555,13 +1553,13 @@ def make_factory_args(self, items: List[str], types: List[Type], kind: int, initializer: Expression = None) -> List[Argument]: return [Argument(Var(item), type, initializer, kind) for item, type in zip(items, types)] - def make_namedtuple_method(self, funcname: str, info: TypeInfo, - ret: Type, args: List[Argument] = []) -> FuncDef: + def add_namedtuple_method(self, funcname: str, symbols: SymbolTable, info: TypeInfo, + ret: Type, args: List[Argument] = []) -> None: types = [cast(Type, None)] + [arg.type_annotation for arg in args] - items = ['__self'] + [arg.variable.name() for arg in args] - arg_kinds = [ARG_POS] + [arg.kind for arg in args] - # TODO: Make sure that the self argument name is not visible? args = [Argument(Var('__self'), NoneTyp(), None, ARG_POS)] + args + items = [arg.variable.name() for arg in args] + arg_kinds = [arg.kind for arg in args] + # TODO: Make sure that the self argument name is not visible? signature = CallableType(types, arg_kinds, items, ret, self.named_type('__builtins__.function'), name=info.name()) @@ -1570,7 +1568,7 @@ def make_namedtuple_method(self, funcname: str, info: TypeInfo, Block([]), typ=signature) func.info = info - return func + symbols[funcname] = SymbolTableNode(MDEF, func) def analyze_types(self, items: List[Node]) -> List[Type]: result = [] # type: List[Type] diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index efd6ed94f58e5..d9f252e765720 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -65,155 +65,3 @@ n = N('x', 'x') # E: Argument 1 to "N" has incompatible type "str"; expected "in n = N(1, b=2) # E: Argument 2 to "N" has incompatible type "int"; expected "str" N(1, 'x') N(b='x', a=1) - -[case testNamedTupleAsBaseClass] -from typing import NamedTuple -N = NamedTuple('N', [('a', int), - ('b', str)]) -class X(N): - pass -x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" -s = '' -i = 0 -s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") -i, s = x -s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") - -[case testNamedTupleAsBaseClass2] -from typing import NamedTuple -class X(NamedTuple('N', [('a', int), - ('b', str)])): - pass -x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" -s = '' -i = 0 -s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") -i, s = x -s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") - -[case testTwoNamedTuplesAsBaseClasses] -from typing import NamedTuple -A = NamedTuple('A', [('a', int)]) -B = NamedTuple('B', [('a', int)]) -class X(A, B): # E: Class has two incompatible bases derived from tuple - pass - -[case testTwoNamedTuplesAsBaseClasses2] -from typing import NamedTuple -A = NamedTuple('A', [('a', int)]) -class X(A, NamedTuple('B', [('a', int)])): # E: Class has two incompatible bases derived from tuple - pass - -[case testSelfTypeWithNamedTupleAsBase] -from typing import NamedTuple -A = NamedTuple('A', [('a', int), ('b', str)]) -class B(A): - def f(self, x: int) -> None: - self.f(self.a) - self.f(self.b) # E: Argument 1 to "f" of "B" has incompatible type "str"; expected "int" - i = 0 - s = '' - i, s = self - i, i = self # E: Incompatible types in assignment (expression has type "str", \ - variable has type "int") -[out] -main: note: In member "f" of class "B": - -[case testTypeReferenceToClassDerivedFromNamedTuple] -from typing import NamedTuple -A = NamedTuple('A', [('a', int), ('b', str)]) -class B(A): - def f(self, x: 'B') -> None: - i = 0 - s = '' - self = x - i, s = x - i, s = x.a, x.b - i, s = x.a, x.a # E: Incompatible types in assignment (expression has type "int", \ - variable has type "str") - i, i = self # E: Incompatible types in assignment (expression has type "str", \ - variable has type "int") -[out] -main: note: In member "f" of class "B": - -[case testNamedTupleSubtyping] -from typing import NamedTuple, Tuple -A = NamedTuple('A', [('a', int), ('b', str)]) -class B(A): pass -a = A(1, '') -b = B(1, '') -t = None # type: Tuple[int, str] -b = a # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "A") -b = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "B") -t = a -t = (1, '') -t = b -a = b - -[case testNamedTupleSimpleTypeInference] -from typing import NamedTuple, Tuple -A = NamedTuple('A', [('a', int)]) -l = [A(1), A(2)] -a = A(1) -a = l[0] -(i,) = l[0] -i, i = l[0] # E: Need more than 1 value to unpack (2 expected) -l = [A(1)] -a = (1,) # E: Incompatible types in assignment (expression has type "Tuple[int]", \ - variable has type "A") -[builtins fixtures/list.py] - -[case testNamedTupleMissingClassAttribute] -import collections -MyNamedTuple = collections.namedtuple('MyNamedTuple', ['spam', 'eggs']) -MyNamedTuple.x # E: "MyNamedTuple" has no attribute "x" - -[case testNamedTupleEmptyItems] -from typing import NamedTuple -A = NamedTuple('A', []) -[builtins fixtures/list.py] - -[case testNamedTupleProperty] -from typing import NamedTuple -A = NamedTuple('A', [('a', int)]) -class B(A): - @property - def b(self) -> int: - return self.a -class C(B): pass -B(1).b -C(2).b -[builtins fixtures/property.py] - -[case testNamedTupleAsDict] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -a = x._asdict() - -[case testNamedTupleReplace] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -x = x._replace(y=5) - -[case testNamedTupleReplaceTyped] -from typing import NamedTuple - -X = NamedTuple('X', [('x', int), ('y', str)]) -x = None # type: X -x = x._replace(x=5) -x = x._replace(y=5) # E: Argument 1 to X has incompatible type "int"; expected "str" - -[case testNamedTupleFields] -from typing import Tuple -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -a = None # type: Tuple[str, str] -a = x._fields() -[builtins fixtures/tuple.py] From 9f96ca7be1c3eddf0bb645ce062c04e85a3ee633 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Wed, 6 Jul 2016 20:19:28 +0300 Subject: [PATCH 40/74] restore tests for _replace and _asdict --- test-data/unit/check-namedtuple.test | 142 +++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index d9f252e765720..9c2d6c4a86d8f 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -65,3 +65,145 @@ n = N('x', 'x') # E: Argument 1 to "N" has incompatible type "str"; expected "in n = N(1, b=2) # E: Argument 2 to "N" has incompatible type "int"; expected "str" N(1, 'x') N(b='x', a=1) + +[case testNamedTupleAsBaseClass] +from typing import NamedTuple +N = NamedTuple('N', [('a', int), + ('b', str)]) +class X(N): + pass +x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" +s = '' +i = 0 +s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") +i, s = x +s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testNamedTupleAsBaseClass2] +from typing import NamedTuple +class X(NamedTuple('N', [('a', int), + ('b', str)])): + pass +x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" +s = '' +i = 0 +s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") +i, s = x +s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testTwoNamedTuplesAsBaseClasses] +from typing import NamedTuple +A = NamedTuple('A', [('a', int)]) +B = NamedTuple('B', [('a', int)]) +class X(A, B): # E: Class has two incompatible bases derived from tuple + pass + +[case testTwoNamedTuplesAsBaseClasses2] +from typing import NamedTuple +A = NamedTuple('A', [('a', int)]) +class X(A, NamedTuple('B', [('a', int)])): # E: Class has two incompatible bases derived from tuple + pass + +[case testSelfTypeWithNamedTupleAsBase] +from typing import NamedTuple +A = NamedTuple('A', [('a', int), ('b', str)]) +class B(A): + def f(self, x: int) -> None: + self.f(self.a) + self.f(self.b) # E: Argument 1 to "f" of "B" has incompatible type "str"; expected "int" + i = 0 + s = '' + i, s = self + i, i = self # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +[out] +main: note: In member "f" of class "B": + +[case testTypeReferenceToClassDerivedFromNamedTuple] +from typing import NamedTuple +A = NamedTuple('A', [('a', int), ('b', str)]) +class B(A): + def f(self, x: 'B') -> None: + i = 0 + s = '' + self = x + i, s = x + i, s = x.a, x.b + i, s = x.a, x.a # E: Incompatible types in assignment (expression has type "int", \ + variable has type "str") + i, i = self # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +[out] +main: note: In member "f" of class "B": + +[case testNamedTupleSubtyping] +from typing import NamedTuple, Tuple +A = NamedTuple('A', [('a', int), ('b', str)]) +class B(A): pass +a = A(1, '') +b = B(1, '') +t = None # type: Tuple[int, str] +b = a # E: Incompatible types in assignment (expression has type "A", variable has type "B") +a = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "A") +b = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "B") +t = a +t = (1, '') +t = b +a = b + +[case testNamedTupleSimpleTypeInference] +from typing import NamedTuple, Tuple +A = NamedTuple('A', [('a', int)]) +l = [A(1), A(2)] +a = A(1) +a = l[0] +(i,) = l[0] +i, i = l[0] # E: Need more than 1 value to unpack (2 expected) +l = [A(1)] +a = (1,) # E: Incompatible types in assignment (expression has type "Tuple[int]", \ + variable has type "A") +[builtins fixtures/list.py] + +[case testNamedTupleMissingClassAttribute] +import collections +MyNamedTuple = collections.namedtuple('MyNamedTuple', ['spam', 'eggs']) +MyNamedTuple.x # E: "MyNamedTuple" has no attribute "x" + +[case testNamedTupleEmptyItems] +from typing import NamedTuple +A = NamedTuple('A', []) +[builtins fixtures/list.py] + +[case testNamedTupleProperty] +from typing import NamedTuple +A = NamedTuple('A', [('a', int)]) +class B(A): + @property + def b(self) -> int: + return self.a +class C(B): pass +B(1).b +C(2).b +[builtins fixtures/property.py] + +[case testNamedTupleAsDict] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +a = x._asdict() + +[case testNamedTupleReplace] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +x = x._replace(y=5) + +[case testNamedTupleReplaceTyped] +from typing import NamedTuple + +X = NamedTuple('X', [('x', int), ('y', str)]) +x = None # type: X +x = x._replace(x=5) +x = x._replace(y=5) # E: Argument 1 to X has incompatible type "int"; expected "str" From 5c2d3acceac999ead1bb7bd3e6492e1fbc3e606d Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Sun, 10 Jul 2016 02:52:21 +0300 Subject: [PATCH 41/74] NamedTupleTypeInfo --- mypy/nodes.py | 12 + mypy/semanal.py | 5672 +++++++++++++------------- test-data/unit/check-namedtuple.test | 419 +- 3 files changed, 3053 insertions(+), 3050 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index b068715d38162..f4c26634ba25b 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1976,6 +1976,18 @@ def deserialize(cls, data: JsonDict) -> 'TypeInfo': return ti +class NamedTupleTypeInfo(TypeInfo): + is_named_tuple = False + def __init__(self, tup, *args): + super().__init__(*args) + tup.fullname = lambda : '__builtins__.tuple' + self.tuple_type = tup + self.bases = [tup] + self.mro = [self, tup] + # self._promote = tup + + + class SymbolTableNode: # Kind of node. Possible values: # - LDEF: local definition (of any kind) diff --git a/mypy/semanal.py b/mypy/semanal.py index 0e7013ce7f149..50bd040f91536 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1,2841 +1,2831 @@ -"""The semantic analyzer. - -Bind names to definitions and do various other simple consistency -checks. For example, consider this program: - - x = 1 - y = x - -Here semantic analysis would detect that the assignment 'x = 1' -defines a new variable, the type of which is to be inferred (in a -later pass; type inference or type checking is not part of semantic -analysis). Also, it would bind both references to 'x' to the same -module-level variable node. The second assignment would also be -analyzed, and the type of 'y' marked as being inferred. - -Semantic analysis is the first analysis pass after parsing, and it is -subdivided into three passes: - - * FirstPass looks up externally visible names defined in a module but - ignores imports and local definitions. It helps enable (some) - cyclic references between modules, such as module 'a' that imports - module 'b' and used names defined in b *and* vice versa. The first - pass can be performed before dependent modules have been processed. - - * SemanticAnalyzer is the second pass. It does the bulk of the work. - It assumes that dependent modules have been semantically analyzed, - up to the second pass, unless there is a import cycle. - - * ThirdPass checks that type argument counts are valid; for example, - it will reject Dict[int]. We don't do this in the second pass, - since we infer the type argument counts of classes during this - pass, and it is possible to refer to classes defined later in a - file, which would not have the type argument count set yet. This - pass also recomputes the method resolution order of each class, in - case one of its bases belongs to a module involved in an import - loop. - -Semantic analysis of types is implemented in module mypy.typeanal. - -TODO: Check if the third pass slows down type checking significantly. - We could probably get rid of it -- for example, we could collect all - analyzed types in a collection and check them without having to - traverse the entire AST. -""" - -from typing import ( - List, Dict, Set, Tuple, cast, Any, overload, TypeVar, Union, Optional, Callable -) - -from mypy.nodes import ( - MypyFile, TypeInfo, Node, AssignmentStmt, FuncDef, OverloadedFuncDef, - ClassDef, Var, GDEF, MODULE_REF, FuncItem, Import, - ImportFrom, ImportAll, Block, LDEF, NameExpr, MemberExpr, - IndexExpr, TupleExpr, ListExpr, ExpressionStmt, ReturnStmt, - RaiseStmt, AssertStmt, OperatorAssignmentStmt, WhileStmt, - ForStmt, BreakStmt, ContinueStmt, IfStmt, TryStmt, WithStmt, DelStmt, - GlobalDecl, SuperExpr, DictExpr, CallExpr, RefExpr, OpExpr, UnaryExpr, - SliceExpr, CastExpr, RevealTypeExpr, TypeApplication, Context, SymbolTable, - SymbolTableNode, BOUND_TVAR, UNBOUND_TVAR, ListComprehension, GeneratorExpr, - FuncExpr, MDEF, FuncBase, Decorator, SetExpr, TypeVarExpr, - StrExpr, BytesExpr, PrintStmt, ConditionalExpr, PromoteExpr, - ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, MroError, type_aliases, - YieldFromExpr, NamedTupleExpr, NonlocalDecl, - SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr, - YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, COVARIANT, CONTRAVARIANT, - IntExpr, FloatExpr, UnicodeExpr, - INVARIANT, UNBOUND_IMPORTED, Expression, EllipsisExpr -) -from mypy.visitor import NodeVisitor -from mypy.traverser import TraverserVisitor -from mypy.errors import Errors, report_internal_error -from mypy.types import ( - NoneTyp, CallableType, Overloaded, Instance, Type, TypeVarType, AnyType, - FunctionLike, UnboundType, TypeList, ErrorType, TypeVarDef, - replace_leading_arg_type, TupleType, UnionType, StarType, EllipsisType -) -from mypy.nodes import function_type, implicit_module_attrs -from mypy.typeanal import TypeAnalyser, TypeAnalyserPass3, analyze_type_alias -from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError -from mypy.lex import lex -from mypy.parsetype import parse_type -from mypy.sametypes import is_same_type -from mypy.erasetype import erase_typevars -from mypy import defaults -from mypy.options import Options - - -T = TypeVar('T') - - -# Inferred value of an expression. -ALWAYS_TRUE = 0 -ALWAYS_FALSE = 1 -TRUTH_VALUE_UNKNOWN = 2 - -# Map from obsolete name to the current spelling. -obsolete_name_mapping = { - 'typing.Function': 'typing.Callable', - 'typing.typevar': 'typing.TypeVar', -} - -# Hard coded type promotions (shared between all Python versions). -# These add extra ad-hoc edges to the subtyping relation. For example, -# int is considered a subtype of float, even though there is no -# subclass relationship. -TYPE_PROMOTIONS = { - 'builtins.int': 'builtins.float', - 'builtins.float': 'builtins.complex', -} - -# Hard coded type promotions for Python 3. -# -# Note that the bytearray -> bytes promotion is a little unsafe -# as some functions only accept bytes objects. Here convenience -# trumps safety. -TYPE_PROMOTIONS_PYTHON3 = TYPE_PROMOTIONS.copy() -TYPE_PROMOTIONS_PYTHON3.update({ - 'builtins.bytearray': 'builtins.bytes', -}) - -# Hard coded type promotions for Python 2. -# -# These promotions are unsafe, but we are doing them anyway -# for convenience and also for Python 3 compatibility -# (bytearray -> str). -TYPE_PROMOTIONS_PYTHON2 = TYPE_PROMOTIONS.copy() -TYPE_PROMOTIONS_PYTHON2.update({ - 'builtins.str': 'builtins.unicode', - 'builtins.bytearray': 'builtins.str', -}) - -# When analyzing a function, should we analyze the whole function in one go, or -# should we only perform one phase of the analysis? The latter is used for -# nested functions. In the first phase we add the function to the symbol table -# but don't process body. In the second phase we process function body. This -# way we can have mutually recursive nested functions. -FUNCTION_BOTH_PHASES = 0 # Everthing in one go -FUNCTION_FIRST_PHASE_POSTPONE_SECOND = 1 # Add to symbol table but postpone body -FUNCTION_SECOND_PHASE = 2 # Only analyze body - - -class SemanticAnalyzer(NodeVisitor): - """Semantically analyze parsed mypy files. - - The analyzer binds names and does various consistency checks for a - parse tree. Note that type checking is performed as a separate - pass. - - This is the second phase of semantic analysis. - """ - - # Library search paths - lib_path = None # type: List[str] - # Module name space - modules = None # type: Dict[str, MypyFile] - # Global name space for current module - globals = None # type: SymbolTable - # Names declared using "global" (separate set for each scope) - global_decls = None # type: List[Set[str]] - # Names declated using "nonlocal" (separate set for each scope) - nonlocal_decls = None # type: List[Set[str]] - # Local names of function scopes; None for non-function scopes. - locals = None # type: List[SymbolTable] - # Nested block depths of scopes - block_depth = None # type: List[int] - # TypeInfo of directly enclosing class (or None) - type = None # type: TypeInfo - # Stack of outer classes (the second tuple item contains tvars). - type_stack = None # type: List[TypeInfo] - # Type variables that are bound by the directly enclosing class - bound_tvars = None # type: List[SymbolTableNode] - # Stack of type variables that were bound by outer classess - tvar_stack = None # type: List[List[SymbolTableNode]] - # Do weak type checking in this file - weak_opts = set() # type: Set[str] - - # Stack of functions being analyzed - function_stack = None # type: List[FuncItem] - # Stack of next available function type variable ids - next_function_tvar_id_stack = None # type: List[int] - - # Status of postponing analysis of nested function bodies. By using this we - # can have mutually recursive nested functions. Values are FUNCTION_x - # constants. Note that separate phasea are not used for methods. - postpone_nested_functions_stack = None # type: List[int] - # Postponed functions collected if - # postpone_nested_functions_stack[-1] == FUNCTION_FIRST_PHASE_POSTPONE_SECOND. - postponed_functions_stack = None # type: List[List[Node]] - - loop_depth = 0 # Depth of breakable loops - cur_mod_id = '' # Current module id (or None) (phase 2) - is_stub_file = False # Are we analyzing a stub file? - imports = None # type: Set[str] # Imported modules (during phase 2 analysis) - errors = None # type: Errors # Keeps track of generated errors - - def __init__(self, - lib_path: List[str], - errors: Errors, - options: Options) -> None: - """Construct semantic analyzer. - - Use lib_path to search for modules, and report analysis errors - using the Errors instance. - """ - self.locals = [None] - self.imports = set() - self.type = None - self.type_stack = [] - self.bound_tvars = None - self.tvar_stack = [] - self.function_stack = [] - self.next_function_tvar_id_stack = [-1] - self.block_depth = [0] - self.loop_depth = 0 - self.lib_path = lib_path - self.errors = errors - self.modules = {} - self.options = options - self.postpone_nested_functions_stack = [FUNCTION_BOTH_PHASES] - self.postponed_functions_stack = [] - self.all_exports = set() # type: Set[str] - - def visit_file(self, file_node: MypyFile, fnam: str) -> None: - self.errors.set_file(fnam) - self.cur_mod_node = file_node - self.cur_mod_id = file_node.fullname() - self.is_stub_file = fnam.lower().endswith('.pyi') - self.globals = file_node.names - self.weak_opts = file_node.weak_opts - - if 'builtins' in self.modules: - self.globals['__builtins__'] = SymbolTableNode( - MODULE_REF, self.modules['builtins'], self.cur_mod_id) - - for name in implicit_module_attrs: - v = self.globals[name].node - if isinstance(v, Var): - v.type = self.anal_type(v.type) - v.is_ready = True - - defs = file_node.defs - for d in defs: - self.accept(d) - - if self.cur_mod_id == 'builtins': - remove_imported_names_from_symtable(self.globals, 'builtins') - - if '__all__' in self.globals: - for name, g in self.globals.items(): - if name not in self.all_exports: - g.module_public = False - - def visit_func_def(self, defn: FuncDef) -> None: - phase_info = self.postpone_nested_functions_stack[-1] - if phase_info != FUNCTION_SECOND_PHASE: - self.function_stack.append(defn) - # First phase of analysis for function. - self.errors.push_function(defn.name()) - self.update_function_type_variables(defn) - self.errors.pop_function() - self.function_stack.pop() - - defn.is_conditional = self.block_depth[-1] > 0 - - # TODO(jukka): Figure out how to share the various cases. It doesn't - # make sense to have (almost) duplicate code (here and elsewhere) for - # 3 cases: module-level, class-level and local names. Maybe implement - # a common stack of namespaces. As the 3 kinds of namespaces have - # different semantics, this wouldn't always work, but it might still - # be a win. - if self.is_class_scope(): - # Method definition - defn.info = self.type - if not defn.is_decorated and not defn.is_overload: - if defn.name() in self.type.names: - # Redefinition. Conditional redefinition is okay. - n = self.type.names[defn.name()].node - if self.is_conditional_func(n, defn): - defn.original_def = cast(FuncDef, n) - else: - self.name_already_defined(defn.name(), defn) - self.type.names[defn.name()] = SymbolTableNode(MDEF, defn) - self.prepare_method_signature(defn) - elif self.is_func_scope(): - # Nested function - if not defn.is_decorated and not defn.is_overload: - if defn.name() in self.locals[-1]: - # Redefinition. Conditional redefinition is okay. - n = self.locals[-1][defn.name()].node - if self.is_conditional_func(n, defn): - defn.original_def = cast(FuncDef, n) - else: - self.name_already_defined(defn.name(), defn) - else: - self.add_local(defn, defn) - else: - # Top-level function - if not defn.is_decorated and not defn.is_overload: - symbol = self.globals.get(defn.name()) - if isinstance(symbol.node, FuncDef) and symbol.node != defn: - # This is redefinition. Conditional redefinition is okay. - original_def = symbol.node - if self.is_conditional_func(original_def, defn): - # Conditional function definition -- multiple defs are ok. - defn.original_def = original_def - else: - # Report error. - self.check_no_global(defn.name(), defn, True) - if phase_info == FUNCTION_FIRST_PHASE_POSTPONE_SECOND: - # Postpone this function (for the second phase). - self.postponed_functions_stack[-1].append(defn) - return - if phase_info != FUNCTION_FIRST_PHASE_POSTPONE_SECOND: - # Second phase of analysis for function. - self.errors.push_function(defn.name()) - self.analyze_function(defn) - self.errors.pop_function() - - def prepare_method_signature(self, func: FuncDef) -> None: - """Check basic signature validity and tweak annotation of self/cls argument.""" - # Only non-static methods are special. - if not func.is_static: - if not func.arguments: - self.fail('Method must have at least one argument', func) - elif func.type: - sig = cast(FunctionLike, func.type) - if func.is_class: - leading_type = self.class_type(self.type) - else: - leading_type = self_type(self.type) - func.type = replace_implicit_first_type(sig, leading_type) - - def is_conditional_func(self, previous: Node, new: FuncDef) -> bool: - """Does 'new' conditionally redefine 'previous'? - - We reject straight redefinitions of functions, as they are usually - a programming error. For example: - - . def f(): ... - . def f(): ... # Error: 'f' redefined - """ - return isinstance(previous, (FuncDef, Var)) and new.is_conditional - - def update_function_type_variables(self, defn: FuncDef) -> None: - """Make any type variables in the signature of defn explicit. - - Update the signature of defn to contain type variable definitions - if defn is generic. - """ - if defn.type: - functype = cast(CallableType, defn.type) - typevars = self.infer_type_variables(functype) - # Do not define a new type variable if already defined in scope. - typevars = [(name, tvar) for name, tvar in typevars - if not self.is_defined_type_var(name, defn)] - if typevars: - next_tvar_id = self.next_function_tvar_id() - defs = [TypeVarDef(tvar[0], next_tvar_id - i, - tvar[1].values, tvar[1].upper_bound, - tvar[1].variance) - for i, tvar in enumerate(typevars)] - functype.variables = defs - - def infer_type_variables(self, - type: CallableType) -> List[Tuple[str, TypeVarExpr]]: - """Return list of unique type variables referred to in a callable.""" - names = [] # type: List[str] - tvars = [] # type: List[TypeVarExpr] - for arg in type.arg_types + [type.ret_type]: - for name, tvar_expr in self.find_type_variables_in_type(arg): - if name not in names: - names.append(name) - tvars.append(tvar_expr) - return list(zip(names, tvars)) - - def find_type_variables_in_type( - self, type: Type) -> List[Tuple[str, TypeVarExpr]]: - """Return a list of all unique type variable references in type. - - This effectively does partial name binding, results of which are mostly thrown away. - """ - result = [] # type: List[Tuple[str, TypeVarExpr]] - if isinstance(type, UnboundType): - name = type.name - node = self.lookup_qualified(name, type) - if node and node.kind == UNBOUND_TVAR: - result.append((name, cast(TypeVarExpr, node.node))) - for arg in type.args: - result.extend(self.find_type_variables_in_type(arg)) - elif isinstance(type, TypeList): - for item in type.items: - result.extend(self.find_type_variables_in_type(item)) - elif isinstance(type, UnionType): - for item in type.items: - result.extend(self.find_type_variables_in_type(item)) - elif isinstance(type, AnyType): - pass - elif isinstance(type, EllipsisType) or isinstance(type, TupleType): - pass - else: - assert False, 'Unsupported type %s' % type - return result - - def is_defined_type_var(self, tvar: str, context: Node) -> bool: - return self.lookup_qualified(tvar, context).kind == BOUND_TVAR - - def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: - t = [] # type: List[CallableType] - for i, item in enumerate(defn.items): - # TODO support decorated overloaded functions properly - item.is_overload = True - item.func.is_overload = True - item.accept(self) - t.append(cast(CallableType, function_type(item.func, - self.builtin_type('builtins.function')))) - if item.func.is_property and i == 0: - # This defines a property, probably with a setter and/or deleter. - self.analyze_property_with_multi_part_definition(defn) - break - if not [dec for dec in item.decorators - if refers_to_fullname(dec, 'typing.overload')]: - self.fail("'overload' decorator expected", item) - - defn.type = Overloaded(t) - defn.type.line = defn.line - - if self.is_class_scope(): - self.type.names[defn.name()] = SymbolTableNode(MDEF, defn, - typ=defn.type) - defn.info = self.type - elif self.is_func_scope(): - self.add_local(defn, defn) - - def analyze_property_with_multi_part_definition(self, defn: OverloadedFuncDef) -> None: - """Analyze a property defined using multiple methods (e.g., using @x.setter). - - Assume that the first method (@property) has already been analyzed. - """ - defn.is_property = True - items = defn.items - for item in items[1:]: - if len(item.decorators) == 1: - node = item.decorators[0] - if isinstance(node, MemberExpr): - if node.name == 'setter': - # The first item represents the entire property. - defn.items[0].var.is_settable_property = True - # Get abstractness from the original definition. - item.func.is_abstract = items[0].func.is_abstract - else: - self.fail("Decorated property not supported", item) - item.func.accept(self) - - def next_function_tvar_id(self) -> int: - return self.next_function_tvar_id_stack[-1] - - def analyze_function(self, defn: FuncItem) -> None: - is_method = self.is_class_scope() - - tvarnodes = self.add_func_type_variables_to_symbol_table(defn) - next_function_tvar_id = min([self.next_function_tvar_id()] + - [n.tvar_def.id.raw_id - 1 for n in tvarnodes]) - self.next_function_tvar_id_stack.append(next_function_tvar_id) - - if defn.type: - # Signature must be analyzed in the surrounding scope so that - # class-level imported names and type variables are in scope. - defn.type = self.anal_type(defn.type) - self.check_function_signature(defn) - if isinstance(defn, FuncDef): - defn.type = set_callable_name(defn.type, defn) - for arg in defn.arguments: - if arg.initializer: - arg.initializer.accept(self) - self.function_stack.append(defn) - self.enter() - for arg in defn.arguments: - self.add_local(arg.variable, defn) - for arg in defn.arguments: - if arg.initialization_statement: - lvalue = arg.initialization_statement.lvalues[0] - lvalue.accept(self) - - # The first argument of a non-static, non-class method is like 'self' - # (though the name could be different), having the enclosing class's - # instance type. - if is_method and not defn.is_static and not defn.is_class and defn.arguments: - defn.arguments[0].variable.is_self = True - - # First analyze body of the function but ignore nested functions. - self.postpone_nested_functions_stack.append(FUNCTION_FIRST_PHASE_POSTPONE_SECOND) - self.postponed_functions_stack.append([]) - defn.body.accept(self) - - # Analyze nested functions (if any) as a second phase. - self.postpone_nested_functions_stack[-1] = FUNCTION_SECOND_PHASE - for postponed in self.postponed_functions_stack[-1]: - postponed.accept(self) - self.postpone_nested_functions_stack.pop() - self.postponed_functions_stack.pop() - - self.next_function_tvar_id_stack.pop() - disable_typevars(tvarnodes) - - self.leave() - self.function_stack.pop() - - def add_func_type_variables_to_symbol_table( - self, defn: FuncItem) -> List[SymbolTableNode]: - nodes = [] # type: List[SymbolTableNode] - if defn.type: - tt = defn.type - names = self.type_var_names() - items = cast(CallableType, tt).variables - for item in items: - name = item.name - if name in names: - self.name_already_defined(name, defn) - node = self.bind_type_var(name, item, defn) - nodes.append(node) - names.add(name) - return nodes - - def type_var_names(self) -> Set[str]: - if not self.type: - return set() - else: - return set(self.type.type_vars) - - def bind_type_var(self, fullname: str, tvar_def: TypeVarDef, - context: Context) -> SymbolTableNode: - node = self.lookup_qualified(fullname, context) - node.kind = BOUND_TVAR - node.tvar_def = tvar_def - return node - - def check_function_signature(self, fdef: FuncItem) -> None: - sig = cast(CallableType, fdef.type) - if len(sig.arg_types) < len(fdef.arguments): - self.fail('Type signature has too few arguments', fdef) - # Add dummy Any arguments to prevent crashes later. - extra_anys = [AnyType()] * (len(fdef.arguments) - len(sig.arg_types)) - sig.arg_types.extend(extra_anys) - elif len(sig.arg_types) > len(fdef.arguments): - self.fail('Type signature has too many arguments', fdef) - - def visit_class_def(self, defn: ClassDef) -> None: - self.clean_up_bases_and_infer_type_variables(defn) - self.setup_class_def_analysis(defn) - - self.bind_class_type_vars(defn) - - self.analyze_base_classes(defn) - self.analyze_metaclass(defn) - - for decorator in defn.decorators: - self.analyze_class_decorator(defn, decorator) - - self.enter_class(defn) - - self.setup_is_builtinclass(defn) - - # Analyze class body. - defn.defs.accept(self) - - self.calculate_abstract_status(defn.info) - self.setup_type_promotion(defn) - - self.leave_class() - self.unbind_class_type_vars() - - def enter_class(self, defn: ClassDef) -> None: - # Remember previous active class - self.type_stack.append(self.type) - self.locals.append(None) # Add class scope - self.block_depth.append(-1) # The class body increments this to 0 - self.postpone_nested_functions_stack.append(FUNCTION_BOTH_PHASES) - self.type = defn.info - - def leave_class(self) -> None: - """ Restore analyzer state. """ - self.postpone_nested_functions_stack.pop() - self.block_depth.pop() - self.locals.pop() - self.type = self.type_stack.pop() - - def bind_class_type_vars(self, defn: ClassDef) -> None: - """ Unbind type variables of previously active class and bind - the type variables for the active class. - """ - if self.bound_tvars: - disable_typevars(self.bound_tvars) - self.tvar_stack.append(self.bound_tvars) - self.bound_tvars = self.bind_class_type_variables_in_symbol_table(defn.info) - - def unbind_class_type_vars(self) -> None: - """ Unbind the active class' type vars and rebind the - type vars of the previously active class. - """ - disable_typevars(self.bound_tvars) - self.bound_tvars = self.tvar_stack.pop() - if self.bound_tvars: - enable_typevars(self.bound_tvars) - - def analyze_class_decorator(self, defn: ClassDef, decorator: Node) -> None: - decorator.accept(self) - - def setup_is_builtinclass(self, defn: ClassDef): - for decorator in defn.decorators: - if refers_to_fullname(decorator, 'typing.builtinclass'): - defn.is_builtinclass = True - if defn.fullname == 'builtins.object': - # Only 'object' is marked as a built-in class, as otherwise things elsewhere - # would break. We need a better way of dealing with built-in classes. - defn.is_builtinclass = True - - def calculate_abstract_status(self, typ: TypeInfo) -> None: - """Calculate abstract status of a class. - - Set is_abstract of the type to True if the type has an unimplemented - abstract attribute. Also compute a list of abstract attributes. - """ - concrete = set() # type: Set[str] - abstract = [] # type: List[str] - for base in typ.mro: - for name, symnode in base.names.items(): - node = symnode.node - if isinstance(node, OverloadedFuncDef): - # Unwrap an overloaded function definition. We can just - # check arbitrarily the first overload item. If the - # different items have a different abstract status, there - # should be an error reported elsewhere. - func = node.items[0] # type: Node - else: - func = node - if isinstance(func, Decorator): - fdef = func.func - if fdef.is_abstract and name not in concrete: - typ.is_abstract = True - abstract.append(name) - concrete.add(name) - typ.abstract_attributes = sorted(abstract) - - def setup_type_promotion(self, defn: ClassDef) -> None: - """Setup extra, ad-hoc subtyping relationships between classes (promotion). - - This includes things like 'int' being compatible with 'float'. - """ - promote_target = None # type: Type - for decorator in defn.decorators: - if isinstance(decorator, CallExpr): - analyzed = decorator.analyzed - if isinstance(analyzed, PromoteExpr): - # _promote class decorator (undocumented faeture). - promote_target = analyzed.type - if not promote_target: - promotions = (TYPE_PROMOTIONS_PYTHON3 if self.options.python_version[0] >= 3 - else TYPE_PROMOTIONS_PYTHON2) - if defn.fullname in promotions: - promote_target = self.named_type_or_none(promotions[defn.fullname]) - defn.info._promote = promote_target - - def clean_up_bases_and_infer_type_variables(self, defn: ClassDef) -> None: - """Remove extra base classes such as Generic and infer type vars. - - For example, consider this class: - - . class Foo(Bar, Generic[T]): ... - - Now we will remove Generic[T] from bases of Foo and infer that the - type variable 'T' is a type argument of Foo. - - Note that this is performed *before* semantic analysis. - """ - removed = [] # type: List[int] - type_vars = [] # type: List[TypeVarDef] - for i, base_expr in enumerate(defn.base_type_exprs): - try: - base = expr_to_unanalyzed_type(base_expr) - except TypeTranslationError: - # This error will be caught later. - continue - tvars = self.analyze_typevar_declaration(base) - if tvars is not None: - if type_vars: - self.fail('Duplicate Generic in bases', defn) - removed.append(i) - for j, (name, tvar_expr) in enumerate(tvars): - type_vars.append(TypeVarDef(name, j + 1, tvar_expr.values, - tvar_expr.upper_bound, tvar_expr.variance)) - if type_vars: - defn.type_vars = type_vars - if defn.info: - defn.info.type_vars = [tv.name for tv in type_vars] - for i in reversed(removed): - del defn.base_type_exprs[i] - - def analyze_typevar_declaration(self, t: Type) -> Optional[List[Tuple[str, TypeVarExpr]]]: - if not isinstance(t, UnboundType): - return None - unbound = t - sym = self.lookup_qualified(unbound.name, unbound) - if sym is None or sym.node is None: - return None - if sym.node.fullname() == 'typing.Generic': - tvars = [] # type: List[Tuple[str, TypeVarExpr]] - for arg in unbound.args: - tvar = self.analyze_unbound_tvar(arg) - if tvar: - tvars.append(tvar) - else: - self.fail('Free type variable expected in %s[...]' % - sym.node.name(), t) - return tvars - return None - - def analyze_unbound_tvar(self, t: Type) -> Tuple[str, TypeVarExpr]: - if not isinstance(t, UnboundType): - return None - unbound = t - sym = self.lookup_qualified(unbound.name, unbound) - if sym is not None and sym.kind == UNBOUND_TVAR: - return unbound.name, cast(TypeVarExpr, sym.node) - return None - - def setup_class_def_analysis(self, defn: ClassDef) -> None: - """Prepare for the analysis of a class definition.""" - if not defn.info: - defn.info = TypeInfo(SymbolTable(), defn) - defn.info._fullname = defn.info.name() - if self.is_func_scope() or self.type: - kind = MDEF - if self.is_func_scope(): - kind = LDEF - self.add_symbol(defn.name, SymbolTableNode(kind, defn.info), defn) - - def analyze_base_classes(self, defn: ClassDef) -> None: - """Analyze and set up base classes. - - This computes several attributes on the corresponding TypeInfo defn.info - related to the base classes: defn.info.bases, defn.info.mro, and - miscellaneous others (at least tuple_type, fallback_to_any, and is_enum.) - """ - - base_types = [] # type: List[Instance] - for base_expr in defn.base_type_exprs: - try: - base = self.expr_to_analyzed_type(base_expr) - except TypeTranslationError: - self.fail('Invalid base class', base_expr) - defn.info.fallback_to_any = True - continue - - if isinstance(base, TupleType): - if defn.info.tuple_type: - self.fail("Class has two incompatible bases derived from tuple", defn) - if (not self.is_stub_file - and not defn.info.is_named_tuple - and base.fallback.type.fullname() == 'builtins.tuple'): - self.fail("Tuple[...] not supported as a base class outside a stub file", defn) - defn.info.tuple_type = base - base_types.append(base.fallback) - elif isinstance(base, Instance): - base_types.append(base) - elif isinstance(base, AnyType): - defn.info.fallback_to_any = True - else: - self.fail('Invalid base class', base_expr) - defn.info.fallback_to_any = True - - # Add 'object' as implicit base if there is no other base class. - if (not base_types and defn.fullname != 'builtins.object'): - base_types.append(self.object_type()) - - defn.info.bases = base_types - - # Calculate the MRO. It might be incomplete at this point if - # the bases of defn include classes imported from other - # modules in an import loop. We'll recompute it in ThirdPass. - if not self.verify_base_classes(defn): - defn.info.mro = [] - return - calculate_class_mro(defn, self.fail) - # If there are cyclic imports, we may be missing 'object' in - # the MRO. Fix MRO if needed. - if defn.info.mro and defn.info.mro[-1].fullname() != 'builtins.object': - defn.info.mro.append(self.object_type().type) - - def expr_to_analyzed_type(self, expr: Node) -> Type: - if isinstance(expr, CallExpr): - expr.accept(self) - info = self.check_namedtuple(expr) - if info is None: - # Some form of namedtuple is the only valid type that looks like a call - # expression. This isn't a valid type. - raise TypeTranslationError() - fallback = Instance(info, []) - return TupleType(info.tuple_type.items, fallback=fallback) - typ = expr_to_unanalyzed_type(expr) - return self.anal_type(typ) - - def verify_base_classes(self, defn: ClassDef) -> bool: - info = defn.info - for base in info.bases: - baseinfo = base.type - if self.is_base_class(info, baseinfo): - self.fail('Cycle in inheritance hierarchy', defn) - # Clear bases to forcefully get rid of the cycle. - info.bases = [] - if baseinfo.fullname() == 'builtins.bool': - self.fail("'%s' is not a valid base class" % - baseinfo.name(), defn) - return False - dup = find_duplicate(info.direct_base_classes()) - if dup: - self.fail('Duplicate base class "%s"' % dup.name(), defn) - return False - return True - - def is_base_class(self, t: TypeInfo, s: TypeInfo) -> bool: - """Determine if t is a base class of s (but do not use mro).""" - # Search the base class graph for t, starting from s. - worklist = [s] - visited = {s} - while worklist: - nxt = worklist.pop() - if nxt == t: - return True - for base in nxt.bases: - if base.type not in visited: - worklist.append(base.type) - visited.add(base.type) - return False - - def analyze_metaclass(self, defn: ClassDef) -> None: - if defn.metaclass: - sym = self.lookup_qualified(defn.metaclass, defn) - if sym is not None and not isinstance(sym.node, TypeInfo): - self.fail("Invalid metaclass '%s'" % defn.metaclass, defn) - - def object_type(self) -> Instance: - return self.named_type('__builtins__.object') - - def class_type(self, info: TypeInfo) -> Type: - # Construct a function type whose fallback is cls. - from mypy import checkmember # To avoid import cycle. - leading_type = checkmember.type_object_type(info, self.builtin_type) - if isinstance(leading_type, Overloaded): - # Overloaded __init__ is too complex to handle. Plus it's stubs only. - return AnyType() - else: - return leading_type - - def named_type(self, qualified_name: str, args: List[Type] = None) -> Instance: - sym = self.lookup_qualified(qualified_name, None) - return Instance(cast(TypeInfo, sym.node), args or []) - - def named_type_or_none(self, qualified_name: str) -> Instance: - sym = self.lookup_fully_qualified_or_none(qualified_name) - if not sym: - return None - return Instance(cast(TypeInfo, sym.node), []) - - def is_instance_type(self, t: Type) -> bool: - return isinstance(t, Instance) - - def bind_class_type_variables_in_symbol_table( - self, info: TypeInfo) -> List[SymbolTableNode]: - nodes = [] # type: List[SymbolTableNode] - for var, binder in zip(info.type_vars, info.defn.type_vars): - node = self.bind_type_var(var, binder, info) - nodes.append(node) - return nodes - - def visit_import(self, i: Import) -> None: - for id, as_id in i.ids: - if as_id is not None: - self.add_module_symbol(id, as_id, module_public=True, context=i) - else: - # Modules imported in a stub file without using 'as x' won't get exported when - # doing 'from m import *'. - module_public = not self.is_stub_file - base = id.split('.')[0] - self.add_module_symbol(base, base, module_public=module_public, - context=i) - - def add_module_symbol(self, id: str, as_id: str, module_public: bool, - context: Context) -> None: - if id in self.modules: - m = self.modules[id] - self.add_symbol(as_id, SymbolTableNode(MODULE_REF, m, self.cur_mod_id, - module_public=module_public), context) - else: - self.add_unknown_symbol(as_id, context) - - def visit_import_from(self, imp: ImportFrom) -> None: - import_id = self.correct_relative_import(imp) - if import_id in self.modules: - module = self.modules[import_id] - for id, as_id in imp.names: - node = module.names.get(id) - if node and node.kind != UNBOUND_IMPORTED: - node = self.normalize_type_alias(node, imp) - if not node: - return - imported_id = as_id or id - existing_symbol = self.globals.get(imported_id) - if existing_symbol: - # Import can redefine a variable. They get special treatment. - if self.process_import_over_existing_name( - imported_id, existing_symbol, node, imp): - continue - # 'from m import x as x' exports x in a stub file. - module_public = not self.is_stub_file or as_id is not None - symbol = SymbolTableNode(node.kind, node.node, - self.cur_mod_id, - node.type_override, - module_public=module_public) - self.add_symbol(imported_id, symbol, imp) - else: - message = "Module has no attribute '{}'".format(id) - extra = self.undefined_name_extra_info('{}.{}'.format(import_id, id)) - if extra: - message += " {}".format(extra) - self.fail(message, imp) - else: - # Missing module. - for id, as_id in imp.names: - self.add_unknown_symbol(as_id or id, imp) - - def process_import_over_existing_name(self, - imported_id: str, existing_symbol: SymbolTableNode, - module_symbol: SymbolTableNode, - import_node: ImportBase) -> bool: - if (existing_symbol.kind in (LDEF, GDEF, MDEF) and - isinstance(existing_symbol.node, (Var, FuncDef))): - # This is a valid import over an existing definition in the file. Construct a dummy - # assignment that we'll use to type check the import. - lvalue = NameExpr(imported_id) - lvalue.kind = existing_symbol.kind - lvalue.node = existing_symbol.node - rvalue = NameExpr(imported_id) - rvalue.kind = module_symbol.kind - rvalue.node = module_symbol.node - assignment = AssignmentStmt([lvalue], rvalue) - for node in assignment, lvalue, rvalue: - node.set_line(import_node) - import_node.assignments.append(assignment) - return True - return False - - def normalize_type_alias(self, node: SymbolTableNode, - ctx: Context) -> SymbolTableNode: - if node.fullname in type_aliases: - # Node refers to an aliased type such as typing.List; normalize. - node = self.lookup_qualified(type_aliases[node.fullname], ctx) - if node.fullname == 'typing.DefaultDict': - self.add_module_symbol('collections', '__mypy_collections__', False, ctx) - node = self.lookup_qualified('__mypy_collections__.defaultdict', ctx) - return node - - def correct_relative_import(self, node: Union[ImportFrom, ImportAll]) -> str: - if node.relative == 0: - return node.id - - parts = self.cur_mod_id.split(".") - cur_mod_id = self.cur_mod_id - - rel = node.relative - if self.cur_mod_node.is_package_init_file(): - rel -= 1 - if len(parts) < rel: - self.fail("Relative import climbs too many namespaces", node) - if rel != 0: - cur_mod_id = ".".join(parts[:-rel]) - - return cur_mod_id + (("." + node.id) if node.id else "") - - def visit_import_all(self, i: ImportAll) -> None: - i_id = self.correct_relative_import(i) - if i_id in self.modules: - m = self.modules[i_id] - for name, node in m.names.items(): - node = self.normalize_type_alias(node, i) - if not name.startswith('_') and node.module_public: - existing_symbol = self.globals.get(name) - if existing_symbol: - # Import can redefine a variable. They get special treatment. - if self.process_import_over_existing_name( - name, existing_symbol, node, i): - continue - self.add_symbol(name, SymbolTableNode(node.kind, node.node, - self.cur_mod_id), i) - else: - # Don't add any dummy symbols for 'from x import *' if 'x' is unknown. - pass - - def add_unknown_symbol(self, name: str, context: Context) -> None: - var = Var(name) - var._fullname = self.qualified_name(name) - var.is_ready = True - var.type = AnyType() - self.add_symbol(name, SymbolTableNode(GDEF, var, self.cur_mod_id), context) - - # - # Statements - # - - def visit_block(self, b: Block) -> None: - if b.is_unreachable: - return - self.block_depth[-1] += 1 - for s in b.body: - self.accept(s) - self.block_depth[-1] -= 1 - - def visit_block_maybe(self, b: Block) -> None: - if b: - self.visit_block(b) - - def anal_type(self, t: Type, allow_tuple_literal: bool = False) -> Type: - if t: - if allow_tuple_literal: - # Types such as (t1, t2, ...) only allowed in assignment statements. They'll - # generate errors elsewhere, and Tuple[t1, t2, ...] must be used instead. - if isinstance(t, TupleType): - # Unlike TypeAnalyser, also allow implicit tuple types (without Tuple[...]). - star_count = sum(1 for item in t.items if isinstance(item, StarType)) - if star_count > 1: - self.fail('At most one star type allowed in a tuple', t) - return TupleType([AnyType() for _ in t.items], - self.builtin_type('builtins.tuple'), t.line) - items = [self.anal_type(item, True) - for item in t.items] - return TupleType(items, self.builtin_type('builtins.tuple'), t.line) - a = TypeAnalyser(self.lookup_qualified, - self.lookup_fully_qualified, - self.fail) - return t.accept(a) - else: - return None - - def visit_assignment_stmt(self, s: AssignmentStmt) -> None: - for lval in s.lvalues: - self.analyze_lvalue(lval, explicit_type=s.type is not None) - s.rvalue.accept(self) - if s.type: - allow_tuple_literal = isinstance(s.lvalues[-1], (TupleExpr, ListExpr)) - s.type = self.anal_type(s.type, allow_tuple_literal) - else: - # For simple assignments, allow binding type aliases. - # Also set the type if the rvalue is a simple literal. - if (s.type is None and len(s.lvalues) == 1 and - isinstance(s.lvalues[0], NameExpr)): - if s.lvalues[0].is_def: - s.type = self.analyze_simple_literal_type(s.rvalue) - res = analyze_type_alias(s.rvalue, - self.lookup_qualified, - self.lookup_fully_qualified, - self.fail) - if res and (not isinstance(res, Instance) or cast(Instance, res).args): - # TODO: What if this gets reassigned? - name = s.lvalues[0] - node = self.lookup(name.name, name) - node.kind = TYPE_ALIAS - node.type_override = res - if isinstance(s.rvalue, IndexExpr): - s.rvalue.analyzed = TypeAliasExpr(res) - if s.type: - # Store type into nodes. - for lvalue in s.lvalues: - self.store_declared_types(lvalue, s.type) - self.check_and_set_up_type_alias(s) - self.process_typevar_declaration(s) - self.process_namedtuple_definition(s) - - if (len(s.lvalues) == 1 and isinstance(s.lvalues[0], NameExpr) and - s.lvalues[0].name == '__all__' and s.lvalues[0].kind == GDEF and - isinstance(s.rvalue, (ListExpr, TupleExpr))): - self.add_exports(*s.rvalue.items) - - def analyze_simple_literal_type(self, rvalue: Node) -> Optional[Type]: - """Return builtins.int if rvalue is an int literal, etc.""" - if self.weak_opts or self.options.semantic_analysis_only or self.function_stack: - # Skip this if any weak options are set. - # Also skip if we're only doing the semantic analysis pass. - # This is mostly to avoid breaking unit tests. - # Also skip inside a function; this is to avoid confusing - # the code that handles dead code due to isinstance() - # inside type variables with value restrictions (like - # AnyStr). - return None - if isinstance(rvalue, IntExpr): - return self.named_type_or_none('builtins.int') - if isinstance(rvalue, FloatExpr): - return self.named_type_or_none('builtins.float') - if isinstance(rvalue, StrExpr): - return self.named_type_or_none('builtins.str') - if isinstance(rvalue, BytesExpr): - return self.named_type_or_none('builtins.bytes') - if isinstance(rvalue, UnicodeExpr): - return self.named_type_or_none('builtins.unicode') - return None - - def check_and_set_up_type_alias(self, s: AssignmentStmt) -> None: - """Check if assignment creates a type alias and set it up as needed.""" - # For now, type aliases only work at the top level of a module. - if (len(s.lvalues) == 1 and not self.is_func_scope() and not self.type - and not s.type): - lvalue = s.lvalues[0] - if isinstance(lvalue, NameExpr): - if not lvalue.is_def: - # Only a definition can create a type alias, not regular assignment. - return - rvalue = s.rvalue - if isinstance(rvalue, RefExpr): - node = rvalue.node - if isinstance(node, TypeInfo): - # TODO: We should record the fact that this is a variable - # that refers to a type, rather than making this - # just an alias for the type. - self.globals[lvalue.name].node = node - - def analyze_lvalue(self, lval: Node, nested: bool = False, - add_global: bool = False, - explicit_type: bool = False) -> None: - """Analyze an lvalue or assignment target. - - Only if add_global is True, add name to globals table. If nested - is true, the lvalue is within a tuple or list lvalue expression. - """ - - if isinstance(lval, NameExpr): - # Top-level definitions within some statements (at least while) are - # not handled in the first pass, so they have to be added now. - nested_global = (not self.is_func_scope() and - self.block_depth[-1] > 0 and - not self.type) - if (add_global or nested_global) and lval.name not in self.globals: - # Define new global name. - v = Var(lval.name) - v._fullname = self.qualified_name(lval.name) - v.is_ready = False # Type not inferred yet - lval.node = v - lval.is_def = True - lval.kind = GDEF - lval.fullname = v._fullname - self.globals[lval.name] = SymbolTableNode(GDEF, v, - self.cur_mod_id) - elif isinstance(lval.node, Var) and lval.is_def: - # Since the is_def flag is set, this must have been analyzed - # already in the first pass and added to the symbol table. - assert lval.node.name() in self.globals - elif (self.is_func_scope() and lval.name not in self.locals[-1] and - lval.name not in self.global_decls[-1] and - lval.name not in self.nonlocal_decls[-1]): - # Define new local name. - v = Var(lval.name) - lval.node = v - lval.is_def = True - lval.kind = LDEF - lval.fullname = lval.name - self.add_local(v, lval) - elif not self.is_func_scope() and (self.type and - lval.name not in self.type.names): - # Define a new attribute within class body. - v = Var(lval.name) - v.info = self.type - v.is_initialized_in_class = True - v.set_line(lval) - lval.node = v - lval.is_def = True - lval.kind = MDEF - lval.fullname = lval.name - self.type.names[lval.name] = SymbolTableNode(MDEF, v) - else: - # Bind to an existing name. - if explicit_type: - self.name_already_defined(lval.name, lval) - lval.accept(self) - self.check_lvalue_validity(lval.node, lval) - elif isinstance(lval, MemberExpr): - if not add_global: - self.analyze_member_lvalue(lval) - if explicit_type and not self.is_self_member_ref(lval): - self.fail('Type cannot be declared in assignment to non-self ' - 'attribute', lval) - elif isinstance(lval, IndexExpr): - if explicit_type: - self.fail('Unexpected type declaration', lval) - if not add_global: - lval.accept(self) - elif (isinstance(lval, TupleExpr) or - isinstance(lval, ListExpr)): - items = cast(Any, lval).items - if len(items) == 0 and isinstance(lval, TupleExpr): - self.fail("Can't assign to ()", lval) - self.analyze_tuple_or_list_lvalue(cast(Union[ListExpr, TupleExpr], lval), - add_global, explicit_type) - elif isinstance(lval, StarExpr): - if nested: - self.analyze_lvalue(lval.expr, nested, add_global, explicit_type) - else: - self.fail('Starred assignment target must be in a list or tuple', lval) - else: - self.fail('Invalid assignment target', lval) - - def analyze_tuple_or_list_lvalue(self, lval: Union[ListExpr, TupleExpr], - add_global: bool = False, - explicit_type: bool = False) -> None: - """Analyze an lvalue or assignment target that is a list or tuple.""" - items = lval.items - star_exprs = [cast(StarExpr, item) - for item in items - if isinstance(item, StarExpr)] - - if len(star_exprs) > 1: - self.fail('Two starred expressions in assignment', lval) - else: - if len(star_exprs) == 1: - star_exprs[0].valid = True - for i in items: - self.analyze_lvalue(i, nested=True, add_global=add_global, - explicit_type = explicit_type) - - def analyze_member_lvalue(self, lval: MemberExpr) -> None: - lval.accept(self) - if (self.is_self_member_ref(lval) and - self.type.get(lval.name) is None): - # Implicit attribute definition in __init__. - lval.is_def = True - v = Var(lval.name) - v.info = self.type - v.is_ready = False - lval.def_var = v - lval.node = v - self.type.names[lval.name] = SymbolTableNode(MDEF, v) - self.check_lvalue_validity(lval.node, lval) - - def is_self_member_ref(self, memberexpr: MemberExpr) -> bool: - """Does memberexpr to refer to an attribute of self?""" - if not isinstance(memberexpr.expr, NameExpr): - return False - node = memberexpr.expr.node - return isinstance(node, Var) and node.is_self - - def check_lvalue_validity(self, node: Node, ctx: Context) -> None: - if isinstance(node, (TypeInfo, TypeVarExpr)): - self.fail('Invalid assignment target', ctx) - - def store_declared_types(self, lvalue: Node, typ: Type) -> None: - if isinstance(typ, StarType) and not isinstance(lvalue, StarExpr): - self.fail('Star type only allowed for starred expressions', lvalue) - if isinstance(lvalue, RefExpr): - lvalue.is_def = False - if isinstance(lvalue.node, Var): - var = lvalue.node - var.type = typ - var.is_ready = True - # If node is not a variable, we'll catch it elsewhere. - elif isinstance(lvalue, TupleExpr): - if isinstance(typ, TupleType): - if len(lvalue.items) != len(typ.items): - self.fail('Incompatible number of tuple items', lvalue) - return - for item, itemtype in zip(lvalue.items, typ.items): - self.store_declared_types(item, itemtype) - else: - self.fail('Tuple type expected for multiple variables', - lvalue) - elif isinstance(lvalue, StarExpr): - if isinstance(typ, StarType): - self.store_declared_types(lvalue.expr, typ.type) - else: - self.fail('Star type expected for starred expression', lvalue) - else: - # This has been flagged elsewhere as an error, so just ignore here. - pass - - def process_typevar_declaration(self, s: AssignmentStmt) -> None: - """Check if s declares a TypeVar; it yes, store it in symbol table.""" - call = self.get_typevar_declaration(s) - if not call: - return - - lvalue = cast(NameExpr, s.lvalues[0]) - name = lvalue.name - if not lvalue.is_def: - if s.type: - self.fail("Cannot declare the type of a type variable", s) - else: - self.fail("Cannot redefine '%s' as a type variable" % name, s) - return - - if not self.check_typevar_name(call, name, s): - return - - # Constraining types - n_values = call.arg_kinds[1:].count(ARG_POS) - values = self.analyze_types(call.args[1:1 + n_values]) - - res = self.process_typevar_parameters(call.args[1 + n_values:], - call.arg_names[1 + n_values:], - call.arg_kinds[1 + n_values:], - bool(values), - s) - if res is None: - return - variance, upper_bound = res - - # Yes, it's a valid type variable definition! Add it to the symbol table. - node = self.lookup(name, s) - node.kind = UNBOUND_TVAR - TypeVar = TypeVarExpr(name, node.fullname, values, upper_bound, variance) - TypeVar.line = call.line - call.analyzed = TypeVar - node.node = TypeVar - - def check_typevar_name(self, call: CallExpr, name: str, context: Context) -> bool: - if len(call.args) < 1: - self.fail("Too few arguments for TypeVar()", context) - return False - if not isinstance(call.args[0], (StrExpr, BytesExpr)) or not call.arg_kinds[0] == ARG_POS: - self.fail("TypeVar() expects a string literal as first argument", context) - return False - if cast(StrExpr, call.args[0]).value != name: - self.fail("Unexpected TypeVar() argument value", context) - return False - return True - - def get_typevar_declaration(self, s: AssignmentStmt) -> Optional[CallExpr]: - """Returns the TypeVar() call expression if `s` is a type var declaration - or None otherwise. - """ - if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): - return None - if not isinstance(s.rvalue, CallExpr): - return None - call = s.rvalue - if not isinstance(call.callee, RefExpr): - return None - callee = call.callee - if callee.fullname != 'typing.TypeVar': - return None - return call - - def process_typevar_parameters(self, args: List[Node], - names: List[Optional[str]], - kinds: List[int], - has_values: bool, - context: Context) -> Optional[Tuple[int, Type]]: - covariant = False - contravariant = False - upper_bound = self.object_type() # type: Type - for param_value, param_name, param_kind in zip(args, names, kinds): - if not param_kind == ARG_NAMED: - self.fail("Unexpected argument to TypeVar()", context) - return None - if param_name == 'covariant': - if isinstance(param_value, NameExpr): - if param_value.name == 'True': - covariant = True - else: - self.fail("TypeVar 'covariant' may only be 'True'", context) - return None - else: - self.fail("TypeVar 'covariant' may only be 'True'", context) - return None - elif param_name == 'contravariant': - if isinstance(param_value, NameExpr): - if param_value.name == 'True': - contravariant = True - else: - self.fail("TypeVar 'contravariant' may only be 'True'", context) - return None - else: - self.fail("TypeVar 'contravariant' may only be 'True'", context) - return None - elif param_name == 'bound': - if has_values: - self.fail("TypeVar cannot have both values and an upper bound", context) - return None - try: - upper_bound = self.expr_to_analyzed_type(param_value) - except TypeTranslationError: - self.fail("TypeVar 'bound' must be a type", param_value) - return None - elif param_name == 'values': - # Probably using obsolete syntax with values=(...). Explain the current syntax. - self.fail("TypeVar 'values' argument not supported", context) - self.fail("Use TypeVar('T', t, ...) instead of TypeVar('T', values=(t, ...))", - context) - return None - else: - self.fail("Unexpected argument to TypeVar(): {}".format(param_name), context) - return None - - if covariant and contravariant: - self.fail("TypeVar cannot be both covariant and contravariant", context) - return None - elif covariant: - variance = COVARIANT - elif contravariant: - variance = CONTRAVARIANT - else: - variance = INVARIANT - return (variance, upper_bound) - - def process_namedtuple_definition(self, s: AssignmentStmt) -> None: - """Check if s defines a namedtuple; if yes, store the definition in symbol table.""" - if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): - return - lvalue = cast(NameExpr, s.lvalues[0]) - name = lvalue.name - named_tuple = self.check_namedtuple(s.rvalue, name) - if named_tuple is None: - return - # Yes, it's a valid namedtuple definition. Add it to the symbol table. - node = self.lookup(name, s) - node.kind = GDEF # TODO locally defined namedtuple - # TODO call.analyzed - node.node = named_tuple - - def check_namedtuple(self, node: Node, var_name: str = None) -> TypeInfo: - """Check if a call defines a namedtuple. - - The optional var_name argument is the name of the variable to - which this is assigned, if any. - - If it does, return the corresponding TypeInfo. Return None otherwise. - - If the definition is invalid but looks like a namedtuple, - report errors but return (some) TypeInfo. - """ - if not isinstance(node, CallExpr): - return None - call = node - if not isinstance(call.callee, RefExpr): - return None - callee = call.callee - fullname = callee.fullname - if fullname not in ('collections.namedtuple', 'typing.NamedTuple'): - return None - items, types, ok = self.parse_namedtuple_args(call, fullname) - if not ok: - # Error. Construct dummy return value. - return self.build_namedtuple_typeinfo('namedtuple', [], []) - else: - # Give it a unique name derived from the line number. - name = cast(StrExpr, call.args[0]).value - if name != var_name: - name += '@' + str(call.line) - info = self.build_namedtuple_typeinfo(name, items, types) - # Store it as a global just in case it would remain anonymous. - self.globals[name] = SymbolTableNode(GDEF, info, self.cur_mod_id) - call.analyzed = NamedTupleExpr(info).set_line(call.line) - return info - - def parse_namedtuple_args(self, call: CallExpr, - fullname: str) -> Tuple[List[str], List[Type], bool]: - # TODO Share code with check_argument_count in checkexpr.py? - args = call.args - if len(args) < 2: - return self.fail_namedtuple_arg("Too few arguments for namedtuple()", call) - if len(args) > 2: - return self.fail_namedtuple_arg("Too many arguments for namedtuple()", call) - if call.arg_kinds != [ARG_POS, ARG_POS]: - return self.fail_namedtuple_arg("Unexpected arguments to namedtuple()", call) - if not isinstance(args[0], (StrExpr, BytesExpr)): - return self.fail_namedtuple_arg( - "namedtuple() expects a string literal as the first argument", call) - types = [] # type: List[Type] - ok = True - if not isinstance(args[1], ListExpr): - if fullname == 'collections.namedtuple' and isinstance(args[1], (StrExpr, BytesExpr)): - str_expr = cast(StrExpr, args[1]) - items = str_expr.value.split() - else: - return self.fail_namedtuple_arg( - "List literal expected as the second argument to namedtuple()", call) - else: - listexpr = args[1] - if fullname == 'collections.namedtuple': - # The fields argument contains just names, with implicit Any types. - if any(not isinstance(item, (StrExpr, BytesExpr)) for item in listexpr.items): - return self.fail_namedtuple_arg("String literal expected as namedtuple() item", - call) - items = [cast(StrExpr, item).value for item in listexpr.items] - else: - # The fields argument contains (name, type) tuples. - items, types, ok = self.parse_namedtuple_fields_with_types(listexpr.items, call) - if not types: - types = [AnyType() for _ in items] - return items, types, ok - - def parse_namedtuple_fields_with_types(self, nodes: List[Node], - context: Context) -> Tuple[List[str], List[Type], bool]: - items = [] # type: List[str] - types = [] # type: List[Type] - for item in nodes: - if isinstance(item, TupleExpr): - if len(item.items) != 2: - return self.fail_namedtuple_arg("Invalid NamedTuple field definition", - item) - name, type_node = item.items - if isinstance(name, (StrExpr, BytesExpr)): - items.append(name.value) - else: - return self.fail_namedtuple_arg("Invalid NamedTuple() field name", item) - try: - type = expr_to_unanalyzed_type(type_node) - except TypeTranslationError: - return self.fail_namedtuple_arg('Invalid field type', type_node) - types.append(self.anal_type(type)) - else: - return self.fail_namedtuple_arg("Tuple expected as NamedTuple() field", item) - return items, types, True - - def fail_namedtuple_arg(self, message: str, - context: Context) -> Tuple[List[str], List[Type], bool]: - self.fail(message, context) - return [], [], False - - def build_namedtuple_typeinfo(self, name: str, items: List[str], - types: List[Type]) -> TypeInfo: - symbols = SymbolTable() - class_def = ClassDef(name, Block([])) - class_def.fullname = self.qualified_name(name) - info = TypeInfo(symbols, class_def) - # Add named tuple items as attributes. - # TODO: Make them read-only. - for item, typ in zip(items, types): - var = Var(item) - var.info = info - var.type = typ - symbols[item] = SymbolTableNode(MDEF, var) - # Add __init__, _replace, _asdict methods and _fields static field - self.add_namedtuple_method('__init__', symbols, info, NoneTyp(), - self.make_factory_args(items, types, ARG_POS)) - # TODO: refine return type to type(self) - self.add_namedtuple_method('_replace', symbols, info, AnyType(), - self.make_factory_args(items, types, ARG_NAMED, initializer=EllipsisExpr())) - # TODO: refine to OrderedDict[Union[types]] - self.add_namedtuple_method('_asdict', symbols, info, AnyType()) - - info.tuple_type = TupleType(types, self.named_type('__builtins__.tuple', [AnyType()])) - info.is_named_tuple = True - info.mro = [info] + info.tuple_type.fallback.type.mro - info.bases = [info.tuple_type.fallback] - return info - - def make_factory_args(self, items: List[str], types: List[Type], - kind: int, initializer: Expression = None) -> List[Argument]: - return [Argument(Var(item), type, initializer, kind) for item, type in zip(items, types)] - - def add_namedtuple_method(self, funcname: str, symbols: SymbolTable, info: TypeInfo, - ret: Type, args: List[Argument] = []) -> None: - types = [cast(Type, None)] + [arg.type_annotation for arg in args] - args = [Argument(Var('__self'), NoneTyp(), None, ARG_POS)] + args - items = [arg.variable.name() for arg in args] - arg_kinds = [arg.kind for arg in args] - # TODO: Make sure that the self argument name is not visible? - signature = CallableType(types, arg_kinds, items, ret, - self.named_type('__builtins__.function'), - name=info.name()) - func = FuncDef(funcname, - args, - Block([]), - typ=signature) - func.info = info - symbols[funcname] = SymbolTableNode(MDEF, func) - - def analyze_types(self, items: List[Node]) -> List[Type]: - result = [] # type: List[Type] - for node in items: - try: - result.append(self.anal_type(expr_to_unanalyzed_type(node))) - except TypeTranslationError: - self.fail('Type expected', node) - result.append(AnyType()) - return result - - def visit_decorator(self, dec: Decorator) -> None: - for d in dec.decorators: - d.accept(self) - removed = [] # type: List[int] - no_type_check = False - for i, d in enumerate(dec.decorators): - # A bunch of decorators are special cased here. - if refers_to_fullname(d, 'abc.abstractmethod'): - removed.append(i) - dec.func.is_abstract = True - self.check_decorated_function_is_method('abstractmethod', dec) - elif refers_to_fullname(d, 'asyncio.tasks.coroutine'): - removed.append(i) - elif refers_to_fullname(d, 'builtins.staticmethod'): - removed.append(i) - dec.func.is_static = True - dec.var.is_staticmethod = True - self.check_decorated_function_is_method('staticmethod', dec) - elif refers_to_fullname(d, 'builtins.classmethod'): - removed.append(i) - dec.func.is_class = True - dec.var.is_classmethod = True - self.check_decorated_function_is_method('classmethod', dec) - elif (refers_to_fullname(d, 'builtins.property') or - refers_to_fullname(d, 'abc.abstractproperty')): - removed.append(i) - dec.func.is_property = True - dec.var.is_property = True - if refers_to_fullname(d, 'abc.abstractproperty'): - dec.func.is_abstract = True - self.check_decorated_function_is_method('property', dec) - if len(dec.func.arguments) > 1: - self.fail('Too many arguments', dec.func) - elif refers_to_fullname(d, 'typing.no_type_check'): - dec.var.type = AnyType() - no_type_check = True - for i in reversed(removed): - del dec.decorators[i] - if not dec.is_overload or dec.var.is_property: - if self.is_func_scope(): - self.add_symbol(dec.var.name(), SymbolTableNode(LDEF, dec), - dec) - elif self.type: - dec.var.info = self.type - dec.var.is_initialized_in_class = True - self.add_symbol(dec.var.name(), SymbolTableNode(MDEF, dec), - dec) - if not no_type_check: - dec.func.accept(self) - if dec.decorators and dec.var.is_property: - self.fail('Decorated property not supported', dec) - - def check_decorated_function_is_method(self, decorator: str, - context: Context) -> None: - if not self.type or self.is_func_scope(): - self.fail("'%s' used with a non-method" % decorator, context) - - def visit_expression_stmt(self, s: ExpressionStmt) -> None: - s.expr.accept(self) - - def visit_return_stmt(self, s: ReturnStmt) -> None: - if not self.is_func_scope(): - self.fail("'return' outside function", s) - if s.expr: - s.expr.accept(self) - - def visit_raise_stmt(self, s: RaiseStmt) -> None: - if s.expr: - s.expr.accept(self) - if s.from_expr: - s.from_expr.accept(self) - - def visit_assert_stmt(self, s: AssertStmt) -> None: - if s.expr: - s.expr.accept(self) - - def visit_operator_assignment_stmt(self, - s: OperatorAssignmentStmt) -> None: - s.lvalue.accept(self) - s.rvalue.accept(self) - if (isinstance(s.lvalue, NameExpr) and s.lvalue.name == '__all__' and - s.lvalue.kind == GDEF and isinstance(s.rvalue, (ListExpr, TupleExpr))): - self.add_exports(*s.rvalue.items) - - def visit_while_stmt(self, s: WhileStmt) -> None: - s.expr.accept(self) - self.loop_depth += 1 - s.body.accept(self) - self.loop_depth -= 1 - self.visit_block_maybe(s.else_body) - - def visit_for_stmt(self, s: ForStmt) -> None: - s.expr.accept(self) - - # Bind index variables and check if they define new names. - self.analyze_lvalue(s.index) - - self.loop_depth += 1 - self.visit_block(s.body) - self.loop_depth -= 1 - - self.visit_block_maybe(s.else_body) - - def visit_break_stmt(self, s: BreakStmt) -> None: - if self.loop_depth == 0: - self.fail("'break' outside loop", s, True, blocker=True) - - def visit_continue_stmt(self, s: ContinueStmt) -> None: - if self.loop_depth == 0: - self.fail("'continue' outside loop", s, True, blocker=True) - - def visit_if_stmt(self, s: IfStmt) -> None: - infer_reachability_of_if_statement(s, pyversion=self.options.python_version) - for i in range(len(s.expr)): - s.expr[i].accept(self) - self.visit_block(s.body[i]) - self.visit_block_maybe(s.else_body) - - def visit_try_stmt(self, s: TryStmt) -> None: - self.analyze_try_stmt(s, self) - - def analyze_try_stmt(self, s: TryStmt, visitor: NodeVisitor, - add_global: bool = False) -> None: - s.body.accept(visitor) - for type, var, handler in zip(s.types, s.vars, s.handlers): - if type: - type.accept(visitor) - if var: - self.analyze_lvalue(var, add_global=add_global) - handler.accept(visitor) - if s.else_body: - s.else_body.accept(visitor) - if s.finally_body: - s.finally_body.accept(visitor) - - def visit_with_stmt(self, s: WithStmt) -> None: - for e, n in zip(s.expr, s.target): - e.accept(self) - if n: - self.analyze_lvalue(n) - self.visit_block(s.body) - - def visit_del_stmt(self, s: DelStmt) -> None: - s.expr.accept(self) - if not self.is_valid_del_target(s.expr): - self.fail('Invalid delete target', s) - - def is_valid_del_target(self, s: Node) -> bool: - if isinstance(s, (IndexExpr, NameExpr, MemberExpr)): - return True - elif isinstance(s, TupleExpr): - return all(self.is_valid_del_target(item) for item in s.items) - - def visit_global_decl(self, g: GlobalDecl) -> None: - for name in g.names: - if name in self.nonlocal_decls[-1]: - self.fail("Name '{}' is nonlocal and global".format(name), g) - self.global_decls[-1].add(name) - - def visit_nonlocal_decl(self, d: NonlocalDecl) -> None: - if not self.is_func_scope(): - self.fail("nonlocal declaration not allowed at module level", d) - else: - for name in d.names: - for table in reversed(self.locals[:-1]): - if table is not None and name in table: - break - else: - self.fail("No binding for nonlocal '{}' found".format(name), d) - - if self.locals[-1] is not None and name in self.locals[-1]: - self.fail("Name '{}' is already defined in local " - "scope before nonlocal declaration".format(name), d) - - if name in self.global_decls[-1]: - self.fail("Name '{}' is nonlocal and global".format(name), d) - self.nonlocal_decls[-1].add(name) - - def visit_print_stmt(self, s: PrintStmt) -> None: - for arg in s.args: - arg.accept(self) - if s.target: - s.target.accept(self) - - def visit_exec_stmt(self, s: ExecStmt) -> None: - s.expr.accept(self) - if s.variables1: - s.variables1.accept(self) - if s.variables2: - s.variables2.accept(self) - - # - # Expressions - # - - def visit_name_expr(self, expr: NameExpr) -> None: - n = self.lookup(expr.name, expr) - if n: - if n.kind == BOUND_TVAR: - self.fail("'{}' is a type variable and only valid in type " - "context".format(expr.name), expr) - else: - expr.kind = n.kind - expr.node = n.node - expr.fullname = n.fullname - - def visit_super_expr(self, expr: SuperExpr) -> None: - if not self.type: - self.fail('"super" used outside class', expr) - return - expr.info = self.type - - def visit_tuple_expr(self, expr: TupleExpr) -> None: - for item in expr.items: - item.accept(self) - - def visit_list_expr(self, expr: ListExpr) -> None: - for item in expr.items: - item.accept(self) - - def visit_set_expr(self, expr: SetExpr) -> None: - for item in expr.items: - item.accept(self) - - def visit_dict_expr(self, expr: DictExpr) -> None: - for key, value in expr.items: - key.accept(self) - value.accept(self) - - def visit_star_expr(self, expr: StarExpr) -> None: - if not expr.valid: - self.fail('Can use starred expression only as assignment target', expr) - else: - expr.expr.accept(self) - - def visit_yield_from_expr(self, e: YieldFromExpr) -> None: - if not self.is_func_scope(): # not sure - self.fail("'yield from' outside function", e, True, blocker=True) - else: - self.function_stack[-1].is_generator = True - if e.expr: - e.expr.accept(self) - - def visit_call_expr(self, expr: CallExpr) -> None: - """Analyze a call expression. - - Some call expressions are recognized as special forms, including - cast(...) and Any(...). - """ - expr.callee.accept(self) - if refers_to_fullname(expr.callee, 'typing.cast'): - # Special form cast(...). - if not self.check_fixed_args(expr, 2, 'cast'): - return - # Translate first argument to an unanalyzed type. - try: - target = expr_to_unanalyzed_type(expr.args[0]) - except TypeTranslationError: - self.fail('Cast target is not a type', expr) - return - # Piggyback CastExpr object to the CallExpr object; it takes - # precedence over the CallExpr semantics. - expr.analyzed = CastExpr(expr.args[1], target) - expr.analyzed.line = expr.line - expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'builtins.reveal_type'): - if not self.check_fixed_args(expr, 1, 'reveal_type'): - return - expr.analyzed = RevealTypeExpr(expr.args[0]) - expr.analyzed.line = expr.line - expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'typing.Any'): - # Special form Any(...). - if not self.check_fixed_args(expr, 1, 'Any'): - return - expr.analyzed = CastExpr(expr.args[0], AnyType()) - expr.analyzed.line = expr.line - expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'typing._promote'): - # Special form _promote(...). - if not self.check_fixed_args(expr, 1, '_promote'): - return - # Translate first argument to an unanalyzed type. - try: - target = expr_to_unanalyzed_type(expr.args[0]) - except TypeTranslationError: - self.fail('Argument 1 to _promote is not a type', expr) - return - expr.analyzed = PromoteExpr(target) - expr.analyzed.line = expr.line - expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'builtins.dict'): - expr.analyzed = self.translate_dict_call(expr) - else: - # Normal call expression. - for a in expr.args: - a.accept(self) - - if (isinstance(expr.callee, MemberExpr) and - isinstance(expr.callee.expr, NameExpr) and - expr.callee.expr.name == '__all__' and - expr.callee.expr.kind == GDEF and - expr.callee.name in ('append', 'extend')): - if expr.callee.name == 'append' and expr.args: - self.add_exports(expr.args[0]) - elif (expr.callee.name == 'extend' and expr.args and - isinstance(expr.args[0], (ListExpr, TupleExpr))): - self.add_exports(*expr.args[0].items) - - def translate_dict_call(self, call: CallExpr) -> Optional[DictExpr]: - """Translate 'dict(x=y, ...)' to {'x': y, ...}. - - For other variants of dict(...), return None. - """ - if not call.args: - return None - if not all(kind == ARG_NAMED for kind in call.arg_kinds): - # Must still accept those args. - for a in call.args: - a.accept(self) - return None - expr = DictExpr([(StrExpr(key), value) - for key, value in zip(call.arg_names, call.args)]) - expr.set_line(call) - expr.accept(self) - return expr - - def check_fixed_args(self, expr: CallExpr, numargs: int, - name: str) -> bool: - """Verify that expr has specified number of positional args. - - Return True if the arguments are valid. - """ - s = 's' - if numargs == 1: - s = '' - if len(expr.args) != numargs: - self.fail("'%s' expects %d argument%s" % (name, numargs, s), - expr) - return False - if expr.arg_kinds != [ARG_POS] * numargs: - self.fail("'%s' must be called with %s positional argument%s" % - (name, numargs, s), expr) - return False - return True - - def visit_member_expr(self, expr: MemberExpr) -> None: - base = expr.expr - base.accept(self) - # Bind references to module attributes. - if isinstance(base, RefExpr) and base.kind == MODULE_REF: - file = cast(MypyFile, base.node) - n = file.names.get(expr.name, None) if file is not None else None - if n: - n = self.normalize_type_alias(n, expr) - if not n: - return - expr.kind = n.kind - expr.fullname = n.fullname - expr.node = n.node - else: - # We only catch some errors here; the rest will be - # catched during type checking. - # - # This way we can report a larger number of errors in - # one type checker run. If we reported errors here, - # the build would terminate after semantic analysis - # and we wouldn't be able to report any type errors. - full_name = '%s.%s' % (file.fullname() if file is not None else None, expr.name) - if full_name in obsolete_name_mapping: - self.fail("Module has no attribute %r (it's now called %r)" % ( - expr.name, obsolete_name_mapping[full_name]), expr) - - def visit_op_expr(self, expr: OpExpr) -> None: - expr.left.accept(self) - expr.right.accept(self) - - def visit_comparison_expr(self, expr: ComparisonExpr) -> None: - for operand in expr.operands: - operand.accept(self) - - def visit_unary_expr(self, expr: UnaryExpr) -> None: - expr.expr.accept(self) - - def visit_index_expr(self, expr: IndexExpr) -> None: - expr.base.accept(self) - if refers_to_class_or_function(expr.base): - # Special form -- type application. - # Translate index to an unanalyzed type. - types = [] # type: List[Type] - if isinstance(expr.index, TupleExpr): - items = expr.index.items - else: - items = [expr.index] - for item in items: - try: - typearg = expr_to_unanalyzed_type(item) - except TypeTranslationError: - self.fail('Type expected within [...]', expr) - return - typearg = self.anal_type(typearg) - types.append(typearg) - expr.analyzed = TypeApplication(expr.base, types) - expr.analyzed.line = expr.line - else: - expr.index.accept(self) - - def visit_slice_expr(self, expr: SliceExpr) -> None: - if expr.begin_index: - expr.begin_index.accept(self) - if expr.end_index: - expr.end_index.accept(self) - if expr.stride: - expr.stride.accept(self) - - def visit_cast_expr(self, expr: CastExpr) -> None: - expr.expr.accept(self) - expr.type = self.anal_type(expr.type) - - def visit_reveal_type_expr(self, expr: RevealTypeExpr) -> None: - expr.expr.accept(self) - - def visit_type_application(self, expr: TypeApplication) -> None: - expr.expr.accept(self) - for i in range(len(expr.types)): - expr.types[i] = self.anal_type(expr.types[i]) - - def visit_list_comprehension(self, expr: ListComprehension) -> None: - expr.generator.accept(self) - - def visit_set_comprehension(self, expr: SetComprehension) -> None: - expr.generator.accept(self) - - def visit_dictionary_comprehension(self, expr: DictionaryComprehension) -> None: - self.enter() - self.analyze_comp_for(expr) - expr.key.accept(self) - expr.value.accept(self) - self.leave() - self.analyze_comp_for_2(expr) - - def visit_generator_expr(self, expr: GeneratorExpr) -> None: - self.enter() - self.analyze_comp_for(expr) - expr.left_expr.accept(self) - self.leave() - self.analyze_comp_for_2(expr) - - def analyze_comp_for(self, expr: Union[GeneratorExpr, - DictionaryComprehension]) -> None: - """Analyses the 'comp_for' part of comprehensions (part 1). - - That is the part after 'for' in (x for x in l if p). This analyzes - variables and conditions which are analyzed in a local scope. - """ - for i, (index, sequence, conditions) in enumerate(zip(expr.indices, - expr.sequences, - expr.condlists)): - if i > 0: - sequence.accept(self) - # Bind index variables. - self.analyze_lvalue(index) - for cond in conditions: - cond.accept(self) - - def analyze_comp_for_2(self, expr: Union[GeneratorExpr, - DictionaryComprehension]) -> None: - """Analyses the 'comp_for' part of comprehensions (part 2). - - That is the part after 'for' in (x for x in l if p). This analyzes - the 'l' part which is analyzed in the surrounding scope. - """ - expr.sequences[0].accept(self) - - def visit_func_expr(self, expr: FuncExpr) -> None: - self.analyze_function(expr) - - def visit_conditional_expr(self, expr: ConditionalExpr) -> None: - expr.if_expr.accept(self) - expr.cond.accept(self) - expr.else_expr.accept(self) - - def visit_backquote_expr(self, expr: BackquoteExpr) -> None: - expr.expr.accept(self) - - def visit__promote_expr(self, expr: PromoteExpr) -> None: - expr.type = self.anal_type(expr.type) - - def visit_yield_expr(self, expr: YieldExpr) -> None: - if not self.is_func_scope(): - self.fail("'yield' outside function", expr, True, blocker=True) - else: - self.function_stack[-1].is_generator = True - if expr.expr: - expr.expr.accept(self) - - # - # Helpers - # - - def lookup(self, name: str, ctx: Context) -> SymbolTableNode: - """Look up an unqualified name in all active namespaces.""" - # 1a. Name declared using 'global x' takes precedence - if name in self.global_decls[-1]: - if name in self.globals: - return self.globals[name] - else: - self.name_not_defined(name, ctx) - return None - # 1b. Name declared using 'nonlocal x' takes precedence - if name in self.nonlocal_decls[-1]: - for table in reversed(self.locals[:-1]): - if table is not None and name in table: - return table[name] - else: - self.name_not_defined(name, ctx) - return None - # 2. Class attributes (if within class definition) - if self.is_class_scope() and name in self.type.names: - return self.type[name] - # 3. Local (function) scopes - for table in reversed(self.locals): - if table is not None and name in table: - return table[name] - # 4. Current file global scope - if name in self.globals: - return self.globals[name] - # 5. Builtins - b = self.globals.get('__builtins__', None) - if b: - table = cast(MypyFile, b.node).names - if name in table: - if name[0] == "_" and name[1] != "_": - self.name_not_defined(name, ctx) - return None - node = table[name] - return node - # Give up. - self.name_not_defined(name, ctx) - self.check_for_obsolete_short_name(name, ctx) - return None - - def check_for_obsolete_short_name(self, name: str, ctx: Context) -> None: - matches = [obsolete_name - for obsolete_name in obsolete_name_mapping - if obsolete_name.rsplit('.', 1)[-1] == name] - if len(matches) == 1: - self.note("(Did you mean '{}'?)".format(obsolete_name_mapping[matches[0]]), ctx) - - def lookup_qualified(self, name: str, ctx: Context) -> SymbolTableNode: - if '.' not in name: - return self.lookup(name, ctx) - else: - parts = name.split('.') - n = self.lookup(parts[0], ctx) # type: SymbolTableNode - if n: - for i in range(1, len(parts)): - if isinstance(n.node, TypeInfo): - if n.node.mro is None: - # We haven't yet analyzed the class `n.node`. Fall back to direct - # lookup in the names declared directly under it, without its base - # classes. This can happen when we have a forward reference to a - # nested class, and the reference is bound before the outer class - # has been fully semantically analyzed. - # - # A better approach would be to introduce a new analysis pass or - # to move things around between passes, but this unblocks a common - # use case even though this is a little limited in case there is - # inheritance involved. - result = n.node.names.get(parts[i]) - else: - result = n.node.get(parts[i]) - n = result - elif isinstance(n.node, MypyFile): - n = n.node.names.get(parts[i], None) - # TODO: What if node is Var or FuncDef? - if not n: - self.name_not_defined(name, ctx) - break - if n: - n = self.normalize_type_alias(n, ctx) - return n - - def builtin_type(self, fully_qualified_name: str) -> Instance: - node = self.lookup_fully_qualified(fully_qualified_name) - info = cast(TypeInfo, node.node) - return Instance(info, []) - - def lookup_fully_qualified(self, name: str) -> SymbolTableNode: - """Lookup a fully qualified name. - - Assume that the name is defined. This happens in the global namespace -- the local - module namespace is ignored. - """ - assert '.' in name - parts = name.split('.') - n = self.modules[parts[0]] - for i in range(1, len(parts) - 1): - n = cast(MypyFile, n.names[parts[i]].node) - return n.names[parts[-1]] - - def lookup_fully_qualified_or_none(self, name: str) -> SymbolTableNode: - """Lookup a fully qualified name. - - Assume that the name is defined. This happens in the global namespace -- the local - module namespace is ignored. - """ - assert '.' in name - parts = name.split('.') - n = self.modules[parts[0]] - for i in range(1, len(parts) - 1): - next_sym = n.names.get(parts[i]) - if not next_sym: - return None - n = cast(MypyFile, next_sym.node) - return n.names.get(parts[-1]) - - def qualified_name(self, n: str) -> str: - return self.cur_mod_id + '.' + n - - def enter(self) -> None: - self.locals.append(SymbolTable()) - self.global_decls.append(set()) - self.nonlocal_decls.append(set()) - # -1 since entering block will increment this to 0. - self.block_depth.append(-1) - - def leave(self) -> None: - self.locals.pop() - self.global_decls.pop() - self.nonlocal_decls.pop() - self.block_depth.pop() - - def is_func_scope(self) -> bool: - return self.locals[-1] is not None - - def is_class_scope(self) -> bool: - return self.type is not None and not self.is_func_scope() - - def add_symbol(self, name: str, node: SymbolTableNode, - context: Context) -> None: - if self.is_func_scope(): - if name in self.locals[-1]: - # Flag redefinition unless this is a reimport of a module. - if not (node.kind == MODULE_REF and - self.locals[-1][name].node == node.node): - self.name_already_defined(name, context) - self.locals[-1][name] = node - elif self.type: - self.type.names[name] = node - else: - existing = self.globals.get(name) - if existing and (not isinstance(node.node, MypyFile) or - existing.node != node.node) and existing.kind != UNBOUND_IMPORTED: - # Modules can be imported multiple times to support import - # of multiple submodules of a package (e.g. a.x and a.y). - ok = False - # Only report an error if the symbol collision provides a different type. - if existing.type and node.type and is_same_type(existing.type, node.type): - ok = True - if not ok: - self.name_already_defined(name, context) - self.globals[name] = node - - def add_var(self, v: Var, ctx: Context) -> None: - if self.is_func_scope(): - self.add_local(v, ctx) - else: - self.globals[v.name()] = SymbolTableNode(GDEF, v, self.cur_mod_id) - v._fullname = self.qualified_name(v.name()) - - def add_local(self, node: Union[Var, FuncBase], ctx: Context) -> None: - name = node.name() - if name in self.locals[-1]: - self.name_already_defined(name, ctx) - node._fullname = name - self.locals[-1][name] = SymbolTableNode(LDEF, node) - - def add_exports(self, *exps: Node) -> None: - for exp in exps: - if isinstance(exp, StrExpr): - self.all_exports.add(exp.value) - - def check_no_global(self, n: str, ctx: Context, - is_overloaded_func: bool = False) -> None: - if n in self.globals: - prev_is_overloaded = isinstance(self.globals[n], OverloadedFuncDef) - if is_overloaded_func and prev_is_overloaded: - self.fail("Nonconsecutive overload {} found".format(n), ctx) - elif prev_is_overloaded: - self.fail("Definition of '{}' missing 'overload'".format(n), ctx) - else: - self.name_already_defined(n, ctx) - - def name_not_defined(self, name: str, ctx: Context) -> None: - message = "Name '{}' is not defined".format(name) - extra = self.undefined_name_extra_info(name) - if extra: - message += ' {}'.format(extra) - self.fail(message, ctx) - - def name_already_defined(self, name: str, ctx: Context) -> None: - self.fail("Name '{}' already defined".format(name), ctx) - - def fail(self, msg: str, ctx: Context, serious: bool = False, *, - blocker: bool = False) -> None: - if (not serious and - not self.options.check_untyped_defs and - self.function_stack and - self.function_stack[-1].is_dynamic()): - return - self.errors.report(ctx.get_line(), msg, blocker=blocker) - - def note(self, msg: str, ctx: Context) -> None: - if (not self.options.check_untyped_defs and - self.function_stack and - self.function_stack[-1].is_dynamic()): - return - self.errors.report(ctx.get_line(), msg, severity='note') - - def undefined_name_extra_info(self, fullname: str) -> Optional[str]: - if fullname in obsolete_name_mapping: - return "(it's now called '{}')".format(obsolete_name_mapping[fullname]) - else: - return None - - def accept(self, node: Node) -> None: - try: - node.accept(self) - except Exception as err: - report_internal_error(err, self.errors.file, node.line) - - -class FirstPass(NodeVisitor): - """First phase of semantic analysis. - - See docstring of 'analyze()' below for a description of what this does. - """ - - def __init__(self, sem: SemanticAnalyzer) -> None: - self.sem = sem - self.pyversion = sem.options.python_version - - def analyze(self, file: MypyFile, fnam: str, mod_id: str) -> None: - """Perform the first analysis pass. - - Populate module global table. Resolve the full names of - definitions not nested within functions and construct type - info structures, but do not resolve inter-definition - references such as base classes. - - Also add implicit definitions such as __name__. - - In this phase we don't resolve imports. For 'from ... import', - we generate dummy symbol table nodes for the imported names, - and these will get resolved in later phases of semantic - analysis. - """ - sem = self.sem - sem.cur_mod_id = mod_id - sem.errors.set_file(fnam) - sem.globals = SymbolTable() - sem.global_decls = [set()] - sem.nonlocal_decls = [set()] - sem.block_depth = [0] - - defs = file.defs - - # Add implicit definitions of module '__name__' etc. - for name, t in implicit_module_attrs.items(): - v = Var(name, UnboundType(t)) - v._fullname = self.sem.qualified_name(name) - self.sem.globals[name] = SymbolTableNode(GDEF, v, self.sem.cur_mod_id) - - for d in defs: - d.accept(self) - - # Add implicit definition of literals/keywords to builtins, as we - # cannot define a variable with them explicitly. - if mod_id == 'builtins': - literal_types = [ - ('None', NoneTyp()), - # reveal_type is a mypy-only function that gives an error with the type of its arg - ('reveal_type', AnyType()), - ] # type: List[Tuple[str, Type]] - - # TODO(ddfisher): This guard is only needed because mypy defines - # fake builtins for its tests which often don't define bool. If - # mypy is fast enough that we no longer need those, this - # conditional check should be removed. - if 'bool' in self.sem.globals: - bool_type = self.sem.named_type('bool') - literal_types.extend([ - ('True', bool_type), - ('False', bool_type), - ('__debug__', bool_type), - ]) - - for name, typ in literal_types: - v = Var(name, typ) - v._fullname = self.sem.qualified_name(name) - self.sem.globals[name] = SymbolTableNode(GDEF, v, self.sem.cur_mod_id) - - def visit_block(self, b: Block) -> None: - if b.is_unreachable: - return - self.sem.block_depth[-1] += 1 - for node in b.body: - node.accept(self) - self.sem.block_depth[-1] -= 1 - - def visit_assignment_stmt(self, s: AssignmentStmt) -> None: - for lval in s.lvalues: - self.analyze_lvalue(lval, explicit_type=s.type is not None) - - def visit_func_def(self, func: FuncDef) -> None: - sem = self.sem - func.is_conditional = sem.block_depth[-1] > 0 - func._fullname = sem.qualified_name(func.name()) - if func.name() in sem.globals: - # Already defined in this module. - original_sym = sem.globals[func.name()] - if original_sym.kind == UNBOUND_IMPORTED: - # Ah this is an imported name. We can't resolve them now, so we'll postpone - # this until the main phase of semantic analysis. - return - original_def = original_sym.node - if sem.is_conditional_func(original_def, func): - # Conditional function definition -- multiple defs are ok. - func.original_def = cast(FuncDef, original_def) - else: - # Report error. - sem.check_no_global(func.name(), func) - else: - sem.globals[func.name()] = SymbolTableNode(GDEF, func, sem.cur_mod_id) - - def visit_overloaded_func_def(self, func: OverloadedFuncDef) -> None: - self.sem.check_no_global(func.name(), func, True) - func._fullname = self.sem.qualified_name(func.name()) - self.sem.globals[func.name()] = SymbolTableNode(GDEF, func, - self.sem.cur_mod_id) - - def visit_class_def(self, cdef: ClassDef) -> None: - self.sem.check_no_global(cdef.name, cdef) - cdef.fullname = self.sem.qualified_name(cdef.name) - info = TypeInfo(SymbolTable(), cdef) - info.set_line(cdef.line) - cdef.info = info - self.sem.globals[cdef.name] = SymbolTableNode(GDEF, info, - self.sem.cur_mod_id) - self.process_nested_classes(cdef) - - def process_nested_classes(self, outer_def: ClassDef) -> None: - for node in outer_def.defs.body: - if isinstance(node, ClassDef): - node.info = TypeInfo(SymbolTable(), node) - if outer_def.fullname: - node.info._fullname = outer_def.fullname + '.' + node.info.name() - else: - node.info._fullname = node.info.name() - symbol = SymbolTableNode(MDEF, node.info) - outer_def.info.names[node.name] = symbol - self.process_nested_classes(node) - - def visit_import_from(self, node: ImportFrom) -> None: - # We can't bind module names during the first pass, as the target module might be - # unprocessed. However, we add dummy unbound imported names to the symbol table so - # that we at least know that the name refers to a module. - node.is_top_level = True - for name, as_name in node.names: - imported_name = as_name or name - if imported_name not in self.sem.globals: - self.sem.add_symbol(imported_name, SymbolTableNode(UNBOUND_IMPORTED, None), node) - - def visit_import(self, node: Import) -> None: - node.is_top_level = True - # This is similar to visit_import_from -- see the comment there. - for id, as_id in node.ids: - imported_id = as_id or id - if imported_id not in self.sem.globals: - self.sem.add_symbol(imported_id, SymbolTableNode(UNBOUND_IMPORTED, None), node) - else: - # If the previous symbol is a variable, this should take precedence. - self.sem.globals[imported_id] = SymbolTableNode(UNBOUND_IMPORTED, None) - - def visit_import_all(self, node: ImportAll) -> None: - node.is_top_level = True - - def visit_while_stmt(self, s: WhileStmt) -> None: - s.body.accept(self) - if s.else_body: - s.else_body.accept(self) - - def visit_for_stmt(self, s: ForStmt) -> None: - self.analyze_lvalue(s.index) - s.body.accept(self) - if s.else_body: - s.else_body.accept(self) - - def visit_with_stmt(self, s: WithStmt) -> None: - for n in s.target: - if n: - self.analyze_lvalue(n) - s.body.accept(self) - - def visit_decorator(self, d: Decorator) -> None: - d.var._fullname = self.sem.qualified_name(d.var.name()) - self.sem.add_symbol(d.var.name(), SymbolTableNode(GDEF, d.var), d) - - def visit_if_stmt(self, s: IfStmt) -> None: - infer_reachability_of_if_statement(s, pyversion=self.pyversion) - for node in s.body: - node.accept(self) - if s.else_body: - s.else_body.accept(self) - - def visit_try_stmt(self, s: TryStmt) -> None: - self.sem.analyze_try_stmt(s, self, add_global=True) - - def analyze_lvalue(self, lvalue: Node, explicit_type: bool = False) -> None: - self.sem.analyze_lvalue(lvalue, add_global=True, explicit_type=explicit_type) - - -class ThirdPass(TraverserVisitor): - """The third and final pass of semantic analysis. - - Check type argument counts and values of generic types, and perform some - straightforward type inference. - """ - - def __init__(self, modules: Dict[str, MypyFile], errors: Errors) -> None: - self.modules = modules - self.errors = errors - - def visit_file(self, file_node: MypyFile, fnam: str) -> None: - self.errors.set_file(fnam) - self.accept(file_node) - - def accept(self, node: Node) -> None: - try: - node.accept(self) - except Exception as err: - report_internal_error(err, self.errors.file, node.line) - - def visit_block(self, b: Block) -> None: - if b.is_unreachable: - return - super().visit_block(b) - - def visit_func_def(self, fdef: FuncDef) -> None: - self.errors.push_function(fdef.name()) - self.analyze(fdef.type) - super().visit_func_def(fdef) - self.errors.pop_function() - - def visit_class_def(self, tdef: ClassDef) -> None: - for type in tdef.info.bases: - self.analyze(type) - # Recompute MRO now that we have analyzed all modules, to pick - # up superclasses of bases imported from other modules in an - # import loop. (Only do so if we succeeded the first time.) - if tdef.info.mro: - tdef.info.mro = [] # Force recomputation - calculate_class_mro(tdef, self.fail) - super().visit_class_def(tdef) - - def visit_decorator(self, dec: Decorator) -> None: - """Try to infer the type of the decorated function. - - This lets us resolve references to decorated functions during - type checking when there are cyclic imports, as otherwise the - type might not be available when we need it. - - This basically uses a simple special-purpose type inference - engine just for decorators. - """ - super().visit_decorator(dec) - if dec.var.is_property: - # Decorators are expected to have a callable type (it's a little odd). - if dec.func.type is None: - dec.var.type = CallableType( - [AnyType()], - [ARG_POS], - [None], - AnyType(), - self.builtin_type('function'), - name=dec.var.name()) - elif isinstance(dec.func.type, CallableType): - dec.var.type = dec.func.type - return - decorator_preserves_type = True - for expr in dec.decorators: - preserve_type = False - if isinstance(expr, RefExpr) and isinstance(expr.node, FuncDef): - if is_identity_signature(expr.node.type): - preserve_type = True - if not preserve_type: - decorator_preserves_type = False - break - if decorator_preserves_type: - # No non-identity decorators left. We can trivially infer the type - # of the function here. - dec.var.type = function_type(dec.func, self.builtin_type('function')) - if dec.decorators: - if returns_any_if_called(dec.decorators[0]): - # The outermost decorator will return Any so we know the type of the - # decorated function. - dec.var.type = AnyType() - sig = find_fixed_callable_return(dec.decorators[0]) - if sig: - # The outermost decorator always returns the same kind of function, - # so we know that this is the type of the decoratored function. - orig_sig = function_type(dec.func, self.builtin_type('function')) - sig.name = orig_sig.items()[0].name - dec.var.type = sig - - def visit_assignment_stmt(self, s: AssignmentStmt) -> None: - self.analyze(s.type) - super().visit_assignment_stmt(s) - - def visit_cast_expr(self, e: CastExpr) -> None: - self.analyze(e.type) - super().visit_cast_expr(e) - - def visit_reveal_type_expr(self, e: RevealTypeExpr) -> None: - super().visit_reveal_type_expr(e) - - def visit_type_application(self, e: TypeApplication) -> None: - for type in e.types: - self.analyze(type) - super().visit_type_application(e) - - # Helpers - - def analyze(self, type: Type) -> None: - if type: - analyzer = TypeAnalyserPass3(self.fail) - type.accept(analyzer) - - def fail(self, msg: str, ctx: Context) -> None: - self.errors.report(ctx.get_line(), msg) - - def builtin_type(self, name: str, args: List[Type] = None) -> Instance: - names = self.modules['builtins'] - sym = names.names[name] - assert isinstance(sym.node, TypeInfo) - return Instance(sym.node, args or []) - - -def self_type(typ: TypeInfo) -> Union[Instance, TupleType]: - """For a non-generic type, return instance type representing the type. - For a generic G type with parameters T1, .., Tn, return G[T1, ..., Tn]. - """ - tv = [] # type: List[Type] - for i in range(len(typ.type_vars)): - tv.append(TypeVarType(typ.defn.type_vars[i])) - inst = Instance(typ, tv) - if typ.tuple_type is None: - return inst - else: - return TupleType(typ.tuple_type.items, inst) - - -def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: - if isinstance(sig, CallableType): - return replace_leading_arg_type(sig, new) - else: - sig = cast(Overloaded, sig) - return Overloaded([cast(CallableType, replace_implicit_first_type(i, new)) - for i in sig.items()]) - - -def set_callable_name(sig: Type, fdef: FuncDef) -> Type: - if isinstance(sig, FunctionLike): - if fdef.info: - return sig.with_name( - '"{}" of "{}"'.format(fdef.name(), fdef.info.name())) - else: - return sig.with_name('"{}"'.format(fdef.name())) - else: - return sig - - -def refers_to_fullname(node: Node, fullname: str) -> bool: - """Is node a name or member expression with the given full name?""" - return isinstance(node, RefExpr) and node.fullname == fullname - - -def refers_to_class_or_function(node: Node) -> bool: - """Does semantically analyzed node refer to a class?""" - return (isinstance(node, RefExpr) and - isinstance(node.node, (TypeInfo, FuncDef, OverloadedFuncDef))) - - -def calculate_class_mro(defn: ClassDef, fail: Callable[[str, Context], None]) -> None: - try: - defn.info.calculate_mro() - except MroError: - fail("Cannot determine consistent method resolution order " - '(MRO) for "%s"' % defn.name, defn) - defn.info.mro = [] - # The property of falling back to Any is inherited. - defn.info.fallback_to_any = any(baseinfo.fallback_to_any for baseinfo in defn.info.mro) - - -def find_duplicate(list: List[T]) -> T: - """If the list has duplicates, return one of the duplicates. - - Otherwise, return None. - """ - for i in range(1, len(list)): - if list[i] in list[:i]: - return list[i] - return None - - -def disable_typevars(nodes: List[SymbolTableNode]) -> None: - for node in nodes: - assert node.kind in (BOUND_TVAR, UNBOUND_TVAR) - node.kind = UNBOUND_TVAR - - -def enable_typevars(nodes: List[SymbolTableNode]) -> None: - for node in nodes: - assert node.kind in (BOUND_TVAR, UNBOUND_TVAR) - node.kind = BOUND_TVAR - - -def remove_imported_names_from_symtable(names: SymbolTable, - module: str) -> None: - """Remove all imported names from the symbol table of a module.""" - removed = [] # type: List[str] - for name, node in names.items(): - if node.node is None: - continue - fullname = node.node.fullname() - prefix = fullname[:fullname.rfind('.')] - if prefix != module: - removed.append(name) - for name in removed: - del names[name] - - -def infer_reachability_of_if_statement(s: IfStmt, - pyversion: Tuple[int, int]) -> None: - for i in range(len(s.expr)): - result = infer_if_condition_value(s.expr[i], pyversion) - if result == ALWAYS_FALSE: - # The condition is always false, so we skip the if/elif body. - mark_block_unreachable(s.body[i]) - elif result == ALWAYS_TRUE: - # This condition is always true, so all of the remaining - # elif/else bodies will never be executed. - for body in s.body[i + 1:]: - mark_block_unreachable(body) - if s.else_body: - mark_block_unreachable(s.else_body) - break - - -def infer_if_condition_value(expr: Node, pyversion: Tuple[int, int]) -> int: - """Infer whether if condition is always true/false. - - Return ALWAYS_TRUE if always true, ALWAYS_FALSE if always false, - and TRUTH_VALUE_UNKNOWN otherwise. - """ - name = '' - negated = False - alias = expr - if isinstance(alias, UnaryExpr): - if alias.op == 'not': - expr = alias.expr - negated = True - if isinstance(expr, NameExpr): - name = expr.name - elif isinstance(expr, MemberExpr): - name = expr.name - result = TRUTH_VALUE_UNKNOWN - if name == 'PY2': - result = ALWAYS_TRUE if pyversion[0] == 2 else ALWAYS_FALSE - elif name == 'PY3': - result = ALWAYS_TRUE if pyversion[0] == 3 else ALWAYS_FALSE - elif name == 'MYPY': - result = ALWAYS_TRUE - if negated: - if result == ALWAYS_TRUE: - result = ALWAYS_FALSE - elif result == ALWAYS_FALSE: - result = ALWAYS_TRUE - return result - - -def mark_block_unreachable(block: Block) -> None: - block.is_unreachable = True - block.accept(MarkImportsUnreachableVisitor()) - - -class MarkImportsUnreachableVisitor(TraverserVisitor): - """Visitor that flags all imports nested within a node as unreachable.""" - - def visit_import(self, node: Import) -> None: - node.is_unreachable = True - - def visit_import_from(self, node: ImportFrom) -> None: - node.is_unreachable = True - - def visit_import_all(self, node: ImportAll) -> None: - node.is_unreachable = True - - -def is_identity_signature(sig: Type) -> bool: - """Is type a callable of form T -> T (where T is a type variable)?""" - if isinstance(sig, CallableType) and sig.arg_kinds == [ARG_POS]: - if isinstance(sig.arg_types[0], TypeVarType) and isinstance(sig.ret_type, TypeVarType): - return sig.arg_types[0].id == sig.ret_type.id - return False - - -def returns_any_if_called(expr: Node) -> bool: - """Return True if we can predict that expr will return Any if called. - - This only uses information available during semantic analysis so this - will sometimes return False because of insufficient information (as - type inference hasn't run yet). - """ - if isinstance(expr, RefExpr): - if isinstance(expr.node, FuncDef): - typ = expr.node.type - if typ is None: - # No signature -> default to Any. - return True - # Explicit Any return? - return isinstance(typ, CallableType) and isinstance(typ.ret_type, AnyType) - elif isinstance(expr.node, Var): - typ = expr.node.type - return typ is None or isinstance(typ, AnyType) - elif isinstance(expr, CallExpr): - return returns_any_if_called(expr.callee) - return False - - -def find_fixed_callable_return(expr: Node) -> Optional[CallableType]: - if isinstance(expr, RefExpr): - if isinstance(expr.node, FuncDef): - typ = expr.node.type - if typ: - if isinstance(typ, CallableType) and has_no_typevars(typ.ret_type): - if isinstance(typ.ret_type, CallableType): - return typ.ret_type - elif isinstance(expr, CallExpr): - t = find_fixed_callable_return(expr.callee) - if t: - if isinstance(t.ret_type, CallableType): - return t.ret_type - return None - - -def has_no_typevars(typ: Type) -> bool: - return is_same_type(typ, erase_typevars(typ)) +"""The semantic analyzer. + +Bind names to definitions and do various other simple consistency +checks. For example, consider this program: + + x = 1 + y = x + +Here semantic analysis would detect that the assignment 'x = 1' +defines a new variable, the type of which is to be inferred (in a +later pass; type inference or type checking is not part of semantic +analysis). Also, it would bind both references to 'x' to the same +module-level variable node. The second assignment would also be +analyzed, and the type of 'y' marked as being inferred. + +Semantic analysis is the first analysis pass after parsing, and it is +subdivided into three passes: + + * FirstPass looks up externally visible names defined in a module but + ignores imports and local definitions. It helps enable (some) + cyclic references between modules, such as module 'a' that imports + module 'b' and used names defined in b *and* vice versa. The first + pass can be performed before dependent modules have been processed. + + * SemanticAnalyzer is the second pass. It does the bulk of the work. + It assumes that dependent modules have been semantically analyzed, + up to the second pass, unless there is a import cycle. + + * ThirdPass checks that type argument counts are valid; for example, + it will reject Dict[int]. We don't do this in the second pass, + since we infer the type argument counts of classes during this + pass, and it is possible to refer to classes defined later in a + file, which would not have the type argument count set yet. This + pass also recomputes the method resolution order of each class, in + case one of its bases belongs to a module involved in an import + loop. + +Semantic analysis of types is implemented in module mypy.typeanal. + +TODO: Check if the third pass slows down type checking significantly. + We could probably get rid of it -- for example, we could collect all + analyzed types in a collection and check them without having to + traverse the entire AST. +""" + +from typing import ( + List, Dict, Set, Tuple, cast, Any, overload, TypeVar, Union, Optional, Callable +) + +from mypy.nodes import ( + MypyFile, TypeInfo, Node, AssignmentStmt, FuncDef, OverloadedFuncDef, + ClassDef, Var, GDEF, MODULE_REF, FuncItem, Import, + ImportFrom, ImportAll, Block, LDEF, NameExpr, MemberExpr, + IndexExpr, TupleExpr, ListExpr, ExpressionStmt, ReturnStmt, + RaiseStmt, AssertStmt, OperatorAssignmentStmt, WhileStmt, + ForStmt, BreakStmt, ContinueStmt, IfStmt, TryStmt, WithStmt, DelStmt, + GlobalDecl, SuperExpr, DictExpr, CallExpr, RefExpr, OpExpr, UnaryExpr, + SliceExpr, CastExpr, RevealTypeExpr, TypeApplication, Context, SymbolTable, + SymbolTableNode, BOUND_TVAR, UNBOUND_TVAR, ListComprehension, GeneratorExpr, + FuncExpr, MDEF, FuncBase, Decorator, SetExpr, TypeVarExpr, + StrExpr, BytesExpr, PrintStmt, ConditionalExpr, PromoteExpr, + ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, MroError, type_aliases, + YieldFromExpr, NamedTupleExpr, NonlocalDecl, + SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr, + YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, COVARIANT, CONTRAVARIANT, + IntExpr, FloatExpr, UnicodeExpr, + INVARIANT, UNBOUND_IMPORTED, Expression, EllipsisExpr, NamedTupleTypeInfo +) +from mypy.visitor import NodeVisitor +from mypy.traverser import TraverserVisitor +from mypy.errors import Errors, report_internal_error +from mypy.types import ( + NoneTyp, CallableType, Overloaded, Instance, Type, TypeVarType, AnyType, + FunctionLike, UnboundType, TypeList, ErrorType, TypeVarDef, + replace_leading_arg_type, TupleType, UnionType, StarType, EllipsisType +) +from mypy.nodes import function_type, implicit_module_attrs +from mypy.typeanal import TypeAnalyser, TypeAnalyserPass3, analyze_type_alias +from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError +from mypy.lex import lex +from mypy.parsetype import parse_type +from mypy.sametypes import is_same_type +from mypy.erasetype import erase_typevars +from mypy import defaults +from mypy.options import Options + + +T = TypeVar('T') + + +# Inferred value of an expression. +ALWAYS_TRUE = 0 +ALWAYS_FALSE = 1 +TRUTH_VALUE_UNKNOWN = 2 + +# Map from obsolete name to the current spelling. +obsolete_name_mapping = { + 'typing.Function': 'typing.Callable', + 'typing.typevar': 'typing.TypeVar', +} + +# Hard coded type promotions (shared between all Python versions). +# These add extra ad-hoc edges to the subtyping relation. For example, +# int is considered a subtype of float, even though there is no +# subclass relationship. +TYPE_PROMOTIONS = { + 'builtins.int': 'builtins.float', + 'builtins.float': 'builtins.complex', +} + +# Hard coded type promotions for Python 3. +# +# Note that the bytearray -> bytes promotion is a little unsafe +# as some functions only accept bytes objects. Here convenience +# trumps safety. +TYPE_PROMOTIONS_PYTHON3 = TYPE_PROMOTIONS.copy() +TYPE_PROMOTIONS_PYTHON3.update({ + 'builtins.bytearray': 'builtins.bytes', +}) + +# Hard coded type promotions for Python 2. +# +# These promotions are unsafe, but we are doing them anyway +# for convenience and also for Python 3 compatibility +# (bytearray -> str). +TYPE_PROMOTIONS_PYTHON2 = TYPE_PROMOTIONS.copy() +TYPE_PROMOTIONS_PYTHON2.update({ + 'builtins.str': 'builtins.unicode', + 'builtins.bytearray': 'builtins.str', +}) + +# When analyzing a function, should we analyze the whole function in one go, or +# should we only perform one phase of the analysis? The latter is used for +# nested functions. In the first phase we add the function to the symbol table +# but don't process body. In the second phase we process function body. This +# way we can have mutually recursive nested functions. +FUNCTION_BOTH_PHASES = 0 # Everthing in one go +FUNCTION_FIRST_PHASE_POSTPONE_SECOND = 1 # Add to symbol table but postpone body +FUNCTION_SECOND_PHASE = 2 # Only analyze body + + +class SemanticAnalyzer(NodeVisitor): + """Semantically analyze parsed mypy files. + + The analyzer binds names and does various consistency checks for a + parse tree. Note that type checking is performed as a separate + pass. + + This is the second phase of semantic analysis. + """ + + # Library search paths + lib_path = None # type: List[str] + # Module name space + modules = None # type: Dict[str, MypyFile] + # Global name space for current module + globals = None # type: SymbolTable + # Names declared using "global" (separate set for each scope) + global_decls = None # type: List[Set[str]] + # Names declated using "nonlocal" (separate set for each scope) + nonlocal_decls = None # type: List[Set[str]] + # Local names of function scopes; None for non-function scopes. + locals = None # type: List[SymbolTable] + # Nested block depths of scopes + block_depth = None # type: List[int] + # TypeInfo of directly enclosing class (or None) + type = None # type: TypeInfo + # Stack of outer classes (the second tuple item contains tvars). + type_stack = None # type: List[TypeInfo] + # Type variables that are bound by the directly enclosing class + bound_tvars = None # type: List[SymbolTableNode] + # Stack of type variables that were bound by outer classess + tvar_stack = None # type: List[List[SymbolTableNode]] + # Do weak type checking in this file + weak_opts = set() # type: Set[str] + + # Stack of functions being analyzed + function_stack = None # type: List[FuncItem] + # Stack of next available function type variable ids + next_function_tvar_id_stack = None # type: List[int] + + # Status of postponing analysis of nested function bodies. By using this we + # can have mutually recursive nested functions. Values are FUNCTION_x + # constants. Note that separate phasea are not used for methods. + postpone_nested_functions_stack = None # type: List[int] + # Postponed functions collected if + # postpone_nested_functions_stack[-1] == FUNCTION_FIRST_PHASE_POSTPONE_SECOND. + postponed_functions_stack = None # type: List[List[Node]] + + loop_depth = 0 # Depth of breakable loops + cur_mod_id = '' # Current module id (or None) (phase 2) + is_stub_file = False # Are we analyzing a stub file? + imports = None # type: Set[str] # Imported modules (during phase 2 analysis) + errors = None # type: Errors # Keeps track of generated errors + + def __init__(self, + lib_path: List[str], + errors: Errors, + options: Options) -> None: + """Construct semantic analyzer. + + Use lib_path to search for modules, and report analysis errors + using the Errors instance. + """ + self.locals = [None] + self.imports = set() + self.type = None + self.type_stack = [] + self.bound_tvars = None + self.tvar_stack = [] + self.function_stack = [] + self.next_function_tvar_id_stack = [-1] + self.block_depth = [0] + self.loop_depth = 0 + self.lib_path = lib_path + self.errors = errors + self.modules = {} + self.options = options + self.postpone_nested_functions_stack = [FUNCTION_BOTH_PHASES] + self.postponed_functions_stack = [] + self.all_exports = set() # type: Set[str] + + def visit_file(self, file_node: MypyFile, fnam: str) -> None: + self.errors.set_file(fnam) + self.cur_mod_node = file_node + self.cur_mod_id = file_node.fullname() + self.is_stub_file = fnam.lower().endswith('.pyi') + self.globals = file_node.names + self.weak_opts = file_node.weak_opts + + if 'builtins' in self.modules: + self.globals['__builtins__'] = SymbolTableNode( + MODULE_REF, self.modules['builtins'], self.cur_mod_id) + + for name in implicit_module_attrs: + v = self.globals[name].node + if isinstance(v, Var): + v.type = self.anal_type(v.type) + v.is_ready = True + + defs = file_node.defs + for d in defs: + self.accept(d) + + if self.cur_mod_id == 'builtins': + remove_imported_names_from_symtable(self.globals, 'builtins') + + if '__all__' in self.globals: + for name, g in self.globals.items(): + if name not in self.all_exports: + g.module_public = False + + def visit_func_def(self, defn: FuncDef) -> None: + phase_info = self.postpone_nested_functions_stack[-1] + if phase_info != FUNCTION_SECOND_PHASE: + self.function_stack.append(defn) + # First phase of analysis for function. + self.errors.push_function(defn.name()) + self.update_function_type_variables(defn) + self.errors.pop_function() + self.function_stack.pop() + + defn.is_conditional = self.block_depth[-1] > 0 + + # TODO(jukka): Figure out how to share the various cases. It doesn't + # make sense to have (almost) duplicate code (here and elsewhere) for + # 3 cases: module-level, class-level and local names. Maybe implement + # a common stack of namespaces. As the 3 kinds of namespaces have + # different semantics, this wouldn't always work, but it might still + # be a win. + if self.is_class_scope(): + # Method definition + defn.info = self.type + if not defn.is_decorated and not defn.is_overload: + if defn.name() in self.type.names: + # Redefinition. Conditional redefinition is okay. + n = self.type.names[defn.name()].node + if self.is_conditional_func(n, defn): + defn.original_def = cast(FuncDef, n) + else: + self.name_already_defined(defn.name(), defn) + self.type.names[defn.name()] = SymbolTableNode(MDEF, defn) + self.prepare_method_signature(defn) + elif self.is_func_scope(): + # Nested function + if not defn.is_decorated and not defn.is_overload: + if defn.name() in self.locals[-1]: + # Redefinition. Conditional redefinition is okay. + n = self.locals[-1][defn.name()].node + if self.is_conditional_func(n, defn): + defn.original_def = cast(FuncDef, n) + else: + self.name_already_defined(defn.name(), defn) + else: + self.add_local(defn, defn) + else: + # Top-level function + if not defn.is_decorated and not defn.is_overload: + symbol = self.globals.get(defn.name()) + if isinstance(symbol.node, FuncDef) and symbol.node != defn: + # This is redefinition. Conditional redefinition is okay. + original_def = symbol.node + if self.is_conditional_func(original_def, defn): + # Conditional function definition -- multiple defs are ok. + defn.original_def = original_def + else: + # Report error. + self.check_no_global(defn.name(), defn, True) + if phase_info == FUNCTION_FIRST_PHASE_POSTPONE_SECOND: + # Postpone this function (for the second phase). + self.postponed_functions_stack[-1].append(defn) + return + if phase_info != FUNCTION_FIRST_PHASE_POSTPONE_SECOND: + # Second phase of analysis for function. + self.errors.push_function(defn.name()) + self.analyze_function(defn) + self.errors.pop_function() + + def prepare_method_signature(self, func: FuncDef) -> None: + """Check basic signature validity and tweak annotation of self/cls argument.""" + # Only non-static methods are special. + if not func.is_static: + if not func.arguments: + self.fail('Method must have at least one argument', func) + elif func.type: + sig = cast(FunctionLike, func.type) + if func.is_class: + leading_type = self.class_type(self.type) + else: + leading_type = self_type(self.type) + func.type = replace_implicit_first_type(sig, leading_type) + + def is_conditional_func(self, previous: Node, new: FuncDef) -> bool: + """Does 'new' conditionally redefine 'previous'? + + We reject straight redefinitions of functions, as they are usually + a programming error. For example: + + . def f(): ... + . def f(): ... # Error: 'f' redefined + """ + return isinstance(previous, (FuncDef, Var)) and new.is_conditional + + def update_function_type_variables(self, defn: FuncDef) -> None: + """Make any type variables in the signature of defn explicit. + + Update the signature of defn to contain type variable definitions + if defn is generic. + """ + if defn.type: + functype = cast(CallableType, defn.type) + typevars = self.infer_type_variables(functype) + # Do not define a new type variable if already defined in scope. + typevars = [(name, tvar) for name, tvar in typevars + if not self.is_defined_type_var(name, defn)] + if typevars: + next_tvar_id = self.next_function_tvar_id() + defs = [TypeVarDef(tvar[0], next_tvar_id - i, + tvar[1].values, tvar[1].upper_bound, + tvar[1].variance) + for i, tvar in enumerate(typevars)] + functype.variables = defs + + def infer_type_variables(self, + type: CallableType) -> List[Tuple[str, TypeVarExpr]]: + """Return list of unique type variables referred to in a callable.""" + names = [] # type: List[str] + tvars = [] # type: List[TypeVarExpr] + for arg in type.arg_types + [type.ret_type]: + for name, tvar_expr in self.find_type_variables_in_type(arg): + if name not in names: + names.append(name) + tvars.append(tvar_expr) + return list(zip(names, tvars)) + + def find_type_variables_in_type( + self, type: Type) -> List[Tuple[str, TypeVarExpr]]: + """Return a list of all unique type variable references in type. + + This effectively does partial name binding, results of which are mostly thrown away. + """ + result = [] # type: List[Tuple[str, TypeVarExpr]] + if isinstance(type, UnboundType): + name = type.name + node = self.lookup_qualified(name, type) + if node and node.kind == UNBOUND_TVAR: + result.append((name, cast(TypeVarExpr, node.node))) + for arg in type.args: + result.extend(self.find_type_variables_in_type(arg)) + elif isinstance(type, TypeList): + for item in type.items: + result.extend(self.find_type_variables_in_type(item)) + elif isinstance(type, UnionType): + for item in type.items: + result.extend(self.find_type_variables_in_type(item)) + elif isinstance(type, AnyType): + pass + elif isinstance(type, EllipsisType) or isinstance(type, TupleType): + pass + else: + assert False, 'Unsupported type %s' % type + return result + + def is_defined_type_var(self, tvar: str, context: Node) -> bool: + return self.lookup_qualified(tvar, context).kind == BOUND_TVAR + + def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: + t = [] # type: List[CallableType] + for i, item in enumerate(defn.items): + # TODO support decorated overloaded functions properly + item.is_overload = True + item.func.is_overload = True + item.accept(self) + t.append(cast(CallableType, function_type(item.func, + self.builtin_type('builtins.function')))) + if item.func.is_property and i == 0: + # This defines a property, probably with a setter and/or deleter. + self.analyze_property_with_multi_part_definition(defn) + break + if not [dec for dec in item.decorators + if refers_to_fullname(dec, 'typing.overload')]: + self.fail("'overload' decorator expected", item) + + defn.type = Overloaded(t) + defn.type.line = defn.line + + if self.is_class_scope(): + self.type.names[defn.name()] = SymbolTableNode(MDEF, defn, + typ=defn.type) + defn.info = self.type + elif self.is_func_scope(): + self.add_local(defn, defn) + + def analyze_property_with_multi_part_definition(self, defn: OverloadedFuncDef) -> None: + """Analyze a property defined using multiple methods (e.g., using @x.setter). + + Assume that the first method (@property) has already been analyzed. + """ + defn.is_property = True + items = defn.items + for item in items[1:]: + if len(item.decorators) == 1: + node = item.decorators[0] + if isinstance(node, MemberExpr): + if node.name == 'setter': + # The first item represents the entire property. + defn.items[0].var.is_settable_property = True + # Get abstractness from the original definition. + item.func.is_abstract = items[0].func.is_abstract + else: + self.fail("Decorated property not supported", item) + item.func.accept(self) + + def next_function_tvar_id(self) -> int: + return self.next_function_tvar_id_stack[-1] + + def analyze_function(self, defn: FuncItem) -> None: + is_method = self.is_class_scope() + + tvarnodes = self.add_func_type_variables_to_symbol_table(defn) + next_function_tvar_id = min([self.next_function_tvar_id()] + + [n.tvar_def.id.raw_id - 1 for n in tvarnodes]) + self.next_function_tvar_id_stack.append(next_function_tvar_id) + + if defn.type: + # Signature must be analyzed in the surrounding scope so that + # class-level imported names and type variables are in scope. + defn.type = self.anal_type(defn.type) + self.check_function_signature(defn) + if isinstance(defn, FuncDef): + defn.type = set_callable_name(defn.type, defn) + for arg in defn.arguments: + if arg.initializer: + arg.initializer.accept(self) + self.function_stack.append(defn) + self.enter() + for arg in defn.arguments: + self.add_local(arg.variable, defn) + for arg in defn.arguments: + if arg.initialization_statement: + lvalue = arg.initialization_statement.lvalues[0] + lvalue.accept(self) + + # The first argument of a non-static, non-class method is like 'self' + # (though the name could be different), having the enclosing class's + # instance type. + if is_method and not defn.is_static and not defn.is_class and defn.arguments: + defn.arguments[0].variable.is_self = True + + # First analyze body of the function but ignore nested functions. + self.postpone_nested_functions_stack.append(FUNCTION_FIRST_PHASE_POSTPONE_SECOND) + self.postponed_functions_stack.append([]) + defn.body.accept(self) + + # Analyze nested functions (if any) as a second phase. + self.postpone_nested_functions_stack[-1] = FUNCTION_SECOND_PHASE + for postponed in self.postponed_functions_stack[-1]: + postponed.accept(self) + self.postpone_nested_functions_stack.pop() + self.postponed_functions_stack.pop() + + self.next_function_tvar_id_stack.pop() + disable_typevars(tvarnodes) + + self.leave() + self.function_stack.pop() + + def add_func_type_variables_to_symbol_table( + self, defn: FuncItem) -> List[SymbolTableNode]: + nodes = [] # type: List[SymbolTableNode] + if defn.type: + tt = defn.type + names = self.type_var_names() + items = cast(CallableType, tt).variables + for item in items: + name = item.name + if name in names: + self.name_already_defined(name, defn) + node = self.bind_type_var(name, item, defn) + nodes.append(node) + names.add(name) + return nodes + + def type_var_names(self) -> Set[str]: + if not self.type: + return set() + else: + return set(self.type.type_vars) + + def bind_type_var(self, fullname: str, tvar_def: TypeVarDef, + context: Context) -> SymbolTableNode: + node = self.lookup_qualified(fullname, context) + node.kind = BOUND_TVAR + node.tvar_def = tvar_def + return node + + def check_function_signature(self, fdef: FuncItem) -> None: + sig = cast(CallableType, fdef.type) + if len(sig.arg_types) < len(fdef.arguments): + self.fail('Type signature has too few arguments', fdef) + # Add dummy Any arguments to prevent crashes later. + extra_anys = [AnyType()] * (len(fdef.arguments) - len(sig.arg_types)) + sig.arg_types.extend(extra_anys) + elif len(sig.arg_types) > len(fdef.arguments): + self.fail('Type signature has too many arguments', fdef) + + def visit_class_def(self, defn: ClassDef) -> None: + self.clean_up_bases_and_infer_type_variables(defn) + self.setup_class_def_analysis(defn) + + self.bind_class_type_vars(defn) + + self.analyze_base_classes(defn) + self.analyze_metaclass(defn) + + for decorator in defn.decorators: + self.analyze_class_decorator(defn, decorator) + + self.enter_class(defn) + + self.setup_is_builtinclass(defn) + + # Analyze class body. + defn.defs.accept(self) + + self.calculate_abstract_status(defn.info) + self.setup_type_promotion(defn) + + self.leave_class() + self.unbind_class_type_vars() + + def enter_class(self, defn: ClassDef) -> None: + # Remember previous active class + self.type_stack.append(self.type) + self.locals.append(None) # Add class scope + self.block_depth.append(-1) # The class body increments this to 0 + self.postpone_nested_functions_stack.append(FUNCTION_BOTH_PHASES) + self.type = defn.info + + def leave_class(self) -> None: + """ Restore analyzer state. """ + self.postpone_nested_functions_stack.pop() + self.block_depth.pop() + self.locals.pop() + self.type = self.type_stack.pop() + + def bind_class_type_vars(self, defn: ClassDef) -> None: + """ Unbind type variables of previously active class and bind + the type variables for the active class. + """ + if self.bound_tvars: + disable_typevars(self.bound_tvars) + self.tvar_stack.append(self.bound_tvars) + self.bound_tvars = self.bind_class_type_variables_in_symbol_table(defn.info) + + def unbind_class_type_vars(self) -> None: + """ Unbind the active class' type vars and rebind the + type vars of the previously active class. + """ + disable_typevars(self.bound_tvars) + self.bound_tvars = self.tvar_stack.pop() + if self.bound_tvars: + enable_typevars(self.bound_tvars) + + def analyze_class_decorator(self, defn: ClassDef, decorator: Node) -> None: + decorator.accept(self) + + def setup_is_builtinclass(self, defn: ClassDef): + for decorator in defn.decorators: + if refers_to_fullname(decorator, 'typing.builtinclass'): + defn.is_builtinclass = True + if defn.fullname == 'builtins.object': + # Only 'object' is marked as a built-in class, as otherwise things elsewhere + # would break. We need a better way of dealing with built-in classes. + defn.is_builtinclass = True + + def calculate_abstract_status(self, typ: TypeInfo) -> None: + """Calculate abstract status of a class. + + Set is_abstract of the type to True if the type has an unimplemented + abstract attribute. Also compute a list of abstract attributes. + """ + concrete = set() # type: Set[str] + abstract = [] # type: List[str] + for base in typ.mro: + for name, symnode in base.names.items(): + node = symnode.node + if isinstance(node, OverloadedFuncDef): + # Unwrap an overloaded function definition. We can just + # check arbitrarily the first overload item. If the + # different items have a different abstract status, there + # should be an error reported elsewhere. + func = node.items[0] # type: Node + else: + func = node + if isinstance(func, Decorator): + fdef = func.func + if fdef.is_abstract and name not in concrete: + typ.is_abstract = True + abstract.append(name) + concrete.add(name) + typ.abstract_attributes = sorted(abstract) + + def setup_type_promotion(self, defn: ClassDef) -> None: + """Setup extra, ad-hoc subtyping relationships between classes (promotion). + + This includes things like 'int' being compatible with 'float'. + """ + promote_target = None # type: Type + for decorator in defn.decorators: + if isinstance(decorator, CallExpr): + analyzed = decorator.analyzed + if isinstance(analyzed, PromoteExpr): + # _promote class decorator (undocumented faeture). + promote_target = analyzed.type + if not promote_target: + promotions = (TYPE_PROMOTIONS_PYTHON3 if self.options.python_version[0] >= 3 + else TYPE_PROMOTIONS_PYTHON2) + if defn.fullname in promotions: + promote_target = self.named_type_or_none(promotions[defn.fullname]) + defn.info._promote = promote_target + + def clean_up_bases_and_infer_type_variables(self, defn: ClassDef) -> None: + """Remove extra base classes such as Generic and infer type vars. + + For example, consider this class: + + . class Foo(Bar, Generic[T]): ... + + Now we will remove Generic[T] from bases of Foo and infer that the + type variable 'T' is a type argument of Foo. + + Note that this is performed *before* semantic analysis. + """ + removed = [] # type: List[int] + type_vars = [] # type: List[TypeVarDef] + for i, base_expr in enumerate(defn.base_type_exprs): + try: + base = expr_to_unanalyzed_type(base_expr) + except TypeTranslationError: + # This error will be caught later. + continue + tvars = self.analyze_typevar_declaration(base) + if tvars is not None: + if type_vars: + self.fail('Duplicate Generic in bases', defn) + removed.append(i) + for j, (name, tvar_expr) in enumerate(tvars): + type_vars.append(TypeVarDef(name, j + 1, tvar_expr.values, + tvar_expr.upper_bound, tvar_expr.variance)) + if type_vars: + defn.type_vars = type_vars + if defn.info: + defn.info.type_vars = [tv.name for tv in type_vars] + for i in reversed(removed): + del defn.base_type_exprs[i] + + def analyze_typevar_declaration(self, t: Type) -> Optional[List[Tuple[str, TypeVarExpr]]]: + if not isinstance(t, UnboundType): + return None + unbound = t + sym = self.lookup_qualified(unbound.name, unbound) + if sym is None or sym.node is None: + return None + if sym.node.fullname() == 'typing.Generic': + tvars = [] # type: List[Tuple[str, TypeVarExpr]] + for arg in unbound.args: + tvar = self.analyze_unbound_tvar(arg) + if tvar: + tvars.append(tvar) + else: + self.fail('Free type variable expected in %s[...]' % + sym.node.name(), t) + return tvars + return None + + def analyze_unbound_tvar(self, t: Type) -> Tuple[str, TypeVarExpr]: + if not isinstance(t, UnboundType): + return None + unbound = t + sym = self.lookup_qualified(unbound.name, unbound) + if sym is not None and sym.kind == UNBOUND_TVAR: + return unbound.name, cast(TypeVarExpr, sym.node) + return None + + def setup_class_def_analysis(self, defn: ClassDef) -> None: + """Prepare for the analysis of a class definition.""" + if not defn.info: + defn.info = TypeInfo(SymbolTable(), defn) + defn.info._fullname = defn.info.name() + if self.is_func_scope() or self.type: + kind = MDEF + if self.is_func_scope(): + kind = LDEF + self.add_symbol(defn.name, SymbolTableNode(kind, defn.info), defn) + + def analyze_base_classes(self, defn: ClassDef) -> None: + """Analyze and set up base classes. + + This computes several attributes on the corresponding TypeInfo defn.info + related to the base classes: defn.info.bases, defn.info.mro, and + miscellaneous others (at least tuple_type, fallback_to_any, and is_enum.) + """ + + base_types = [] # type: List[Instance] + for base_expr in defn.base_type_exprs: + try: + base = self.expr_to_analyzed_type(base_expr) + except TypeTranslationError: + self.fail('Invalid base class', base_expr) + defn.info.fallback_to_any = True + continue + + if isinstance(base, TupleType): + if defn.info.tuple_type: + self.fail("Class has two incompatible bases derived from tuple", defn) + if (not self.is_stub_file + and not defn.info.is_named_tuple + and base.fallback.type.fullname() == 'builtins.tuple'): + self.fail("Tuple[...] not supported as a base class outside a stub file", defn) + defn.info.tuple_type = base + base_types.append(base.fallback) + elif isinstance(base, Instance): + base_types.append(base) + elif isinstance(base, AnyType): + defn.info.fallback_to_any = True + else: + self.fail('Invalid base class', base_expr) + defn.info.fallback_to_any = True + + # Add 'object' as implicit base if there is no other base class. + if (not base_types and defn.fullname != 'builtins.object'): + base_types.append(self.object_type()) + + defn.info.bases = base_types + + # Calculate the MRO. It might be incomplete at this point if + # the bases of defn include classes imported from other + # modules in an import loop. We'll recompute it in ThirdPass. + if not self.verify_base_classes(defn): + defn.info.mro = [] + return + calculate_class_mro(defn, self.fail) + # If there are cyclic imports, we may be missing 'object' in + # the MRO. Fix MRO if needed. + if defn.info.mro and defn.info.mro[-1].fullname() != 'builtins.object': + defn.info.mro.append(self.object_type().type) + + def expr_to_analyzed_type(self, expr: Node) -> Type: + if isinstance(expr, CallExpr): + expr.accept(self) + info = self.check_namedtuple(expr) + if info is None: + # Some form of namedtuple is the only valid type that looks like a call + # expression. This isn't a valid type. + raise TypeTranslationError() + fallback = Instance(info, []) + return TupleType(info.tuple_type.items, fallback=fallback) + typ = expr_to_unanalyzed_type(expr) + return self.anal_type(typ) + + def verify_base_classes(self, defn: ClassDef) -> bool: + info = defn.info + for base in info.bases: + baseinfo = base.type + if self.is_base_class(info, baseinfo): + self.fail('Cycle in inheritance hierarchy', defn) + # Clear bases to forcefully get rid of the cycle. + info.bases = [] + if baseinfo.fullname() == 'builtins.bool': + self.fail("'%s' is not a valid base class" % + baseinfo.name(), defn) + return False + dup = find_duplicate(info.direct_base_classes()) + if dup: + self.fail('Duplicate base class "%s"' % dup.name(), defn) + return False + return True + + def is_base_class(self, t: TypeInfo, s: TypeInfo) -> bool: + """Determine if t is a base class of s (but do not use mro).""" + # Search the base class graph for t, starting from s. + worklist = [s] + visited = {s} + while worklist: + nxt = worklist.pop() + if nxt == t: + return True + for base in nxt.bases: + if base.type not in visited: + worklist.append(base.type) + visited.add(base.type) + return False + + def analyze_metaclass(self, defn: ClassDef) -> None: + if defn.metaclass: + sym = self.lookup_qualified(defn.metaclass, defn) + if sym is not None and not isinstance(sym.node, TypeInfo): + self.fail("Invalid metaclass '%s'" % defn.metaclass, defn) + + def object_type(self) -> Instance: + return self.named_type('__builtins__.object') + + def class_type(self, info: TypeInfo) -> Type: + # Construct a function type whose fallback is cls. + from mypy import checkmember # To avoid import cycle. + leading_type = checkmember.type_object_type(info, self.builtin_type) + if isinstance(leading_type, Overloaded): + # Overloaded __init__ is too complex to handle. Plus it's stubs only. + return AnyType() + else: + return leading_type + + def named_type(self, qualified_name: str, args: List[Type] = None) -> Instance: + sym = self.lookup_qualified(qualified_name, None) + return Instance(cast(TypeInfo, sym.node), args or []) + + def named_type_or_none(self, qualified_name: str) -> Instance: + sym = self.lookup_fully_qualified_or_none(qualified_name) + if not sym: + return None + return Instance(cast(TypeInfo, sym.node), []) + + def is_instance_type(self, t: Type) -> bool: + return isinstance(t, Instance) + + def bind_class_type_variables_in_symbol_table( + self, info: TypeInfo) -> List[SymbolTableNode]: + nodes = [] # type: List[SymbolTableNode] + for var, binder in zip(info.type_vars, info.defn.type_vars): + node = self.bind_type_var(var, binder, info) + nodes.append(node) + return nodes + + def visit_import(self, i: Import) -> None: + for id, as_id in i.ids: + if as_id is not None: + self.add_module_symbol(id, as_id, module_public=True, context=i) + else: + # Modules imported in a stub file without using 'as x' won't get exported when + # doing 'from m import *'. + module_public = not self.is_stub_file + base = id.split('.')[0] + self.add_module_symbol(base, base, module_public=module_public, + context=i) + + def add_module_symbol(self, id: str, as_id: str, module_public: bool, + context: Context) -> None: + if id in self.modules: + m = self.modules[id] + self.add_symbol(as_id, SymbolTableNode(MODULE_REF, m, self.cur_mod_id, + module_public=module_public), context) + else: + self.add_unknown_symbol(as_id, context) + + def visit_import_from(self, imp: ImportFrom) -> None: + import_id = self.correct_relative_import(imp) + if import_id in self.modules: + module = self.modules[import_id] + for id, as_id in imp.names: + node = module.names.get(id) + if node and node.kind != UNBOUND_IMPORTED: + node = self.normalize_type_alias(node, imp) + if not node: + return + imported_id = as_id or id + existing_symbol = self.globals.get(imported_id) + if existing_symbol: + # Import can redefine a variable. They get special treatment. + if self.process_import_over_existing_name( + imported_id, existing_symbol, node, imp): + continue + # 'from m import x as x' exports x in a stub file. + module_public = not self.is_stub_file or as_id is not None + symbol = SymbolTableNode(node.kind, node.node, + self.cur_mod_id, + node.type_override, + module_public=module_public) + self.add_symbol(imported_id, symbol, imp) + else: + message = "Module has no attribute '{}'".format(id) + extra = self.undefined_name_extra_info('{}.{}'.format(import_id, id)) + if extra: + message += " {}".format(extra) + self.fail(message, imp) + else: + # Missing module. + for id, as_id in imp.names: + self.add_unknown_symbol(as_id or id, imp) + + def process_import_over_existing_name(self, + imported_id: str, existing_symbol: SymbolTableNode, + module_symbol: SymbolTableNode, + import_node: ImportBase) -> bool: + if (existing_symbol.kind in (LDEF, GDEF, MDEF) and + isinstance(existing_symbol.node, (Var, FuncDef))): + # This is a valid import over an existing definition in the file. Construct a dummy + # assignment that we'll use to type check the import. + lvalue = NameExpr(imported_id) + lvalue.kind = existing_symbol.kind + lvalue.node = existing_symbol.node + rvalue = NameExpr(imported_id) + rvalue.kind = module_symbol.kind + rvalue.node = module_symbol.node + assignment = AssignmentStmt([lvalue], rvalue) + for node in assignment, lvalue, rvalue: + node.set_line(import_node) + import_node.assignments.append(assignment) + return True + return False + + def normalize_type_alias(self, node: SymbolTableNode, + ctx: Context) -> SymbolTableNode: + if node.fullname in type_aliases: + # Node refers to an aliased type such as typing.List; normalize. + node = self.lookup_qualified(type_aliases[node.fullname], ctx) + if node.fullname == 'typing.DefaultDict': + self.add_module_symbol('collections', '__mypy_collections__', False, ctx) + node = self.lookup_qualified('__mypy_collections__.defaultdict', ctx) + return node + + def correct_relative_import(self, node: Union[ImportFrom, ImportAll]) -> str: + if node.relative == 0: + return node.id + + parts = self.cur_mod_id.split(".") + cur_mod_id = self.cur_mod_id + + rel = node.relative + if self.cur_mod_node.is_package_init_file(): + rel -= 1 + if len(parts) < rel: + self.fail("Relative import climbs too many namespaces", node) + if rel != 0: + cur_mod_id = ".".join(parts[:-rel]) + + return cur_mod_id + (("." + node.id) if node.id else "") + + def visit_import_all(self, i: ImportAll) -> None: + i_id = self.correct_relative_import(i) + if i_id in self.modules: + m = self.modules[i_id] + for name, node in m.names.items(): + node = self.normalize_type_alias(node, i) + if not name.startswith('_') and node.module_public: + existing_symbol = self.globals.get(name) + if existing_symbol: + # Import can redefine a variable. They get special treatment. + if self.process_import_over_existing_name( + name, existing_symbol, node, i): + continue + self.add_symbol(name, SymbolTableNode(node.kind, node.node, + self.cur_mod_id), i) + else: + # Don't add any dummy symbols for 'from x import *' if 'x' is unknown. + pass + + def add_unknown_symbol(self, name: str, context: Context) -> None: + var = Var(name) + var._fullname = self.qualified_name(name) + var.is_ready = True + var.type = AnyType() + self.add_symbol(name, SymbolTableNode(GDEF, var, self.cur_mod_id), context) + + # + # Statements + # + + def visit_block(self, b: Block) -> None: + if b.is_unreachable: + return + self.block_depth[-1] += 1 + for s in b.body: + self.accept(s) + self.block_depth[-1] -= 1 + + def visit_block_maybe(self, b: Block) -> None: + if b: + self.visit_block(b) + + def anal_type(self, t: Type, allow_tuple_literal: bool = False) -> Type: + if t: + if allow_tuple_literal: + # Types such as (t1, t2, ...) only allowed in assignment statements. They'll + # generate errors elsewhere, and Tuple[t1, t2, ...] must be used instead. + if isinstance(t, TupleType): + # Unlike TypeAnalyser, also allow implicit tuple types (without Tuple[...]). + star_count = sum(1 for item in t.items if isinstance(item, StarType)) + if star_count > 1: + self.fail('At most one star type allowed in a tuple', t) + return TupleType([AnyType() for _ in t.items], + self.builtin_type('builtins.tuple'), t.line) + items = [self.anal_type(item, True) + for item in t.items] + return TupleType(items, self.builtin_type('builtins.tuple'), t.line) + a = TypeAnalyser(self.lookup_qualified, + self.lookup_fully_qualified, + self.fail) + return t.accept(a) + else: + return None + + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + for lval in s.lvalues: + self.analyze_lvalue(lval, explicit_type=s.type is not None) + s.rvalue.accept(self) + if s.type: + allow_tuple_literal = isinstance(s.lvalues[-1], (TupleExpr, ListExpr)) + s.type = self.anal_type(s.type, allow_tuple_literal) + else: + # For simple assignments, allow binding type aliases. + # Also set the type if the rvalue is a simple literal. + if (s.type is None and len(s.lvalues) == 1 and + isinstance(s.lvalues[0], NameExpr)): + if s.lvalues[0].is_def: + s.type = self.analyze_simple_literal_type(s.rvalue) + res = analyze_type_alias(s.rvalue, + self.lookup_qualified, + self.lookup_fully_qualified, + self.fail) + if res and (not isinstance(res, Instance) or cast(Instance, res).args): + # TODO: What if this gets reassigned? + name = s.lvalues[0] + node = self.lookup(name.name, name) + node.kind = TYPE_ALIAS + node.type_override = res + if isinstance(s.rvalue, IndexExpr): + s.rvalue.analyzed = TypeAliasExpr(res) + if s.type: + # Store type into nodes. + for lvalue in s.lvalues: + self.store_declared_types(lvalue, s.type) + self.check_and_set_up_type_alias(s) + self.process_typevar_declaration(s) + self.process_namedtuple_definition(s) + + if (len(s.lvalues) == 1 and isinstance(s.lvalues[0], NameExpr) and + s.lvalues[0].name == '__all__' and s.lvalues[0].kind == GDEF and + isinstance(s.rvalue, (ListExpr, TupleExpr))): + self.add_exports(*s.rvalue.items) + + def analyze_simple_literal_type(self, rvalue: Node) -> Optional[Type]: + """Return builtins.int if rvalue is an int literal, etc.""" + if self.weak_opts or self.options.semantic_analysis_only or self.function_stack: + # Skip this if any weak options are set. + # Also skip if we're only doing the semantic analysis pass. + # This is mostly to avoid breaking unit tests. + # Also skip inside a function; this is to avoid confusing + # the code that handles dead code due to isinstance() + # inside type variables with value restrictions (like + # AnyStr). + return None + if isinstance(rvalue, IntExpr): + return self.named_type_or_none('builtins.int') + if isinstance(rvalue, FloatExpr): + return self.named_type_or_none('builtins.float') + if isinstance(rvalue, StrExpr): + return self.named_type_or_none('builtins.str') + if isinstance(rvalue, BytesExpr): + return self.named_type_or_none('builtins.bytes') + if isinstance(rvalue, UnicodeExpr): + return self.named_type_or_none('builtins.unicode') + return None + + def check_and_set_up_type_alias(self, s: AssignmentStmt) -> None: + """Check if assignment creates a type alias and set it up as needed.""" + # For now, type aliases only work at the top level of a module. + if (len(s.lvalues) == 1 and not self.is_func_scope() and not self.type + and not s.type): + lvalue = s.lvalues[0] + if isinstance(lvalue, NameExpr): + if not lvalue.is_def: + # Only a definition can create a type alias, not regular assignment. + return + rvalue = s.rvalue + if isinstance(rvalue, RefExpr): + node = rvalue.node + if isinstance(node, TypeInfo): + # TODO: We should record the fact that this is a variable + # that refers to a type, rather than making this + # just an alias for the type. + self.globals[lvalue.name].node = node + + def analyze_lvalue(self, lval: Node, nested: bool = False, + add_global: bool = False, + explicit_type: bool = False) -> None: + """Analyze an lvalue or assignment target. + + Only if add_global is True, add name to globals table. If nested + is true, the lvalue is within a tuple or list lvalue expression. + """ + + if isinstance(lval, NameExpr): + # Top-level definitions within some statements (at least while) are + # not handled in the first pass, so they have to be added now. + nested_global = (not self.is_func_scope() and + self.block_depth[-1] > 0 and + not self.type) + if (add_global or nested_global) and lval.name not in self.globals: + # Define new global name. + v = Var(lval.name) + v._fullname = self.qualified_name(lval.name) + v.is_ready = False # Type not inferred yet + lval.node = v + lval.is_def = True + lval.kind = GDEF + lval.fullname = v._fullname + self.globals[lval.name] = SymbolTableNode(GDEF, v, + self.cur_mod_id) + elif isinstance(lval.node, Var) and lval.is_def: + # Since the is_def flag is set, this must have been analyzed + # already in the first pass and added to the symbol table. + assert lval.node.name() in self.globals + elif (self.is_func_scope() and lval.name not in self.locals[-1] and + lval.name not in self.global_decls[-1] and + lval.name not in self.nonlocal_decls[-1]): + # Define new local name. + v = Var(lval.name) + lval.node = v + lval.is_def = True + lval.kind = LDEF + lval.fullname = lval.name + self.add_local(v, lval) + elif not self.is_func_scope() and (self.type and + lval.name not in self.type.names): + # Define a new attribute within class body. + v = Var(lval.name) + v.info = self.type + v.is_initialized_in_class = True + v.set_line(lval) + lval.node = v + lval.is_def = True + lval.kind = MDEF + lval.fullname = lval.name + self.type.names[lval.name] = SymbolTableNode(MDEF, v) + else: + # Bind to an existing name. + if explicit_type: + self.name_already_defined(lval.name, lval) + lval.accept(self) + self.check_lvalue_validity(lval.node, lval) + elif isinstance(lval, MemberExpr): + if not add_global: + self.analyze_member_lvalue(lval) + if explicit_type and not self.is_self_member_ref(lval): + self.fail('Type cannot be declared in assignment to non-self ' + 'attribute', lval) + elif isinstance(lval, IndexExpr): + if explicit_type: + self.fail('Unexpected type declaration', lval) + if not add_global: + lval.accept(self) + elif (isinstance(lval, TupleExpr) or + isinstance(lval, ListExpr)): + items = cast(Any, lval).items + if len(items) == 0 and isinstance(lval, TupleExpr): + self.fail("Can't assign to ()", lval) + self.analyze_tuple_or_list_lvalue(cast(Union[ListExpr, TupleExpr], lval), + add_global, explicit_type) + elif isinstance(lval, StarExpr): + if nested: + self.analyze_lvalue(lval.expr, nested, add_global, explicit_type) + else: + self.fail('Starred assignment target must be in a list or tuple', lval) + else: + self.fail('Invalid assignment target', lval) + + def analyze_tuple_or_list_lvalue(self, lval: Union[ListExpr, TupleExpr], + add_global: bool = False, + explicit_type: bool = False) -> None: + """Analyze an lvalue or assignment target that is a list or tuple.""" + items = lval.items + star_exprs = [cast(StarExpr, item) + for item in items + if isinstance(item, StarExpr)] + + if len(star_exprs) > 1: + self.fail('Two starred expressions in assignment', lval) + else: + if len(star_exprs) == 1: + star_exprs[0].valid = True + for i in items: + self.analyze_lvalue(i, nested=True, add_global=add_global, + explicit_type = explicit_type) + + def analyze_member_lvalue(self, lval: MemberExpr) -> None: + lval.accept(self) + if (self.is_self_member_ref(lval) and + self.type.get(lval.name) is None): + # Implicit attribute definition in __init__. + lval.is_def = True + v = Var(lval.name) + v.info = self.type + v.is_ready = False + lval.def_var = v + lval.node = v + self.type.names[lval.name] = SymbolTableNode(MDEF, v) + self.check_lvalue_validity(lval.node, lval) + + def is_self_member_ref(self, memberexpr: MemberExpr) -> bool: + """Does memberexpr to refer to an attribute of self?""" + if not isinstance(memberexpr.expr, NameExpr): + return False + node = memberexpr.expr.node + return isinstance(node, Var) and node.is_self + + def check_lvalue_validity(self, node: Node, ctx: Context) -> None: + if isinstance(node, (TypeInfo, TypeVarExpr)): + self.fail('Invalid assignment target', ctx) + + def store_declared_types(self, lvalue: Node, typ: Type) -> None: + if isinstance(typ, StarType) and not isinstance(lvalue, StarExpr): + self.fail('Star type only allowed for starred expressions', lvalue) + if isinstance(lvalue, RefExpr): + lvalue.is_def = False + if isinstance(lvalue.node, Var): + var = lvalue.node + var.type = typ + var.is_ready = True + # If node is not a variable, we'll catch it elsewhere. + elif isinstance(lvalue, TupleExpr): + if isinstance(typ, TupleType): + if len(lvalue.items) != len(typ.items): + self.fail('Incompatible number of tuple items', lvalue) + return + for item, itemtype in zip(lvalue.items, typ.items): + self.store_declared_types(item, itemtype) + else: + self.fail('Tuple type expected for multiple variables', + lvalue) + elif isinstance(lvalue, StarExpr): + if isinstance(typ, StarType): + self.store_declared_types(lvalue.expr, typ.type) + else: + self.fail('Star type expected for starred expression', lvalue) + else: + # This has been flagged elsewhere as an error, so just ignore here. + pass + + def process_typevar_declaration(self, s: AssignmentStmt) -> None: + """Check if s declares a TypeVar; it yes, store it in symbol table.""" + call = self.get_typevar_declaration(s) + if not call: + return + + lvalue = cast(NameExpr, s.lvalues[0]) + name = lvalue.name + if not lvalue.is_def: + if s.type: + self.fail("Cannot declare the type of a type variable", s) + else: + self.fail("Cannot redefine '%s' as a type variable" % name, s) + return + + if not self.check_typevar_name(call, name, s): + return + + # Constraining types + n_values = call.arg_kinds[1:].count(ARG_POS) + values = self.analyze_types(call.args[1:1 + n_values]) + + res = self.process_typevar_parameters(call.args[1 + n_values:], + call.arg_names[1 + n_values:], + call.arg_kinds[1 + n_values:], + bool(values), + s) + if res is None: + return + variance, upper_bound = res + + # Yes, it's a valid type variable definition! Add it to the symbol table. + node = self.lookup(name, s) + node.kind = UNBOUND_TVAR + TypeVar = TypeVarExpr(name, node.fullname, values, upper_bound, variance) + TypeVar.line = call.line + call.analyzed = TypeVar + node.node = TypeVar + + def check_typevar_name(self, call: CallExpr, name: str, context: Context) -> bool: + if len(call.args) < 1: + self.fail("Too few arguments for TypeVar()", context) + return False + if not isinstance(call.args[0], (StrExpr, BytesExpr)) or not call.arg_kinds[0] == ARG_POS: + self.fail("TypeVar() expects a string literal as first argument", context) + return False + if cast(StrExpr, call.args[0]).value != name: + self.fail("Unexpected TypeVar() argument value", context) + return False + return True + + def get_typevar_declaration(self, s: AssignmentStmt) -> Optional[CallExpr]: + """Returns the TypeVar() call expression if `s` is a type var declaration + or None otherwise. + """ + if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): + return None + if not isinstance(s.rvalue, CallExpr): + return None + call = s.rvalue + if not isinstance(call.callee, RefExpr): + return None + callee = call.callee + if callee.fullname != 'typing.TypeVar': + return None + return call + + def process_typevar_parameters(self, args: List[Node], + names: List[Optional[str]], + kinds: List[int], + has_values: bool, + context: Context) -> Optional[Tuple[int, Type]]: + covariant = False + contravariant = False + upper_bound = self.object_type() # type: Type + for param_value, param_name, param_kind in zip(args, names, kinds): + if not param_kind == ARG_NAMED: + self.fail("Unexpected argument to TypeVar()", context) + return None + if param_name == 'covariant': + if isinstance(param_value, NameExpr): + if param_value.name == 'True': + covariant = True + else: + self.fail("TypeVar 'covariant' may only be 'True'", context) + return None + else: + self.fail("TypeVar 'covariant' may only be 'True'", context) + return None + elif param_name == 'contravariant': + if isinstance(param_value, NameExpr): + if param_value.name == 'True': + contravariant = True + else: + self.fail("TypeVar 'contravariant' may only be 'True'", context) + return None + else: + self.fail("TypeVar 'contravariant' may only be 'True'", context) + return None + elif param_name == 'bound': + if has_values: + self.fail("TypeVar cannot have both values and an upper bound", context) + return None + try: + upper_bound = self.expr_to_analyzed_type(param_value) + except TypeTranslationError: + self.fail("TypeVar 'bound' must be a type", param_value) + return None + elif param_name == 'values': + # Probably using obsolete syntax with values=(...). Explain the current syntax. + self.fail("TypeVar 'values' argument not supported", context) + self.fail("Use TypeVar('T', t, ...) instead of TypeVar('T', values=(t, ...))", + context) + return None + else: + self.fail("Unexpected argument to TypeVar(): {}".format(param_name), context) + return None + + if covariant and contravariant: + self.fail("TypeVar cannot be both covariant and contravariant", context) + return None + elif covariant: + variance = COVARIANT + elif contravariant: + variance = CONTRAVARIANT + else: + variance = INVARIANT + return (variance, upper_bound) + + def process_namedtuple_definition(self, s: AssignmentStmt) -> None: + """Check if s defines a namedtuple; if yes, store the definition in symbol table.""" + if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): + return + lvalue = cast(NameExpr, s.lvalues[0]) + name = lvalue.name + named_tuple = self.check_namedtuple(s.rvalue, name) + if named_tuple is None: + return + # Yes, it's a valid namedtuple definition. Add it to the symbol table. + node = self.lookup(name, s) + node.kind = TYPE_ALIAS # TODO locally defined namedtuple + # TODO call.analyzed + node.node = named_tuple + + def check_namedtuple(self, node: Node, var_name: str = None) -> TypeInfo: + """Check if a call defines a namedtuple. + + The optional var_name argument is the name of the variable to + which this is assigned, if any. + + If it does, return the corresponding TypeInfo. Return None otherwise. + + If the definition is invalid but looks like a namedtuple, + report errors but return (some) TypeInfo. + """ + if not isinstance(node, CallExpr): + return None + call = node + if not isinstance(call.callee, RefExpr): + return None + callee = call.callee + fullname = callee.fullname + if fullname not in ('collections.namedtuple', 'typing.NamedTuple'): + return None + items, types, ok = self.parse_namedtuple_args(call, fullname) + if not ok: + # Error. Construct dummy return value. + return self.build_namedtuple_typeinfo('namedtuple', [], []) + else: + # Give it a unique name derived from the line number. + name = cast(StrExpr, call.args[0]).value + if name != var_name: + name += '@' + str(call.line) + info = self.build_namedtuple_typeinfo(name, items, types) + # Store it as a global just in case it would remain anonymous. + self.globals[name] = SymbolTableNode(GDEF, info, self.cur_mod_id) + call.analyzed = NamedTupleExpr(info).set_line(call.line) + return info + + def parse_namedtuple_args(self, call: CallExpr, + fullname: str) -> Tuple[List[str], List[Type], bool]: + # TODO Share code with check_argument_count in checkexpr.py? + args = call.args + if len(args) < 2: + return self.fail_namedtuple_arg("Too few arguments for namedtuple()", call) + if len(args) > 2: + return self.fail_namedtuple_arg("Too many arguments for namedtuple()", call) + if call.arg_kinds != [ARG_POS, ARG_POS]: + return self.fail_namedtuple_arg("Unexpected arguments to namedtuple()", call) + if not isinstance(args[0], (StrExpr, BytesExpr)): + return self.fail_namedtuple_arg( + "namedtuple() expects a string literal as the first argument", call) + types = [] # type: List[Type] + ok = True + if not isinstance(args[1], ListExpr): + if fullname == 'collections.namedtuple' and isinstance(args[1], (StrExpr, BytesExpr)): + str_expr = cast(StrExpr, args[1]) + items = str_expr.value.split() + else: + return self.fail_namedtuple_arg( + "List literal expected as the second argument to namedtuple()", call) + else: + listexpr = args[1] + if fullname == 'collections.namedtuple': + # The fields argument contains just names, with implicit Any types. + if any(not isinstance(item, (StrExpr, BytesExpr)) for item in listexpr.items): + return self.fail_namedtuple_arg("String literal expected as namedtuple() item", + call) + items = [cast(StrExpr, item).value for item in listexpr.items] + else: + # The fields argument contains (name, type) tuples. + items, types, ok = self.parse_namedtuple_fields_with_types(listexpr.items, call) + if not types: + types = [AnyType() for _ in items] + return items, types, ok + + def parse_namedtuple_fields_with_types(self, nodes: List[Node], + context: Context) -> Tuple[List[str], List[Type], bool]: + items = [] # type: List[str] + types = [] # type: List[Type] + for item in nodes: + if isinstance(item, TupleExpr): + if len(item.items) != 2: + return self.fail_namedtuple_arg("Invalid NamedTuple field definition", + item) + name, type_node = item.items + if isinstance(name, (StrExpr, BytesExpr)): + items.append(name.value) + else: + return self.fail_namedtuple_arg("Invalid NamedTuple() field name", item) + try: + type = expr_to_unanalyzed_type(type_node) + except TypeTranslationError: + return self.fail_namedtuple_arg('Invalid field type', type_node) + types.append(self.anal_type(type)) + else: + return self.fail_namedtuple_arg("Tuple expected as NamedTuple() field", item) + return items, types, True + + def fail_namedtuple_arg(self, message: str, + context: Context) -> Tuple[List[str], List[Type], bool]: + self.fail(message, context) + return [], [], False + + def build_namedtuple_typeinfo(self, name: str, items: List[str], + types: List[Type]) -> TypeInfo: + symbols = SymbolTable() + tup = TupleType(types, self.named_type('__builtins__.tuple', [AnyType()])) + class_def = ClassDef(name, Block([])) + class_def.fullname = self.qualified_name(name) + info = NamedTupleTypeInfo(tup, symbols, class_def) + vars = [Var(item, typ) for item, typ in zip(items, types)] + # Add named tuple items as attributes. + # TODO: Make them read-only. + for var in vars: + var.info = info + symbols[var.name] = SymbolTableNode(MDEF, var) + # Add __init__, _replace, _asdict methods and _fields static field + self.add_namedtuple_method('__init__', symbols, info, NoneTyp(), + self.make_factory_args(vars, ARG_POS)) + self.add_namedtuple_method('_replace', symbols, info, Instance(info, []), + self.make_factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) + # TODO: refine to OrderedDict[Union[types]] + self.add_namedtuple_method('_asdict', symbols, info, AnyType()) + return info + + def make_factory_args(self, vars: List[Var], kind: int, + initializer: Expression = None) -> List[Argument]: + return [Argument(var, var.type, initializer, kind) for var in vars] + + def add_namedtuple_method(self, funcname: str, symbols: SymbolTable, info: TypeInfo, + ret: Type, args: List[Argument] = []) -> None: + types = [cast(Type, None)] + [arg.type_annotation for arg in args] + args = [Argument(Var('__self'), NoneTyp(), None, ARG_POS)] + args + items = [arg.variable.name() for arg in args] + arg_kinds = [arg.kind for arg in args] + # TODO: Make sure that the self argument name is not visible? + signature = CallableType(types, arg_kinds, items, ret, + self.named_type('__builtins__.function'), + name=info.name()) + func = FuncDef(funcname, + args, + Block([]), + typ=signature) + func.info = info + symbols[funcname] = SymbolTableNode(MDEF, func) + + def analyze_types(self, items: List[Node]) -> List[Type]: + result = [] # type: List[Type] + for node in items: + try: + result.append(self.anal_type(expr_to_unanalyzed_type(node))) + except TypeTranslationError: + self.fail('Type expected', node) + result.append(AnyType()) + return result + + def visit_decorator(self, dec: Decorator) -> None: + for d in dec.decorators: + d.accept(self) + removed = [] # type: List[int] + no_type_check = False + for i, d in enumerate(dec.decorators): + # A bunch of decorators are special cased here. + if refers_to_fullname(d, 'abc.abstractmethod'): + removed.append(i) + dec.func.is_abstract = True + self.check_decorated_function_is_method('abstractmethod', dec) + elif refers_to_fullname(d, 'asyncio.tasks.coroutine'): + removed.append(i) + elif refers_to_fullname(d, 'builtins.staticmethod'): + removed.append(i) + dec.func.is_static = True + dec.var.is_staticmethod = True + self.check_decorated_function_is_method('staticmethod', dec) + elif refers_to_fullname(d, 'builtins.classmethod'): + removed.append(i) + dec.func.is_class = True + dec.var.is_classmethod = True + self.check_decorated_function_is_method('classmethod', dec) + elif (refers_to_fullname(d, 'builtins.property') or + refers_to_fullname(d, 'abc.abstractproperty')): + removed.append(i) + dec.func.is_property = True + dec.var.is_property = True + if refers_to_fullname(d, 'abc.abstractproperty'): + dec.func.is_abstract = True + self.check_decorated_function_is_method('property', dec) + if len(dec.func.arguments) > 1: + self.fail('Too many arguments', dec.func) + elif refers_to_fullname(d, 'typing.no_type_check'): + dec.var.type = AnyType() + no_type_check = True + for i in reversed(removed): + del dec.decorators[i] + if not dec.is_overload or dec.var.is_property: + if self.is_func_scope(): + self.add_symbol(dec.var.name(), SymbolTableNode(LDEF, dec), + dec) + elif self.type: + dec.var.info = self.type + dec.var.is_initialized_in_class = True + self.add_symbol(dec.var.name(), SymbolTableNode(MDEF, dec), + dec) + if not no_type_check: + dec.func.accept(self) + if dec.decorators and dec.var.is_property: + self.fail('Decorated property not supported', dec) + + def check_decorated_function_is_method(self, decorator: str, + context: Context) -> None: + if not self.type or self.is_func_scope(): + self.fail("'%s' used with a non-method" % decorator, context) + + def visit_expression_stmt(self, s: ExpressionStmt) -> None: + s.expr.accept(self) + + def visit_return_stmt(self, s: ReturnStmt) -> None: + if not self.is_func_scope(): + self.fail("'return' outside function", s) + if s.expr: + s.expr.accept(self) + + def visit_raise_stmt(self, s: RaiseStmt) -> None: + if s.expr: + s.expr.accept(self) + if s.from_expr: + s.from_expr.accept(self) + + def visit_assert_stmt(self, s: AssertStmt) -> None: + if s.expr: + s.expr.accept(self) + + def visit_operator_assignment_stmt(self, + s: OperatorAssignmentStmt) -> None: + s.lvalue.accept(self) + s.rvalue.accept(self) + if (isinstance(s.lvalue, NameExpr) and s.lvalue.name == '__all__' and + s.lvalue.kind == GDEF and isinstance(s.rvalue, (ListExpr, TupleExpr))): + self.add_exports(*s.rvalue.items) + + def visit_while_stmt(self, s: WhileStmt) -> None: + s.expr.accept(self) + self.loop_depth += 1 + s.body.accept(self) + self.loop_depth -= 1 + self.visit_block_maybe(s.else_body) + + def visit_for_stmt(self, s: ForStmt) -> None: + s.expr.accept(self) + + # Bind index variables and check if they define new names. + self.analyze_lvalue(s.index) + + self.loop_depth += 1 + self.visit_block(s.body) + self.loop_depth -= 1 + + self.visit_block_maybe(s.else_body) + + def visit_break_stmt(self, s: BreakStmt) -> None: + if self.loop_depth == 0: + self.fail("'break' outside loop", s, True, blocker=True) + + def visit_continue_stmt(self, s: ContinueStmt) -> None: + if self.loop_depth == 0: + self.fail("'continue' outside loop", s, True, blocker=True) + + def visit_if_stmt(self, s: IfStmt) -> None: + infer_reachability_of_if_statement(s, pyversion=self.options.python_version) + for i in range(len(s.expr)): + s.expr[i].accept(self) + self.visit_block(s.body[i]) + self.visit_block_maybe(s.else_body) + + def visit_try_stmt(self, s: TryStmt) -> None: + self.analyze_try_stmt(s, self) + + def analyze_try_stmt(self, s: TryStmt, visitor: NodeVisitor, + add_global: bool = False) -> None: + s.body.accept(visitor) + for type, var, handler in zip(s.types, s.vars, s.handlers): + if type: + type.accept(visitor) + if var: + self.analyze_lvalue(var, add_global=add_global) + handler.accept(visitor) + if s.else_body: + s.else_body.accept(visitor) + if s.finally_body: + s.finally_body.accept(visitor) + + def visit_with_stmt(self, s: WithStmt) -> None: + for e, n in zip(s.expr, s.target): + e.accept(self) + if n: + self.analyze_lvalue(n) + self.visit_block(s.body) + + def visit_del_stmt(self, s: DelStmt) -> None: + s.expr.accept(self) + if not self.is_valid_del_target(s.expr): + self.fail('Invalid delete target', s) + + def is_valid_del_target(self, s: Node) -> bool: + if isinstance(s, (IndexExpr, NameExpr, MemberExpr)): + return True + elif isinstance(s, TupleExpr): + return all(self.is_valid_del_target(item) for item in s.items) + + def visit_global_decl(self, g: GlobalDecl) -> None: + for name in g.names: + if name in self.nonlocal_decls[-1]: + self.fail("Name '{}' is nonlocal and global".format(name), g) + self.global_decls[-1].add(name) + + def visit_nonlocal_decl(self, d: NonlocalDecl) -> None: + if not self.is_func_scope(): + self.fail("nonlocal declaration not allowed at module level", d) + else: + for name in d.names: + for table in reversed(self.locals[:-1]): + if table is not None and name in table: + break + else: + self.fail("No binding for nonlocal '{}' found".format(name), d) + + if self.locals[-1] is not None and name in self.locals[-1]: + self.fail("Name '{}' is already defined in local " + "scope before nonlocal declaration".format(name), d) + + if name in self.global_decls[-1]: + self.fail("Name '{}' is nonlocal and global".format(name), d) + self.nonlocal_decls[-1].add(name) + + def visit_print_stmt(self, s: PrintStmt) -> None: + for arg in s.args: + arg.accept(self) + if s.target: + s.target.accept(self) + + def visit_exec_stmt(self, s: ExecStmt) -> None: + s.expr.accept(self) + if s.variables1: + s.variables1.accept(self) + if s.variables2: + s.variables2.accept(self) + + # + # Expressions + # + + def visit_name_expr(self, expr: NameExpr) -> None: + n = self.lookup(expr.name, expr) + if n: + if n.kind == BOUND_TVAR: + self.fail("'{}' is a type variable and only valid in type " + "context".format(expr.name), expr) + else: + expr.kind = n.kind + expr.node = n.node + expr.fullname = n.fullname + + def visit_super_expr(self, expr: SuperExpr) -> None: + if not self.type: + self.fail('"super" used outside class', expr) + return + expr.info = self.type + + def visit_tuple_expr(self, expr: TupleExpr) -> None: + for item in expr.items: + item.accept(self) + + def visit_list_expr(self, expr: ListExpr) -> None: + for item in expr.items: + item.accept(self) + + def visit_set_expr(self, expr: SetExpr) -> None: + for item in expr.items: + item.accept(self) + + def visit_dict_expr(self, expr: DictExpr) -> None: + for key, value in expr.items: + key.accept(self) + value.accept(self) + + def visit_star_expr(self, expr: StarExpr) -> None: + if not expr.valid: + self.fail('Can use starred expression only as assignment target', expr) + else: + expr.expr.accept(self) + + def visit_yield_from_expr(self, e: YieldFromExpr) -> None: + if not self.is_func_scope(): # not sure + self.fail("'yield from' outside function", e, True, blocker=True) + else: + self.function_stack[-1].is_generator = True + if e.expr: + e.expr.accept(self) + + def visit_call_expr(self, expr: CallExpr) -> None: + """Analyze a call expression. + + Some call expressions are recognized as special forms, including + cast(...) and Any(...). + """ + expr.callee.accept(self) + if refers_to_fullname(expr.callee, 'typing.cast'): + # Special form cast(...). + if not self.check_fixed_args(expr, 2, 'cast'): + return + # Translate first argument to an unanalyzed type. + try: + target = expr_to_unanalyzed_type(expr.args[0]) + except TypeTranslationError: + self.fail('Cast target is not a type', expr) + return + # Piggyback CastExpr object to the CallExpr object; it takes + # precedence over the CallExpr semantics. + expr.analyzed = CastExpr(expr.args[1], target) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, 'builtins.reveal_type'): + if not self.check_fixed_args(expr, 1, 'reveal_type'): + return + expr.analyzed = RevealTypeExpr(expr.args[0]) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, 'typing.Any'): + # Special form Any(...). + if not self.check_fixed_args(expr, 1, 'Any'): + return + expr.analyzed = CastExpr(expr.args[0], AnyType()) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, 'typing._promote'): + # Special form _promote(...). + if not self.check_fixed_args(expr, 1, '_promote'): + return + # Translate first argument to an unanalyzed type. + try: + target = expr_to_unanalyzed_type(expr.args[0]) + except TypeTranslationError: + self.fail('Argument 1 to _promote is not a type', expr) + return + expr.analyzed = PromoteExpr(target) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, 'builtins.dict'): + expr.analyzed = self.translate_dict_call(expr) + else: + # Normal call expression. + for a in expr.args: + a.accept(self) + + if (isinstance(expr.callee, MemberExpr) and + isinstance(expr.callee.expr, NameExpr) and + expr.callee.expr.name == '__all__' and + expr.callee.expr.kind == GDEF and + expr.callee.name in ('append', 'extend')): + if expr.callee.name == 'append' and expr.args: + self.add_exports(expr.args[0]) + elif (expr.callee.name == 'extend' and expr.args and + isinstance(expr.args[0], (ListExpr, TupleExpr))): + self.add_exports(*expr.args[0].items) + + def translate_dict_call(self, call: CallExpr) -> Optional[DictExpr]: + """Translate 'dict(x=y, ...)' to {'x': y, ...}. + + For other variants of dict(...), return None. + """ + if not call.args: + return None + if not all(kind == ARG_NAMED for kind in call.arg_kinds): + # Must still accept those args. + for a in call.args: + a.accept(self) + return None + expr = DictExpr([(StrExpr(key), value) + for key, value in zip(call.arg_names, call.args)]) + expr.set_line(call) + expr.accept(self) + return expr + + def check_fixed_args(self, expr: CallExpr, numargs: int, + name: str) -> bool: + """Verify that expr has specified number of positional args. + + Return True if the arguments are valid. + """ + s = 's' + if numargs == 1: + s = '' + if len(expr.args) != numargs: + self.fail("'%s' expects %d argument%s" % (name, numargs, s), + expr) + return False + if expr.arg_kinds != [ARG_POS] * numargs: + self.fail("'%s' must be called with %s positional argument%s" % + (name, numargs, s), expr) + return False + return True + + def visit_member_expr(self, expr: MemberExpr) -> None: + base = expr.expr + base.accept(self) + # Bind references to module attributes. + if isinstance(base, RefExpr) and base.kind == MODULE_REF: + file = cast(MypyFile, base.node) + n = file.names.get(expr.name, None) if file is not None else None + if n: + n = self.normalize_type_alias(n, expr) + if not n: + return + expr.kind = n.kind + expr.fullname = n.fullname + expr.node = n.node + else: + # We only catch some errors here; the rest will be + # catched during type checking. + # + # This way we can report a larger number of errors in + # one type checker run. If we reported errors here, + # the build would terminate after semantic analysis + # and we wouldn't be able to report any type errors. + full_name = '%s.%s' % (file.fullname() if file is not None else None, expr.name) + if full_name in obsolete_name_mapping: + self.fail("Module has no attribute %r (it's now called %r)" % ( + expr.name, obsolete_name_mapping[full_name]), expr) + + def visit_op_expr(self, expr: OpExpr) -> None: + expr.left.accept(self) + expr.right.accept(self) + + def visit_comparison_expr(self, expr: ComparisonExpr) -> None: + for operand in expr.operands: + operand.accept(self) + + def visit_unary_expr(self, expr: UnaryExpr) -> None: + expr.expr.accept(self) + + def visit_index_expr(self, expr: IndexExpr) -> None: + expr.base.accept(self) + if refers_to_class_or_function(expr.base): + # Special form -- type application. + # Translate index to an unanalyzed type. + types = [] # type: List[Type] + if isinstance(expr.index, TupleExpr): + items = expr.index.items + else: + items = [expr.index] + for item in items: + try: + typearg = expr_to_unanalyzed_type(item) + except TypeTranslationError: + self.fail('Type expected within [...]', expr) + return + typearg = self.anal_type(typearg) + types.append(typearg) + expr.analyzed = TypeApplication(expr.base, types) + expr.analyzed.line = expr.line + else: + expr.index.accept(self) + + def visit_slice_expr(self, expr: SliceExpr) -> None: + if expr.begin_index: + expr.begin_index.accept(self) + if expr.end_index: + expr.end_index.accept(self) + if expr.stride: + expr.stride.accept(self) + + def visit_cast_expr(self, expr: CastExpr) -> None: + expr.expr.accept(self) + expr.type = self.anal_type(expr.type) + + def visit_reveal_type_expr(self, expr: RevealTypeExpr) -> None: + expr.expr.accept(self) + + def visit_type_application(self, expr: TypeApplication) -> None: + expr.expr.accept(self) + for i in range(len(expr.types)): + expr.types[i] = self.anal_type(expr.types[i]) + + def visit_list_comprehension(self, expr: ListComprehension) -> None: + expr.generator.accept(self) + + def visit_set_comprehension(self, expr: SetComprehension) -> None: + expr.generator.accept(self) + + def visit_dictionary_comprehension(self, expr: DictionaryComprehension) -> None: + self.enter() + self.analyze_comp_for(expr) + expr.key.accept(self) + expr.value.accept(self) + self.leave() + self.analyze_comp_for_2(expr) + + def visit_generator_expr(self, expr: GeneratorExpr) -> None: + self.enter() + self.analyze_comp_for(expr) + expr.left_expr.accept(self) + self.leave() + self.analyze_comp_for_2(expr) + + def analyze_comp_for(self, expr: Union[GeneratorExpr, + DictionaryComprehension]) -> None: + """Analyses the 'comp_for' part of comprehensions (part 1). + + That is the part after 'for' in (x for x in l if p). This analyzes + variables and conditions which are analyzed in a local scope. + """ + for i, (index, sequence, conditions) in enumerate(zip(expr.indices, + expr.sequences, + expr.condlists)): + if i > 0: + sequence.accept(self) + # Bind index variables. + self.analyze_lvalue(index) + for cond in conditions: + cond.accept(self) + + def analyze_comp_for_2(self, expr: Union[GeneratorExpr, + DictionaryComprehension]) -> None: + """Analyses the 'comp_for' part of comprehensions (part 2). + + That is the part after 'for' in (x for x in l if p). This analyzes + the 'l' part which is analyzed in the surrounding scope. + """ + expr.sequences[0].accept(self) + + def visit_func_expr(self, expr: FuncExpr) -> None: + self.analyze_function(expr) + + def visit_conditional_expr(self, expr: ConditionalExpr) -> None: + expr.if_expr.accept(self) + expr.cond.accept(self) + expr.else_expr.accept(self) + + def visit_backquote_expr(self, expr: BackquoteExpr) -> None: + expr.expr.accept(self) + + def visit__promote_expr(self, expr: PromoteExpr) -> None: + expr.type = self.anal_type(expr.type) + + def visit_yield_expr(self, expr: YieldExpr) -> None: + if not self.is_func_scope(): + self.fail("'yield' outside function", expr, True, blocker=True) + else: + self.function_stack[-1].is_generator = True + if expr.expr: + expr.expr.accept(self) + + # + # Helpers + # + + def lookup(self, name: str, ctx: Context) -> SymbolTableNode: + """Look up an unqualified name in all active namespaces.""" + # 1a. Name declared using 'global x' takes precedence + if name in self.global_decls[-1]: + if name in self.globals: + return self.globals[name] + else: + self.name_not_defined(name, ctx) + return None + # 1b. Name declared using 'nonlocal x' takes precedence + if name in self.nonlocal_decls[-1]: + for table in reversed(self.locals[:-1]): + if table is not None and name in table: + return table[name] + else: + self.name_not_defined(name, ctx) + return None + # 2. Class attributes (if within class definition) + if self.is_class_scope() and name in self.type.names: + return self.type[name] + # 3. Local (function) scopes + for table in reversed(self.locals): + if table is not None and name in table: + return table[name] + # 4. Current file global scope + if name in self.globals: + return self.globals[name] + # 5. Builtins + b = self.globals.get('__builtins__', None) + if b: + table = cast(MypyFile, b.node).names + if name in table: + if name[0] == "_" and name[1] != "_": + self.name_not_defined(name, ctx) + return None + node = table[name] + return node + # Give up. + self.name_not_defined(name, ctx) + self.check_for_obsolete_short_name(name, ctx) + return None + + def check_for_obsolete_short_name(self, name: str, ctx: Context) -> None: + matches = [obsolete_name + for obsolete_name in obsolete_name_mapping + if obsolete_name.rsplit('.', 1)[-1] == name] + if len(matches) == 1: + self.note("(Did you mean '{}'?)".format(obsolete_name_mapping[matches[0]]), ctx) + + def lookup_qualified(self, name: str, ctx: Context) -> SymbolTableNode: + if '.' not in name: + return self.lookup(name, ctx) + else: + parts = name.split('.') + n = self.lookup(parts[0], ctx) # type: SymbolTableNode + if n: + for i in range(1, len(parts)): + if isinstance(n.node, TypeInfo): + if n.node.mro is None: + # We haven't yet analyzed the class `n.node`. Fall back to direct + # lookup in the names declared directly under it, without its base + # classes. This can happen when we have a forward reference to a + # nested class, and the reference is bound before the outer class + # has been fully semantically analyzed. + # + # A better approach would be to introduce a new analysis pass or + # to move things around between passes, but this unblocks a common + # use case even though this is a little limited in case there is + # inheritance involved. + result = n.node.names.get(parts[i]) + else: + result = n.node.get(parts[i]) + n = result + elif isinstance(n.node, MypyFile): + n = n.node.names.get(parts[i], None) + # TODO: What if node is Var or FuncDef? + if not n: + self.name_not_defined(name, ctx) + break + if n: + n = self.normalize_type_alias(n, ctx) + return n + + def builtin_type(self, fully_qualified_name: str) -> Instance: + node = self.lookup_fully_qualified(fully_qualified_name) + info = cast(TypeInfo, node.node) + return Instance(info, []) + + def lookup_fully_qualified(self, name: str) -> SymbolTableNode: + """Lookup a fully qualified name. + + Assume that the name is defined. This happens in the global namespace -- the local + module namespace is ignored. + """ + assert '.' in name + parts = name.split('.') + n = self.modules[parts[0]] + for i in range(1, len(parts) - 1): + n = cast(MypyFile, n.names[parts[i]].node) + return n.names[parts[-1]] + + def lookup_fully_qualified_or_none(self, name: str) -> SymbolTableNode: + """Lookup a fully qualified name. + + Assume that the name is defined. This happens in the global namespace -- the local + module namespace is ignored. + """ + assert '.' in name + parts = name.split('.') + n = self.modules[parts[0]] + for i in range(1, len(parts) - 1): + next_sym = n.names.get(parts[i]) + if not next_sym: + return None + n = cast(MypyFile, next_sym.node) + return n.names.get(parts[-1]) + + def qualified_name(self, n: str) -> str: + return self.cur_mod_id + '.' + n + + def enter(self) -> None: + self.locals.append(SymbolTable()) + self.global_decls.append(set()) + self.nonlocal_decls.append(set()) + # -1 since entering block will increment this to 0. + self.block_depth.append(-1) + + def leave(self) -> None: + self.locals.pop() + self.global_decls.pop() + self.nonlocal_decls.pop() + self.block_depth.pop() + + def is_func_scope(self) -> bool: + return self.locals[-1] is not None + + def is_class_scope(self) -> bool: + return self.type is not None and not self.is_func_scope() + + def add_symbol(self, name: str, node: SymbolTableNode, + context: Context) -> None: + if self.is_func_scope(): + if name in self.locals[-1]: + # Flag redefinition unless this is a reimport of a module. + if not (node.kind == MODULE_REF and + self.locals[-1][name].node == node.node): + self.name_already_defined(name, context) + self.locals[-1][name] = node + elif self.type: + self.type.names[name] = node + else: + existing = self.globals.get(name) + if existing and (not isinstance(node.node, MypyFile) or + existing.node != node.node) and existing.kind != UNBOUND_IMPORTED: + # Modules can be imported multiple times to support import + # of multiple submodules of a package (e.g. a.x and a.y). + ok = False + # Only report an error if the symbol collision provides a different type. + if existing.type and node.type and is_same_type(existing.type, node.type): + ok = True + if not ok: + self.name_already_defined(name, context) + self.globals[name] = node + + def add_var(self, v: Var, ctx: Context) -> None: + if self.is_func_scope(): + self.add_local(v, ctx) + else: + self.globals[v.name()] = SymbolTableNode(GDEF, v, self.cur_mod_id) + v._fullname = self.qualified_name(v.name()) + + def add_local(self, node: Union[Var, FuncBase], ctx: Context) -> None: + name = node.name() + if name in self.locals[-1]: + self.name_already_defined(name, ctx) + node._fullname = name + self.locals[-1][name] = SymbolTableNode(LDEF, node) + + def add_exports(self, *exps: Node) -> None: + for exp in exps: + if isinstance(exp, StrExpr): + self.all_exports.add(exp.value) + + def check_no_global(self, n: str, ctx: Context, + is_overloaded_func: bool = False) -> None: + if n in self.globals: + prev_is_overloaded = isinstance(self.globals[n], OverloadedFuncDef) + if is_overloaded_func and prev_is_overloaded: + self.fail("Nonconsecutive overload {} found".format(n), ctx) + elif prev_is_overloaded: + self.fail("Definition of '{}' missing 'overload'".format(n), ctx) + else: + self.name_already_defined(n, ctx) + + def name_not_defined(self, name: str, ctx: Context) -> None: + message = "Name '{}' is not defined".format(name) + extra = self.undefined_name_extra_info(name) + if extra: + message += ' {}'.format(extra) + self.fail(message, ctx) + + def name_already_defined(self, name: str, ctx: Context) -> None: + self.fail("Name '{}' already defined".format(name), ctx) + + def fail(self, msg: str, ctx: Context, serious: bool = False, *, + blocker: bool = False) -> None: + if (not serious and + not self.options.check_untyped_defs and + self.function_stack and + self.function_stack[-1].is_dynamic()): + return + self.errors.report(ctx.get_line(), msg, blocker=blocker) + + def note(self, msg: str, ctx: Context) -> None: + if (not self.options.check_untyped_defs and + self.function_stack and + self.function_stack[-1].is_dynamic()): + return + self.errors.report(ctx.get_line(), msg, severity='note') + + def undefined_name_extra_info(self, fullname: str) -> Optional[str]: + if fullname in obsolete_name_mapping: + return "(it's now called '{}')".format(obsolete_name_mapping[fullname]) + else: + return None + + def accept(self, node: Node) -> None: + try: + node.accept(self) + except Exception as err: + report_internal_error(err, self.errors.file, node.line) + + +class FirstPass(NodeVisitor): + """First phase of semantic analysis. + + See docstring of 'analyze()' below for a description of what this does. + """ + + def __init__(self, sem: SemanticAnalyzer) -> None: + self.sem = sem + self.pyversion = sem.options.python_version + + def analyze(self, file: MypyFile, fnam: str, mod_id: str) -> None: + """Perform the first analysis pass. + + Populate module global table. Resolve the full names of + definitions not nested within functions and construct type + info structures, but do not resolve inter-definition + references such as base classes. + + Also add implicit definitions such as __name__. + + In this phase we don't resolve imports. For 'from ... import', + we generate dummy symbol table nodes for the imported names, + and these will get resolved in later phases of semantic + analysis. + """ + sem = self.sem + sem.cur_mod_id = mod_id + sem.errors.set_file(fnam) + sem.globals = SymbolTable() + sem.global_decls = [set()] + sem.nonlocal_decls = [set()] + sem.block_depth = [0] + + defs = file.defs + + # Add implicit definitions of module '__name__' etc. + for name, t in implicit_module_attrs.items(): + v = Var(name, UnboundType(t)) + v._fullname = self.sem.qualified_name(name) + self.sem.globals[name] = SymbolTableNode(GDEF, v, self.sem.cur_mod_id) + + for d in defs: + d.accept(self) + + # Add implicit definition of literals/keywords to builtins, as we + # cannot define a variable with them explicitly. + if mod_id == 'builtins': + literal_types = [ + ('None', NoneTyp()), + # reveal_type is a mypy-only function that gives an error with the type of its arg + ('reveal_type', AnyType()), + ] # type: List[Tuple[str, Type]] + + # TODO(ddfisher): This guard is only needed because mypy defines + # fake builtins for its tests which often don't define bool. If + # mypy is fast enough that we no longer need those, this + # conditional check should be removed. + if 'bool' in self.sem.globals: + bool_type = self.sem.named_type('bool') + literal_types.extend([ + ('True', bool_type), + ('False', bool_type), + ('__debug__', bool_type), + ]) + + for name, typ in literal_types: + v = Var(name, typ) + v._fullname = self.sem.qualified_name(name) + self.sem.globals[name] = SymbolTableNode(GDEF, v, self.sem.cur_mod_id) + + def visit_block(self, b: Block) -> None: + if b.is_unreachable: + return + self.sem.block_depth[-1] += 1 + for node in b.body: + node.accept(self) + self.sem.block_depth[-1] -= 1 + + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + for lval in s.lvalues: + self.analyze_lvalue(lval, explicit_type=s.type is not None) + + def visit_func_def(self, func: FuncDef) -> None: + sem = self.sem + func.is_conditional = sem.block_depth[-1] > 0 + func._fullname = sem.qualified_name(func.name()) + if func.name() in sem.globals: + # Already defined in this module. + original_sym = sem.globals[func.name()] + if original_sym.kind == UNBOUND_IMPORTED: + # Ah this is an imported name. We can't resolve them now, so we'll postpone + # this until the main phase of semantic analysis. + return + original_def = original_sym.node + if sem.is_conditional_func(original_def, func): + # Conditional function definition -- multiple defs are ok. + func.original_def = cast(FuncDef, original_def) + else: + # Report error. + sem.check_no_global(func.name(), func) + else: + sem.globals[func.name()] = SymbolTableNode(GDEF, func, sem.cur_mod_id) + + def visit_overloaded_func_def(self, func: OverloadedFuncDef) -> None: + self.sem.check_no_global(func.name(), func, True) + func._fullname = self.sem.qualified_name(func.name()) + self.sem.globals[func.name()] = SymbolTableNode(GDEF, func, + self.sem.cur_mod_id) + + def visit_class_def(self, cdef: ClassDef) -> None: + self.sem.check_no_global(cdef.name, cdef) + cdef.fullname = self.sem.qualified_name(cdef.name) + info = TypeInfo(SymbolTable(), cdef) + info.set_line(cdef.line) + cdef.info = info + self.sem.globals[cdef.name] = SymbolTableNode(GDEF, info, + self.sem.cur_mod_id) + self.process_nested_classes(cdef) + + def process_nested_classes(self, outer_def: ClassDef) -> None: + for node in outer_def.defs.body: + if isinstance(node, ClassDef): + node.info = TypeInfo(SymbolTable(), node) + if outer_def.fullname: + node.info._fullname = outer_def.fullname + '.' + node.info.name() + else: + node.info._fullname = node.info.name() + symbol = SymbolTableNode(MDEF, node.info) + outer_def.info.names[node.name] = symbol + self.process_nested_classes(node) + + def visit_import_from(self, node: ImportFrom) -> None: + # We can't bind module names during the first pass, as the target module might be + # unprocessed. However, we add dummy unbound imported names to the symbol table so + # that we at least know that the name refers to a module. + node.is_top_level = True + for name, as_name in node.names: + imported_name = as_name or name + if imported_name not in self.sem.globals: + self.sem.add_symbol(imported_name, SymbolTableNode(UNBOUND_IMPORTED, None), node) + + def visit_import(self, node: Import) -> None: + node.is_top_level = True + # This is similar to visit_import_from -- see the comment there. + for id, as_id in node.ids: + imported_id = as_id or id + if imported_id not in self.sem.globals: + self.sem.add_symbol(imported_id, SymbolTableNode(UNBOUND_IMPORTED, None), node) + else: + # If the previous symbol is a variable, this should take precedence. + self.sem.globals[imported_id] = SymbolTableNode(UNBOUND_IMPORTED, None) + + def visit_import_all(self, node: ImportAll) -> None: + node.is_top_level = True + + def visit_while_stmt(self, s: WhileStmt) -> None: + s.body.accept(self) + if s.else_body: + s.else_body.accept(self) + + def visit_for_stmt(self, s: ForStmt) -> None: + self.analyze_lvalue(s.index) + s.body.accept(self) + if s.else_body: + s.else_body.accept(self) + + def visit_with_stmt(self, s: WithStmt) -> None: + for n in s.target: + if n: + self.analyze_lvalue(n) + s.body.accept(self) + + def visit_decorator(self, d: Decorator) -> None: + d.var._fullname = self.sem.qualified_name(d.var.name()) + self.sem.add_symbol(d.var.name(), SymbolTableNode(GDEF, d.var), d) + + def visit_if_stmt(self, s: IfStmt) -> None: + infer_reachability_of_if_statement(s, pyversion=self.pyversion) + for node in s.body: + node.accept(self) + if s.else_body: + s.else_body.accept(self) + + def visit_try_stmt(self, s: TryStmt) -> None: + self.sem.analyze_try_stmt(s, self, add_global=True) + + def analyze_lvalue(self, lvalue: Node, explicit_type: bool = False) -> None: + self.sem.analyze_lvalue(lvalue, add_global=True, explicit_type=explicit_type) + + +class ThirdPass(TraverserVisitor): + """The third and final pass of semantic analysis. + + Check type argument counts and values of generic types, and perform some + straightforward type inference. + """ + + def __init__(self, modules: Dict[str, MypyFile], errors: Errors) -> None: + self.modules = modules + self.errors = errors + + def visit_file(self, file_node: MypyFile, fnam: str) -> None: + self.errors.set_file(fnam) + self.accept(file_node) + + def accept(self, node: Node) -> None: + try: + node.accept(self) + except Exception as err: + report_internal_error(err, self.errors.file, node.line) + + def visit_block(self, b: Block) -> None: + if b.is_unreachable: + return + super().visit_block(b) + + def visit_func_def(self, fdef: FuncDef) -> None: + self.errors.push_function(fdef.name()) + self.analyze(fdef.type) + super().visit_func_def(fdef) + self.errors.pop_function() + + def visit_class_def(self, tdef: ClassDef) -> None: + for type in tdef.info.bases: + self.analyze(type) + # Recompute MRO now that we have analyzed all modules, to pick + # up superclasses of bases imported from other modules in an + # import loop. (Only do so if we succeeded the first time.) + if tdef.info.mro: + tdef.info.mro = [] # Force recomputation + calculate_class_mro(tdef, self.fail) + super().visit_class_def(tdef) + + def visit_decorator(self, dec: Decorator) -> None: + """Try to infer the type of the decorated function. + + This lets us resolve references to decorated functions during + type checking when there are cyclic imports, as otherwise the + type might not be available when we need it. + + This basically uses a simple special-purpose type inference + engine just for decorators. + """ + super().visit_decorator(dec) + if dec.var.is_property: + # Decorators are expected to have a callable type (it's a little odd). + if dec.func.type is None: + dec.var.type = CallableType( + [AnyType()], + [ARG_POS], + [None], + AnyType(), + self.builtin_type('function'), + name=dec.var.name()) + elif isinstance(dec.func.type, CallableType): + dec.var.type = dec.func.type + return + decorator_preserves_type = True + for expr in dec.decorators: + preserve_type = False + if isinstance(expr, RefExpr) and isinstance(expr.node, FuncDef): + if is_identity_signature(expr.node.type): + preserve_type = True + if not preserve_type: + decorator_preserves_type = False + break + if decorator_preserves_type: + # No non-identity decorators left. We can trivially infer the type + # of the function here. + dec.var.type = function_type(dec.func, self.builtin_type('function')) + if dec.decorators: + if returns_any_if_called(dec.decorators[0]): + # The outermost decorator will return Any so we know the type of the + # decorated function. + dec.var.type = AnyType() + sig = find_fixed_callable_return(dec.decorators[0]) + if sig: + # The outermost decorator always returns the same kind of function, + # so we know that this is the type of the decoratored function. + orig_sig = function_type(dec.func, self.builtin_type('function')) + sig.name = orig_sig.items()[0].name + dec.var.type = sig + + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + self.analyze(s.type) + super().visit_assignment_stmt(s) + + def visit_cast_expr(self, e: CastExpr) -> None: + self.analyze(e.type) + super().visit_cast_expr(e) + + def visit_reveal_type_expr(self, e: RevealTypeExpr) -> None: + super().visit_reveal_type_expr(e) + + def visit_type_application(self, e: TypeApplication) -> None: + for type in e.types: + self.analyze(type) + super().visit_type_application(e) + + # Helpers + + def analyze(self, type: Type) -> None: + if type: + analyzer = TypeAnalyserPass3(self.fail) + type.accept(analyzer) + + def fail(self, msg: str, ctx: Context) -> None: + self.errors.report(ctx.get_line(), msg) + + def builtin_type(self, name: str, args: List[Type] = None) -> Instance: + names = self.modules['builtins'] + sym = names.names[name] + assert isinstance(sym.node, TypeInfo) + return Instance(sym.node, args or []) + + +def self_type(typ: TypeInfo) -> Union[Instance, TupleType]: + """For a non-generic type, return instance type representing the type. + For a generic G type with parameters T1, .., Tn, return G[T1, ..., Tn]. + """ + tv = [] # type: List[Type] + for i in range(len(typ.type_vars)): + tv.append(TypeVarType(typ.defn.type_vars[i])) + return Instance(typ, tv) + + +def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: + if isinstance(sig, CallableType): + return replace_leading_arg_type(sig, new) + else: + sig = cast(Overloaded, sig) + return Overloaded([cast(CallableType, replace_implicit_first_type(i, new)) + for i in sig.items()]) + + +def set_callable_name(sig: Type, fdef: FuncDef) -> Type: + if isinstance(sig, FunctionLike): + if fdef.info: + return sig.with_name( + '"{}" of "{}"'.format(fdef.name(), fdef.info.name())) + else: + return sig.with_name('"{}"'.format(fdef.name())) + else: + return sig + + +def refers_to_fullname(node: Node, fullname: str) -> bool: + """Is node a name or member expression with the given full name?""" + return isinstance(node, RefExpr) and node.fullname == fullname + + +def refers_to_class_or_function(node: Node) -> bool: + """Does semantically analyzed node refer to a class?""" + return (isinstance(node, RefExpr) and + isinstance(node.node, (TypeInfo, FuncDef, OverloadedFuncDef))) + + +def calculate_class_mro(defn: ClassDef, fail: Callable[[str, Context], None]) -> None: + try: + defn.info.calculate_mro() + except MroError: + fail("Cannot determine consistent method resolution order " + '(MRO) for "%s"' % defn.name, defn) + defn.info.mro = [] + # The property of falling back to Any is inherited. + defn.info.fallback_to_any = any(baseinfo.fallback_to_any for baseinfo in defn.info.mro) + + +def find_duplicate(list: List[T]) -> T: + """If the list has duplicates, return one of the duplicates. + + Otherwise, return None. + """ + for i in range(1, len(list)): + if list[i] in list[:i]: + return list[i] + return None + + +def disable_typevars(nodes: List[SymbolTableNode]) -> None: + for node in nodes: + assert node.kind in (BOUND_TVAR, UNBOUND_TVAR) + node.kind = UNBOUND_TVAR + + +def enable_typevars(nodes: List[SymbolTableNode]) -> None: + for node in nodes: + assert node.kind in (BOUND_TVAR, UNBOUND_TVAR) + node.kind = BOUND_TVAR + + +def remove_imported_names_from_symtable(names: SymbolTable, + module: str) -> None: + """Remove all imported names from the symbol table of a module.""" + removed = [] # type: List[str] + for name, node in names.items(): + if node.node is None: + continue + fullname = node.node.fullname() + prefix = fullname[:fullname.rfind('.')] + if prefix != module: + removed.append(name) + for name in removed: + del names[name] + + +def infer_reachability_of_if_statement(s: IfStmt, + pyversion: Tuple[int, int]) -> None: + for i in range(len(s.expr)): + result = infer_if_condition_value(s.expr[i], pyversion) + if result == ALWAYS_FALSE: + # The condition is always false, so we skip the if/elif body. + mark_block_unreachable(s.body[i]) + elif result == ALWAYS_TRUE: + # This condition is always true, so all of the remaining + # elif/else bodies will never be executed. + for body in s.body[i + 1:]: + mark_block_unreachable(body) + if s.else_body: + mark_block_unreachable(s.else_body) + break + + +def infer_if_condition_value(expr: Node, pyversion: Tuple[int, int]) -> int: + """Infer whether if condition is always true/false. + + Return ALWAYS_TRUE if always true, ALWAYS_FALSE if always false, + and TRUTH_VALUE_UNKNOWN otherwise. + """ + name = '' + negated = False + alias = expr + if isinstance(alias, UnaryExpr): + if alias.op == 'not': + expr = alias.expr + negated = True + if isinstance(expr, NameExpr): + name = expr.name + elif isinstance(expr, MemberExpr): + name = expr.name + result = TRUTH_VALUE_UNKNOWN + if name == 'PY2': + result = ALWAYS_TRUE if pyversion[0] == 2 else ALWAYS_FALSE + elif name == 'PY3': + result = ALWAYS_TRUE if pyversion[0] == 3 else ALWAYS_FALSE + elif name == 'MYPY': + result = ALWAYS_TRUE + if negated: + if result == ALWAYS_TRUE: + result = ALWAYS_FALSE + elif result == ALWAYS_FALSE: + result = ALWAYS_TRUE + return result + + +def mark_block_unreachable(block: Block) -> None: + block.is_unreachable = True + block.accept(MarkImportsUnreachableVisitor()) + + +class MarkImportsUnreachableVisitor(TraverserVisitor): + """Visitor that flags all imports nested within a node as unreachable.""" + + def visit_import(self, node: Import) -> None: + node.is_unreachable = True + + def visit_import_from(self, node: ImportFrom) -> None: + node.is_unreachable = True + + def visit_import_all(self, node: ImportAll) -> None: + node.is_unreachable = True + + +def is_identity_signature(sig: Type) -> bool: + """Is type a callable of form T -> T (where T is a type variable)?""" + if isinstance(sig, CallableType) and sig.arg_kinds == [ARG_POS]: + if isinstance(sig.arg_types[0], TypeVarType) and isinstance(sig.ret_type, TypeVarType): + return sig.arg_types[0].id == sig.ret_type.id + return False + + +def returns_any_if_called(expr: Node) -> bool: + """Return True if we can predict that expr will return Any if called. + + This only uses information available during semantic analysis so this + will sometimes return False because of insufficient information (as + type inference hasn't run yet). + """ + if isinstance(expr, RefExpr): + if isinstance(expr.node, FuncDef): + typ = expr.node.type + if typ is None: + # No signature -> default to Any. + return True + # Explicit Any return? + return isinstance(typ, CallableType) and isinstance(typ.ret_type, AnyType) + elif isinstance(expr.node, Var): + typ = expr.node.type + return typ is None or isinstance(typ, AnyType) + elif isinstance(expr, CallExpr): + return returns_any_if_called(expr.callee) + return False + + +def find_fixed_callable_return(expr: Node) -> Optional[CallableType]: + if isinstance(expr, RefExpr): + if isinstance(expr.node, FuncDef): + typ = expr.node.type + if typ: + if isinstance(typ, CallableType) and has_no_typevars(typ.ret_type): + if isinstance(typ.ret_type, CallableType): + return typ.ret_type + elif isinstance(expr, CallExpr): + t = find_fixed_callable_return(expr.callee) + if t: + if isinstance(t.ret_type, CallableType): + return t.ret_type + return None + + +def has_no_typevars(typ: Type) -> bool: + return is_same_type(typ, erase_typevars(typ)) diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 9c2d6c4a86d8f..cd23b4fea08bd 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -1,209 +1,210 @@ -[case testNamedTupleUsedAsTuple] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -a, b = x -b = x[0] -a = x[1] -a, b, c = x # E: Need more than 2 values to unpack (3 expected) -x[2] # E: Tuple index out of range - -[case testAccessingNamedTupleAttributes] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -x.x -x.y -x.z # E: "X" has no attribute "z" - -[case testCreateNamedTupleWithPositionalArguments] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = X(1, 'x') -x.x -x.z # E: "X" has no attribute "z" -x = X(1) # E: Too few arguments for "X" -x = X(1, 2, 3) # E: Too many arguments for "X" - -[case testCreateNamedTupleWithKeywordArguments] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = X(x=1, y='x') -x = X(1, y='x') -x = X(x=1, z=1) # E: Unexpected keyword argument "z" for "X" -x = X(y=1) # E: Missing positional argument "x" in call to "X" - -[case testCreateNamedTupleAndUseAsTuple] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = X(1, 'x') -a, b = x -a, b, c = x # E: Need more than 2 values to unpack (3 expected) - -[case testNamedTupleWithItemTypes] -from typing import NamedTuple -N = NamedTuple('N', [('a', int), - ('b', str)]) -n = N(1, 'x') -s = n.a # type: str # E: Incompatible types in assignment (expression has type "int", \ - variable has type "str") -i = n.b # type: int # E: Incompatible types in assignment (expression has type "str", \ - variable has type "int") -x, y = n -x = y # E: Incompatible types in assignment (expression has type "str", variable has type "int") - -[case testNamedTupleConstructorArgumentTypes] -from typing import NamedTuple -N = NamedTuple('N', [('a', int), - ('b', str)]) -n = N('x', 'x') # E: Argument 1 to "N" has incompatible type "str"; expected "int" -n = N(1, b=2) # E: Argument 2 to "N" has incompatible type "int"; expected "str" -N(1, 'x') -N(b='x', a=1) - -[case testNamedTupleAsBaseClass] -from typing import NamedTuple -N = NamedTuple('N', [('a', int), - ('b', str)]) -class X(N): - pass -x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" -s = '' -i = 0 -s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") -i, s = x -s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") - -[case testNamedTupleAsBaseClass2] -from typing import NamedTuple -class X(NamedTuple('N', [('a', int), - ('b', str)])): - pass -x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" -s = '' -i = 0 -s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") -i, s = x -s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") - -[case testTwoNamedTuplesAsBaseClasses] -from typing import NamedTuple -A = NamedTuple('A', [('a', int)]) -B = NamedTuple('B', [('a', int)]) -class X(A, B): # E: Class has two incompatible bases derived from tuple - pass - -[case testTwoNamedTuplesAsBaseClasses2] -from typing import NamedTuple -A = NamedTuple('A', [('a', int)]) -class X(A, NamedTuple('B', [('a', int)])): # E: Class has two incompatible bases derived from tuple - pass - -[case testSelfTypeWithNamedTupleAsBase] -from typing import NamedTuple -A = NamedTuple('A', [('a', int), ('b', str)]) -class B(A): - def f(self, x: int) -> None: - self.f(self.a) - self.f(self.b) # E: Argument 1 to "f" of "B" has incompatible type "str"; expected "int" - i = 0 - s = '' - i, s = self - i, i = self # E: Incompatible types in assignment (expression has type "str", \ - variable has type "int") -[out] -main: note: In member "f" of class "B": - -[case testTypeReferenceToClassDerivedFromNamedTuple] -from typing import NamedTuple -A = NamedTuple('A', [('a', int), ('b', str)]) -class B(A): - def f(self, x: 'B') -> None: - i = 0 - s = '' - self = x - i, s = x - i, s = x.a, x.b - i, s = x.a, x.a # E: Incompatible types in assignment (expression has type "int", \ - variable has type "str") - i, i = self # E: Incompatible types in assignment (expression has type "str", \ - variable has type "int") -[out] -main: note: In member "f" of class "B": - -[case testNamedTupleSubtyping] -from typing import NamedTuple, Tuple -A = NamedTuple('A', [('a', int), ('b', str)]) -class B(A): pass -a = A(1, '') -b = B(1, '') -t = None # type: Tuple[int, str] -b = a # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "A") -b = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "B") -t = a -t = (1, '') -t = b -a = b - -[case testNamedTupleSimpleTypeInference] -from typing import NamedTuple, Tuple -A = NamedTuple('A', [('a', int)]) -l = [A(1), A(2)] -a = A(1) -a = l[0] -(i,) = l[0] -i, i = l[0] # E: Need more than 1 value to unpack (2 expected) -l = [A(1)] -a = (1,) # E: Incompatible types in assignment (expression has type "Tuple[int]", \ - variable has type "A") -[builtins fixtures/list.py] - -[case testNamedTupleMissingClassAttribute] -import collections -MyNamedTuple = collections.namedtuple('MyNamedTuple', ['spam', 'eggs']) -MyNamedTuple.x # E: "MyNamedTuple" has no attribute "x" - -[case testNamedTupleEmptyItems] -from typing import NamedTuple -A = NamedTuple('A', []) -[builtins fixtures/list.py] - -[case testNamedTupleProperty] -from typing import NamedTuple -A = NamedTuple('A', [('a', int)]) -class B(A): - @property - def b(self) -> int: - return self.a -class C(B): pass -B(1).b -C(2).b -[builtins fixtures/property.py] - -[case testNamedTupleAsDict] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -a = x._asdict() - -[case testNamedTupleReplace] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -x = x._replace(y=5) - -[case testNamedTupleReplaceTyped] -from typing import NamedTuple - -X = NamedTuple('X', [('x', int), ('y', str)]) -x = None # type: X -x = x._replace(x=5) -x = x._replace(y=5) # E: Argument 1 to X has incompatible type "int"; expected "str" +[case testNamedTupleUsedAsTuple] +import typing +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = X(1, 2) # type: X # None does not seem to work for TYPE_ALIAS +a, b = x +b = x[0] +a = x[1] +a, b, c = x # E: Need more than 2 values to unpack (3 expected) +x[2] # E: Tuple index out of range + +[case testAccessingNamedTupleAttributes] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +x.x +x.y +x.z # E: "X" has no attribute "z" + +[case testCreateNamedTupleWithPositionalArguments] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = X(1, 'x') +x.x +x.z # E: "X" has no attribute "z" +x = X(1) # E: Too few arguments for "X" +x = X(1, 2, 3) # E: Too many arguments for "X" + +[case testCreateNamedTupleWithKeywordArguments] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = X(x=1, y='x') +x = X(1, y='x') +x = X(x=1, z=1) # E: Unexpected keyword argument "z" for "X" +x = X(y=1) # E: Missing positional argument "x" in call to "X" + +[case testCreateNamedTupleAndUseAsTuple] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = X(1, 'x') +a, b = x +a, b, c = x # E: Need more than 2 values to unpack (3 expected) + +[case testNamedTupleWithItemTypes] +from typing import NamedTuple +N = NamedTuple('N', [('a', int), + ('b', str)]) +n = N(1, 'x') +s = n.a # type: str # E: Incompatible types in assignment (expression has type "int", \ + variable has type "str") +i = n.b # type: int # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +x, y = n +x = y # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testNamedTupleConstructorArgumentTypes] +from typing import NamedTuple +N = NamedTuple('N', [('a', int), + ('b', str)]) +n = N('x', 'x') # E: Argument 1 to "N" has incompatible type "str"; expected "int" +n = N(1, b=2) # E: Argument 2 to "N" has incompatible type "int"; expected "str" +N(1, 'x') +N(b='x', a=1) + +[case testNamedTupleAsBaseClass] +from typing import NamedTuple +N = NamedTuple('N', [('a', int), + ('b', str)]) +class X(N): + pass +x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" +s = '' +i = 0 +s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") +i, s = x +s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testNamedTupleAsBaseClass2] +from typing import NamedTuple +class X(NamedTuple('N', [('a', int), + ('b', str)])): + pass +x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" +s = '' +i = 0 +s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") +i, s = x +s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testTwoNamedTuplesAsBaseClasses] +from typing import NamedTuple +A = NamedTuple('A', [('a', int)]) +B = NamedTuple('B', [('a', int)]) +class X(A, B): # E: Class has two incompatible bases derived from tuple + pass + +[case testTwoNamedTuplesAsBaseClasses2] +from typing import NamedTuple +A = NamedTuple('A', [('a', int)]) +class X(A, NamedTuple('B', [('a', int)])): # E: Class has two incompatible bases derived from tuple + pass + +[case testSelfTypeWithNamedTupleAsBase] +from typing import NamedTuple +A = NamedTuple('A', [('a', int), ('b', str)]) +class B(A): + def f(self, x: int) -> None: + self.f(self.a) + self.f(self.b) # E: Argument 1 to "f" of "B" has incompatible type "str"; expected "int" + i = 0 + s = '' + i, s = self + i, i = self # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +[out] +main: note: In member "f" of class "B": + +[case testTypeReferenceToClassDerivedFromNamedTuple] +from typing import NamedTuple +A = NamedTuple('A', [('a', int), ('b', str)]) +class B(A): + def f(self, x: 'B') -> None: + i = 0 + s = '' + self = x + i, s = x + i, s = x.a, x.b + i, s = x.a, x.a # E: Incompatible types in assignment (expression has type "int", \ + variable has type "str") + i, i = self # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +[out] +main: note: In member "f" of class "B": + +[case testNamedTupleSubtyping] +from typing import NamedTuple, Tuple +A = NamedTuple('A', [('a', int), ('b', str)]) +class B(A): pass +a = A(1, '') +b = B(1, '') +t = None # type: Tuple[int, str] +b = a # E: Incompatible types in assignment (expression has type "A", variable has type "B") +a = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "A") +b = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "B") +t = a +t = (1, '') +t = b +a = b + +[case testNamedTupleSimpleTypeInference] +from typing import NamedTuple, Tuple +A = NamedTuple('A', [('a', int)]) +l = [A(1), A(2)] +a = A(1) +a = l[0] +(i,) = l[0] +i, i = l[0] # E: Need more than 1 value to unpack (2 expected) +l = [A(1)] +a = (1,) # E: Incompatible types in assignment (expression has type "Tuple[int]", \ + variable has type "A") +[builtins fixtures/list.py] + +[case testNamedTupleMissingClassAttribute] +import collections +MyNamedTuple = collections.namedtuple('MyNamedTuple', ['spam', 'eggs']) +MyNamedTuple.x # E: "MyNamedTuple" has no attribute "x" + +[case testNamedTupleEmptyItems] +from typing import NamedTuple +A = NamedTuple('A', []) +[builtins fixtures/list.py] + +[case testNamedTupleProperty] +from typing import NamedTuple +A = NamedTuple('A', [('a', int)]) +class B(A): + @property + def b(self) -> int: + return self.a +class C(B): pass +B(1).b +C(2).b +[builtins fixtures/property.py] + +[case testNamedTupleAsDict] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = X(1, 2) # type: X +a = x._asdict() + +[case testNamedTupleReplace] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = X(1, 2) # type: X +x = x._replace(y=5) + +[case testNamedTupleReplaceTyped] +from typing import NamedTuple + +X = NamedTuple('X', [('x', int), ('y', str)]) +x = X(1, 'a') # type: X +x = x._replace(x=5) +x = x._replace(y=5) # E: Argument 1 to X has incompatible type "int"; expected "str" From 41ed07580df2eecb91b82fae13c7690b79b66ad3 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Sun, 10 Jul 2016 10:33:21 +0300 Subject: [PATCH 42/74] run both Replace and AsTuple --- mypy/nodes.py | 2 +- mypy/semanal.py | 15 +++++++++++---- mypy/types.py | 8 ++++++++ 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index f4c26634ba25b..645c54dd015e7 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1977,7 +1977,7 @@ def deserialize(cls, data: JsonDict) -> 'TypeInfo': class NamedTupleTypeInfo(TypeInfo): - is_named_tuple = False + is_named_tuple = True def __init__(self, tup, *args): super().__init__(*args) tup.fullname = lambda : '__builtins__.tuple' diff --git a/mypy/semanal.py b/mypy/semanal.py index 50bd040f91536..e89d3560fcc6f 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -72,7 +72,8 @@ from mypy.types import ( NoneTyp, CallableType, Overloaded, Instance, Type, TypeVarType, AnyType, FunctionLike, UnboundType, TypeList, ErrorType, TypeVarDef, - replace_leading_arg_type, TupleType, UnionType, StarType, EllipsisType + replace_leading_arg_type, TupleType, UnionType, StarType, EllipsisType, + NamedTupleType ) from mypy.nodes import function_type, implicit_module_attrs from mypy.typeanal import TypeAnalyser, TypeAnalyserPass3, analyze_type_alias @@ -1524,7 +1525,7 @@ def fail_namedtuple_arg(self, message: str, def build_namedtuple_typeinfo(self, name: str, items: List[str], types: List[Type]) -> TypeInfo: symbols = SymbolTable() - tup = TupleType(types, self.named_type('__builtins__.tuple', [AnyType()])) + tup = NamedTupleType(items, types, self.named_type('__builtins__.tuple', [AnyType()])) class_def = ClassDef(name, Block([])) class_def.fullname = self.qualified_name(name) info = NamedTupleTypeInfo(tup, symbols, class_def) @@ -1537,7 +1538,7 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], # Add __init__, _replace, _asdict methods and _fields static field self.add_namedtuple_method('__init__', symbols, info, NoneTyp(), self.make_factory_args(vars, ARG_POS)) - self.add_namedtuple_method('_replace', symbols, info, Instance(info, []), + self.add_namedtuple_method('_replace', symbols, info, self_type(info), self.make_factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) # TODO: refine to OrderedDict[Union[types]] self.add_namedtuple_method('_asdict', symbols, info, AnyType()) @@ -2630,7 +2631,13 @@ def self_type(typ: TypeInfo) -> Union[Instance, TupleType]: tv = [] # type: List[Type] for i in range(len(typ.type_vars)): tv.append(TypeVarType(typ.defn.type_vars[i])) - return Instance(typ, tv) + inst = Instance(typ, tv) + if typ.tuple_type is None: + return inst + if not typ.is_named_tuple: + inst = TupleType(typ.tuple_type.items, inst) + return inst + return NamedTupleType(typ.tuple_type.names, typ.tuple_type.items, inst) def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: diff --git a/mypy/types.py b/mypy/types.py index 8f0f938d09d1c..26dc26bf6c12a 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -761,6 +761,14 @@ def deserialize(cls, data: JsonDict) -> 'TupleType': implicit=data['implicit']) +class NamedTupleType(TupleType): + names = None # type: List[str] + + def __init__(self, names: List[str], *args) -> None: + self.names = names + super().__init__(*args) + + class StarType(Type): """The star type *type_parameter. From 6dd568f14fc66ff6428140d89accc092484a3e21 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Mon, 11 Jul 2016 00:09:04 +0300 Subject: [PATCH 43/74] fix most of the tests --- mypy/nodes.py | 8 ++++---- mypy/semanal.py | 11 ++++------- mypy/types.py | 12 +++++++++--- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index 645c54dd015e7..0f512d0b6d718 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1978,12 +1978,12 @@ def deserialize(cls, data: JsonDict) -> 'TypeInfo': class NamedTupleTypeInfo(TypeInfo): is_named_tuple = True - def __init__(self, tup, *args): + bases = None # type: List[mypy.types.Instance] + def __init__(self, tup: 'mypy.types.NamedTupleType', *args) -> None: super().__init__(*args) - tup.fullname = lambda : '__builtins__.tuple' self.tuple_type = tup - self.bases = [tup] - self.mro = [self, tup] + self.bases = [] + self.mro = [self] # self._promote = tup diff --git a/mypy/semanal.py b/mypy/semanal.py index e89d3560fcc6f..232f918f23a48 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1418,7 +1418,7 @@ def process_namedtuple_definition(self, s: AssignmentStmt) -> None: return # Yes, it's a valid namedtuple definition. Add it to the symbol table. node = self.lookup(name, s) - node.kind = TYPE_ALIAS # TODO locally defined namedtuple + node.kind = GDEF # TODO locally defined namedtuple # TODO call.analyzed node.node = named_tuple @@ -1525,7 +1525,7 @@ def fail_namedtuple_arg(self, message: str, def build_namedtuple_typeinfo(self, name: str, items: List[str], types: List[Type]) -> TypeInfo: symbols = SymbolTable() - tup = NamedTupleType(items, types, self.named_type('__builtins__.tuple', [AnyType()])) + tup = NamedTupleType(items, types, self.named_type('__builtins__.tuple', types)) class_def = ClassDef(name, Block([])) class_def.fullname = self.qualified_name(name) info = NamedTupleTypeInfo(tup, symbols, class_def) @@ -1534,7 +1534,7 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], # TODO: Make them read-only. for var in vars: var.info = info - symbols[var.name] = SymbolTableNode(MDEF, var) + symbols[var.name()] = SymbolTableNode(MDEF, var) # Add __init__, _replace, _asdict methods and _fields static field self.add_namedtuple_method('__init__', symbols, info, NoneTyp(), self.make_factory_args(vars, ARG_POS)) @@ -2634,10 +2634,7 @@ def self_type(typ: TypeInfo) -> Union[Instance, TupleType]: inst = Instance(typ, tv) if typ.tuple_type is None: return inst - if not typ.is_named_tuple: - inst = TupleType(typ.tuple_type.items, inst) - return inst - return NamedTupleType(typ.tuple_type.names, typ.tuple_type.items, inst) + return typ.tuple_type.self_type(inst) def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: diff --git a/mypy/types.py b/mypy/types.py index 26dc26bf6c12a..bd31bf25d49f3 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -760,13 +760,19 @@ def deserialize(cls, data: JsonDict) -> 'TupleType': Instance.deserialize(data['fallback']), implicit=data['implicit']) + def self_type(self, inst: Instance) -> 'TupleType': + return TupleType(self.items, inst) + class NamedTupleType(TupleType): - names = None # type: List[str] + attrs = None # type: List[str] - def __init__(self, names: List[str], *args) -> None: - self.names = names + def __init__(self, attrs: List[str], *args) -> None: + self.attrs = attrs super().__init__(*args) + + def self_type(self, inst: Instance) -> 'NamedTupleType': + return NamedTupleType(self.attrs, self.items, inst) class StarType(Type): From b18fc25c36fc169b4760944765288a2c859739b1 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Mon, 11 Jul 2016 01:32:18 +0300 Subject: [PATCH 44/74] complete support for _replace --- mypy/checker.py | 4 +++- mypy/nodes.py | 6 ++---- mypy/semanal.py | 22 ++++++++++++---------- test-data/unit/check-namedtuple.test | 9 ++++----- 4 files changed, 21 insertions(+), 20 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index e2aef9334ea7e..7063758096dfa 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -801,7 +801,9 @@ def visit_class_def(self, defn: ClassDef) -> Type: with self.binder.frame_context(): self.accept(defn.defs) self.binder = old_binder - self.check_multiple_inheritance(typ) + if not hasattr(defn, 'incompatible_baseclass'): + # Otherwise we've already found errors; more errors are not useful + self.check_multiple_inheritance(typ) self.leave_partial_types() self.errors.pop_type() diff --git a/mypy/nodes.py b/mypy/nodes.py index 0f512d0b6d718..b027c6223e528 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1982,10 +1982,8 @@ class NamedTupleTypeInfo(TypeInfo): def __init__(self, tup: 'mypy.types.NamedTupleType', *args) -> None: super().__init__(*args) self.tuple_type = tup - self.bases = [] - self.mro = [self] - # self._promote = tup - + self.bases = [tup.fallback] + self.mro = [cast(TypeInfo, self)] + tup.fallback.type.mro class SymbolTableNode: diff --git a/mypy/semanal.py b/mypy/semanal.py index 232f918f23a48..a19287787a9f0 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -744,48 +744,50 @@ def analyze_base_classes(self, defn: ClassDef) -> None: """ base_types = [] # type: List[Instance] + info = defn.info for base_expr in defn.base_type_exprs: try: base = self.expr_to_analyzed_type(base_expr) except TypeTranslationError: self.fail('Invalid base class', base_expr) - defn.info.fallback_to_any = True + info.fallback_to_any = True continue if isinstance(base, TupleType): - if defn.info.tuple_type: + if info.tuple_type: self.fail("Class has two incompatible bases derived from tuple", defn) + defn.incompatible_baseclass = True if (not self.is_stub_file - and not defn.info.is_named_tuple + and not info.is_named_tuple and base.fallback.type.fullname() == 'builtins.tuple'): self.fail("Tuple[...] not supported as a base class outside a stub file", defn) - defn.info.tuple_type = base + info.tuple_type = base base_types.append(base.fallback) elif isinstance(base, Instance): base_types.append(base) elif isinstance(base, AnyType): - defn.info.fallback_to_any = True + info.fallback_to_any = True else: self.fail('Invalid base class', base_expr) - defn.info.fallback_to_any = True + info.fallback_to_any = True # Add 'object' as implicit base if there is no other base class. if (not base_types and defn.fullname != 'builtins.object'): base_types.append(self.object_type()) - defn.info.bases = base_types + info.bases = base_types # Calculate the MRO. It might be incomplete at this point if # the bases of defn include classes imported from other # modules in an import loop. We'll recompute it in ThirdPass. if not self.verify_base_classes(defn): - defn.info.mro = [] + info.mro = [] return calculate_class_mro(defn, self.fail) # If there are cyclic imports, we may be missing 'object' in # the MRO. Fix MRO if needed. - if defn.info.mro and defn.info.mro[-1].fullname() != 'builtins.object': - defn.info.mro.append(self.object_type().type) + if info.mro and info.mro[-1].fullname() != 'builtins.object': + info.mro.append(self.object_type().type) def expr_to_analyzed_type(self, expr: Node) -> Type: if isinstance(expr, CallExpr): diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index cd23b4fea08bd..bc37c40208198 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -1,9 +1,8 @@ [case testNamedTupleUsedAsTuple] -import typing from collections import namedtuple X = namedtuple('X', ['x', 'y']) -x = X(1, 2) # type: X # None does not seem to work for TYPE_ALIAS +x = None # type: X a, b = x b = x[0] a = x[1] @@ -191,20 +190,20 @@ C(2).b from collections import namedtuple X = namedtuple('X', ['x', 'y']) -x = X(1, 2) # type: X +x = None # type: X a = x._asdict() [case testNamedTupleReplace] from collections import namedtuple X = namedtuple('X', ['x', 'y']) -x = X(1, 2) # type: X +x = None # type: X x = x._replace(y=5) [case testNamedTupleReplaceTyped] from typing import NamedTuple X = NamedTuple('X', [('x', int), ('y', str)]) -x = X(1, 'a') # type: X +x = None # type: X x = x._replace(x=5) x = x._replace(y=5) # E: Argument 1 to X has incompatible type "int"; expected "str" From d7dbd3f27da636a249bec60b51914260b1d7ea19 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Mon, 11 Jul 2016 01:38:41 +0300 Subject: [PATCH 45/74] flake8-ing --- mypy/nodes.py | 1 + mypy/semanal.py | 5674 +++++++++++++------------- mypy/types.py | 2 +- test-data/unit/check-namedtuple.test | 418 +- 4 files changed, 3048 insertions(+), 3047 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index b027c6223e528..9211d65485106 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1979,6 +1979,7 @@ def deserialize(cls, data: JsonDict) -> 'TypeInfo': class NamedTupleTypeInfo(TypeInfo): is_named_tuple = True bases = None # type: List[mypy.types.Instance] + def __init__(self, tup: 'mypy.types.NamedTupleType', *args) -> None: super().__init__(*args) self.tuple_type = tup diff --git a/mypy/semanal.py b/mypy/semanal.py index a19287787a9f0..fed7475ae557a 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1,2837 +1,2837 @@ -"""The semantic analyzer. - -Bind names to definitions and do various other simple consistency -checks. For example, consider this program: - - x = 1 - y = x - -Here semantic analysis would detect that the assignment 'x = 1' -defines a new variable, the type of which is to be inferred (in a -later pass; type inference or type checking is not part of semantic -analysis). Also, it would bind both references to 'x' to the same -module-level variable node. The second assignment would also be -analyzed, and the type of 'y' marked as being inferred. - -Semantic analysis is the first analysis pass after parsing, and it is -subdivided into three passes: - - * FirstPass looks up externally visible names defined in a module but - ignores imports and local definitions. It helps enable (some) - cyclic references between modules, such as module 'a' that imports - module 'b' and used names defined in b *and* vice versa. The first - pass can be performed before dependent modules have been processed. - - * SemanticAnalyzer is the second pass. It does the bulk of the work. - It assumes that dependent modules have been semantically analyzed, - up to the second pass, unless there is a import cycle. - - * ThirdPass checks that type argument counts are valid; for example, - it will reject Dict[int]. We don't do this in the second pass, - since we infer the type argument counts of classes during this - pass, and it is possible to refer to classes defined later in a - file, which would not have the type argument count set yet. This - pass also recomputes the method resolution order of each class, in - case one of its bases belongs to a module involved in an import - loop. - -Semantic analysis of types is implemented in module mypy.typeanal. - -TODO: Check if the third pass slows down type checking significantly. - We could probably get rid of it -- for example, we could collect all - analyzed types in a collection and check them without having to - traverse the entire AST. -""" - -from typing import ( - List, Dict, Set, Tuple, cast, Any, overload, TypeVar, Union, Optional, Callable -) - -from mypy.nodes import ( - MypyFile, TypeInfo, Node, AssignmentStmt, FuncDef, OverloadedFuncDef, - ClassDef, Var, GDEF, MODULE_REF, FuncItem, Import, - ImportFrom, ImportAll, Block, LDEF, NameExpr, MemberExpr, - IndexExpr, TupleExpr, ListExpr, ExpressionStmt, ReturnStmt, - RaiseStmt, AssertStmt, OperatorAssignmentStmt, WhileStmt, - ForStmt, BreakStmt, ContinueStmt, IfStmt, TryStmt, WithStmt, DelStmt, - GlobalDecl, SuperExpr, DictExpr, CallExpr, RefExpr, OpExpr, UnaryExpr, - SliceExpr, CastExpr, RevealTypeExpr, TypeApplication, Context, SymbolTable, - SymbolTableNode, BOUND_TVAR, UNBOUND_TVAR, ListComprehension, GeneratorExpr, - FuncExpr, MDEF, FuncBase, Decorator, SetExpr, TypeVarExpr, - StrExpr, BytesExpr, PrintStmt, ConditionalExpr, PromoteExpr, - ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, MroError, type_aliases, - YieldFromExpr, NamedTupleExpr, NonlocalDecl, - SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr, - YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, COVARIANT, CONTRAVARIANT, - IntExpr, FloatExpr, UnicodeExpr, - INVARIANT, UNBOUND_IMPORTED, Expression, EllipsisExpr, NamedTupleTypeInfo -) -from mypy.visitor import NodeVisitor -from mypy.traverser import TraverserVisitor -from mypy.errors import Errors, report_internal_error -from mypy.types import ( - NoneTyp, CallableType, Overloaded, Instance, Type, TypeVarType, AnyType, - FunctionLike, UnboundType, TypeList, ErrorType, TypeVarDef, - replace_leading_arg_type, TupleType, UnionType, StarType, EllipsisType, - NamedTupleType -) -from mypy.nodes import function_type, implicit_module_attrs -from mypy.typeanal import TypeAnalyser, TypeAnalyserPass3, analyze_type_alias -from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError -from mypy.lex import lex -from mypy.parsetype import parse_type -from mypy.sametypes import is_same_type -from mypy.erasetype import erase_typevars -from mypy import defaults -from mypy.options import Options - - -T = TypeVar('T') - - -# Inferred value of an expression. -ALWAYS_TRUE = 0 -ALWAYS_FALSE = 1 -TRUTH_VALUE_UNKNOWN = 2 - -# Map from obsolete name to the current spelling. -obsolete_name_mapping = { - 'typing.Function': 'typing.Callable', - 'typing.typevar': 'typing.TypeVar', -} - -# Hard coded type promotions (shared between all Python versions). -# These add extra ad-hoc edges to the subtyping relation. For example, -# int is considered a subtype of float, even though there is no -# subclass relationship. -TYPE_PROMOTIONS = { - 'builtins.int': 'builtins.float', - 'builtins.float': 'builtins.complex', -} - -# Hard coded type promotions for Python 3. -# -# Note that the bytearray -> bytes promotion is a little unsafe -# as some functions only accept bytes objects. Here convenience -# trumps safety. -TYPE_PROMOTIONS_PYTHON3 = TYPE_PROMOTIONS.copy() -TYPE_PROMOTIONS_PYTHON3.update({ - 'builtins.bytearray': 'builtins.bytes', -}) - -# Hard coded type promotions for Python 2. -# -# These promotions are unsafe, but we are doing them anyway -# for convenience and also for Python 3 compatibility -# (bytearray -> str). -TYPE_PROMOTIONS_PYTHON2 = TYPE_PROMOTIONS.copy() -TYPE_PROMOTIONS_PYTHON2.update({ - 'builtins.str': 'builtins.unicode', - 'builtins.bytearray': 'builtins.str', -}) - -# When analyzing a function, should we analyze the whole function in one go, or -# should we only perform one phase of the analysis? The latter is used for -# nested functions. In the first phase we add the function to the symbol table -# but don't process body. In the second phase we process function body. This -# way we can have mutually recursive nested functions. -FUNCTION_BOTH_PHASES = 0 # Everthing in one go -FUNCTION_FIRST_PHASE_POSTPONE_SECOND = 1 # Add to symbol table but postpone body -FUNCTION_SECOND_PHASE = 2 # Only analyze body - - -class SemanticAnalyzer(NodeVisitor): - """Semantically analyze parsed mypy files. - - The analyzer binds names and does various consistency checks for a - parse tree. Note that type checking is performed as a separate - pass. - - This is the second phase of semantic analysis. - """ - - # Library search paths - lib_path = None # type: List[str] - # Module name space - modules = None # type: Dict[str, MypyFile] - # Global name space for current module - globals = None # type: SymbolTable - # Names declared using "global" (separate set for each scope) - global_decls = None # type: List[Set[str]] - # Names declated using "nonlocal" (separate set for each scope) - nonlocal_decls = None # type: List[Set[str]] - # Local names of function scopes; None for non-function scopes. - locals = None # type: List[SymbolTable] - # Nested block depths of scopes - block_depth = None # type: List[int] - # TypeInfo of directly enclosing class (or None) - type = None # type: TypeInfo - # Stack of outer classes (the second tuple item contains tvars). - type_stack = None # type: List[TypeInfo] - # Type variables that are bound by the directly enclosing class - bound_tvars = None # type: List[SymbolTableNode] - # Stack of type variables that were bound by outer classess - tvar_stack = None # type: List[List[SymbolTableNode]] - # Do weak type checking in this file - weak_opts = set() # type: Set[str] - - # Stack of functions being analyzed - function_stack = None # type: List[FuncItem] - # Stack of next available function type variable ids - next_function_tvar_id_stack = None # type: List[int] - - # Status of postponing analysis of nested function bodies. By using this we - # can have mutually recursive nested functions. Values are FUNCTION_x - # constants. Note that separate phasea are not used for methods. - postpone_nested_functions_stack = None # type: List[int] - # Postponed functions collected if - # postpone_nested_functions_stack[-1] == FUNCTION_FIRST_PHASE_POSTPONE_SECOND. - postponed_functions_stack = None # type: List[List[Node]] - - loop_depth = 0 # Depth of breakable loops - cur_mod_id = '' # Current module id (or None) (phase 2) - is_stub_file = False # Are we analyzing a stub file? - imports = None # type: Set[str] # Imported modules (during phase 2 analysis) - errors = None # type: Errors # Keeps track of generated errors - - def __init__(self, - lib_path: List[str], - errors: Errors, - options: Options) -> None: - """Construct semantic analyzer. - - Use lib_path to search for modules, and report analysis errors - using the Errors instance. - """ - self.locals = [None] - self.imports = set() - self.type = None - self.type_stack = [] - self.bound_tvars = None - self.tvar_stack = [] - self.function_stack = [] - self.next_function_tvar_id_stack = [-1] - self.block_depth = [0] - self.loop_depth = 0 - self.lib_path = lib_path - self.errors = errors - self.modules = {} - self.options = options - self.postpone_nested_functions_stack = [FUNCTION_BOTH_PHASES] - self.postponed_functions_stack = [] - self.all_exports = set() # type: Set[str] - - def visit_file(self, file_node: MypyFile, fnam: str) -> None: - self.errors.set_file(fnam) - self.cur_mod_node = file_node - self.cur_mod_id = file_node.fullname() - self.is_stub_file = fnam.lower().endswith('.pyi') - self.globals = file_node.names - self.weak_opts = file_node.weak_opts - - if 'builtins' in self.modules: - self.globals['__builtins__'] = SymbolTableNode( - MODULE_REF, self.modules['builtins'], self.cur_mod_id) - - for name in implicit_module_attrs: - v = self.globals[name].node - if isinstance(v, Var): - v.type = self.anal_type(v.type) - v.is_ready = True - - defs = file_node.defs - for d in defs: - self.accept(d) - - if self.cur_mod_id == 'builtins': - remove_imported_names_from_symtable(self.globals, 'builtins') - - if '__all__' in self.globals: - for name, g in self.globals.items(): - if name not in self.all_exports: - g.module_public = False - - def visit_func_def(self, defn: FuncDef) -> None: - phase_info = self.postpone_nested_functions_stack[-1] - if phase_info != FUNCTION_SECOND_PHASE: - self.function_stack.append(defn) - # First phase of analysis for function. - self.errors.push_function(defn.name()) - self.update_function_type_variables(defn) - self.errors.pop_function() - self.function_stack.pop() - - defn.is_conditional = self.block_depth[-1] > 0 - - # TODO(jukka): Figure out how to share the various cases. It doesn't - # make sense to have (almost) duplicate code (here and elsewhere) for - # 3 cases: module-level, class-level and local names. Maybe implement - # a common stack of namespaces. As the 3 kinds of namespaces have - # different semantics, this wouldn't always work, but it might still - # be a win. - if self.is_class_scope(): - # Method definition - defn.info = self.type - if not defn.is_decorated and not defn.is_overload: - if defn.name() in self.type.names: - # Redefinition. Conditional redefinition is okay. - n = self.type.names[defn.name()].node - if self.is_conditional_func(n, defn): - defn.original_def = cast(FuncDef, n) - else: - self.name_already_defined(defn.name(), defn) - self.type.names[defn.name()] = SymbolTableNode(MDEF, defn) - self.prepare_method_signature(defn) - elif self.is_func_scope(): - # Nested function - if not defn.is_decorated and not defn.is_overload: - if defn.name() in self.locals[-1]: - # Redefinition. Conditional redefinition is okay. - n = self.locals[-1][defn.name()].node - if self.is_conditional_func(n, defn): - defn.original_def = cast(FuncDef, n) - else: - self.name_already_defined(defn.name(), defn) - else: - self.add_local(defn, defn) - else: - # Top-level function - if not defn.is_decorated and not defn.is_overload: - symbol = self.globals.get(defn.name()) - if isinstance(symbol.node, FuncDef) and symbol.node != defn: - # This is redefinition. Conditional redefinition is okay. - original_def = symbol.node - if self.is_conditional_func(original_def, defn): - # Conditional function definition -- multiple defs are ok. - defn.original_def = original_def - else: - # Report error. - self.check_no_global(defn.name(), defn, True) - if phase_info == FUNCTION_FIRST_PHASE_POSTPONE_SECOND: - # Postpone this function (for the second phase). - self.postponed_functions_stack[-1].append(defn) - return - if phase_info != FUNCTION_FIRST_PHASE_POSTPONE_SECOND: - # Second phase of analysis for function. - self.errors.push_function(defn.name()) - self.analyze_function(defn) - self.errors.pop_function() - - def prepare_method_signature(self, func: FuncDef) -> None: - """Check basic signature validity and tweak annotation of self/cls argument.""" - # Only non-static methods are special. - if not func.is_static: - if not func.arguments: - self.fail('Method must have at least one argument', func) - elif func.type: - sig = cast(FunctionLike, func.type) - if func.is_class: - leading_type = self.class_type(self.type) - else: - leading_type = self_type(self.type) - func.type = replace_implicit_first_type(sig, leading_type) - - def is_conditional_func(self, previous: Node, new: FuncDef) -> bool: - """Does 'new' conditionally redefine 'previous'? - - We reject straight redefinitions of functions, as they are usually - a programming error. For example: - - . def f(): ... - . def f(): ... # Error: 'f' redefined - """ - return isinstance(previous, (FuncDef, Var)) and new.is_conditional - - def update_function_type_variables(self, defn: FuncDef) -> None: - """Make any type variables in the signature of defn explicit. - - Update the signature of defn to contain type variable definitions - if defn is generic. - """ - if defn.type: - functype = cast(CallableType, defn.type) - typevars = self.infer_type_variables(functype) - # Do not define a new type variable if already defined in scope. - typevars = [(name, tvar) for name, tvar in typevars - if not self.is_defined_type_var(name, defn)] - if typevars: - next_tvar_id = self.next_function_tvar_id() - defs = [TypeVarDef(tvar[0], next_tvar_id - i, - tvar[1].values, tvar[1].upper_bound, - tvar[1].variance) - for i, tvar in enumerate(typevars)] - functype.variables = defs - - def infer_type_variables(self, - type: CallableType) -> List[Tuple[str, TypeVarExpr]]: - """Return list of unique type variables referred to in a callable.""" - names = [] # type: List[str] - tvars = [] # type: List[TypeVarExpr] - for arg in type.arg_types + [type.ret_type]: - for name, tvar_expr in self.find_type_variables_in_type(arg): - if name not in names: - names.append(name) - tvars.append(tvar_expr) - return list(zip(names, tvars)) - - def find_type_variables_in_type( - self, type: Type) -> List[Tuple[str, TypeVarExpr]]: - """Return a list of all unique type variable references in type. - - This effectively does partial name binding, results of which are mostly thrown away. - """ - result = [] # type: List[Tuple[str, TypeVarExpr]] - if isinstance(type, UnboundType): - name = type.name - node = self.lookup_qualified(name, type) - if node and node.kind == UNBOUND_TVAR: - result.append((name, cast(TypeVarExpr, node.node))) - for arg in type.args: - result.extend(self.find_type_variables_in_type(arg)) - elif isinstance(type, TypeList): - for item in type.items: - result.extend(self.find_type_variables_in_type(item)) - elif isinstance(type, UnionType): - for item in type.items: - result.extend(self.find_type_variables_in_type(item)) - elif isinstance(type, AnyType): - pass - elif isinstance(type, EllipsisType) or isinstance(type, TupleType): - pass - else: - assert False, 'Unsupported type %s' % type - return result - - def is_defined_type_var(self, tvar: str, context: Node) -> bool: - return self.lookup_qualified(tvar, context).kind == BOUND_TVAR - - def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: - t = [] # type: List[CallableType] - for i, item in enumerate(defn.items): - # TODO support decorated overloaded functions properly - item.is_overload = True - item.func.is_overload = True - item.accept(self) - t.append(cast(CallableType, function_type(item.func, - self.builtin_type('builtins.function')))) - if item.func.is_property and i == 0: - # This defines a property, probably with a setter and/or deleter. - self.analyze_property_with_multi_part_definition(defn) - break - if not [dec for dec in item.decorators - if refers_to_fullname(dec, 'typing.overload')]: - self.fail("'overload' decorator expected", item) - - defn.type = Overloaded(t) - defn.type.line = defn.line - - if self.is_class_scope(): - self.type.names[defn.name()] = SymbolTableNode(MDEF, defn, - typ=defn.type) - defn.info = self.type - elif self.is_func_scope(): - self.add_local(defn, defn) - - def analyze_property_with_multi_part_definition(self, defn: OverloadedFuncDef) -> None: - """Analyze a property defined using multiple methods (e.g., using @x.setter). - - Assume that the first method (@property) has already been analyzed. - """ - defn.is_property = True - items = defn.items - for item in items[1:]: - if len(item.decorators) == 1: - node = item.decorators[0] - if isinstance(node, MemberExpr): - if node.name == 'setter': - # The first item represents the entire property. - defn.items[0].var.is_settable_property = True - # Get abstractness from the original definition. - item.func.is_abstract = items[0].func.is_abstract - else: - self.fail("Decorated property not supported", item) - item.func.accept(self) - - def next_function_tvar_id(self) -> int: - return self.next_function_tvar_id_stack[-1] - - def analyze_function(self, defn: FuncItem) -> None: - is_method = self.is_class_scope() - - tvarnodes = self.add_func_type_variables_to_symbol_table(defn) - next_function_tvar_id = min([self.next_function_tvar_id()] + - [n.tvar_def.id.raw_id - 1 for n in tvarnodes]) - self.next_function_tvar_id_stack.append(next_function_tvar_id) - - if defn.type: - # Signature must be analyzed in the surrounding scope so that - # class-level imported names and type variables are in scope. - defn.type = self.anal_type(defn.type) - self.check_function_signature(defn) - if isinstance(defn, FuncDef): - defn.type = set_callable_name(defn.type, defn) - for arg in defn.arguments: - if arg.initializer: - arg.initializer.accept(self) - self.function_stack.append(defn) - self.enter() - for arg in defn.arguments: - self.add_local(arg.variable, defn) - for arg in defn.arguments: - if arg.initialization_statement: - lvalue = arg.initialization_statement.lvalues[0] - lvalue.accept(self) - - # The first argument of a non-static, non-class method is like 'self' - # (though the name could be different), having the enclosing class's - # instance type. - if is_method and not defn.is_static and not defn.is_class and defn.arguments: - defn.arguments[0].variable.is_self = True - - # First analyze body of the function but ignore nested functions. - self.postpone_nested_functions_stack.append(FUNCTION_FIRST_PHASE_POSTPONE_SECOND) - self.postponed_functions_stack.append([]) - defn.body.accept(self) - - # Analyze nested functions (if any) as a second phase. - self.postpone_nested_functions_stack[-1] = FUNCTION_SECOND_PHASE - for postponed in self.postponed_functions_stack[-1]: - postponed.accept(self) - self.postpone_nested_functions_stack.pop() - self.postponed_functions_stack.pop() - - self.next_function_tvar_id_stack.pop() - disable_typevars(tvarnodes) - - self.leave() - self.function_stack.pop() - - def add_func_type_variables_to_symbol_table( - self, defn: FuncItem) -> List[SymbolTableNode]: - nodes = [] # type: List[SymbolTableNode] - if defn.type: - tt = defn.type - names = self.type_var_names() - items = cast(CallableType, tt).variables - for item in items: - name = item.name - if name in names: - self.name_already_defined(name, defn) - node = self.bind_type_var(name, item, defn) - nodes.append(node) - names.add(name) - return nodes - - def type_var_names(self) -> Set[str]: - if not self.type: - return set() - else: - return set(self.type.type_vars) - - def bind_type_var(self, fullname: str, tvar_def: TypeVarDef, - context: Context) -> SymbolTableNode: - node = self.lookup_qualified(fullname, context) - node.kind = BOUND_TVAR - node.tvar_def = tvar_def - return node - - def check_function_signature(self, fdef: FuncItem) -> None: - sig = cast(CallableType, fdef.type) - if len(sig.arg_types) < len(fdef.arguments): - self.fail('Type signature has too few arguments', fdef) - # Add dummy Any arguments to prevent crashes later. - extra_anys = [AnyType()] * (len(fdef.arguments) - len(sig.arg_types)) - sig.arg_types.extend(extra_anys) - elif len(sig.arg_types) > len(fdef.arguments): - self.fail('Type signature has too many arguments', fdef) - - def visit_class_def(self, defn: ClassDef) -> None: - self.clean_up_bases_and_infer_type_variables(defn) - self.setup_class_def_analysis(defn) - - self.bind_class_type_vars(defn) - - self.analyze_base_classes(defn) - self.analyze_metaclass(defn) - - for decorator in defn.decorators: - self.analyze_class_decorator(defn, decorator) - - self.enter_class(defn) - - self.setup_is_builtinclass(defn) - - # Analyze class body. - defn.defs.accept(self) - - self.calculate_abstract_status(defn.info) - self.setup_type_promotion(defn) - - self.leave_class() - self.unbind_class_type_vars() - - def enter_class(self, defn: ClassDef) -> None: - # Remember previous active class - self.type_stack.append(self.type) - self.locals.append(None) # Add class scope - self.block_depth.append(-1) # The class body increments this to 0 - self.postpone_nested_functions_stack.append(FUNCTION_BOTH_PHASES) - self.type = defn.info - - def leave_class(self) -> None: - """ Restore analyzer state. """ - self.postpone_nested_functions_stack.pop() - self.block_depth.pop() - self.locals.pop() - self.type = self.type_stack.pop() - - def bind_class_type_vars(self, defn: ClassDef) -> None: - """ Unbind type variables of previously active class and bind - the type variables for the active class. - """ - if self.bound_tvars: - disable_typevars(self.bound_tvars) - self.tvar_stack.append(self.bound_tvars) - self.bound_tvars = self.bind_class_type_variables_in_symbol_table(defn.info) - - def unbind_class_type_vars(self) -> None: - """ Unbind the active class' type vars and rebind the - type vars of the previously active class. - """ - disable_typevars(self.bound_tvars) - self.bound_tvars = self.tvar_stack.pop() - if self.bound_tvars: - enable_typevars(self.bound_tvars) - - def analyze_class_decorator(self, defn: ClassDef, decorator: Node) -> None: - decorator.accept(self) - - def setup_is_builtinclass(self, defn: ClassDef): - for decorator in defn.decorators: - if refers_to_fullname(decorator, 'typing.builtinclass'): - defn.is_builtinclass = True - if defn.fullname == 'builtins.object': - # Only 'object' is marked as a built-in class, as otherwise things elsewhere - # would break. We need a better way of dealing with built-in classes. - defn.is_builtinclass = True - - def calculate_abstract_status(self, typ: TypeInfo) -> None: - """Calculate abstract status of a class. - - Set is_abstract of the type to True if the type has an unimplemented - abstract attribute. Also compute a list of abstract attributes. - """ - concrete = set() # type: Set[str] - abstract = [] # type: List[str] - for base in typ.mro: - for name, symnode in base.names.items(): - node = symnode.node - if isinstance(node, OverloadedFuncDef): - # Unwrap an overloaded function definition. We can just - # check arbitrarily the first overload item. If the - # different items have a different abstract status, there - # should be an error reported elsewhere. - func = node.items[0] # type: Node - else: - func = node - if isinstance(func, Decorator): - fdef = func.func - if fdef.is_abstract and name not in concrete: - typ.is_abstract = True - abstract.append(name) - concrete.add(name) - typ.abstract_attributes = sorted(abstract) - - def setup_type_promotion(self, defn: ClassDef) -> None: - """Setup extra, ad-hoc subtyping relationships between classes (promotion). - - This includes things like 'int' being compatible with 'float'. - """ - promote_target = None # type: Type - for decorator in defn.decorators: - if isinstance(decorator, CallExpr): - analyzed = decorator.analyzed - if isinstance(analyzed, PromoteExpr): - # _promote class decorator (undocumented faeture). - promote_target = analyzed.type - if not promote_target: - promotions = (TYPE_PROMOTIONS_PYTHON3 if self.options.python_version[0] >= 3 - else TYPE_PROMOTIONS_PYTHON2) - if defn.fullname in promotions: - promote_target = self.named_type_or_none(promotions[defn.fullname]) - defn.info._promote = promote_target - - def clean_up_bases_and_infer_type_variables(self, defn: ClassDef) -> None: - """Remove extra base classes such as Generic and infer type vars. - - For example, consider this class: - - . class Foo(Bar, Generic[T]): ... - - Now we will remove Generic[T] from bases of Foo and infer that the - type variable 'T' is a type argument of Foo. - - Note that this is performed *before* semantic analysis. - """ - removed = [] # type: List[int] - type_vars = [] # type: List[TypeVarDef] - for i, base_expr in enumerate(defn.base_type_exprs): - try: - base = expr_to_unanalyzed_type(base_expr) - except TypeTranslationError: - # This error will be caught later. - continue - tvars = self.analyze_typevar_declaration(base) - if tvars is not None: - if type_vars: - self.fail('Duplicate Generic in bases', defn) - removed.append(i) - for j, (name, tvar_expr) in enumerate(tvars): - type_vars.append(TypeVarDef(name, j + 1, tvar_expr.values, - tvar_expr.upper_bound, tvar_expr.variance)) - if type_vars: - defn.type_vars = type_vars - if defn.info: - defn.info.type_vars = [tv.name for tv in type_vars] - for i in reversed(removed): - del defn.base_type_exprs[i] - - def analyze_typevar_declaration(self, t: Type) -> Optional[List[Tuple[str, TypeVarExpr]]]: - if not isinstance(t, UnboundType): - return None - unbound = t - sym = self.lookup_qualified(unbound.name, unbound) - if sym is None or sym.node is None: - return None - if sym.node.fullname() == 'typing.Generic': - tvars = [] # type: List[Tuple[str, TypeVarExpr]] - for arg in unbound.args: - tvar = self.analyze_unbound_tvar(arg) - if tvar: - tvars.append(tvar) - else: - self.fail('Free type variable expected in %s[...]' % - sym.node.name(), t) - return tvars - return None - - def analyze_unbound_tvar(self, t: Type) -> Tuple[str, TypeVarExpr]: - if not isinstance(t, UnboundType): - return None - unbound = t - sym = self.lookup_qualified(unbound.name, unbound) - if sym is not None and sym.kind == UNBOUND_TVAR: - return unbound.name, cast(TypeVarExpr, sym.node) - return None - - def setup_class_def_analysis(self, defn: ClassDef) -> None: - """Prepare for the analysis of a class definition.""" - if not defn.info: - defn.info = TypeInfo(SymbolTable(), defn) - defn.info._fullname = defn.info.name() - if self.is_func_scope() or self.type: - kind = MDEF - if self.is_func_scope(): - kind = LDEF - self.add_symbol(defn.name, SymbolTableNode(kind, defn.info), defn) - - def analyze_base_classes(self, defn: ClassDef) -> None: - """Analyze and set up base classes. - - This computes several attributes on the corresponding TypeInfo defn.info - related to the base classes: defn.info.bases, defn.info.mro, and - miscellaneous others (at least tuple_type, fallback_to_any, and is_enum.) - """ - - base_types = [] # type: List[Instance] - info = defn.info - for base_expr in defn.base_type_exprs: - try: - base = self.expr_to_analyzed_type(base_expr) - except TypeTranslationError: - self.fail('Invalid base class', base_expr) - info.fallback_to_any = True - continue - - if isinstance(base, TupleType): - if info.tuple_type: - self.fail("Class has two incompatible bases derived from tuple", defn) - defn.incompatible_baseclass = True - if (not self.is_stub_file - and not info.is_named_tuple - and base.fallback.type.fullname() == 'builtins.tuple'): - self.fail("Tuple[...] not supported as a base class outside a stub file", defn) - info.tuple_type = base - base_types.append(base.fallback) - elif isinstance(base, Instance): - base_types.append(base) - elif isinstance(base, AnyType): - info.fallback_to_any = True - else: - self.fail('Invalid base class', base_expr) - info.fallback_to_any = True - - # Add 'object' as implicit base if there is no other base class. - if (not base_types and defn.fullname != 'builtins.object'): - base_types.append(self.object_type()) - - info.bases = base_types - - # Calculate the MRO. It might be incomplete at this point if - # the bases of defn include classes imported from other - # modules in an import loop. We'll recompute it in ThirdPass. - if not self.verify_base_classes(defn): - info.mro = [] - return - calculate_class_mro(defn, self.fail) - # If there are cyclic imports, we may be missing 'object' in - # the MRO. Fix MRO if needed. - if info.mro and info.mro[-1].fullname() != 'builtins.object': - info.mro.append(self.object_type().type) - - def expr_to_analyzed_type(self, expr: Node) -> Type: - if isinstance(expr, CallExpr): - expr.accept(self) - info = self.check_namedtuple(expr) - if info is None: - # Some form of namedtuple is the only valid type that looks like a call - # expression. This isn't a valid type. - raise TypeTranslationError() - fallback = Instance(info, []) - return TupleType(info.tuple_type.items, fallback=fallback) - typ = expr_to_unanalyzed_type(expr) - return self.anal_type(typ) - - def verify_base_classes(self, defn: ClassDef) -> bool: - info = defn.info - for base in info.bases: - baseinfo = base.type - if self.is_base_class(info, baseinfo): - self.fail('Cycle in inheritance hierarchy', defn) - # Clear bases to forcefully get rid of the cycle. - info.bases = [] - if baseinfo.fullname() == 'builtins.bool': - self.fail("'%s' is not a valid base class" % - baseinfo.name(), defn) - return False - dup = find_duplicate(info.direct_base_classes()) - if dup: - self.fail('Duplicate base class "%s"' % dup.name(), defn) - return False - return True - - def is_base_class(self, t: TypeInfo, s: TypeInfo) -> bool: - """Determine if t is a base class of s (but do not use mro).""" - # Search the base class graph for t, starting from s. - worklist = [s] - visited = {s} - while worklist: - nxt = worklist.pop() - if nxt == t: - return True - for base in nxt.bases: - if base.type not in visited: - worklist.append(base.type) - visited.add(base.type) - return False - - def analyze_metaclass(self, defn: ClassDef) -> None: - if defn.metaclass: - sym = self.lookup_qualified(defn.metaclass, defn) - if sym is not None and not isinstance(sym.node, TypeInfo): - self.fail("Invalid metaclass '%s'" % defn.metaclass, defn) - - def object_type(self) -> Instance: - return self.named_type('__builtins__.object') - - def class_type(self, info: TypeInfo) -> Type: - # Construct a function type whose fallback is cls. - from mypy import checkmember # To avoid import cycle. - leading_type = checkmember.type_object_type(info, self.builtin_type) - if isinstance(leading_type, Overloaded): - # Overloaded __init__ is too complex to handle. Plus it's stubs only. - return AnyType() - else: - return leading_type - - def named_type(self, qualified_name: str, args: List[Type] = None) -> Instance: - sym = self.lookup_qualified(qualified_name, None) - return Instance(cast(TypeInfo, sym.node), args or []) - - def named_type_or_none(self, qualified_name: str) -> Instance: - sym = self.lookup_fully_qualified_or_none(qualified_name) - if not sym: - return None - return Instance(cast(TypeInfo, sym.node), []) - - def is_instance_type(self, t: Type) -> bool: - return isinstance(t, Instance) - - def bind_class_type_variables_in_symbol_table( - self, info: TypeInfo) -> List[SymbolTableNode]: - nodes = [] # type: List[SymbolTableNode] - for var, binder in zip(info.type_vars, info.defn.type_vars): - node = self.bind_type_var(var, binder, info) - nodes.append(node) - return nodes - - def visit_import(self, i: Import) -> None: - for id, as_id in i.ids: - if as_id is not None: - self.add_module_symbol(id, as_id, module_public=True, context=i) - else: - # Modules imported in a stub file without using 'as x' won't get exported when - # doing 'from m import *'. - module_public = not self.is_stub_file - base = id.split('.')[0] - self.add_module_symbol(base, base, module_public=module_public, - context=i) - - def add_module_symbol(self, id: str, as_id: str, module_public: bool, - context: Context) -> None: - if id in self.modules: - m = self.modules[id] - self.add_symbol(as_id, SymbolTableNode(MODULE_REF, m, self.cur_mod_id, - module_public=module_public), context) - else: - self.add_unknown_symbol(as_id, context) - - def visit_import_from(self, imp: ImportFrom) -> None: - import_id = self.correct_relative_import(imp) - if import_id in self.modules: - module = self.modules[import_id] - for id, as_id in imp.names: - node = module.names.get(id) - if node and node.kind != UNBOUND_IMPORTED: - node = self.normalize_type_alias(node, imp) - if not node: - return - imported_id = as_id or id - existing_symbol = self.globals.get(imported_id) - if existing_symbol: - # Import can redefine a variable. They get special treatment. - if self.process_import_over_existing_name( - imported_id, existing_symbol, node, imp): - continue - # 'from m import x as x' exports x in a stub file. - module_public = not self.is_stub_file or as_id is not None - symbol = SymbolTableNode(node.kind, node.node, - self.cur_mod_id, - node.type_override, - module_public=module_public) - self.add_symbol(imported_id, symbol, imp) - else: - message = "Module has no attribute '{}'".format(id) - extra = self.undefined_name_extra_info('{}.{}'.format(import_id, id)) - if extra: - message += " {}".format(extra) - self.fail(message, imp) - else: - # Missing module. - for id, as_id in imp.names: - self.add_unknown_symbol(as_id or id, imp) - - def process_import_over_existing_name(self, - imported_id: str, existing_symbol: SymbolTableNode, - module_symbol: SymbolTableNode, - import_node: ImportBase) -> bool: - if (existing_symbol.kind in (LDEF, GDEF, MDEF) and - isinstance(existing_symbol.node, (Var, FuncDef))): - # This is a valid import over an existing definition in the file. Construct a dummy - # assignment that we'll use to type check the import. - lvalue = NameExpr(imported_id) - lvalue.kind = existing_symbol.kind - lvalue.node = existing_symbol.node - rvalue = NameExpr(imported_id) - rvalue.kind = module_symbol.kind - rvalue.node = module_symbol.node - assignment = AssignmentStmt([lvalue], rvalue) - for node in assignment, lvalue, rvalue: - node.set_line(import_node) - import_node.assignments.append(assignment) - return True - return False - - def normalize_type_alias(self, node: SymbolTableNode, - ctx: Context) -> SymbolTableNode: - if node.fullname in type_aliases: - # Node refers to an aliased type such as typing.List; normalize. - node = self.lookup_qualified(type_aliases[node.fullname], ctx) - if node.fullname == 'typing.DefaultDict': - self.add_module_symbol('collections', '__mypy_collections__', False, ctx) - node = self.lookup_qualified('__mypy_collections__.defaultdict', ctx) - return node - - def correct_relative_import(self, node: Union[ImportFrom, ImportAll]) -> str: - if node.relative == 0: - return node.id - - parts = self.cur_mod_id.split(".") - cur_mod_id = self.cur_mod_id - - rel = node.relative - if self.cur_mod_node.is_package_init_file(): - rel -= 1 - if len(parts) < rel: - self.fail("Relative import climbs too many namespaces", node) - if rel != 0: - cur_mod_id = ".".join(parts[:-rel]) - - return cur_mod_id + (("." + node.id) if node.id else "") - - def visit_import_all(self, i: ImportAll) -> None: - i_id = self.correct_relative_import(i) - if i_id in self.modules: - m = self.modules[i_id] - for name, node in m.names.items(): - node = self.normalize_type_alias(node, i) - if not name.startswith('_') and node.module_public: - existing_symbol = self.globals.get(name) - if existing_symbol: - # Import can redefine a variable. They get special treatment. - if self.process_import_over_existing_name( - name, existing_symbol, node, i): - continue - self.add_symbol(name, SymbolTableNode(node.kind, node.node, - self.cur_mod_id), i) - else: - # Don't add any dummy symbols for 'from x import *' if 'x' is unknown. - pass - - def add_unknown_symbol(self, name: str, context: Context) -> None: - var = Var(name) - var._fullname = self.qualified_name(name) - var.is_ready = True - var.type = AnyType() - self.add_symbol(name, SymbolTableNode(GDEF, var, self.cur_mod_id), context) - - # - # Statements - # - - def visit_block(self, b: Block) -> None: - if b.is_unreachable: - return - self.block_depth[-1] += 1 - for s in b.body: - self.accept(s) - self.block_depth[-1] -= 1 - - def visit_block_maybe(self, b: Block) -> None: - if b: - self.visit_block(b) - - def anal_type(self, t: Type, allow_tuple_literal: bool = False) -> Type: - if t: - if allow_tuple_literal: - # Types such as (t1, t2, ...) only allowed in assignment statements. They'll - # generate errors elsewhere, and Tuple[t1, t2, ...] must be used instead. - if isinstance(t, TupleType): - # Unlike TypeAnalyser, also allow implicit tuple types (without Tuple[...]). - star_count = sum(1 for item in t.items if isinstance(item, StarType)) - if star_count > 1: - self.fail('At most one star type allowed in a tuple', t) - return TupleType([AnyType() for _ in t.items], - self.builtin_type('builtins.tuple'), t.line) - items = [self.anal_type(item, True) - for item in t.items] - return TupleType(items, self.builtin_type('builtins.tuple'), t.line) - a = TypeAnalyser(self.lookup_qualified, - self.lookup_fully_qualified, - self.fail) - return t.accept(a) - else: - return None - - def visit_assignment_stmt(self, s: AssignmentStmt) -> None: - for lval in s.lvalues: - self.analyze_lvalue(lval, explicit_type=s.type is not None) - s.rvalue.accept(self) - if s.type: - allow_tuple_literal = isinstance(s.lvalues[-1], (TupleExpr, ListExpr)) - s.type = self.anal_type(s.type, allow_tuple_literal) - else: - # For simple assignments, allow binding type aliases. - # Also set the type if the rvalue is a simple literal. - if (s.type is None and len(s.lvalues) == 1 and - isinstance(s.lvalues[0], NameExpr)): - if s.lvalues[0].is_def: - s.type = self.analyze_simple_literal_type(s.rvalue) - res = analyze_type_alias(s.rvalue, - self.lookup_qualified, - self.lookup_fully_qualified, - self.fail) - if res and (not isinstance(res, Instance) or cast(Instance, res).args): - # TODO: What if this gets reassigned? - name = s.lvalues[0] - node = self.lookup(name.name, name) - node.kind = TYPE_ALIAS - node.type_override = res - if isinstance(s.rvalue, IndexExpr): - s.rvalue.analyzed = TypeAliasExpr(res) - if s.type: - # Store type into nodes. - for lvalue in s.lvalues: - self.store_declared_types(lvalue, s.type) - self.check_and_set_up_type_alias(s) - self.process_typevar_declaration(s) - self.process_namedtuple_definition(s) - - if (len(s.lvalues) == 1 and isinstance(s.lvalues[0], NameExpr) and - s.lvalues[0].name == '__all__' and s.lvalues[0].kind == GDEF and - isinstance(s.rvalue, (ListExpr, TupleExpr))): - self.add_exports(*s.rvalue.items) - - def analyze_simple_literal_type(self, rvalue: Node) -> Optional[Type]: - """Return builtins.int if rvalue is an int literal, etc.""" - if self.weak_opts or self.options.semantic_analysis_only or self.function_stack: - # Skip this if any weak options are set. - # Also skip if we're only doing the semantic analysis pass. - # This is mostly to avoid breaking unit tests. - # Also skip inside a function; this is to avoid confusing - # the code that handles dead code due to isinstance() - # inside type variables with value restrictions (like - # AnyStr). - return None - if isinstance(rvalue, IntExpr): - return self.named_type_or_none('builtins.int') - if isinstance(rvalue, FloatExpr): - return self.named_type_or_none('builtins.float') - if isinstance(rvalue, StrExpr): - return self.named_type_or_none('builtins.str') - if isinstance(rvalue, BytesExpr): - return self.named_type_or_none('builtins.bytes') - if isinstance(rvalue, UnicodeExpr): - return self.named_type_or_none('builtins.unicode') - return None - - def check_and_set_up_type_alias(self, s: AssignmentStmt) -> None: - """Check if assignment creates a type alias and set it up as needed.""" - # For now, type aliases only work at the top level of a module. - if (len(s.lvalues) == 1 and not self.is_func_scope() and not self.type - and not s.type): - lvalue = s.lvalues[0] - if isinstance(lvalue, NameExpr): - if not lvalue.is_def: - # Only a definition can create a type alias, not regular assignment. - return - rvalue = s.rvalue - if isinstance(rvalue, RefExpr): - node = rvalue.node - if isinstance(node, TypeInfo): - # TODO: We should record the fact that this is a variable - # that refers to a type, rather than making this - # just an alias for the type. - self.globals[lvalue.name].node = node - - def analyze_lvalue(self, lval: Node, nested: bool = False, - add_global: bool = False, - explicit_type: bool = False) -> None: - """Analyze an lvalue or assignment target. - - Only if add_global is True, add name to globals table. If nested - is true, the lvalue is within a tuple or list lvalue expression. - """ - - if isinstance(lval, NameExpr): - # Top-level definitions within some statements (at least while) are - # not handled in the first pass, so they have to be added now. - nested_global = (not self.is_func_scope() and - self.block_depth[-1] > 0 and - not self.type) - if (add_global or nested_global) and lval.name not in self.globals: - # Define new global name. - v = Var(lval.name) - v._fullname = self.qualified_name(lval.name) - v.is_ready = False # Type not inferred yet - lval.node = v - lval.is_def = True - lval.kind = GDEF - lval.fullname = v._fullname - self.globals[lval.name] = SymbolTableNode(GDEF, v, - self.cur_mod_id) - elif isinstance(lval.node, Var) and lval.is_def: - # Since the is_def flag is set, this must have been analyzed - # already in the first pass and added to the symbol table. - assert lval.node.name() in self.globals - elif (self.is_func_scope() and lval.name not in self.locals[-1] and - lval.name not in self.global_decls[-1] and - lval.name not in self.nonlocal_decls[-1]): - # Define new local name. - v = Var(lval.name) - lval.node = v - lval.is_def = True - lval.kind = LDEF - lval.fullname = lval.name - self.add_local(v, lval) - elif not self.is_func_scope() and (self.type and - lval.name not in self.type.names): - # Define a new attribute within class body. - v = Var(lval.name) - v.info = self.type - v.is_initialized_in_class = True - v.set_line(lval) - lval.node = v - lval.is_def = True - lval.kind = MDEF - lval.fullname = lval.name - self.type.names[lval.name] = SymbolTableNode(MDEF, v) - else: - # Bind to an existing name. - if explicit_type: - self.name_already_defined(lval.name, lval) - lval.accept(self) - self.check_lvalue_validity(lval.node, lval) - elif isinstance(lval, MemberExpr): - if not add_global: - self.analyze_member_lvalue(lval) - if explicit_type and not self.is_self_member_ref(lval): - self.fail('Type cannot be declared in assignment to non-self ' - 'attribute', lval) - elif isinstance(lval, IndexExpr): - if explicit_type: - self.fail('Unexpected type declaration', lval) - if not add_global: - lval.accept(self) - elif (isinstance(lval, TupleExpr) or - isinstance(lval, ListExpr)): - items = cast(Any, lval).items - if len(items) == 0 and isinstance(lval, TupleExpr): - self.fail("Can't assign to ()", lval) - self.analyze_tuple_or_list_lvalue(cast(Union[ListExpr, TupleExpr], lval), - add_global, explicit_type) - elif isinstance(lval, StarExpr): - if nested: - self.analyze_lvalue(lval.expr, nested, add_global, explicit_type) - else: - self.fail('Starred assignment target must be in a list or tuple', lval) - else: - self.fail('Invalid assignment target', lval) - - def analyze_tuple_or_list_lvalue(self, lval: Union[ListExpr, TupleExpr], - add_global: bool = False, - explicit_type: bool = False) -> None: - """Analyze an lvalue or assignment target that is a list or tuple.""" - items = lval.items - star_exprs = [cast(StarExpr, item) - for item in items - if isinstance(item, StarExpr)] - - if len(star_exprs) > 1: - self.fail('Two starred expressions in assignment', lval) - else: - if len(star_exprs) == 1: - star_exprs[0].valid = True - for i in items: - self.analyze_lvalue(i, nested=True, add_global=add_global, - explicit_type = explicit_type) - - def analyze_member_lvalue(self, lval: MemberExpr) -> None: - lval.accept(self) - if (self.is_self_member_ref(lval) and - self.type.get(lval.name) is None): - # Implicit attribute definition in __init__. - lval.is_def = True - v = Var(lval.name) - v.info = self.type - v.is_ready = False - lval.def_var = v - lval.node = v - self.type.names[lval.name] = SymbolTableNode(MDEF, v) - self.check_lvalue_validity(lval.node, lval) - - def is_self_member_ref(self, memberexpr: MemberExpr) -> bool: - """Does memberexpr to refer to an attribute of self?""" - if not isinstance(memberexpr.expr, NameExpr): - return False - node = memberexpr.expr.node - return isinstance(node, Var) and node.is_self - - def check_lvalue_validity(self, node: Node, ctx: Context) -> None: - if isinstance(node, (TypeInfo, TypeVarExpr)): - self.fail('Invalid assignment target', ctx) - - def store_declared_types(self, lvalue: Node, typ: Type) -> None: - if isinstance(typ, StarType) and not isinstance(lvalue, StarExpr): - self.fail('Star type only allowed for starred expressions', lvalue) - if isinstance(lvalue, RefExpr): - lvalue.is_def = False - if isinstance(lvalue.node, Var): - var = lvalue.node - var.type = typ - var.is_ready = True - # If node is not a variable, we'll catch it elsewhere. - elif isinstance(lvalue, TupleExpr): - if isinstance(typ, TupleType): - if len(lvalue.items) != len(typ.items): - self.fail('Incompatible number of tuple items', lvalue) - return - for item, itemtype in zip(lvalue.items, typ.items): - self.store_declared_types(item, itemtype) - else: - self.fail('Tuple type expected for multiple variables', - lvalue) - elif isinstance(lvalue, StarExpr): - if isinstance(typ, StarType): - self.store_declared_types(lvalue.expr, typ.type) - else: - self.fail('Star type expected for starred expression', lvalue) - else: - # This has been flagged elsewhere as an error, so just ignore here. - pass - - def process_typevar_declaration(self, s: AssignmentStmt) -> None: - """Check if s declares a TypeVar; it yes, store it in symbol table.""" - call = self.get_typevar_declaration(s) - if not call: - return - - lvalue = cast(NameExpr, s.lvalues[0]) - name = lvalue.name - if not lvalue.is_def: - if s.type: - self.fail("Cannot declare the type of a type variable", s) - else: - self.fail("Cannot redefine '%s' as a type variable" % name, s) - return - - if not self.check_typevar_name(call, name, s): - return - - # Constraining types - n_values = call.arg_kinds[1:].count(ARG_POS) - values = self.analyze_types(call.args[1:1 + n_values]) - - res = self.process_typevar_parameters(call.args[1 + n_values:], - call.arg_names[1 + n_values:], - call.arg_kinds[1 + n_values:], - bool(values), - s) - if res is None: - return - variance, upper_bound = res - - # Yes, it's a valid type variable definition! Add it to the symbol table. - node = self.lookup(name, s) - node.kind = UNBOUND_TVAR - TypeVar = TypeVarExpr(name, node.fullname, values, upper_bound, variance) - TypeVar.line = call.line - call.analyzed = TypeVar - node.node = TypeVar - - def check_typevar_name(self, call: CallExpr, name: str, context: Context) -> bool: - if len(call.args) < 1: - self.fail("Too few arguments for TypeVar()", context) - return False - if not isinstance(call.args[0], (StrExpr, BytesExpr)) or not call.arg_kinds[0] == ARG_POS: - self.fail("TypeVar() expects a string literal as first argument", context) - return False - if cast(StrExpr, call.args[0]).value != name: - self.fail("Unexpected TypeVar() argument value", context) - return False - return True - - def get_typevar_declaration(self, s: AssignmentStmt) -> Optional[CallExpr]: - """Returns the TypeVar() call expression if `s` is a type var declaration - or None otherwise. - """ - if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): - return None - if not isinstance(s.rvalue, CallExpr): - return None - call = s.rvalue - if not isinstance(call.callee, RefExpr): - return None - callee = call.callee - if callee.fullname != 'typing.TypeVar': - return None - return call - - def process_typevar_parameters(self, args: List[Node], - names: List[Optional[str]], - kinds: List[int], - has_values: bool, - context: Context) -> Optional[Tuple[int, Type]]: - covariant = False - contravariant = False - upper_bound = self.object_type() # type: Type - for param_value, param_name, param_kind in zip(args, names, kinds): - if not param_kind == ARG_NAMED: - self.fail("Unexpected argument to TypeVar()", context) - return None - if param_name == 'covariant': - if isinstance(param_value, NameExpr): - if param_value.name == 'True': - covariant = True - else: - self.fail("TypeVar 'covariant' may only be 'True'", context) - return None - else: - self.fail("TypeVar 'covariant' may only be 'True'", context) - return None - elif param_name == 'contravariant': - if isinstance(param_value, NameExpr): - if param_value.name == 'True': - contravariant = True - else: - self.fail("TypeVar 'contravariant' may only be 'True'", context) - return None - else: - self.fail("TypeVar 'contravariant' may only be 'True'", context) - return None - elif param_name == 'bound': - if has_values: - self.fail("TypeVar cannot have both values and an upper bound", context) - return None - try: - upper_bound = self.expr_to_analyzed_type(param_value) - except TypeTranslationError: - self.fail("TypeVar 'bound' must be a type", param_value) - return None - elif param_name == 'values': - # Probably using obsolete syntax with values=(...). Explain the current syntax. - self.fail("TypeVar 'values' argument not supported", context) - self.fail("Use TypeVar('T', t, ...) instead of TypeVar('T', values=(t, ...))", - context) - return None - else: - self.fail("Unexpected argument to TypeVar(): {}".format(param_name), context) - return None - - if covariant and contravariant: - self.fail("TypeVar cannot be both covariant and contravariant", context) - return None - elif covariant: - variance = COVARIANT - elif contravariant: - variance = CONTRAVARIANT - else: - variance = INVARIANT - return (variance, upper_bound) - - def process_namedtuple_definition(self, s: AssignmentStmt) -> None: - """Check if s defines a namedtuple; if yes, store the definition in symbol table.""" - if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): - return - lvalue = cast(NameExpr, s.lvalues[0]) - name = lvalue.name - named_tuple = self.check_namedtuple(s.rvalue, name) - if named_tuple is None: - return - # Yes, it's a valid namedtuple definition. Add it to the symbol table. - node = self.lookup(name, s) - node.kind = GDEF # TODO locally defined namedtuple - # TODO call.analyzed - node.node = named_tuple - - def check_namedtuple(self, node: Node, var_name: str = None) -> TypeInfo: - """Check if a call defines a namedtuple. - - The optional var_name argument is the name of the variable to - which this is assigned, if any. - - If it does, return the corresponding TypeInfo. Return None otherwise. - - If the definition is invalid but looks like a namedtuple, - report errors but return (some) TypeInfo. - """ - if not isinstance(node, CallExpr): - return None - call = node - if not isinstance(call.callee, RefExpr): - return None - callee = call.callee - fullname = callee.fullname - if fullname not in ('collections.namedtuple', 'typing.NamedTuple'): - return None - items, types, ok = self.parse_namedtuple_args(call, fullname) - if not ok: - # Error. Construct dummy return value. - return self.build_namedtuple_typeinfo('namedtuple', [], []) - else: - # Give it a unique name derived from the line number. - name = cast(StrExpr, call.args[0]).value - if name != var_name: - name += '@' + str(call.line) - info = self.build_namedtuple_typeinfo(name, items, types) - # Store it as a global just in case it would remain anonymous. - self.globals[name] = SymbolTableNode(GDEF, info, self.cur_mod_id) - call.analyzed = NamedTupleExpr(info).set_line(call.line) - return info - - def parse_namedtuple_args(self, call: CallExpr, - fullname: str) -> Tuple[List[str], List[Type], bool]: - # TODO Share code with check_argument_count in checkexpr.py? - args = call.args - if len(args) < 2: - return self.fail_namedtuple_arg("Too few arguments for namedtuple()", call) - if len(args) > 2: - return self.fail_namedtuple_arg("Too many arguments for namedtuple()", call) - if call.arg_kinds != [ARG_POS, ARG_POS]: - return self.fail_namedtuple_arg("Unexpected arguments to namedtuple()", call) - if not isinstance(args[0], (StrExpr, BytesExpr)): - return self.fail_namedtuple_arg( - "namedtuple() expects a string literal as the first argument", call) - types = [] # type: List[Type] - ok = True - if not isinstance(args[1], ListExpr): - if fullname == 'collections.namedtuple' and isinstance(args[1], (StrExpr, BytesExpr)): - str_expr = cast(StrExpr, args[1]) - items = str_expr.value.split() - else: - return self.fail_namedtuple_arg( - "List literal expected as the second argument to namedtuple()", call) - else: - listexpr = args[1] - if fullname == 'collections.namedtuple': - # The fields argument contains just names, with implicit Any types. - if any(not isinstance(item, (StrExpr, BytesExpr)) for item in listexpr.items): - return self.fail_namedtuple_arg("String literal expected as namedtuple() item", - call) - items = [cast(StrExpr, item).value for item in listexpr.items] - else: - # The fields argument contains (name, type) tuples. - items, types, ok = self.parse_namedtuple_fields_with_types(listexpr.items, call) - if not types: - types = [AnyType() for _ in items] - return items, types, ok - - def parse_namedtuple_fields_with_types(self, nodes: List[Node], - context: Context) -> Tuple[List[str], List[Type], bool]: - items = [] # type: List[str] - types = [] # type: List[Type] - for item in nodes: - if isinstance(item, TupleExpr): - if len(item.items) != 2: - return self.fail_namedtuple_arg("Invalid NamedTuple field definition", - item) - name, type_node = item.items - if isinstance(name, (StrExpr, BytesExpr)): - items.append(name.value) - else: - return self.fail_namedtuple_arg("Invalid NamedTuple() field name", item) - try: - type = expr_to_unanalyzed_type(type_node) - except TypeTranslationError: - return self.fail_namedtuple_arg('Invalid field type', type_node) - types.append(self.anal_type(type)) - else: - return self.fail_namedtuple_arg("Tuple expected as NamedTuple() field", item) - return items, types, True - - def fail_namedtuple_arg(self, message: str, - context: Context) -> Tuple[List[str], List[Type], bool]: - self.fail(message, context) - return [], [], False - - def build_namedtuple_typeinfo(self, name: str, items: List[str], - types: List[Type]) -> TypeInfo: - symbols = SymbolTable() - tup = NamedTupleType(items, types, self.named_type('__builtins__.tuple', types)) - class_def = ClassDef(name, Block([])) - class_def.fullname = self.qualified_name(name) - info = NamedTupleTypeInfo(tup, symbols, class_def) - vars = [Var(item, typ) for item, typ in zip(items, types)] - # Add named tuple items as attributes. - # TODO: Make them read-only. - for var in vars: - var.info = info - symbols[var.name()] = SymbolTableNode(MDEF, var) - # Add __init__, _replace, _asdict methods and _fields static field - self.add_namedtuple_method('__init__', symbols, info, NoneTyp(), - self.make_factory_args(vars, ARG_POS)) - self.add_namedtuple_method('_replace', symbols, info, self_type(info), - self.make_factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) - # TODO: refine to OrderedDict[Union[types]] - self.add_namedtuple_method('_asdict', symbols, info, AnyType()) - return info - - def make_factory_args(self, vars: List[Var], kind: int, - initializer: Expression = None) -> List[Argument]: - return [Argument(var, var.type, initializer, kind) for var in vars] - - def add_namedtuple_method(self, funcname: str, symbols: SymbolTable, info: TypeInfo, - ret: Type, args: List[Argument] = []) -> None: - types = [cast(Type, None)] + [arg.type_annotation for arg in args] - args = [Argument(Var('__self'), NoneTyp(), None, ARG_POS)] + args - items = [arg.variable.name() for arg in args] - arg_kinds = [arg.kind for arg in args] - # TODO: Make sure that the self argument name is not visible? - signature = CallableType(types, arg_kinds, items, ret, - self.named_type('__builtins__.function'), - name=info.name()) - func = FuncDef(funcname, - args, - Block([]), - typ=signature) - func.info = info - symbols[funcname] = SymbolTableNode(MDEF, func) - - def analyze_types(self, items: List[Node]) -> List[Type]: - result = [] # type: List[Type] - for node in items: - try: - result.append(self.anal_type(expr_to_unanalyzed_type(node))) - except TypeTranslationError: - self.fail('Type expected', node) - result.append(AnyType()) - return result - - def visit_decorator(self, dec: Decorator) -> None: - for d in dec.decorators: - d.accept(self) - removed = [] # type: List[int] - no_type_check = False - for i, d in enumerate(dec.decorators): - # A bunch of decorators are special cased here. - if refers_to_fullname(d, 'abc.abstractmethod'): - removed.append(i) - dec.func.is_abstract = True - self.check_decorated_function_is_method('abstractmethod', dec) - elif refers_to_fullname(d, 'asyncio.tasks.coroutine'): - removed.append(i) - elif refers_to_fullname(d, 'builtins.staticmethod'): - removed.append(i) - dec.func.is_static = True - dec.var.is_staticmethod = True - self.check_decorated_function_is_method('staticmethod', dec) - elif refers_to_fullname(d, 'builtins.classmethod'): - removed.append(i) - dec.func.is_class = True - dec.var.is_classmethod = True - self.check_decorated_function_is_method('classmethod', dec) - elif (refers_to_fullname(d, 'builtins.property') or - refers_to_fullname(d, 'abc.abstractproperty')): - removed.append(i) - dec.func.is_property = True - dec.var.is_property = True - if refers_to_fullname(d, 'abc.abstractproperty'): - dec.func.is_abstract = True - self.check_decorated_function_is_method('property', dec) - if len(dec.func.arguments) > 1: - self.fail('Too many arguments', dec.func) - elif refers_to_fullname(d, 'typing.no_type_check'): - dec.var.type = AnyType() - no_type_check = True - for i in reversed(removed): - del dec.decorators[i] - if not dec.is_overload or dec.var.is_property: - if self.is_func_scope(): - self.add_symbol(dec.var.name(), SymbolTableNode(LDEF, dec), - dec) - elif self.type: - dec.var.info = self.type - dec.var.is_initialized_in_class = True - self.add_symbol(dec.var.name(), SymbolTableNode(MDEF, dec), - dec) - if not no_type_check: - dec.func.accept(self) - if dec.decorators and dec.var.is_property: - self.fail('Decorated property not supported', dec) - - def check_decorated_function_is_method(self, decorator: str, - context: Context) -> None: - if not self.type or self.is_func_scope(): - self.fail("'%s' used with a non-method" % decorator, context) - - def visit_expression_stmt(self, s: ExpressionStmt) -> None: - s.expr.accept(self) - - def visit_return_stmt(self, s: ReturnStmt) -> None: - if not self.is_func_scope(): - self.fail("'return' outside function", s) - if s.expr: - s.expr.accept(self) - - def visit_raise_stmt(self, s: RaiseStmt) -> None: - if s.expr: - s.expr.accept(self) - if s.from_expr: - s.from_expr.accept(self) - - def visit_assert_stmt(self, s: AssertStmt) -> None: - if s.expr: - s.expr.accept(self) - - def visit_operator_assignment_stmt(self, - s: OperatorAssignmentStmt) -> None: - s.lvalue.accept(self) - s.rvalue.accept(self) - if (isinstance(s.lvalue, NameExpr) and s.lvalue.name == '__all__' and - s.lvalue.kind == GDEF and isinstance(s.rvalue, (ListExpr, TupleExpr))): - self.add_exports(*s.rvalue.items) - - def visit_while_stmt(self, s: WhileStmt) -> None: - s.expr.accept(self) - self.loop_depth += 1 - s.body.accept(self) - self.loop_depth -= 1 - self.visit_block_maybe(s.else_body) - - def visit_for_stmt(self, s: ForStmt) -> None: - s.expr.accept(self) - - # Bind index variables and check if they define new names. - self.analyze_lvalue(s.index) - - self.loop_depth += 1 - self.visit_block(s.body) - self.loop_depth -= 1 - - self.visit_block_maybe(s.else_body) - - def visit_break_stmt(self, s: BreakStmt) -> None: - if self.loop_depth == 0: - self.fail("'break' outside loop", s, True, blocker=True) - - def visit_continue_stmt(self, s: ContinueStmt) -> None: - if self.loop_depth == 0: - self.fail("'continue' outside loop", s, True, blocker=True) - - def visit_if_stmt(self, s: IfStmt) -> None: - infer_reachability_of_if_statement(s, pyversion=self.options.python_version) - for i in range(len(s.expr)): - s.expr[i].accept(self) - self.visit_block(s.body[i]) - self.visit_block_maybe(s.else_body) - - def visit_try_stmt(self, s: TryStmt) -> None: - self.analyze_try_stmt(s, self) - - def analyze_try_stmt(self, s: TryStmt, visitor: NodeVisitor, - add_global: bool = False) -> None: - s.body.accept(visitor) - for type, var, handler in zip(s.types, s.vars, s.handlers): - if type: - type.accept(visitor) - if var: - self.analyze_lvalue(var, add_global=add_global) - handler.accept(visitor) - if s.else_body: - s.else_body.accept(visitor) - if s.finally_body: - s.finally_body.accept(visitor) - - def visit_with_stmt(self, s: WithStmt) -> None: - for e, n in zip(s.expr, s.target): - e.accept(self) - if n: - self.analyze_lvalue(n) - self.visit_block(s.body) - - def visit_del_stmt(self, s: DelStmt) -> None: - s.expr.accept(self) - if not self.is_valid_del_target(s.expr): - self.fail('Invalid delete target', s) - - def is_valid_del_target(self, s: Node) -> bool: - if isinstance(s, (IndexExpr, NameExpr, MemberExpr)): - return True - elif isinstance(s, TupleExpr): - return all(self.is_valid_del_target(item) for item in s.items) - - def visit_global_decl(self, g: GlobalDecl) -> None: - for name in g.names: - if name in self.nonlocal_decls[-1]: - self.fail("Name '{}' is nonlocal and global".format(name), g) - self.global_decls[-1].add(name) - - def visit_nonlocal_decl(self, d: NonlocalDecl) -> None: - if not self.is_func_scope(): - self.fail("nonlocal declaration not allowed at module level", d) - else: - for name in d.names: - for table in reversed(self.locals[:-1]): - if table is not None and name in table: - break - else: - self.fail("No binding for nonlocal '{}' found".format(name), d) - - if self.locals[-1] is not None and name in self.locals[-1]: - self.fail("Name '{}' is already defined in local " - "scope before nonlocal declaration".format(name), d) - - if name in self.global_decls[-1]: - self.fail("Name '{}' is nonlocal and global".format(name), d) - self.nonlocal_decls[-1].add(name) - - def visit_print_stmt(self, s: PrintStmt) -> None: - for arg in s.args: - arg.accept(self) - if s.target: - s.target.accept(self) - - def visit_exec_stmt(self, s: ExecStmt) -> None: - s.expr.accept(self) - if s.variables1: - s.variables1.accept(self) - if s.variables2: - s.variables2.accept(self) - - # - # Expressions - # - - def visit_name_expr(self, expr: NameExpr) -> None: - n = self.lookup(expr.name, expr) - if n: - if n.kind == BOUND_TVAR: - self.fail("'{}' is a type variable and only valid in type " - "context".format(expr.name), expr) - else: - expr.kind = n.kind - expr.node = n.node - expr.fullname = n.fullname - - def visit_super_expr(self, expr: SuperExpr) -> None: - if not self.type: - self.fail('"super" used outside class', expr) - return - expr.info = self.type - - def visit_tuple_expr(self, expr: TupleExpr) -> None: - for item in expr.items: - item.accept(self) - - def visit_list_expr(self, expr: ListExpr) -> None: - for item in expr.items: - item.accept(self) - - def visit_set_expr(self, expr: SetExpr) -> None: - for item in expr.items: - item.accept(self) - - def visit_dict_expr(self, expr: DictExpr) -> None: - for key, value in expr.items: - key.accept(self) - value.accept(self) - - def visit_star_expr(self, expr: StarExpr) -> None: - if not expr.valid: - self.fail('Can use starred expression only as assignment target', expr) - else: - expr.expr.accept(self) - - def visit_yield_from_expr(self, e: YieldFromExpr) -> None: - if not self.is_func_scope(): # not sure - self.fail("'yield from' outside function", e, True, blocker=True) - else: - self.function_stack[-1].is_generator = True - if e.expr: - e.expr.accept(self) - - def visit_call_expr(self, expr: CallExpr) -> None: - """Analyze a call expression. - - Some call expressions are recognized as special forms, including - cast(...) and Any(...). - """ - expr.callee.accept(self) - if refers_to_fullname(expr.callee, 'typing.cast'): - # Special form cast(...). - if not self.check_fixed_args(expr, 2, 'cast'): - return - # Translate first argument to an unanalyzed type. - try: - target = expr_to_unanalyzed_type(expr.args[0]) - except TypeTranslationError: - self.fail('Cast target is not a type', expr) - return - # Piggyback CastExpr object to the CallExpr object; it takes - # precedence over the CallExpr semantics. - expr.analyzed = CastExpr(expr.args[1], target) - expr.analyzed.line = expr.line - expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'builtins.reveal_type'): - if not self.check_fixed_args(expr, 1, 'reveal_type'): - return - expr.analyzed = RevealTypeExpr(expr.args[0]) - expr.analyzed.line = expr.line - expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'typing.Any'): - # Special form Any(...). - if not self.check_fixed_args(expr, 1, 'Any'): - return - expr.analyzed = CastExpr(expr.args[0], AnyType()) - expr.analyzed.line = expr.line - expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'typing._promote'): - # Special form _promote(...). - if not self.check_fixed_args(expr, 1, '_promote'): - return - # Translate first argument to an unanalyzed type. - try: - target = expr_to_unanalyzed_type(expr.args[0]) - except TypeTranslationError: - self.fail('Argument 1 to _promote is not a type', expr) - return - expr.analyzed = PromoteExpr(target) - expr.analyzed.line = expr.line - expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'builtins.dict'): - expr.analyzed = self.translate_dict_call(expr) - else: - # Normal call expression. - for a in expr.args: - a.accept(self) - - if (isinstance(expr.callee, MemberExpr) and - isinstance(expr.callee.expr, NameExpr) and - expr.callee.expr.name == '__all__' and - expr.callee.expr.kind == GDEF and - expr.callee.name in ('append', 'extend')): - if expr.callee.name == 'append' and expr.args: - self.add_exports(expr.args[0]) - elif (expr.callee.name == 'extend' and expr.args and - isinstance(expr.args[0], (ListExpr, TupleExpr))): - self.add_exports(*expr.args[0].items) - - def translate_dict_call(self, call: CallExpr) -> Optional[DictExpr]: - """Translate 'dict(x=y, ...)' to {'x': y, ...}. - - For other variants of dict(...), return None. - """ - if not call.args: - return None - if not all(kind == ARG_NAMED for kind in call.arg_kinds): - # Must still accept those args. - for a in call.args: - a.accept(self) - return None - expr = DictExpr([(StrExpr(key), value) - for key, value in zip(call.arg_names, call.args)]) - expr.set_line(call) - expr.accept(self) - return expr - - def check_fixed_args(self, expr: CallExpr, numargs: int, - name: str) -> bool: - """Verify that expr has specified number of positional args. - - Return True if the arguments are valid. - """ - s = 's' - if numargs == 1: - s = '' - if len(expr.args) != numargs: - self.fail("'%s' expects %d argument%s" % (name, numargs, s), - expr) - return False - if expr.arg_kinds != [ARG_POS] * numargs: - self.fail("'%s' must be called with %s positional argument%s" % - (name, numargs, s), expr) - return False - return True - - def visit_member_expr(self, expr: MemberExpr) -> None: - base = expr.expr - base.accept(self) - # Bind references to module attributes. - if isinstance(base, RefExpr) and base.kind == MODULE_REF: - file = cast(MypyFile, base.node) - n = file.names.get(expr.name, None) if file is not None else None - if n: - n = self.normalize_type_alias(n, expr) - if not n: - return - expr.kind = n.kind - expr.fullname = n.fullname - expr.node = n.node - else: - # We only catch some errors here; the rest will be - # catched during type checking. - # - # This way we can report a larger number of errors in - # one type checker run. If we reported errors here, - # the build would terminate after semantic analysis - # and we wouldn't be able to report any type errors. - full_name = '%s.%s' % (file.fullname() if file is not None else None, expr.name) - if full_name in obsolete_name_mapping: - self.fail("Module has no attribute %r (it's now called %r)" % ( - expr.name, obsolete_name_mapping[full_name]), expr) - - def visit_op_expr(self, expr: OpExpr) -> None: - expr.left.accept(self) - expr.right.accept(self) - - def visit_comparison_expr(self, expr: ComparisonExpr) -> None: - for operand in expr.operands: - operand.accept(self) - - def visit_unary_expr(self, expr: UnaryExpr) -> None: - expr.expr.accept(self) - - def visit_index_expr(self, expr: IndexExpr) -> None: - expr.base.accept(self) - if refers_to_class_or_function(expr.base): - # Special form -- type application. - # Translate index to an unanalyzed type. - types = [] # type: List[Type] - if isinstance(expr.index, TupleExpr): - items = expr.index.items - else: - items = [expr.index] - for item in items: - try: - typearg = expr_to_unanalyzed_type(item) - except TypeTranslationError: - self.fail('Type expected within [...]', expr) - return - typearg = self.anal_type(typearg) - types.append(typearg) - expr.analyzed = TypeApplication(expr.base, types) - expr.analyzed.line = expr.line - else: - expr.index.accept(self) - - def visit_slice_expr(self, expr: SliceExpr) -> None: - if expr.begin_index: - expr.begin_index.accept(self) - if expr.end_index: - expr.end_index.accept(self) - if expr.stride: - expr.stride.accept(self) - - def visit_cast_expr(self, expr: CastExpr) -> None: - expr.expr.accept(self) - expr.type = self.anal_type(expr.type) - - def visit_reveal_type_expr(self, expr: RevealTypeExpr) -> None: - expr.expr.accept(self) - - def visit_type_application(self, expr: TypeApplication) -> None: - expr.expr.accept(self) - for i in range(len(expr.types)): - expr.types[i] = self.anal_type(expr.types[i]) - - def visit_list_comprehension(self, expr: ListComprehension) -> None: - expr.generator.accept(self) - - def visit_set_comprehension(self, expr: SetComprehension) -> None: - expr.generator.accept(self) - - def visit_dictionary_comprehension(self, expr: DictionaryComprehension) -> None: - self.enter() - self.analyze_comp_for(expr) - expr.key.accept(self) - expr.value.accept(self) - self.leave() - self.analyze_comp_for_2(expr) - - def visit_generator_expr(self, expr: GeneratorExpr) -> None: - self.enter() - self.analyze_comp_for(expr) - expr.left_expr.accept(self) - self.leave() - self.analyze_comp_for_2(expr) - - def analyze_comp_for(self, expr: Union[GeneratorExpr, - DictionaryComprehension]) -> None: - """Analyses the 'comp_for' part of comprehensions (part 1). - - That is the part after 'for' in (x for x in l if p). This analyzes - variables and conditions which are analyzed in a local scope. - """ - for i, (index, sequence, conditions) in enumerate(zip(expr.indices, - expr.sequences, - expr.condlists)): - if i > 0: - sequence.accept(self) - # Bind index variables. - self.analyze_lvalue(index) - for cond in conditions: - cond.accept(self) - - def analyze_comp_for_2(self, expr: Union[GeneratorExpr, - DictionaryComprehension]) -> None: - """Analyses the 'comp_for' part of comprehensions (part 2). - - That is the part after 'for' in (x for x in l if p). This analyzes - the 'l' part which is analyzed in the surrounding scope. - """ - expr.sequences[0].accept(self) - - def visit_func_expr(self, expr: FuncExpr) -> None: - self.analyze_function(expr) - - def visit_conditional_expr(self, expr: ConditionalExpr) -> None: - expr.if_expr.accept(self) - expr.cond.accept(self) - expr.else_expr.accept(self) - - def visit_backquote_expr(self, expr: BackquoteExpr) -> None: - expr.expr.accept(self) - - def visit__promote_expr(self, expr: PromoteExpr) -> None: - expr.type = self.anal_type(expr.type) - - def visit_yield_expr(self, expr: YieldExpr) -> None: - if not self.is_func_scope(): - self.fail("'yield' outside function", expr, True, blocker=True) - else: - self.function_stack[-1].is_generator = True - if expr.expr: - expr.expr.accept(self) - - # - # Helpers - # - - def lookup(self, name: str, ctx: Context) -> SymbolTableNode: - """Look up an unqualified name in all active namespaces.""" - # 1a. Name declared using 'global x' takes precedence - if name in self.global_decls[-1]: - if name in self.globals: - return self.globals[name] - else: - self.name_not_defined(name, ctx) - return None - # 1b. Name declared using 'nonlocal x' takes precedence - if name in self.nonlocal_decls[-1]: - for table in reversed(self.locals[:-1]): - if table is not None and name in table: - return table[name] - else: - self.name_not_defined(name, ctx) - return None - # 2. Class attributes (if within class definition) - if self.is_class_scope() and name in self.type.names: - return self.type[name] - # 3. Local (function) scopes - for table in reversed(self.locals): - if table is not None and name in table: - return table[name] - # 4. Current file global scope - if name in self.globals: - return self.globals[name] - # 5. Builtins - b = self.globals.get('__builtins__', None) - if b: - table = cast(MypyFile, b.node).names - if name in table: - if name[0] == "_" and name[1] != "_": - self.name_not_defined(name, ctx) - return None - node = table[name] - return node - # Give up. - self.name_not_defined(name, ctx) - self.check_for_obsolete_short_name(name, ctx) - return None - - def check_for_obsolete_short_name(self, name: str, ctx: Context) -> None: - matches = [obsolete_name - for obsolete_name in obsolete_name_mapping - if obsolete_name.rsplit('.', 1)[-1] == name] - if len(matches) == 1: - self.note("(Did you mean '{}'?)".format(obsolete_name_mapping[matches[0]]), ctx) - - def lookup_qualified(self, name: str, ctx: Context) -> SymbolTableNode: - if '.' not in name: - return self.lookup(name, ctx) - else: - parts = name.split('.') - n = self.lookup(parts[0], ctx) # type: SymbolTableNode - if n: - for i in range(1, len(parts)): - if isinstance(n.node, TypeInfo): - if n.node.mro is None: - # We haven't yet analyzed the class `n.node`. Fall back to direct - # lookup in the names declared directly under it, without its base - # classes. This can happen when we have a forward reference to a - # nested class, and the reference is bound before the outer class - # has been fully semantically analyzed. - # - # A better approach would be to introduce a new analysis pass or - # to move things around between passes, but this unblocks a common - # use case even though this is a little limited in case there is - # inheritance involved. - result = n.node.names.get(parts[i]) - else: - result = n.node.get(parts[i]) - n = result - elif isinstance(n.node, MypyFile): - n = n.node.names.get(parts[i], None) - # TODO: What if node is Var or FuncDef? - if not n: - self.name_not_defined(name, ctx) - break - if n: - n = self.normalize_type_alias(n, ctx) - return n - - def builtin_type(self, fully_qualified_name: str) -> Instance: - node = self.lookup_fully_qualified(fully_qualified_name) - info = cast(TypeInfo, node.node) - return Instance(info, []) - - def lookup_fully_qualified(self, name: str) -> SymbolTableNode: - """Lookup a fully qualified name. - - Assume that the name is defined. This happens in the global namespace -- the local - module namespace is ignored. - """ - assert '.' in name - parts = name.split('.') - n = self.modules[parts[0]] - for i in range(1, len(parts) - 1): - n = cast(MypyFile, n.names[parts[i]].node) - return n.names[parts[-1]] - - def lookup_fully_qualified_or_none(self, name: str) -> SymbolTableNode: - """Lookup a fully qualified name. - - Assume that the name is defined. This happens in the global namespace -- the local - module namespace is ignored. - """ - assert '.' in name - parts = name.split('.') - n = self.modules[parts[0]] - for i in range(1, len(parts) - 1): - next_sym = n.names.get(parts[i]) - if not next_sym: - return None - n = cast(MypyFile, next_sym.node) - return n.names.get(parts[-1]) - - def qualified_name(self, n: str) -> str: - return self.cur_mod_id + '.' + n - - def enter(self) -> None: - self.locals.append(SymbolTable()) - self.global_decls.append(set()) - self.nonlocal_decls.append(set()) - # -1 since entering block will increment this to 0. - self.block_depth.append(-1) - - def leave(self) -> None: - self.locals.pop() - self.global_decls.pop() - self.nonlocal_decls.pop() - self.block_depth.pop() - - def is_func_scope(self) -> bool: - return self.locals[-1] is not None - - def is_class_scope(self) -> bool: - return self.type is not None and not self.is_func_scope() - - def add_symbol(self, name: str, node: SymbolTableNode, - context: Context) -> None: - if self.is_func_scope(): - if name in self.locals[-1]: - # Flag redefinition unless this is a reimport of a module. - if not (node.kind == MODULE_REF and - self.locals[-1][name].node == node.node): - self.name_already_defined(name, context) - self.locals[-1][name] = node - elif self.type: - self.type.names[name] = node - else: - existing = self.globals.get(name) - if existing and (not isinstance(node.node, MypyFile) or - existing.node != node.node) and existing.kind != UNBOUND_IMPORTED: - # Modules can be imported multiple times to support import - # of multiple submodules of a package (e.g. a.x and a.y). - ok = False - # Only report an error if the symbol collision provides a different type. - if existing.type and node.type and is_same_type(existing.type, node.type): - ok = True - if not ok: - self.name_already_defined(name, context) - self.globals[name] = node - - def add_var(self, v: Var, ctx: Context) -> None: - if self.is_func_scope(): - self.add_local(v, ctx) - else: - self.globals[v.name()] = SymbolTableNode(GDEF, v, self.cur_mod_id) - v._fullname = self.qualified_name(v.name()) - - def add_local(self, node: Union[Var, FuncBase], ctx: Context) -> None: - name = node.name() - if name in self.locals[-1]: - self.name_already_defined(name, ctx) - node._fullname = name - self.locals[-1][name] = SymbolTableNode(LDEF, node) - - def add_exports(self, *exps: Node) -> None: - for exp in exps: - if isinstance(exp, StrExpr): - self.all_exports.add(exp.value) - - def check_no_global(self, n: str, ctx: Context, - is_overloaded_func: bool = False) -> None: - if n in self.globals: - prev_is_overloaded = isinstance(self.globals[n], OverloadedFuncDef) - if is_overloaded_func and prev_is_overloaded: - self.fail("Nonconsecutive overload {} found".format(n), ctx) - elif prev_is_overloaded: - self.fail("Definition of '{}' missing 'overload'".format(n), ctx) - else: - self.name_already_defined(n, ctx) - - def name_not_defined(self, name: str, ctx: Context) -> None: - message = "Name '{}' is not defined".format(name) - extra = self.undefined_name_extra_info(name) - if extra: - message += ' {}'.format(extra) - self.fail(message, ctx) - - def name_already_defined(self, name: str, ctx: Context) -> None: - self.fail("Name '{}' already defined".format(name), ctx) - - def fail(self, msg: str, ctx: Context, serious: bool = False, *, - blocker: bool = False) -> None: - if (not serious and - not self.options.check_untyped_defs and - self.function_stack and - self.function_stack[-1].is_dynamic()): - return - self.errors.report(ctx.get_line(), msg, blocker=blocker) - - def note(self, msg: str, ctx: Context) -> None: - if (not self.options.check_untyped_defs and - self.function_stack and - self.function_stack[-1].is_dynamic()): - return - self.errors.report(ctx.get_line(), msg, severity='note') - - def undefined_name_extra_info(self, fullname: str) -> Optional[str]: - if fullname in obsolete_name_mapping: - return "(it's now called '{}')".format(obsolete_name_mapping[fullname]) - else: - return None - - def accept(self, node: Node) -> None: - try: - node.accept(self) - except Exception as err: - report_internal_error(err, self.errors.file, node.line) - - -class FirstPass(NodeVisitor): - """First phase of semantic analysis. - - See docstring of 'analyze()' below for a description of what this does. - """ - - def __init__(self, sem: SemanticAnalyzer) -> None: - self.sem = sem - self.pyversion = sem.options.python_version - - def analyze(self, file: MypyFile, fnam: str, mod_id: str) -> None: - """Perform the first analysis pass. - - Populate module global table. Resolve the full names of - definitions not nested within functions and construct type - info structures, but do not resolve inter-definition - references such as base classes. - - Also add implicit definitions such as __name__. - - In this phase we don't resolve imports. For 'from ... import', - we generate dummy symbol table nodes for the imported names, - and these will get resolved in later phases of semantic - analysis. - """ - sem = self.sem - sem.cur_mod_id = mod_id - sem.errors.set_file(fnam) - sem.globals = SymbolTable() - sem.global_decls = [set()] - sem.nonlocal_decls = [set()] - sem.block_depth = [0] - - defs = file.defs - - # Add implicit definitions of module '__name__' etc. - for name, t in implicit_module_attrs.items(): - v = Var(name, UnboundType(t)) - v._fullname = self.sem.qualified_name(name) - self.sem.globals[name] = SymbolTableNode(GDEF, v, self.sem.cur_mod_id) - - for d in defs: - d.accept(self) - - # Add implicit definition of literals/keywords to builtins, as we - # cannot define a variable with them explicitly. - if mod_id == 'builtins': - literal_types = [ - ('None', NoneTyp()), - # reveal_type is a mypy-only function that gives an error with the type of its arg - ('reveal_type', AnyType()), - ] # type: List[Tuple[str, Type]] - - # TODO(ddfisher): This guard is only needed because mypy defines - # fake builtins for its tests which often don't define bool. If - # mypy is fast enough that we no longer need those, this - # conditional check should be removed. - if 'bool' in self.sem.globals: - bool_type = self.sem.named_type('bool') - literal_types.extend([ - ('True', bool_type), - ('False', bool_type), - ('__debug__', bool_type), - ]) - - for name, typ in literal_types: - v = Var(name, typ) - v._fullname = self.sem.qualified_name(name) - self.sem.globals[name] = SymbolTableNode(GDEF, v, self.sem.cur_mod_id) - - def visit_block(self, b: Block) -> None: - if b.is_unreachable: - return - self.sem.block_depth[-1] += 1 - for node in b.body: - node.accept(self) - self.sem.block_depth[-1] -= 1 - - def visit_assignment_stmt(self, s: AssignmentStmt) -> None: - for lval in s.lvalues: - self.analyze_lvalue(lval, explicit_type=s.type is not None) - - def visit_func_def(self, func: FuncDef) -> None: - sem = self.sem - func.is_conditional = sem.block_depth[-1] > 0 - func._fullname = sem.qualified_name(func.name()) - if func.name() in sem.globals: - # Already defined in this module. - original_sym = sem.globals[func.name()] - if original_sym.kind == UNBOUND_IMPORTED: - # Ah this is an imported name. We can't resolve them now, so we'll postpone - # this until the main phase of semantic analysis. - return - original_def = original_sym.node - if sem.is_conditional_func(original_def, func): - # Conditional function definition -- multiple defs are ok. - func.original_def = cast(FuncDef, original_def) - else: - # Report error. - sem.check_no_global(func.name(), func) - else: - sem.globals[func.name()] = SymbolTableNode(GDEF, func, sem.cur_mod_id) - - def visit_overloaded_func_def(self, func: OverloadedFuncDef) -> None: - self.sem.check_no_global(func.name(), func, True) - func._fullname = self.sem.qualified_name(func.name()) - self.sem.globals[func.name()] = SymbolTableNode(GDEF, func, - self.sem.cur_mod_id) - - def visit_class_def(self, cdef: ClassDef) -> None: - self.sem.check_no_global(cdef.name, cdef) - cdef.fullname = self.sem.qualified_name(cdef.name) - info = TypeInfo(SymbolTable(), cdef) - info.set_line(cdef.line) - cdef.info = info - self.sem.globals[cdef.name] = SymbolTableNode(GDEF, info, - self.sem.cur_mod_id) - self.process_nested_classes(cdef) - - def process_nested_classes(self, outer_def: ClassDef) -> None: - for node in outer_def.defs.body: - if isinstance(node, ClassDef): - node.info = TypeInfo(SymbolTable(), node) - if outer_def.fullname: - node.info._fullname = outer_def.fullname + '.' + node.info.name() - else: - node.info._fullname = node.info.name() - symbol = SymbolTableNode(MDEF, node.info) - outer_def.info.names[node.name] = symbol - self.process_nested_classes(node) - - def visit_import_from(self, node: ImportFrom) -> None: - # We can't bind module names during the first pass, as the target module might be - # unprocessed. However, we add dummy unbound imported names to the symbol table so - # that we at least know that the name refers to a module. - node.is_top_level = True - for name, as_name in node.names: - imported_name = as_name or name - if imported_name not in self.sem.globals: - self.sem.add_symbol(imported_name, SymbolTableNode(UNBOUND_IMPORTED, None), node) - - def visit_import(self, node: Import) -> None: - node.is_top_level = True - # This is similar to visit_import_from -- see the comment there. - for id, as_id in node.ids: - imported_id = as_id or id - if imported_id not in self.sem.globals: - self.sem.add_symbol(imported_id, SymbolTableNode(UNBOUND_IMPORTED, None), node) - else: - # If the previous symbol is a variable, this should take precedence. - self.sem.globals[imported_id] = SymbolTableNode(UNBOUND_IMPORTED, None) - - def visit_import_all(self, node: ImportAll) -> None: - node.is_top_level = True - - def visit_while_stmt(self, s: WhileStmt) -> None: - s.body.accept(self) - if s.else_body: - s.else_body.accept(self) - - def visit_for_stmt(self, s: ForStmt) -> None: - self.analyze_lvalue(s.index) - s.body.accept(self) - if s.else_body: - s.else_body.accept(self) - - def visit_with_stmt(self, s: WithStmt) -> None: - for n in s.target: - if n: - self.analyze_lvalue(n) - s.body.accept(self) - - def visit_decorator(self, d: Decorator) -> None: - d.var._fullname = self.sem.qualified_name(d.var.name()) - self.sem.add_symbol(d.var.name(), SymbolTableNode(GDEF, d.var), d) - - def visit_if_stmt(self, s: IfStmt) -> None: - infer_reachability_of_if_statement(s, pyversion=self.pyversion) - for node in s.body: - node.accept(self) - if s.else_body: - s.else_body.accept(self) - - def visit_try_stmt(self, s: TryStmt) -> None: - self.sem.analyze_try_stmt(s, self, add_global=True) - - def analyze_lvalue(self, lvalue: Node, explicit_type: bool = False) -> None: - self.sem.analyze_lvalue(lvalue, add_global=True, explicit_type=explicit_type) - - -class ThirdPass(TraverserVisitor): - """The third and final pass of semantic analysis. - - Check type argument counts and values of generic types, and perform some - straightforward type inference. - """ - - def __init__(self, modules: Dict[str, MypyFile], errors: Errors) -> None: - self.modules = modules - self.errors = errors - - def visit_file(self, file_node: MypyFile, fnam: str) -> None: - self.errors.set_file(fnam) - self.accept(file_node) - - def accept(self, node: Node) -> None: - try: - node.accept(self) - except Exception as err: - report_internal_error(err, self.errors.file, node.line) - - def visit_block(self, b: Block) -> None: - if b.is_unreachable: - return - super().visit_block(b) - - def visit_func_def(self, fdef: FuncDef) -> None: - self.errors.push_function(fdef.name()) - self.analyze(fdef.type) - super().visit_func_def(fdef) - self.errors.pop_function() - - def visit_class_def(self, tdef: ClassDef) -> None: - for type in tdef.info.bases: - self.analyze(type) - # Recompute MRO now that we have analyzed all modules, to pick - # up superclasses of bases imported from other modules in an - # import loop. (Only do so if we succeeded the first time.) - if tdef.info.mro: - tdef.info.mro = [] # Force recomputation - calculate_class_mro(tdef, self.fail) - super().visit_class_def(tdef) - - def visit_decorator(self, dec: Decorator) -> None: - """Try to infer the type of the decorated function. - - This lets us resolve references to decorated functions during - type checking when there are cyclic imports, as otherwise the - type might not be available when we need it. - - This basically uses a simple special-purpose type inference - engine just for decorators. - """ - super().visit_decorator(dec) - if dec.var.is_property: - # Decorators are expected to have a callable type (it's a little odd). - if dec.func.type is None: - dec.var.type = CallableType( - [AnyType()], - [ARG_POS], - [None], - AnyType(), - self.builtin_type('function'), - name=dec.var.name()) - elif isinstance(dec.func.type, CallableType): - dec.var.type = dec.func.type - return - decorator_preserves_type = True - for expr in dec.decorators: - preserve_type = False - if isinstance(expr, RefExpr) and isinstance(expr.node, FuncDef): - if is_identity_signature(expr.node.type): - preserve_type = True - if not preserve_type: - decorator_preserves_type = False - break - if decorator_preserves_type: - # No non-identity decorators left. We can trivially infer the type - # of the function here. - dec.var.type = function_type(dec.func, self.builtin_type('function')) - if dec.decorators: - if returns_any_if_called(dec.decorators[0]): - # The outermost decorator will return Any so we know the type of the - # decorated function. - dec.var.type = AnyType() - sig = find_fixed_callable_return(dec.decorators[0]) - if sig: - # The outermost decorator always returns the same kind of function, - # so we know that this is the type of the decoratored function. - orig_sig = function_type(dec.func, self.builtin_type('function')) - sig.name = orig_sig.items()[0].name - dec.var.type = sig - - def visit_assignment_stmt(self, s: AssignmentStmt) -> None: - self.analyze(s.type) - super().visit_assignment_stmt(s) - - def visit_cast_expr(self, e: CastExpr) -> None: - self.analyze(e.type) - super().visit_cast_expr(e) - - def visit_reveal_type_expr(self, e: RevealTypeExpr) -> None: - super().visit_reveal_type_expr(e) - - def visit_type_application(self, e: TypeApplication) -> None: - for type in e.types: - self.analyze(type) - super().visit_type_application(e) - - # Helpers - - def analyze(self, type: Type) -> None: - if type: - analyzer = TypeAnalyserPass3(self.fail) - type.accept(analyzer) - - def fail(self, msg: str, ctx: Context) -> None: - self.errors.report(ctx.get_line(), msg) - - def builtin_type(self, name: str, args: List[Type] = None) -> Instance: - names = self.modules['builtins'] - sym = names.names[name] - assert isinstance(sym.node, TypeInfo) - return Instance(sym.node, args or []) - - -def self_type(typ: TypeInfo) -> Union[Instance, TupleType]: - """For a non-generic type, return instance type representing the type. - For a generic G type with parameters T1, .., Tn, return G[T1, ..., Tn]. - """ - tv = [] # type: List[Type] - for i in range(len(typ.type_vars)): - tv.append(TypeVarType(typ.defn.type_vars[i])) - inst = Instance(typ, tv) - if typ.tuple_type is None: - return inst - return typ.tuple_type.self_type(inst) - - -def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: - if isinstance(sig, CallableType): - return replace_leading_arg_type(sig, new) - else: - sig = cast(Overloaded, sig) - return Overloaded([cast(CallableType, replace_implicit_first_type(i, new)) - for i in sig.items()]) - - -def set_callable_name(sig: Type, fdef: FuncDef) -> Type: - if isinstance(sig, FunctionLike): - if fdef.info: - return sig.with_name( - '"{}" of "{}"'.format(fdef.name(), fdef.info.name())) - else: - return sig.with_name('"{}"'.format(fdef.name())) - else: - return sig - - -def refers_to_fullname(node: Node, fullname: str) -> bool: - """Is node a name or member expression with the given full name?""" - return isinstance(node, RefExpr) and node.fullname == fullname - - -def refers_to_class_or_function(node: Node) -> bool: - """Does semantically analyzed node refer to a class?""" - return (isinstance(node, RefExpr) and - isinstance(node.node, (TypeInfo, FuncDef, OverloadedFuncDef))) - - -def calculate_class_mro(defn: ClassDef, fail: Callable[[str, Context], None]) -> None: - try: - defn.info.calculate_mro() - except MroError: - fail("Cannot determine consistent method resolution order " - '(MRO) for "%s"' % defn.name, defn) - defn.info.mro = [] - # The property of falling back to Any is inherited. - defn.info.fallback_to_any = any(baseinfo.fallback_to_any for baseinfo in defn.info.mro) - - -def find_duplicate(list: List[T]) -> T: - """If the list has duplicates, return one of the duplicates. - - Otherwise, return None. - """ - for i in range(1, len(list)): - if list[i] in list[:i]: - return list[i] - return None - - -def disable_typevars(nodes: List[SymbolTableNode]) -> None: - for node in nodes: - assert node.kind in (BOUND_TVAR, UNBOUND_TVAR) - node.kind = UNBOUND_TVAR - - -def enable_typevars(nodes: List[SymbolTableNode]) -> None: - for node in nodes: - assert node.kind in (BOUND_TVAR, UNBOUND_TVAR) - node.kind = BOUND_TVAR - - -def remove_imported_names_from_symtable(names: SymbolTable, - module: str) -> None: - """Remove all imported names from the symbol table of a module.""" - removed = [] # type: List[str] - for name, node in names.items(): - if node.node is None: - continue - fullname = node.node.fullname() - prefix = fullname[:fullname.rfind('.')] - if prefix != module: - removed.append(name) - for name in removed: - del names[name] - - -def infer_reachability_of_if_statement(s: IfStmt, - pyversion: Tuple[int, int]) -> None: - for i in range(len(s.expr)): - result = infer_if_condition_value(s.expr[i], pyversion) - if result == ALWAYS_FALSE: - # The condition is always false, so we skip the if/elif body. - mark_block_unreachable(s.body[i]) - elif result == ALWAYS_TRUE: - # This condition is always true, so all of the remaining - # elif/else bodies will never be executed. - for body in s.body[i + 1:]: - mark_block_unreachable(body) - if s.else_body: - mark_block_unreachable(s.else_body) - break - - -def infer_if_condition_value(expr: Node, pyversion: Tuple[int, int]) -> int: - """Infer whether if condition is always true/false. - - Return ALWAYS_TRUE if always true, ALWAYS_FALSE if always false, - and TRUTH_VALUE_UNKNOWN otherwise. - """ - name = '' - negated = False - alias = expr - if isinstance(alias, UnaryExpr): - if alias.op == 'not': - expr = alias.expr - negated = True - if isinstance(expr, NameExpr): - name = expr.name - elif isinstance(expr, MemberExpr): - name = expr.name - result = TRUTH_VALUE_UNKNOWN - if name == 'PY2': - result = ALWAYS_TRUE if pyversion[0] == 2 else ALWAYS_FALSE - elif name == 'PY3': - result = ALWAYS_TRUE if pyversion[0] == 3 else ALWAYS_FALSE - elif name == 'MYPY': - result = ALWAYS_TRUE - if negated: - if result == ALWAYS_TRUE: - result = ALWAYS_FALSE - elif result == ALWAYS_FALSE: - result = ALWAYS_TRUE - return result - - -def mark_block_unreachable(block: Block) -> None: - block.is_unreachable = True - block.accept(MarkImportsUnreachableVisitor()) - - -class MarkImportsUnreachableVisitor(TraverserVisitor): - """Visitor that flags all imports nested within a node as unreachable.""" - - def visit_import(self, node: Import) -> None: - node.is_unreachable = True - - def visit_import_from(self, node: ImportFrom) -> None: - node.is_unreachable = True - - def visit_import_all(self, node: ImportAll) -> None: - node.is_unreachable = True - - -def is_identity_signature(sig: Type) -> bool: - """Is type a callable of form T -> T (where T is a type variable)?""" - if isinstance(sig, CallableType) and sig.arg_kinds == [ARG_POS]: - if isinstance(sig.arg_types[0], TypeVarType) and isinstance(sig.ret_type, TypeVarType): - return sig.arg_types[0].id == sig.ret_type.id - return False - - -def returns_any_if_called(expr: Node) -> bool: - """Return True if we can predict that expr will return Any if called. - - This only uses information available during semantic analysis so this - will sometimes return False because of insufficient information (as - type inference hasn't run yet). - """ - if isinstance(expr, RefExpr): - if isinstance(expr.node, FuncDef): - typ = expr.node.type - if typ is None: - # No signature -> default to Any. - return True - # Explicit Any return? - return isinstance(typ, CallableType) and isinstance(typ.ret_type, AnyType) - elif isinstance(expr.node, Var): - typ = expr.node.type - return typ is None or isinstance(typ, AnyType) - elif isinstance(expr, CallExpr): - return returns_any_if_called(expr.callee) - return False - - -def find_fixed_callable_return(expr: Node) -> Optional[CallableType]: - if isinstance(expr, RefExpr): - if isinstance(expr.node, FuncDef): - typ = expr.node.type - if typ: - if isinstance(typ, CallableType) and has_no_typevars(typ.ret_type): - if isinstance(typ.ret_type, CallableType): - return typ.ret_type - elif isinstance(expr, CallExpr): - t = find_fixed_callable_return(expr.callee) - if t: - if isinstance(t.ret_type, CallableType): - return t.ret_type - return None - - -def has_no_typevars(typ: Type) -> bool: - return is_same_type(typ, erase_typevars(typ)) +"""The semantic analyzer. + +Bind names to definitions and do various other simple consistency +checks. For example, consider this program: + + x = 1 + y = x + +Here semantic analysis would detect that the assignment 'x = 1' +defines a new variable, the type of which is to be inferred (in a +later pass; type inference or type checking is not part of semantic +analysis). Also, it would bind both references to 'x' to the same +module-level variable node. The second assignment would also be +analyzed, and the type of 'y' marked as being inferred. + +Semantic analysis is the first analysis pass after parsing, and it is +subdivided into three passes: + + * FirstPass looks up externally visible names defined in a module but + ignores imports and local definitions. It helps enable (some) + cyclic references between modules, such as module 'a' that imports + module 'b' and used names defined in b *and* vice versa. The first + pass can be performed before dependent modules have been processed. + + * SemanticAnalyzer is the second pass. It does the bulk of the work. + It assumes that dependent modules have been semantically analyzed, + up to the second pass, unless there is a import cycle. + + * ThirdPass checks that type argument counts are valid; for example, + it will reject Dict[int]. We don't do this in the second pass, + since we infer the type argument counts of classes during this + pass, and it is possible to refer to classes defined later in a + file, which would not have the type argument count set yet. This + pass also recomputes the method resolution order of each class, in + case one of its bases belongs to a module involved in an import + loop. + +Semantic analysis of types is implemented in module mypy.typeanal. + +TODO: Check if the third pass slows down type checking significantly. + We could probably get rid of it -- for example, we could collect all + analyzed types in a collection and check them without having to + traverse the entire AST. +""" + +from typing import ( + List, Dict, Set, Tuple, cast, Any, overload, TypeVar, Union, Optional, Callable +) + +from mypy.nodes import ( + MypyFile, TypeInfo, Node, AssignmentStmt, FuncDef, OverloadedFuncDef, + ClassDef, Var, GDEF, MODULE_REF, FuncItem, Import, + ImportFrom, ImportAll, Block, LDEF, NameExpr, MemberExpr, + IndexExpr, TupleExpr, ListExpr, ExpressionStmt, ReturnStmt, + RaiseStmt, AssertStmt, OperatorAssignmentStmt, WhileStmt, + ForStmt, BreakStmt, ContinueStmt, IfStmt, TryStmt, WithStmt, DelStmt, + GlobalDecl, SuperExpr, DictExpr, CallExpr, RefExpr, OpExpr, UnaryExpr, + SliceExpr, CastExpr, RevealTypeExpr, TypeApplication, Context, SymbolTable, + SymbolTableNode, BOUND_TVAR, UNBOUND_TVAR, ListComprehension, GeneratorExpr, + FuncExpr, MDEF, FuncBase, Decorator, SetExpr, TypeVarExpr, + StrExpr, BytesExpr, PrintStmt, ConditionalExpr, PromoteExpr, + ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, MroError, type_aliases, + YieldFromExpr, NamedTupleExpr, NonlocalDecl, + SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr, + YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, COVARIANT, CONTRAVARIANT, + IntExpr, FloatExpr, UnicodeExpr, + INVARIANT, UNBOUND_IMPORTED, Expression, EllipsisExpr, NamedTupleTypeInfo +) +from mypy.visitor import NodeVisitor +from mypy.traverser import TraverserVisitor +from mypy.errors import Errors, report_internal_error +from mypy.types import ( + NoneTyp, CallableType, Overloaded, Instance, Type, TypeVarType, AnyType, + FunctionLike, UnboundType, TypeList, ErrorType, TypeVarDef, + replace_leading_arg_type, TupleType, UnionType, StarType, EllipsisType, + NamedTupleType +) +from mypy.nodes import function_type, implicit_module_attrs +from mypy.typeanal import TypeAnalyser, TypeAnalyserPass3, analyze_type_alias +from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError +from mypy.lex import lex +from mypy.parsetype import parse_type +from mypy.sametypes import is_same_type +from mypy.erasetype import erase_typevars +from mypy import defaults +from mypy.options import Options + + +T = TypeVar('T') + + +# Inferred value of an expression. +ALWAYS_TRUE = 0 +ALWAYS_FALSE = 1 +TRUTH_VALUE_UNKNOWN = 2 + +# Map from obsolete name to the current spelling. +obsolete_name_mapping = { + 'typing.Function': 'typing.Callable', + 'typing.typevar': 'typing.TypeVar', +} + +# Hard coded type promotions (shared between all Python versions). +# These add extra ad-hoc edges to the subtyping relation. For example, +# int is considered a subtype of float, even though there is no +# subclass relationship. +TYPE_PROMOTIONS = { + 'builtins.int': 'builtins.float', + 'builtins.float': 'builtins.complex', +} + +# Hard coded type promotions for Python 3. +# +# Note that the bytearray -> bytes promotion is a little unsafe +# as some functions only accept bytes objects. Here convenience +# trumps safety. +TYPE_PROMOTIONS_PYTHON3 = TYPE_PROMOTIONS.copy() +TYPE_PROMOTIONS_PYTHON3.update({ + 'builtins.bytearray': 'builtins.bytes', +}) + +# Hard coded type promotions for Python 2. +# +# These promotions are unsafe, but we are doing them anyway +# for convenience and also for Python 3 compatibility +# (bytearray -> str). +TYPE_PROMOTIONS_PYTHON2 = TYPE_PROMOTIONS.copy() +TYPE_PROMOTIONS_PYTHON2.update({ + 'builtins.str': 'builtins.unicode', + 'builtins.bytearray': 'builtins.str', +}) + +# When analyzing a function, should we analyze the whole function in one go, or +# should we only perform one phase of the analysis? The latter is used for +# nested functions. In the first phase we add the function to the symbol table +# but don't process body. In the second phase we process function body. This +# way we can have mutually recursive nested functions. +FUNCTION_BOTH_PHASES = 0 # Everthing in one go +FUNCTION_FIRST_PHASE_POSTPONE_SECOND = 1 # Add to symbol table but postpone body +FUNCTION_SECOND_PHASE = 2 # Only analyze body + + +class SemanticAnalyzer(NodeVisitor): + """Semantically analyze parsed mypy files. + + The analyzer binds names and does various consistency checks for a + parse tree. Note that type checking is performed as a separate + pass. + + This is the second phase of semantic analysis. + """ + + # Library search paths + lib_path = None # type: List[str] + # Module name space + modules = None # type: Dict[str, MypyFile] + # Global name space for current module + globals = None # type: SymbolTable + # Names declared using "global" (separate set for each scope) + global_decls = None # type: List[Set[str]] + # Names declated using "nonlocal" (separate set for each scope) + nonlocal_decls = None # type: List[Set[str]] + # Local names of function scopes; None for non-function scopes. + locals = None # type: List[SymbolTable] + # Nested block depths of scopes + block_depth = None # type: List[int] + # TypeInfo of directly enclosing class (or None) + type = None # type: TypeInfo + # Stack of outer classes (the second tuple item contains tvars). + type_stack = None # type: List[TypeInfo] + # Type variables that are bound by the directly enclosing class + bound_tvars = None # type: List[SymbolTableNode] + # Stack of type variables that were bound by outer classess + tvar_stack = None # type: List[List[SymbolTableNode]] + # Do weak type checking in this file + weak_opts = set() # type: Set[str] + + # Stack of functions being analyzed + function_stack = None # type: List[FuncItem] + # Stack of next available function type variable ids + next_function_tvar_id_stack = None # type: List[int] + + # Status of postponing analysis of nested function bodies. By using this we + # can have mutually recursive nested functions. Values are FUNCTION_x + # constants. Note that separate phasea are not used for methods. + postpone_nested_functions_stack = None # type: List[int] + # Postponed functions collected if + # postpone_nested_functions_stack[-1] == FUNCTION_FIRST_PHASE_POSTPONE_SECOND. + postponed_functions_stack = None # type: List[List[Node]] + + loop_depth = 0 # Depth of breakable loops + cur_mod_id = '' # Current module id (or None) (phase 2) + is_stub_file = False # Are we analyzing a stub file? + imports = None # type: Set[str] # Imported modules (during phase 2 analysis) + errors = None # type: Errors # Keeps track of generated errors + + def __init__(self, + lib_path: List[str], + errors: Errors, + options: Options) -> None: + """Construct semantic analyzer. + + Use lib_path to search for modules, and report analysis errors + using the Errors instance. + """ + self.locals = [None] + self.imports = set() + self.type = None + self.type_stack = [] + self.bound_tvars = None + self.tvar_stack = [] + self.function_stack = [] + self.next_function_tvar_id_stack = [-1] + self.block_depth = [0] + self.loop_depth = 0 + self.lib_path = lib_path + self.errors = errors + self.modules = {} + self.options = options + self.postpone_nested_functions_stack = [FUNCTION_BOTH_PHASES] + self.postponed_functions_stack = [] + self.all_exports = set() # type: Set[str] + + def visit_file(self, file_node: MypyFile, fnam: str) -> None: + self.errors.set_file(fnam) + self.cur_mod_node = file_node + self.cur_mod_id = file_node.fullname() + self.is_stub_file = fnam.lower().endswith('.pyi') + self.globals = file_node.names + self.weak_opts = file_node.weak_opts + + if 'builtins' in self.modules: + self.globals['__builtins__'] = SymbolTableNode( + MODULE_REF, self.modules['builtins'], self.cur_mod_id) + + for name in implicit_module_attrs: + v = self.globals[name].node + if isinstance(v, Var): + v.type = self.anal_type(v.type) + v.is_ready = True + + defs = file_node.defs + for d in defs: + self.accept(d) + + if self.cur_mod_id == 'builtins': + remove_imported_names_from_symtable(self.globals, 'builtins') + + if '__all__' in self.globals: + for name, g in self.globals.items(): + if name not in self.all_exports: + g.module_public = False + + def visit_func_def(self, defn: FuncDef) -> None: + phase_info = self.postpone_nested_functions_stack[-1] + if phase_info != FUNCTION_SECOND_PHASE: + self.function_stack.append(defn) + # First phase of analysis for function. + self.errors.push_function(defn.name()) + self.update_function_type_variables(defn) + self.errors.pop_function() + self.function_stack.pop() + + defn.is_conditional = self.block_depth[-1] > 0 + + # TODO(jukka): Figure out how to share the various cases. It doesn't + # make sense to have (almost) duplicate code (here and elsewhere) for + # 3 cases: module-level, class-level and local names. Maybe implement + # a common stack of namespaces. As the 3 kinds of namespaces have + # different semantics, this wouldn't always work, but it might still + # be a win. + if self.is_class_scope(): + # Method definition + defn.info = self.type + if not defn.is_decorated and not defn.is_overload: + if defn.name() in self.type.names: + # Redefinition. Conditional redefinition is okay. + n = self.type.names[defn.name()].node + if self.is_conditional_func(n, defn): + defn.original_def = cast(FuncDef, n) + else: + self.name_already_defined(defn.name(), defn) + self.type.names[defn.name()] = SymbolTableNode(MDEF, defn) + self.prepare_method_signature(defn) + elif self.is_func_scope(): + # Nested function + if not defn.is_decorated and not defn.is_overload: + if defn.name() in self.locals[-1]: + # Redefinition. Conditional redefinition is okay. + n = self.locals[-1][defn.name()].node + if self.is_conditional_func(n, defn): + defn.original_def = cast(FuncDef, n) + else: + self.name_already_defined(defn.name(), defn) + else: + self.add_local(defn, defn) + else: + # Top-level function + if not defn.is_decorated and not defn.is_overload: + symbol = self.globals.get(defn.name()) + if isinstance(symbol.node, FuncDef) and symbol.node != defn: + # This is redefinition. Conditional redefinition is okay. + original_def = symbol.node + if self.is_conditional_func(original_def, defn): + # Conditional function definition -- multiple defs are ok. + defn.original_def = original_def + else: + # Report error. + self.check_no_global(defn.name(), defn, True) + if phase_info == FUNCTION_FIRST_PHASE_POSTPONE_SECOND: + # Postpone this function (for the second phase). + self.postponed_functions_stack[-1].append(defn) + return + if phase_info != FUNCTION_FIRST_PHASE_POSTPONE_SECOND: + # Second phase of analysis for function. + self.errors.push_function(defn.name()) + self.analyze_function(defn) + self.errors.pop_function() + + def prepare_method_signature(self, func: FuncDef) -> None: + """Check basic signature validity and tweak annotation of self/cls argument.""" + # Only non-static methods are special. + if not func.is_static: + if not func.arguments: + self.fail('Method must have at least one argument', func) + elif func.type: + sig = cast(FunctionLike, func.type) + if func.is_class: + leading_type = self.class_type(self.type) + else: + leading_type = self_type(self.type) + func.type = replace_implicit_first_type(sig, leading_type) + + def is_conditional_func(self, previous: Node, new: FuncDef) -> bool: + """Does 'new' conditionally redefine 'previous'? + + We reject straight redefinitions of functions, as they are usually + a programming error. For example: + + . def f(): ... + . def f(): ... # Error: 'f' redefined + """ + return isinstance(previous, (FuncDef, Var)) and new.is_conditional + + def update_function_type_variables(self, defn: FuncDef) -> None: + """Make any type variables in the signature of defn explicit. + + Update the signature of defn to contain type variable definitions + if defn is generic. + """ + if defn.type: + functype = cast(CallableType, defn.type) + typevars = self.infer_type_variables(functype) + # Do not define a new type variable if already defined in scope. + typevars = [(name, tvar) for name, tvar in typevars + if not self.is_defined_type_var(name, defn)] + if typevars: + next_tvar_id = self.next_function_tvar_id() + defs = [TypeVarDef(tvar[0], next_tvar_id - i, + tvar[1].values, tvar[1].upper_bound, + tvar[1].variance) + for i, tvar in enumerate(typevars)] + functype.variables = defs + + def infer_type_variables(self, + type: CallableType) -> List[Tuple[str, TypeVarExpr]]: + """Return list of unique type variables referred to in a callable.""" + names = [] # type: List[str] + tvars = [] # type: List[TypeVarExpr] + for arg in type.arg_types + [type.ret_type]: + for name, tvar_expr in self.find_type_variables_in_type(arg): + if name not in names: + names.append(name) + tvars.append(tvar_expr) + return list(zip(names, tvars)) + + def find_type_variables_in_type( + self, type: Type) -> List[Tuple[str, TypeVarExpr]]: + """Return a list of all unique type variable references in type. + + This effectively does partial name binding, results of which are mostly thrown away. + """ + result = [] # type: List[Tuple[str, TypeVarExpr]] + if isinstance(type, UnboundType): + name = type.name + node = self.lookup_qualified(name, type) + if node and node.kind == UNBOUND_TVAR: + result.append((name, cast(TypeVarExpr, node.node))) + for arg in type.args: + result.extend(self.find_type_variables_in_type(arg)) + elif isinstance(type, TypeList): + for item in type.items: + result.extend(self.find_type_variables_in_type(item)) + elif isinstance(type, UnionType): + for item in type.items: + result.extend(self.find_type_variables_in_type(item)) + elif isinstance(type, AnyType): + pass + elif isinstance(type, EllipsisType) or isinstance(type, TupleType): + pass + else: + assert False, 'Unsupported type %s' % type + return result + + def is_defined_type_var(self, tvar: str, context: Node) -> bool: + return self.lookup_qualified(tvar, context).kind == BOUND_TVAR + + def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: + t = [] # type: List[CallableType] + for i, item in enumerate(defn.items): + # TODO support decorated overloaded functions properly + item.is_overload = True + item.func.is_overload = True + item.accept(self) + t.append(cast(CallableType, function_type(item.func, + self.builtin_type('builtins.function')))) + if item.func.is_property and i == 0: + # This defines a property, probably with a setter and/or deleter. + self.analyze_property_with_multi_part_definition(defn) + break + if not [dec for dec in item.decorators + if refers_to_fullname(dec, 'typing.overload')]: + self.fail("'overload' decorator expected", item) + + defn.type = Overloaded(t) + defn.type.line = defn.line + + if self.is_class_scope(): + self.type.names[defn.name()] = SymbolTableNode(MDEF, defn, + typ=defn.type) + defn.info = self.type + elif self.is_func_scope(): + self.add_local(defn, defn) + + def analyze_property_with_multi_part_definition(self, defn: OverloadedFuncDef) -> None: + """Analyze a property defined using multiple methods (e.g., using @x.setter). + + Assume that the first method (@property) has already been analyzed. + """ + defn.is_property = True + items = defn.items + for item in items[1:]: + if len(item.decorators) == 1: + node = item.decorators[0] + if isinstance(node, MemberExpr): + if node.name == 'setter': + # The first item represents the entire property. + defn.items[0].var.is_settable_property = True + # Get abstractness from the original definition. + item.func.is_abstract = items[0].func.is_abstract + else: + self.fail("Decorated property not supported", item) + item.func.accept(self) + + def next_function_tvar_id(self) -> int: + return self.next_function_tvar_id_stack[-1] + + def analyze_function(self, defn: FuncItem) -> None: + is_method = self.is_class_scope() + + tvarnodes = self.add_func_type_variables_to_symbol_table(defn) + next_function_tvar_id = min([self.next_function_tvar_id()] + + [n.tvar_def.id.raw_id - 1 for n in tvarnodes]) + self.next_function_tvar_id_stack.append(next_function_tvar_id) + + if defn.type: + # Signature must be analyzed in the surrounding scope so that + # class-level imported names and type variables are in scope. + defn.type = self.anal_type(defn.type) + self.check_function_signature(defn) + if isinstance(defn, FuncDef): + defn.type = set_callable_name(defn.type, defn) + for arg in defn.arguments: + if arg.initializer: + arg.initializer.accept(self) + self.function_stack.append(defn) + self.enter() + for arg in defn.arguments: + self.add_local(arg.variable, defn) + for arg in defn.arguments: + if arg.initialization_statement: + lvalue = arg.initialization_statement.lvalues[0] + lvalue.accept(self) + + # The first argument of a non-static, non-class method is like 'self' + # (though the name could be different), having the enclosing class's + # instance type. + if is_method and not defn.is_static and not defn.is_class and defn.arguments: + defn.arguments[0].variable.is_self = True + + # First analyze body of the function but ignore nested functions. + self.postpone_nested_functions_stack.append(FUNCTION_FIRST_PHASE_POSTPONE_SECOND) + self.postponed_functions_stack.append([]) + defn.body.accept(self) + + # Analyze nested functions (if any) as a second phase. + self.postpone_nested_functions_stack[-1] = FUNCTION_SECOND_PHASE + for postponed in self.postponed_functions_stack[-1]: + postponed.accept(self) + self.postpone_nested_functions_stack.pop() + self.postponed_functions_stack.pop() + + self.next_function_tvar_id_stack.pop() + disable_typevars(tvarnodes) + + self.leave() + self.function_stack.pop() + + def add_func_type_variables_to_symbol_table( + self, defn: FuncItem) -> List[SymbolTableNode]: + nodes = [] # type: List[SymbolTableNode] + if defn.type: + tt = defn.type + names = self.type_var_names() + items = cast(CallableType, tt).variables + for item in items: + name = item.name + if name in names: + self.name_already_defined(name, defn) + node = self.bind_type_var(name, item, defn) + nodes.append(node) + names.add(name) + return nodes + + def type_var_names(self) -> Set[str]: + if not self.type: + return set() + else: + return set(self.type.type_vars) + + def bind_type_var(self, fullname: str, tvar_def: TypeVarDef, + context: Context) -> SymbolTableNode: + node = self.lookup_qualified(fullname, context) + node.kind = BOUND_TVAR + node.tvar_def = tvar_def + return node + + def check_function_signature(self, fdef: FuncItem) -> None: + sig = cast(CallableType, fdef.type) + if len(sig.arg_types) < len(fdef.arguments): + self.fail('Type signature has too few arguments', fdef) + # Add dummy Any arguments to prevent crashes later. + extra_anys = [AnyType()] * (len(fdef.arguments) - len(sig.arg_types)) + sig.arg_types.extend(extra_anys) + elif len(sig.arg_types) > len(fdef.arguments): + self.fail('Type signature has too many arguments', fdef) + + def visit_class_def(self, defn: ClassDef) -> None: + self.clean_up_bases_and_infer_type_variables(defn) + self.setup_class_def_analysis(defn) + + self.bind_class_type_vars(defn) + + self.analyze_base_classes(defn) + self.analyze_metaclass(defn) + + for decorator in defn.decorators: + self.analyze_class_decorator(defn, decorator) + + self.enter_class(defn) + + self.setup_is_builtinclass(defn) + + # Analyze class body. + defn.defs.accept(self) + + self.calculate_abstract_status(defn.info) + self.setup_type_promotion(defn) + + self.leave_class() + self.unbind_class_type_vars() + + def enter_class(self, defn: ClassDef) -> None: + # Remember previous active class + self.type_stack.append(self.type) + self.locals.append(None) # Add class scope + self.block_depth.append(-1) # The class body increments this to 0 + self.postpone_nested_functions_stack.append(FUNCTION_BOTH_PHASES) + self.type = defn.info + + def leave_class(self) -> None: + """ Restore analyzer state. """ + self.postpone_nested_functions_stack.pop() + self.block_depth.pop() + self.locals.pop() + self.type = self.type_stack.pop() + + def bind_class_type_vars(self, defn: ClassDef) -> None: + """ Unbind type variables of previously active class and bind + the type variables for the active class. + """ + if self.bound_tvars: + disable_typevars(self.bound_tvars) + self.tvar_stack.append(self.bound_tvars) + self.bound_tvars = self.bind_class_type_variables_in_symbol_table(defn.info) + + def unbind_class_type_vars(self) -> None: + """ Unbind the active class' type vars and rebind the + type vars of the previously active class. + """ + disable_typevars(self.bound_tvars) + self.bound_tvars = self.tvar_stack.pop() + if self.bound_tvars: + enable_typevars(self.bound_tvars) + + def analyze_class_decorator(self, defn: ClassDef, decorator: Node) -> None: + decorator.accept(self) + + def setup_is_builtinclass(self, defn: ClassDef): + for decorator in defn.decorators: + if refers_to_fullname(decorator, 'typing.builtinclass'): + defn.is_builtinclass = True + if defn.fullname == 'builtins.object': + # Only 'object' is marked as a built-in class, as otherwise things elsewhere + # would break. We need a better way of dealing with built-in classes. + defn.is_builtinclass = True + + def calculate_abstract_status(self, typ: TypeInfo) -> None: + """Calculate abstract status of a class. + + Set is_abstract of the type to True if the type has an unimplemented + abstract attribute. Also compute a list of abstract attributes. + """ + concrete = set() # type: Set[str] + abstract = [] # type: List[str] + for base in typ.mro: + for name, symnode in base.names.items(): + node = symnode.node + if isinstance(node, OverloadedFuncDef): + # Unwrap an overloaded function definition. We can just + # check arbitrarily the first overload item. If the + # different items have a different abstract status, there + # should be an error reported elsewhere. + func = node.items[0] # type: Node + else: + func = node + if isinstance(func, Decorator): + fdef = func.func + if fdef.is_abstract and name not in concrete: + typ.is_abstract = True + abstract.append(name) + concrete.add(name) + typ.abstract_attributes = sorted(abstract) + + def setup_type_promotion(self, defn: ClassDef) -> None: + """Setup extra, ad-hoc subtyping relationships between classes (promotion). + + This includes things like 'int' being compatible with 'float'. + """ + promote_target = None # type: Type + for decorator in defn.decorators: + if isinstance(decorator, CallExpr): + analyzed = decorator.analyzed + if isinstance(analyzed, PromoteExpr): + # _promote class decorator (undocumented faeture). + promote_target = analyzed.type + if not promote_target: + promotions = (TYPE_PROMOTIONS_PYTHON3 if self.options.python_version[0] >= 3 + else TYPE_PROMOTIONS_PYTHON2) + if defn.fullname in promotions: + promote_target = self.named_type_or_none(promotions[defn.fullname]) + defn.info._promote = promote_target + + def clean_up_bases_and_infer_type_variables(self, defn: ClassDef) -> None: + """Remove extra base classes such as Generic and infer type vars. + + For example, consider this class: + + . class Foo(Bar, Generic[T]): ... + + Now we will remove Generic[T] from bases of Foo and infer that the + type variable 'T' is a type argument of Foo. + + Note that this is performed *before* semantic analysis. + """ + removed = [] # type: List[int] + type_vars = [] # type: List[TypeVarDef] + for i, base_expr in enumerate(defn.base_type_exprs): + try: + base = expr_to_unanalyzed_type(base_expr) + except TypeTranslationError: + # This error will be caught later. + continue + tvars = self.analyze_typevar_declaration(base) + if tvars is not None: + if type_vars: + self.fail('Duplicate Generic in bases', defn) + removed.append(i) + for j, (name, tvar_expr) in enumerate(tvars): + type_vars.append(TypeVarDef(name, j + 1, tvar_expr.values, + tvar_expr.upper_bound, tvar_expr.variance)) + if type_vars: + defn.type_vars = type_vars + if defn.info: + defn.info.type_vars = [tv.name for tv in type_vars] + for i in reversed(removed): + del defn.base_type_exprs[i] + + def analyze_typevar_declaration(self, t: Type) -> Optional[List[Tuple[str, TypeVarExpr]]]: + if not isinstance(t, UnboundType): + return None + unbound = t + sym = self.lookup_qualified(unbound.name, unbound) + if sym is None or sym.node is None: + return None + if sym.node.fullname() == 'typing.Generic': + tvars = [] # type: List[Tuple[str, TypeVarExpr]] + for arg in unbound.args: + tvar = self.analyze_unbound_tvar(arg) + if tvar: + tvars.append(tvar) + else: + self.fail('Free type variable expected in %s[...]' % + sym.node.name(), t) + return tvars + return None + + def analyze_unbound_tvar(self, t: Type) -> Tuple[str, TypeVarExpr]: + if not isinstance(t, UnboundType): + return None + unbound = t + sym = self.lookup_qualified(unbound.name, unbound) + if sym is not None and sym.kind == UNBOUND_TVAR: + return unbound.name, cast(TypeVarExpr, sym.node) + return None + + def setup_class_def_analysis(self, defn: ClassDef) -> None: + """Prepare for the analysis of a class definition.""" + if not defn.info: + defn.info = TypeInfo(SymbolTable(), defn) + defn.info._fullname = defn.info.name() + if self.is_func_scope() or self.type: + kind = MDEF + if self.is_func_scope(): + kind = LDEF + self.add_symbol(defn.name, SymbolTableNode(kind, defn.info), defn) + + def analyze_base_classes(self, defn: ClassDef) -> None: + """Analyze and set up base classes. + + This computes several attributes on the corresponding TypeInfo defn.info + related to the base classes: defn.info.bases, defn.info.mro, and + miscellaneous others (at least tuple_type, fallback_to_any, and is_enum.) + """ + + base_types = [] # type: List[Instance] + info = defn.info + for base_expr in defn.base_type_exprs: + try: + base = self.expr_to_analyzed_type(base_expr) + except TypeTranslationError: + self.fail('Invalid base class', base_expr) + info.fallback_to_any = True + continue + + if isinstance(base, TupleType): + if info.tuple_type: + self.fail("Class has two incompatible bases derived from tuple", defn) + defn.incompatible_baseclass = True + if (not self.is_stub_file + and not info.is_named_tuple + and base.fallback.type.fullname() == 'builtins.tuple'): + self.fail("Tuple[...] not supported as a base class outside a stub file", defn) + info.tuple_type = base + base_types.append(base.fallback) + elif isinstance(base, Instance): + base_types.append(base) + elif isinstance(base, AnyType): + info.fallback_to_any = True + else: + self.fail('Invalid base class', base_expr) + info.fallback_to_any = True + + # Add 'object' as implicit base if there is no other base class. + if (not base_types and defn.fullname != 'builtins.object'): + base_types.append(self.object_type()) + + info.bases = base_types + + # Calculate the MRO. It might be incomplete at this point if + # the bases of defn include classes imported from other + # modules in an import loop. We'll recompute it in ThirdPass. + if not self.verify_base_classes(defn): + info.mro = [] + return + calculate_class_mro(defn, self.fail) + # If there are cyclic imports, we may be missing 'object' in + # the MRO. Fix MRO if needed. + if info.mro and info.mro[-1].fullname() != 'builtins.object': + info.mro.append(self.object_type().type) + + def expr_to_analyzed_type(self, expr: Node) -> Type: + if isinstance(expr, CallExpr): + expr.accept(self) + info = self.check_namedtuple(expr) + if info is None: + # Some form of namedtuple is the only valid type that looks like a call + # expression. This isn't a valid type. + raise TypeTranslationError() + fallback = Instance(info, []) + return TupleType(info.tuple_type.items, fallback=fallback) + typ = expr_to_unanalyzed_type(expr) + return self.anal_type(typ) + + def verify_base_classes(self, defn: ClassDef) -> bool: + info = defn.info + for base in info.bases: + baseinfo = base.type + if self.is_base_class(info, baseinfo): + self.fail('Cycle in inheritance hierarchy', defn) + # Clear bases to forcefully get rid of the cycle. + info.bases = [] + if baseinfo.fullname() == 'builtins.bool': + self.fail("'%s' is not a valid base class" % + baseinfo.name(), defn) + return False + dup = find_duplicate(info.direct_base_classes()) + if dup: + self.fail('Duplicate base class "%s"' % dup.name(), defn) + return False + return True + + def is_base_class(self, t: TypeInfo, s: TypeInfo) -> bool: + """Determine if t is a base class of s (but do not use mro).""" + # Search the base class graph for t, starting from s. + worklist = [s] + visited = {s} + while worklist: + nxt = worklist.pop() + if nxt == t: + return True + for base in nxt.bases: + if base.type not in visited: + worklist.append(base.type) + visited.add(base.type) + return False + + def analyze_metaclass(self, defn: ClassDef) -> None: + if defn.metaclass: + sym = self.lookup_qualified(defn.metaclass, defn) + if sym is not None and not isinstance(sym.node, TypeInfo): + self.fail("Invalid metaclass '%s'" % defn.metaclass, defn) + + def object_type(self) -> Instance: + return self.named_type('__builtins__.object') + + def class_type(self, info: TypeInfo) -> Type: + # Construct a function type whose fallback is cls. + from mypy import checkmember # To avoid import cycle. + leading_type = checkmember.type_object_type(info, self.builtin_type) + if isinstance(leading_type, Overloaded): + # Overloaded __init__ is too complex to handle. Plus it's stubs only. + return AnyType() + else: + return leading_type + + def named_type(self, qualified_name: str, args: List[Type] = None) -> Instance: + sym = self.lookup_qualified(qualified_name, None) + return Instance(cast(TypeInfo, sym.node), args or []) + + def named_type_or_none(self, qualified_name: str) -> Instance: + sym = self.lookup_fully_qualified_or_none(qualified_name) + if not sym: + return None + return Instance(cast(TypeInfo, sym.node), []) + + def is_instance_type(self, t: Type) -> bool: + return isinstance(t, Instance) + + def bind_class_type_variables_in_symbol_table( + self, info: TypeInfo) -> List[SymbolTableNode]: + nodes = [] # type: List[SymbolTableNode] + for var, binder in zip(info.type_vars, info.defn.type_vars): + node = self.bind_type_var(var, binder, info) + nodes.append(node) + return nodes + + def visit_import(self, i: Import) -> None: + for id, as_id in i.ids: + if as_id is not None: + self.add_module_symbol(id, as_id, module_public=True, context=i) + else: + # Modules imported in a stub file without using 'as x' won't get exported when + # doing 'from m import *'. + module_public = not self.is_stub_file + base = id.split('.')[0] + self.add_module_symbol(base, base, module_public=module_public, + context=i) + + def add_module_symbol(self, id: str, as_id: str, module_public: bool, + context: Context) -> None: + if id in self.modules: + m = self.modules[id] + self.add_symbol(as_id, SymbolTableNode(MODULE_REF, m, self.cur_mod_id, + module_public=module_public), context) + else: + self.add_unknown_symbol(as_id, context) + + def visit_import_from(self, imp: ImportFrom) -> None: + import_id = self.correct_relative_import(imp) + if import_id in self.modules: + module = self.modules[import_id] + for id, as_id in imp.names: + node = module.names.get(id) + if node and node.kind != UNBOUND_IMPORTED: + node = self.normalize_type_alias(node, imp) + if not node: + return + imported_id = as_id or id + existing_symbol = self.globals.get(imported_id) + if existing_symbol: + # Import can redefine a variable. They get special treatment. + if self.process_import_over_existing_name( + imported_id, existing_symbol, node, imp): + continue + # 'from m import x as x' exports x in a stub file. + module_public = not self.is_stub_file or as_id is not None + symbol = SymbolTableNode(node.kind, node.node, + self.cur_mod_id, + node.type_override, + module_public=module_public) + self.add_symbol(imported_id, symbol, imp) + else: + message = "Module has no attribute '{}'".format(id) + extra = self.undefined_name_extra_info('{}.{}'.format(import_id, id)) + if extra: + message += " {}".format(extra) + self.fail(message, imp) + else: + # Missing module. + for id, as_id in imp.names: + self.add_unknown_symbol(as_id or id, imp) + + def process_import_over_existing_name(self, + imported_id: str, existing_symbol: SymbolTableNode, + module_symbol: SymbolTableNode, + import_node: ImportBase) -> bool: + if (existing_symbol.kind in (LDEF, GDEF, MDEF) and + isinstance(existing_symbol.node, (Var, FuncDef))): + # This is a valid import over an existing definition in the file. Construct a dummy + # assignment that we'll use to type check the import. + lvalue = NameExpr(imported_id) + lvalue.kind = existing_symbol.kind + lvalue.node = existing_symbol.node + rvalue = NameExpr(imported_id) + rvalue.kind = module_symbol.kind + rvalue.node = module_symbol.node + assignment = AssignmentStmt([lvalue], rvalue) + for node in assignment, lvalue, rvalue: + node.set_line(import_node) + import_node.assignments.append(assignment) + return True + return False + + def normalize_type_alias(self, node: SymbolTableNode, + ctx: Context) -> SymbolTableNode: + if node.fullname in type_aliases: + # Node refers to an aliased type such as typing.List; normalize. + node = self.lookup_qualified(type_aliases[node.fullname], ctx) + if node.fullname == 'typing.DefaultDict': + self.add_module_symbol('collections', '__mypy_collections__', False, ctx) + node = self.lookup_qualified('__mypy_collections__.defaultdict', ctx) + return node + + def correct_relative_import(self, node: Union[ImportFrom, ImportAll]) -> str: + if node.relative == 0: + return node.id + + parts = self.cur_mod_id.split(".") + cur_mod_id = self.cur_mod_id + + rel = node.relative + if self.cur_mod_node.is_package_init_file(): + rel -= 1 + if len(parts) < rel: + self.fail("Relative import climbs too many namespaces", node) + if rel != 0: + cur_mod_id = ".".join(parts[:-rel]) + + return cur_mod_id + (("." + node.id) if node.id else "") + + def visit_import_all(self, i: ImportAll) -> None: + i_id = self.correct_relative_import(i) + if i_id in self.modules: + m = self.modules[i_id] + for name, node in m.names.items(): + node = self.normalize_type_alias(node, i) + if not name.startswith('_') and node.module_public: + existing_symbol = self.globals.get(name) + if existing_symbol: + # Import can redefine a variable. They get special treatment. + if self.process_import_over_existing_name( + name, existing_symbol, node, i): + continue + self.add_symbol(name, SymbolTableNode(node.kind, node.node, + self.cur_mod_id), i) + else: + # Don't add any dummy symbols for 'from x import *' if 'x' is unknown. + pass + + def add_unknown_symbol(self, name: str, context: Context) -> None: + var = Var(name) + var._fullname = self.qualified_name(name) + var.is_ready = True + var.type = AnyType() + self.add_symbol(name, SymbolTableNode(GDEF, var, self.cur_mod_id), context) + + # + # Statements + # + + def visit_block(self, b: Block) -> None: + if b.is_unreachable: + return + self.block_depth[-1] += 1 + for s in b.body: + self.accept(s) + self.block_depth[-1] -= 1 + + def visit_block_maybe(self, b: Block) -> None: + if b: + self.visit_block(b) + + def anal_type(self, t: Type, allow_tuple_literal: bool = False) -> Type: + if t: + if allow_tuple_literal: + # Types such as (t1, t2, ...) only allowed in assignment statements. They'll + # generate errors elsewhere, and Tuple[t1, t2, ...] must be used instead. + if isinstance(t, TupleType): + # Unlike TypeAnalyser, also allow implicit tuple types (without Tuple[...]). + star_count = sum(1 for item in t.items if isinstance(item, StarType)) + if star_count > 1: + self.fail('At most one star type allowed in a tuple', t) + return TupleType([AnyType() for _ in t.items], + self.builtin_type('builtins.tuple'), t.line) + items = [self.anal_type(item, True) + for item in t.items] + return TupleType(items, self.builtin_type('builtins.tuple'), t.line) + a = TypeAnalyser(self.lookup_qualified, + self.lookup_fully_qualified, + self.fail) + return t.accept(a) + else: + return None + + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + for lval in s.lvalues: + self.analyze_lvalue(lval, explicit_type=s.type is not None) + s.rvalue.accept(self) + if s.type: + allow_tuple_literal = isinstance(s.lvalues[-1], (TupleExpr, ListExpr)) + s.type = self.anal_type(s.type, allow_tuple_literal) + else: + # For simple assignments, allow binding type aliases. + # Also set the type if the rvalue is a simple literal. + if (s.type is None and len(s.lvalues) == 1 and + isinstance(s.lvalues[0], NameExpr)): + if s.lvalues[0].is_def: + s.type = self.analyze_simple_literal_type(s.rvalue) + res = analyze_type_alias(s.rvalue, + self.lookup_qualified, + self.lookup_fully_qualified, + self.fail) + if res and (not isinstance(res, Instance) or cast(Instance, res).args): + # TODO: What if this gets reassigned? + name = s.lvalues[0] + node = self.lookup(name.name, name) + node.kind = TYPE_ALIAS + node.type_override = res + if isinstance(s.rvalue, IndexExpr): + s.rvalue.analyzed = TypeAliasExpr(res) + if s.type: + # Store type into nodes. + for lvalue in s.lvalues: + self.store_declared_types(lvalue, s.type) + self.check_and_set_up_type_alias(s) + self.process_typevar_declaration(s) + self.process_namedtuple_definition(s) + + if (len(s.lvalues) == 1 and isinstance(s.lvalues[0], NameExpr) and + s.lvalues[0].name == '__all__' and s.lvalues[0].kind == GDEF and + isinstance(s.rvalue, (ListExpr, TupleExpr))): + self.add_exports(*s.rvalue.items) + + def analyze_simple_literal_type(self, rvalue: Node) -> Optional[Type]: + """Return builtins.int if rvalue is an int literal, etc.""" + if self.weak_opts or self.options.semantic_analysis_only or self.function_stack: + # Skip this if any weak options are set. + # Also skip if we're only doing the semantic analysis pass. + # This is mostly to avoid breaking unit tests. + # Also skip inside a function; this is to avoid confusing + # the code that handles dead code due to isinstance() + # inside type variables with value restrictions (like + # AnyStr). + return None + if isinstance(rvalue, IntExpr): + return self.named_type_or_none('builtins.int') + if isinstance(rvalue, FloatExpr): + return self.named_type_or_none('builtins.float') + if isinstance(rvalue, StrExpr): + return self.named_type_or_none('builtins.str') + if isinstance(rvalue, BytesExpr): + return self.named_type_or_none('builtins.bytes') + if isinstance(rvalue, UnicodeExpr): + return self.named_type_or_none('builtins.unicode') + return None + + def check_and_set_up_type_alias(self, s: AssignmentStmt) -> None: + """Check if assignment creates a type alias and set it up as needed.""" + # For now, type aliases only work at the top level of a module. + if (len(s.lvalues) == 1 and not self.is_func_scope() and not self.type + and not s.type): + lvalue = s.lvalues[0] + if isinstance(lvalue, NameExpr): + if not lvalue.is_def: + # Only a definition can create a type alias, not regular assignment. + return + rvalue = s.rvalue + if isinstance(rvalue, RefExpr): + node = rvalue.node + if isinstance(node, TypeInfo): + # TODO: We should record the fact that this is a variable + # that refers to a type, rather than making this + # just an alias for the type. + self.globals[lvalue.name].node = node + + def analyze_lvalue(self, lval: Node, nested: bool = False, + add_global: bool = False, + explicit_type: bool = False) -> None: + """Analyze an lvalue or assignment target. + + Only if add_global is True, add name to globals table. If nested + is true, the lvalue is within a tuple or list lvalue expression. + """ + + if isinstance(lval, NameExpr): + # Top-level definitions within some statements (at least while) are + # not handled in the first pass, so they have to be added now. + nested_global = (not self.is_func_scope() and + self.block_depth[-1] > 0 and + not self.type) + if (add_global or nested_global) and lval.name not in self.globals: + # Define new global name. + v = Var(lval.name) + v._fullname = self.qualified_name(lval.name) + v.is_ready = False # Type not inferred yet + lval.node = v + lval.is_def = True + lval.kind = GDEF + lval.fullname = v._fullname + self.globals[lval.name] = SymbolTableNode(GDEF, v, + self.cur_mod_id) + elif isinstance(lval.node, Var) and lval.is_def: + # Since the is_def flag is set, this must have been analyzed + # already in the first pass and added to the symbol table. + assert lval.node.name() in self.globals + elif (self.is_func_scope() and lval.name not in self.locals[-1] and + lval.name not in self.global_decls[-1] and + lval.name not in self.nonlocal_decls[-1]): + # Define new local name. + v = Var(lval.name) + lval.node = v + lval.is_def = True + lval.kind = LDEF + lval.fullname = lval.name + self.add_local(v, lval) + elif not self.is_func_scope() and (self.type and + lval.name not in self.type.names): + # Define a new attribute within class body. + v = Var(lval.name) + v.info = self.type + v.is_initialized_in_class = True + v.set_line(lval) + lval.node = v + lval.is_def = True + lval.kind = MDEF + lval.fullname = lval.name + self.type.names[lval.name] = SymbolTableNode(MDEF, v) + else: + # Bind to an existing name. + if explicit_type: + self.name_already_defined(lval.name, lval) + lval.accept(self) + self.check_lvalue_validity(lval.node, lval) + elif isinstance(lval, MemberExpr): + if not add_global: + self.analyze_member_lvalue(lval) + if explicit_type and not self.is_self_member_ref(lval): + self.fail('Type cannot be declared in assignment to non-self ' + 'attribute', lval) + elif isinstance(lval, IndexExpr): + if explicit_type: + self.fail('Unexpected type declaration', lval) + if not add_global: + lval.accept(self) + elif (isinstance(lval, TupleExpr) or + isinstance(lval, ListExpr)): + items = cast(Any, lval).items + if len(items) == 0 and isinstance(lval, TupleExpr): + self.fail("Can't assign to ()", lval) + self.analyze_tuple_or_list_lvalue(cast(Union[ListExpr, TupleExpr], lval), + add_global, explicit_type) + elif isinstance(lval, StarExpr): + if nested: + self.analyze_lvalue(lval.expr, nested, add_global, explicit_type) + else: + self.fail('Starred assignment target must be in a list or tuple', lval) + else: + self.fail('Invalid assignment target', lval) + + def analyze_tuple_or_list_lvalue(self, lval: Union[ListExpr, TupleExpr], + add_global: bool = False, + explicit_type: bool = False) -> None: + """Analyze an lvalue or assignment target that is a list or tuple.""" + items = lval.items + star_exprs = [cast(StarExpr, item) + for item in items + if isinstance(item, StarExpr)] + + if len(star_exprs) > 1: + self.fail('Two starred expressions in assignment', lval) + else: + if len(star_exprs) == 1: + star_exprs[0].valid = True + for i in items: + self.analyze_lvalue(i, nested=True, add_global=add_global, + explicit_type = explicit_type) + + def analyze_member_lvalue(self, lval: MemberExpr) -> None: + lval.accept(self) + if (self.is_self_member_ref(lval) and + self.type.get(lval.name) is None): + # Implicit attribute definition in __init__. + lval.is_def = True + v = Var(lval.name) + v.info = self.type + v.is_ready = False + lval.def_var = v + lval.node = v + self.type.names[lval.name] = SymbolTableNode(MDEF, v) + self.check_lvalue_validity(lval.node, lval) + + def is_self_member_ref(self, memberexpr: MemberExpr) -> bool: + """Does memberexpr to refer to an attribute of self?""" + if not isinstance(memberexpr.expr, NameExpr): + return False + node = memberexpr.expr.node + return isinstance(node, Var) and node.is_self + + def check_lvalue_validity(self, node: Node, ctx: Context) -> None: + if isinstance(node, (TypeInfo, TypeVarExpr)): + self.fail('Invalid assignment target', ctx) + + def store_declared_types(self, lvalue: Node, typ: Type) -> None: + if isinstance(typ, StarType) and not isinstance(lvalue, StarExpr): + self.fail('Star type only allowed for starred expressions', lvalue) + if isinstance(lvalue, RefExpr): + lvalue.is_def = False + if isinstance(lvalue.node, Var): + var = lvalue.node + var.type = typ + var.is_ready = True + # If node is not a variable, we'll catch it elsewhere. + elif isinstance(lvalue, TupleExpr): + if isinstance(typ, TupleType): + if len(lvalue.items) != len(typ.items): + self.fail('Incompatible number of tuple items', lvalue) + return + for item, itemtype in zip(lvalue.items, typ.items): + self.store_declared_types(item, itemtype) + else: + self.fail('Tuple type expected for multiple variables', + lvalue) + elif isinstance(lvalue, StarExpr): + if isinstance(typ, StarType): + self.store_declared_types(lvalue.expr, typ.type) + else: + self.fail('Star type expected for starred expression', lvalue) + else: + # This has been flagged elsewhere as an error, so just ignore here. + pass + + def process_typevar_declaration(self, s: AssignmentStmt) -> None: + """Check if s declares a TypeVar; it yes, store it in symbol table.""" + call = self.get_typevar_declaration(s) + if not call: + return + + lvalue = cast(NameExpr, s.lvalues[0]) + name = lvalue.name + if not lvalue.is_def: + if s.type: + self.fail("Cannot declare the type of a type variable", s) + else: + self.fail("Cannot redefine '%s' as a type variable" % name, s) + return + + if not self.check_typevar_name(call, name, s): + return + + # Constraining types + n_values = call.arg_kinds[1:].count(ARG_POS) + values = self.analyze_types(call.args[1:1 + n_values]) + + res = self.process_typevar_parameters(call.args[1 + n_values:], + call.arg_names[1 + n_values:], + call.arg_kinds[1 + n_values:], + bool(values), + s) + if res is None: + return + variance, upper_bound = res + + # Yes, it's a valid type variable definition! Add it to the symbol table. + node = self.lookup(name, s) + node.kind = UNBOUND_TVAR + TypeVar = TypeVarExpr(name, node.fullname, values, upper_bound, variance) + TypeVar.line = call.line + call.analyzed = TypeVar + node.node = TypeVar + + def check_typevar_name(self, call: CallExpr, name: str, context: Context) -> bool: + if len(call.args) < 1: + self.fail("Too few arguments for TypeVar()", context) + return False + if not isinstance(call.args[0], (StrExpr, BytesExpr)) or not call.arg_kinds[0] == ARG_POS: + self.fail("TypeVar() expects a string literal as first argument", context) + return False + if cast(StrExpr, call.args[0]).value != name: + self.fail("Unexpected TypeVar() argument value", context) + return False + return True + + def get_typevar_declaration(self, s: AssignmentStmt) -> Optional[CallExpr]: + """Returns the TypeVar() call expression if `s` is a type var declaration + or None otherwise. + """ + if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): + return None + if not isinstance(s.rvalue, CallExpr): + return None + call = s.rvalue + if not isinstance(call.callee, RefExpr): + return None + callee = call.callee + if callee.fullname != 'typing.TypeVar': + return None + return call + + def process_typevar_parameters(self, args: List[Node], + names: List[Optional[str]], + kinds: List[int], + has_values: bool, + context: Context) -> Optional[Tuple[int, Type]]: + covariant = False + contravariant = False + upper_bound = self.object_type() # type: Type + for param_value, param_name, param_kind in zip(args, names, kinds): + if not param_kind == ARG_NAMED: + self.fail("Unexpected argument to TypeVar()", context) + return None + if param_name == 'covariant': + if isinstance(param_value, NameExpr): + if param_value.name == 'True': + covariant = True + else: + self.fail("TypeVar 'covariant' may only be 'True'", context) + return None + else: + self.fail("TypeVar 'covariant' may only be 'True'", context) + return None + elif param_name == 'contravariant': + if isinstance(param_value, NameExpr): + if param_value.name == 'True': + contravariant = True + else: + self.fail("TypeVar 'contravariant' may only be 'True'", context) + return None + else: + self.fail("TypeVar 'contravariant' may only be 'True'", context) + return None + elif param_name == 'bound': + if has_values: + self.fail("TypeVar cannot have both values and an upper bound", context) + return None + try: + upper_bound = self.expr_to_analyzed_type(param_value) + except TypeTranslationError: + self.fail("TypeVar 'bound' must be a type", param_value) + return None + elif param_name == 'values': + # Probably using obsolete syntax with values=(...). Explain the current syntax. + self.fail("TypeVar 'values' argument not supported", context) + self.fail("Use TypeVar('T', t, ...) instead of TypeVar('T', values=(t, ...))", + context) + return None + else: + self.fail("Unexpected argument to TypeVar(): {}".format(param_name), context) + return None + + if covariant and contravariant: + self.fail("TypeVar cannot be both covariant and contravariant", context) + return None + elif covariant: + variance = COVARIANT + elif contravariant: + variance = CONTRAVARIANT + else: + variance = INVARIANT + return (variance, upper_bound) + + def process_namedtuple_definition(self, s: AssignmentStmt) -> None: + """Check if s defines a namedtuple; if yes, store the definition in symbol table.""" + if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): + return + lvalue = cast(NameExpr, s.lvalues[0]) + name = lvalue.name + named_tuple = self.check_namedtuple(s.rvalue, name) + if named_tuple is None: + return + # Yes, it's a valid namedtuple definition. Add it to the symbol table. + node = self.lookup(name, s) + node.kind = GDEF # TODO locally defined namedtuple + # TODO call.analyzed + node.node = named_tuple + + def check_namedtuple(self, node: Node, var_name: str = None) -> TypeInfo: + """Check if a call defines a namedtuple. + + The optional var_name argument is the name of the variable to + which this is assigned, if any. + + If it does, return the corresponding TypeInfo. Return None otherwise. + + If the definition is invalid but looks like a namedtuple, + report errors but return (some) TypeInfo. + """ + if not isinstance(node, CallExpr): + return None + call = node + if not isinstance(call.callee, RefExpr): + return None + callee = call.callee + fullname = callee.fullname + if fullname not in ('collections.namedtuple', 'typing.NamedTuple'): + return None + items, types, ok = self.parse_namedtuple_args(call, fullname) + if not ok: + # Error. Construct dummy return value. + return self.build_namedtuple_typeinfo('namedtuple', [], []) + else: + # Give it a unique name derived from the line number. + name = cast(StrExpr, call.args[0]).value + if name != var_name: + name += '@' + str(call.line) + info = self.build_namedtuple_typeinfo(name, items, types) + # Store it as a global just in case it would remain anonymous. + self.globals[name] = SymbolTableNode(GDEF, info, self.cur_mod_id) + call.analyzed = NamedTupleExpr(info).set_line(call.line) + return info + + def parse_namedtuple_args(self, call: CallExpr, + fullname: str) -> Tuple[List[str], List[Type], bool]: + # TODO Share code with check_argument_count in checkexpr.py? + args = call.args + if len(args) < 2: + return self.fail_namedtuple_arg("Too few arguments for namedtuple()", call) + if len(args) > 2: + return self.fail_namedtuple_arg("Too many arguments for namedtuple()", call) + if call.arg_kinds != [ARG_POS, ARG_POS]: + return self.fail_namedtuple_arg("Unexpected arguments to namedtuple()", call) + if not isinstance(args[0], (StrExpr, BytesExpr)): + return self.fail_namedtuple_arg( + "namedtuple() expects a string literal as the first argument", call) + types = [] # type: List[Type] + ok = True + if not isinstance(args[1], ListExpr): + if fullname == 'collections.namedtuple' and isinstance(args[1], (StrExpr, BytesExpr)): + str_expr = cast(StrExpr, args[1]) + items = str_expr.value.split() + else: + return self.fail_namedtuple_arg( + "List literal expected as the second argument to namedtuple()", call) + else: + listexpr = args[1] + if fullname == 'collections.namedtuple': + # The fields argument contains just names, with implicit Any types. + if any(not isinstance(item, (StrExpr, BytesExpr)) for item in listexpr.items): + return self.fail_namedtuple_arg("String literal expected as namedtuple() item", + call) + items = [cast(StrExpr, item).value for item in listexpr.items] + else: + # The fields argument contains (name, type) tuples. + items, types, ok = self.parse_namedtuple_fields_with_types(listexpr.items, call) + if not types: + types = [AnyType() for _ in items] + return items, types, ok + + def parse_namedtuple_fields_with_types(self, nodes: List[Node], + context: Context) -> Tuple[List[str], List[Type], bool]: + items = [] # type: List[str] + types = [] # type: List[Type] + for item in nodes: + if isinstance(item, TupleExpr): + if len(item.items) != 2: + return self.fail_namedtuple_arg("Invalid NamedTuple field definition", + item) + name, type_node = item.items + if isinstance(name, (StrExpr, BytesExpr)): + items.append(name.value) + else: + return self.fail_namedtuple_arg("Invalid NamedTuple() field name", item) + try: + type = expr_to_unanalyzed_type(type_node) + except TypeTranslationError: + return self.fail_namedtuple_arg('Invalid field type', type_node) + types.append(self.anal_type(type)) + else: + return self.fail_namedtuple_arg("Tuple expected as NamedTuple() field", item) + return items, types, True + + def fail_namedtuple_arg(self, message: str, + context: Context) -> Tuple[List[str], List[Type], bool]: + self.fail(message, context) + return [], [], False + + def build_namedtuple_typeinfo(self, name: str, items: List[str], + types: List[Type]) -> TypeInfo: + symbols = SymbolTable() + tup = NamedTupleType(items, types, self.named_type('__builtins__.tuple', types)) + class_def = ClassDef(name, Block([])) + class_def.fullname = self.qualified_name(name) + info = NamedTupleTypeInfo(tup, symbols, class_def) + vars = [Var(item, typ) for item, typ in zip(items, types)] + # Add named tuple items as attributes. + # TODO: Make them read-only. + for var in vars: + var.info = info + symbols[var.name()] = SymbolTableNode(MDEF, var) + # Add __init__, _replace, _asdict methods and _fields static field + self.add_namedtuple_method('__init__', symbols, info, NoneTyp(), + self.make_factory_args(vars, ARG_POS)) + self.add_namedtuple_method('_replace', symbols, info, self_type(info), + self.make_factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) + # TODO: refine to OrderedDict[Union[types]] + self.add_namedtuple_method('_asdict', symbols, info, AnyType()) + return info + + def make_factory_args(self, vars: List[Var], kind: int, + initializer: Expression = None) -> List[Argument]: + return [Argument(var, var.type, initializer, kind) for var in vars] + + def add_namedtuple_method(self, funcname: str, symbols: SymbolTable, info: TypeInfo, + ret: Type, args: List[Argument] = []) -> None: + types = [cast(Type, None)] + [arg.type_annotation for arg in args] + args = [Argument(Var('__self'), NoneTyp(), None, ARG_POS)] + args + items = [arg.variable.name() for arg in args] + arg_kinds = [arg.kind for arg in args] + # TODO: Make sure that the self argument name is not visible? + signature = CallableType(types, arg_kinds, items, ret, + self.named_type('__builtins__.function'), + name=info.name()) + func = FuncDef(funcname, + args, + Block([]), + typ=signature) + func.info = info + symbols[funcname] = SymbolTableNode(MDEF, func) + + def analyze_types(self, items: List[Node]) -> List[Type]: + result = [] # type: List[Type] + for node in items: + try: + result.append(self.anal_type(expr_to_unanalyzed_type(node))) + except TypeTranslationError: + self.fail('Type expected', node) + result.append(AnyType()) + return result + + def visit_decorator(self, dec: Decorator) -> None: + for d in dec.decorators: + d.accept(self) + removed = [] # type: List[int] + no_type_check = False + for i, d in enumerate(dec.decorators): + # A bunch of decorators are special cased here. + if refers_to_fullname(d, 'abc.abstractmethod'): + removed.append(i) + dec.func.is_abstract = True + self.check_decorated_function_is_method('abstractmethod', dec) + elif refers_to_fullname(d, 'asyncio.tasks.coroutine'): + removed.append(i) + elif refers_to_fullname(d, 'builtins.staticmethod'): + removed.append(i) + dec.func.is_static = True + dec.var.is_staticmethod = True + self.check_decorated_function_is_method('staticmethod', dec) + elif refers_to_fullname(d, 'builtins.classmethod'): + removed.append(i) + dec.func.is_class = True + dec.var.is_classmethod = True + self.check_decorated_function_is_method('classmethod', dec) + elif (refers_to_fullname(d, 'builtins.property') or + refers_to_fullname(d, 'abc.abstractproperty')): + removed.append(i) + dec.func.is_property = True + dec.var.is_property = True + if refers_to_fullname(d, 'abc.abstractproperty'): + dec.func.is_abstract = True + self.check_decorated_function_is_method('property', dec) + if len(dec.func.arguments) > 1: + self.fail('Too many arguments', dec.func) + elif refers_to_fullname(d, 'typing.no_type_check'): + dec.var.type = AnyType() + no_type_check = True + for i in reversed(removed): + del dec.decorators[i] + if not dec.is_overload or dec.var.is_property: + if self.is_func_scope(): + self.add_symbol(dec.var.name(), SymbolTableNode(LDEF, dec), + dec) + elif self.type: + dec.var.info = self.type + dec.var.is_initialized_in_class = True + self.add_symbol(dec.var.name(), SymbolTableNode(MDEF, dec), + dec) + if not no_type_check: + dec.func.accept(self) + if dec.decorators and dec.var.is_property: + self.fail('Decorated property not supported', dec) + + def check_decorated_function_is_method(self, decorator: str, + context: Context) -> None: + if not self.type or self.is_func_scope(): + self.fail("'%s' used with a non-method" % decorator, context) + + def visit_expression_stmt(self, s: ExpressionStmt) -> None: + s.expr.accept(self) + + def visit_return_stmt(self, s: ReturnStmt) -> None: + if not self.is_func_scope(): + self.fail("'return' outside function", s) + if s.expr: + s.expr.accept(self) + + def visit_raise_stmt(self, s: RaiseStmt) -> None: + if s.expr: + s.expr.accept(self) + if s.from_expr: + s.from_expr.accept(self) + + def visit_assert_stmt(self, s: AssertStmt) -> None: + if s.expr: + s.expr.accept(self) + + def visit_operator_assignment_stmt(self, + s: OperatorAssignmentStmt) -> None: + s.lvalue.accept(self) + s.rvalue.accept(self) + if (isinstance(s.lvalue, NameExpr) and s.lvalue.name == '__all__' and + s.lvalue.kind == GDEF and isinstance(s.rvalue, (ListExpr, TupleExpr))): + self.add_exports(*s.rvalue.items) + + def visit_while_stmt(self, s: WhileStmt) -> None: + s.expr.accept(self) + self.loop_depth += 1 + s.body.accept(self) + self.loop_depth -= 1 + self.visit_block_maybe(s.else_body) + + def visit_for_stmt(self, s: ForStmt) -> None: + s.expr.accept(self) + + # Bind index variables and check if they define new names. + self.analyze_lvalue(s.index) + + self.loop_depth += 1 + self.visit_block(s.body) + self.loop_depth -= 1 + + self.visit_block_maybe(s.else_body) + + def visit_break_stmt(self, s: BreakStmt) -> None: + if self.loop_depth == 0: + self.fail("'break' outside loop", s, True, blocker=True) + + def visit_continue_stmt(self, s: ContinueStmt) -> None: + if self.loop_depth == 0: + self.fail("'continue' outside loop", s, True, blocker=True) + + def visit_if_stmt(self, s: IfStmt) -> None: + infer_reachability_of_if_statement(s, pyversion=self.options.python_version) + for i in range(len(s.expr)): + s.expr[i].accept(self) + self.visit_block(s.body[i]) + self.visit_block_maybe(s.else_body) + + def visit_try_stmt(self, s: TryStmt) -> None: + self.analyze_try_stmt(s, self) + + def analyze_try_stmt(self, s: TryStmt, visitor: NodeVisitor, + add_global: bool = False) -> None: + s.body.accept(visitor) + for type, var, handler in zip(s.types, s.vars, s.handlers): + if type: + type.accept(visitor) + if var: + self.analyze_lvalue(var, add_global=add_global) + handler.accept(visitor) + if s.else_body: + s.else_body.accept(visitor) + if s.finally_body: + s.finally_body.accept(visitor) + + def visit_with_stmt(self, s: WithStmt) -> None: + for e, n in zip(s.expr, s.target): + e.accept(self) + if n: + self.analyze_lvalue(n) + self.visit_block(s.body) + + def visit_del_stmt(self, s: DelStmt) -> None: + s.expr.accept(self) + if not self.is_valid_del_target(s.expr): + self.fail('Invalid delete target', s) + + def is_valid_del_target(self, s: Node) -> bool: + if isinstance(s, (IndexExpr, NameExpr, MemberExpr)): + return True + elif isinstance(s, TupleExpr): + return all(self.is_valid_del_target(item) for item in s.items) + + def visit_global_decl(self, g: GlobalDecl) -> None: + for name in g.names: + if name in self.nonlocal_decls[-1]: + self.fail("Name '{}' is nonlocal and global".format(name), g) + self.global_decls[-1].add(name) + + def visit_nonlocal_decl(self, d: NonlocalDecl) -> None: + if not self.is_func_scope(): + self.fail("nonlocal declaration not allowed at module level", d) + else: + for name in d.names: + for table in reversed(self.locals[:-1]): + if table is not None and name in table: + break + else: + self.fail("No binding for nonlocal '{}' found".format(name), d) + + if self.locals[-1] is not None and name in self.locals[-1]: + self.fail("Name '{}' is already defined in local " + "scope before nonlocal declaration".format(name), d) + + if name in self.global_decls[-1]: + self.fail("Name '{}' is nonlocal and global".format(name), d) + self.nonlocal_decls[-1].add(name) + + def visit_print_stmt(self, s: PrintStmt) -> None: + for arg in s.args: + arg.accept(self) + if s.target: + s.target.accept(self) + + def visit_exec_stmt(self, s: ExecStmt) -> None: + s.expr.accept(self) + if s.variables1: + s.variables1.accept(self) + if s.variables2: + s.variables2.accept(self) + + # + # Expressions + # + + def visit_name_expr(self, expr: NameExpr) -> None: + n = self.lookup(expr.name, expr) + if n: + if n.kind == BOUND_TVAR: + self.fail("'{}' is a type variable and only valid in type " + "context".format(expr.name), expr) + else: + expr.kind = n.kind + expr.node = n.node + expr.fullname = n.fullname + + def visit_super_expr(self, expr: SuperExpr) -> None: + if not self.type: + self.fail('"super" used outside class', expr) + return + expr.info = self.type + + def visit_tuple_expr(self, expr: TupleExpr) -> None: + for item in expr.items: + item.accept(self) + + def visit_list_expr(self, expr: ListExpr) -> None: + for item in expr.items: + item.accept(self) + + def visit_set_expr(self, expr: SetExpr) -> None: + for item in expr.items: + item.accept(self) + + def visit_dict_expr(self, expr: DictExpr) -> None: + for key, value in expr.items: + key.accept(self) + value.accept(self) + + def visit_star_expr(self, expr: StarExpr) -> None: + if not expr.valid: + self.fail('Can use starred expression only as assignment target', expr) + else: + expr.expr.accept(self) + + def visit_yield_from_expr(self, e: YieldFromExpr) -> None: + if not self.is_func_scope(): # not sure + self.fail("'yield from' outside function", e, True, blocker=True) + else: + self.function_stack[-1].is_generator = True + if e.expr: + e.expr.accept(self) + + def visit_call_expr(self, expr: CallExpr) -> None: + """Analyze a call expression. + + Some call expressions are recognized as special forms, including + cast(...) and Any(...). + """ + expr.callee.accept(self) + if refers_to_fullname(expr.callee, 'typing.cast'): + # Special form cast(...). + if not self.check_fixed_args(expr, 2, 'cast'): + return + # Translate first argument to an unanalyzed type. + try: + target = expr_to_unanalyzed_type(expr.args[0]) + except TypeTranslationError: + self.fail('Cast target is not a type', expr) + return + # Piggyback CastExpr object to the CallExpr object; it takes + # precedence over the CallExpr semantics. + expr.analyzed = CastExpr(expr.args[1], target) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, 'builtins.reveal_type'): + if not self.check_fixed_args(expr, 1, 'reveal_type'): + return + expr.analyzed = RevealTypeExpr(expr.args[0]) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, 'typing.Any'): + # Special form Any(...). + if not self.check_fixed_args(expr, 1, 'Any'): + return + expr.analyzed = CastExpr(expr.args[0], AnyType()) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, 'typing._promote'): + # Special form _promote(...). + if not self.check_fixed_args(expr, 1, '_promote'): + return + # Translate first argument to an unanalyzed type. + try: + target = expr_to_unanalyzed_type(expr.args[0]) + except TypeTranslationError: + self.fail('Argument 1 to _promote is not a type', expr) + return + expr.analyzed = PromoteExpr(target) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, 'builtins.dict'): + expr.analyzed = self.translate_dict_call(expr) + else: + # Normal call expression. + for a in expr.args: + a.accept(self) + + if (isinstance(expr.callee, MemberExpr) and + isinstance(expr.callee.expr, NameExpr) and + expr.callee.expr.name == '__all__' and + expr.callee.expr.kind == GDEF and + expr.callee.name in ('append', 'extend')): + if expr.callee.name == 'append' and expr.args: + self.add_exports(expr.args[0]) + elif (expr.callee.name == 'extend' and expr.args and + isinstance(expr.args[0], (ListExpr, TupleExpr))): + self.add_exports(*expr.args[0].items) + + def translate_dict_call(self, call: CallExpr) -> Optional[DictExpr]: + """Translate 'dict(x=y, ...)' to {'x': y, ...}. + + For other variants of dict(...), return None. + """ + if not call.args: + return None + if not all(kind == ARG_NAMED for kind in call.arg_kinds): + # Must still accept those args. + for a in call.args: + a.accept(self) + return None + expr = DictExpr([(StrExpr(key), value) + for key, value in zip(call.arg_names, call.args)]) + expr.set_line(call) + expr.accept(self) + return expr + + def check_fixed_args(self, expr: CallExpr, numargs: int, + name: str) -> bool: + """Verify that expr has specified number of positional args. + + Return True if the arguments are valid. + """ + s = 's' + if numargs == 1: + s = '' + if len(expr.args) != numargs: + self.fail("'%s' expects %d argument%s" % (name, numargs, s), + expr) + return False + if expr.arg_kinds != [ARG_POS] * numargs: + self.fail("'%s' must be called with %s positional argument%s" % + (name, numargs, s), expr) + return False + return True + + def visit_member_expr(self, expr: MemberExpr) -> None: + base = expr.expr + base.accept(self) + # Bind references to module attributes. + if isinstance(base, RefExpr) and base.kind == MODULE_REF: + file = cast(MypyFile, base.node) + n = file.names.get(expr.name, None) if file is not None else None + if n: + n = self.normalize_type_alias(n, expr) + if not n: + return + expr.kind = n.kind + expr.fullname = n.fullname + expr.node = n.node + else: + # We only catch some errors here; the rest will be + # catched during type checking. + # + # This way we can report a larger number of errors in + # one type checker run. If we reported errors here, + # the build would terminate after semantic analysis + # and we wouldn't be able to report any type errors. + full_name = '%s.%s' % (file.fullname() if file is not None else None, expr.name) + if full_name in obsolete_name_mapping: + self.fail("Module has no attribute %r (it's now called %r)" % ( + expr.name, obsolete_name_mapping[full_name]), expr) + + def visit_op_expr(self, expr: OpExpr) -> None: + expr.left.accept(self) + expr.right.accept(self) + + def visit_comparison_expr(self, expr: ComparisonExpr) -> None: + for operand in expr.operands: + operand.accept(self) + + def visit_unary_expr(self, expr: UnaryExpr) -> None: + expr.expr.accept(self) + + def visit_index_expr(self, expr: IndexExpr) -> None: + expr.base.accept(self) + if refers_to_class_or_function(expr.base): + # Special form -- type application. + # Translate index to an unanalyzed type. + types = [] # type: List[Type] + if isinstance(expr.index, TupleExpr): + items = expr.index.items + else: + items = [expr.index] + for item in items: + try: + typearg = expr_to_unanalyzed_type(item) + except TypeTranslationError: + self.fail('Type expected within [...]', expr) + return + typearg = self.anal_type(typearg) + types.append(typearg) + expr.analyzed = TypeApplication(expr.base, types) + expr.analyzed.line = expr.line + else: + expr.index.accept(self) + + def visit_slice_expr(self, expr: SliceExpr) -> None: + if expr.begin_index: + expr.begin_index.accept(self) + if expr.end_index: + expr.end_index.accept(self) + if expr.stride: + expr.stride.accept(self) + + def visit_cast_expr(self, expr: CastExpr) -> None: + expr.expr.accept(self) + expr.type = self.anal_type(expr.type) + + def visit_reveal_type_expr(self, expr: RevealTypeExpr) -> None: + expr.expr.accept(self) + + def visit_type_application(self, expr: TypeApplication) -> None: + expr.expr.accept(self) + for i in range(len(expr.types)): + expr.types[i] = self.anal_type(expr.types[i]) + + def visit_list_comprehension(self, expr: ListComprehension) -> None: + expr.generator.accept(self) + + def visit_set_comprehension(self, expr: SetComprehension) -> None: + expr.generator.accept(self) + + def visit_dictionary_comprehension(self, expr: DictionaryComprehension) -> None: + self.enter() + self.analyze_comp_for(expr) + expr.key.accept(self) + expr.value.accept(self) + self.leave() + self.analyze_comp_for_2(expr) + + def visit_generator_expr(self, expr: GeneratorExpr) -> None: + self.enter() + self.analyze_comp_for(expr) + expr.left_expr.accept(self) + self.leave() + self.analyze_comp_for_2(expr) + + def analyze_comp_for(self, expr: Union[GeneratorExpr, + DictionaryComprehension]) -> None: + """Analyses the 'comp_for' part of comprehensions (part 1). + + That is the part after 'for' in (x for x in l if p). This analyzes + variables and conditions which are analyzed in a local scope. + """ + for i, (index, sequence, conditions) in enumerate(zip(expr.indices, + expr.sequences, + expr.condlists)): + if i > 0: + sequence.accept(self) + # Bind index variables. + self.analyze_lvalue(index) + for cond in conditions: + cond.accept(self) + + def analyze_comp_for_2(self, expr: Union[GeneratorExpr, + DictionaryComprehension]) -> None: + """Analyses the 'comp_for' part of comprehensions (part 2). + + That is the part after 'for' in (x for x in l if p). This analyzes + the 'l' part which is analyzed in the surrounding scope. + """ + expr.sequences[0].accept(self) + + def visit_func_expr(self, expr: FuncExpr) -> None: + self.analyze_function(expr) + + def visit_conditional_expr(self, expr: ConditionalExpr) -> None: + expr.if_expr.accept(self) + expr.cond.accept(self) + expr.else_expr.accept(self) + + def visit_backquote_expr(self, expr: BackquoteExpr) -> None: + expr.expr.accept(self) + + def visit__promote_expr(self, expr: PromoteExpr) -> None: + expr.type = self.anal_type(expr.type) + + def visit_yield_expr(self, expr: YieldExpr) -> None: + if not self.is_func_scope(): + self.fail("'yield' outside function", expr, True, blocker=True) + else: + self.function_stack[-1].is_generator = True + if expr.expr: + expr.expr.accept(self) + + # + # Helpers + # + + def lookup(self, name: str, ctx: Context) -> SymbolTableNode: + """Look up an unqualified name in all active namespaces.""" + # 1a. Name declared using 'global x' takes precedence + if name in self.global_decls[-1]: + if name in self.globals: + return self.globals[name] + else: + self.name_not_defined(name, ctx) + return None + # 1b. Name declared using 'nonlocal x' takes precedence + if name in self.nonlocal_decls[-1]: + for table in reversed(self.locals[:-1]): + if table is not None and name in table: + return table[name] + else: + self.name_not_defined(name, ctx) + return None + # 2. Class attributes (if within class definition) + if self.is_class_scope() and name in self.type.names: + return self.type[name] + # 3. Local (function) scopes + for table in reversed(self.locals): + if table is not None and name in table: + return table[name] + # 4. Current file global scope + if name in self.globals: + return self.globals[name] + # 5. Builtins + b = self.globals.get('__builtins__', None) + if b: + table = cast(MypyFile, b.node).names + if name in table: + if name[0] == "_" and name[1] != "_": + self.name_not_defined(name, ctx) + return None + node = table[name] + return node + # Give up. + self.name_not_defined(name, ctx) + self.check_for_obsolete_short_name(name, ctx) + return None + + def check_for_obsolete_short_name(self, name: str, ctx: Context) -> None: + matches = [obsolete_name + for obsolete_name in obsolete_name_mapping + if obsolete_name.rsplit('.', 1)[-1] == name] + if len(matches) == 1: + self.note("(Did you mean '{}'?)".format(obsolete_name_mapping[matches[0]]), ctx) + + def lookup_qualified(self, name: str, ctx: Context) -> SymbolTableNode: + if '.' not in name: + return self.lookup(name, ctx) + else: + parts = name.split('.') + n = self.lookup(parts[0], ctx) # type: SymbolTableNode + if n: + for i in range(1, len(parts)): + if isinstance(n.node, TypeInfo): + if n.node.mro is None: + # We haven't yet analyzed the class `n.node`. Fall back to direct + # lookup in the names declared directly under it, without its base + # classes. This can happen when we have a forward reference to a + # nested class, and the reference is bound before the outer class + # has been fully semantically analyzed. + # + # A better approach would be to introduce a new analysis pass or + # to move things around between passes, but this unblocks a common + # use case even though this is a little limited in case there is + # inheritance involved. + result = n.node.names.get(parts[i]) + else: + result = n.node.get(parts[i]) + n = result + elif isinstance(n.node, MypyFile): + n = n.node.names.get(parts[i], None) + # TODO: What if node is Var or FuncDef? + if not n: + self.name_not_defined(name, ctx) + break + if n: + n = self.normalize_type_alias(n, ctx) + return n + + def builtin_type(self, fully_qualified_name: str) -> Instance: + node = self.lookup_fully_qualified(fully_qualified_name) + info = cast(TypeInfo, node.node) + return Instance(info, []) + + def lookup_fully_qualified(self, name: str) -> SymbolTableNode: + """Lookup a fully qualified name. + + Assume that the name is defined. This happens in the global namespace -- the local + module namespace is ignored. + """ + assert '.' in name + parts = name.split('.') + n = self.modules[parts[0]] + for i in range(1, len(parts) - 1): + n = cast(MypyFile, n.names[parts[i]].node) + return n.names[parts[-1]] + + def lookup_fully_qualified_or_none(self, name: str) -> SymbolTableNode: + """Lookup a fully qualified name. + + Assume that the name is defined. This happens in the global namespace -- the local + module namespace is ignored. + """ + assert '.' in name + parts = name.split('.') + n = self.modules[parts[0]] + for i in range(1, len(parts) - 1): + next_sym = n.names.get(parts[i]) + if not next_sym: + return None + n = cast(MypyFile, next_sym.node) + return n.names.get(parts[-1]) + + def qualified_name(self, n: str) -> str: + return self.cur_mod_id + '.' + n + + def enter(self) -> None: + self.locals.append(SymbolTable()) + self.global_decls.append(set()) + self.nonlocal_decls.append(set()) + # -1 since entering block will increment this to 0. + self.block_depth.append(-1) + + def leave(self) -> None: + self.locals.pop() + self.global_decls.pop() + self.nonlocal_decls.pop() + self.block_depth.pop() + + def is_func_scope(self) -> bool: + return self.locals[-1] is not None + + def is_class_scope(self) -> bool: + return self.type is not None and not self.is_func_scope() + + def add_symbol(self, name: str, node: SymbolTableNode, + context: Context) -> None: + if self.is_func_scope(): + if name in self.locals[-1]: + # Flag redefinition unless this is a reimport of a module. + if not (node.kind == MODULE_REF and + self.locals[-1][name].node == node.node): + self.name_already_defined(name, context) + self.locals[-1][name] = node + elif self.type: + self.type.names[name] = node + else: + existing = self.globals.get(name) + if existing and (not isinstance(node.node, MypyFile) or + existing.node != node.node) and existing.kind != UNBOUND_IMPORTED: + # Modules can be imported multiple times to support import + # of multiple submodules of a package (e.g. a.x and a.y). + ok = False + # Only report an error if the symbol collision provides a different type. + if existing.type and node.type and is_same_type(existing.type, node.type): + ok = True + if not ok: + self.name_already_defined(name, context) + self.globals[name] = node + + def add_var(self, v: Var, ctx: Context) -> None: + if self.is_func_scope(): + self.add_local(v, ctx) + else: + self.globals[v.name()] = SymbolTableNode(GDEF, v, self.cur_mod_id) + v._fullname = self.qualified_name(v.name()) + + def add_local(self, node: Union[Var, FuncBase], ctx: Context) -> None: + name = node.name() + if name in self.locals[-1]: + self.name_already_defined(name, ctx) + node._fullname = name + self.locals[-1][name] = SymbolTableNode(LDEF, node) + + def add_exports(self, *exps: Node) -> None: + for exp in exps: + if isinstance(exp, StrExpr): + self.all_exports.add(exp.value) + + def check_no_global(self, n: str, ctx: Context, + is_overloaded_func: bool = False) -> None: + if n in self.globals: + prev_is_overloaded = isinstance(self.globals[n], OverloadedFuncDef) + if is_overloaded_func and prev_is_overloaded: + self.fail("Nonconsecutive overload {} found".format(n), ctx) + elif prev_is_overloaded: + self.fail("Definition of '{}' missing 'overload'".format(n), ctx) + else: + self.name_already_defined(n, ctx) + + def name_not_defined(self, name: str, ctx: Context) -> None: + message = "Name '{}' is not defined".format(name) + extra = self.undefined_name_extra_info(name) + if extra: + message += ' {}'.format(extra) + self.fail(message, ctx) + + def name_already_defined(self, name: str, ctx: Context) -> None: + self.fail("Name '{}' already defined".format(name), ctx) + + def fail(self, msg: str, ctx: Context, serious: bool = False, *, + blocker: bool = False) -> None: + if (not serious and + not self.options.check_untyped_defs and + self.function_stack and + self.function_stack[-1].is_dynamic()): + return + self.errors.report(ctx.get_line(), msg, blocker=blocker) + + def note(self, msg: str, ctx: Context) -> None: + if (not self.options.check_untyped_defs and + self.function_stack and + self.function_stack[-1].is_dynamic()): + return + self.errors.report(ctx.get_line(), msg, severity='note') + + def undefined_name_extra_info(self, fullname: str) -> Optional[str]: + if fullname in obsolete_name_mapping: + return "(it's now called '{}')".format(obsolete_name_mapping[fullname]) + else: + return None + + def accept(self, node: Node) -> None: + try: + node.accept(self) + except Exception as err: + report_internal_error(err, self.errors.file, node.line) + + +class FirstPass(NodeVisitor): + """First phase of semantic analysis. + + See docstring of 'analyze()' below for a description of what this does. + """ + + def __init__(self, sem: SemanticAnalyzer) -> None: + self.sem = sem + self.pyversion = sem.options.python_version + + def analyze(self, file: MypyFile, fnam: str, mod_id: str) -> None: + """Perform the first analysis pass. + + Populate module global table. Resolve the full names of + definitions not nested within functions and construct type + info structures, but do not resolve inter-definition + references such as base classes. + + Also add implicit definitions such as __name__. + + In this phase we don't resolve imports. For 'from ... import', + we generate dummy symbol table nodes for the imported names, + and these will get resolved in later phases of semantic + analysis. + """ + sem = self.sem + sem.cur_mod_id = mod_id + sem.errors.set_file(fnam) + sem.globals = SymbolTable() + sem.global_decls = [set()] + sem.nonlocal_decls = [set()] + sem.block_depth = [0] + + defs = file.defs + + # Add implicit definitions of module '__name__' etc. + for name, t in implicit_module_attrs.items(): + v = Var(name, UnboundType(t)) + v._fullname = self.sem.qualified_name(name) + self.sem.globals[name] = SymbolTableNode(GDEF, v, self.sem.cur_mod_id) + + for d in defs: + d.accept(self) + + # Add implicit definition of literals/keywords to builtins, as we + # cannot define a variable with them explicitly. + if mod_id == 'builtins': + literal_types = [ + ('None', NoneTyp()), + # reveal_type is a mypy-only function that gives an error with the type of its arg + ('reveal_type', AnyType()), + ] # type: List[Tuple[str, Type]] + + # TODO(ddfisher): This guard is only needed because mypy defines + # fake builtins for its tests which often don't define bool. If + # mypy is fast enough that we no longer need those, this + # conditional check should be removed. + if 'bool' in self.sem.globals: + bool_type = self.sem.named_type('bool') + literal_types.extend([ + ('True', bool_type), + ('False', bool_type), + ('__debug__', bool_type), + ]) + + for name, typ in literal_types: + v = Var(name, typ) + v._fullname = self.sem.qualified_name(name) + self.sem.globals[name] = SymbolTableNode(GDEF, v, self.sem.cur_mod_id) + + def visit_block(self, b: Block) -> None: + if b.is_unreachable: + return + self.sem.block_depth[-1] += 1 + for node in b.body: + node.accept(self) + self.sem.block_depth[-1] -= 1 + + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + for lval in s.lvalues: + self.analyze_lvalue(lval, explicit_type=s.type is not None) + + def visit_func_def(self, func: FuncDef) -> None: + sem = self.sem + func.is_conditional = sem.block_depth[-1] > 0 + func._fullname = sem.qualified_name(func.name()) + if func.name() in sem.globals: + # Already defined in this module. + original_sym = sem.globals[func.name()] + if original_sym.kind == UNBOUND_IMPORTED: + # Ah this is an imported name. We can't resolve them now, so we'll postpone + # this until the main phase of semantic analysis. + return + original_def = original_sym.node + if sem.is_conditional_func(original_def, func): + # Conditional function definition -- multiple defs are ok. + func.original_def = cast(FuncDef, original_def) + else: + # Report error. + sem.check_no_global(func.name(), func) + else: + sem.globals[func.name()] = SymbolTableNode(GDEF, func, sem.cur_mod_id) + + def visit_overloaded_func_def(self, func: OverloadedFuncDef) -> None: + self.sem.check_no_global(func.name(), func, True) + func._fullname = self.sem.qualified_name(func.name()) + self.sem.globals[func.name()] = SymbolTableNode(GDEF, func, + self.sem.cur_mod_id) + + def visit_class_def(self, cdef: ClassDef) -> None: + self.sem.check_no_global(cdef.name, cdef) + cdef.fullname = self.sem.qualified_name(cdef.name) + info = TypeInfo(SymbolTable(), cdef) + info.set_line(cdef.line) + cdef.info = info + self.sem.globals[cdef.name] = SymbolTableNode(GDEF, info, + self.sem.cur_mod_id) + self.process_nested_classes(cdef) + + def process_nested_classes(self, outer_def: ClassDef) -> None: + for node in outer_def.defs.body: + if isinstance(node, ClassDef): + node.info = TypeInfo(SymbolTable(), node) + if outer_def.fullname: + node.info._fullname = outer_def.fullname + '.' + node.info.name() + else: + node.info._fullname = node.info.name() + symbol = SymbolTableNode(MDEF, node.info) + outer_def.info.names[node.name] = symbol + self.process_nested_classes(node) + + def visit_import_from(self, node: ImportFrom) -> None: + # We can't bind module names during the first pass, as the target module might be + # unprocessed. However, we add dummy unbound imported names to the symbol table so + # that we at least know that the name refers to a module. + node.is_top_level = True + for name, as_name in node.names: + imported_name = as_name or name + if imported_name not in self.sem.globals: + self.sem.add_symbol(imported_name, SymbolTableNode(UNBOUND_IMPORTED, None), node) + + def visit_import(self, node: Import) -> None: + node.is_top_level = True + # This is similar to visit_import_from -- see the comment there. + for id, as_id in node.ids: + imported_id = as_id or id + if imported_id not in self.sem.globals: + self.sem.add_symbol(imported_id, SymbolTableNode(UNBOUND_IMPORTED, None), node) + else: + # If the previous symbol is a variable, this should take precedence. + self.sem.globals[imported_id] = SymbolTableNode(UNBOUND_IMPORTED, None) + + def visit_import_all(self, node: ImportAll) -> None: + node.is_top_level = True + + def visit_while_stmt(self, s: WhileStmt) -> None: + s.body.accept(self) + if s.else_body: + s.else_body.accept(self) + + def visit_for_stmt(self, s: ForStmt) -> None: + self.analyze_lvalue(s.index) + s.body.accept(self) + if s.else_body: + s.else_body.accept(self) + + def visit_with_stmt(self, s: WithStmt) -> None: + for n in s.target: + if n: + self.analyze_lvalue(n) + s.body.accept(self) + + def visit_decorator(self, d: Decorator) -> None: + d.var._fullname = self.sem.qualified_name(d.var.name()) + self.sem.add_symbol(d.var.name(), SymbolTableNode(GDEF, d.var), d) + + def visit_if_stmt(self, s: IfStmt) -> None: + infer_reachability_of_if_statement(s, pyversion=self.pyversion) + for node in s.body: + node.accept(self) + if s.else_body: + s.else_body.accept(self) + + def visit_try_stmt(self, s: TryStmt) -> None: + self.sem.analyze_try_stmt(s, self, add_global=True) + + def analyze_lvalue(self, lvalue: Node, explicit_type: bool = False) -> None: + self.sem.analyze_lvalue(lvalue, add_global=True, explicit_type=explicit_type) + + +class ThirdPass(TraverserVisitor): + """The third and final pass of semantic analysis. + + Check type argument counts and values of generic types, and perform some + straightforward type inference. + """ + + def __init__(self, modules: Dict[str, MypyFile], errors: Errors) -> None: + self.modules = modules + self.errors = errors + + def visit_file(self, file_node: MypyFile, fnam: str) -> None: + self.errors.set_file(fnam) + self.accept(file_node) + + def accept(self, node: Node) -> None: + try: + node.accept(self) + except Exception as err: + report_internal_error(err, self.errors.file, node.line) + + def visit_block(self, b: Block) -> None: + if b.is_unreachable: + return + super().visit_block(b) + + def visit_func_def(self, fdef: FuncDef) -> None: + self.errors.push_function(fdef.name()) + self.analyze(fdef.type) + super().visit_func_def(fdef) + self.errors.pop_function() + + def visit_class_def(self, tdef: ClassDef) -> None: + for type in tdef.info.bases: + self.analyze(type) + # Recompute MRO now that we have analyzed all modules, to pick + # up superclasses of bases imported from other modules in an + # import loop. (Only do so if we succeeded the first time.) + if tdef.info.mro: + tdef.info.mro = [] # Force recomputation + calculate_class_mro(tdef, self.fail) + super().visit_class_def(tdef) + + def visit_decorator(self, dec: Decorator) -> None: + """Try to infer the type of the decorated function. + + This lets us resolve references to decorated functions during + type checking when there are cyclic imports, as otherwise the + type might not be available when we need it. + + This basically uses a simple special-purpose type inference + engine just for decorators. + """ + super().visit_decorator(dec) + if dec.var.is_property: + # Decorators are expected to have a callable type (it's a little odd). + if dec.func.type is None: + dec.var.type = CallableType( + [AnyType()], + [ARG_POS], + [None], + AnyType(), + self.builtin_type('function'), + name=dec.var.name()) + elif isinstance(dec.func.type, CallableType): + dec.var.type = dec.func.type + return + decorator_preserves_type = True + for expr in dec.decorators: + preserve_type = False + if isinstance(expr, RefExpr) and isinstance(expr.node, FuncDef): + if is_identity_signature(expr.node.type): + preserve_type = True + if not preserve_type: + decorator_preserves_type = False + break + if decorator_preserves_type: + # No non-identity decorators left. We can trivially infer the type + # of the function here. + dec.var.type = function_type(dec.func, self.builtin_type('function')) + if dec.decorators: + if returns_any_if_called(dec.decorators[0]): + # The outermost decorator will return Any so we know the type of the + # decorated function. + dec.var.type = AnyType() + sig = find_fixed_callable_return(dec.decorators[0]) + if sig: + # The outermost decorator always returns the same kind of function, + # so we know that this is the type of the decoratored function. + orig_sig = function_type(dec.func, self.builtin_type('function')) + sig.name = orig_sig.items()[0].name + dec.var.type = sig + + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + self.analyze(s.type) + super().visit_assignment_stmt(s) + + def visit_cast_expr(self, e: CastExpr) -> None: + self.analyze(e.type) + super().visit_cast_expr(e) + + def visit_reveal_type_expr(self, e: RevealTypeExpr) -> None: + super().visit_reveal_type_expr(e) + + def visit_type_application(self, e: TypeApplication) -> None: + for type in e.types: + self.analyze(type) + super().visit_type_application(e) + + # Helpers + + def analyze(self, type: Type) -> None: + if type: + analyzer = TypeAnalyserPass3(self.fail) + type.accept(analyzer) + + def fail(self, msg: str, ctx: Context) -> None: + self.errors.report(ctx.get_line(), msg) + + def builtin_type(self, name: str, args: List[Type] = None) -> Instance: + names = self.modules['builtins'] + sym = names.names[name] + assert isinstance(sym.node, TypeInfo) + return Instance(sym.node, args or []) + + +def self_type(typ: TypeInfo) -> Union[Instance, TupleType]: + """For a non-generic type, return instance type representing the type. + For a generic G type with parameters T1, .., Tn, return G[T1, ..., Tn]. + """ + tv = [] # type: List[Type] + for i in range(len(typ.type_vars)): + tv.append(TypeVarType(typ.defn.type_vars[i])) + inst = Instance(typ, tv) + if typ.tuple_type is None: + return inst + return typ.tuple_type.self_type(inst) + + +def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: + if isinstance(sig, CallableType): + return replace_leading_arg_type(sig, new) + else: + sig = cast(Overloaded, sig) + return Overloaded([cast(CallableType, replace_implicit_first_type(i, new)) + for i in sig.items()]) + + +def set_callable_name(sig: Type, fdef: FuncDef) -> Type: + if isinstance(sig, FunctionLike): + if fdef.info: + return sig.with_name( + '"{}" of "{}"'.format(fdef.name(), fdef.info.name())) + else: + return sig.with_name('"{}"'.format(fdef.name())) + else: + return sig + + +def refers_to_fullname(node: Node, fullname: str) -> bool: + """Is node a name or member expression with the given full name?""" + return isinstance(node, RefExpr) and node.fullname == fullname + + +def refers_to_class_or_function(node: Node) -> bool: + """Does semantically analyzed node refer to a class?""" + return (isinstance(node, RefExpr) and + isinstance(node.node, (TypeInfo, FuncDef, OverloadedFuncDef))) + + +def calculate_class_mro(defn: ClassDef, fail: Callable[[str, Context], None]) -> None: + try: + defn.info.calculate_mro() + except MroError: + fail("Cannot determine consistent method resolution order " + '(MRO) for "%s"' % defn.name, defn) + defn.info.mro = [] + # The property of falling back to Any is inherited. + defn.info.fallback_to_any = any(baseinfo.fallback_to_any for baseinfo in defn.info.mro) + + +def find_duplicate(list: List[T]) -> T: + """If the list has duplicates, return one of the duplicates. + + Otherwise, return None. + """ + for i in range(1, len(list)): + if list[i] in list[:i]: + return list[i] + return None + + +def disable_typevars(nodes: List[SymbolTableNode]) -> None: + for node in nodes: + assert node.kind in (BOUND_TVAR, UNBOUND_TVAR) + node.kind = UNBOUND_TVAR + + +def enable_typevars(nodes: List[SymbolTableNode]) -> None: + for node in nodes: + assert node.kind in (BOUND_TVAR, UNBOUND_TVAR) + node.kind = BOUND_TVAR + + +def remove_imported_names_from_symtable(names: SymbolTable, + module: str) -> None: + """Remove all imported names from the symbol table of a module.""" + removed = [] # type: List[str] + for name, node in names.items(): + if node.node is None: + continue + fullname = node.node.fullname() + prefix = fullname[:fullname.rfind('.')] + if prefix != module: + removed.append(name) + for name in removed: + del names[name] + + +def infer_reachability_of_if_statement(s: IfStmt, + pyversion: Tuple[int, int]) -> None: + for i in range(len(s.expr)): + result = infer_if_condition_value(s.expr[i], pyversion) + if result == ALWAYS_FALSE: + # The condition is always false, so we skip the if/elif body. + mark_block_unreachable(s.body[i]) + elif result == ALWAYS_TRUE: + # This condition is always true, so all of the remaining + # elif/else bodies will never be executed. + for body in s.body[i + 1:]: + mark_block_unreachable(body) + if s.else_body: + mark_block_unreachable(s.else_body) + break + + +def infer_if_condition_value(expr: Node, pyversion: Tuple[int, int]) -> int: + """Infer whether if condition is always true/false. + + Return ALWAYS_TRUE if always true, ALWAYS_FALSE if always false, + and TRUTH_VALUE_UNKNOWN otherwise. + """ + name = '' + negated = False + alias = expr + if isinstance(alias, UnaryExpr): + if alias.op == 'not': + expr = alias.expr + negated = True + if isinstance(expr, NameExpr): + name = expr.name + elif isinstance(expr, MemberExpr): + name = expr.name + result = TRUTH_VALUE_UNKNOWN + if name == 'PY2': + result = ALWAYS_TRUE if pyversion[0] == 2 else ALWAYS_FALSE + elif name == 'PY3': + result = ALWAYS_TRUE if pyversion[0] == 3 else ALWAYS_FALSE + elif name == 'MYPY': + result = ALWAYS_TRUE + if negated: + if result == ALWAYS_TRUE: + result = ALWAYS_FALSE + elif result == ALWAYS_FALSE: + result = ALWAYS_TRUE + return result + + +def mark_block_unreachable(block: Block) -> None: + block.is_unreachable = True + block.accept(MarkImportsUnreachableVisitor()) + + +class MarkImportsUnreachableVisitor(TraverserVisitor): + """Visitor that flags all imports nested within a node as unreachable.""" + + def visit_import(self, node: Import) -> None: + node.is_unreachable = True + + def visit_import_from(self, node: ImportFrom) -> None: + node.is_unreachable = True + + def visit_import_all(self, node: ImportAll) -> None: + node.is_unreachable = True + + +def is_identity_signature(sig: Type) -> bool: + """Is type a callable of form T -> T (where T is a type variable)?""" + if isinstance(sig, CallableType) and sig.arg_kinds == [ARG_POS]: + if isinstance(sig.arg_types[0], TypeVarType) and isinstance(sig.ret_type, TypeVarType): + return sig.arg_types[0].id == sig.ret_type.id + return False + + +def returns_any_if_called(expr: Node) -> bool: + """Return True if we can predict that expr will return Any if called. + + This only uses information available during semantic analysis so this + will sometimes return False because of insufficient information (as + type inference hasn't run yet). + """ + if isinstance(expr, RefExpr): + if isinstance(expr.node, FuncDef): + typ = expr.node.type + if typ is None: + # No signature -> default to Any. + return True + # Explicit Any return? + return isinstance(typ, CallableType) and isinstance(typ.ret_type, AnyType) + elif isinstance(expr.node, Var): + typ = expr.node.type + return typ is None or isinstance(typ, AnyType) + elif isinstance(expr, CallExpr): + return returns_any_if_called(expr.callee) + return False + + +def find_fixed_callable_return(expr: Node) -> Optional[CallableType]: + if isinstance(expr, RefExpr): + if isinstance(expr.node, FuncDef): + typ = expr.node.type + if typ: + if isinstance(typ, CallableType) and has_no_typevars(typ.ret_type): + if isinstance(typ.ret_type, CallableType): + return typ.ret_type + elif isinstance(expr, CallExpr): + t = find_fixed_callable_return(expr.callee) + if t: + if isinstance(t.ret_type, CallableType): + return t.ret_type + return None + + +def has_no_typevars(typ: Type) -> bool: + return is_same_type(typ, erase_typevars(typ)) diff --git a/mypy/types.py b/mypy/types.py index bd31bf25d49f3..84a0eb12b607b 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -770,7 +770,7 @@ class NamedTupleType(TupleType): def __init__(self, attrs: List[str], *args) -> None: self.attrs = attrs super().__init__(*args) - + def self_type(self, inst: Instance) -> 'NamedTupleType': return NamedTupleType(self.attrs, self.items, inst) diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index bc37c40208198..9c2d6c4a86d8f 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -1,209 +1,209 @@ -[case testNamedTupleUsedAsTuple] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -a, b = x -b = x[0] -a = x[1] -a, b, c = x # E: Need more than 2 values to unpack (3 expected) -x[2] # E: Tuple index out of range - -[case testAccessingNamedTupleAttributes] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -x.x -x.y -x.z # E: "X" has no attribute "z" - -[case testCreateNamedTupleWithPositionalArguments] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = X(1, 'x') -x.x -x.z # E: "X" has no attribute "z" -x = X(1) # E: Too few arguments for "X" -x = X(1, 2, 3) # E: Too many arguments for "X" - -[case testCreateNamedTupleWithKeywordArguments] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = X(x=1, y='x') -x = X(1, y='x') -x = X(x=1, z=1) # E: Unexpected keyword argument "z" for "X" -x = X(y=1) # E: Missing positional argument "x" in call to "X" - -[case testCreateNamedTupleAndUseAsTuple] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = X(1, 'x') -a, b = x -a, b, c = x # E: Need more than 2 values to unpack (3 expected) - -[case testNamedTupleWithItemTypes] -from typing import NamedTuple -N = NamedTuple('N', [('a', int), - ('b', str)]) -n = N(1, 'x') -s = n.a # type: str # E: Incompatible types in assignment (expression has type "int", \ - variable has type "str") -i = n.b # type: int # E: Incompatible types in assignment (expression has type "str", \ - variable has type "int") -x, y = n -x = y # E: Incompatible types in assignment (expression has type "str", variable has type "int") - -[case testNamedTupleConstructorArgumentTypes] -from typing import NamedTuple -N = NamedTuple('N', [('a', int), - ('b', str)]) -n = N('x', 'x') # E: Argument 1 to "N" has incompatible type "str"; expected "int" -n = N(1, b=2) # E: Argument 2 to "N" has incompatible type "int"; expected "str" -N(1, 'x') -N(b='x', a=1) - -[case testNamedTupleAsBaseClass] -from typing import NamedTuple -N = NamedTuple('N', [('a', int), - ('b', str)]) -class X(N): - pass -x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" -s = '' -i = 0 -s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") -i, s = x -s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") - -[case testNamedTupleAsBaseClass2] -from typing import NamedTuple -class X(NamedTuple('N', [('a', int), - ('b', str)])): - pass -x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" -s = '' -i = 0 -s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") -i, s = x -s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") - -[case testTwoNamedTuplesAsBaseClasses] -from typing import NamedTuple -A = NamedTuple('A', [('a', int)]) -B = NamedTuple('B', [('a', int)]) -class X(A, B): # E: Class has two incompatible bases derived from tuple - pass - -[case testTwoNamedTuplesAsBaseClasses2] -from typing import NamedTuple -A = NamedTuple('A', [('a', int)]) -class X(A, NamedTuple('B', [('a', int)])): # E: Class has two incompatible bases derived from tuple - pass - -[case testSelfTypeWithNamedTupleAsBase] -from typing import NamedTuple -A = NamedTuple('A', [('a', int), ('b', str)]) -class B(A): - def f(self, x: int) -> None: - self.f(self.a) - self.f(self.b) # E: Argument 1 to "f" of "B" has incompatible type "str"; expected "int" - i = 0 - s = '' - i, s = self - i, i = self # E: Incompatible types in assignment (expression has type "str", \ - variable has type "int") -[out] -main: note: In member "f" of class "B": - -[case testTypeReferenceToClassDerivedFromNamedTuple] -from typing import NamedTuple -A = NamedTuple('A', [('a', int), ('b', str)]) -class B(A): - def f(self, x: 'B') -> None: - i = 0 - s = '' - self = x - i, s = x - i, s = x.a, x.b - i, s = x.a, x.a # E: Incompatible types in assignment (expression has type "int", \ - variable has type "str") - i, i = self # E: Incompatible types in assignment (expression has type "str", \ - variable has type "int") -[out] -main: note: In member "f" of class "B": - -[case testNamedTupleSubtyping] -from typing import NamedTuple, Tuple -A = NamedTuple('A', [('a', int), ('b', str)]) -class B(A): pass -a = A(1, '') -b = B(1, '') -t = None # type: Tuple[int, str] -b = a # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "A") -b = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "B") -t = a -t = (1, '') -t = b -a = b - -[case testNamedTupleSimpleTypeInference] -from typing import NamedTuple, Tuple -A = NamedTuple('A', [('a', int)]) -l = [A(1), A(2)] -a = A(1) -a = l[0] -(i,) = l[0] -i, i = l[0] # E: Need more than 1 value to unpack (2 expected) -l = [A(1)] -a = (1,) # E: Incompatible types in assignment (expression has type "Tuple[int]", \ - variable has type "A") -[builtins fixtures/list.py] - -[case testNamedTupleMissingClassAttribute] -import collections -MyNamedTuple = collections.namedtuple('MyNamedTuple', ['spam', 'eggs']) -MyNamedTuple.x # E: "MyNamedTuple" has no attribute "x" - -[case testNamedTupleEmptyItems] -from typing import NamedTuple -A = NamedTuple('A', []) -[builtins fixtures/list.py] - -[case testNamedTupleProperty] -from typing import NamedTuple -A = NamedTuple('A', [('a', int)]) -class B(A): - @property - def b(self) -> int: - return self.a -class C(B): pass -B(1).b -C(2).b -[builtins fixtures/property.py] - -[case testNamedTupleAsDict] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -a = x._asdict() - -[case testNamedTupleReplace] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -x = x._replace(y=5) - -[case testNamedTupleReplaceTyped] -from typing import NamedTuple - -X = NamedTuple('X', [('x', int), ('y', str)]) -x = None # type: X -x = x._replace(x=5) -x = x._replace(y=5) # E: Argument 1 to X has incompatible type "int"; expected "str" +[case testNamedTupleUsedAsTuple] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +a, b = x +b = x[0] +a = x[1] +a, b, c = x # E: Need more than 2 values to unpack (3 expected) +x[2] # E: Tuple index out of range + +[case testAccessingNamedTupleAttributes] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +x.x +x.y +x.z # E: "X" has no attribute "z" + +[case testCreateNamedTupleWithPositionalArguments] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = X(1, 'x') +x.x +x.z # E: "X" has no attribute "z" +x = X(1) # E: Too few arguments for "X" +x = X(1, 2, 3) # E: Too many arguments for "X" + +[case testCreateNamedTupleWithKeywordArguments] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = X(x=1, y='x') +x = X(1, y='x') +x = X(x=1, z=1) # E: Unexpected keyword argument "z" for "X" +x = X(y=1) # E: Missing positional argument "x" in call to "X" + +[case testCreateNamedTupleAndUseAsTuple] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = X(1, 'x') +a, b = x +a, b, c = x # E: Need more than 2 values to unpack (3 expected) + +[case testNamedTupleWithItemTypes] +from typing import NamedTuple +N = NamedTuple('N', [('a', int), + ('b', str)]) +n = N(1, 'x') +s = n.a # type: str # E: Incompatible types in assignment (expression has type "int", \ + variable has type "str") +i = n.b # type: int # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +x, y = n +x = y # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testNamedTupleConstructorArgumentTypes] +from typing import NamedTuple +N = NamedTuple('N', [('a', int), + ('b', str)]) +n = N('x', 'x') # E: Argument 1 to "N" has incompatible type "str"; expected "int" +n = N(1, b=2) # E: Argument 2 to "N" has incompatible type "int"; expected "str" +N(1, 'x') +N(b='x', a=1) + +[case testNamedTupleAsBaseClass] +from typing import NamedTuple +N = NamedTuple('N', [('a', int), + ('b', str)]) +class X(N): + pass +x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" +s = '' +i = 0 +s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") +i, s = x +s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testNamedTupleAsBaseClass2] +from typing import NamedTuple +class X(NamedTuple('N', [('a', int), + ('b', str)])): + pass +x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" +s = '' +i = 0 +s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") +i, s = x +s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testTwoNamedTuplesAsBaseClasses] +from typing import NamedTuple +A = NamedTuple('A', [('a', int)]) +B = NamedTuple('B', [('a', int)]) +class X(A, B): # E: Class has two incompatible bases derived from tuple + pass + +[case testTwoNamedTuplesAsBaseClasses2] +from typing import NamedTuple +A = NamedTuple('A', [('a', int)]) +class X(A, NamedTuple('B', [('a', int)])): # E: Class has two incompatible bases derived from tuple + pass + +[case testSelfTypeWithNamedTupleAsBase] +from typing import NamedTuple +A = NamedTuple('A', [('a', int), ('b', str)]) +class B(A): + def f(self, x: int) -> None: + self.f(self.a) + self.f(self.b) # E: Argument 1 to "f" of "B" has incompatible type "str"; expected "int" + i = 0 + s = '' + i, s = self + i, i = self # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +[out] +main: note: In member "f" of class "B": + +[case testTypeReferenceToClassDerivedFromNamedTuple] +from typing import NamedTuple +A = NamedTuple('A', [('a', int), ('b', str)]) +class B(A): + def f(self, x: 'B') -> None: + i = 0 + s = '' + self = x + i, s = x + i, s = x.a, x.b + i, s = x.a, x.a # E: Incompatible types in assignment (expression has type "int", \ + variable has type "str") + i, i = self # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +[out] +main: note: In member "f" of class "B": + +[case testNamedTupleSubtyping] +from typing import NamedTuple, Tuple +A = NamedTuple('A', [('a', int), ('b', str)]) +class B(A): pass +a = A(1, '') +b = B(1, '') +t = None # type: Tuple[int, str] +b = a # E: Incompatible types in assignment (expression has type "A", variable has type "B") +a = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "A") +b = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "B") +t = a +t = (1, '') +t = b +a = b + +[case testNamedTupleSimpleTypeInference] +from typing import NamedTuple, Tuple +A = NamedTuple('A', [('a', int)]) +l = [A(1), A(2)] +a = A(1) +a = l[0] +(i,) = l[0] +i, i = l[0] # E: Need more than 1 value to unpack (2 expected) +l = [A(1)] +a = (1,) # E: Incompatible types in assignment (expression has type "Tuple[int]", \ + variable has type "A") +[builtins fixtures/list.py] + +[case testNamedTupleMissingClassAttribute] +import collections +MyNamedTuple = collections.namedtuple('MyNamedTuple', ['spam', 'eggs']) +MyNamedTuple.x # E: "MyNamedTuple" has no attribute "x" + +[case testNamedTupleEmptyItems] +from typing import NamedTuple +A = NamedTuple('A', []) +[builtins fixtures/list.py] + +[case testNamedTupleProperty] +from typing import NamedTuple +A = NamedTuple('A', [('a', int)]) +class B(A): + @property + def b(self) -> int: + return self.a +class C(B): pass +B(1).b +C(2).b +[builtins fixtures/property.py] + +[case testNamedTupleAsDict] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +a = x._asdict() + +[case testNamedTupleReplace] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +x = x._replace(y=5) + +[case testNamedTupleReplaceTyped] +from typing import NamedTuple + +X = NamedTuple('X', [('x', int), ('y', str)]) +x = None # type: X +x = x._replace(x=5) +x = x._replace(y=5) # E: Argument 1 to X has incompatible type "int"; expected "str" From 2b6738039f9e05561e9cfbe7729f08a7bc52b22a Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Tue, 12 Jul 2016 14:16:32 +0300 Subject: [PATCH 46/74] Add partial support for calling '_replace' statically --- mypy/messages.py | 1 + mypy/nodes.py | 18 ++++++++---------- mypy/semanal.py | 20 +++++++++++--------- mypy/subtypes.py | 16 +++++++++++++--- mypy/typeanal.py | 14 +++++++++++++- mypy/types.py | 19 ++++++++++++++++--- test-data/unit/check-namedtuple.test | 10 ++++++++++ 7 files changed, 72 insertions(+), 26 deletions(-) diff --git a/mypy/messages.py b/mypy/messages.py index 209010764eca3..dd21dc4e2d7f7 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -486,6 +486,7 @@ def incompatible_argument(self, n: int, m: int, callee: CallableType, arg_type: except IndexError: # Varargs callees expected_type = callee.arg_types[-1] arg_type_str, expected_type_str = self.format_distinctly(arg_type, expected_type) + import pdb; pdb.set_trace() if arg_kind == ARG_STAR: arg_type_str = '*' + arg_type_str elif arg_kind == ARG_STAR2: diff --git a/mypy/nodes.py b/mypy/nodes.py index 9211d65485106..6e1d3bd353271 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1179,6 +1179,7 @@ def __init__(self, callee: Expression, args: List[Expression], arg_kinds: List[i arg_names: List[str] = None, analyzed: Expression = None) -> None: if not arg_names: arg_names = [None] * len(args) + self.callee = callee self.args = args self.arg_kinds = arg_kinds @@ -1976,16 +1977,13 @@ def deserialize(cls, data: JsonDict) -> 'TypeInfo': return ti -class NamedTupleTypeInfo(TypeInfo): - is_named_tuple = True - bases = None # type: List[mypy.types.Instance] - - def __init__(self, tup: 'mypy.types.NamedTupleType', *args) -> None: - super().__init__(*args) - self.tuple_type = tup - self.bases = [tup.fallback] - self.mro = [cast(TypeInfo, self)] + tup.fallback.type.mro - +def NamedTupleTypeInfo(tup: 'mypy.types.NamedTupleType', *args) -> None: + info = TypeInfo(*args) + info.tuple_type = tup + info.bases = [tup.fallback] + info.is_named_tuple = True + info.mro = [info] + tup.fallback.type.mro + return info class SymbolTableNode: # Kind of node. Possible values: diff --git a/mypy/semanal.py b/mypy/semanal.py index fed7475ae557a..8f77f908a41f8 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1527,23 +1527,25 @@ def fail_namedtuple_arg(self, message: str, def build_namedtuple_typeinfo(self, name: str, items: List[str], types: List[Type]) -> TypeInfo: symbols = SymbolTable() - tup = NamedTupleType(items, types, self.named_type('__builtins__.tuple', types)) + tup = NamedTupleType(name, items, types, self.named_type('__builtins__.tuple', types)) class_def = ClassDef(name, Block([])) class_def.fullname = self.qualified_name(name) info = NamedTupleTypeInfo(tup, symbols, class_def) + info.attrs = items vars = [Var(item, typ) for item, typ in zip(items, types)] # Add named tuple items as attributes. # TODO: Make them read-only. for var in vars: var.info = info symbols[var.name()] = SymbolTableNode(MDEF, var) + this_type = self_type(info) # Add __init__, _replace, _asdict methods and _fields static field - self.add_namedtuple_method('__init__', symbols, info, NoneTyp(), - self.make_factory_args(vars, ARG_POS)) - self.add_namedtuple_method('_replace', symbols, info, self_type(info), - self.make_factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) + self.add_namedtuple_method('_replace', symbols, info, this_type, this_type, + self.make_factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) + self.add_namedtuple_method('__init__', symbols, info, this_type, NoneTyp(), + self.make_factory_args(vars, ARG_NAMED)) # TODO: refine to OrderedDict[Union[types]] - self.add_namedtuple_method('_asdict', symbols, info, AnyType()) + self.add_namedtuple_method('_asdict', symbols, info, this_type, AnyType(), []) return info def make_factory_args(self, vars: List[Var], kind: int, @@ -1551,9 +1553,9 @@ def make_factory_args(self, vars: List[Var], kind: int, return [Argument(var, var.type, initializer, kind) for var in vars] def add_namedtuple_method(self, funcname: str, symbols: SymbolTable, info: TypeInfo, - ret: Type, args: List[Argument] = []) -> None: - types = [cast(Type, None)] + [arg.type_annotation for arg in args] - args = [Argument(Var('__self'), NoneTyp(), None, ARG_POS)] + args + this_type: Type, ret: Type, args: List[Argument]) -> None: + args = [Argument(Var('self'), this_type, None, ARG_POS)] + args + types = [arg.type_annotation for arg in args] items = [arg.variable.name() for arg in args] arg_kinds = [arg.kind for arg in args] # TODO: Make sure that the self argument name is not visible? diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 8e9017a7b355f..ee08cf452413e 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -2,7 +2,7 @@ from mypy.types import ( Type, AnyType, UnboundType, TypeVisitor, ErrorType, Void, NoneTyp, - Instance, TypeVarType, CallableType, TupleType, UnionType, Overloaded, ErasedType, TypeList, + Instance, TypeVarType, CallableType, TupleType, NamedTupleType, UnionType, Overloaded, ErasedType, TypeList, PartialType, DeletedType, UninhabitedType, TypeType, is_named_instance ) import mypy.applytype @@ -178,11 +178,13 @@ def visit_tuple_type(self, left: TupleType) -> bool: iter_type = right.args[0] return all(is_subtype(li, iter_type) for li in left.items) return False + elif isinstance(right, NamedTupleType) and not isinstance(left, NamedTupleType): + return False elif isinstance(right, TupleType): if len(left.items) != len(right.items): return False - for i in range(len(left.items)): - if not is_subtype(left.items[i], right.items[i], self.check_type_parameter): + for l, r in zip(left.items, right.items): + if not is_subtype(l, r, self.check_type_parameter): return False if not is_subtype(left.fallback, right.fallback, self.check_type_parameter): return False @@ -190,6 +192,14 @@ def visit_tuple_type(self, left: TupleType) -> bool: else: return False + def visit_namedtuple_type(self, left: NamedTupleType) -> bool: + if isinstance(self.right, NamedTupleType): + # left.attrs.issubset(right.attrs) is insufficient + # (a: int, b: str) is not subtype of (b: int, a: str) + if any(l != r for l, r in zip(left.attrs, self.right.attrs)): + return False + return self.visit_tuple_type(left) + def visit_overloaded(self, left: Overloaded) -> bool: right = self.right if isinstance(right, Instance): diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 0493d3a15f6b8..cce852a1be525 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -3,7 +3,7 @@ from typing import Callable, cast, List, Tuple from mypy.types import ( - Type, UnboundType, TypeVarType, TupleType, UnionType, Instance, AnyType, CallableType, + Type, UnboundType, TypeVarType, TupleType, NamedTupleType, UnionType, Instance, AnyType, CallableType, Void, NoneTyp, DeletedType, TypeList, TypeVarDef, TypeVisitor, StarType, PartialType, EllipsisType, UninhabitedType, TypeType ) @@ -169,6 +169,12 @@ def visit_unbound_type(self, t: UnboundType) -> Type: if t.args: self.fail('Generic tuple types not supported', t) return AnyType() + # import pdb; pdb.set_trace() + if info.is_named_tuple: + return NamedTupleType(info.name, info.attrs, + self.anal_array(info.tuple_type.items), + fallback=instance, + line=t.line) return TupleType(self.anal_array(info.tuple_type.items), fallback=instance, line=t.line) @@ -217,6 +223,12 @@ def visit_tuple_type(self, t: TupleType) -> Type: fallback = t.fallback if t.fallback else self.builtin_type('builtins.tuple', [AnyType()]) return TupleType(self.anal_array(t.items), fallback, t.line) + def visit_namedtuple_type(self, t: NamedTupleType) -> Type: + return NamedTupleType(t.name, t.attrs, + self.anal_array(t.items), + fallback=t.fallback, + line=t.line) + def visit_star_type(self, t: StarType) -> Type: return StarType(t.type.accept(self), t.line) diff --git a/mypy/types.py b/mypy/types.py index 84a0eb12b607b..61005b6c6c975 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -735,6 +735,7 @@ class TupleType(Type): def __init__(self, items: List[Type], fallback: Instance, line: int = -1, implicit: bool = False) -> None: + #import pdb; pdb.set_trace() self.items = items self.fallback = fallback self.implicit = implicit @@ -766,13 +767,21 @@ def self_type(self, inst: Instance) -> 'TupleType': class NamedTupleType(TupleType): attrs = None # type: List[str] + name = None # type: str - def __init__(self, attrs: List[str], *args) -> None: + def __init__(self, name: str, attrs: List[str], *args, **kwargs) -> None: self.attrs = attrs - super().__init__(*args) + self.name = name + super().__init__(*args, **kwargs) def self_type(self, inst: Instance) -> 'NamedTupleType': - return NamedTupleType(self.attrs, self.items, inst) + return NamedTupleType(self.name, self.attrs, self.items, inst) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + try: + return visitor.visit_namedtuple_type(self) + except: + return super().accept(visitor) class StarType(Type): @@ -1238,6 +1247,10 @@ def visit_tuple_type(self, t): return 'Tuple[{}, fallback={}]'.format(s, t.fallback.accept(self)) return 'Tuple[{}]'.format(s) + def visit_namedtuple_type(self, t): + s = self.list_str(zip(t.attrs, t.items)) + return '{}[{}]'.format(t.name, s) + def visit_star_type(self, t): s = t.type.accept(self) return '*{}'.format(s) diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 9c2d6c4a86d8f..3268209cc3f3b 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -199,6 +199,16 @@ from collections import namedtuple X = namedtuple('X', ['x', 'y']) x = None # type: X x = x._replace(y=5) +x = x._replace(z=5) # E: Unexpected keyword argument "z" for X +x = x._replace(5) # E: Too many positional arguments for X + +[case testNamedTupleReplaceAsClass] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +X._replace(x, x=1, y=2) +X._replace(x=1, y=2) # E: Missing positional argument "self" in call to X._replace [case testNamedTupleReplaceTyped] from typing import NamedTuple From 525c081b36fff84de595c260f268d4707c610f2c Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Thu, 11 Aug 2016 22:55:22 +0300 Subject: [PATCH 47/74] fix method name --- mypy/checkexpr.py | 3 +-- mypy/expandtype.py | 2 +- mypy/messages.py | 2 +- mypy/nodes.py | 2 +- mypy/semanal.py | 6 +++--- mypy/subtypes.py | 1 + mypy/typeanal.py | 2 +- mypy/types.py | 15 +++++++++++++-- test-data/unit/check-namedtuple.test | 12 ++++++------ test-data/unit/check-tuples.test | 2 +- 10 files changed, 29 insertions(+), 18 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index e29a1b383d716..97a50142901eb 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1221,8 +1221,7 @@ def visit_tuple_slice_helper(self, left_type: TupleType, slic: SliceExpr): slic.stride) return AnyType() - return TupleType(left_type.items[begin:end:stride], left_type.fallback, - left_type.line, left_type.implicit) + return left_type.slice(begin, stride, end) def _get_value(self, index: Node) -> Optional[int]: if isinstance(index, IntExpr): diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 87b1641daecc1..ce6d5190fac6a 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -88,7 +88,7 @@ def visit_overloaded(self, t: Overloaded) -> Type: return Overloaded(items) def visit_tuple_type(self, t: TupleType) -> Type: - return TupleType(self.expand_types(t.items), t.fallback, t.line) + return t.with_types(self.expand_types(t.items)) def visit_union_type(self, t: UnionType) -> Type: # After substituting for type variables in t.items, diff --git a/mypy/messages.py b/mypy/messages.py index dd21dc4e2d7f7..698bc7a131b93 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -486,7 +486,7 @@ def incompatible_argument(self, n: int, m: int, callee: CallableType, arg_type: except IndexError: # Varargs callees expected_type = callee.arg_types[-1] arg_type_str, expected_type_str = self.format_distinctly(arg_type, expected_type) - import pdb; pdb.set_trace() + #import pdb; pdb.set_trace() if arg_kind == ARG_STAR: arg_type_str = '*' + arg_type_str elif arg_kind == ARG_STAR2: diff --git a/mypy/nodes.py b/mypy/nodes.py index 6e1d3bd353271..60d7b5d18944f 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1977,7 +1977,7 @@ def deserialize(cls, data: JsonDict) -> 'TypeInfo': return ti -def NamedTupleTypeInfo(tup: 'mypy.types.NamedTupleType', *args) -> None: +def NamedTupleTypeInfo(tup: 'mypy.types.NamedTupleType', *args) -> TypeInfo: info = TypeInfo(*args) info.tuple_type = tup info.bases = [tup.fallback] diff --git a/mypy/semanal.py b/mypy/semanal.py index 8f77f908a41f8..0f4331499d7cc 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1543,7 +1543,7 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], self.add_namedtuple_method('_replace', symbols, info, this_type, this_type, self.make_factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) self.add_namedtuple_method('__init__', symbols, info, this_type, NoneTyp(), - self.make_factory_args(vars, ARG_NAMED)) + self.make_factory_args(vars, ARG_POS), info.name()) # TODO: refine to OrderedDict[Union[types]] self.add_namedtuple_method('_asdict', symbols, info, this_type, AnyType(), []) return info @@ -1553,7 +1553,7 @@ def make_factory_args(self, vars: List[Var], kind: int, return [Argument(var, var.type, initializer, kind) for var in vars] def add_namedtuple_method(self, funcname: str, symbols: SymbolTable, info: TypeInfo, - this_type: Type, ret: Type, args: List[Argument]) -> None: + this_type: Type, ret: Type, args: List[Argument], name=None) -> None: args = [Argument(Var('self'), this_type, None, ARG_POS)] + args types = [arg.type_annotation for arg in args] items = [arg.variable.name() for arg in args] @@ -1561,7 +1561,7 @@ def add_namedtuple_method(self, funcname: str, symbols: SymbolTable, info: TypeI # TODO: Make sure that the self argument name is not visible? signature = CallableType(types, arg_kinds, items, ret, self.named_type('__builtins__.function'), - name=info.name()) + name=name or info.name() + '.' + funcname) func = FuncDef(funcname, args, Block([]), diff --git a/mypy/subtypes.py b/mypy/subtypes.py index ee08cf452413e..2205d014261d6 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -193,6 +193,7 @@ def visit_tuple_type(self, left: TupleType) -> bool: return False def visit_namedtuple_type(self, left: NamedTupleType) -> bool: + #import pdb; pdb.set_trace() if isinstance(self.right, NamedTupleType): # left.attrs.issubset(right.attrs) is insufficient # (a: int, b: str) is not subtype of (b: int, a: str) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index cce852a1be525..1841f283a19e1 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -171,7 +171,7 @@ def visit_unbound_type(self, t: UnboundType) -> Type: return AnyType() # import pdb; pdb.set_trace() if info.is_named_tuple: - return NamedTupleType(info.name, info.attrs, + return NamedTupleType(info.name(), info.attrs, self.anal_array(info.tuple_type.items), fallback=instance, line=t.line) diff --git a/mypy/types.py b/mypy/types.py index 61005b6c6c975..ae08876e8bca0 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -762,7 +762,14 @@ def deserialize(cls, data: JsonDict) -> 'TupleType': implicit=data['implicit']) def self_type(self, inst: Instance) -> 'TupleType': - return TupleType(self.items, inst) + return TupleType(self.items, inst, self.line, self.implicit) + + def with_types(self, items: List[Type]) -> 'TupleType': + return TupleType(items, self.fallback, self.line, self.implicit) + + def slice(self, begin: int, stride: int, end: int) -> 'TupleType': + return TupleType(self.items[begin:end:stride], self.fallback, + self.line, self.implicit) class NamedTupleType(TupleType): @@ -775,7 +782,11 @@ def __init__(self, name: str, attrs: List[str], *args, **kwargs) -> None: super().__init__(*args, **kwargs) def self_type(self, inst: Instance) -> 'NamedTupleType': - return NamedTupleType(self.name, self.attrs, self.items, inst) + return NamedTupleType(self.name, self.attrs, self.items, inst, self.line) + + def with_types(self, items: List[Type]) -> 'NamedTupleType': + return NamedTupleType(self.name, self.attrs, items, + self.fallback, self.line, self.implicit) def accept(self, visitor: 'TypeVisitor[T]') -> T: try: diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 3268209cc3f3b..a14cdd9ab3c77 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -143,9 +143,9 @@ class B(A): pass a = A(1, '') b = B(1, '') t = None # type: Tuple[int, str] -b = a # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "A") -b = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "B") +b = a # E: Incompatible types in assignment (expression has type "A", variable has type "B") +a = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "A") +b = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "B") t = a t = (1, '') t = b @@ -199,8 +199,8 @@ from collections import namedtuple X = namedtuple('X', ['x', 'y']) x = None # type: X x = x._replace(y=5) -x = x._replace(z=5) # E: Unexpected keyword argument "z" for X -x = x._replace(5) # E: Too many positional arguments for X +x = x._replace(z=5) # E: Unexpected keyword argument "z" for X._replace +x = x._replace(5) # E: Too many positional arguments for X._replace [case testNamedTupleReplaceAsClass] from collections import namedtuple @@ -216,4 +216,4 @@ from typing import NamedTuple X = NamedTuple('X', [('x', int), ('y', str)]) x = None # type: X x = x._replace(x=5) -x = x._replace(y=5) # E: Argument 1 to X has incompatible type "int"; expected "str" +x = x._replace(y=5) # E: Argument 1 to X._replace has incompatible type "int"; expected "str" diff --git a/test-data/unit/check-tuples.test b/test-data/unit/check-tuples.test index 7c03014fd5c81..50d1b3e64598d 100644 --- a/test-data/unit/check-tuples.test +++ b/test-data/unit/check-tuples.test @@ -2,7 +2,7 @@ -- ------------------------------- -[case testAssignmentWithTupleTypes] +[case testTupleAssignmentWithTupleTypes] from typing import Tuple t1 = None # type: Tuple[A] t2 = None # type: Tuple[B] From 15acfe012e9f0aa2a6ccadadc7ef0e87e03ad2ef Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Sun, 14 Aug 2016 18:11:37 +0300 Subject: [PATCH 48/74] namedtuple tests pass. list has problems --- mypy/join.py | 9 +++++++-- mypy/messages.py | 1 + mypy/subtypes.py | 3 --- mypy/types.py | 21 +++++++++++++++++++++ test-data/unit/check-lists.test | 2 ++ test-data/unit/check-namedtuple.test | 6 +++++- 6 files changed, 36 insertions(+), 6 deletions(-) diff --git a/mypy/join.py b/mypy/join.py index c28216b5de69d..684d62cfeaebd 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -1,12 +1,12 @@ """Calculation of the least upper bound types (joins).""" -from typing import cast, List +from typing import List from mypy.types import ( Type, AnyType, NoneTyp, Void, TypeVisitor, Instance, UnboundType, ErrorType, TypeVarType, CallableType, TupleType, ErasedType, TypeList, UnionType, FunctionLike, Overloaded, PartialType, DeletedType, - UninhabitedType, TypeType + UninhabitedType, TypeType, NamedTupleType ) from mypy.maptype import map_instance_to_supertype from mypy.subtypes import is_subtype, is_equivalent, is_subtype_ignoring_tvars @@ -210,10 +210,15 @@ def visit_overloaded(self, t: Overloaded) -> Type: return join_types(t.fallback, s) def visit_tuple_type(self, t: TupleType) -> Type: + if self.s == t: + return self.s if isinstance(self.s, TupleType) and self.s.length() == t.length(): items = [] # type: List[Type] for i in range(t.length()): items.append(self.join(t.items[i], self.s.items[i])) + if (not (isinstance(t, NamedTupleType) and isinstance(self.s, NamedTupleType)) + or self.s.attrs == t.attrs): + return self.s.with_types(items) # TODO: What if the fallback types are different? return TupleType(items, t.fallback) else: diff --git a/mypy/messages.py b/mypy/messages.py index 698bc7a131b93..5d82c4845782a 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -146,6 +146,7 @@ def report(self, msg: str, context: Context, severity: str, file: str = None) -> def fail(self, msg: str, context: Context, file: str = None) -> None: """Report an error message (unless disabled).""" + # import pdb; pdb.set_trace() self.report(msg, context, 'error', file=file) def note(self, msg: str, context: Context, file: str = None) -> None: diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 2205d014261d6..6872e297c3a64 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -178,8 +178,6 @@ def visit_tuple_type(self, left: TupleType) -> bool: iter_type = right.args[0] return all(is_subtype(li, iter_type) for li in left.items) return False - elif isinstance(right, NamedTupleType) and not isinstance(left, NamedTupleType): - return False elif isinstance(right, TupleType): if len(left.items) != len(right.items): return False @@ -193,7 +191,6 @@ def visit_tuple_type(self, left: TupleType) -> bool: return False def visit_namedtuple_type(self, left: NamedTupleType) -> bool: - #import pdb; pdb.set_trace() if isinstance(self.right, NamedTupleType): # left.attrs.issubset(right.attrs) is insufficient # (a: int, b: str) is not subtype of (b: int, a: str) diff --git a/mypy/types.py b/mypy/types.py index ae08876e8bca0..7d8dcb1c1a603 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -788,6 +788,27 @@ def with_types(self, items: List[Type]) -> 'NamedTupleType': return NamedTupleType(self.name, self.attrs, items, self.fallback, self.line, self.implicit) + def __eq__(self, other) -> bool: + return self.serialize() == other.serialize() + + def serialize(self) -> JsonDict: + d = super().serialize() + d.update({'.class': 'NamedTupleType', + 'name': self.name, + 'attrs': self.attrs}) + return d + + @classmethod + def deserialize(cls, data: JsonDict) -> 'NamedTupleType': + assert data['.class'] == 'NamedTupleType' + data['.class'] = 'TupleType' + tup = TupleType.deserialize(data) + return NamedTupleType(data['name'], + data['attrs'], + tup.items, + tup.fallback, + implicit=tup.implicit) + def accept(self, visitor: 'TypeVisitor[T]') -> T: try: return visitor.visit_namedtuple_type(self) diff --git a/test-data/unit/check-lists.test b/test-data/unit/check-lists.test index 3bafa99245d18..032e02314f5c4 100644 --- a/test-data/unit/check-lists.test +++ b/test-data/unit/check-lists.test @@ -1,5 +1,7 @@ -- Nested list assignment -- ----------------------------- +[case testListSimplest] +l = [1, 1] [case testNestedListAssignment] diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index a14cdd9ab3c77..adb07ec522705 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -1,3 +1,8 @@ +[case testNamedTupleDeclaration] +from typing import NamedTuple + +A = NamedTuple('A', [('x', str), ('y', int)]) + [case testNamedTupleUsedAsTuple] from collections import namedtuple @@ -159,7 +164,6 @@ a = A(1) a = l[0] (i,) = l[0] i, i = l[0] # E: Need more than 1 value to unpack (2 expected) -l = [A(1)] a = (1,) # E: Incompatible types in assignment (expression has type "Tuple[int]", \ variable has type "A") [builtins fixtures/list.py] From fb9ba7f5f14a1f5b7894cc2cc9d48886744687e1 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Sun, 14 Aug 2016 18:50:35 +0300 Subject: [PATCH 49/74] tests pass. OrderedDict unsupported. cleanup still required --- mypy/build.py | 3356 +++++++-------- mypy/checker.py | 4918 +++++++++++----------- mypy/checkexpr.py | 3644 ++++++++--------- mypy/expandtype.py | 224 +- mypy/join.py | 734 ++-- mypy/meet.py | 570 +-- mypy/messages.py | 1840 ++++----- mypy/nodes.py | 4484 ++++++++++---------- mypy/parse.py | 4068 +++++++++--------- mypy/semanal.py | 5678 +++++++++++++------------- mypy/subtypes.py | 818 ++-- mypy/typeanal.py | 860 ++-- mypy/types.py | 2902 ++++++------- test-data/unit/check-lists.test | 129 +- test-data/unit/check-namedtuple.test | 446 +- test-data/unit/check-tuples.test | 1540 +++---- 16 files changed, 18104 insertions(+), 18107 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index e8a84c97c7c76..a930614989f90 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -1,1678 +1,1678 @@ -"""Facilities to analyze entire programs, including imported modules. - -Parse and analyze the source files of a program in the correct order -(based on file dependencies), and collect the results. - -This module only directs a build, which is performed in multiple passes per -file. The individual passes are implemented in separate modules. - -The function build() is the main interface to this module. -""" -# TODO: More consistent terminology, e.g. path/fnam, module/id, state/file - -import binascii -import collections -import contextlib -import json -import os -import os.path -import sys -import time -from os.path import dirname, basename - -from typing import (AbstractSet, Dict, Iterable, Iterator, List, - NamedTuple, Optional, Set, Tuple, Union, Mapping) - -from mypy.types import Type -from mypy.nodes import (MypyFile, Node, Import, ImportFrom, ImportAll, - SymbolTableNode, MODULE_REF) -from mypy.semanal import FirstPass, SemanticAnalyzer, ThirdPass -from mypy.checker import TypeChecker -from mypy.errors import Errors, CompileError, DecodeError, report_internal_error -from mypy import fixup -from mypy.report import Reports -from mypy import defaults -from mypy import moduleinfo -from mypy import util -from mypy.fixup import fixup_module_pass_one, fixup_module_pass_two -from mypy.options import Options -from mypy.parse import parse -from mypy.stats import dump_type_stats - - -# We need to know the location of this file to load data, but -# until Python 3.4, __file__ is relative. -__file__ = os.path.realpath(__file__) - -PYTHON_EXTENSIONS = ['.pyi', '.py'] - - -class BuildResult: - """The result of a successful build. - - Attributes: - manager: The build manager. - files: Dictionary from module name to related AST node. - types: Dictionary from parse tree node to its inferred type. - errors: List of error messages. - """ - - def __init__(self, manager: 'BuildManager') -> None: - self.manager = manager - self.files = manager.modules - self.types = manager.type_checker.type_map - self.errors = manager.errors.messages() - - -class BuildSource: - def __init__(self, path: Optional[str], module: Optional[str], - text: Optional[str]) -> None: - self.path = path - self.module = module or '__main__' - self.text = text - - @property - def effective_path(self) -> str: - """Return the effective path (ie, if its from in memory)""" - return self.path or '' - - -class BuildSourceSet: - """Efficiently test a file's membership in the set of build sources.""" - - def __init__(self, sources: List[BuildSource]) -> None: - self.source_text_present = False - self.source_modules = set() # type: Set[str] - self.source_paths = set() # type: Set[str] - - for source in sources: - if source.text is not None: - self.source_text_present = True - elif source.path: - self.source_paths.add(source.path) - else: - self.source_modules.add(source.module) - - def is_source(self, file: MypyFile) -> bool: - if file.path and file.path in self.source_paths: - return True - elif file._fullname in self.source_modules: - return True - elif file.path is None and self.source_text_present: - return True - else: - return False - - -def build(sources: List[BuildSource], - options: Options, - alt_lib_path: str = None, - bin_dir: str = None) -> BuildResult: - """Analyze a program. - - A single call to build performs parsing, semantic analysis and optionally - type checking for the program *and* all imported modules, recursively. - - Return BuildResult if successful or only non-blocking errors were found; - otherwise raise CompileError. - - Args: - sources: list of sources to build - options: build options - alt_lib_path: an additional directory for looking up library modules - (takes precedence over other directories) - bin_dir: directory containing the mypy script, used for finding data - directories; if omitted, use '.' as the data directory - """ - - data_dir = default_data_dir(bin_dir) - - find_module_clear_caches() - - # Determine the default module search path. - lib_path = default_lib_path(data_dir, options.python_version) - - if options.use_builtins_fixtures: - # Use stub builtins (to speed up test cases and to make them easier to - # debug). This is a test-only feature, so assume our files are laid out - # as in the source tree. - root_dir = os.path.dirname(os.path.dirname(__file__)) - lib_path.insert(0, os.path.join(root_dir, 'test-data', 'unit', 'lib-stub')) - else: - for source in sources: - if source.path: - # Include directory of the program file in the module search path. - dir = remove_cwd_prefix_from_path(dirname(source.path)) - if dir not in lib_path: - lib_path.insert(0, dir) - - # Do this even if running as a file, for sanity (mainly because with - # multiple builds, there could be a mix of files/modules, so its easier - # to just define the semantics that we always add the current director - # to the lib_path - lib_path.insert(0, os.getcwd()) - - # Add MYPYPATH environment variable to front of library path, if defined. - lib_path[:0] = mypy_path() - - # If provided, insert the caller-supplied extra module path to the - # beginning (highest priority) of the search path. - if alt_lib_path: - lib_path.insert(0, alt_lib_path) - - reports = Reports(data_dir, options.report_dirs) - - source_set = BuildSourceSet(sources) - - # Construct a build manager object to hold state during the build. - # - # Ignore current directory prefix in error messages. - manager = BuildManager(data_dir, lib_path, - ignore_prefix=os.getcwd(), - source_set=source_set, - reports=reports, - options=options) - - try: - dispatch(sources, manager) - return BuildResult(manager) - finally: - manager.log("Build finished with %d modules, %d types, and %d errors" % - (len(manager.modules), - len(manager.type_checker.type_map), - manager.errors.num_messages())) - # Finish the HTML or XML reports even if CompileError was raised. - reports.finish() - - -def default_data_dir(bin_dir: Optional[str]) -> str: - """Returns directory containing typeshed directory - - Args: - bin_dir: directory containing the mypy script - """ - if not bin_dir: - mypy_package = os.path.dirname(__file__) - parent = os.path.dirname(mypy_package) - if (os.path.basename(parent) == 'site-packages' or - os.path.basename(parent) == 'dist-packages'): - # Installed in site-packages or dist-packages, but invoked with python3 -m mypy; - # __file__ is .../blah/lib/python3.N/site-packages/mypy/build.py - # or .../blah/lib/python3.N/dist-packages/mypy/build.py (Debian) - # or .../blah/lib/site-packages/mypy/build.py (Windows) - # blah may be a virtualenv or /usr/local. We want .../blah/lib/mypy. - lib = parent - for i in range(2): - lib = os.path.dirname(lib) - if os.path.basename(lib) == 'lib': - return os.path.join(lib, 'mypy') - subdir = os.path.join(parent, 'lib', 'mypy') - if os.path.isdir(subdir): - # If installed via buildout, the __file__ is - # somewhere/mypy/__init__.py and what we want is - # somewhere/lib/mypy. - return subdir - # Default to directory containing this file's parent. - return parent - base = os.path.basename(bin_dir) - dir = os.path.dirname(bin_dir) - if (sys.platform == 'win32' and base.lower() == 'scripts' - and not os.path.isdir(os.path.join(dir, 'typeshed'))): - # Installed, on Windows. - return os.path.join(dir, 'Lib', 'mypy') - elif base == 'scripts': - # Assume that we have a repo check out or unpacked source tarball. - return dir - elif base == 'bin': - # Installed to somewhere (can be under /usr/local or anywhere). - return os.path.join(dir, 'lib', 'mypy') - elif base == 'python3': - # Assume we installed python3 with brew on os x - return os.path.join(os.path.dirname(dir), 'lib', 'mypy') - elif dir.endswith('python-exec'): - # Gentoo uses a python wrapper in /usr/lib to which mypy is a symlink. - return os.path.join(os.path.dirname(dir), 'mypy') - else: - # Don't know where to find the data files! - raise RuntimeError("Broken installation: can't determine base dir") - - -def mypy_path() -> List[str]: - path_env = os.getenv('MYPYPATH') - if not path_env: - return [] - return path_env.split(os.pathsep) - - -def default_lib_path(data_dir: str, pyversion: Tuple[int, int]) -> List[str]: - """Return default standard library search paths.""" - # IDEA: Make this more portable. - path = [] # type: List[str] - - auto = os.path.join(data_dir, 'stubs-auto') - if os.path.isdir(auto): - data_dir = auto - - # We allow a module for e.g. version 3.5 to be in 3.4/. The assumption - # is that a module added with 3.4 will still be present in Python 3.5. - versions = ["%d.%d" % (pyversion[0], minor) - for minor in reversed(range(pyversion[1] + 1))] - # E.g. for Python 3.2, try 3.2/, 3.1/, 3.0/, 3/, 2and3/. - # (Note that 3.1 and 3.0 aren't really supported, but we don't care.) - for v in versions + [str(pyversion[0]), '2and3']: - for lib_type in ['stdlib', 'third_party']: - stubdir = os.path.join(data_dir, 'typeshed', lib_type, v) - if os.path.isdir(stubdir): - path.append(stubdir) - - # Add fallback path that can be used if we have a broken installation. - if sys.platform != 'win32': - path.append('/usr/local/lib/mypy') - - return path - - -CacheMeta = NamedTuple('CacheMeta', - [('id', str), - ('path', str), - ('mtime', float), - ('size', int), - ('dependencies', List[str]), # names of imported modules - ('data_mtime', float), # mtime of data_json - ('data_json', str), # path of .data.json - ('suppressed', List[str]), # dependencies that weren't imported - ('options', Optional[Dict[str, bool]]), # build options - ('dep_prios', List[int]), - ]) -# NOTE: dependencies + suppressed == all reachable imports; -# suppressed contains those reachable imports that were prevented by -# --silent-imports or simply not found. - - -# Priorities used for imports. (Here, top-level includes inside a class.) -# These are used to determine a more predictable order in which the -# nodes in an import cycle are processed. -PRI_HIGH = 5 # top-level "from X import blah" -PRI_MED = 10 # top-level "import X" -PRI_LOW = 20 # either form inside a function -PRI_ALL = 99 # include all priorities - - -class BuildManager: - """This class holds shared state for building a mypy program. - - It is used to coordinate parsing, import processing, semantic - analysis and type checking. The actual build steps are carried - out by dispatch(). - - Attributes: - data_dir: Mypy data directory (contains stubs) - lib_path: Library path for looking up modules - modules: Mapping of module ID to MypyFile (shared by the passes) - semantic_analyzer: - Semantic analyzer, pass 2 - semantic_analyzer_pass3: - Semantic analyzer, pass 3 - type_checker: Type checker - errors: Used for reporting all errors - options: Build options - missing_modules: Set of modules that could not be imported encountered so far - """ - - def __init__(self, data_dir: str, - lib_path: List[str], - ignore_prefix: str, - source_set: BuildSourceSet, - reports: Reports, - options: Options) -> None: - self.start_time = time.time() - self.data_dir = data_dir - self.errors = Errors() - self.errors.set_ignore_prefix(ignore_prefix) - self.lib_path = tuple(lib_path) - self.source_set = source_set - self.reports = reports - self.options = options - self.semantic_analyzer = SemanticAnalyzer(lib_path, self.errors, options=options) - self.modules = self.semantic_analyzer.modules - self.semantic_analyzer_pass3 = ThirdPass(self.modules, self.errors) - self.type_checker = TypeChecker(self.errors, self.modules, options=options) - self.missing_modules = set() # type: Set[str] - - def all_imported_modules_in_file(self, - file: MypyFile) -> List[Tuple[int, str, int]]: - """Find all reachable import statements in a file. - - Return list of tuples (priority, module id, import line number) - for all modules imported in file; lower numbers == higher priority. - """ - - def correct_rel_imp(imp: Union[ImportFrom, ImportAll]) -> str: - """Function to correct for relative imports.""" - file_id = file.fullname() - rel = imp.relative - if rel == 0: - return imp.id - if os.path.basename(file.path).startswith('__init__.'): - rel -= 1 - if rel != 0: - file_id = ".".join(file_id.split(".")[:-rel]) - new_id = file_id + "." + imp.id if imp.id else file_id - - return new_id - - res = [] # type: List[Tuple[int, str, int]] - for imp in file.imports: - if not imp.is_unreachable: - if isinstance(imp, Import): - pri = PRI_MED if imp.is_top_level else PRI_LOW - for id, _ in imp.ids: - res.append((pri, id, imp.line)) - elif isinstance(imp, ImportFrom): - cur_id = correct_rel_imp(imp) - pos = len(res) - all_are_submodules = True - # Also add any imported names that are submodules. - pri = PRI_MED if imp.is_top_level else PRI_LOW - for name, __ in imp.names: - sub_id = cur_id + '.' + name - if self.is_module(sub_id): - res.append((pri, sub_id, imp.line)) - else: - all_are_submodules = False - # If all imported names are submodules, don't add - # cur_id as a dependency. Otherwise (i.e., if at - # least one imported name isn't a submodule) - # cur_id is also a dependency, and we should - # insert it *before* any submodules. - if not all_are_submodules: - pri = PRI_HIGH if imp.is_top_level else PRI_LOW - res.insert(pos, ((pri, cur_id, imp.line))) - elif isinstance(imp, ImportAll): - pri = PRI_HIGH if imp.is_top_level else PRI_LOW - res.append((pri, correct_rel_imp(imp), imp.line)) - - return res - - def is_module(self, id: str) -> bool: - """Is there a file in the file system corresponding to module id?""" - return find_module(id, self.lib_path) is not None - - def parse_file(self, id: str, path: str, source: str) -> MypyFile: - """Parse the source of a file with the given name. - - Raise CompileError if there is a parse error. - """ - num_errs = self.errors.num_messages() - tree = parse(source, path, self.errors, options=self.options) - tree._fullname = id - - # We don't want to warn about 'type: ignore' comments on - # imports, but we're about to modify tree.imports, so grab - # these first. - import_lines = set(node.line for node in tree.imports) - - # Skip imports that have been ignored (so that we can ignore a C extension module without - # stub, for example), except for 'from x import *', because we wouldn't be able to - # determine which names should be defined unless we process the module. We can still - # ignore errors such as redefinitions when using the latter form. - imports = [node for node in tree.imports - if node.line not in tree.ignored_lines or isinstance(node, ImportAll)] - tree.imports = imports - - if self.errors.num_messages() != num_errs: - self.log("Bailing due to parse errors") - self.errors.raise_error() - - self.errors.set_file_ignored_lines(path, tree.ignored_lines) - self.errors.mark_file_ignored_lines_used(path, import_lines) - return tree - - def module_not_found(self, path: str, line: int, id: str) -> None: - self.errors.set_file(path) - stub_msg = "(Stub files are from https://github.com/python/typeshed)" - if ((self.options.python_version[0] == 2 and moduleinfo.is_py2_std_lib_module(id)) or - (self.options.python_version[0] >= 3 and moduleinfo.is_py3_std_lib_module(id))): - self.errors.report( - line, "No library stub file for standard library module '{}'".format(id)) - self.errors.report(line, stub_msg, severity='note', only_once=True) - elif moduleinfo.is_third_party_module(id): - self.errors.report(line, "No library stub file for module '{}'".format(id)) - self.errors.report(line, stub_msg, severity='note', only_once=True) - else: - self.errors.report(line, "Cannot find module named '{}'".format(id)) - self.errors.report(line, '(Perhaps setting MYPYPATH ' - 'or using the "--silent-imports" flag would help)', - severity='note', only_once=True) - - def report_file(self, file: MypyFile) -> None: - if self.source_set.is_source(file): - self.reports.file(file, type_map=self.type_checker.type_map) - - def log(self, *message: str) -> None: - if self.options.verbosity >= 1: - print('%.3f:LOG: ' % (time.time() - self.start_time), *message, file=sys.stderr) - sys.stderr.flush() - - def trace(self, *message: str) -> None: - if self.options.verbosity >= 2: - print('%.3f:TRACE:' % (time.time() - self.start_time), *message, file=sys.stderr) - sys.stderr.flush() - - -def remove_cwd_prefix_from_path(p: str) -> str: - """Remove current working directory prefix from p, if present. - - Also crawl up until a directory without __init__.py is found. - - If the result would be empty, return '.' instead. - """ - cur = os.getcwd() - # Add separator to the end of the path, unless one is already present. - if basename(cur) != '': - cur += os.sep - # Compute root path. - while (p and - (os.path.isfile(os.path.join(p, '__init__.py')) or - os.path.isfile(os.path.join(p, '__init__.pyi')))): - dir, base = os.path.split(p) - if not base: - break - p = dir - # Remove current directory prefix from the path, if present. - if p.startswith(cur): - p = p[len(cur):] - # Avoid returning an empty path; replace that with '.'. - if p == '': - p = '.' - return p - - -# Cache find_module: (id, lib_path) -> result. -find_module_cache = {} # type: Dict[Tuple[str, Tuple[str, ...]], str] - -# Cache some repeated work within distinct find_module calls: finding which -# elements of lib_path have even the subdirectory they'd need for the module -# to exist. This is shared among different module ids when they differ only -# in the last component. -find_module_dir_cache = {} # type: Dict[Tuple[str, Tuple[str, ...]], List[str]] - -# Cache directory listings. We assume that while one os.listdir() -# call may be more expensive than one os.stat() call, a small number -# of os.stat() calls is quickly more expensive than caching the -# os.listdir() outcome, and the advantage of the latter is that it -# gives us the case-correct filename on Windows and Mac. -find_module_listdir_cache = {} # type: Dict[str, Optional[List[str]]] - - -def find_module_clear_caches(): - find_module_cache.clear() - find_module_dir_cache.clear() - find_module_listdir_cache.clear() - - -def list_dir(path: str) -> Optional[List[str]]: - """Return a cached directory listing. - - Returns None if the path doesn't exist or isn't a directory. - """ - if path in find_module_listdir_cache: - return find_module_listdir_cache[path] - try: - res = os.listdir(path) - except OSError: - res = None - find_module_listdir_cache[path] = res - return res - - -def is_file(path: str) -> bool: - """Return whether path exists and is a file. - - On case-insensitive filesystems (like Mac or Windows) this returns - False if the case of the path's last component does not exactly - match the case found in the filesystem. - """ - head, tail = os.path.split(path) - if not tail: - return False - names = list_dir(head) - if not names: - return False - if tail not in names: - return False - return os.path.isfile(path) - - -def find_module(id: str, lib_path: Iterable[str]) -> str: - """Return the path of the module source file, or None if not found.""" - if not isinstance(lib_path, tuple): - lib_path = tuple(lib_path) - - def find(): - # If we're looking for a module like 'foo.bar.baz', it's likely that most of the - # many elements of lib_path don't even have a subdirectory 'foo/bar'. Discover - # that only once and cache it for when we look for modules like 'foo.bar.blah' - # that will require the same subdirectory. - components = id.split('.') - dir_chain = os.sep.join(components[:-1]) # e.g., 'foo/bar' - if (dir_chain, lib_path) not in find_module_dir_cache: - dirs = [] - for pathitem in lib_path: - # e.g., '/usr/lib/python3.4/foo/bar' - dir = os.path.normpath(os.path.join(pathitem, dir_chain)) - if os.path.isdir(dir): - dirs.append(dir) - find_module_dir_cache[dir_chain, lib_path] = dirs - candidate_base_dirs = find_module_dir_cache[dir_chain, lib_path] - - # If we're looking for a module like 'foo.bar.baz', then candidate_base_dirs now - # contains just the subdirectories 'foo/bar' that actually exist under the - # elements of lib_path. This is probably much shorter than lib_path itself. - # Now just look for 'baz.pyi', 'baz/__init__.py', etc., inside those directories. - seplast = os.sep + components[-1] # so e.g. '/baz' - sepinit = os.sep + '__init__' - for base_dir in candidate_base_dirs: - base_path = base_dir + seplast # so e.g. '/usr/lib/python3.4/foo/bar/baz' - # Prefer package over module, i.e. baz/__init__.py* over baz.py*. - for extension in PYTHON_EXTENSIONS: - path = base_path + sepinit + extension - if is_file(path) and verify_module(id, path): - return path - # No package, look for module. - for extension in PYTHON_EXTENSIONS: - path = base_path + extension - if is_file(path) and verify_module(id, path): - return path - return None - - key = (id, lib_path) - if key not in find_module_cache: - find_module_cache[key] = find() - return find_module_cache[key] - - -def find_modules_recursive(module: str, lib_path: List[str]) -> List[BuildSource]: - module_path = find_module(module, lib_path) - if not module_path: - return [] - result = [BuildSource(module_path, module, None)] - if module_path.endswith(('__init__.py', '__init__.pyi')): - # Subtle: this code prefers the .pyi over the .py if both - # exists, and also prefers packages over modules if both x/ - # and x.py* exist. How? We sort the directory items, so x - # comes before x.py and x.pyi. But the preference for .pyi - # over .py is encoded in find_module(); even though we see - # x.py before x.pyi, find_module() will find x.pyi first. We - # use hits to avoid adding it a second time when we see x.pyi. - # This also avoids both x.py and x.pyi when x/ was seen first. - hits = set() # type: Set[str] - for item in sorted(os.listdir(os.path.dirname(module_path))): - abs_path = os.path.join(os.path.dirname(module_path), item) - if os.path.isdir(abs_path) and \ - (os.path.isfile(os.path.join(abs_path, '__init__.py')) or - os.path.isfile(os.path.join(abs_path, '__init__.pyi'))): - hits.add(item) - result += find_modules_recursive(module + '.' + item, lib_path) - elif item != '__init__.py' and item != '__init__.pyi' and \ - item.endswith(('.py', '.pyi')): - mod = item.split('.')[0] - if mod not in hits: - hits.add(mod) - result += find_modules_recursive( - module + '.' + mod, lib_path) - return result - - -def verify_module(id: str, path: str) -> bool: - """Check that all packages containing id have a __init__ file.""" - if path.endswith(('__init__.py', '__init__.pyi')): - path = dirname(path) - for i in range(id.count('.')): - path = dirname(path) - if not any(os.path.isfile(os.path.join(path, '__init__{}'.format(extension))) - for extension in PYTHON_EXTENSIONS): - return False - return True - - -def read_with_python_encoding(path: str, pyversion: Tuple[int, int]) -> str: - """Read the Python file with while obeying PEP-263 encoding detection""" - source_bytearray = bytearray() - encoding = 'utf8' if pyversion[0] >= 3 else 'ascii' - - with open(path, 'rb') as f: - # read first two lines and check if PEP-263 coding is present - source_bytearray.extend(f.readline()) - source_bytearray.extend(f.readline()) - - # check for BOM UTF-8 encoding and strip it out if present - if source_bytearray.startswith(b'\xef\xbb\xbf'): - encoding = 'utf8' - source_bytearray = source_bytearray[3:] - else: - _encoding, _ = util.find_python_encoding(source_bytearray, pyversion) - # check that the coding isn't mypy. We skip it since - # registering may not have happened yet - if _encoding != 'mypy': - encoding = _encoding - - source_bytearray.extend(f.read()) - try: - source_bytearray.decode(encoding) - except LookupError as lookuperr: - raise DecodeError(str(lookuperr)) - return source_bytearray.decode(encoding) - - -MYPY_CACHE = '.mypy_cache' - - -def get_cache_names(id: str, path: str, pyversion: Tuple[int, int]) -> Tuple[str, str]: - """Return the file names for the cache files. - - Args: - id: module ID - path: module path (used to recognize packages) - pyversion: Python version (major, minor) - - Returns: - A tuple with the file names to be used for the meta JSON and the - data JSON, respectively. - """ - prefix = os.path.join(MYPY_CACHE, '%d.%d' % pyversion, *id.split('.')) - is_package = os.path.basename(path).startswith('__init__.py') - if is_package: - prefix = os.path.join(prefix, '__init__') - return (prefix + '.meta.json', prefix + '.data.json') - - -def find_cache_meta(id: str, path: str, manager: BuildManager) -> Optional[CacheMeta]: - """Find cache data for a module. - - Args: - id: module ID - path: module path - manager: the build manager (for pyversion, log/trace, and build options) - - Returns: - A CacheMeta instance if the cache data was found and appears - valid; otherwise None. - """ - # TODO: May need to take more build options into account - meta_json, data_json = get_cache_names(id, path, manager.options.python_version) - manager.trace('Looking for {} {}'.format(id, data_json)) - if not os.path.exists(meta_json): - return None - with open(meta_json, 'r') as f: - meta_str = f.read() - manager.trace('Meta {} {}'.format(id, meta_str.rstrip())) - meta = json.loads(meta_str) # TODO: Errors - if not isinstance(meta, dict): - return None - path = os.path.abspath(path) - m = CacheMeta( - meta.get('id'), - meta.get('path'), - meta.get('mtime'), - meta.get('size'), - meta.get('dependencies', []), - meta.get('data_mtime'), - data_json, - meta.get('suppressed', []), - meta.get('options'), - meta.get('dep_prios', []), - ) - if (m.id != id or m.path != path or - m.mtime is None or m.size is None or - m.dependencies is None or m.data_mtime is None): - return None - - # Ignore cache if generated by an older mypy version. - if m.options is None or len(m.dependencies) != len(m.dep_prios): - return None - - # Ignore cache if (relevant) options aren't the same. - cached_options = m.options - current_options = select_options_affecting_cache(manager.options) - if cached_options != current_options: - return None - - # TODO: Share stat() outcome with find_module() - st = os.stat(path) # TODO: Errors - if st.st_mtime != m.mtime or st.st_size != m.size: - manager.log('Metadata abandoned because of modified file {}'.format(path)) - return None - # It's a match on (id, path, mtime, size). - # Check data_json; assume if its mtime matches it's good. - # TODO: stat() errors - if os.path.getmtime(data_json) != m.data_mtime: - return None - manager.log('Found {} {}'.format(id, meta_json)) - return m - - -def select_options_affecting_cache(options: Options) -> Mapping[str, bool]: - return {opt: getattr(options, opt) for opt in OPTIONS_AFFECTING_CACHE} - - -OPTIONS_AFFECTING_CACHE = [ - "silent_imports", - "almost_silent", - "disallow_untyped_calls", - "disallow_untyped_defs", - "check_untyped_defs", -] - - -def random_string(): - return binascii.hexlify(os.urandom(8)).decode('ascii') - - -def write_cache(id: str, path: str, tree: MypyFile, - dependencies: List[str], suppressed: List[str], - dep_prios: List[int], - manager: BuildManager) -> None: - """Write cache files for a module. - - Args: - id: module ID - path: module path - tree: the fully checked module data - dependencies: module IDs on which this module depends - suppressed: module IDs which were suppressed as dependencies - dep_prios: priorities (parallel array to dependencies) - manager: the build manager (for pyversion, log/trace) - """ - path = os.path.abspath(path) - manager.trace('Dumping {} {}'.format(id, path)) - st = os.stat(path) # TODO: Errors - mtime = st.st_mtime - size = st.st_size - meta_json, data_json = get_cache_names(id, path, manager.options.python_version) - manager.log('Writing {} {} {}'.format(id, meta_json, data_json)) - data = tree.serialize() - parent = os.path.dirname(data_json) - if not os.path.isdir(parent): - os.makedirs(parent) - assert os.path.dirname(meta_json) == parent - nonce = '.' + random_string() - data_json_tmp = data_json + nonce - meta_json_tmp = meta_json + nonce - with open(data_json_tmp, 'w') as f: - json.dump(data, f, indent=2, sort_keys=True) - f.write('\n') - data_mtime = os.path.getmtime(data_json_tmp) - meta = {'id': id, - 'path': path, - 'mtime': mtime, - 'size': size, - 'data_mtime': data_mtime, - 'dependencies': dependencies, - 'suppressed': suppressed, - 'options': select_options_affecting_cache(manager.options), - 'dep_prios': dep_prios, - } - with open(meta_json_tmp, 'w') as f: - json.dump(meta, f, sort_keys=True) - f.write('\n') - os.replace(data_json_tmp, data_json) - os.replace(meta_json_tmp, meta_json) - - -"""Dependency manager. - -Design -====== - -Ideally -------- - -A. Collapse cycles (each SCC -- strongly connected component -- - becomes one "supernode"). - -B. Topologically sort nodes based on dependencies. - -C. Process from leaves towards roots. - -Wrinkles --------- - -a. Need to parse source modules to determine dependencies. - -b. Processing order for modules within an SCC. - -c. Must order mtimes of files to decide whether to re-process; depends - on clock never resetting. - -d. from P import M; checks filesystem whether module P.M exists in - filesystem. - -e. Race conditions, where somebody modifies a file while we're - processing. I propose not to modify the algorithm to handle this, - but to detect when this could lead to inconsistencies. (For - example, when we decide on the dependencies based on cache - metadata, and then we decide to re-parse a file because of a stale - dependency, if the re-parsing leads to a different list of - dependencies we should warn the user or start over.) - -Steps ------ - -1. For each explicitly given module find the source file location. - -2. For each such module load and check the cache metadata, and decide - whether it's valid. - -3. Now recursively (or iteratively) find dependencies and add those to - the graph: - - - for cached nodes use the list of dependencies from the cache - metadata (this will be valid even if we later end up re-parsing - the same source); - - - for uncached nodes parse the file and process all imports found, - taking care of (a) above. - -Step 3 should also address (d) above. - -Once step 3 terminates we have the entire dependency graph, and for -each module we've either loaded the cache metadata or parsed the -source code. (However, we may still need to parse those modules for -which we have cache metadata but that depend, directly or indirectly, -on at least one module for which the cache metadata is stale.) - -Now we can execute steps A-C from the first section. Finding SCCs for -step A shouldn't be hard; there's a recipe here: -http://code.activestate.com/recipes/578507/. There's also a plethora -of topsort recipes, e.g. http://code.activestate.com/recipes/577413/. - -For single nodes, processing is simple. If the node was cached, we -deserialize the cache data and fix up cross-references. Otherwise, we -do semantic analysis followed by type checking. We also handle (c) -above; if a module has valid cache data *but* any of its -dependencies was processed from source, then the module should be -processed from source. - -A relatively simple optimization (outside SCCs) we might do in the -future is as follows: if a node's cache data is valid, but one or more -of its dependencies are out of date so we have to re-parse the node -from source, once we have fully type-checked the node, we can decide -whether its symbol table actually changed compared to the cache data -(by reading the cache data and comparing it to the data we would be -writing). If there is no change we can declare the node up to date, -and any node that depends (and for which we have cached data, and -whose other dependencies are up to date) on it won't need to be -re-parsed from source. - -Import cycles -------------- - -Finally we have to decide how to handle (c), import cycles. Here -we'll need a modified version of the original state machine -(build.py), but we only need to do this per SCC, and we won't have to -deal with changes to the list of nodes while we're processing it. - -If all nodes in the SCC have valid cache metadata and all dependencies -outside the SCC are still valid, we can proceed as follows: - - 1. Load cache data for all nodes in the SCC. - - 2. Fix up cross-references for all nodes in the SCC. - -Otherwise, the simplest (but potentially slow) way to proceed is to -invalidate all cache data in the SCC and re-parse all nodes in the SCC -from source. We can do this as follows: - - 1. Parse source for all nodes in the SCC. - - 2. Semantic analysis for all nodes in the SCC. - - 3. Type check all nodes in the SCC. - -(If there are more passes the process is the same -- each pass should -be done for all nodes before starting the next pass for any nodes in -the SCC.) - -We could process the nodes in the SCC in any order. For sentimental -reasons, I've decided to process them in the reverse order in which we -encountered them when originally constructing the graph. That's how -the old build.py deals with cycles, and at least this reproduces the -previous implementation more accurately. - -Can we do better than re-parsing all nodes in the SCC when any of its -dependencies are out of date? It's doubtful. The optimization -mentioned at the end of the previous section would require re-parsing -and type-checking a node and then comparing its symbol table to the -cached data; but because the node is part of a cycle we can't -technically type-check it until the semantic analysis of all other -nodes in the cycle has completed. (This is an important issue because -Dropbox has a very large cycle in production code. But I'd like to -deal with it later.) - -Additional wrinkles -------------------- - -During implementation more wrinkles were found. - -- When a submodule of a package (e.g. x.y) is encountered, the parent - package (e.g. x) must also be loaded, but it is not strictly a - dependency. See State.add_ancestors() below. -""" - - -class ModuleNotFound(Exception): - """Control flow exception to signal that a module was not found.""" - - -class State: - """The state for a module. - - The source is only used for the -c command line option; in that - case path is None. Otherwise source is None and path isn't. - """ - - manager = None # type: BuildManager - order_counter = 0 # Class variable - order = None # type: int # Order in which modules were encountered - id = None # type: str # Fully qualified module name - path = None # type: Optional[str] # Path to module source - xpath = None # type: str # Path or '' - source = None # type: Optional[str] # Module source code - meta = None # type: Optional[CacheMeta] - data = None # type: Optional[str] - tree = None # type: Optional[MypyFile] - dependencies = None # type: List[str] - suppressed = None # type: List[str] # Suppressed/missing dependencies - priorities = None # type: Dict[str, int] - - # Map each dependency to the line number where it is first imported - dep_line_map = None # type: Dict[str, int] - - # Parent package, its parent, etc. - ancestors = None # type: Optional[List[str]] - - # List of (path, line number) tuples giving context for import - import_context = None # type: List[Tuple[str, int]] - - # The State from which this module was imported, if any - caller_state = None # type: Optional[State] - - # If caller_state is set, the line number in the caller where the import occurred - caller_line = 0 - - def __init__(self, - id: Optional[str], - path: Optional[str], - source: Optional[str], - manager: BuildManager, - caller_state: 'State' = None, - caller_line: int = 0, - ancestor_for: 'State' = None, - ) -> None: - assert id or path or source is not None, "Neither id, path nor source given" - self.manager = manager - State.order_counter += 1 - self.order = State.order_counter - self.caller_state = caller_state - self.caller_line = caller_line - if caller_state: - self.import_context = caller_state.import_context[:] - self.import_context.append((caller_state.xpath, caller_line)) - else: - self.import_context = [] - self.id = id or '__main__' - if not path and source is None: - file_id = id - if id == 'builtins' and manager.options.python_version[0] == 2: - # The __builtin__ module is called internally by mypy - # 'builtins' in Python 2 mode (similar to Python 3), - # but the stub file is __builtin__.pyi. The reason is - # that a lot of code hard-codes 'builtins.x' and it's - # easier to work it around like this. It also means - # that the implementation can mostly ignore the - # difference and just assume 'builtins' everywhere, - # which simplifies code. - file_id = '__builtin__' - path = find_module(file_id, manager.lib_path) - if path: - # In silent mode, don't import .py files, except from stubs. - if (manager.options.silent_imports and - path.endswith('.py') and (caller_state or ancestor_for)): - # (Never silence builtins, even if it's a .py file; - # this can happen in tests!) - if (id != 'builtins' and - not ((caller_state and - caller_state.tree and - caller_state.tree.is_stub))): - if manager.options.almost_silent: - if ancestor_for: - self.skipping_ancestor(id, path, ancestor_for) - else: - self.skipping_module(id, path) - path = None - manager.missing_modules.add(id) - raise ModuleNotFound - else: - # Could not find a module. Typically the reason is a - # misspelled module name, missing stub, module not in - # search path or the module has not been installed. - if caller_state: - suppress_message = ((manager.options.silent_imports and - not manager.options.almost_silent) or - (caller_state.tree is not None and - 'import' in caller_state.tree.weak_opts)) - if not suppress_message: - save_import_context = manager.errors.import_context() - manager.errors.set_import_context(caller_state.import_context) - manager.module_not_found(caller_state.xpath, caller_line, id) - manager.errors.set_import_context(save_import_context) - manager.missing_modules.add(id) - raise ModuleNotFound - else: - # If we can't find a root source it's always fatal. - # TODO: This might hide non-fatal errors from - # root sources processed earlier. - raise CompileError(["mypy: can't find module '%s'" % id]) - self.path = path - self.xpath = path or '' - self.source = source - if path and source is None and manager.options.incremental: - self.meta = find_cache_meta(self.id, self.path, manager) - # TODO: Get mtime if not cached. - self.add_ancestors() - if self.meta: - # Make copies, since we may modify these and want to - # compare them to the originals later. - self.dependencies = list(self.meta.dependencies) - self.suppressed = list(self.meta.suppressed) - assert len(self.meta.dependencies) == len(self.meta.dep_prios) - self.priorities = {id: pri - for id, pri in zip(self.meta.dependencies, self.meta.dep_prios)} - self.dep_line_map = {} - else: - # Parse the file (and then some) to get the dependencies. - self.parse_file() - self.suppressed = [] - - def skipping_ancestor(self, id: str, path: str, ancestor_for: 'State') -> None: - # TODO: Read the path (the __init__.py file) and return - # immediately if it's empty or only contains comments. - # But beware, some package may be the ancestor of many modules, - # so we'd need to cache the decision. - manager = self.manager - manager.errors.set_import_context([]) - manager.errors.set_file(ancestor_for.xpath) - manager.errors.report(-1, "Ancestor package '%s' silently ignored" % (id,), - severity='note', only_once=True) - manager.errors.report(-1, "(Using --silent-imports, submodule passed on command line)", - severity='note', only_once=True) - manager.errors.report(-1, "(This note brought to you by --almost-silent)", - severity='note', only_once=True) - - def skipping_module(self, id: str, path: str) -> None: - assert self.caller_state, (id, path) - manager = self.manager - save_import_context = manager.errors.import_context() - manager.errors.set_import_context(self.caller_state.import_context) - manager.errors.set_file(self.caller_state.xpath) - line = self.caller_line - manager.errors.report(line, "Import of '%s' silently ignored" % (id,), - severity='note') - manager.errors.report(line, "(Using --silent-imports, module not passed on command line)", - severity='note', only_once=True) - manager.errors.report(line, "(This note courtesy of --almost-silent)", - severity='note', only_once=True) - manager.errors.set_import_context(save_import_context) - - def add_ancestors(self) -> None: - # All parent packages are new ancestors. - ancestors = [] - parent = self.id - while '.' in parent: - parent, _ = parent.rsplit('.', 1) - ancestors.append(parent) - self.ancestors = ancestors - - def is_fresh(self) -> bool: - """Return whether the cache data for this file is fresh.""" - # NOTE: self.dependencies may differ from - # self.meta.dependencies when a dependency is dropped due to - # suppression by --silent-imports. However when a suppressed - # dependency is added back we find out later in the process. - return self.meta is not None and self.dependencies == self.meta.dependencies - - def mark_stale(self) -> None: - """Throw away the cache data for this file, marking it as stale.""" - self.meta = None - - def check_blockers(self) -> None: - """Raise CompileError if a blocking error is detected.""" - if self.manager.errors.is_blockers(): - self.manager.log("Bailing due to blocking errors") - self.manager.errors.raise_error() - - @contextlib.contextmanager - def wrap_context(self) -> Iterator[None]: - save_import_context = self.manager.errors.import_context() - self.manager.errors.set_import_context(self.import_context) - try: - yield - except CompileError: - raise - except Exception as err: - report_internal_error(err, self.path, 0) - self.manager.errors.set_import_context(save_import_context) - self.check_blockers() - - # Methods for processing cached modules. - - def load_tree(self) -> None: - with open(self.meta.data_json) as f: - data = json.load(f) - # TODO: Assert data file wasn't changed. - self.tree = MypyFile.deserialize(data) - self.manager.modules[self.id] = self.tree - - def fix_cross_refs(self) -> None: - fixup_module_pass_one(self.tree, self.manager.modules) - - def calculate_mros(self) -> None: - fixup_module_pass_two(self.tree, self.manager.modules) - - # Methods for processing modules from source code. - - def parse_file(self) -> None: - if self.tree is not None: - # The file was already parsed (in __init__()). - return - - manager = self.manager - modules = manager.modules - manager.log("Parsing %s (%s)" % (self.xpath, self.id)) - - with self.wrap_context(): - source = self.source - self.source = None # We won't need it again. - if self.path and source is None: - try: - source = read_with_python_encoding(self.path, manager.options.python_version) - except IOError as ioerr: - raise CompileError([ - "mypy: can't read file '{}': {}".format(self.path, ioerr.strerror)]) - except (UnicodeDecodeError, DecodeError) as decodeerr: - raise CompileError([ - "mypy: can't decode file '{}': {}".format(self.path, str(decodeerr))]) - self.tree = manager.parse_file(self.id, self.xpath, source) - - modules[self.id] = self.tree - - # Do the first pass of semantic analysis: add top-level - # definitions in the file to the symbol table. We must do - # this before processing imports, since this may mark some - # import statements as unreachable. - first = FirstPass(manager.semantic_analyzer) - first.analyze(self.tree, self.xpath, self.id) - - # Initialize module symbol table, which was populated by the - # semantic analyzer. - # TODO: Why can't FirstPass .analyze() do this? - self.tree.names = manager.semantic_analyzer.globals - - # Compute (direct) dependencies. - # Add all direct imports (this is why we needed the first pass). - # Also keep track of each dependency's source line. - dependencies = [] - suppressed = [] - priorities = {} # type: Dict[str, int] # id -> priority - dep_line_map = {} # type: Dict[str, int] # id -> line - for pri, id, line in manager.all_imported_modules_in_file(self.tree): - priorities[id] = min(pri, priorities.get(id, PRI_ALL)) - if id == self.id: - continue - # Omit missing modules, as otherwise we could not type-check - # programs with missing modules. - if id in manager.missing_modules: - if id not in dep_line_map: - suppressed.append(id) - dep_line_map[id] = line - continue - if id == '': - # Must be from a relative import. - manager.errors.set_file(self.xpath) - manager.errors.report(line, "No parent module -- cannot perform relative import", - blocker=True) - continue - if id not in dep_line_map: - dependencies.append(id) - dep_line_map[id] = line - # Every module implicitly depends on builtins. - if self.id != 'builtins' and 'builtins' not in dep_line_map: - dependencies.append('builtins') - - # If self.dependencies is already set, it was read from the - # cache, but for some reason we're re-parsing the file. - # NOTE: What to do about race conditions (like editing the - # file while mypy runs)? A previous version of this code - # explicitly checked for this, but ran afoul of other reasons - # for differences (e.g. --silent-imports). - self.dependencies = dependencies - self.suppressed = suppressed - self.priorities = priorities - self.dep_line_map = dep_line_map - self.check_blockers() - - def patch_parent(self) -> None: - # Include module in the symbol table of the enclosing package. - if '.' not in self.id: - return - manager = self.manager - modules = manager.modules - parent, child = self.id.rsplit('.', 1) - if parent in modules: - manager.trace("Added %s.%s" % (parent, child)) - modules[parent].names[child] = SymbolTableNode(MODULE_REF, self.tree, parent) - else: - manager.log("Hm... couldn't add %s.%s" % (parent, child)) - - def semantic_analysis(self) -> None: - with self.wrap_context(): - self.manager.semantic_analyzer.visit_file(self.tree, self.xpath) - - def semantic_analysis_pass_three(self) -> None: - with self.wrap_context(): - self.manager.semantic_analyzer_pass3.visit_file(self.tree, self.xpath) - if self.manager.options.dump_type_stats: - dump_type_stats(self.tree, self.xpath) - - def type_check(self) -> None: - manager = self.manager - if manager.options.semantic_analysis_only: - return - with self.wrap_context(): - manager.type_checker.visit_file(self.tree, self.xpath) - if manager.options.dump_inference_stats: - dump_type_stats(self.tree, self.xpath, inferred=True, - typemap=manager.type_checker.type_map) - manager.report_file(self.tree) - - def write_cache(self) -> None: - if self.path and self.manager.options.incremental and not self.manager.errors.is_errors(): - dep_prios = [self.priorities.get(dep, PRI_HIGH) for dep in self.dependencies] - write_cache(self.id, self.path, self.tree, - list(self.dependencies), list(self.suppressed), - dep_prios, - self.manager) - - -Graph = Dict[str, State] - - -def dispatch(sources: List[BuildSource], manager: BuildManager) -> None: - manager.log("Using new dependency manager") - graph = load_graph(sources, manager) - manager.log("Loaded graph with %d nodes" % len(graph)) - process_graph(graph, manager) - if manager.options.warn_unused_ignores: - manager.errors.generate_unused_ignore_notes() - - -def load_graph(sources: List[BuildSource], manager: BuildManager) -> Graph: - """Given some source files, load the full dependency graph.""" - graph = {} # type: Graph - # The deque is used to implement breadth-first traversal. - # TODO: Consider whether to go depth-first instead. This may - # affect the order in which we process files within import cycles. - new = collections.deque() # type: collections.deque[State] - # Seed the graph with the initial root sources. - for bs in sources: - try: - st = State(id=bs.module, path=bs.path, source=bs.text, manager=manager) - except ModuleNotFound: - continue - if st.id in graph: - manager.errors.set_file(st.xpath) - manager.errors.report(1, "Duplicate module named '%s'" % st.id) - manager.errors.raise_error() - graph[st.id] = st - new.append(st) - # Collect dependencies. We go breadth-first. - while new: - st = new.popleft() - for dep in st.ancestors + st.dependencies: - if dep not in graph: - try: - if dep in st.ancestors: - # TODO: Why not 'if dep not in st.dependencies' ? - # Ancestors don't have import context. - newst = State(id=dep, path=None, source=None, manager=manager, - ancestor_for=st) - else: - newst = State(id=dep, path=None, source=None, manager=manager, - caller_state=st, caller_line=st.dep_line_map.get(dep, 1)) - except ModuleNotFound: - if dep in st.dependencies: - st.dependencies.remove(dep) - st.suppressed.append(dep) - else: - assert newst.id not in graph, newst.id - graph[newst.id] = newst - new.append(newst) - return graph - - -def process_graph(graph: Graph, manager: BuildManager) -> None: - """Process everything in dependency order.""" - sccs = sorted_components(graph) - manager.log("Found %d SCCs; largest has %d nodes" % - (len(sccs), max(len(scc) for scc in sccs))) - # We're processing SCCs from leaves (those without further - # dependencies) to roots (those from which everything else can be - # reached). - for ascc in sccs: - # Order the SCC's nodes using a heuristic. - # Note that ascc is a set, and scc is a list. - scc = order_ascc(graph, ascc) - # If builtins is in the list, move it last. (This is a bit of - # a hack, but it's necessary because the builtins module is - # part of a small cycle involving at least {builtins, abc, - # typing}. Of these, builtins must be processed last or else - # some builtin objects will be incompletely processed.) - if 'builtins' in ascc: - scc.remove('builtins') - scc.append('builtins') - if manager.options.verbosity >= 2: - for id in scc: - manager.trace("Priorities for %s:" % id, - " ".join("%s:%d" % (x, graph[id].priorities[x]) - for x in graph[id].dependencies - if x in ascc and x in graph[id].priorities)) - # Because the SCCs are presented in topological sort order, we - # don't need to look at dependencies recursively for staleness - # -- the immediate dependencies are sufficient. - stale_scc = {id for id in scc if not graph[id].is_fresh()} - fresh = not stale_scc - deps = set() - for id in scc: - deps.update(graph[id].dependencies) - deps -= ascc - stale_deps = {id for id in deps if not graph[id].is_fresh()} - fresh = fresh and not stale_deps - undeps = set() - if fresh: - # Check if any dependencies that were suppressed according - # to the cache have heen added back in this run. - # NOTE: Newly suppressed dependencies are handled by is_fresh(). - for id in scc: - undeps.update(graph[id].suppressed) - undeps &= graph.keys() - if undeps: - fresh = False - if fresh: - # All cache files are fresh. Check that no dependency's - # cache file is newer than any scc node's cache file. - oldest_in_scc = min(graph[id].meta.data_mtime for id in scc) - newest_in_deps = 0 if not deps else max(graph[dep].meta.data_mtime for dep in deps) - if manager.options.verbosity >= 3: # Dump all mtimes for extreme debugging. - all_ids = sorted(ascc | deps, key=lambda id: graph[id].meta.data_mtime) - for id in all_ids: - if id in scc: - if graph[id].meta.data_mtime < newest_in_deps: - key = "*id:" - else: - key = "id:" - else: - if graph[id].meta.data_mtime > oldest_in_scc: - key = "+dep:" - else: - key = "dep:" - manager.trace(" %5s %.0f %s" % (key, graph[id].meta.data_mtime, id)) - # If equal, give the benefit of the doubt, due to 1-sec time granularity - # (on some platforms). - if oldest_in_scc < newest_in_deps: - fresh = False - fresh_msg = "out of date by %.0f seconds" % (newest_in_deps - oldest_in_scc) - else: - fresh_msg = "fresh" - elif undeps: - fresh_msg = "stale due to changed suppression (%s)" % " ".join(sorted(undeps)) - elif stale_scc: - fresh_msg = "inherently stale (%s)" % " ".join(sorted(stale_scc)) - if stale_deps: - fresh_msg += " with stale deps (%s)" % " ".join(sorted(stale_deps)) - else: - fresh_msg = "stale due to deps (%s)" % " ".join(sorted(stale_deps)) - if len(scc) == 1: - manager.log("Processing SCC singleton (%s) as %s" % (" ".join(scc), fresh_msg)) - else: - manager.log("Processing SCC of size %d (%s) as %s" % - (len(scc), " ".join(scc), fresh_msg)) - if fresh: - process_fresh_scc(graph, scc) - else: - process_stale_scc(graph, scc) - - -def order_ascc(graph: Graph, ascc: AbstractSet[str], pri_max: int = PRI_ALL) -> List[str]: - """Come up with the ideal processing order within an SCC. - - Using the priorities assigned by all_imported_modules_in_file(), - try to reduce the cycle to a DAG, by omitting arcs representing - dependencies of lower priority. - - In the simplest case, if we have A <--> B where A has a top-level - "import B" (medium priority) but B only has the reverse "import A" - inside a function (low priority), we turn the cycle into a DAG by - dropping the B --> A arc, which leaves only A --> B. - - If all arcs have the same priority, we fall back to sorting by - reverse global order (the order in which modules were first - encountered). - - The algorithm is recursive, as follows: when as arcs of different - priorities are present, drop all arcs of the lowest priority, - identify SCCs in the resulting graph, and apply the algorithm to - each SCC thus found. The recursion is bounded because at each - recursion the spread in priorities is (at least) one less. - - In practice there are only a few priority levels (currently - N=3) and in the worst case we just carry out the same algorithm - for finding SCCs N times. Thus the complexity is no worse than - the complexity of the original SCC-finding algorithm -- see - strongly_connected_components() below for a reference. - """ - if len(ascc) == 1: - return [s for s in ascc] - pri_spread = set() - for id in ascc: - state = graph[id] - for dep in state.dependencies: - if dep in ascc: - pri = state.priorities.get(dep, PRI_HIGH) - if pri < pri_max: - pri_spread.add(pri) - if len(pri_spread) == 1: - # Filtered dependencies are uniform -- order by global order. - return sorted(ascc, key=lambda id: -graph[id].order) - pri_max = max(pri_spread) - sccs = sorted_components(graph, ascc, pri_max) - # The recursion is bounded by the len(pri_spread) check above. - return [s for ss in sccs for s in order_ascc(graph, ss, pri_max)] - - -def process_fresh_scc(graph: Graph, scc: List[str]) -> None: - """Process the modules in one SCC from their cached data.""" - for id in scc: - graph[id].load_tree() - for id in scc: - graph[id].patch_parent() - for id in scc: - graph[id].fix_cross_refs() - for id in scc: - graph[id].calculate_mros() - - -def process_stale_scc(graph: Graph, scc: List[str]) -> None: - """Process the modules in one SCC from source code.""" - for id in scc: - graph[id].mark_stale() - for id in scc: - # We may already have parsed the module, or not. - # If the former, parse_file() is a no-op. - graph[id].parse_file() - for id in scc: - graph[id].patch_parent() - for id in scc: - graph[id].semantic_analysis() - for id in scc: - graph[id].semantic_analysis_pass_three() - for id in scc: - graph[id].type_check() - graph[id].write_cache() - - -def sorted_components(graph: Graph, - vertices: Optional[AbstractSet[str]] = None, - pri_max: int = PRI_ALL) -> List[AbstractSet[str]]: - """Return the graph's SCCs, topologically sorted by dependencies. - - The sort order is from leaves (nodes without dependencies) to - roots (nodes on which no other nodes depend). - - This works for a subset of the full dependency graph too; - dependencies that aren't present in graph.keys() are ignored. - """ - # Compute SCCs. - if vertices is None: - vertices = set(graph) - edges = {id: deps_filtered(graph, vertices, id, pri_max) for id in vertices} - sccs = list(strongly_connected_components(vertices, edges)) - # Topsort. - sccsmap = {id: frozenset(scc) for scc in sccs for id in scc} - data = {} # type: Dict[AbstractSet[str], Set[AbstractSet[str]]] - for scc in sccs: - deps = set() # type: Set[AbstractSet[str]] - for id in scc: - deps.update(sccsmap[x] for x in deps_filtered(graph, vertices, id, pri_max)) - data[frozenset(scc)] = deps - res = [] - for ready in topsort(data): - # Sort the sets in ready by reversed smallest State.order. Examples: - # - # - If ready is [{x}, {y}], x.order == 1, y.order == 2, we get - # [{y}, {x}]. - # - # - If ready is [{a, b}, {c, d}], a.order == 1, b.order == 3, - # c.order == 2, d.order == 4, the sort keys become [1, 2] - # and the result is [{c, d}, {a, b}]. - res.extend(sorted(ready, - key=lambda scc: -min(graph[id].order for id in scc))) - return res - - -def deps_filtered(graph: Graph, vertices: AbstractSet[str], id: str, pri_max: int) -> List[str]: - """Filter dependencies for id with pri < pri_max.""" - if id not in vertices: - return [] - state = graph[id] - return [dep - for dep in state.dependencies - if dep in vertices and state.priorities.get(dep, PRI_HIGH) < pri_max] - - -def strongly_connected_components(vertices: AbstractSet[str], - edges: Dict[str, List[str]]) -> Iterator[Set[str]]: - """Compute Strongly Connected Components of a directed graph. - - Args: - vertices: the labels for the vertices - edges: for each vertex, gives the target vertices of its outgoing edges - - Returns: - An iterator yielding strongly connected components, each - represented as a set of vertices. Each input vertex will occur - exactly once; vertices not part of a SCC are returned as - singleton sets. - - From http://code.activestate.com/recipes/578507/. - """ - identified = set() # type: Set[str] - stack = [] # type: List[str] - index = {} # type: Dict[str, int] - boundaries = [] # type: List[int] - - def dfs(v: str) -> Iterator[Set[str]]: - index[v] = len(stack) - stack.append(v) - boundaries.append(index[v]) - - for w in edges[v]: - if w not in index: - # For Python >= 3.3, replace with "yield from dfs(w)" - for scc in dfs(w): - yield scc - elif w not in identified: - while index[w] < boundaries[-1]: - boundaries.pop() - - if boundaries[-1] == index[v]: - boundaries.pop() - scc = set(stack[index[v]:]) - del stack[index[v]:] - identified.update(scc) - yield scc - - for v in vertices: - if v not in index: - # For Python >= 3.3, replace with "yield from dfs(v)" - for scc in dfs(v): - yield scc - - -def topsort(data: Dict[AbstractSet[str], - Set[AbstractSet[str]]]) -> Iterable[Set[AbstractSet[str]]]: - """Topological sort. - - Args: - data: A map from SCCs (represented as frozen sets of strings) to - sets of SCCs, its dependencies. NOTE: This data structure - is modified in place -- for normalization purposes, - self-dependencies are removed and entries representing - orphans are added. - - Returns: - An iterator yielding sets of SCCs that have an equivalent - ordering. NOTE: The algorithm doesn't care about the internal - structure of SCCs. - - Example: - Suppose the input has the following structure: - - {A: {B, C}, B: {D}, C: {D}} - - This is normalized to: - - {A: {B, C}, B: {D}, C: {D}, D: {}} - - The algorithm will yield the following values: - - {D} - {B, C} - {A} - - From http://code.activestate.com/recipes/577413/. - """ - # TODO: Use a faster algorithm? - for k, v in data.items(): - v.discard(k) # Ignore self dependencies. - for item in set.union(*data.values()) - set(data.keys()): - data[item] = set() - while True: - ready = {item for item, dep in data.items() if not dep} - if not ready: - break - yield ready - data = {item: (dep - ready) - for item, dep in data.items() - if item not in ready} - assert not data, "A cyclic dependency exists amongst %r" % data +"""Facilities to analyze entire programs, including imported modules. + +Parse and analyze the source files of a program in the correct order +(based on file dependencies), and collect the results. + +This module only directs a build, which is performed in multiple passes per +file. The individual passes are implemented in separate modules. + +The function build() is the main interface to this module. +""" +# TODO: More consistent terminology, e.g. path/fnam, module/id, state/file + +import binascii +import collections +import contextlib +import json +import os +import os.path +import sys +import time +from os.path import dirname, basename + +from typing import (AbstractSet, Dict, Iterable, Iterator, List, + NamedTuple, Optional, Set, Tuple, Union, Mapping) + +from mypy.types import Type +from mypy.nodes import (MypyFile, Node, Import, ImportFrom, ImportAll, + SymbolTableNode, MODULE_REF) +from mypy.semanal import FirstPass, SemanticAnalyzer, ThirdPass +from mypy.checker import TypeChecker +from mypy.errors import Errors, CompileError, DecodeError, report_internal_error +from mypy import fixup +from mypy.report import Reports +from mypy import defaults +from mypy import moduleinfo +from mypy import util +from mypy.fixup import fixup_module_pass_one, fixup_module_pass_two +from mypy.options import Options +from mypy.parse import parse +from mypy.stats import dump_type_stats + + +# We need to know the location of this file to load data, but +# until Python 3.4, __file__ is relative. +__file__ = os.path.realpath(__file__) + +PYTHON_EXTENSIONS = ['.pyi', '.py'] + + +class BuildResult: + """The result of a successful build. + + Attributes: + manager: The build manager. + files: Dictionary from module name to related AST node. + types: Dictionary from parse tree node to its inferred type. + errors: List of error messages. + """ + + def __init__(self, manager: 'BuildManager') -> None: + self.manager = manager + self.files = manager.modules + self.types = manager.type_checker.type_map + self.errors = manager.errors.messages() + + +class BuildSource: + def __init__(self, path: Optional[str], module: Optional[str], + text: Optional[str]) -> None: + self.path = path + self.module = module or '__main__' + self.text = text + + @property + def effective_path(self) -> str: + """Return the effective path (ie, if its from in memory)""" + return self.path or '' + + +class BuildSourceSet: + """Efficiently test a file's membership in the set of build sources.""" + + def __init__(self, sources: List[BuildSource]) -> None: + self.source_text_present = False + self.source_modules = set() # type: Set[str] + self.source_paths = set() # type: Set[str] + + for source in sources: + if source.text is not None: + self.source_text_present = True + elif source.path: + self.source_paths.add(source.path) + else: + self.source_modules.add(source.module) + + def is_source(self, file: MypyFile) -> bool: + if file.path and file.path in self.source_paths: + return True + elif file._fullname in self.source_modules: + return True + elif file.path is None and self.source_text_present: + return True + else: + return False + + +def build(sources: List[BuildSource], + options: Options, + alt_lib_path: str = None, + bin_dir: str = None) -> BuildResult: + """Analyze a program. + + A single call to build performs parsing, semantic analysis and optionally + type checking for the program *and* all imported modules, recursively. + + Return BuildResult if successful or only non-blocking errors were found; + otherwise raise CompileError. + + Args: + sources: list of sources to build + options: build options + alt_lib_path: an additional directory for looking up library modules + (takes precedence over other directories) + bin_dir: directory containing the mypy script, used for finding data + directories; if omitted, use '.' as the data directory + """ + + data_dir = default_data_dir(bin_dir) + + find_module_clear_caches() + + # Determine the default module search path. + lib_path = default_lib_path(data_dir, options.python_version) + + if options.use_builtins_fixtures: + # Use stub builtins (to speed up test cases and to make them easier to + # debug). This is a test-only feature, so assume our files are laid out + # as in the source tree. + root_dir = os.path.dirname(os.path.dirname(__file__)) + lib_path.insert(0, os.path.join(root_dir, 'test-data', 'unit', 'lib-stub')) + else: + for source in sources: + if source.path: + # Include directory of the program file in the module search path. + dir = remove_cwd_prefix_from_path(dirname(source.path)) + if dir not in lib_path: + lib_path.insert(0, dir) + + # Do this even if running as a file, for sanity (mainly because with + # multiple builds, there could be a mix of files/modules, so its easier + # to just define the semantics that we always add the current director + # to the lib_path + lib_path.insert(0, os.getcwd()) + + # Add MYPYPATH environment variable to front of library path, if defined. + lib_path[:0] = mypy_path() + + # If provided, insert the caller-supplied extra module path to the + # beginning (highest priority) of the search path. + if alt_lib_path: + lib_path.insert(0, alt_lib_path) + + reports = Reports(data_dir, options.report_dirs) + + source_set = BuildSourceSet(sources) + + # Construct a build manager object to hold state during the build. + # + # Ignore current directory prefix in error messages. + manager = BuildManager(data_dir, lib_path, + ignore_prefix=os.getcwd(), + source_set=source_set, + reports=reports, + options=options) + + try: + dispatch(sources, manager) + return BuildResult(manager) + finally: + manager.log("Build finished with %d modules, %d types, and %d errors" % + (len(manager.modules), + len(manager.type_checker.type_map), + manager.errors.num_messages())) + # Finish the HTML or XML reports even if CompileError was raised. + reports.finish() + + +def default_data_dir(bin_dir: Optional[str]) -> str: + """Returns directory containing typeshed directory + + Args: + bin_dir: directory containing the mypy script + """ + if not bin_dir: + mypy_package = os.path.dirname(__file__) + parent = os.path.dirname(mypy_package) + if (os.path.basename(parent) == 'site-packages' or + os.path.basename(parent) == 'dist-packages'): + # Installed in site-packages or dist-packages, but invoked with python3 -m mypy; + # __file__ is .../blah/lib/python3.N/site-packages/mypy/build.py + # or .../blah/lib/python3.N/dist-packages/mypy/build.py (Debian) + # or .../blah/lib/site-packages/mypy/build.py (Windows) + # blah may be a virtualenv or /usr/local. We want .../blah/lib/mypy. + lib = parent + for i in range(2): + lib = os.path.dirname(lib) + if os.path.basename(lib) == 'lib': + return os.path.join(lib, 'mypy') + subdir = os.path.join(parent, 'lib', 'mypy') + if os.path.isdir(subdir): + # If installed via buildout, the __file__ is + # somewhere/mypy/__init__.py and what we want is + # somewhere/lib/mypy. + return subdir + # Default to directory containing this file's parent. + return parent + base = os.path.basename(bin_dir) + dir = os.path.dirname(bin_dir) + if (sys.platform == 'win32' and base.lower() == 'scripts' + and not os.path.isdir(os.path.join(dir, 'typeshed'))): + # Installed, on Windows. + return os.path.join(dir, 'Lib', 'mypy') + elif base == 'scripts': + # Assume that we have a repo check out or unpacked source tarball. + return dir + elif base == 'bin': + # Installed to somewhere (can be under /usr/local or anywhere). + return os.path.join(dir, 'lib', 'mypy') + elif base == 'python3': + # Assume we installed python3 with brew on os x + return os.path.join(os.path.dirname(dir), 'lib', 'mypy') + elif dir.endswith('python-exec'): + # Gentoo uses a python wrapper in /usr/lib to which mypy is a symlink. + return os.path.join(os.path.dirname(dir), 'mypy') + else: + # Don't know where to find the data files! + raise RuntimeError("Broken installation: can't determine base dir") + + +def mypy_path() -> List[str]: + path_env = os.getenv('MYPYPATH') + if not path_env: + return [] + return path_env.split(os.pathsep) + + +def default_lib_path(data_dir: str, pyversion: Tuple[int, int]) -> List[str]: + """Return default standard library search paths.""" + # IDEA: Make this more portable. + path = [] # type: List[str] + + auto = os.path.join(data_dir, 'stubs-auto') + if os.path.isdir(auto): + data_dir = auto + + # We allow a module for e.g. version 3.5 to be in 3.4/. The assumption + # is that a module added with 3.4 will still be present in Python 3.5. + versions = ["%d.%d" % (pyversion[0], minor) + for minor in reversed(range(pyversion[1] + 1))] + # E.g. for Python 3.2, try 3.2/, 3.1/, 3.0/, 3/, 2and3/. + # (Note that 3.1 and 3.0 aren't really supported, but we don't care.) + for v in versions + [str(pyversion[0]), '2and3']: + for lib_type in ['stdlib', 'third_party']: + stubdir = os.path.join(data_dir, 'typeshed', lib_type, v) + if os.path.isdir(stubdir): + path.append(stubdir) + + # Add fallback path that can be used if we have a broken installation. + if sys.platform != 'win32': + path.append('/usr/local/lib/mypy') + + return path + + +CacheMeta = NamedTuple('CacheMeta', + [('id', str), + ('path', str), + ('mtime', float), + ('size', int), + ('dependencies', List[str]), # names of imported modules + ('data_mtime', float), # mtime of data_json + ('data_json', str), # path of .data.json + ('suppressed', List[str]), # dependencies that weren't imported + ('options', Optional[Dict[str, bool]]), # build options + ('dep_prios', List[int]), + ]) +# NOTE: dependencies + suppressed == all reachable imports; +# suppressed contains those reachable imports that were prevented by +# --silent-imports or simply not found. + + +# Priorities used for imports. (Here, top-level includes inside a class.) +# These are used to determine a more predictable order in which the +# nodes in an import cycle are processed. +PRI_HIGH = 5 # top-level "from X import blah" +PRI_MED = 10 # top-level "import X" +PRI_LOW = 20 # either form inside a function +PRI_ALL = 99 # include all priorities + + +class BuildManager: + """This class holds shared state for building a mypy program. + + It is used to coordinate parsing, import processing, semantic + analysis and type checking. The actual build steps are carried + out by dispatch(). + + Attributes: + data_dir: Mypy data directory (contains stubs) + lib_path: Library path for looking up modules + modules: Mapping of module ID to MypyFile (shared by the passes) + semantic_analyzer: + Semantic analyzer, pass 2 + semantic_analyzer_pass3: + Semantic analyzer, pass 3 + type_checker: Type checker + errors: Used for reporting all errors + options: Build options + missing_modules: Set of modules that could not be imported encountered so far + """ + + def __init__(self, data_dir: str, + lib_path: List[str], + ignore_prefix: str, + source_set: BuildSourceSet, + reports: Reports, + options: Options) -> None: + self.start_time = time.time() + self.data_dir = data_dir + self.errors = Errors() + self.errors.set_ignore_prefix(ignore_prefix) + self.lib_path = tuple(lib_path) + self.source_set = source_set + self.reports = reports + self.options = options + self.semantic_analyzer = SemanticAnalyzer(lib_path, self.errors, options=options) + self.modules = self.semantic_analyzer.modules + self.semantic_analyzer_pass3 = ThirdPass(self.modules, self.errors) + self.type_checker = TypeChecker(self.errors, self.modules, options=options) + self.missing_modules = set() # type: Set[str] + + def all_imported_modules_in_file(self, + file: MypyFile) -> List[Tuple[int, str, int]]: + """Find all reachable import statements in a file. + + Return list of tuples (priority, module id, import line number) + for all modules imported in file; lower numbers == higher priority. + """ + + def correct_rel_imp(imp: Union[ImportFrom, ImportAll]) -> str: + """Function to correct for relative imports.""" + file_id = file.fullname() + rel = imp.relative + if rel == 0: + return imp.id + if os.path.basename(file.path).startswith('__init__.'): + rel -= 1 + if rel != 0: + file_id = ".".join(file_id.split(".")[:-rel]) + new_id = file_id + "." + imp.id if imp.id else file_id + + return new_id + + res = [] # type: List[Tuple[int, str, int]] + for imp in file.imports: + if not imp.is_unreachable: + if isinstance(imp, Import): + pri = PRI_MED if imp.is_top_level else PRI_LOW + for id, _ in imp.ids: + res.append((pri, id, imp.line)) + elif isinstance(imp, ImportFrom): + cur_id = correct_rel_imp(imp) + pos = len(res) + all_are_submodules = True + # Also add any imported names that are submodules. + pri = PRI_MED if imp.is_top_level else PRI_LOW + for name, __ in imp.names: + sub_id = cur_id + '.' + name + if self.is_module(sub_id): + res.append((pri, sub_id, imp.line)) + else: + all_are_submodules = False + # If all imported names are submodules, don't add + # cur_id as a dependency. Otherwise (i.e., if at + # least one imported name isn't a submodule) + # cur_id is also a dependency, and we should + # insert it *before* any submodules. + if not all_are_submodules: + pri = PRI_HIGH if imp.is_top_level else PRI_LOW + res.insert(pos, ((pri, cur_id, imp.line))) + elif isinstance(imp, ImportAll): + pri = PRI_HIGH if imp.is_top_level else PRI_LOW + res.append((pri, correct_rel_imp(imp), imp.line)) + + return res + + def is_module(self, id: str) -> bool: + """Is there a file in the file system corresponding to module id?""" + return find_module(id, self.lib_path) is not None + + def parse_file(self, id: str, path: str, source: str) -> MypyFile: + """Parse the source of a file with the given name. + + Raise CompileError if there is a parse error. + """ + num_errs = self.errors.num_messages() + tree = parse(source, path, self.errors, options=self.options) + tree._fullname = id + + # We don't want to warn about 'type: ignore' comments on + # imports, but we're about to modify tree.imports, so grab + # these first. + import_lines = set(node.line for node in tree.imports) + + # Skip imports that have been ignored (so that we can ignore a C extension module without + # stub, for example), except for 'from x import *', because we wouldn't be able to + # determine which names should be defined unless we process the module. We can still + # ignore errors such as redefinitions when using the latter form. + imports = [node for node in tree.imports + if node.line not in tree.ignored_lines or isinstance(node, ImportAll)] + tree.imports = imports + + if self.errors.num_messages() != num_errs: + self.log("Bailing due to parse errors") + self.errors.raise_error() + + self.errors.set_file_ignored_lines(path, tree.ignored_lines) + self.errors.mark_file_ignored_lines_used(path, import_lines) + return tree + + def module_not_found(self, path: str, line: int, id: str) -> None: + self.errors.set_file(path) + stub_msg = "(Stub files are from https://github.com/python/typeshed)" + if ((self.options.python_version[0] == 2 and moduleinfo.is_py2_std_lib_module(id)) or + (self.options.python_version[0] >= 3 and moduleinfo.is_py3_std_lib_module(id))): + self.errors.report( + line, "No library stub file for standard library module '{}'".format(id)) + self.errors.report(line, stub_msg, severity='note', only_once=True) + elif moduleinfo.is_third_party_module(id): + self.errors.report(line, "No library stub file for module '{}'".format(id)) + self.errors.report(line, stub_msg, severity='note', only_once=True) + else: + self.errors.report(line, "Cannot find module named '{}'".format(id)) + self.errors.report(line, '(Perhaps setting MYPYPATH ' + 'or using the "--silent-imports" flag would help)', + severity='note', only_once=True) + + def report_file(self, file: MypyFile) -> None: + if self.source_set.is_source(file): + self.reports.file(file, type_map=self.type_checker.type_map) + + def log(self, *message: str) -> None: + if self.options.verbosity >= 1: + print('%.3f:LOG: ' % (time.time() - self.start_time), *message, file=sys.stderr) + sys.stderr.flush() + + def trace(self, *message: str) -> None: + if self.options.verbosity >= 2: + print('%.3f:TRACE:' % (time.time() - self.start_time), *message, file=sys.stderr) + sys.stderr.flush() + + +def remove_cwd_prefix_from_path(p: str) -> str: + """Remove current working directory prefix from p, if present. + + Also crawl up until a directory without __init__.py is found. + + If the result would be empty, return '.' instead. + """ + cur = os.getcwd() + # Add separator to the end of the path, unless one is already present. + if basename(cur) != '': + cur += os.sep + # Compute root path. + while (p and + (os.path.isfile(os.path.join(p, '__init__.py')) or + os.path.isfile(os.path.join(p, '__init__.pyi')))): + dir, base = os.path.split(p) + if not base: + break + p = dir + # Remove current directory prefix from the path, if present. + if p.startswith(cur): + p = p[len(cur):] + # Avoid returning an empty path; replace that with '.'. + if p == '': + p = '.' + return p + + +# Cache find_module: (id, lib_path) -> result. +find_module_cache = {} # type: Dict[Tuple[str, Tuple[str, ...]], str] + +# Cache some repeated work within distinct find_module calls: finding which +# elements of lib_path have even the subdirectory they'd need for the module +# to exist. This is shared among different module ids when they differ only +# in the last component. +find_module_dir_cache = {} # type: Dict[Tuple[str, Tuple[str, ...]], List[str]] + +# Cache directory listings. We assume that while one os.listdir() +# call may be more expensive than one os.stat() call, a small number +# of os.stat() calls is quickly more expensive than caching the +# os.listdir() outcome, and the advantage of the latter is that it +# gives us the case-correct filename on Windows and Mac. +find_module_listdir_cache = {} # type: Dict[str, Optional[List[str]]] + + +def find_module_clear_caches(): + find_module_cache.clear() + find_module_dir_cache.clear() + find_module_listdir_cache.clear() + + +def list_dir(path: str) -> Optional[List[str]]: + """Return a cached directory listing. + + Returns None if the path doesn't exist or isn't a directory. + """ + if path in find_module_listdir_cache: + return find_module_listdir_cache[path] + try: + res = os.listdir(path) + except OSError: + res = None + find_module_listdir_cache[path] = res + return res + + +def is_file(path: str) -> bool: + """Return whether path exists and is a file. + + On case-insensitive filesystems (like Mac or Windows) this returns + False if the case of the path's last component does not exactly + match the case found in the filesystem. + """ + head, tail = os.path.split(path) + if not tail: + return False + names = list_dir(head) + if not names: + return False + if tail not in names: + return False + return os.path.isfile(path) + + +def find_module(id: str, lib_path: Iterable[str]) -> str: + """Return the path of the module source file, or None if not found.""" + if not isinstance(lib_path, tuple): + lib_path = tuple(lib_path) + + def find(): + # If we're looking for a module like 'foo.bar.baz', it's likely that most of the + # many elements of lib_path don't even have a subdirectory 'foo/bar'. Discover + # that only once and cache it for when we look for modules like 'foo.bar.blah' + # that will require the same subdirectory. + components = id.split('.') + dir_chain = os.sep.join(components[:-1]) # e.g., 'foo/bar' + if (dir_chain, lib_path) not in find_module_dir_cache: + dirs = [] + for pathitem in lib_path: + # e.g., '/usr/lib/python3.4/foo/bar' + dir = os.path.normpath(os.path.join(pathitem, dir_chain)) + if os.path.isdir(dir): + dirs.append(dir) + find_module_dir_cache[dir_chain, lib_path] = dirs + candidate_base_dirs = find_module_dir_cache[dir_chain, lib_path] + + # If we're looking for a module like 'foo.bar.baz', then candidate_base_dirs now + # contains just the subdirectories 'foo/bar' that actually exist under the + # elements of lib_path. This is probably much shorter than lib_path itself. + # Now just look for 'baz.pyi', 'baz/__init__.py', etc., inside those directories. + seplast = os.sep + components[-1] # so e.g. '/baz' + sepinit = os.sep + '__init__' + for base_dir in candidate_base_dirs: + base_path = base_dir + seplast # so e.g. '/usr/lib/python3.4/foo/bar/baz' + # Prefer package over module, i.e. baz/__init__.py* over baz.py*. + for extension in PYTHON_EXTENSIONS: + path = base_path + sepinit + extension + if is_file(path) and verify_module(id, path): + return path + # No package, look for module. + for extension in PYTHON_EXTENSIONS: + path = base_path + extension + if is_file(path) and verify_module(id, path): + return path + return None + + key = (id, lib_path) + if key not in find_module_cache: + find_module_cache[key] = find() + return find_module_cache[key] + + +def find_modules_recursive(module: str, lib_path: List[str]) -> List[BuildSource]: + module_path = find_module(module, lib_path) + if not module_path: + return [] + result = [BuildSource(module_path, module, None)] + if module_path.endswith(('__init__.py', '__init__.pyi')): + # Subtle: this code prefers the .pyi over the .py if both + # exists, and also prefers packages over modules if both x/ + # and x.py* exist. How? We sort the directory items, so x + # comes before x.py and x.pyi. But the preference for .pyi + # over .py is encoded in find_module(); even though we see + # x.py before x.pyi, find_module() will find x.pyi first. We + # use hits to avoid adding it a second time when we see x.pyi. + # This also avoids both x.py and x.pyi when x/ was seen first. + hits = set() # type: Set[str] + for item in sorted(os.listdir(os.path.dirname(module_path))): + abs_path = os.path.join(os.path.dirname(module_path), item) + if os.path.isdir(abs_path) and \ + (os.path.isfile(os.path.join(abs_path, '__init__.py')) or + os.path.isfile(os.path.join(abs_path, '__init__.pyi'))): + hits.add(item) + result += find_modules_recursive(module + '.' + item, lib_path) + elif item != '__init__.py' and item != '__init__.pyi' and \ + item.endswith(('.py', '.pyi')): + mod = item.split('.')[0] + if mod not in hits: + hits.add(mod) + result += find_modules_recursive( + module + '.' + mod, lib_path) + return result + + +def verify_module(id: str, path: str) -> bool: + """Check that all packages containing id have a __init__ file.""" + if path.endswith(('__init__.py', '__init__.pyi')): + path = dirname(path) + for i in range(id.count('.')): + path = dirname(path) + if not any(os.path.isfile(os.path.join(path, '__init__{}'.format(extension))) + for extension in PYTHON_EXTENSIONS): + return False + return True + + +def read_with_python_encoding(path: str, pyversion: Tuple[int, int]) -> str: + """Read the Python file with while obeying PEP-263 encoding detection""" + source_bytearray = bytearray() + encoding = 'utf8' if pyversion[0] >= 3 else 'ascii' + + with open(path, 'rb') as f: + # read first two lines and check if PEP-263 coding is present + source_bytearray.extend(f.readline()) + source_bytearray.extend(f.readline()) + + # check for BOM UTF-8 encoding and strip it out if present + if source_bytearray.startswith(b'\xef\xbb\xbf'): + encoding = 'utf8' + source_bytearray = source_bytearray[3:] + else: + _encoding, _ = util.find_python_encoding(source_bytearray, pyversion) + # check that the coding isn't mypy. We skip it since + # registering may not have happened yet + if _encoding != 'mypy': + encoding = _encoding + + source_bytearray.extend(f.read()) + try: + source_bytearray.decode(encoding) + except LookupError as lookuperr: + raise DecodeError(str(lookuperr)) + return source_bytearray.decode(encoding) + + +MYPY_CACHE = '.mypy_cache' + + +def get_cache_names(id: str, path: str, pyversion: Tuple[int, int]) -> Tuple[str, str]: + """Return the file names for the cache files. + + Args: + id: module ID + path: module path (used to recognize packages) + pyversion: Python version (major, minor) + + Returns: + A tuple with the file names to be used for the meta JSON and the + data JSON, respectively. + """ + prefix = os.path.join(MYPY_CACHE, '%d.%d' % pyversion, *id.split('.')) + is_package = os.path.basename(path).startswith('__init__.py') + if is_package: + prefix = os.path.join(prefix, '__init__') + return (prefix + '.meta.json', prefix + '.data.json') + + +def find_cache_meta(id: str, path: str, manager: BuildManager) -> Optional[CacheMeta]: + """Find cache data for a module. + + Args: + id: module ID + path: module path + manager: the build manager (for pyversion, log/trace, and build options) + + Returns: + A CacheMeta instance if the cache data was found and appears + valid; otherwise None. + """ + # TODO: May need to take more build options into account + meta_json, data_json = get_cache_names(id, path, manager.options.python_version) + manager.trace('Looking for {} {}'.format(id, data_json)) + if not os.path.exists(meta_json): + return None + with open(meta_json, 'r') as f: + meta_str = f.read() + manager.trace('Meta {} {}'.format(id, meta_str.rstrip())) + meta = json.loads(meta_str) # TODO: Errors + if not isinstance(meta, dict): + return None + path = os.path.abspath(path) + m = CacheMeta( + meta.get('id'), + meta.get('path'), + meta.get('mtime'), + meta.get('size'), + meta.get('dependencies', []), + meta.get('data_mtime'), + data_json, + meta.get('suppressed', []), + meta.get('options'), + meta.get('dep_prios', []), + ) + if (m.id != id or m.path != path or + m.mtime is None or m.size is None or + m.dependencies is None or m.data_mtime is None): + return None + + # Ignore cache if generated by an older mypy version. + if m.options is None or len(m.dependencies) != len(m.dep_prios): + return None + + # Ignore cache if (relevant) options aren't the same. + cached_options = m.options + current_options = select_options_affecting_cache(manager.options) + if cached_options != current_options: + return None + + # TODO: Share stat() outcome with find_module() + st = os.stat(path) # TODO: Errors + if st.st_mtime != m.mtime or st.st_size != m.size: + manager.log('Metadata abandoned because of modified file {}'.format(path)) + return None + # It's a match on (id, path, mtime, size). + # Check data_json; assume if its mtime matches it's good. + # TODO: stat() errors + if os.path.getmtime(data_json) != m.data_mtime: + return None + manager.log('Found {} {}'.format(id, meta_json)) + return m + + +def select_options_affecting_cache(options: Options) -> Mapping[str, bool]: + return {opt: getattr(options, opt) for opt in OPTIONS_AFFECTING_CACHE} + + +OPTIONS_AFFECTING_CACHE = [ + "silent_imports", + "almost_silent", + "disallow_untyped_calls", + "disallow_untyped_defs", + "check_untyped_defs", +] + + +def random_string(): + return binascii.hexlify(os.urandom(8)).decode('ascii') + + +def write_cache(id: str, path: str, tree: MypyFile, + dependencies: List[str], suppressed: List[str], + dep_prios: List[int], + manager: BuildManager) -> None: + """Write cache files for a module. + + Args: + id: module ID + path: module path + tree: the fully checked module data + dependencies: module IDs on which this module depends + suppressed: module IDs which were suppressed as dependencies + dep_prios: priorities (parallel array to dependencies) + manager: the build manager (for pyversion, log/trace) + """ + path = os.path.abspath(path) + manager.trace('Dumping {} {}'.format(id, path)) + st = os.stat(path) # TODO: Errors + mtime = st.st_mtime + size = st.st_size + meta_json, data_json = get_cache_names(id, path, manager.options.python_version) + manager.log('Writing {} {} {}'.format(id, meta_json, data_json)) + data = tree.serialize() + parent = os.path.dirname(data_json) + if not os.path.isdir(parent): + os.makedirs(parent) + assert os.path.dirname(meta_json) == parent + nonce = '.' + random_string() + data_json_tmp = data_json + nonce + meta_json_tmp = meta_json + nonce + with open(data_json_tmp, 'w') as f: + json.dump(data, f, indent=2, sort_keys=True) + f.write('\n') + data_mtime = os.path.getmtime(data_json_tmp) + meta = {'id': id, + 'path': path, + 'mtime': mtime, + 'size': size, + 'data_mtime': data_mtime, + 'dependencies': dependencies, + 'suppressed': suppressed, + 'options': select_options_affecting_cache(manager.options), + 'dep_prios': dep_prios, + } + with open(meta_json_tmp, 'w') as f: + json.dump(meta, f, sort_keys=True) + f.write('\n') + os.replace(data_json_tmp, data_json) + os.replace(meta_json_tmp, meta_json) + + +"""Dependency manager. + +Design +====== + +Ideally +------- + +A. Collapse cycles (each SCC -- strongly connected component -- + becomes one "supernode"). + +B. Topologically sort nodes based on dependencies. + +C. Process from leaves towards roots. + +Wrinkles +-------- + +a. Need to parse source modules to determine dependencies. + +b. Processing order for modules within an SCC. + +c. Must order mtimes of files to decide whether to re-process; depends + on clock never resetting. + +d. from P import M; checks filesystem whether module P.M exists in + filesystem. + +e. Race conditions, where somebody modifies a file while we're + processing. I propose not to modify the algorithm to handle this, + but to detect when this could lead to inconsistencies. (For + example, when we decide on the dependencies based on cache + metadata, and then we decide to re-parse a file because of a stale + dependency, if the re-parsing leads to a different list of + dependencies we should warn the user or start over.) + +Steps +----- + +1. For each explicitly given module find the source file location. + +2. For each such module load and check the cache metadata, and decide + whether it's valid. + +3. Now recursively (or iteratively) find dependencies and add those to + the graph: + + - for cached nodes use the list of dependencies from the cache + metadata (this will be valid even if we later end up re-parsing + the same source); + + - for uncached nodes parse the file and process all imports found, + taking care of (a) above. + +Step 3 should also address (d) above. + +Once step 3 terminates we have the entire dependency graph, and for +each module we've either loaded the cache metadata or parsed the +source code. (However, we may still need to parse those modules for +which we have cache metadata but that depend, directly or indirectly, +on at least one module for which the cache metadata is stale.) + +Now we can execute steps A-C from the first section. Finding SCCs for +step A shouldn't be hard; there's a recipe here: +http://code.activestate.com/recipes/578507/. There's also a plethora +of topsort recipes, e.g. http://code.activestate.com/recipes/577413/. + +For single nodes, processing is simple. If the node was cached, we +deserialize the cache data and fix up cross-references. Otherwise, we +do semantic analysis followed by type checking. We also handle (c) +above; if a module has valid cache data *but* any of its +dependencies was processed from source, then the module should be +processed from source. + +A relatively simple optimization (outside SCCs) we might do in the +future is as follows: if a node's cache data is valid, but one or more +of its dependencies are out of date so we have to re-parse the node +from source, once we have fully type-checked the node, we can decide +whether its symbol table actually changed compared to the cache data +(by reading the cache data and comparing it to the data we would be +writing). If there is no change we can declare the node up to date, +and any node that depends (and for which we have cached data, and +whose other dependencies are up to date) on it won't need to be +re-parsed from source. + +Import cycles +------------- + +Finally we have to decide how to handle (c), import cycles. Here +we'll need a modified version of the original state machine +(build.py), but we only need to do this per SCC, and we won't have to +deal with changes to the list of nodes while we're processing it. + +If all nodes in the SCC have valid cache metadata and all dependencies +outside the SCC are still valid, we can proceed as follows: + + 1. Load cache data for all nodes in the SCC. + + 2. Fix up cross-references for all nodes in the SCC. + +Otherwise, the simplest (but potentially slow) way to proceed is to +invalidate all cache data in the SCC and re-parse all nodes in the SCC +from source. We can do this as follows: + + 1. Parse source for all nodes in the SCC. + + 2. Semantic analysis for all nodes in the SCC. + + 3. Type check all nodes in the SCC. + +(If there are more passes the process is the same -- each pass should +be done for all nodes before starting the next pass for any nodes in +the SCC.) + +We could process the nodes in the SCC in any order. For sentimental +reasons, I've decided to process them in the reverse order in which we +encountered them when originally constructing the graph. That's how +the old build.py deals with cycles, and at least this reproduces the +previous implementation more accurately. + +Can we do better than re-parsing all nodes in the SCC when any of its +dependencies are out of date? It's doubtful. The optimization +mentioned at the end of the previous section would require re-parsing +and type-checking a node and then comparing its symbol table to the +cached data; but because the node is part of a cycle we can't +technically type-check it until the semantic analysis of all other +nodes in the cycle has completed. (This is an important issue because +Dropbox has a very large cycle in production code. But I'd like to +deal with it later.) + +Additional wrinkles +------------------- + +During implementation more wrinkles were found. + +- When a submodule of a package (e.g. x.y) is encountered, the parent + package (e.g. x) must also be loaded, but it is not strictly a + dependency. See State.add_ancestors() below. +""" + + +class ModuleNotFound(Exception): + """Control flow exception to signal that a module was not found.""" + + +class State: + """The state for a module. + + The source is only used for the -c command line option; in that + case path is None. Otherwise source is None and path isn't. + """ + + manager = None # type: BuildManager + order_counter = 0 # Class variable + order = None # type: int # Order in which modules were encountered + id = None # type: str # Fully qualified module name + path = None # type: Optional[str] # Path to module source + xpath = None # type: str # Path or '' + source = None # type: Optional[str] # Module source code + meta = None # type: Optional[CacheMeta] + data = None # type: Optional[str] + tree = None # type: Optional[MypyFile] + dependencies = None # type: List[str] + suppressed = None # type: List[str] # Suppressed/missing dependencies + priorities = None # type: Dict[str, int] + + # Map each dependency to the line number where it is first imported + dep_line_map = None # type: Dict[str, int] + + # Parent package, its parent, etc. + ancestors = None # type: Optional[List[str]] + + # List of (path, line number) tuples giving context for import + import_context = None # type: List[Tuple[str, int]] + + # The State from which this module was imported, if any + caller_state = None # type: Optional[State] + + # If caller_state is set, the line number in the caller where the import occurred + caller_line = 0 + + def __init__(self, + id: Optional[str], + path: Optional[str], + source: Optional[str], + manager: BuildManager, + caller_state: 'State' = None, + caller_line: int = 0, + ancestor_for: 'State' = None, + ) -> None: + assert id or path or source is not None, "Neither id, path nor source given" + self.manager = manager + State.order_counter += 1 + self.order = State.order_counter + self.caller_state = caller_state + self.caller_line = caller_line + if caller_state: + self.import_context = caller_state.import_context[:] + self.import_context.append((caller_state.xpath, caller_line)) + else: + self.import_context = [] + self.id = id or '__main__' + if not path and source is None: + file_id = id + if id == 'builtins' and manager.options.python_version[0] == 2: + # The __builtin__ module is called internally by mypy + # 'builtins' in Python 2 mode (similar to Python 3), + # but the stub file is __builtin__.pyi. The reason is + # that a lot of code hard-codes 'builtins.x' and it's + # easier to work it around like this. It also means + # that the implementation can mostly ignore the + # difference and just assume 'builtins' everywhere, + # which simplifies code. + file_id = '__builtin__' + path = find_module(file_id, manager.lib_path) + if path: + # In silent mode, don't import .py files, except from stubs. + if (manager.options.silent_imports and + path.endswith('.py') and (caller_state or ancestor_for)): + # (Never silence builtins, even if it's a .py file; + # this can happen in tests!) + if (id != 'builtins' and + not ((caller_state and + caller_state.tree and + caller_state.tree.is_stub))): + if manager.options.almost_silent: + if ancestor_for: + self.skipping_ancestor(id, path, ancestor_for) + else: + self.skipping_module(id, path) + path = None + manager.missing_modules.add(id) + raise ModuleNotFound + else: + # Could not find a module. Typically the reason is a + # misspelled module name, missing stub, module not in + # search path or the module has not been installed. + if caller_state: + suppress_message = ((manager.options.silent_imports and + not manager.options.almost_silent) or + (caller_state.tree is not None and + 'import' in caller_state.tree.weak_opts)) + if not suppress_message: + save_import_context = manager.errors.import_context() + manager.errors.set_import_context(caller_state.import_context) + manager.module_not_found(caller_state.xpath, caller_line, id) + manager.errors.set_import_context(save_import_context) + manager.missing_modules.add(id) + raise ModuleNotFound + else: + # If we can't find a root source it's always fatal. + # TODO: This might hide non-fatal errors from + # root sources processed earlier. + raise CompileError(["mypy: can't find module '%s'" % id]) + self.path = path + self.xpath = path or '' + self.source = source + if path and source is None and manager.options.incremental: + self.meta = find_cache_meta(self.id, self.path, manager) + # TODO: Get mtime if not cached. + self.add_ancestors() + if self.meta: + # Make copies, since we may modify these and want to + # compare them to the originals later. + self.dependencies = list(self.meta.dependencies) + self.suppressed = list(self.meta.suppressed) + assert len(self.meta.dependencies) == len(self.meta.dep_prios) + self.priorities = {id: pri + for id, pri in zip(self.meta.dependencies, self.meta.dep_prios)} + self.dep_line_map = {} + else: + # Parse the file (and then some) to get the dependencies. + self.parse_file() + self.suppressed = [] + + def skipping_ancestor(self, id: str, path: str, ancestor_for: 'State') -> None: + # TODO: Read the path (the __init__.py file) and return + # immediately if it's empty or only contains comments. + # But beware, some package may be the ancestor of many modules, + # so we'd need to cache the decision. + manager = self.manager + manager.errors.set_import_context([]) + manager.errors.set_file(ancestor_for.xpath) + manager.errors.report(-1, "Ancestor package '%s' silently ignored" % (id,), + severity='note', only_once=True) + manager.errors.report(-1, "(Using --silent-imports, submodule passed on command line)", + severity='note', only_once=True) + manager.errors.report(-1, "(This note brought to you by --almost-silent)", + severity='note', only_once=True) + + def skipping_module(self, id: str, path: str) -> None: + assert self.caller_state, (id, path) + manager = self.manager + save_import_context = manager.errors.import_context() + manager.errors.set_import_context(self.caller_state.import_context) + manager.errors.set_file(self.caller_state.xpath) + line = self.caller_line + manager.errors.report(line, "Import of '%s' silently ignored" % (id,), + severity='note') + manager.errors.report(line, "(Using --silent-imports, module not passed on command line)", + severity='note', only_once=True) + manager.errors.report(line, "(This note courtesy of --almost-silent)", + severity='note', only_once=True) + manager.errors.set_import_context(save_import_context) + + def add_ancestors(self) -> None: + # All parent packages are new ancestors. + ancestors = [] + parent = self.id + while '.' in parent: + parent, _ = parent.rsplit('.', 1) + ancestors.append(parent) + self.ancestors = ancestors + + def is_fresh(self) -> bool: + """Return whether the cache data for this file is fresh.""" + # NOTE: self.dependencies may differ from + # self.meta.dependencies when a dependency is dropped due to + # suppression by --silent-imports. However when a suppressed + # dependency is added back we find out later in the process. + return self.meta is not None and self.dependencies == self.meta.dependencies + + def mark_stale(self) -> None: + """Throw away the cache data for this file, marking it as stale.""" + self.meta = None + + def check_blockers(self) -> None: + """Raise CompileError if a blocking error is detected.""" + if self.manager.errors.is_blockers(): + self.manager.log("Bailing due to blocking errors") + self.manager.errors.raise_error() + + @contextlib.contextmanager + def wrap_context(self) -> Iterator[None]: + save_import_context = self.manager.errors.import_context() + self.manager.errors.set_import_context(self.import_context) + try: + yield + except CompileError: + raise + except Exception as err: + report_internal_error(err, self.path, 0) + self.manager.errors.set_import_context(save_import_context) + self.check_blockers() + + # Methods for processing cached modules. + + def load_tree(self) -> None: + with open(self.meta.data_json) as f: + data = json.load(f) + # TODO: Assert data file wasn't changed. + self.tree = MypyFile.deserialize(data) + self.manager.modules[self.id] = self.tree + + def fix_cross_refs(self) -> None: + fixup_module_pass_one(self.tree, self.manager.modules) + + def calculate_mros(self) -> None: + fixup_module_pass_two(self.tree, self.manager.modules) + + # Methods for processing modules from source code. + + def parse_file(self) -> None: + if self.tree is not None: + # The file was already parsed (in __init__()). + return + + manager = self.manager + modules = manager.modules + manager.log("Parsing %s (%s)" % (self.xpath, self.id)) + + with self.wrap_context(): + source = self.source + self.source = None # We won't need it again. + if self.path and source is None: + try: + source = read_with_python_encoding(self.path, manager.options.python_version) + except IOError as ioerr: + raise CompileError([ + "mypy: can't read file '{}': {}".format(self.path, ioerr.strerror)]) + except (UnicodeDecodeError, DecodeError) as decodeerr: + raise CompileError([ + "mypy: can't decode file '{}': {}".format(self.path, str(decodeerr))]) + self.tree = manager.parse_file(self.id, self.xpath, source) + + modules[self.id] = self.tree + + # Do the first pass of semantic analysis: add top-level + # definitions in the file to the symbol table. We must do + # this before processing imports, since this may mark some + # import statements as unreachable. + first = FirstPass(manager.semantic_analyzer) + first.analyze(self.tree, self.xpath, self.id) + + # Initialize module symbol table, which was populated by the + # semantic analyzer. + # TODO: Why can't FirstPass .analyze() do this? + self.tree.names = manager.semantic_analyzer.globals + + # Compute (direct) dependencies. + # Add all direct imports (this is why we needed the first pass). + # Also keep track of each dependency's source line. + dependencies = [] + suppressed = [] + priorities = {} # type: Dict[str, int] # id -> priority + dep_line_map = {} # type: Dict[str, int] # id -> line + for pri, id, line in manager.all_imported_modules_in_file(self.tree): + priorities[id] = min(pri, priorities.get(id, PRI_ALL)) + if id == self.id: + continue + # Omit missing modules, as otherwise we could not type-check + # programs with missing modules. + if id in manager.missing_modules: + if id not in dep_line_map: + suppressed.append(id) + dep_line_map[id] = line + continue + if id == '': + # Must be from a relative import. + manager.errors.set_file(self.xpath) + manager.errors.report(line, "No parent module -- cannot perform relative import", + blocker=True) + continue + if id not in dep_line_map: + dependencies.append(id) + dep_line_map[id] = line + # Every module implicitly depends on builtins. + if self.id != 'builtins' and 'builtins' not in dep_line_map: + dependencies.append('builtins') + + # If self.dependencies is already set, it was read from the + # cache, but for some reason we're re-parsing the file. + # NOTE: What to do about race conditions (like editing the + # file while mypy runs)? A previous version of this code + # explicitly checked for this, but ran afoul of other reasons + # for differences (e.g. --silent-imports). + self.dependencies = dependencies + self.suppressed = suppressed + self.priorities = priorities + self.dep_line_map = dep_line_map + self.check_blockers() + + def patch_parent(self) -> None: + # Include module in the symbol table of the enclosing package. + if '.' not in self.id: + return + manager = self.manager + modules = manager.modules + parent, child = self.id.rsplit('.', 1) + if parent in modules: + manager.trace("Added %s.%s" % (parent, child)) + modules[parent].names[child] = SymbolTableNode(MODULE_REF, self.tree, parent) + else: + manager.log("Hm... couldn't add %s.%s" % (parent, child)) + + def semantic_analysis(self) -> None: + with self.wrap_context(): + self.manager.semantic_analyzer.visit_file(self.tree, self.xpath) + + def semantic_analysis_pass_three(self) -> None: + with self.wrap_context(): + self.manager.semantic_analyzer_pass3.visit_file(self.tree, self.xpath) + if self.manager.options.dump_type_stats: + dump_type_stats(self.tree, self.xpath) + + def type_check(self) -> None: + manager = self.manager + if manager.options.semantic_analysis_only: + return + with self.wrap_context(): + manager.type_checker.visit_file(self.tree, self.xpath) + if manager.options.dump_inference_stats: + dump_type_stats(self.tree, self.xpath, inferred=True, + typemap=manager.type_checker.type_map) + manager.report_file(self.tree) + + def write_cache(self) -> None: + if self.path and self.manager.options.incremental and not self.manager.errors.is_errors(): + dep_prios = [self.priorities.get(dep, PRI_HIGH) for dep in self.dependencies] + write_cache(self.id, self.path, self.tree, + list(self.dependencies), list(self.suppressed), + dep_prios, + self.manager) + + +Graph = Dict[str, State] + + +def dispatch(sources: List[BuildSource], manager: BuildManager) -> None: + manager.log("Using new dependency manager") + graph = load_graph(sources, manager) + manager.log("Loaded graph with %d nodes" % len(graph)) + process_graph(graph, manager) + if manager.options.warn_unused_ignores: + manager.errors.generate_unused_ignore_notes() + + +def load_graph(sources: List[BuildSource], manager: BuildManager) -> Graph: + """Given some source files, load the full dependency graph.""" + graph = {} # type: Graph + # The deque is used to implement breadth-first traversal. + # TODO: Consider whether to go depth-first instead. This may + # affect the order in which we process files within import cycles. + new = collections.deque() # type: collections.deque[State] + # Seed the graph with the initial root sources. + for bs in sources: + try: + st = State(id=bs.module, path=bs.path, source=bs.text, manager=manager) + except ModuleNotFound: + continue + if st.id in graph: + manager.errors.set_file(st.xpath) + manager.errors.report(1, "Duplicate module named '%s'" % st.id) + manager.errors.raise_error() + graph[st.id] = st + new.append(st) + # Collect dependencies. We go breadth-first. + while new: + st = new.popleft() + for dep in st.ancestors + st.dependencies: + if dep not in graph: + try: + if dep in st.ancestors: + # TODO: Why not 'if dep not in st.dependencies' ? + # Ancestors don't have import context. + newst = State(id=dep, path=None, source=None, manager=manager, + ancestor_for=st) + else: + newst = State(id=dep, path=None, source=None, manager=manager, + caller_state=st, caller_line=st.dep_line_map.get(dep, 1)) + except ModuleNotFound: + if dep in st.dependencies: + st.dependencies.remove(dep) + st.suppressed.append(dep) + else: + assert newst.id not in graph, newst.id + graph[newst.id] = newst + new.append(newst) + return graph + + +def process_graph(graph: Graph, manager: BuildManager) -> None: + """Process everything in dependency order.""" + sccs = sorted_components(graph) + manager.log("Found %d SCCs; largest has %d nodes" % + (len(sccs), max(len(scc) for scc in sccs))) + # We're processing SCCs from leaves (those without further + # dependencies) to roots (those from which everything else can be + # reached). + for ascc in sccs: + # Order the SCC's nodes using a heuristic. + # Note that ascc is a set, and scc is a list. + scc = order_ascc(graph, ascc) + # If builtins is in the list, move it last. (This is a bit of + # a hack, but it's necessary because the builtins module is + # part of a small cycle involving at least {builtins, abc, + # typing}. Of these, builtins must be processed last or else + # some builtin objects will be incompletely processed.) + if 'builtins' in ascc: + scc.remove('builtins') + scc.append('builtins') + if manager.options.verbosity >= 2: + for id in scc: + manager.trace("Priorities for %s:" % id, + " ".join("%s:%d" % (x, graph[id].priorities[x]) + for x in graph[id].dependencies + if x in ascc and x in graph[id].priorities)) + # Because the SCCs are presented in topological sort order, we + # don't need to look at dependencies recursively for staleness + # -- the immediate dependencies are sufficient. + stale_scc = {id for id in scc if not graph[id].is_fresh()} + fresh = not stale_scc + deps = set() + for id in scc: + deps.update(graph[id].dependencies) + deps -= ascc + stale_deps = {id for id in deps if not graph[id].is_fresh()} + fresh = fresh and not stale_deps + undeps = set() + if fresh: + # Check if any dependencies that were suppressed according + # to the cache have heen added back in this run. + # NOTE: Newly suppressed dependencies are handled by is_fresh(). + for id in scc: + undeps.update(graph[id].suppressed) + undeps &= graph.keys() + if undeps: + fresh = False + if fresh: + # All cache files are fresh. Check that no dependency's + # cache file is newer than any scc node's cache file. + oldest_in_scc = min(graph[id].meta.data_mtime for id in scc) + newest_in_deps = 0 if not deps else max(graph[dep].meta.data_mtime for dep in deps) + if manager.options.verbosity >= 3: # Dump all mtimes for extreme debugging. + all_ids = sorted(ascc | deps, key=lambda id: graph[id].meta.data_mtime) + for id in all_ids: + if id in scc: + if graph[id].meta.data_mtime < newest_in_deps: + key = "*id:" + else: + key = "id:" + else: + if graph[id].meta.data_mtime > oldest_in_scc: + key = "+dep:" + else: + key = "dep:" + manager.trace(" %5s %.0f %s" % (key, graph[id].meta.data_mtime, id)) + # If equal, give the benefit of the doubt, due to 1-sec time granularity + # (on some platforms). + if oldest_in_scc < newest_in_deps: + fresh = False + fresh_msg = "out of date by %.0f seconds" % (newest_in_deps - oldest_in_scc) + else: + fresh_msg = "fresh" + elif undeps: + fresh_msg = "stale due to changed suppression (%s)" % " ".join(sorted(undeps)) + elif stale_scc: + fresh_msg = "inherently stale (%s)" % " ".join(sorted(stale_scc)) + if stale_deps: + fresh_msg += " with stale deps (%s)" % " ".join(sorted(stale_deps)) + else: + fresh_msg = "stale due to deps (%s)" % " ".join(sorted(stale_deps)) + if len(scc) == 1: + manager.log("Processing SCC singleton (%s) as %s" % (" ".join(scc), fresh_msg)) + else: + manager.log("Processing SCC of size %d (%s) as %s" % + (len(scc), " ".join(scc), fresh_msg)) + if fresh: + process_fresh_scc(graph, scc) + else: + process_stale_scc(graph, scc) + + +def order_ascc(graph: Graph, ascc: AbstractSet[str], pri_max: int = PRI_ALL) -> List[str]: + """Come up with the ideal processing order within an SCC. + + Using the priorities assigned by all_imported_modules_in_file(), + try to reduce the cycle to a DAG, by omitting arcs representing + dependencies of lower priority. + + In the simplest case, if we have A <--> B where A has a top-level + "import B" (medium priority) but B only has the reverse "import A" + inside a function (low priority), we turn the cycle into a DAG by + dropping the B --> A arc, which leaves only A --> B. + + If all arcs have the same priority, we fall back to sorting by + reverse global order (the order in which modules were first + encountered). + + The algorithm is recursive, as follows: when as arcs of different + priorities are present, drop all arcs of the lowest priority, + identify SCCs in the resulting graph, and apply the algorithm to + each SCC thus found. The recursion is bounded because at each + recursion the spread in priorities is (at least) one less. + + In practice there are only a few priority levels (currently + N=3) and in the worst case we just carry out the same algorithm + for finding SCCs N times. Thus the complexity is no worse than + the complexity of the original SCC-finding algorithm -- see + strongly_connected_components() below for a reference. + """ + if len(ascc) == 1: + return [s for s in ascc] + pri_spread = set() + for id in ascc: + state = graph[id] + for dep in state.dependencies: + if dep in ascc: + pri = state.priorities.get(dep, PRI_HIGH) + if pri < pri_max: + pri_spread.add(pri) + if len(pri_spread) == 1: + # Filtered dependencies are uniform -- order by global order. + return sorted(ascc, key=lambda id: -graph[id].order) + pri_max = max(pri_spread) + sccs = sorted_components(graph, ascc, pri_max) + # The recursion is bounded by the len(pri_spread) check above. + return [s for ss in sccs for s in order_ascc(graph, ss, pri_max)] + + +def process_fresh_scc(graph: Graph, scc: List[str]) -> None: + """Process the modules in one SCC from their cached data.""" + for id in scc: + graph[id].load_tree() + for id in scc: + graph[id].patch_parent() + for id in scc: + graph[id].fix_cross_refs() + for id in scc: + graph[id].calculate_mros() + + +def process_stale_scc(graph: Graph, scc: List[str]) -> None: + """Process the modules in one SCC from source code.""" + for id in scc: + graph[id].mark_stale() + for id in scc: + # We may already have parsed the module, or not. + # If the former, parse_file() is a no-op. + graph[id].parse_file() + for id in scc: + graph[id].patch_parent() + for id in scc: + graph[id].semantic_analysis() + for id in scc: + graph[id].semantic_analysis_pass_three() + for id in scc: + graph[id].type_check() + graph[id].write_cache() + + +def sorted_components(graph: Graph, + vertices: Optional[AbstractSet[str]] = None, + pri_max: int = PRI_ALL) -> List[AbstractSet[str]]: + """Return the graph's SCCs, topologically sorted by dependencies. + + The sort order is from leaves (nodes without dependencies) to + roots (nodes on which no other nodes depend). + + This works for a subset of the full dependency graph too; + dependencies that aren't present in graph.keys() are ignored. + """ + # Compute SCCs. + if vertices is None: + vertices = set(graph) + edges = {id: deps_filtered(graph, vertices, id, pri_max) for id in vertices} + sccs = list(strongly_connected_components(vertices, edges)) + # Topsort. + sccsmap = {id: frozenset(scc) for scc in sccs for id in scc} + data = {} # type: Dict[AbstractSet[str], Set[AbstractSet[str]]] + for scc in sccs: + deps = set() # type: Set[AbstractSet[str]] + for id in scc: + deps.update(sccsmap[x] for x in deps_filtered(graph, vertices, id, pri_max)) + data[frozenset(scc)] = deps + res = [] + for ready in topsort(data): + # Sort the sets in ready by reversed smallest State.order. Examples: + # + # - If ready is [{x}, {y}], x.order == 1, y.order == 2, we get + # [{y}, {x}]. + # + # - If ready is [{a, b}, {c, d}], a.order == 1, b.order == 3, + # c.order == 2, d.order == 4, the sort keys become [1, 2] + # and the result is [{c, d}, {a, b}]. + res.extend(sorted(ready, + key=lambda scc: -min(graph[id].order for id in scc))) + return res + + +def deps_filtered(graph: Graph, vertices: AbstractSet[str], id: str, pri_max: int) -> List[str]: + """Filter dependencies for id with pri < pri_max.""" + if id not in vertices: + return [] + state = graph[id] + return [dep + for dep in state.dependencies + if dep in vertices and state.priorities.get(dep, PRI_HIGH) < pri_max] + + +def strongly_connected_components(vertices: AbstractSet[str], + edges: Dict[str, List[str]]) -> Iterator[Set[str]]: + """Compute Strongly Connected Components of a directed graph. + + Args: + vertices: the labels for the vertices + edges: for each vertex, gives the target vertices of its outgoing edges + + Returns: + An iterator yielding strongly connected components, each + represented as a set of vertices. Each input vertex will occur + exactly once; vertices not part of a SCC are returned as + singleton sets. + + From http://code.activestate.com/recipes/578507/. + """ + identified = set() # type: Set[str] + stack = [] # type: List[str] + index = {} # type: Dict[str, int] + boundaries = [] # type: List[int] + + def dfs(v: str) -> Iterator[Set[str]]: + index[v] = len(stack) + stack.append(v) + boundaries.append(index[v]) + + for w in edges[v]: + if w not in index: + # For Python >= 3.3, replace with "yield from dfs(w)" + for scc in dfs(w): + yield scc + elif w not in identified: + while index[w] < boundaries[-1]: + boundaries.pop() + + if boundaries[-1] == index[v]: + boundaries.pop() + scc = set(stack[index[v]:]) + del stack[index[v]:] + identified.update(scc) + yield scc + + for v in vertices: + if v not in index: + # For Python >= 3.3, replace with "yield from dfs(v)" + for scc in dfs(v): + yield scc + + +def topsort(data: Dict[AbstractSet[str], + Set[AbstractSet[str]]]) -> Iterable[Set[AbstractSet[str]]]: + """Topological sort. + + Args: + data: A map from SCCs (represented as frozen sets of strings) to + sets of SCCs, its dependencies. NOTE: This data structure + is modified in place -- for normalization purposes, + self-dependencies are removed and entries representing + orphans are added. + + Returns: + An iterator yielding sets of SCCs that have an equivalent + ordering. NOTE: The algorithm doesn't care about the internal + structure of SCCs. + + Example: + Suppose the input has the following structure: + + {A: {B, C}, B: {D}, C: {D}} + + This is normalized to: + + {A: {B, C}, B: {D}, C: {D}, D: {}} + + The algorithm will yield the following values: + + {D} + {B, C} + {A} + + From http://code.activestate.com/recipes/577413/. + """ + # TODO: Use a faster algorithm? + for k, v in data.items(): + v.discard(k) # Ignore self dependencies. + for item in set.union(*data.values()) - set(data.keys()): + data[item] = set() + while True: + ready = {item for item, dep in data.items() if not dep} + if not ready: + break + yield ready + data = {item: (dep - ready) + for item, dep in data.items() + if item not in ready} + assert not data, "A cyclic dependency exists amongst %r" % data diff --git a/mypy/checker.py b/mypy/checker.py index 7063758096dfa..62f4c250ea272 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -1,2459 +1,2459 @@ -"""Mypy type checker.""" - -import itertools -import contextlib -import os -import os.path - -from typing import ( - Any, Dict, Set, List, cast, Tuple, TypeVar, Union, Optional, NamedTuple -) - -from mypy.errors import Errors, report_internal_error -from mypy.nodes import ( - SymbolTable, Node, MypyFile, Var, - OverloadedFuncDef, FuncDef, FuncItem, FuncBase, TypeInfo, - ClassDef, GDEF, Block, AssignmentStmt, NameExpr, MemberExpr, IndexExpr, - TupleExpr, ListExpr, ExpressionStmt, ReturnStmt, IfStmt, - WhileStmt, OperatorAssignmentStmt, WithStmt, AssertStmt, - RaiseStmt, TryStmt, ForStmt, DelStmt, CallExpr, IntExpr, StrExpr, - BytesExpr, UnicodeExpr, FloatExpr, OpExpr, UnaryExpr, CastExpr, RevealTypeExpr, SuperExpr, - TypeApplication, DictExpr, SliceExpr, FuncExpr, TempNode, SymbolTableNode, - Context, ListComprehension, ConditionalExpr, GeneratorExpr, - Decorator, SetExpr, TypeVarExpr, PrintStmt, - LITERAL_TYPE, BreakStmt, ContinueStmt, ComparisonExpr, StarExpr, - YieldFromExpr, NamedTupleExpr, SetComprehension, - DictionaryComprehension, ComplexExpr, EllipsisExpr, TypeAliasExpr, - RefExpr, YieldExpr, BackquoteExpr, ImportFrom, ImportAll, ImportBase, - CONTRAVARIANT, COVARIANT -) -from mypy.nodes import function_type, method_type, method_type_with_fallback -from mypy import nodes -from mypy.types import ( - Type, AnyType, CallableType, Void, FunctionLike, Overloaded, TupleType, - Instance, NoneTyp, ErrorType, strip_type, - UnionType, TypeVarId, TypeVarType, PartialType, DeletedType, UninhabitedType -) -from mypy.sametypes import is_same_type -from mypy.messages import MessageBuilder -import mypy.checkexpr -from mypy.checkmember import map_type_from_supertype -from mypy import defaults -from mypy import messages -from mypy.subtypes import ( - is_subtype, is_equivalent, is_proper_subtype, - is_more_precise, restrict_subtype_away -) -from mypy.maptype import map_instance_to_supertype -from mypy.semanal import self_type, set_callable_name, refers_to_fullname -from mypy.erasetype import erase_typevars -from mypy.expandtype import expand_type_by_instance, expand_type -from mypy.visitor import NodeVisitor -from mypy.join import join_types -from mypy.treetransform import TransformVisitor -from mypy.meet import meet_simple, nearest_builtin_ancestor, is_overlapping_types -from mypy.binder import ConditionalTypeBinder -from mypy.options import Options - -from mypy import experiments - - -T = TypeVar('T') - - -# A node which is postponed to be type checked during the next pass. -DeferredNode = NamedTuple( - 'DeferredNode', - [ - ('node', Node), - ('context_type_name', Optional[str]), # Name of the surrounding class (for error messages) - ]) - - -class TypeChecker(NodeVisitor[Type]): - """Mypy type checker. - - Type check mypy source files that have been semantically analyzed. - """ - - # Are we type checking a stub? - is_stub = False - # Error message reporter - errors = None # type: Errors - # Utility for generating messages - msg = None # type: MessageBuilder - # Types of type checked nodes - type_map = None # type: Dict[Node, Type] - - # Helper for managing conditional types - binder = None # type: ConditionalTypeBinder - # Helper for type checking expressions - expr_checker = None # type: mypy.checkexpr.ExpressionChecker - - # Stack of function return types - return_types = None # type: List[Type] - # Type context for type inference - type_context = None # type: List[Type] - # Flags; true for dynamically typed functions - dynamic_funcs = None # type: List[bool] - # Stack of functions being type checked - function_stack = None # type: List[FuncItem] - # Do weak type checking in this file - weak_opts = set() # type: Set[str] - # Stack of collections of variables with partial types - partial_types = None # type: List[Dict[Var, Context]] - globals = None # type: SymbolTable - modules = None # type: Dict[str, MypyFile] - # Nodes that couldn't be checked because some types weren't available. We'll run - # another pass and try these again. - deferred_nodes = None # type: List[DeferredNode] - # Type checking pass number (0 = first pass) - pass_num = 0 - # Have we deferred the current function? If yes, don't infer additional - # types during this pass within the function. - current_node_deferred = False - is_typeshed_stub = False - options = None # type: Options - - def __init__(self, errors: Errors, modules: Dict[str, MypyFile], options: Options) -> None: - """Construct a type checker. - - Use errors to report type check errors. - """ - self.errors = errors - self.modules = modules - self.options = options - self.msg = MessageBuilder(errors, modules) - self.type_map = {} - self.binder = ConditionalTypeBinder() - self.expr_checker = mypy.checkexpr.ExpressionChecker(self, self.msg) - self.return_types = [] - self.type_context = [] - self.dynamic_funcs = [] - self.function_stack = [] - self.weak_opts = set() # type: Set[str] - self.partial_types = [] - self.deferred_nodes = [] - self.pass_num = 0 - self.current_node_deferred = False - - def visit_file(self, file_node: MypyFile, path: str) -> None: - """Type check a mypy file with the given path.""" - self.pass_num = 0 - self.is_stub = file_node.is_stub - self.errors.set_file(path) - self.globals = file_node.names - self.weak_opts = file_node.weak_opts - self.enter_partial_types() - # gross, but no other clear way to tell - self.is_typeshed_stub = self.is_stub and 'typeshed' in os.path.normpath(path).split(os.sep) - - for d in file_node.defs: - self.accept(d) - - self.leave_partial_types() - - if self.deferred_nodes: - self.check_second_pass() - - self.current_node_deferred = False - - all_ = self.globals.get('__all__') - if all_ is not None and all_.type is not None: - seq_str = self.named_generic_type('typing.Sequence', - [self.named_type('builtins.str')]) - if not is_subtype(all_.type, seq_str): - str_seq_s, all_s = self.msg.format_distinctly(seq_str, all_.type) - self.fail(messages.ALL_MUST_BE_SEQ_STR.format(str_seq_s, all_s), - all_.node) - - def check_second_pass(self): - """Run second pass of type checking which goes through deferred nodes.""" - self.pass_num = 1 - for node, type_name in self.deferred_nodes: - if type_name: - self.errors.push_type(type_name) - self.accept(node) - if type_name: - self.errors.pop_type() - self.deferred_nodes = [] - - def handle_cannot_determine_type(self, name: str, context: Context) -> None: - if self.pass_num == 0 and self.function_stack: - # Don't report an error yet. Just defer. - node = self.function_stack[-1] - if self.errors.type_name: - type_name = self.errors.type_name[-1] - else: - type_name = None - self.deferred_nodes.append(DeferredNode(node, type_name)) - # Set a marker so that we won't infer additional types in this - # function. Any inferred types could be bogus, because there's at - # least one type that we don't know. - self.current_node_deferred = True - else: - self.msg.cannot_determine_type(name, context) - - def accept(self, node: Node, type_context: Type = None) -> Type: - """Type check a node in the given type context.""" - self.type_context.append(type_context) - try: - typ = node.accept(self) - except Exception as err: - report_internal_error(err, self.errors.file, node.line) - self.type_context.pop() - self.store_type(node, typ) - if self.typing_mode_none(): - return AnyType() - else: - return typ - - def accept_loop(self, body: Node, else_body: Node = None) -> Type: - """Repeatedly type check a loop body until the frame doesn't change. - - Then check the else_body. - """ - # The outer frame accumulates the results of all iterations - with self.binder.frame_context(1) as outer_frame: - self.binder.push_loop_frame() - while True: - with self.binder.frame_context(1): - # We may skip each iteration - self.binder.options_on_return[-1].append(outer_frame) - self.accept(body) - if not self.binder.last_pop_changed: - break - self.binder.pop_loop_frame() - if else_body: - self.accept(else_body) - - # - # Definitions - # - - def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> Type: - num_abstract = 0 - if defn.is_property: - # HACK: Infer the type of the property. - self.visit_decorator(defn.items[0]) - for fdef in defn.items: - self.check_func_item(fdef.func, name=fdef.func.name()) - if fdef.func.is_abstract: - num_abstract += 1 - if num_abstract not in (0, len(defn.items)): - self.fail(messages.INCONSISTENT_ABSTRACT_OVERLOAD, defn) - if defn.info: - self.check_method_override(defn) - self.check_inplace_operator_method(defn) - self.check_overlapping_overloads(defn) - - def check_overlapping_overloads(self, defn: OverloadedFuncDef) -> None: - for i, item in enumerate(defn.items): - for j, item2 in enumerate(defn.items[i + 1:]): - # TODO overloads involving decorators - sig1 = self.function_type(item.func) - sig2 = self.function_type(item2.func) - if is_unsafe_overlapping_signatures(sig1, sig2): - self.msg.overloaded_signatures_overlap(i + 1, i + j + 2, - item.func) - - def is_generator_return_type(self, typ: Type) -> bool: - return is_subtype(self.named_generic_type('typing.Generator', - [AnyType(), AnyType(), AnyType()]), - typ) - - def get_generator_yield_type(self, return_type: Type) -> Type: - if isinstance(return_type, AnyType): - return AnyType() - elif not self.is_generator_return_type(return_type): - # If the function doesn't have a proper Generator (or superclass) return type, anything - # is permissible. - return AnyType() - elif not isinstance(return_type, Instance): - # Same as above, but written as a separate branch so the typechecker can understand. - return AnyType() - elif return_type.args: - return return_type.args[0] - else: - # If the function's declared supertype of Generator has no type - # parameters (i.e. is `object`), then the yielded values can't - # be accessed so any type is acceptable. - return AnyType() - - def get_generator_receive_type(self, return_type: Type) -> Type: - if isinstance(return_type, AnyType): - return AnyType() - elif not self.is_generator_return_type(return_type): - # If the function doesn't have a proper Generator (or superclass) return type, anything - # is permissible. - return AnyType() - elif not isinstance(return_type, Instance): - # Same as above, but written as a separate branch so the typechecker can understand. - return AnyType() - elif return_type.type.fullname() == 'typing.Generator': - # Generator is the only type which specifies the type of values it can receive. - return return_type.args[1] - else: - # `return_type` is a supertype of Generator, so callers won't be able to send it - # values. - return Void() - - def get_generator_return_type(self, return_type: Type) -> Type: - if isinstance(return_type, AnyType): - return AnyType() - elif not self.is_generator_return_type(return_type): - # If the function doesn't have a proper Generator (or superclass) return type, anything - # is permissible. - return AnyType() - elif not isinstance(return_type, Instance): - # Same as above, but written as a separate branch so the typechecker can understand. - return AnyType() - elif return_type.type.fullname() == 'typing.Generator': - # Generator is the only type which specifies the type of values it returns into - # `yield from` expressions. - return return_type.args[2] - else: - # `return_type` is supertype of Generator, so callers won't be able to see the return - # type when used in a `yield from` expression. - return AnyType() - - def visit_func_def(self, defn: FuncDef) -> Type: - """Type check a function definition.""" - self.check_func_item(defn, name=defn.name()) - if defn.info: - if not defn.is_dynamic(): - self.check_method_override(defn) - self.check_inplace_operator_method(defn) - if defn.original_def: - # Override previous definition. - new_type = self.function_type(defn) - if isinstance(defn.original_def, FuncDef): - # Function definition overrides function definition. - if not is_same_type(new_type, self.function_type(defn.original_def)): - self.msg.incompatible_conditional_function_def(defn) - else: - # Function definition overrides a variable initialized via assignment. - orig_type = defn.original_def.type - if orig_type is None: - # XXX This can be None, as happens in - # test_testcheck_TypeCheckSuite.testRedefinedFunctionInTryWithElse - self.msg.note("Internal mypy error checking function redefinition.", defn) - return None - if isinstance(orig_type, PartialType): - if orig_type.type is None: - # Ah this is a partial type. Give it the type of the function. - var = defn.original_def - partial_types = self.find_partial_types(var) - if partial_types is not None: - var.type = new_type - del partial_types[var] - else: - # Trying to redefine something like partial empty list as function. - self.fail(messages.INCOMPATIBLE_REDEFINITION, defn) - else: - # TODO: Update conditional type binder. - self.check_subtype(new_type, orig_type, defn, - messages.INCOMPATIBLE_REDEFINITION, - 'redefinition with type', - 'original type') - - def check_func_item(self, defn: FuncItem, - type_override: CallableType = None, - name: str = None) -> Type: - """Type check a function. - - If type_override is provided, use it as the function type. - """ - # We may be checking a function definition or an anonymous function. In - # the first case, set up another reference with the precise type. - fdef = None # type: FuncDef - if isinstance(defn, FuncDef): - fdef = defn - - self.function_stack.append(defn) - self.dynamic_funcs.append(defn.is_dynamic() and not type_override) - - if fdef: - self.errors.push_function(fdef.name()) - - self.enter_partial_types() - - typ = self.function_type(defn) - if type_override: - typ = type_override - if isinstance(typ, CallableType): - self.check_func_def(defn, typ, name) - else: - raise RuntimeError('Not supported') - - self.leave_partial_types() - - if fdef: - self.errors.pop_function() - - self.dynamic_funcs.pop() - self.function_stack.pop() - self.current_node_deferred = False - - def check_func_def(self, defn: FuncItem, typ: CallableType, name: str) -> None: - """Type check a function definition.""" - # Expand type variables with value restrictions to ordinary types. - for item, typ in self.expand_typevars(defn, typ): - old_binder = self.binder - self.binder = ConditionalTypeBinder() - with self.binder.frame_context(): - defn.expanded.append(item) - - # We may be checking a function definition or an anonymous - # function. In the first case, set up another reference with the - # precise type. - if isinstance(item, FuncDef): - fdef = item - else: - fdef = None - - if fdef: - # Check if __init__ has an invalid, non-None return type. - if (fdef.info and fdef.name() == '__init__' and - not isinstance(typ.ret_type, Void) and - not self.dynamic_funcs[-1]): - self.fail(messages.INIT_MUST_HAVE_NONE_RETURN_TYPE, - item.type) - - show_untyped = not self.is_typeshed_stub or self.options.warn_incomplete_stub - if self.options.disallow_untyped_defs and show_untyped: - # Check for functions with unspecified/not fully specified types. - def is_implicit_any(t: Type) -> bool: - return isinstance(t, AnyType) and t.implicit - - if fdef.type is None: - self.fail(messages.FUNCTION_TYPE_EXPECTED, fdef) - elif isinstance(fdef.type, CallableType): - if is_implicit_any(fdef.type.ret_type): - self.fail(messages.RETURN_TYPE_EXPECTED, fdef) - if any(is_implicit_any(t) for t in fdef.type.arg_types): - self.fail(messages.ARGUMENT_TYPE_EXPECTED, fdef) - - if name in nodes.reverse_op_method_set: - self.check_reverse_op_method(item, typ, name) - elif name == '__getattr__': - self.check_getattr_method(typ, defn) - - # Refuse contravariant return type variable - if isinstance(typ.ret_type, TypeVarType): - if typ.ret_type.variance == CONTRAVARIANT: - self.fail(messages.RETURN_TYPE_CANNOT_BE_CONTRAVARIANT, - typ.ret_type) - - # Check that Generator functions have the appropriate return type. - if defn.is_generator: - if not self.is_generator_return_type(typ.ret_type): - self.fail(messages.INVALID_RETURN_TYPE_FOR_GENERATOR, typ) - - # Python 2 generators aren't allowed to return values. - if (self.options.python_version[0] == 2 and - isinstance(typ.ret_type, Instance) and - typ.ret_type.type.fullname() == 'typing.Generator'): - if not (isinstance(typ.ret_type.args[2], Void) - or isinstance(typ.ret_type.args[2], AnyType)): - self.fail(messages.INVALID_GENERATOR_RETURN_ITEM_TYPE, typ) - - # Push return type. - self.return_types.append(typ.ret_type) - - # Store argument types. - for i in range(len(typ.arg_types)): - arg_type = typ.arg_types[i] - - # Refuse covariant parameter type variables - if isinstance(arg_type, TypeVarType): - if arg_type.variance == COVARIANT: - self.fail(messages.FUNCTION_PARAMETER_CANNOT_BE_COVARIANT, - arg_type) - - if typ.arg_kinds[i] == nodes.ARG_STAR: - # builtins.tuple[T] is typing.Tuple[T, ...] - arg_type = self.named_generic_type('builtins.tuple', - [arg_type]) - elif typ.arg_kinds[i] == nodes.ARG_STAR2: - arg_type = self.named_generic_type('builtins.dict', - [self.str_type(), - arg_type]) - item.arguments[i].variable.type = arg_type - - # Type check initialization expressions. - for arg in item.arguments: - init = arg.initialization_statement - if init: - self.accept(init) - - # Type check body in a new scope. - with self.binder.frame_context(): - self.accept(item.body) - - self.return_types.pop() - - self.binder = old_binder - - def check_reverse_op_method(self, defn: FuncItem, typ: CallableType, - method: str) -> None: - """Check a reverse operator method such as __radd__.""" - - # This used to check for some very obscure scenario. It now - # just decides whether it's worth calling - # check_overlapping_op_methods(). - - if method in ('__eq__', '__ne__'): - # These are defined for all objects => can't cause trouble. - return - - # With 'Any' or 'object' return type we are happy, since any possible - # return value is valid. - ret_type = typ.ret_type - if isinstance(ret_type, AnyType): - return - if isinstance(ret_type, Instance): - if ret_type.type.fullname() == 'builtins.object': - return - # Plausibly the method could have too few arguments, which would result - # in an error elsewhere. - if len(typ.arg_types) <= 2: - # TODO check self argument kind - - # Check for the issue described above. - arg_type = typ.arg_types[1] - other_method = nodes.normal_from_reverse_op[method] - if isinstance(arg_type, Instance): - if not arg_type.type.has_readable_member(other_method): - return - elif isinstance(arg_type, AnyType): - return - elif isinstance(arg_type, UnionType): - if not arg_type.has_readable_member(other_method): - return - else: - return - - typ2 = self.expr_checker.analyze_external_member_access( - other_method, arg_type, defn) - self.check_overlapping_op_methods( - typ, method, defn.info, - typ2, other_method, cast(Instance, arg_type), - defn) - - def check_overlapping_op_methods(self, - reverse_type: CallableType, - reverse_name: str, - reverse_class: TypeInfo, - forward_type: Type, - forward_name: str, - forward_base: Instance, - context: Context) -> None: - """Check for overlapping method and reverse method signatures. - - Assume reverse method has valid argument count and kinds. - """ - - # Reverse operator method that overlaps unsafely with the - # forward operator method can result in type unsafety. This is - # similar to overlapping overload variants. - # - # This example illustrates the issue: - # - # class X: pass - # class A: - # def __add__(self, x: X) -> int: - # if isinstance(x, X): - # return 1 - # return NotImplemented - # class B: - # def __radd__(self, x: A) -> str: return 'x' - # class C(X, B): pass - # def f(b: B) -> None: - # A() + b # Result is 1, even though static type seems to be str! - # f(C()) - # - # The reason for the problem is that B and X are overlapping - # types, and the return types are different. Also, if the type - # of x in __radd__ would not be A, the methods could be - # non-overlapping. - - if isinstance(forward_type, CallableType): - # TODO check argument kinds - if len(forward_type.arg_types) < 1: - # Not a valid operator method -- can't succeed anyway. - return - - # Construct normalized function signatures corresponding to the - # operator methods. The first argument is the left operand and the - # second operand is the right argument -- we switch the order of - # the arguments of the reverse method. - forward_tweaked = CallableType( - [forward_base, forward_type.arg_types[0]], - [nodes.ARG_POS] * 2, - [None] * 2, - forward_type.ret_type, - forward_type.fallback, - name=forward_type.name) - reverse_args = reverse_type.arg_types - reverse_tweaked = CallableType( - [reverse_args[1], reverse_args[0]], - [nodes.ARG_POS] * 2, - [None] * 2, - reverse_type.ret_type, - fallback=self.named_type('builtins.function'), - name=reverse_type.name) - - if is_unsafe_overlapping_signatures(forward_tweaked, - reverse_tweaked): - self.msg.operator_method_signatures_overlap( - reverse_class.name(), reverse_name, - forward_base.type.name(), forward_name, context) - elif isinstance(forward_type, Overloaded): - for item in forward_type.items(): - self.check_overlapping_op_methods( - reverse_type, reverse_name, reverse_class, - item, forward_name, forward_base, context) - elif not isinstance(forward_type, AnyType): - self.msg.forward_operator_not_callable(forward_name, context) - - def check_inplace_operator_method(self, defn: FuncBase) -> None: - """Check an inplace operator method such as __iadd__. - - They cannot arbitrarily overlap with __add__. - """ - method = defn.name() - if method not in nodes.inplace_operator_methods: - return - typ = self.method_type(defn) - cls = defn.info - other_method = '__' + method[3:] - if cls.has_readable_member(other_method): - instance = self_type(cls) - typ2 = self.expr_checker.analyze_external_member_access( - other_method, instance, defn) - fail = False - if isinstance(typ2, FunctionLike): - if not is_more_general_arg_prefix(typ, typ2): - fail = True - else: - # TODO overloads - fail = True - if fail: - self.msg.signatures_incompatible(method, other_method, defn) - - def check_getattr_method(self, typ: CallableType, context: Context) -> None: - method_type = CallableType([AnyType(), self.named_type('builtins.str')], - [nodes.ARG_POS, nodes.ARG_POS], - [None], - AnyType(), - self.named_type('builtins.function')) - if not is_subtype(typ, method_type): - self.msg.invalid_signature(typ, context) - - def expand_typevars(self, defn: FuncItem, - typ: CallableType) -> List[Tuple[FuncItem, CallableType]]: - # TODO use generator - subst = [] # type: List[List[Tuple[TypeVarId, Type]]] - tvars = typ.variables or [] - tvars = tvars[:] - if defn.info: - # Class type variables - tvars += defn.info.defn.type_vars or [] - for tvar in tvars: - if tvar.values: - subst.append([(tvar.id, value) - for value in tvar.values]) - if subst: - result = [] # type: List[Tuple[FuncItem, CallableType]] - for substitutions in itertools.product(*subst): - mapping = dict(substitutions) - expanded = cast(CallableType, expand_type(typ, mapping)) - result.append((expand_func(defn, mapping), expanded)) - return result - else: - return [(defn, typ)] - - def check_method_override(self, defn: FuncBase) -> None: - """Check if function definition is compatible with base classes.""" - # Check against definitions in base classes. - for base in defn.info.mro[1:]: - self.check_method_or_accessor_override_for_base(defn, base) - - def check_method_or_accessor_override_for_base(self, defn: FuncBase, - base: TypeInfo) -> None: - """Check if method definition is compatible with a base class.""" - if base: - name = defn.name() - if name not in ('__init__', '__new__'): - # Check method override (__init__ and __new__ are special). - self.check_method_override_for_base_with_name(defn, name, base) - if name in nodes.inplace_operator_methods: - # Figure out the name of the corresponding operator method. - method = '__' + name[3:] - # An inplace operator method such as __iadd__ might not be - # always introduced safely if a base class defined __add__. - # TODO can't come up with an example where this is - # necessary; now it's "just in case" - self.check_method_override_for_base_with_name(defn, method, - base) - - def check_method_override_for_base_with_name( - self, defn: FuncBase, name: str, base: TypeInfo) -> None: - base_attr = base.names.get(name) - if base_attr: - # The name of the method is defined in the base class. - - # Construct the type of the overriding method. - typ = self.method_type(defn) - # Map the overridden method type to subtype context so that - # it can be checked for compatibility. - original_type = base_attr.type - if original_type is None and isinstance(base_attr.node, - FuncDef): - original_type = self.function_type(base_attr.node) - if isinstance(original_type, FunctionLike): - original = map_type_from_supertype( - method_type(original_type), - defn.info, base) - # Check that the types are compatible. - # TODO overloaded signatures - self.check_override(typ, - cast(FunctionLike, original), - defn.name(), - name, - base.name(), - defn) - else: - assert original_type is not None - self.msg.signature_incompatible_with_supertype( - defn.name(), name, base.name(), defn) - - def check_override(self, override: FunctionLike, original: FunctionLike, - name: str, name_in_super: str, supertype: str, - node: Context) -> None: - """Check a method override with given signatures. - - Arguments: - override: The signature of the overriding method. - original: The signature of the original supertype method. - name: The name of the subtype. This and the next argument are - only used for generating error messages. - supertype: The name of the supertype. - """ - # Use boolean variable to clarify code. - fail = False - if not is_subtype(override, original): - fail = True - elif (not isinstance(original, Overloaded) and - isinstance(override, Overloaded) and - name in nodes.reverse_op_methods.keys()): - # Operator method overrides cannot introduce overloading, as - # this could be unsafe with reverse operator methods. - fail = True - - if fail: - emitted_msg = False - if (isinstance(override, CallableType) and - isinstance(original, CallableType) and - len(override.arg_types) == len(original.arg_types) and - override.min_args == original.min_args): - # Give more detailed messages for the common case of both - # signatures having the same number of arguments and no - # overloads. - - # override might have its own generic function type - # variables. If an argument or return type of override - # does not have the correct subtyping relationship - # with the original type even after these variables - # are erased, then it is definitely an incompatiblity. - - override_ids = override.type_var_ids() - - def erase_override(t: Type) -> Type: - return erase_typevars(t, ids_to_erase=override_ids) - - for i in range(len(override.arg_types)): - if not is_subtype(original.arg_types[i], - erase_override(override.arg_types[i])): - self.msg.argument_incompatible_with_supertype( - i + 1, name, name_in_super, supertype, node) - emitted_msg = True - - if not is_subtype(erase_override(override.ret_type), - original.ret_type): - self.msg.return_type_incompatible_with_supertype( - name, name_in_super, supertype, node) - emitted_msg = True - - if not emitted_msg: - # Fall back to generic incompatibility message. - self.msg.signature_incompatible_with_supertype( - name, name_in_super, supertype, node) - - def visit_class_def(self, defn: ClassDef) -> Type: - """Type check a class definition.""" - typ = defn.info - self.errors.push_type(defn.name) - self.enter_partial_types() - old_binder = self.binder - self.binder = ConditionalTypeBinder() - with self.binder.frame_context(): - self.accept(defn.defs) - self.binder = old_binder - if not hasattr(defn, 'incompatible_baseclass'): - # Otherwise we've already found errors; more errors are not useful - self.check_multiple_inheritance(typ) - self.leave_partial_types() - self.errors.pop_type() - - def check_multiple_inheritance(self, typ: TypeInfo) -> None: - """Check for multiple inheritance related errors.""" - - if len(typ.bases) <= 1: - # No multiple inheritance. - return - # Verify that inherited attributes are compatible. - mro = typ.mro[1:] - for i, base in enumerate(mro): - for name in base.names: - for base2 in mro[i + 1:]: - # We only need to check compatibility of attributes from classes not - # in a subclass relationship. For subclasses, normal (single inheritance) - # checks suffice (these are implemented elsewhere). - if name in base2.names and base2 not in base.mro: - self.check_compatibility(name, base, base2, typ) - # Verify that base class layouts are compatible. - builtin_bases = [nearest_builtin_ancestor(base.type) - for base in typ.bases] - for base1 in builtin_bases: - for base2 in builtin_bases: - if not (base1 in base2.mro or base2 in base1.mro): - self.fail(messages.INSTANCE_LAYOUT_CONFLICT, typ) - - def check_compatibility(self, name: str, base1: TypeInfo, - base2: TypeInfo, ctx: Context) -> None: - """Check if attribute name in base1 is compatible with base2 in multiple inheritance. - - Assume base1 comes before base2 in the MRO, and that base1 and base2 don't have - a direct subclass relationship (i.e., the compatibility requirement only derives from - multiple inheritance). - """ - if name == '__init__': - # __init__ can be incompatible -- it's a special case. - return - first = base1[name] - second = base2[name] - first_type = first.type - if first_type is None and isinstance(first.node, FuncDef): - first_type = self.function_type(first.node) - second_type = second.type - if second_type is None and isinstance(second.node, FuncDef): - second_type = self.function_type(second.node) - # TODO: What if some classes are generic? - if (isinstance(first_type, FunctionLike) and - isinstance(second_type, FunctionLike)): - # Method override - first_sig = method_type(first_type) - second_sig = method_type(second_type) - ok = is_subtype(first_sig, second_sig) - elif first_type and second_type: - ok = is_equivalent(first_type, second_type) - else: - if first_type is None: - self.msg.cannot_determine_type_in_base(name, base1.name(), ctx) - if second_type is None: - self.msg.cannot_determine_type_in_base(name, base2.name(), ctx) - ok = True - if not ok: - self.msg.base_class_definitions_incompatible(name, base1, base2, - ctx) - - def visit_import_from(self, node: ImportFrom) -> Type: - self.check_import(node) - - def visit_import_all(self, node: ImportAll) -> Type: - self.check_import(node) - - def check_import(self, node: ImportBase) -> Type: - for assign in node.assignments: - lvalue = assign.lvalues[0] - lvalue_type, _, __ = self.check_lvalue(lvalue) - if lvalue_type is None: - # TODO: This is broken. - lvalue_type = AnyType() - message = '{} "{}"'.format(messages.INCOMPATIBLE_IMPORT_OF, - cast(NameExpr, assign.rvalue).name) - self.check_simple_assignment(lvalue_type, assign.rvalue, node, - msg=message, lvalue_name='local name', - rvalue_name='imported name') - - # - # Statements - # - - def visit_block(self, b: Block) -> Type: - if b.is_unreachable: - return None - for s in b.body: - self.accept(s) - if self.binder.breaking_out: - break - - def visit_assignment_stmt(self, s: AssignmentStmt) -> Type: - """Type check an assignment statement. - - Handle all kinds of assignment statements (simple, indexed, multiple). - """ - self.check_assignment(s.lvalues[-1], s.rvalue, s.type is None) - - if len(s.lvalues) > 1: - # Chained assignment (e.g. x = y = ...). - # Make sure that rvalue type will not be reinferred. - if s.rvalue not in self.type_map: - self.accept(s.rvalue) - rvalue = self.temp_node(self.type_map[s.rvalue], s) - for lv in s.lvalues[:-1]: - self.check_assignment(lv, rvalue, s.type is None) - - def check_assignment(self, lvalue: Node, rvalue: Node, infer_lvalue_type: bool = True) -> None: - """Type check a single assignment: lvalue = rvalue.""" - if isinstance(lvalue, TupleExpr) or isinstance(lvalue, ListExpr): - ltuple = cast(Union[TupleExpr, ListExpr], lvalue) - - self.check_assignment_to_multiple_lvalues(ltuple.items, rvalue, lvalue, - infer_lvalue_type) - else: - lvalue_type, index_lvalue, inferred = self.check_lvalue(lvalue) - if lvalue_type: - if isinstance(lvalue_type, PartialType) and lvalue_type.type is None: - # Try to infer a proper type for a variable with a partial None type. - rvalue_type = self.accept(rvalue) - if isinstance(rvalue_type, NoneTyp): - # This doesn't actually provide any additional information -- multiple - # None initializers preserve the partial None type. - return - if is_valid_inferred_type(rvalue_type): - var = lvalue_type.var - partial_types = self.find_partial_types(var) - if partial_types is not None: - if not self.current_node_deferred: - if experiments.STRICT_OPTIONAL: - var.type = UnionType.make_simplified_union( - [rvalue_type, NoneTyp()]) - else: - var.type = rvalue_type - else: - var.type = None - del partial_types[var] - # Try to infer a partial type. No need to check the return value, as - # an error will be reported elsewhere. - self.infer_partial_type(lvalue_type.var, lvalue, rvalue_type) - return - if (is_literal_none(rvalue) and - isinstance(lvalue, NameExpr) and - isinstance(lvalue.node, Var) and - lvalue.node.is_initialized_in_class): - # Allow None's to be assigned to class variables with non-Optional types. - rvalue_type = lvalue_type - else: - rvalue_type = self.check_simple_assignment(lvalue_type, rvalue, lvalue) - - if rvalue_type and infer_lvalue_type: - self.binder.assign_type(lvalue, - rvalue_type, - lvalue_type, - self.typing_mode_weak()) - elif index_lvalue: - self.check_indexed_assignment(index_lvalue, rvalue, rvalue) - - if inferred: - self.infer_variable_type(inferred, lvalue, self.accept(rvalue), - rvalue) - - def check_assignment_to_multiple_lvalues(self, lvalues: List[Node], rvalue: Node, - context: Context, - infer_lvalue_type: bool = True) -> None: - if isinstance(rvalue, TupleExpr) or isinstance(rvalue, ListExpr): - # Recursively go into Tuple or List expression rhs instead of - # using the type of rhs, because this allowed more fine grained - # control in cases like: a, b = [int, str] where rhs would get - # type List[object] - - rvalues = cast(Union[TupleExpr, ListExpr], rvalue).items - - if self.check_rvalue_count_in_assignment(lvalues, len(rvalues), context): - star_index = next((i for i, lv in enumerate(lvalues) if - isinstance(lv, StarExpr)), len(lvalues)) - - left_lvs = lvalues[:star_index] - star_lv = cast(StarExpr, - lvalues[star_index]) if star_index != len(lvalues) else None - right_lvs = lvalues[star_index + 1:] - - left_rvs, star_rvs, right_rvs = self.split_around_star( - rvalues, star_index, len(lvalues)) - - lr_pairs = list(zip(left_lvs, left_rvs)) - if star_lv: - rv_list = ListExpr(star_rvs) - rv_list.set_line(rvalue.get_line()) - lr_pairs.append((star_lv.expr, rv_list)) - lr_pairs.extend(zip(right_lvs, right_rvs)) - - for lv, rv in lr_pairs: - self.check_assignment(lv, rv, infer_lvalue_type) - else: - self.check_multi_assignment(lvalues, rvalue, context, infer_lvalue_type) - - def check_rvalue_count_in_assignment(self, lvalues: List[Node], rvalue_count: int, - context: Context) -> bool: - if any(isinstance(lvalue, StarExpr) for lvalue in lvalues): - if len(lvalues) - 1 > rvalue_count: - self.msg.wrong_number_values_to_unpack(rvalue_count, - len(lvalues) - 1, context) - return False - elif rvalue_count != len(lvalues): - self.msg.wrong_number_values_to_unpack(rvalue_count, - len(lvalues), context) - return False - return True - - def check_multi_assignment(self, lvalues: List[Node], - rvalue: Node, - context: Context, - infer_lvalue_type: bool = True, - msg: str = None) -> None: - """Check the assignment of one rvalue to a number of lvalues.""" - - # Infer the type of an ordinary rvalue expression. - rvalue_type = self.accept(rvalue) # TODO maybe elsewhere; redundant - undefined_rvalue = False - - if isinstance(rvalue_type, AnyType): - for lv in lvalues: - if isinstance(lv, StarExpr): - lv = lv.expr - self.check_assignment(lv, self.temp_node(AnyType(), context), infer_lvalue_type) - elif isinstance(rvalue_type, TupleType): - self.check_multi_assignment_from_tuple(lvalues, rvalue, rvalue_type, - context, undefined_rvalue, infer_lvalue_type) - else: - self.check_multi_assignment_from_iterable(lvalues, rvalue_type, - context, infer_lvalue_type) - - def check_multi_assignment_from_tuple(self, lvalues: List[Node], rvalue: Node, - rvalue_type: TupleType, context: Context, - undefined_rvalue: bool, - infer_lvalue_type: bool = True) -> None: - if self.check_rvalue_count_in_assignment(lvalues, len(rvalue_type.items), context): - star_index = next((i for i, lv in enumerate(lvalues) - if isinstance(lv, StarExpr)), len(lvalues)) - - left_lvs = lvalues[:star_index] - star_lv = cast(StarExpr, lvalues[star_index]) if star_index != len(lvalues) else None - right_lvs = lvalues[star_index + 1:] - - if not undefined_rvalue: - # Infer rvalue again, now in the correct type context. - lvalue_type = self.lvalue_type_for_inference(lvalues, rvalue_type) - rvalue_type = cast(TupleType, self.accept(rvalue, lvalue_type)) - - left_rv_types, star_rv_types, right_rv_types = self.split_around_star( - rvalue_type.items, star_index, len(lvalues)) - - for lv, rv_type in zip(left_lvs, left_rv_types): - self.check_assignment(lv, self.temp_node(rv_type, context), infer_lvalue_type) - if star_lv: - nodes = [self.temp_node(rv_type, context) for rv_type in star_rv_types] - list_expr = ListExpr(nodes) - list_expr.set_line(context.get_line()) - self.check_assignment(star_lv.expr, list_expr, infer_lvalue_type) - for lv, rv_type in zip(right_lvs, right_rv_types): - self.check_assignment(lv, self.temp_node(rv_type, context), infer_lvalue_type) - - def lvalue_type_for_inference(self, lvalues: List[Node], rvalue_type: TupleType) -> Type: - star_index = next((i for i, lv in enumerate(lvalues) - if isinstance(lv, StarExpr)), len(lvalues)) - left_lvs = lvalues[:star_index] - star_lv = cast(StarExpr, lvalues[star_index]) if star_index != len(lvalues) else None - right_lvs = lvalues[star_index + 1:] - left_rv_types, star_rv_types, right_rv_types = self.split_around_star( - rvalue_type.items, star_index, len(lvalues)) - - type_parameters = [] # type: List[Type] - - def append_types_for_inference(lvs: List[Node], rv_types: List[Type]) -> None: - for lv, rv_type in zip(lvs, rv_types): - sub_lvalue_type, index_expr, inferred = self.check_lvalue(lv) - if sub_lvalue_type: - type_parameters.append(sub_lvalue_type) - else: # index lvalue - # TODO Figure out more precise type context, probably - # based on the type signature of the _set method. - type_parameters.append(rv_type) - - append_types_for_inference(left_lvs, left_rv_types) - - if star_lv: - sub_lvalue_type, index_expr, inferred = self.check_lvalue(star_lv.expr) - if sub_lvalue_type: - type_parameters.extend([sub_lvalue_type] * len(star_rv_types)) - else: # index lvalue - # TODO Figure out more precise type context, probably - # based on the type signature of the _set method. - type_parameters.extend(star_rv_types) - - append_types_for_inference(right_lvs, right_rv_types) - - return TupleType(type_parameters, self.named_type('builtins.tuple')) - - def split_around_star(self, items: List[T], star_index: int, - length: int) -> Tuple[List[T], List[T], List[T]]: - """Splits a list of items in three to match another list of length 'length' - that contains a starred expression at 'star_index' in the following way: - - star_index = 2, length = 5 (i.e., [a,b,*,c,d]), items = [1,2,3,4,5,6,7] - returns in: ([1,2], [3,4,5], [6,7]) - """ - nr_right_of_star = length - star_index - 1 - right_index = nr_right_of_star if -nr_right_of_star != 0 else len(items) - left = items[:star_index] - star = items[star_index:right_index] - right = items[right_index:] - return (left, star, right) - - def type_is_iterable(self, type: Type) -> bool: - return (is_subtype(type, self.named_generic_type('typing.Iterable', - [AnyType()])) and - isinstance(type, Instance)) - - def check_multi_assignment_from_iterable(self, lvalues: List[Node], rvalue_type: Type, - context: Context, - infer_lvalue_type: bool = True) -> None: - if self.type_is_iterable(rvalue_type): - item_type = self.iterable_item_type(cast(Instance, rvalue_type)) - for lv in lvalues: - if isinstance(lv, StarExpr): - self.check_assignment(lv.expr, self.temp_node(rvalue_type, context), - infer_lvalue_type) - else: - self.check_assignment(lv, self.temp_node(item_type, context), - infer_lvalue_type) - else: - self.msg.type_not_iterable(rvalue_type, context) - - def check_lvalue(self, lvalue: Node) -> Tuple[Type, IndexExpr, Var]: - lvalue_type = None # type: Type - index_lvalue = None # type: IndexExpr - inferred = None # type: Var - - if self.is_definition(lvalue): - if isinstance(lvalue, NameExpr): - inferred = cast(Var, lvalue.node) - assert isinstance(inferred, Var) - else: - m = cast(MemberExpr, lvalue) - self.accept(m.expr) - inferred = m.def_var - elif isinstance(lvalue, IndexExpr): - index_lvalue = lvalue - elif isinstance(lvalue, MemberExpr): - lvalue_type = self.expr_checker.analyze_ordinary_member_access(lvalue, - True) - self.store_type(lvalue, lvalue_type) - elif isinstance(lvalue, NameExpr): - lvalue_type = self.expr_checker.analyze_ref_expr(lvalue, lvalue=True) - self.store_type(lvalue, lvalue_type) - elif isinstance(lvalue, TupleExpr) or isinstance(lvalue, ListExpr): - lv = cast(Union[TupleExpr, ListExpr], lvalue) - types = [self.check_lvalue(sub_expr)[0] for sub_expr in lv.items] - lvalue_type = TupleType(types, self.named_type('builtins.tuple')) - else: - lvalue_type = self.accept(lvalue) - - return lvalue_type, index_lvalue, inferred - - def is_definition(self, s: Node) -> bool: - if isinstance(s, NameExpr): - if s.is_def: - return True - # If the node type is not defined, this must the first assignment - # that we process => this is a definition, even though the semantic - # analyzer did not recognize this as such. This can arise in code - # that uses isinstance checks, if type checking of the primary - # definition is skipped due to an always False type check. - node = s.node - if isinstance(node, Var): - return node.type is None - elif isinstance(s, MemberExpr): - return s.is_def - return False - - def infer_variable_type(self, name: Var, lvalue: Node, - init_type: Type, context: Context) -> None: - """Infer the type of initialized variables from initializer type.""" - if self.typing_mode_weak(): - self.set_inferred_type(name, lvalue, AnyType()) - self.binder.assign_type(lvalue, init_type, self.binder.get_declaration(lvalue), True) - elif isinstance(init_type, Void): - self.check_not_void(init_type, context) - self.set_inference_error_fallback_type(name, lvalue, init_type, context) - elif isinstance(init_type, DeletedType): - self.msg.deleted_as_rvalue(init_type, context) - elif not is_valid_inferred_type(init_type): - # We cannot use the type of the initialization expression for full type - # inference (it's not specific enough), but we might be able to give - # partial type which will be made more specific later. A partial type - # gets generated in assignment like 'x = []' where item type is not known. - if not self.infer_partial_type(name, lvalue, init_type): - self.fail(messages.NEED_ANNOTATION_FOR_VAR, context) - self.set_inference_error_fallback_type(name, lvalue, init_type, context) - else: - # Infer type of the target. - - # Make the type more general (strip away function names etc.). - init_type = strip_type(init_type) - - self.set_inferred_type(name, lvalue, init_type) - - def infer_partial_type(self, name: Var, lvalue: Node, init_type: Type) -> bool: - if isinstance(init_type, (NoneTyp, UninhabitedType)): - partial_type = PartialType(None, name, [init_type]) - elif isinstance(init_type, Instance): - fullname = init_type.type.fullname() - if (isinstance(lvalue, NameExpr) and - (fullname == 'builtins.list' or - fullname == 'builtins.set' or - fullname == 'builtins.dict') and - all(isinstance(t, (NoneTyp, UninhabitedType)) for t in init_type.args)): - partial_type = PartialType(init_type.type, name, init_type.args) - else: - return False - else: - return False - self.set_inferred_type(name, lvalue, partial_type) - self.partial_types[-1][name] = lvalue - return True - - def set_inferred_type(self, var: Var, lvalue: Node, type: Type) -> None: - """Store inferred variable type. - - Store the type to both the variable node and the expression node that - refers to the variable (lvalue). If var is None, do nothing. - """ - if var and not self.current_node_deferred: - var.type = type - self.store_type(lvalue, type) - - def set_inference_error_fallback_type(self, var: Var, lvalue: Node, type: Type, - context: Context) -> None: - """If errors on context line are ignored, store dummy type for variable. - - If a program ignores error on type inference error, the variable should get some - inferred type so that if can used later on in the program. Example: - - x = [] # type: ignore - x.append(1) # Should be ok! - - We implement this here by giving x a valid type (Any). - """ - if context.get_line() in self.errors.ignored_lines[self.errors.file]: - self.set_inferred_type(var, lvalue, AnyType()) - - def narrow_type_from_binder(self, expr: Node, known_type: Type) -> Type: - if expr.literal >= LITERAL_TYPE: - restriction = self.binder.get(expr) - if restriction: - ans = meet_simple(known_type, restriction) - return ans - return known_type - - def check_simple_assignment(self, lvalue_type: Type, rvalue: Node, - context: Node, - msg: str = messages.INCOMPATIBLE_TYPES_IN_ASSIGNMENT, - lvalue_name: str = 'variable', - rvalue_name: str = 'expression') -> Type: - if self.is_stub and isinstance(rvalue, EllipsisExpr): - # '...' is always a valid initializer in a stub. - return AnyType() - else: - rvalue_type = self.accept(rvalue, lvalue_type) - if isinstance(rvalue_type, DeletedType): - self.msg.deleted_as_rvalue(rvalue_type, context) - if self.typing_mode_weak(): - return rvalue_type - if isinstance(lvalue_type, DeletedType): - self.msg.deleted_as_lvalue(lvalue_type, context) - else: - self.check_subtype(rvalue_type, lvalue_type, context, msg, - '{} has type'.format(rvalue_name), - '{} has type'.format(lvalue_name)) - return rvalue_type - - def check_indexed_assignment(self, lvalue: IndexExpr, - rvalue: Node, context: Context) -> None: - """Type check indexed assignment base[index] = rvalue. - - The lvalue argument is the base[index] expression. - """ - self.try_infer_partial_type_from_indexed_assignment(lvalue, rvalue) - basetype = self.accept(lvalue.base) - method_type = self.expr_checker.analyze_external_member_access( - '__setitem__', basetype, context) - lvalue.method_type = method_type - self.expr_checker.check_call(method_type, [lvalue.index, rvalue], - [nodes.ARG_POS, nodes.ARG_POS], - context) - - def try_infer_partial_type_from_indexed_assignment( - self, lvalue: IndexExpr, rvalue: Node) -> None: - # TODO: Should we share some of this with try_infer_partial_type? - if isinstance(lvalue.base, RefExpr) and isinstance(lvalue.base.node, Var): - var = lvalue.base.node - if isinstance(var.type, PartialType): - type_type = var.type.type - if type_type is None: - return # The partial type is None. - partial_types = self.find_partial_types(var) - if partial_types is None: - return - typename = type_type.fullname() - if typename == 'builtins.dict': - # TODO: Don't infer things twice. - key_type = self.accept(lvalue.index) - value_type = self.accept(rvalue) - full_key_type = UnionType.make_simplified_union( - [key_type, var.type.inner_types[0]]) - full_value_type = UnionType.make_simplified_union( - [value_type, var.type.inner_types[1]]) - if (is_valid_inferred_type(full_key_type) and - is_valid_inferred_type(full_value_type)): - if not self.current_node_deferred: - var.type = self.named_generic_type('builtins.dict', - [full_key_type, full_value_type]) - del partial_types[var] - - def visit_expression_stmt(self, s: ExpressionStmt) -> Type: - self.accept(s.expr) - - def visit_return_stmt(self, s: ReturnStmt) -> Type: - """Type check a return statement.""" - self.binder.breaking_out = True - if self.is_within_function(): - if self.function_stack[-1].is_generator: - return_type = self.get_generator_return_type(self.return_types[-1]) - else: - return_type = self.return_types[-1] - - if s.expr: - # Return with a value. - typ = self.accept(s.expr, return_type) - # Returning a value of type Any is always fine. - if isinstance(typ, AnyType): - return None - - if isinstance(return_type, Void): - # Lambdas are allowed to have a Void return. - # Functions returning a value of type None are allowed to have a Void return. - if isinstance(self.function_stack[-1], FuncExpr) or isinstance(typ, NoneTyp): - return None - self.fail(messages.NO_RETURN_VALUE_EXPECTED, s) - else: - self.check_subtype( - subtype_label='got', - subtype=typ, - supertype_label='expected', - supertype=return_type, - context=s, - msg=messages.INCOMPATIBLE_RETURN_VALUE_TYPE) - else: - # Empty returns are valid in Generators with Any typed returns. - if (self.function_stack[-1].is_generator and isinstance(return_type, AnyType)): - return None - - if isinstance(return_type, Void): - return None - - if isinstance(return_type, AnyType): - return None - - if self.typing_mode_full(): - self.fail(messages.RETURN_VALUE_EXPECTED, s) - - def wrap_generic_type(self, typ: Instance, rtyp: Instance, check_type: - str, context: Context) -> Type: - n_diff = self.count_nested_types(rtyp, check_type) - self.count_nested_types(typ, - check_type) - if n_diff == 1: - return self.named_generic_type(check_type, [typ]) - elif n_diff == 0 or n_diff > 1: - self.fail(messages.INCOMPATIBLE_RETURN_VALUE_TYPE - + ": expected {}, got {}".format(rtyp, typ), context) - return typ - return typ - - def count_nested_types(self, typ: Instance, check_type: str) -> int: - c = 0 - while is_subtype(typ, self.named_type(check_type)): - c += 1 - typ = map_instance_to_supertype(self.named_generic_type(check_type, typ.args), - self.lookup_typeinfo(check_type)) - if typ.args: - typ = cast(Instance, typ.args[0]) - else: - return c - return c - - def visit_if_stmt(self, s: IfStmt) -> Type: - """Type check an if statement.""" - breaking_out = True - # This frame records the knowledge from previous if/elif clauses not being taken. - with self.binder.frame_context(): - for e, b in zip(s.expr, s.body): - t = self.accept(e) - self.check_not_void(t, e) - if_map, else_map = find_isinstance_check( - e, self.type_map, - self.typing_mode_weak() - ) - if if_map is None: - # The condition is always false - # XXX should issue a warning? - pass - else: - # Only type check body if the if condition can be true. - with self.binder.frame_context(2): - if if_map: - for var, type in if_map.items(): - self.binder.push(var, type) - - self.accept(b) - breaking_out = breaking_out and self.binder.last_pop_breaking_out - - if else_map: - for var, type in else_map.items(): - self.binder.push(var, type) - if else_map is None: - # The condition is always true => remaining elif/else blocks - # can never be reached. - - # Might also want to issue a warning - # print("Warning: isinstance always true") - break - else: # Didn't break => can't prove one of the conditions is always true - with self.binder.frame_context(2): - if s.else_body: - self.accept(s.else_body) - breaking_out = breaking_out and self.binder.last_pop_breaking_out - if breaking_out: - self.binder.breaking_out = True - return None - - def visit_while_stmt(self, s: WhileStmt) -> Type: - """Type check a while statement.""" - self.accept_loop(IfStmt([s.expr], [s.body], None), s.else_body) - - def visit_operator_assignment_stmt(self, - s: OperatorAssignmentStmt) -> Type: - """Type check an operator assignment statement, e.g. x += 1.""" - lvalue_type = self.accept(s.lvalue) - method = infer_operator_assignment_method(lvalue_type, s.op) - rvalue_type, method_type = self.expr_checker.check_op( - method, lvalue_type, s.rvalue, s) - - if isinstance(s.lvalue, IndexExpr): - self.check_indexed_assignment(s.lvalue, s.rvalue, s.rvalue) - else: - if not is_subtype(rvalue_type, lvalue_type): - self.msg.incompatible_operator_assignment(s.op, s) - - def visit_assert_stmt(self, s: AssertStmt) -> Type: - self.accept(s.expr) - - # If this is asserting some isinstance check, bind that type in the following code - true_map, _ = find_isinstance_check( - s.expr, self.type_map, - self.typing_mode_weak() - ) - - if true_map: - for var, type in true_map.items(): - self.binder.push(var, type) - - def visit_raise_stmt(self, s: RaiseStmt) -> Type: - """Type check a raise statement.""" - self.binder.breaking_out = True - if s.expr: - self.type_check_raise(s.expr, s) - if s.from_expr: - self.type_check_raise(s.from_expr, s) - - def type_check_raise(self, e: Node, s: RaiseStmt) -> None: - typ = self.accept(e) - if isinstance(typ, FunctionLike): - if typ.is_type_obj(): - # Cases like "raise/from ExceptionClass". - typeinfo = typ.type_object() - base = self.lookup_typeinfo('builtins.BaseException') - if base in typeinfo.mro or typeinfo.fallback_to_any: - # Good! - return - # Else fall back to the checks below (which will fail). - if isinstance(typ, TupleType) and self.options.python_version[0] == 2: - # allow `raise type, value, traceback` - # https://docs.python.org/2/reference/simple_stmts.html#the-raise-statement - # TODO: Also check tuple item types. - if len(typ.items) in (2, 3): - return - if isinstance(typ, Instance) and typ.type.fallback_to_any: - # OK! - return - self.check_subtype(typ, - self.named_type('builtins.BaseException'), s, - messages.INVALID_EXCEPTION) - - def visit_try_stmt(self, s: TryStmt) -> Type: - """Type check a try statement.""" - # Our enclosing frame will get the result if the try/except falls through. - # This one gets all possible intermediate states - with self.binder.frame_context(): - if s.finally_body: - self.binder.try_frames.add(len(self.binder.frames) - 1) - breaking_out = self.visit_try_without_finally(s) - self.binder.try_frames.remove(len(self.binder.frames) - 1) - # First we check finally_body is type safe for all intermediate frames - self.accept(s.finally_body) - breaking_out = breaking_out or self.binder.breaking_out - else: - breaking_out = self.visit_try_without_finally(s) - - if not breaking_out and s.finally_body: - # Then we try again for the more restricted set of options that can fall through - self.accept(s.finally_body) - self.binder.breaking_out = breaking_out - return None - - def visit_try_without_finally(self, s: TryStmt) -> bool: - """Type check a try statement, ignoring the finally block. - - Return whether we are guaranteed to be breaking out. - Otherwise, it will place the results possible frames of - that don't break out into self.binder.frames[-2]. - """ - breaking_out = True - # This frame records the possible states that exceptions can leave variables in - # during the try: block - with self.binder.frame_context(): - with self.binder.frame_context(3): - self.binder.try_frames.add(len(self.binder.frames) - 2) - self.accept(s.body) - self.binder.try_frames.remove(len(self.binder.frames) - 2) - if s.else_body: - self.accept(s.else_body) - breaking_out = breaking_out and self.binder.last_pop_breaking_out - for i in range(len(s.handlers)): - with self.binder.frame_context(3): - if s.types[i]: - t = self.visit_except_handler_test(s.types[i]) - if s.vars[i]: - # To support local variables, we make this a definition line, - # causing assignment to set the variable's type. - s.vars[i].is_def = True - self.check_assignment(s.vars[i], self.temp_node(t, s.vars[i])) - self.accept(s.handlers[i]) - if s.vars[i]: - # Exception variables are deleted in python 3 but not python 2. - # But, since it's bad form in python 2 and the type checking - # wouldn't work very well, we delete it anyway. - - # Unfortunately, this doesn't let us detect usage before the - # try/except block. - if self.options.python_version[0] >= 3: - source = s.vars[i].name - else: - source = ('(exception variable "{}", which we do not accept outside' - 'except: blocks even in python 2)'.format(s.vars[i].name)) - var = cast(Var, s.vars[i].node) - var.type = DeletedType(source=source) - self.binder.cleanse(s.vars[i]) - breaking_out = breaking_out and self.binder.last_pop_breaking_out - return breaking_out - - def visit_except_handler_test(self, n: Node) -> Type: - """Type check an exception handler test clause.""" - type = self.accept(n) - - all_types = [] # type: List[Type] - test_types = type.items if isinstance(type, TupleType) else [type] - - for ttype in test_types: - if isinstance(ttype, AnyType): - all_types.append(ttype) - continue - - if not isinstance(ttype, FunctionLike): - self.fail(messages.INVALID_EXCEPTION_TYPE, n) - return AnyType() - - item = ttype.items()[0] - ret_type = item.ret_type - if not (is_subtype(ret_type, self.named_type('builtins.BaseException')) - and item.is_type_obj()): - self.fail(messages.INVALID_EXCEPTION_TYPE, n) - return AnyType() - - all_types.append(ret_type) - - return UnionType.make_simplified_union(all_types) - - def visit_for_stmt(self, s: ForStmt) -> Type: - """Type check a for statement.""" - item_type = self.analyze_iterable_item_type(s.expr) - self.analyze_index_variables(s.index, item_type, s) - self.accept_loop(s.body, s.else_body) - - def analyze_iterable_item_type(self, expr: Node) -> Type: - """Analyse iterable expression and return iterator item type.""" - iterable = self.accept(expr) - - self.check_not_void(iterable, expr) - if isinstance(iterable, TupleType): - if experiments.STRICT_OPTIONAL: - joined = UninhabitedType() # type: Type - else: - joined = NoneTyp() - for item in iterable.items: - joined = join_types(joined, item) - if isinstance(joined, ErrorType): - self.fail(messages.CANNOT_INFER_ITEM_TYPE, expr) - return AnyType() - return joined - else: - # Non-tuple iterable. - self.check_subtype(iterable, - self.named_generic_type('typing.Iterable', - [AnyType()]), - expr, messages.ITERABLE_EXPECTED) - - echk = self.expr_checker - method = echk.analyze_external_member_access('__iter__', iterable, - expr) - iterator = echk.check_call(method, [], [], expr)[0] - if self.options.python_version[0] >= 3: - nextmethod = '__next__' - else: - nextmethod = 'next' - method = echk.analyze_external_member_access(nextmethod, iterator, - expr) - return echk.check_call(method, [], [], expr)[0] - - def analyze_index_variables(self, index: Node, item_type: Type, - context: Context) -> None: - """Type check or infer for loop or list comprehension index vars.""" - self.check_assignment(index, self.temp_node(item_type, context)) - - def visit_del_stmt(self, s: DelStmt) -> Type: - if isinstance(s.expr, IndexExpr): - e = s.expr - m = MemberExpr(e.base, '__delitem__') - m.line = s.line - c = CallExpr(m, [e.index], [nodes.ARG_POS], [None]) - c.line = s.line - return c.accept(self) - else: - def flatten(t: Node) -> List[Node]: - """Flatten a nested sequence of tuples/lists into one list of nodes.""" - if isinstance(t, TupleExpr) or isinstance(t, ListExpr): - t = cast(Union[TupleExpr, ListExpr], t) - return [b for a in t.items for b in flatten(a)] - else: - return [t] - - s.expr.accept(self) - for elt in flatten(s.expr): - if isinstance(elt, NameExpr): - self.binder.assign_type(elt, - DeletedType(source=elt.name), - self.binder.get_declaration(elt), - self.typing_mode_weak()) - return None - - def visit_decorator(self, e: Decorator) -> Type: - for d in e.decorators: - if isinstance(d, RefExpr): - if d.fullname == 'typing.no_type_check': - e.var.type = AnyType() - e.var.is_ready = True - return NoneTyp() - - e.func.accept(self) - sig = self.function_type(e.func) # type: Type - # Process decorators from the inside out. - for i in range(len(e.decorators)): - n = len(e.decorators) - 1 - i - d = e.decorators[n] - if isinstance(d, NameExpr) and d.fullname == 'typing.overload': - self.fail('Single overload definition, multiple required', e) - continue - dec = self.accept(d) - temp = self.temp_node(sig) - sig, t2 = self.expr_checker.check_call(dec, [temp], - [nodes.ARG_POS], e) - sig = cast(FunctionLike, sig) - sig = set_callable_name(sig, e.func) - e.var.type = sig - e.var.is_ready = True - if e.func.is_property: - self.check_incompatible_property_override(e) - - def check_incompatible_property_override(self, e: Decorator) -> None: - if not e.var.is_settable_property and e.func.info is not None: - name = e.func.name() - for base in e.func.info.mro[1:]: - base_attr = base.names.get(name) - if not base_attr: - continue - if (isinstance(base_attr.node, OverloadedFuncDef) and - base_attr.node.is_property and - base_attr.node.items[0].var.is_settable_property): - self.fail(messages.READ_ONLY_PROPERTY_OVERRIDES_READ_WRITE, e) - - def visit_with_stmt(self, s: WithStmt) -> Type: - echk = self.expr_checker - for expr, target in zip(s.expr, s.target): - ctx = self.accept(expr) - enter = echk.analyze_external_member_access('__enter__', ctx, expr) - obj = echk.check_call(enter, [], [], expr)[0] - if target: - self.check_assignment(target, self.temp_node(obj, expr)) - exit = echk.analyze_external_member_access('__exit__', ctx, expr) - arg = self.temp_node(AnyType(), expr) - echk.check_call(exit, [arg] * 3, [nodes.ARG_POS] * 3, expr) - self.accept(s.body) - - def visit_print_stmt(self, s: PrintStmt) -> Type: - for arg in s.args: - self.accept(arg) - if s.target: - target_type = self.accept(s.target) - if not isinstance(target_type, NoneTyp): - # TODO: Also verify the type of 'write'. - self.expr_checker.analyze_external_member_access('write', target_type, s.target) - - # - # Expressions - # - - def visit_name_expr(self, e: NameExpr) -> Type: - return self.expr_checker.visit_name_expr(e) - - def visit_call_expr(self, e: CallExpr) -> Type: - return self.expr_checker.visit_call_expr(e) - - def visit_yield_from_expr(self, e: YieldFromExpr) -> Type: - return_type = self.return_types[-1] - subexpr_type = self.accept(e.expr, return_type) - iter_type = None # type: Type - - # Check that the expr is an instance of Iterable and get the type of the iterator produced - # by __iter__. - if isinstance(subexpr_type, AnyType): - iter_type = AnyType() - elif (isinstance(subexpr_type, Instance) and - is_subtype(subexpr_type, self.named_type('typing.Iterable'))): - iter_method_type = self.expr_checker.analyze_external_member_access( - '__iter__', - subexpr_type, - AnyType()) - - generic_generator_type = self.named_generic_type('typing.Generator', - [AnyType(), AnyType(), AnyType()]) - iter_type, _ = self.expr_checker.check_call(iter_method_type, [], [], - context=generic_generator_type) - else: - self.msg.yield_from_invalid_operand_type(subexpr_type, e) - iter_type = AnyType() - - # Check that the iterator's item type matches the type yielded by the Generator function - # containing this `yield from` expression. - expected_item_type = self.get_generator_yield_type(return_type) - actual_item_type = self.get_generator_yield_type(iter_type) - - self.check_subtype(actual_item_type, expected_item_type, e, - messages.INCOMPATIBLE_TYPES_IN_YIELD_FROM, - 'actual type', 'expected type') - - # Determine the type of the entire yield from expression. - if (isinstance(iter_type, Instance) and - iter_type.type.fullname() == 'typing.Generator'): - return self.get_generator_return_type(iter_type) - else: - # Non-Generators don't return anything from `yield from` expressions. - return Void() - - def visit_member_expr(self, e: MemberExpr) -> Type: - return self.expr_checker.visit_member_expr(e) - - def visit_break_stmt(self, s: BreakStmt) -> Type: - self.binder.breaking_out = True - self.binder.allow_jump(self.binder.loop_frames[-1] - 1) - return None - - def visit_continue_stmt(self, s: ContinueStmt) -> Type: - self.binder.breaking_out = True - self.binder.allow_jump(self.binder.loop_frames[-1]) - return None - - def visit_int_expr(self, e: IntExpr) -> Type: - return self.expr_checker.visit_int_expr(e) - - def visit_str_expr(self, e: StrExpr) -> Type: - return self.expr_checker.visit_str_expr(e) - - def visit_bytes_expr(self, e: BytesExpr) -> Type: - return self.expr_checker.visit_bytes_expr(e) - - def visit_unicode_expr(self, e: UnicodeExpr) -> Type: - return self.expr_checker.visit_unicode_expr(e) - - def visit_float_expr(self, e: FloatExpr) -> Type: - return self.expr_checker.visit_float_expr(e) - - def visit_complex_expr(self, e: ComplexExpr) -> Type: - return self.expr_checker.visit_complex_expr(e) - - def visit_ellipsis(self, e: EllipsisExpr) -> Type: - return self.expr_checker.visit_ellipsis(e) - - def visit_op_expr(self, e: OpExpr) -> Type: - return self.expr_checker.visit_op_expr(e) - - def visit_comparison_expr(self, e: ComparisonExpr) -> Type: - return self.expr_checker.visit_comparison_expr(e) - - def visit_unary_expr(self, e: UnaryExpr) -> Type: - return self.expr_checker.visit_unary_expr(e) - - def visit_index_expr(self, e: IndexExpr) -> Type: - return self.expr_checker.visit_index_expr(e) - - def visit_cast_expr(self, e: CastExpr) -> Type: - return self.expr_checker.visit_cast_expr(e) - - def visit_reveal_type_expr(self, e: RevealTypeExpr) -> Type: - return self.expr_checker.visit_reveal_type_expr(e) - - def visit_super_expr(self, e: SuperExpr) -> Type: - return self.expr_checker.visit_super_expr(e) - - def visit_type_application(self, e: TypeApplication) -> Type: - return self.expr_checker.visit_type_application(e) - - def visit_type_alias_expr(self, e: TypeAliasExpr) -> Type: - return self.expr_checker.visit_type_alias_expr(e) - - def visit_type_var_expr(self, e: TypeVarExpr) -> Type: - # TODO: Perhaps return a special type used for type variables only? - return AnyType() - - def visit_namedtuple_expr(self, e: NamedTupleExpr) -> Type: - # TODO: Perhaps return a type object type? - return AnyType() - - def visit_list_expr(self, e: ListExpr) -> Type: - return self.expr_checker.visit_list_expr(e) - - def visit_set_expr(self, e: SetExpr) -> Type: - return self.expr_checker.visit_set_expr(e) - - def visit_tuple_expr(self, e: TupleExpr) -> Type: - return self.expr_checker.visit_tuple_expr(e) - - def visit_dict_expr(self, e: DictExpr) -> Type: - return self.expr_checker.visit_dict_expr(e) - - def visit_slice_expr(self, e: SliceExpr) -> Type: - return self.expr_checker.visit_slice_expr(e) - - def visit_func_expr(self, e: FuncExpr) -> Type: - return self.expr_checker.visit_func_expr(e) - - def visit_list_comprehension(self, e: ListComprehension) -> Type: - return self.expr_checker.visit_list_comprehension(e) - - def visit_set_comprehension(self, e: SetComprehension) -> Type: - return self.expr_checker.visit_set_comprehension(e) - - def visit_generator_expr(self, e: GeneratorExpr) -> Type: - return self.expr_checker.visit_generator_expr(e) - - def visit_dictionary_comprehension(self, e: DictionaryComprehension) -> Type: - return self.expr_checker.visit_dictionary_comprehension(e) - - def visit_temp_node(self, e: TempNode) -> Type: - return e.type - - def visit_conditional_expr(self, e: ConditionalExpr) -> Type: - return self.expr_checker.visit_conditional_expr(e) - - def visit_backquote_expr(self, e: BackquoteExpr) -> Type: - return self.expr_checker.visit_backquote_expr(e) - - def visit_yield_expr(self, e: YieldExpr) -> Type: - return_type = self.return_types[-1] - expected_item_type = self.get_generator_yield_type(return_type) - if e.expr is None: - if (not (isinstance(expected_item_type, Void) or - isinstance(expected_item_type, AnyType)) - and self.typing_mode_full()): - self.fail(messages.YIELD_VALUE_EXPECTED, e) - else: - actual_item_type = self.accept(e.expr, expected_item_type) - self.check_subtype(actual_item_type, expected_item_type, e, - messages.INCOMPATIBLE_TYPES_IN_YIELD, - 'actual type', 'expected type') - return self.get_generator_receive_type(return_type) - - # - # Helpers - # - - def check_subtype(self, subtype: Type, supertype: Type, context: Context, - msg: str = messages.INCOMPATIBLE_TYPES, - subtype_label: str = None, - supertype_label: str = None) -> None: - """Generate an error if the subtype is not compatible with - supertype.""" - if not is_subtype(subtype, supertype): - if isinstance(subtype, Void): - self.msg.does_not_return_value(subtype, context) - else: - extra_info = [] # type: List[str] - if subtype_label is not None or supertype_label is not None: - subtype_str, supertype_str = self.msg.format_distinctly(subtype, supertype) - if subtype_label is not None: - extra_info.append(subtype_label + ' ' + subtype_str) - if supertype_label is not None: - extra_info.append(supertype_label + ' ' + supertype_str) - if extra_info: - msg += ' (' + ', '.join(extra_info) + ')' - self.fail(msg, context) - - def named_type(self, name: str) -> Instance: - """Return an instance type with type given by the name and no - type arguments. For example, named_type('builtins.object') - produces the object type. - """ - # Assume that the name refers to a type. - sym = self.lookup_qualified(name) - return Instance(cast(TypeInfo, sym.node), []) - - def named_generic_type(self, name: str, args: List[Type]) -> Instance: - """Return an instance with the given name and type arguments. - - Assume that the number of arguments is correct. Assume that - the name refers to a compatible generic type. - """ - return Instance(self.lookup_typeinfo(name), args) - - def lookup_typeinfo(self, fullname: str) -> TypeInfo: - # Assume that the name refers to a class. - sym = self.lookup_qualified(fullname) - return cast(TypeInfo, sym.node) - - def type_type(self) -> Instance: - """Return instance type 'type'.""" - return self.named_type('builtins.type') - - def object_type(self) -> Instance: - """Return instance type 'object'.""" - return self.named_type('builtins.object') - - def bool_type(self) -> Instance: - """Return instance type 'bool'.""" - return self.named_type('builtins.bool') - - def str_type(self) -> Instance: - """Return instance type 'str'.""" - return self.named_type('builtins.str') - - def check_type_equivalency(self, t1: Type, t2: Type, node: Context, - msg: str = messages.INCOMPATIBLE_TYPES) -> None: - """Generate an error if the types are not equivalent. The - dynamic type is equivalent with all types. - """ - if not is_equivalent(t1, t2): - self.fail(msg, node) - - def store_type(self, node: Node, typ: Type) -> None: - """Store the type of a node in the type map.""" - self.type_map[node] = typ - - def typing_mode_none(self) -> bool: - if self.is_dynamic_function() and not self.options.check_untyped_defs: - return not self.weak_opts - elif self.function_stack: - return False - else: - return False - - def typing_mode_weak(self) -> bool: - if self.is_dynamic_function() and not self.options.check_untyped_defs: - return bool(self.weak_opts) - elif self.function_stack: - return False - else: - return 'global' in self.weak_opts - - def typing_mode_full(self) -> bool: - if self.is_dynamic_function() and not self.options.check_untyped_defs: - return False - elif self.function_stack: - return True - else: - return 'global' not in self.weak_opts - - def is_dynamic_function(self) -> bool: - return len(self.dynamic_funcs) > 0 and self.dynamic_funcs[-1] - - def lookup(self, name: str, kind: int) -> SymbolTableNode: - """Look up a definition from the symbol table with the given name. - TODO remove kind argument - """ - if name in self.globals: - return self.globals[name] - else: - b = self.globals.get('__builtins__', None) - if b: - table = cast(MypyFile, b.node).names - if name in table: - return table[name] - raise KeyError('Failed lookup: {}'.format(name)) - - def lookup_qualified(self, name: str) -> SymbolTableNode: - if '.' not in name: - return self.lookup(name, GDEF) # FIX kind - else: - parts = name.split('.') - n = self.modules[parts[0]] - for i in range(1, len(parts) - 1): - n = cast(MypyFile, n.names.get(parts[i], None).node) - return n.names[parts[-1]] - - def enter_partial_types(self) -> None: - """Push a new scope for collecting partial types.""" - self.partial_types.append({}) - - def leave_partial_types(self) -> None: - """Pop partial type scope. - - Also report errors for variables which still have partial - types, i.e. we couldn't infer a complete type. - """ - partial_types = self.partial_types.pop() - if not self.current_node_deferred: - for var, context in partial_types.items(): - if experiments.STRICT_OPTIONAL and cast(PartialType, var.type).type is None: - # None partial type: assume variable is intended to have type None - var.type = NoneTyp() - else: - self.msg.fail(messages.NEED_ANNOTATION_FOR_VAR, context) - var.type = AnyType() - - def find_partial_types(self, var: Var) -> Optional[Dict[Var, Context]]: - for partial_types in reversed(self.partial_types): - if var in partial_types: - return partial_types - return None - - def is_within_function(self) -> bool: - """Are we currently type checking within a function? - - I.e. not at class body or at the top level. - """ - return self.return_types != [] - - def check_not_void(self, typ: Type, context: Context) -> None: - """Generate an error if the type is Void.""" - if isinstance(typ, Void): - self.msg.does_not_return_value(typ, context) - - def temp_node(self, t: Type, context: Context = None) -> Node: - """Create a temporary node with the given, fixed type.""" - temp = TempNode(t) - if context: - temp.set_line(context.get_line()) - return temp - - def fail(self, msg: str, context: Context) -> None: - """Produce an error message.""" - self.msg.fail(msg, context) - - def iterable_item_type(self, instance: Instance) -> Type: - iterable = map_instance_to_supertype( - instance, - self.lookup_typeinfo('typing.Iterable')) - return iterable.args[0] - - def function_type(self, func: FuncBase) -> FunctionLike: - return function_type(func, self.named_type('builtins.function')) - - def method_type(self, func: FuncBase) -> FunctionLike: - return method_type_with_fallback(func, self.named_type('builtins.function')) - - -# Data structure returned by find_isinstance_check representing -# information learned from the truth or falsehood of a condition. The -# dict maps nodes representing expressions like 'a[0].x' to their -# refined types under the assumption that the condition has a -# particular truth value. A value of None means that the condition can -# never have that truth value. - -# NB: The keys of this dict are nodes in the original source program, -# which are compared by reference equality--effectively, being *the -# same* expression of the program, not just two identical expressions -# (such as two references to the same variable). TODO: it would -# probably be better to have the dict keyed by the nodes' literal_hash -# field instead. - -# NB: This should be `TypeMap = Optional[Dict[Node, Type]]`! -# But see https://github.com/python/mypy/issues/1637 -TypeMap = Dict[Node, Type] - - -def conditional_type_map(expr: Node, - current_type: Optional[Type], - proposed_type: Optional[Type], - *, - weak: bool = False - ) -> Tuple[TypeMap, TypeMap]: - """Takes in an expression, the current type of the expression, and a - proposed type of that expression. - - Returns a 2-tuple: The first element is a map from the expression to - the proposed type, if the expression can be the proposed type. The - second element is a map from the expression to the type it would hold - if it was not the proposed type, if any.""" - if proposed_type: - if current_type: - if is_proper_subtype(current_type, proposed_type): - return {expr: proposed_type}, None - elif not is_overlapping_types(current_type, proposed_type): - return None, {expr: current_type} - else: - remaining_type = restrict_subtype_away(current_type, proposed_type) - return {expr: proposed_type}, {expr: remaining_type} - else: - return {expr: proposed_type}, {} - else: - # An isinstance check, but we don't understand the type - if weak: - return {expr: AnyType()}, {expr: current_type} - else: - return {}, {} - - -def is_literal_none(n: Node) -> bool: - return isinstance(n, NameExpr) and n.fullname == 'builtins.None' - - -def and_conditional_maps(m1: TypeMap, m2: TypeMap) -> TypeMap: - """Calculate what information we can learn from the truth of (e1 and e2) - in terms of the information that we can learn from the truth of e1 and - the truth of e2. - """ - - if m1 is None or m2 is None: - # One of the conditions can never be true. - return None - # Both conditions can be true; combine the information. Anything - # we learn from either conditions's truth is valid. If the same - # expression's type is refined by both conditions, we somewhat - # arbitrarily give precedence to m2. (In the future, we could use - # an intersection type.) - result = m2.copy() - m2_keys = set(n2.literal_hash for n2 in m2) - for n1 in m1: - if n1.literal_hash not in m2_keys: - result[n1] = m1[n1] - return result - - -def or_conditional_maps(m1: TypeMap, m2: TypeMap) -> TypeMap: - """Calculate what information we can learn from the truth of (e1 or e2) - in terms of the information that we can learn from the truth of e1 and - the truth of e2. - """ - - if m1 is None: - return m2 - if m2 is None: - return m1 - # Both conditions can be true. Combine information about - # expressions whose type is refined by both conditions. (We do not - # learn anything about expressions whose type is refined by only - # one condition.) - result = {} - for n1 in m1: - for n2 in m2: - if n1.literal_hash == n2.literal_hash: - result[n1] = UnionType.make_simplified_union([m1[n1], m2[n2]]) - return result - - -def find_isinstance_check(node: Node, - type_map: Dict[Node, Type], - weak: bool=False - ) -> Tuple[TypeMap, TypeMap]: - """Find any isinstance checks (within a chain of ands). Includes - implicit and explicit checks for None. - - Return value is a map of variables to their types if the condition - is true and a map of variables to their types if the condition is false. - - If either of the values in the tuple is None, then that particular - branch can never occur. - - Guaranteed to not return None, None. (But may return {}, {}) - """ - if isinstance(node, CallExpr): - if refers_to_fullname(node.callee, 'builtins.isinstance'): - expr = node.args[0] - if expr.literal == LITERAL_TYPE: - vartype = type_map[expr] - type = get_isinstance_type(node.args[1], type_map) - return conditional_type_map(expr, vartype, type, weak=weak) - elif (isinstance(node, ComparisonExpr) and any(is_literal_none(n) for n in node.operands) and - experiments.STRICT_OPTIONAL): - # Check for `x is None` and `x is not None`. - is_not = node.operators == ['is not'] - if is_not or node.operators == ['is']: - if_vars = {} # type: Dict[Node, Type] - else_vars = {} # type: Dict[Node, Type] - for expr in node.operands: - if expr.literal == LITERAL_TYPE and not is_literal_none(expr) and expr in type_map: - # This should only be true at most once: there should be - # two elements in node.operands, and at least one of them - # should represent a None. - vartype = type_map[expr] - if_vars, else_vars = conditional_type_map(expr, vartype, NoneTyp(), weak=weak) - break - - if is_not: - if_vars, else_vars = else_vars, if_vars - return if_vars, else_vars - elif isinstance(node, RefExpr) and experiments.STRICT_OPTIONAL: - # The type could be falsy, so we can't deduce anything new about the else branch - vartype = type_map[node] - _, if_vars = conditional_type_map(node, vartype, NoneTyp(), weak=weak) - return if_vars, {} - elif isinstance(node, OpExpr) and node.op == 'and': - left_if_vars, left_else_vars = find_isinstance_check( - node.left, - type_map, - weak, - ) - - right_if_vars, right_else_vars = find_isinstance_check( - node.right, - type_map, - weak, - ) - - # (e1 and e2) is true if both e1 and e2 are true, - # and false if at least one of e1 and e2 is false. - return (and_conditional_maps(left_if_vars, right_if_vars), - or_conditional_maps(left_else_vars, right_else_vars)) - elif isinstance(node, OpExpr) and node.op == 'or': - left_if_vars, left_else_vars = find_isinstance_check( - node.left, - type_map, - weak, - ) - - right_if_vars, right_else_vars = find_isinstance_check( - node.right, - type_map, - weak, - ) - - # (e1 or e2) is true if at least one of e1 or e2 is true, - # and false if both e1 and e2 are false. - return (or_conditional_maps(left_if_vars, right_if_vars), - and_conditional_maps(left_else_vars, right_else_vars)) - elif isinstance(node, UnaryExpr) and node.op == 'not': - left, right = find_isinstance_check(node.expr, type_map, weak) - return right, left - - # Not a supported isinstance check - return {}, {} - - -def get_isinstance_type(node: Node, type_map: Dict[Node, Type]) -> Type: - type = type_map[node] - - if isinstance(type, TupleType): - all_types = type.items - else: - all_types = [type] - - types = [] # type: List[Type] - - for type in all_types: - if isinstance(type, FunctionLike): - if type.is_type_obj(): - # Type variables may be present -- erase them, which is the best - # we can do (outside disallowing them here). - type = erase_typevars(type.items()[0].ret_type) - - types.append(type) - - if len(types) == 0: - return None - elif len(types) == 1: - return types[0] - else: - return UnionType(types) - - -def expand_node(defn: Node, map: Dict[TypeVarId, Type]) -> Node: - visitor = TypeTransformVisitor(map) - return defn.accept(visitor) - - -def expand_func(defn: FuncItem, map: Dict[TypeVarId, Type]) -> FuncItem: - return cast(FuncItem, expand_node(defn, map)) - - -class TypeTransformVisitor(TransformVisitor): - def __init__(self, map: Dict[TypeVarId, Type]) -> None: - super().__init__() - self.map = map - - def type(self, type: Type) -> Type: - return expand_type(type, self.map) - - -def is_unsafe_overlapping_signatures(signature: Type, other: Type) -> bool: - """Check if two signatures may be unsafely overlapping. - - Two signatures s and t are overlapping if both can be valid for the same - statically typed values and the return types are incompatible. - - Assume calls are first checked against 'signature', then against 'other'. - Thus if 'signature' is more general than 'other', there is no unsafe - overlapping. - - TODO If argument types vary covariantly, the return type may vary - covariantly as well. - """ - if isinstance(signature, CallableType): - if isinstance(other, CallableType): - # TODO varargs - # TODO keyword args - # TODO erasure - # TODO allow to vary covariantly - # Check if the argument counts are overlapping. - min_args = max(signature.min_args, other.min_args) - max_args = min(len(signature.arg_types), len(other.arg_types)) - if min_args > max_args: - # Argument counts are not overlapping. - return False - # Signatures are overlapping iff if they are overlapping for the - # smallest common argument count. - for i in range(min_args): - t1 = signature.arg_types[i] - t2 = other.arg_types[i] - if not is_overlapping_types(t1, t2): - return False - # All arguments types for the smallest common argument count are - # overlapping => the signature is overlapping. The overlapping is - # safe if the return types are identical. - if is_same_type(signature.ret_type, other.ret_type): - return False - # If the first signature has more general argument types, the - # latter will never be called - if is_more_general_arg_prefix(signature, other): - return False - return not is_more_precise_signature(signature, other) - return True - - -def is_more_general_arg_prefix(t: FunctionLike, s: FunctionLike) -> bool: - """Does t have wider arguments than s?""" - # TODO should an overload with additional items be allowed to be more - # general than one with fewer items (or just one item)? - # TODO check argument kinds - if isinstance(t, CallableType): - if isinstance(s, CallableType): - return all(is_proper_subtype(args, argt) - for argt, args in zip(t.arg_types, s.arg_types)) - elif isinstance(t, FunctionLike): - if isinstance(s, FunctionLike): - if len(t.items()) == len(s.items()): - return all(is_same_arg_prefix(items, itemt) - for items, itemt in zip(t.items(), s.items())) - return False - - -def is_same_arg_prefix(t: CallableType, s: CallableType) -> bool: - # TODO check argument kinds - return all(is_same_type(argt, args) - for argt, args in zip(t.arg_types, s.arg_types)) - - -def is_more_precise_signature(t: CallableType, s: CallableType) -> bool: - """Is t more precise than s? - - A signature t is more precise than s if all argument types and the return - type of t are more precise than the corresponding types in s. - - Assume that the argument kinds and names are compatible, and that the - argument counts are overlapping. - """ - # TODO generic function types - # Only consider the common prefix of argument types. - for argt, args in zip(t.arg_types, s.arg_types): - if not is_more_precise(argt, args): - return False - return is_more_precise(t.ret_type, s.ret_type) - - -def infer_operator_assignment_method(type: Type, operator: str) -> str: - """Return the method used for operator assignment for given value type. - - For example, if operator is '+', return '__iadd__' or '__add__' depending - on which method is supported by the type. - """ - method = nodes.op_methods[operator] - if isinstance(type, Instance): - if operator in nodes.ops_with_inplace_method: - inplace = '__i' + method[2:] - if type.type.has_readable_member(inplace): - method = inplace - return method - - -def is_valid_inferred_type(typ: Type) -> bool: - """Is an inferred type valid? - - Examples of invalid types include the None type or a type with a None component. - """ - if is_same_type(typ, NoneTyp()): - # With strict Optional checking, we *may* eventually infer NoneTyp, but - # we only do that if we can't infer a specific Optional type. This - # resolution happens in leave_partial_types when we pop a partial types - # scope. - return False - if is_same_type(typ, UninhabitedType()): - return False - elif isinstance(typ, Instance): - for arg in typ.args: - if not is_valid_inferred_type(arg): - return False - elif isinstance(typ, TupleType): - for item in typ.items: - if not is_valid_inferred_type(item): - return False - return True +"""Mypy type checker.""" + +import itertools +import contextlib +import os +import os.path + +from typing import ( + Any, Dict, Set, List, cast, Tuple, TypeVar, Union, Optional, NamedTuple +) + +from mypy.errors import Errors, report_internal_error +from mypy.nodes import ( + SymbolTable, Node, MypyFile, Var, + OverloadedFuncDef, FuncDef, FuncItem, FuncBase, TypeInfo, + ClassDef, GDEF, Block, AssignmentStmt, NameExpr, MemberExpr, IndexExpr, + TupleExpr, ListExpr, ExpressionStmt, ReturnStmt, IfStmt, + WhileStmt, OperatorAssignmentStmt, WithStmt, AssertStmt, + RaiseStmt, TryStmt, ForStmt, DelStmt, CallExpr, IntExpr, StrExpr, + BytesExpr, UnicodeExpr, FloatExpr, OpExpr, UnaryExpr, CastExpr, RevealTypeExpr, SuperExpr, + TypeApplication, DictExpr, SliceExpr, FuncExpr, TempNode, SymbolTableNode, + Context, ListComprehension, ConditionalExpr, GeneratorExpr, + Decorator, SetExpr, TypeVarExpr, PrintStmt, + LITERAL_TYPE, BreakStmt, ContinueStmt, ComparisonExpr, StarExpr, + YieldFromExpr, NamedTupleExpr, SetComprehension, + DictionaryComprehension, ComplexExpr, EllipsisExpr, TypeAliasExpr, + RefExpr, YieldExpr, BackquoteExpr, ImportFrom, ImportAll, ImportBase, + CONTRAVARIANT, COVARIANT +) +from mypy.nodes import function_type, method_type, method_type_with_fallback +from mypy import nodes +from mypy.types import ( + Type, AnyType, CallableType, Void, FunctionLike, Overloaded, TupleType, + Instance, NoneTyp, ErrorType, strip_type, + UnionType, TypeVarId, TypeVarType, PartialType, DeletedType, UninhabitedType +) +from mypy.sametypes import is_same_type +from mypy.messages import MessageBuilder +import mypy.checkexpr +from mypy.checkmember import map_type_from_supertype +from mypy import defaults +from mypy import messages +from mypy.subtypes import ( + is_subtype, is_equivalent, is_proper_subtype, + is_more_precise, restrict_subtype_away +) +from mypy.maptype import map_instance_to_supertype +from mypy.semanal import self_type, set_callable_name, refers_to_fullname +from mypy.erasetype import erase_typevars +from mypy.expandtype import expand_type_by_instance, expand_type +from mypy.visitor import NodeVisitor +from mypy.join import join_types +from mypy.treetransform import TransformVisitor +from mypy.meet import meet_simple, nearest_builtin_ancestor, is_overlapping_types +from mypy.binder import ConditionalTypeBinder +from mypy.options import Options + +from mypy import experiments + + +T = TypeVar('T') + + +# A node which is postponed to be type checked during the next pass. +DeferredNode = NamedTuple( + 'DeferredNode', + [ + ('node', Node), + ('context_type_name', Optional[str]), # Name of the surrounding class (for error messages) + ]) + + +class TypeChecker(NodeVisitor[Type]): + """Mypy type checker. + + Type check mypy source files that have been semantically analyzed. + """ + + # Are we type checking a stub? + is_stub = False + # Error message reporter + errors = None # type: Errors + # Utility for generating messages + msg = None # type: MessageBuilder + # Types of type checked nodes + type_map = None # type: Dict[Node, Type] + + # Helper for managing conditional types + binder = None # type: ConditionalTypeBinder + # Helper for type checking expressions + expr_checker = None # type: mypy.checkexpr.ExpressionChecker + + # Stack of function return types + return_types = None # type: List[Type] + # Type context for type inference + type_context = None # type: List[Type] + # Flags; true for dynamically typed functions + dynamic_funcs = None # type: List[bool] + # Stack of functions being type checked + function_stack = None # type: List[FuncItem] + # Do weak type checking in this file + weak_opts = set() # type: Set[str] + # Stack of collections of variables with partial types + partial_types = None # type: List[Dict[Var, Context]] + globals = None # type: SymbolTable + modules = None # type: Dict[str, MypyFile] + # Nodes that couldn't be checked because some types weren't available. We'll run + # another pass and try these again. + deferred_nodes = None # type: List[DeferredNode] + # Type checking pass number (0 = first pass) + pass_num = 0 + # Have we deferred the current function? If yes, don't infer additional + # types during this pass within the function. + current_node_deferred = False + is_typeshed_stub = False + options = None # type: Options + + def __init__(self, errors: Errors, modules: Dict[str, MypyFile], options: Options) -> None: + """Construct a type checker. + + Use errors to report type check errors. + """ + self.errors = errors + self.modules = modules + self.options = options + self.msg = MessageBuilder(errors, modules) + self.type_map = {} + self.binder = ConditionalTypeBinder() + self.expr_checker = mypy.checkexpr.ExpressionChecker(self, self.msg) + self.return_types = [] + self.type_context = [] + self.dynamic_funcs = [] + self.function_stack = [] + self.weak_opts = set() # type: Set[str] + self.partial_types = [] + self.deferred_nodes = [] + self.pass_num = 0 + self.current_node_deferred = False + + def visit_file(self, file_node: MypyFile, path: str) -> None: + """Type check a mypy file with the given path.""" + self.pass_num = 0 + self.is_stub = file_node.is_stub + self.errors.set_file(path) + self.globals = file_node.names + self.weak_opts = file_node.weak_opts + self.enter_partial_types() + # gross, but no other clear way to tell + self.is_typeshed_stub = self.is_stub and 'typeshed' in os.path.normpath(path).split(os.sep) + + for d in file_node.defs: + self.accept(d) + + self.leave_partial_types() + + if self.deferred_nodes: + self.check_second_pass() + + self.current_node_deferred = False + + all_ = self.globals.get('__all__') + if all_ is not None and all_.type is not None: + seq_str = self.named_generic_type('typing.Sequence', + [self.named_type('builtins.str')]) + if not is_subtype(all_.type, seq_str): + str_seq_s, all_s = self.msg.format_distinctly(seq_str, all_.type) + self.fail(messages.ALL_MUST_BE_SEQ_STR.format(str_seq_s, all_s), + all_.node) + + def check_second_pass(self): + """Run second pass of type checking which goes through deferred nodes.""" + self.pass_num = 1 + for node, type_name in self.deferred_nodes: + if type_name: + self.errors.push_type(type_name) + self.accept(node) + if type_name: + self.errors.pop_type() + self.deferred_nodes = [] + + def handle_cannot_determine_type(self, name: str, context: Context) -> None: + if self.pass_num == 0 and self.function_stack: + # Don't report an error yet. Just defer. + node = self.function_stack[-1] + if self.errors.type_name: + type_name = self.errors.type_name[-1] + else: + type_name = None + self.deferred_nodes.append(DeferredNode(node, type_name)) + # Set a marker so that we won't infer additional types in this + # function. Any inferred types could be bogus, because there's at + # least one type that we don't know. + self.current_node_deferred = True + else: + self.msg.cannot_determine_type(name, context) + + def accept(self, node: Node, type_context: Type = None) -> Type: + """Type check a node in the given type context.""" + self.type_context.append(type_context) + try: + typ = node.accept(self) + except Exception as err: + report_internal_error(err, self.errors.file, node.line) + self.type_context.pop() + self.store_type(node, typ) + if self.typing_mode_none(): + return AnyType() + else: + return typ + + def accept_loop(self, body: Node, else_body: Node = None) -> Type: + """Repeatedly type check a loop body until the frame doesn't change. + + Then check the else_body. + """ + # The outer frame accumulates the results of all iterations + with self.binder.frame_context(1) as outer_frame: + self.binder.push_loop_frame() + while True: + with self.binder.frame_context(1): + # We may skip each iteration + self.binder.options_on_return[-1].append(outer_frame) + self.accept(body) + if not self.binder.last_pop_changed: + break + self.binder.pop_loop_frame() + if else_body: + self.accept(else_body) + + # + # Definitions + # + + def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> Type: + num_abstract = 0 + if defn.is_property: + # HACK: Infer the type of the property. + self.visit_decorator(defn.items[0]) + for fdef in defn.items: + self.check_func_item(fdef.func, name=fdef.func.name()) + if fdef.func.is_abstract: + num_abstract += 1 + if num_abstract not in (0, len(defn.items)): + self.fail(messages.INCONSISTENT_ABSTRACT_OVERLOAD, defn) + if defn.info: + self.check_method_override(defn) + self.check_inplace_operator_method(defn) + self.check_overlapping_overloads(defn) + + def check_overlapping_overloads(self, defn: OverloadedFuncDef) -> None: + for i, item in enumerate(defn.items): + for j, item2 in enumerate(defn.items[i + 1:]): + # TODO overloads involving decorators + sig1 = self.function_type(item.func) + sig2 = self.function_type(item2.func) + if is_unsafe_overlapping_signatures(sig1, sig2): + self.msg.overloaded_signatures_overlap(i + 1, i + j + 2, + item.func) + + def is_generator_return_type(self, typ: Type) -> bool: + return is_subtype(self.named_generic_type('typing.Generator', + [AnyType(), AnyType(), AnyType()]), + typ) + + def get_generator_yield_type(self, return_type: Type) -> Type: + if isinstance(return_type, AnyType): + return AnyType() + elif not self.is_generator_return_type(return_type): + # If the function doesn't have a proper Generator (or superclass) return type, anything + # is permissible. + return AnyType() + elif not isinstance(return_type, Instance): + # Same as above, but written as a separate branch so the typechecker can understand. + return AnyType() + elif return_type.args: + return return_type.args[0] + else: + # If the function's declared supertype of Generator has no type + # parameters (i.e. is `object`), then the yielded values can't + # be accessed so any type is acceptable. + return AnyType() + + def get_generator_receive_type(self, return_type: Type) -> Type: + if isinstance(return_type, AnyType): + return AnyType() + elif not self.is_generator_return_type(return_type): + # If the function doesn't have a proper Generator (or superclass) return type, anything + # is permissible. + return AnyType() + elif not isinstance(return_type, Instance): + # Same as above, but written as a separate branch so the typechecker can understand. + return AnyType() + elif return_type.type.fullname() == 'typing.Generator': + # Generator is the only type which specifies the type of values it can receive. + return return_type.args[1] + else: + # `return_type` is a supertype of Generator, so callers won't be able to send it + # values. + return Void() + + def get_generator_return_type(self, return_type: Type) -> Type: + if isinstance(return_type, AnyType): + return AnyType() + elif not self.is_generator_return_type(return_type): + # If the function doesn't have a proper Generator (or superclass) return type, anything + # is permissible. + return AnyType() + elif not isinstance(return_type, Instance): + # Same as above, but written as a separate branch so the typechecker can understand. + return AnyType() + elif return_type.type.fullname() == 'typing.Generator': + # Generator is the only type which specifies the type of values it returns into + # `yield from` expressions. + return return_type.args[2] + else: + # `return_type` is supertype of Generator, so callers won't be able to see the return + # type when used in a `yield from` expression. + return AnyType() + + def visit_func_def(self, defn: FuncDef) -> Type: + """Type check a function definition.""" + self.check_func_item(defn, name=defn.name()) + if defn.info: + if not defn.is_dynamic(): + self.check_method_override(defn) + self.check_inplace_operator_method(defn) + if defn.original_def: + # Override previous definition. + new_type = self.function_type(defn) + if isinstance(defn.original_def, FuncDef): + # Function definition overrides function definition. + if not is_same_type(new_type, self.function_type(defn.original_def)): + self.msg.incompatible_conditional_function_def(defn) + else: + # Function definition overrides a variable initialized via assignment. + orig_type = defn.original_def.type + if orig_type is None: + # XXX This can be None, as happens in + # test_testcheck_TypeCheckSuite.testRedefinedFunctionInTryWithElse + self.msg.note("Internal mypy error checking function redefinition.", defn) + return None + if isinstance(orig_type, PartialType): + if orig_type.type is None: + # Ah this is a partial type. Give it the type of the function. + var = defn.original_def + partial_types = self.find_partial_types(var) + if partial_types is not None: + var.type = new_type + del partial_types[var] + else: + # Trying to redefine something like partial empty list as function. + self.fail(messages.INCOMPATIBLE_REDEFINITION, defn) + else: + # TODO: Update conditional type binder. + self.check_subtype(new_type, orig_type, defn, + messages.INCOMPATIBLE_REDEFINITION, + 'redefinition with type', + 'original type') + + def check_func_item(self, defn: FuncItem, + type_override: CallableType = None, + name: str = None) -> Type: + """Type check a function. + + If type_override is provided, use it as the function type. + """ + # We may be checking a function definition or an anonymous function. In + # the first case, set up another reference with the precise type. + fdef = None # type: FuncDef + if isinstance(defn, FuncDef): + fdef = defn + + self.function_stack.append(defn) + self.dynamic_funcs.append(defn.is_dynamic() and not type_override) + + if fdef: + self.errors.push_function(fdef.name()) + + self.enter_partial_types() + + typ = self.function_type(defn) + if type_override: + typ = type_override + if isinstance(typ, CallableType): + self.check_func_def(defn, typ, name) + else: + raise RuntimeError('Not supported') + + self.leave_partial_types() + + if fdef: + self.errors.pop_function() + + self.dynamic_funcs.pop() + self.function_stack.pop() + self.current_node_deferred = False + + def check_func_def(self, defn: FuncItem, typ: CallableType, name: str) -> None: + """Type check a function definition.""" + # Expand type variables with value restrictions to ordinary types. + for item, typ in self.expand_typevars(defn, typ): + old_binder = self.binder + self.binder = ConditionalTypeBinder() + with self.binder.frame_context(): + defn.expanded.append(item) + + # We may be checking a function definition or an anonymous + # function. In the first case, set up another reference with the + # precise type. + if isinstance(item, FuncDef): + fdef = item + else: + fdef = None + + if fdef: + # Check if __init__ has an invalid, non-None return type. + if (fdef.info and fdef.name() == '__init__' and + not isinstance(typ.ret_type, Void) and + not self.dynamic_funcs[-1]): + self.fail(messages.INIT_MUST_HAVE_NONE_RETURN_TYPE, + item.type) + + show_untyped = not self.is_typeshed_stub or self.options.warn_incomplete_stub + if self.options.disallow_untyped_defs and show_untyped: + # Check for functions with unspecified/not fully specified types. + def is_implicit_any(t: Type) -> bool: + return isinstance(t, AnyType) and t.implicit + + if fdef.type is None: + self.fail(messages.FUNCTION_TYPE_EXPECTED, fdef) + elif isinstance(fdef.type, CallableType): + if is_implicit_any(fdef.type.ret_type): + self.fail(messages.RETURN_TYPE_EXPECTED, fdef) + if any(is_implicit_any(t) for t in fdef.type.arg_types): + self.fail(messages.ARGUMENT_TYPE_EXPECTED, fdef) + + if name in nodes.reverse_op_method_set: + self.check_reverse_op_method(item, typ, name) + elif name == '__getattr__': + self.check_getattr_method(typ, defn) + + # Refuse contravariant return type variable + if isinstance(typ.ret_type, TypeVarType): + if typ.ret_type.variance == CONTRAVARIANT: + self.fail(messages.RETURN_TYPE_CANNOT_BE_CONTRAVARIANT, + typ.ret_type) + + # Check that Generator functions have the appropriate return type. + if defn.is_generator: + if not self.is_generator_return_type(typ.ret_type): + self.fail(messages.INVALID_RETURN_TYPE_FOR_GENERATOR, typ) + + # Python 2 generators aren't allowed to return values. + if (self.options.python_version[0] == 2 and + isinstance(typ.ret_type, Instance) and + typ.ret_type.type.fullname() == 'typing.Generator'): + if not (isinstance(typ.ret_type.args[2], Void) + or isinstance(typ.ret_type.args[2], AnyType)): + self.fail(messages.INVALID_GENERATOR_RETURN_ITEM_TYPE, typ) + + # Push return type. + self.return_types.append(typ.ret_type) + + # Store argument types. + for i in range(len(typ.arg_types)): + arg_type = typ.arg_types[i] + + # Refuse covariant parameter type variables + if isinstance(arg_type, TypeVarType): + if arg_type.variance == COVARIANT: + self.fail(messages.FUNCTION_PARAMETER_CANNOT_BE_COVARIANT, + arg_type) + + if typ.arg_kinds[i] == nodes.ARG_STAR: + # builtins.tuple[T] is typing.Tuple[T, ...] + arg_type = self.named_generic_type('builtins.tuple', + [arg_type]) + elif typ.arg_kinds[i] == nodes.ARG_STAR2: + arg_type = self.named_generic_type('builtins.dict', + [self.str_type(), + arg_type]) + item.arguments[i].variable.type = arg_type + + # Type check initialization expressions. + for arg in item.arguments: + init = arg.initialization_statement + if init: + self.accept(init) + + # Type check body in a new scope. + with self.binder.frame_context(): + self.accept(item.body) + + self.return_types.pop() + + self.binder = old_binder + + def check_reverse_op_method(self, defn: FuncItem, typ: CallableType, + method: str) -> None: + """Check a reverse operator method such as __radd__.""" + + # This used to check for some very obscure scenario. It now + # just decides whether it's worth calling + # check_overlapping_op_methods(). + + if method in ('__eq__', '__ne__'): + # These are defined for all objects => can't cause trouble. + return + + # With 'Any' or 'object' return type we are happy, since any possible + # return value is valid. + ret_type = typ.ret_type + if isinstance(ret_type, AnyType): + return + if isinstance(ret_type, Instance): + if ret_type.type.fullname() == 'builtins.object': + return + # Plausibly the method could have too few arguments, which would result + # in an error elsewhere. + if len(typ.arg_types) <= 2: + # TODO check self argument kind + + # Check for the issue described above. + arg_type = typ.arg_types[1] + other_method = nodes.normal_from_reverse_op[method] + if isinstance(arg_type, Instance): + if not arg_type.type.has_readable_member(other_method): + return + elif isinstance(arg_type, AnyType): + return + elif isinstance(arg_type, UnionType): + if not arg_type.has_readable_member(other_method): + return + else: + return + + typ2 = self.expr_checker.analyze_external_member_access( + other_method, arg_type, defn) + self.check_overlapping_op_methods( + typ, method, defn.info, + typ2, other_method, cast(Instance, arg_type), + defn) + + def check_overlapping_op_methods(self, + reverse_type: CallableType, + reverse_name: str, + reverse_class: TypeInfo, + forward_type: Type, + forward_name: str, + forward_base: Instance, + context: Context) -> None: + """Check for overlapping method and reverse method signatures. + + Assume reverse method has valid argument count and kinds. + """ + + # Reverse operator method that overlaps unsafely with the + # forward operator method can result in type unsafety. This is + # similar to overlapping overload variants. + # + # This example illustrates the issue: + # + # class X: pass + # class A: + # def __add__(self, x: X) -> int: + # if isinstance(x, X): + # return 1 + # return NotImplemented + # class B: + # def __radd__(self, x: A) -> str: return 'x' + # class C(X, B): pass + # def f(b: B) -> None: + # A() + b # Result is 1, even though static type seems to be str! + # f(C()) + # + # The reason for the problem is that B and X are overlapping + # types, and the return types are different. Also, if the type + # of x in __radd__ would not be A, the methods could be + # non-overlapping. + + if isinstance(forward_type, CallableType): + # TODO check argument kinds + if len(forward_type.arg_types) < 1: + # Not a valid operator method -- can't succeed anyway. + return + + # Construct normalized function signatures corresponding to the + # operator methods. The first argument is the left operand and the + # second operand is the right argument -- we switch the order of + # the arguments of the reverse method. + forward_tweaked = CallableType( + [forward_base, forward_type.arg_types[0]], + [nodes.ARG_POS] * 2, + [None] * 2, + forward_type.ret_type, + forward_type.fallback, + name=forward_type.name) + reverse_args = reverse_type.arg_types + reverse_tweaked = CallableType( + [reverse_args[1], reverse_args[0]], + [nodes.ARG_POS] * 2, + [None] * 2, + reverse_type.ret_type, + fallback=self.named_type('builtins.function'), + name=reverse_type.name) + + if is_unsafe_overlapping_signatures(forward_tweaked, + reverse_tweaked): + self.msg.operator_method_signatures_overlap( + reverse_class.name(), reverse_name, + forward_base.type.name(), forward_name, context) + elif isinstance(forward_type, Overloaded): + for item in forward_type.items(): + self.check_overlapping_op_methods( + reverse_type, reverse_name, reverse_class, + item, forward_name, forward_base, context) + elif not isinstance(forward_type, AnyType): + self.msg.forward_operator_not_callable(forward_name, context) + + def check_inplace_operator_method(self, defn: FuncBase) -> None: + """Check an inplace operator method such as __iadd__. + + They cannot arbitrarily overlap with __add__. + """ + method = defn.name() + if method not in nodes.inplace_operator_methods: + return + typ = self.method_type(defn) + cls = defn.info + other_method = '__' + method[3:] + if cls.has_readable_member(other_method): + instance = self_type(cls) + typ2 = self.expr_checker.analyze_external_member_access( + other_method, instance, defn) + fail = False + if isinstance(typ2, FunctionLike): + if not is_more_general_arg_prefix(typ, typ2): + fail = True + else: + # TODO overloads + fail = True + if fail: + self.msg.signatures_incompatible(method, other_method, defn) + + def check_getattr_method(self, typ: CallableType, context: Context) -> None: + method_type = CallableType([AnyType(), self.named_type('builtins.str')], + [nodes.ARG_POS, nodes.ARG_POS], + [None], + AnyType(), + self.named_type('builtins.function')) + if not is_subtype(typ, method_type): + self.msg.invalid_signature(typ, context) + + def expand_typevars(self, defn: FuncItem, + typ: CallableType) -> List[Tuple[FuncItem, CallableType]]: + # TODO use generator + subst = [] # type: List[List[Tuple[TypeVarId, Type]]] + tvars = typ.variables or [] + tvars = tvars[:] + if defn.info: + # Class type variables + tvars += defn.info.defn.type_vars or [] + for tvar in tvars: + if tvar.values: + subst.append([(tvar.id, value) + for value in tvar.values]) + if subst: + result = [] # type: List[Tuple[FuncItem, CallableType]] + for substitutions in itertools.product(*subst): + mapping = dict(substitutions) + expanded = cast(CallableType, expand_type(typ, mapping)) + result.append((expand_func(defn, mapping), expanded)) + return result + else: + return [(defn, typ)] + + def check_method_override(self, defn: FuncBase) -> None: + """Check if function definition is compatible with base classes.""" + # Check against definitions in base classes. + for base in defn.info.mro[1:]: + self.check_method_or_accessor_override_for_base(defn, base) + + def check_method_or_accessor_override_for_base(self, defn: FuncBase, + base: TypeInfo) -> None: + """Check if method definition is compatible with a base class.""" + if base: + name = defn.name() + if name not in ('__init__', '__new__'): + # Check method override (__init__ and __new__ are special). + self.check_method_override_for_base_with_name(defn, name, base) + if name in nodes.inplace_operator_methods: + # Figure out the name of the corresponding operator method. + method = '__' + name[3:] + # An inplace operator method such as __iadd__ might not be + # always introduced safely if a base class defined __add__. + # TODO can't come up with an example where this is + # necessary; now it's "just in case" + self.check_method_override_for_base_with_name(defn, method, + base) + + def check_method_override_for_base_with_name( + self, defn: FuncBase, name: str, base: TypeInfo) -> None: + base_attr = base.names.get(name) + if base_attr: + # The name of the method is defined in the base class. + + # Construct the type of the overriding method. + typ = self.method_type(defn) + # Map the overridden method type to subtype context so that + # it can be checked for compatibility. + original_type = base_attr.type + if original_type is None and isinstance(base_attr.node, + FuncDef): + original_type = self.function_type(base_attr.node) + if isinstance(original_type, FunctionLike): + original = map_type_from_supertype( + method_type(original_type), + defn.info, base) + # Check that the types are compatible. + # TODO overloaded signatures + self.check_override(typ, + cast(FunctionLike, original), + defn.name(), + name, + base.name(), + defn) + else: + assert original_type is not None + self.msg.signature_incompatible_with_supertype( + defn.name(), name, base.name(), defn) + + def check_override(self, override: FunctionLike, original: FunctionLike, + name: str, name_in_super: str, supertype: str, + node: Context) -> None: + """Check a method override with given signatures. + + Arguments: + override: The signature of the overriding method. + original: The signature of the original supertype method. + name: The name of the subtype. This and the next argument are + only used for generating error messages. + supertype: The name of the supertype. + """ + # Use boolean variable to clarify code. + fail = False + if not is_subtype(override, original): + fail = True + elif (not isinstance(original, Overloaded) and + isinstance(override, Overloaded) and + name in nodes.reverse_op_methods.keys()): + # Operator method overrides cannot introduce overloading, as + # this could be unsafe with reverse operator methods. + fail = True + + if fail: + emitted_msg = False + if (isinstance(override, CallableType) and + isinstance(original, CallableType) and + len(override.arg_types) == len(original.arg_types) and + override.min_args == original.min_args): + # Give more detailed messages for the common case of both + # signatures having the same number of arguments and no + # overloads. + + # override might have its own generic function type + # variables. If an argument or return type of override + # does not have the correct subtyping relationship + # with the original type even after these variables + # are erased, then it is definitely an incompatiblity. + + override_ids = override.type_var_ids() + + def erase_override(t: Type) -> Type: + return erase_typevars(t, ids_to_erase=override_ids) + + for i in range(len(override.arg_types)): + if not is_subtype(original.arg_types[i], + erase_override(override.arg_types[i])): + self.msg.argument_incompatible_with_supertype( + i + 1, name, name_in_super, supertype, node) + emitted_msg = True + + if not is_subtype(erase_override(override.ret_type), + original.ret_type): + self.msg.return_type_incompatible_with_supertype( + name, name_in_super, supertype, node) + emitted_msg = True + + if not emitted_msg: + # Fall back to generic incompatibility message. + self.msg.signature_incompatible_with_supertype( + name, name_in_super, supertype, node) + + def visit_class_def(self, defn: ClassDef) -> Type: + """Type check a class definition.""" + typ = defn.info + self.errors.push_type(defn.name) + self.enter_partial_types() + old_binder = self.binder + self.binder = ConditionalTypeBinder() + with self.binder.frame_context(): + self.accept(defn.defs) + self.binder = old_binder + if not hasattr(defn, 'incompatible_baseclass'): + # Otherwise we've already found errors; more errors are not useful + self.check_multiple_inheritance(typ) + self.leave_partial_types() + self.errors.pop_type() + + def check_multiple_inheritance(self, typ: TypeInfo) -> None: + """Check for multiple inheritance related errors.""" + + if len(typ.bases) <= 1: + # No multiple inheritance. + return + # Verify that inherited attributes are compatible. + mro = typ.mro[1:] + for i, base in enumerate(mro): + for name in base.names: + for base2 in mro[i + 1:]: + # We only need to check compatibility of attributes from classes not + # in a subclass relationship. For subclasses, normal (single inheritance) + # checks suffice (these are implemented elsewhere). + if name in base2.names and base2 not in base.mro: + self.check_compatibility(name, base, base2, typ) + # Verify that base class layouts are compatible. + builtin_bases = [nearest_builtin_ancestor(base.type) + for base in typ.bases] + for base1 in builtin_bases: + for base2 in builtin_bases: + if not (base1 in base2.mro or base2 in base1.mro): + self.fail(messages.INSTANCE_LAYOUT_CONFLICT, typ) + + def check_compatibility(self, name: str, base1: TypeInfo, + base2: TypeInfo, ctx: Context) -> None: + """Check if attribute name in base1 is compatible with base2 in multiple inheritance. + + Assume base1 comes before base2 in the MRO, and that base1 and base2 don't have + a direct subclass relationship (i.e., the compatibility requirement only derives from + multiple inheritance). + """ + if name == '__init__': + # __init__ can be incompatible -- it's a special case. + return + first = base1[name] + second = base2[name] + first_type = first.type + if first_type is None and isinstance(first.node, FuncDef): + first_type = self.function_type(first.node) + second_type = second.type + if second_type is None and isinstance(second.node, FuncDef): + second_type = self.function_type(second.node) + # TODO: What if some classes are generic? + if (isinstance(first_type, FunctionLike) and + isinstance(second_type, FunctionLike)): + # Method override + first_sig = method_type(first_type) + second_sig = method_type(second_type) + ok = is_subtype(first_sig, second_sig) + elif first_type and second_type: + ok = is_equivalent(first_type, second_type) + else: + if first_type is None: + self.msg.cannot_determine_type_in_base(name, base1.name(), ctx) + if second_type is None: + self.msg.cannot_determine_type_in_base(name, base2.name(), ctx) + ok = True + if not ok: + self.msg.base_class_definitions_incompatible(name, base1, base2, + ctx) + + def visit_import_from(self, node: ImportFrom) -> Type: + self.check_import(node) + + def visit_import_all(self, node: ImportAll) -> Type: + self.check_import(node) + + def check_import(self, node: ImportBase) -> Type: + for assign in node.assignments: + lvalue = assign.lvalues[0] + lvalue_type, _, __ = self.check_lvalue(lvalue) + if lvalue_type is None: + # TODO: This is broken. + lvalue_type = AnyType() + message = '{} "{}"'.format(messages.INCOMPATIBLE_IMPORT_OF, + cast(NameExpr, assign.rvalue).name) + self.check_simple_assignment(lvalue_type, assign.rvalue, node, + msg=message, lvalue_name='local name', + rvalue_name='imported name') + + # + # Statements + # + + def visit_block(self, b: Block) -> Type: + if b.is_unreachable: + return None + for s in b.body: + self.accept(s) + if self.binder.breaking_out: + break + + def visit_assignment_stmt(self, s: AssignmentStmt) -> Type: + """Type check an assignment statement. + + Handle all kinds of assignment statements (simple, indexed, multiple). + """ + self.check_assignment(s.lvalues[-1], s.rvalue, s.type is None) + + if len(s.lvalues) > 1: + # Chained assignment (e.g. x = y = ...). + # Make sure that rvalue type will not be reinferred. + if s.rvalue not in self.type_map: + self.accept(s.rvalue) + rvalue = self.temp_node(self.type_map[s.rvalue], s) + for lv in s.lvalues[:-1]: + self.check_assignment(lv, rvalue, s.type is None) + + def check_assignment(self, lvalue: Node, rvalue: Node, infer_lvalue_type: bool = True) -> None: + """Type check a single assignment: lvalue = rvalue.""" + if isinstance(lvalue, TupleExpr) or isinstance(lvalue, ListExpr): + ltuple = cast(Union[TupleExpr, ListExpr], lvalue) + + self.check_assignment_to_multiple_lvalues(ltuple.items, rvalue, lvalue, + infer_lvalue_type) + else: + lvalue_type, index_lvalue, inferred = self.check_lvalue(lvalue) + if lvalue_type: + if isinstance(lvalue_type, PartialType) and lvalue_type.type is None: + # Try to infer a proper type for a variable with a partial None type. + rvalue_type = self.accept(rvalue) + if isinstance(rvalue_type, NoneTyp): + # This doesn't actually provide any additional information -- multiple + # None initializers preserve the partial None type. + return + if is_valid_inferred_type(rvalue_type): + var = lvalue_type.var + partial_types = self.find_partial_types(var) + if partial_types is not None: + if not self.current_node_deferred: + if experiments.STRICT_OPTIONAL: + var.type = UnionType.make_simplified_union( + [rvalue_type, NoneTyp()]) + else: + var.type = rvalue_type + else: + var.type = None + del partial_types[var] + # Try to infer a partial type. No need to check the return value, as + # an error will be reported elsewhere. + self.infer_partial_type(lvalue_type.var, lvalue, rvalue_type) + return + if (is_literal_none(rvalue) and + isinstance(lvalue, NameExpr) and + isinstance(lvalue.node, Var) and + lvalue.node.is_initialized_in_class): + # Allow None's to be assigned to class variables with non-Optional types. + rvalue_type = lvalue_type + else: + rvalue_type = self.check_simple_assignment(lvalue_type, rvalue, lvalue) + + if rvalue_type and infer_lvalue_type: + self.binder.assign_type(lvalue, + rvalue_type, + lvalue_type, + self.typing_mode_weak()) + elif index_lvalue: + self.check_indexed_assignment(index_lvalue, rvalue, rvalue) + + if inferred: + self.infer_variable_type(inferred, lvalue, self.accept(rvalue), + rvalue) + + def check_assignment_to_multiple_lvalues(self, lvalues: List[Node], rvalue: Node, + context: Context, + infer_lvalue_type: bool = True) -> None: + if isinstance(rvalue, TupleExpr) or isinstance(rvalue, ListExpr): + # Recursively go into Tuple or List expression rhs instead of + # using the type of rhs, because this allowed more fine grained + # control in cases like: a, b = [int, str] where rhs would get + # type List[object] + + rvalues = cast(Union[TupleExpr, ListExpr], rvalue).items + + if self.check_rvalue_count_in_assignment(lvalues, len(rvalues), context): + star_index = next((i for i, lv in enumerate(lvalues) if + isinstance(lv, StarExpr)), len(lvalues)) + + left_lvs = lvalues[:star_index] + star_lv = cast(StarExpr, + lvalues[star_index]) if star_index != len(lvalues) else None + right_lvs = lvalues[star_index + 1:] + + left_rvs, star_rvs, right_rvs = self.split_around_star( + rvalues, star_index, len(lvalues)) + + lr_pairs = list(zip(left_lvs, left_rvs)) + if star_lv: + rv_list = ListExpr(star_rvs) + rv_list.set_line(rvalue.get_line()) + lr_pairs.append((star_lv.expr, rv_list)) + lr_pairs.extend(zip(right_lvs, right_rvs)) + + for lv, rv in lr_pairs: + self.check_assignment(lv, rv, infer_lvalue_type) + else: + self.check_multi_assignment(lvalues, rvalue, context, infer_lvalue_type) + + def check_rvalue_count_in_assignment(self, lvalues: List[Node], rvalue_count: int, + context: Context) -> bool: + if any(isinstance(lvalue, StarExpr) for lvalue in lvalues): + if len(lvalues) - 1 > rvalue_count: + self.msg.wrong_number_values_to_unpack(rvalue_count, + len(lvalues) - 1, context) + return False + elif rvalue_count != len(lvalues): + self.msg.wrong_number_values_to_unpack(rvalue_count, + len(lvalues), context) + return False + return True + + def check_multi_assignment(self, lvalues: List[Node], + rvalue: Node, + context: Context, + infer_lvalue_type: bool = True, + msg: str = None) -> None: + """Check the assignment of one rvalue to a number of lvalues.""" + + # Infer the type of an ordinary rvalue expression. + rvalue_type = self.accept(rvalue) # TODO maybe elsewhere; redundant + undefined_rvalue = False + + if isinstance(rvalue_type, AnyType): + for lv in lvalues: + if isinstance(lv, StarExpr): + lv = lv.expr + self.check_assignment(lv, self.temp_node(AnyType(), context), infer_lvalue_type) + elif isinstance(rvalue_type, TupleType): + self.check_multi_assignment_from_tuple(lvalues, rvalue, rvalue_type, + context, undefined_rvalue, infer_lvalue_type) + else: + self.check_multi_assignment_from_iterable(lvalues, rvalue_type, + context, infer_lvalue_type) + + def check_multi_assignment_from_tuple(self, lvalues: List[Node], rvalue: Node, + rvalue_type: TupleType, context: Context, + undefined_rvalue: bool, + infer_lvalue_type: bool = True) -> None: + if self.check_rvalue_count_in_assignment(lvalues, len(rvalue_type.items), context): + star_index = next((i for i, lv in enumerate(lvalues) + if isinstance(lv, StarExpr)), len(lvalues)) + + left_lvs = lvalues[:star_index] + star_lv = cast(StarExpr, lvalues[star_index]) if star_index != len(lvalues) else None + right_lvs = lvalues[star_index + 1:] + + if not undefined_rvalue: + # Infer rvalue again, now in the correct type context. + lvalue_type = self.lvalue_type_for_inference(lvalues, rvalue_type) + rvalue_type = cast(TupleType, self.accept(rvalue, lvalue_type)) + + left_rv_types, star_rv_types, right_rv_types = self.split_around_star( + rvalue_type.items, star_index, len(lvalues)) + + for lv, rv_type in zip(left_lvs, left_rv_types): + self.check_assignment(lv, self.temp_node(rv_type, context), infer_lvalue_type) + if star_lv: + nodes = [self.temp_node(rv_type, context) for rv_type in star_rv_types] + list_expr = ListExpr(nodes) + list_expr.set_line(context.get_line()) + self.check_assignment(star_lv.expr, list_expr, infer_lvalue_type) + for lv, rv_type in zip(right_lvs, right_rv_types): + self.check_assignment(lv, self.temp_node(rv_type, context), infer_lvalue_type) + + def lvalue_type_for_inference(self, lvalues: List[Node], rvalue_type: TupleType) -> Type: + star_index = next((i for i, lv in enumerate(lvalues) + if isinstance(lv, StarExpr)), len(lvalues)) + left_lvs = lvalues[:star_index] + star_lv = cast(StarExpr, lvalues[star_index]) if star_index != len(lvalues) else None + right_lvs = lvalues[star_index + 1:] + left_rv_types, star_rv_types, right_rv_types = self.split_around_star( + rvalue_type.items, star_index, len(lvalues)) + + type_parameters = [] # type: List[Type] + + def append_types_for_inference(lvs: List[Node], rv_types: List[Type]) -> None: + for lv, rv_type in zip(lvs, rv_types): + sub_lvalue_type, index_expr, inferred = self.check_lvalue(lv) + if sub_lvalue_type: + type_parameters.append(sub_lvalue_type) + else: # index lvalue + # TODO Figure out more precise type context, probably + # based on the type signature of the _set method. + type_parameters.append(rv_type) + + append_types_for_inference(left_lvs, left_rv_types) + + if star_lv: + sub_lvalue_type, index_expr, inferred = self.check_lvalue(star_lv.expr) + if sub_lvalue_type: + type_parameters.extend([sub_lvalue_type] * len(star_rv_types)) + else: # index lvalue + # TODO Figure out more precise type context, probably + # based on the type signature of the _set method. + type_parameters.extend(star_rv_types) + + append_types_for_inference(right_lvs, right_rv_types) + + return TupleType(type_parameters, self.named_type('builtins.tuple')) + + def split_around_star(self, items: List[T], star_index: int, + length: int) -> Tuple[List[T], List[T], List[T]]: + """Splits a list of items in three to match another list of length 'length' + that contains a starred expression at 'star_index' in the following way: + + star_index = 2, length = 5 (i.e., [a,b,*,c,d]), items = [1,2,3,4,5,6,7] + returns in: ([1,2], [3,4,5], [6,7]) + """ + nr_right_of_star = length - star_index - 1 + right_index = nr_right_of_star if -nr_right_of_star != 0 else len(items) + left = items[:star_index] + star = items[star_index:right_index] + right = items[right_index:] + return (left, star, right) + + def type_is_iterable(self, type: Type) -> bool: + return (is_subtype(type, self.named_generic_type('typing.Iterable', + [AnyType()])) and + isinstance(type, Instance)) + + def check_multi_assignment_from_iterable(self, lvalues: List[Node], rvalue_type: Type, + context: Context, + infer_lvalue_type: bool = True) -> None: + if self.type_is_iterable(rvalue_type): + item_type = self.iterable_item_type(cast(Instance, rvalue_type)) + for lv in lvalues: + if isinstance(lv, StarExpr): + self.check_assignment(lv.expr, self.temp_node(rvalue_type, context), + infer_lvalue_type) + else: + self.check_assignment(lv, self.temp_node(item_type, context), + infer_lvalue_type) + else: + self.msg.type_not_iterable(rvalue_type, context) + + def check_lvalue(self, lvalue: Node) -> Tuple[Type, IndexExpr, Var]: + lvalue_type = None # type: Type + index_lvalue = None # type: IndexExpr + inferred = None # type: Var + + if self.is_definition(lvalue): + if isinstance(lvalue, NameExpr): + inferred = cast(Var, lvalue.node) + assert isinstance(inferred, Var) + else: + m = cast(MemberExpr, lvalue) + self.accept(m.expr) + inferred = m.def_var + elif isinstance(lvalue, IndexExpr): + index_lvalue = lvalue + elif isinstance(lvalue, MemberExpr): + lvalue_type = self.expr_checker.analyze_ordinary_member_access(lvalue, + True) + self.store_type(lvalue, lvalue_type) + elif isinstance(lvalue, NameExpr): + lvalue_type = self.expr_checker.analyze_ref_expr(lvalue, lvalue=True) + self.store_type(lvalue, lvalue_type) + elif isinstance(lvalue, TupleExpr) or isinstance(lvalue, ListExpr): + lv = cast(Union[TupleExpr, ListExpr], lvalue) + types = [self.check_lvalue(sub_expr)[0] for sub_expr in lv.items] + lvalue_type = TupleType(types, self.named_type('builtins.tuple')) + else: + lvalue_type = self.accept(lvalue) + + return lvalue_type, index_lvalue, inferred + + def is_definition(self, s: Node) -> bool: + if isinstance(s, NameExpr): + if s.is_def: + return True + # If the node type is not defined, this must the first assignment + # that we process => this is a definition, even though the semantic + # analyzer did not recognize this as such. This can arise in code + # that uses isinstance checks, if type checking of the primary + # definition is skipped due to an always False type check. + node = s.node + if isinstance(node, Var): + return node.type is None + elif isinstance(s, MemberExpr): + return s.is_def + return False + + def infer_variable_type(self, name: Var, lvalue: Node, + init_type: Type, context: Context) -> None: + """Infer the type of initialized variables from initializer type.""" + if self.typing_mode_weak(): + self.set_inferred_type(name, lvalue, AnyType()) + self.binder.assign_type(lvalue, init_type, self.binder.get_declaration(lvalue), True) + elif isinstance(init_type, Void): + self.check_not_void(init_type, context) + self.set_inference_error_fallback_type(name, lvalue, init_type, context) + elif isinstance(init_type, DeletedType): + self.msg.deleted_as_rvalue(init_type, context) + elif not is_valid_inferred_type(init_type): + # We cannot use the type of the initialization expression for full type + # inference (it's not specific enough), but we might be able to give + # partial type which will be made more specific later. A partial type + # gets generated in assignment like 'x = []' where item type is not known. + if not self.infer_partial_type(name, lvalue, init_type): + self.fail(messages.NEED_ANNOTATION_FOR_VAR, context) + self.set_inference_error_fallback_type(name, lvalue, init_type, context) + else: + # Infer type of the target. + + # Make the type more general (strip away function names etc.). + init_type = strip_type(init_type) + + self.set_inferred_type(name, lvalue, init_type) + + def infer_partial_type(self, name: Var, lvalue: Node, init_type: Type) -> bool: + if isinstance(init_type, (NoneTyp, UninhabitedType)): + partial_type = PartialType(None, name, [init_type]) + elif isinstance(init_type, Instance): + fullname = init_type.type.fullname() + if (isinstance(lvalue, NameExpr) and + (fullname == 'builtins.list' or + fullname == 'builtins.set' or + fullname == 'builtins.dict') and + all(isinstance(t, (NoneTyp, UninhabitedType)) for t in init_type.args)): + partial_type = PartialType(init_type.type, name, init_type.args) + else: + return False + else: + return False + self.set_inferred_type(name, lvalue, partial_type) + self.partial_types[-1][name] = lvalue + return True + + def set_inferred_type(self, var: Var, lvalue: Node, type: Type) -> None: + """Store inferred variable type. + + Store the type to both the variable node and the expression node that + refers to the variable (lvalue). If var is None, do nothing. + """ + if var and not self.current_node_deferred: + var.type = type + self.store_type(lvalue, type) + + def set_inference_error_fallback_type(self, var: Var, lvalue: Node, type: Type, + context: Context) -> None: + """If errors on context line are ignored, store dummy type for variable. + + If a program ignores error on type inference error, the variable should get some + inferred type so that if can used later on in the program. Example: + + x = [] # type: ignore + x.append(1) # Should be ok! + + We implement this here by giving x a valid type (Any). + """ + if context.get_line() in self.errors.ignored_lines[self.errors.file]: + self.set_inferred_type(var, lvalue, AnyType()) + + def narrow_type_from_binder(self, expr: Node, known_type: Type) -> Type: + if expr.literal >= LITERAL_TYPE: + restriction = self.binder.get(expr) + if restriction: + ans = meet_simple(known_type, restriction) + return ans + return known_type + + def check_simple_assignment(self, lvalue_type: Type, rvalue: Node, + context: Node, + msg: str = messages.INCOMPATIBLE_TYPES_IN_ASSIGNMENT, + lvalue_name: str = 'variable', + rvalue_name: str = 'expression') -> Type: + if self.is_stub and isinstance(rvalue, EllipsisExpr): + # '...' is always a valid initializer in a stub. + return AnyType() + else: + rvalue_type = self.accept(rvalue, lvalue_type) + if isinstance(rvalue_type, DeletedType): + self.msg.deleted_as_rvalue(rvalue_type, context) + if self.typing_mode_weak(): + return rvalue_type + if isinstance(lvalue_type, DeletedType): + self.msg.deleted_as_lvalue(lvalue_type, context) + else: + self.check_subtype(rvalue_type, lvalue_type, context, msg, + '{} has type'.format(rvalue_name), + '{} has type'.format(lvalue_name)) + return rvalue_type + + def check_indexed_assignment(self, lvalue: IndexExpr, + rvalue: Node, context: Context) -> None: + """Type check indexed assignment base[index] = rvalue. + + The lvalue argument is the base[index] expression. + """ + self.try_infer_partial_type_from_indexed_assignment(lvalue, rvalue) + basetype = self.accept(lvalue.base) + method_type = self.expr_checker.analyze_external_member_access( + '__setitem__', basetype, context) + lvalue.method_type = method_type + self.expr_checker.check_call(method_type, [lvalue.index, rvalue], + [nodes.ARG_POS, nodes.ARG_POS], + context) + + def try_infer_partial_type_from_indexed_assignment( + self, lvalue: IndexExpr, rvalue: Node) -> None: + # TODO: Should we share some of this with try_infer_partial_type? + if isinstance(lvalue.base, RefExpr) and isinstance(lvalue.base.node, Var): + var = lvalue.base.node + if isinstance(var.type, PartialType): + type_type = var.type.type + if type_type is None: + return # The partial type is None. + partial_types = self.find_partial_types(var) + if partial_types is None: + return + typename = type_type.fullname() + if typename == 'builtins.dict': + # TODO: Don't infer things twice. + key_type = self.accept(lvalue.index) + value_type = self.accept(rvalue) + full_key_type = UnionType.make_simplified_union( + [key_type, var.type.inner_types[0]]) + full_value_type = UnionType.make_simplified_union( + [value_type, var.type.inner_types[1]]) + if (is_valid_inferred_type(full_key_type) and + is_valid_inferred_type(full_value_type)): + if not self.current_node_deferred: + var.type = self.named_generic_type('builtins.dict', + [full_key_type, full_value_type]) + del partial_types[var] + + def visit_expression_stmt(self, s: ExpressionStmt) -> Type: + self.accept(s.expr) + + def visit_return_stmt(self, s: ReturnStmt) -> Type: + """Type check a return statement.""" + self.binder.breaking_out = True + if self.is_within_function(): + if self.function_stack[-1].is_generator: + return_type = self.get_generator_return_type(self.return_types[-1]) + else: + return_type = self.return_types[-1] + + if s.expr: + # Return with a value. + typ = self.accept(s.expr, return_type) + # Returning a value of type Any is always fine. + if isinstance(typ, AnyType): + return None + + if isinstance(return_type, Void): + # Lambdas are allowed to have a Void return. + # Functions returning a value of type None are allowed to have a Void return. + if isinstance(self.function_stack[-1], FuncExpr) or isinstance(typ, NoneTyp): + return None + self.fail(messages.NO_RETURN_VALUE_EXPECTED, s) + else: + self.check_subtype( + subtype_label='got', + subtype=typ, + supertype_label='expected', + supertype=return_type, + context=s, + msg=messages.INCOMPATIBLE_RETURN_VALUE_TYPE) + else: + # Empty returns are valid in Generators with Any typed returns. + if (self.function_stack[-1].is_generator and isinstance(return_type, AnyType)): + return None + + if isinstance(return_type, Void): + return None + + if isinstance(return_type, AnyType): + return None + + if self.typing_mode_full(): + self.fail(messages.RETURN_VALUE_EXPECTED, s) + + def wrap_generic_type(self, typ: Instance, rtyp: Instance, check_type: + str, context: Context) -> Type: + n_diff = self.count_nested_types(rtyp, check_type) - self.count_nested_types(typ, + check_type) + if n_diff == 1: + return self.named_generic_type(check_type, [typ]) + elif n_diff == 0 or n_diff > 1: + self.fail(messages.INCOMPATIBLE_RETURN_VALUE_TYPE + + ": expected {}, got {}".format(rtyp, typ), context) + return typ + return typ + + def count_nested_types(self, typ: Instance, check_type: str) -> int: + c = 0 + while is_subtype(typ, self.named_type(check_type)): + c += 1 + typ = map_instance_to_supertype(self.named_generic_type(check_type, typ.args), + self.lookup_typeinfo(check_type)) + if typ.args: + typ = cast(Instance, typ.args[0]) + else: + return c + return c + + def visit_if_stmt(self, s: IfStmt) -> Type: + """Type check an if statement.""" + breaking_out = True + # This frame records the knowledge from previous if/elif clauses not being taken. + with self.binder.frame_context(): + for e, b in zip(s.expr, s.body): + t = self.accept(e) + self.check_not_void(t, e) + if_map, else_map = find_isinstance_check( + e, self.type_map, + self.typing_mode_weak() + ) + if if_map is None: + # The condition is always false + # XXX should issue a warning? + pass + else: + # Only type check body if the if condition can be true. + with self.binder.frame_context(2): + if if_map: + for var, type in if_map.items(): + self.binder.push(var, type) + + self.accept(b) + breaking_out = breaking_out and self.binder.last_pop_breaking_out + + if else_map: + for var, type in else_map.items(): + self.binder.push(var, type) + if else_map is None: + # The condition is always true => remaining elif/else blocks + # can never be reached. + + # Might also want to issue a warning + # print("Warning: isinstance always true") + break + else: # Didn't break => can't prove one of the conditions is always true + with self.binder.frame_context(2): + if s.else_body: + self.accept(s.else_body) + breaking_out = breaking_out and self.binder.last_pop_breaking_out + if breaking_out: + self.binder.breaking_out = True + return None + + def visit_while_stmt(self, s: WhileStmt) -> Type: + """Type check a while statement.""" + self.accept_loop(IfStmt([s.expr], [s.body], None), s.else_body) + + def visit_operator_assignment_stmt(self, + s: OperatorAssignmentStmt) -> Type: + """Type check an operator assignment statement, e.g. x += 1.""" + lvalue_type = self.accept(s.lvalue) + method = infer_operator_assignment_method(lvalue_type, s.op) + rvalue_type, method_type = self.expr_checker.check_op( + method, lvalue_type, s.rvalue, s) + + if isinstance(s.lvalue, IndexExpr): + self.check_indexed_assignment(s.lvalue, s.rvalue, s.rvalue) + else: + if not is_subtype(rvalue_type, lvalue_type): + self.msg.incompatible_operator_assignment(s.op, s) + + def visit_assert_stmt(self, s: AssertStmt) -> Type: + self.accept(s.expr) + + # If this is asserting some isinstance check, bind that type in the following code + true_map, _ = find_isinstance_check( + s.expr, self.type_map, + self.typing_mode_weak() + ) + + if true_map: + for var, type in true_map.items(): + self.binder.push(var, type) + + def visit_raise_stmt(self, s: RaiseStmt) -> Type: + """Type check a raise statement.""" + self.binder.breaking_out = True + if s.expr: + self.type_check_raise(s.expr, s) + if s.from_expr: + self.type_check_raise(s.from_expr, s) + + def type_check_raise(self, e: Node, s: RaiseStmt) -> None: + typ = self.accept(e) + if isinstance(typ, FunctionLike): + if typ.is_type_obj(): + # Cases like "raise/from ExceptionClass". + typeinfo = typ.type_object() + base = self.lookup_typeinfo('builtins.BaseException') + if base in typeinfo.mro or typeinfo.fallback_to_any: + # Good! + return + # Else fall back to the checks below (which will fail). + if isinstance(typ, TupleType) and self.options.python_version[0] == 2: + # allow `raise type, value, traceback` + # https://docs.python.org/2/reference/simple_stmts.html#the-raise-statement + # TODO: Also check tuple item types. + if len(typ.items) in (2, 3): + return + if isinstance(typ, Instance) and typ.type.fallback_to_any: + # OK! + return + self.check_subtype(typ, + self.named_type('builtins.BaseException'), s, + messages.INVALID_EXCEPTION) + + def visit_try_stmt(self, s: TryStmt) -> Type: + """Type check a try statement.""" + # Our enclosing frame will get the result if the try/except falls through. + # This one gets all possible intermediate states + with self.binder.frame_context(): + if s.finally_body: + self.binder.try_frames.add(len(self.binder.frames) - 1) + breaking_out = self.visit_try_without_finally(s) + self.binder.try_frames.remove(len(self.binder.frames) - 1) + # First we check finally_body is type safe for all intermediate frames + self.accept(s.finally_body) + breaking_out = breaking_out or self.binder.breaking_out + else: + breaking_out = self.visit_try_without_finally(s) + + if not breaking_out and s.finally_body: + # Then we try again for the more restricted set of options that can fall through + self.accept(s.finally_body) + self.binder.breaking_out = breaking_out + return None + + def visit_try_without_finally(self, s: TryStmt) -> bool: + """Type check a try statement, ignoring the finally block. + + Return whether we are guaranteed to be breaking out. + Otherwise, it will place the results possible frames of + that don't break out into self.binder.frames[-2]. + """ + breaking_out = True + # This frame records the possible states that exceptions can leave variables in + # during the try: block + with self.binder.frame_context(): + with self.binder.frame_context(3): + self.binder.try_frames.add(len(self.binder.frames) - 2) + self.accept(s.body) + self.binder.try_frames.remove(len(self.binder.frames) - 2) + if s.else_body: + self.accept(s.else_body) + breaking_out = breaking_out and self.binder.last_pop_breaking_out + for i in range(len(s.handlers)): + with self.binder.frame_context(3): + if s.types[i]: + t = self.visit_except_handler_test(s.types[i]) + if s.vars[i]: + # To support local variables, we make this a definition line, + # causing assignment to set the variable's type. + s.vars[i].is_def = True + self.check_assignment(s.vars[i], self.temp_node(t, s.vars[i])) + self.accept(s.handlers[i]) + if s.vars[i]: + # Exception variables are deleted in python 3 but not python 2. + # But, since it's bad form in python 2 and the type checking + # wouldn't work very well, we delete it anyway. + + # Unfortunately, this doesn't let us detect usage before the + # try/except block. + if self.options.python_version[0] >= 3: + source = s.vars[i].name + else: + source = ('(exception variable "{}", which we do not accept outside' + 'except: blocks even in python 2)'.format(s.vars[i].name)) + var = cast(Var, s.vars[i].node) + var.type = DeletedType(source=source) + self.binder.cleanse(s.vars[i]) + breaking_out = breaking_out and self.binder.last_pop_breaking_out + return breaking_out + + def visit_except_handler_test(self, n: Node) -> Type: + """Type check an exception handler test clause.""" + type = self.accept(n) + + all_types = [] # type: List[Type] + test_types = type.items if isinstance(type, TupleType) else [type] + + for ttype in test_types: + if isinstance(ttype, AnyType): + all_types.append(ttype) + continue + + if not isinstance(ttype, FunctionLike): + self.fail(messages.INVALID_EXCEPTION_TYPE, n) + return AnyType() + + item = ttype.items()[0] + ret_type = item.ret_type + if not (is_subtype(ret_type, self.named_type('builtins.BaseException')) + and item.is_type_obj()): + self.fail(messages.INVALID_EXCEPTION_TYPE, n) + return AnyType() + + all_types.append(ret_type) + + return UnionType.make_simplified_union(all_types) + + def visit_for_stmt(self, s: ForStmt) -> Type: + """Type check a for statement.""" + item_type = self.analyze_iterable_item_type(s.expr) + self.analyze_index_variables(s.index, item_type, s) + self.accept_loop(s.body, s.else_body) + + def analyze_iterable_item_type(self, expr: Node) -> Type: + """Analyse iterable expression and return iterator item type.""" + iterable = self.accept(expr) + + self.check_not_void(iterable, expr) + if isinstance(iterable, TupleType): + if experiments.STRICT_OPTIONAL: + joined = UninhabitedType() # type: Type + else: + joined = NoneTyp() + for item in iterable.items: + joined = join_types(joined, item) + if isinstance(joined, ErrorType): + self.fail(messages.CANNOT_INFER_ITEM_TYPE, expr) + return AnyType() + return joined + else: + # Non-tuple iterable. + self.check_subtype(iterable, + self.named_generic_type('typing.Iterable', + [AnyType()]), + expr, messages.ITERABLE_EXPECTED) + + echk = self.expr_checker + method = echk.analyze_external_member_access('__iter__', iterable, + expr) + iterator = echk.check_call(method, [], [], expr)[0] + if self.options.python_version[0] >= 3: + nextmethod = '__next__' + else: + nextmethod = 'next' + method = echk.analyze_external_member_access(nextmethod, iterator, + expr) + return echk.check_call(method, [], [], expr)[0] + + def analyze_index_variables(self, index: Node, item_type: Type, + context: Context) -> None: + """Type check or infer for loop or list comprehension index vars.""" + self.check_assignment(index, self.temp_node(item_type, context)) + + def visit_del_stmt(self, s: DelStmt) -> Type: + if isinstance(s.expr, IndexExpr): + e = s.expr + m = MemberExpr(e.base, '__delitem__') + m.line = s.line + c = CallExpr(m, [e.index], [nodes.ARG_POS], [None]) + c.line = s.line + return c.accept(self) + else: + def flatten(t: Node) -> List[Node]: + """Flatten a nested sequence of tuples/lists into one list of nodes.""" + if isinstance(t, TupleExpr) or isinstance(t, ListExpr): + t = cast(Union[TupleExpr, ListExpr], t) + return [b for a in t.items for b in flatten(a)] + else: + return [t] + + s.expr.accept(self) + for elt in flatten(s.expr): + if isinstance(elt, NameExpr): + self.binder.assign_type(elt, + DeletedType(source=elt.name), + self.binder.get_declaration(elt), + self.typing_mode_weak()) + return None + + def visit_decorator(self, e: Decorator) -> Type: + for d in e.decorators: + if isinstance(d, RefExpr): + if d.fullname == 'typing.no_type_check': + e.var.type = AnyType() + e.var.is_ready = True + return NoneTyp() + + e.func.accept(self) + sig = self.function_type(e.func) # type: Type + # Process decorators from the inside out. + for i in range(len(e.decorators)): + n = len(e.decorators) - 1 - i + d = e.decorators[n] + if isinstance(d, NameExpr) and d.fullname == 'typing.overload': + self.fail('Single overload definition, multiple required', e) + continue + dec = self.accept(d) + temp = self.temp_node(sig) + sig, t2 = self.expr_checker.check_call(dec, [temp], + [nodes.ARG_POS], e) + sig = cast(FunctionLike, sig) + sig = set_callable_name(sig, e.func) + e.var.type = sig + e.var.is_ready = True + if e.func.is_property: + self.check_incompatible_property_override(e) + + def check_incompatible_property_override(self, e: Decorator) -> None: + if not e.var.is_settable_property and e.func.info is not None: + name = e.func.name() + for base in e.func.info.mro[1:]: + base_attr = base.names.get(name) + if not base_attr: + continue + if (isinstance(base_attr.node, OverloadedFuncDef) and + base_attr.node.is_property and + base_attr.node.items[0].var.is_settable_property): + self.fail(messages.READ_ONLY_PROPERTY_OVERRIDES_READ_WRITE, e) + + def visit_with_stmt(self, s: WithStmt) -> Type: + echk = self.expr_checker + for expr, target in zip(s.expr, s.target): + ctx = self.accept(expr) + enter = echk.analyze_external_member_access('__enter__', ctx, expr) + obj = echk.check_call(enter, [], [], expr)[0] + if target: + self.check_assignment(target, self.temp_node(obj, expr)) + exit = echk.analyze_external_member_access('__exit__', ctx, expr) + arg = self.temp_node(AnyType(), expr) + echk.check_call(exit, [arg] * 3, [nodes.ARG_POS] * 3, expr) + self.accept(s.body) + + def visit_print_stmt(self, s: PrintStmt) -> Type: + for arg in s.args: + self.accept(arg) + if s.target: + target_type = self.accept(s.target) + if not isinstance(target_type, NoneTyp): + # TODO: Also verify the type of 'write'. + self.expr_checker.analyze_external_member_access('write', target_type, s.target) + + # + # Expressions + # + + def visit_name_expr(self, e: NameExpr) -> Type: + return self.expr_checker.visit_name_expr(e) + + def visit_call_expr(self, e: CallExpr) -> Type: + return self.expr_checker.visit_call_expr(e) + + def visit_yield_from_expr(self, e: YieldFromExpr) -> Type: + return_type = self.return_types[-1] + subexpr_type = self.accept(e.expr, return_type) + iter_type = None # type: Type + + # Check that the expr is an instance of Iterable and get the type of the iterator produced + # by __iter__. + if isinstance(subexpr_type, AnyType): + iter_type = AnyType() + elif (isinstance(subexpr_type, Instance) and + is_subtype(subexpr_type, self.named_type('typing.Iterable'))): + iter_method_type = self.expr_checker.analyze_external_member_access( + '__iter__', + subexpr_type, + AnyType()) + + generic_generator_type = self.named_generic_type('typing.Generator', + [AnyType(), AnyType(), AnyType()]) + iter_type, _ = self.expr_checker.check_call(iter_method_type, [], [], + context=generic_generator_type) + else: + self.msg.yield_from_invalid_operand_type(subexpr_type, e) + iter_type = AnyType() + + # Check that the iterator's item type matches the type yielded by the Generator function + # containing this `yield from` expression. + expected_item_type = self.get_generator_yield_type(return_type) + actual_item_type = self.get_generator_yield_type(iter_type) + + self.check_subtype(actual_item_type, expected_item_type, e, + messages.INCOMPATIBLE_TYPES_IN_YIELD_FROM, + 'actual type', 'expected type') + + # Determine the type of the entire yield from expression. + if (isinstance(iter_type, Instance) and + iter_type.type.fullname() == 'typing.Generator'): + return self.get_generator_return_type(iter_type) + else: + # Non-Generators don't return anything from `yield from` expressions. + return Void() + + def visit_member_expr(self, e: MemberExpr) -> Type: + return self.expr_checker.visit_member_expr(e) + + def visit_break_stmt(self, s: BreakStmt) -> Type: + self.binder.breaking_out = True + self.binder.allow_jump(self.binder.loop_frames[-1] - 1) + return None + + def visit_continue_stmt(self, s: ContinueStmt) -> Type: + self.binder.breaking_out = True + self.binder.allow_jump(self.binder.loop_frames[-1]) + return None + + def visit_int_expr(self, e: IntExpr) -> Type: + return self.expr_checker.visit_int_expr(e) + + def visit_str_expr(self, e: StrExpr) -> Type: + return self.expr_checker.visit_str_expr(e) + + def visit_bytes_expr(self, e: BytesExpr) -> Type: + return self.expr_checker.visit_bytes_expr(e) + + def visit_unicode_expr(self, e: UnicodeExpr) -> Type: + return self.expr_checker.visit_unicode_expr(e) + + def visit_float_expr(self, e: FloatExpr) -> Type: + return self.expr_checker.visit_float_expr(e) + + def visit_complex_expr(self, e: ComplexExpr) -> Type: + return self.expr_checker.visit_complex_expr(e) + + def visit_ellipsis(self, e: EllipsisExpr) -> Type: + return self.expr_checker.visit_ellipsis(e) + + def visit_op_expr(self, e: OpExpr) -> Type: + return self.expr_checker.visit_op_expr(e) + + def visit_comparison_expr(self, e: ComparisonExpr) -> Type: + return self.expr_checker.visit_comparison_expr(e) + + def visit_unary_expr(self, e: UnaryExpr) -> Type: + return self.expr_checker.visit_unary_expr(e) + + def visit_index_expr(self, e: IndexExpr) -> Type: + return self.expr_checker.visit_index_expr(e) + + def visit_cast_expr(self, e: CastExpr) -> Type: + return self.expr_checker.visit_cast_expr(e) + + def visit_reveal_type_expr(self, e: RevealTypeExpr) -> Type: + return self.expr_checker.visit_reveal_type_expr(e) + + def visit_super_expr(self, e: SuperExpr) -> Type: + return self.expr_checker.visit_super_expr(e) + + def visit_type_application(self, e: TypeApplication) -> Type: + return self.expr_checker.visit_type_application(e) + + def visit_type_alias_expr(self, e: TypeAliasExpr) -> Type: + return self.expr_checker.visit_type_alias_expr(e) + + def visit_type_var_expr(self, e: TypeVarExpr) -> Type: + # TODO: Perhaps return a special type used for type variables only? + return AnyType() + + def visit_namedtuple_expr(self, e: NamedTupleExpr) -> Type: + # TODO: Perhaps return a type object type? + return AnyType() + + def visit_list_expr(self, e: ListExpr) -> Type: + return self.expr_checker.visit_list_expr(e) + + def visit_set_expr(self, e: SetExpr) -> Type: + return self.expr_checker.visit_set_expr(e) + + def visit_tuple_expr(self, e: TupleExpr) -> Type: + return self.expr_checker.visit_tuple_expr(e) + + def visit_dict_expr(self, e: DictExpr) -> Type: + return self.expr_checker.visit_dict_expr(e) + + def visit_slice_expr(self, e: SliceExpr) -> Type: + return self.expr_checker.visit_slice_expr(e) + + def visit_func_expr(self, e: FuncExpr) -> Type: + return self.expr_checker.visit_func_expr(e) + + def visit_list_comprehension(self, e: ListComprehension) -> Type: + return self.expr_checker.visit_list_comprehension(e) + + def visit_set_comprehension(self, e: SetComprehension) -> Type: + return self.expr_checker.visit_set_comprehension(e) + + def visit_generator_expr(self, e: GeneratorExpr) -> Type: + return self.expr_checker.visit_generator_expr(e) + + def visit_dictionary_comprehension(self, e: DictionaryComprehension) -> Type: + return self.expr_checker.visit_dictionary_comprehension(e) + + def visit_temp_node(self, e: TempNode) -> Type: + return e.type + + def visit_conditional_expr(self, e: ConditionalExpr) -> Type: + return self.expr_checker.visit_conditional_expr(e) + + def visit_backquote_expr(self, e: BackquoteExpr) -> Type: + return self.expr_checker.visit_backquote_expr(e) + + def visit_yield_expr(self, e: YieldExpr) -> Type: + return_type = self.return_types[-1] + expected_item_type = self.get_generator_yield_type(return_type) + if e.expr is None: + if (not (isinstance(expected_item_type, Void) or + isinstance(expected_item_type, AnyType)) + and self.typing_mode_full()): + self.fail(messages.YIELD_VALUE_EXPECTED, e) + else: + actual_item_type = self.accept(e.expr, expected_item_type) + self.check_subtype(actual_item_type, expected_item_type, e, + messages.INCOMPATIBLE_TYPES_IN_YIELD, + 'actual type', 'expected type') + return self.get_generator_receive_type(return_type) + + # + # Helpers + # + + def check_subtype(self, subtype: Type, supertype: Type, context: Context, + msg: str = messages.INCOMPATIBLE_TYPES, + subtype_label: str = None, + supertype_label: str = None) -> None: + """Generate an error if the subtype is not compatible with + supertype.""" + if not is_subtype(subtype, supertype): + if isinstance(subtype, Void): + self.msg.does_not_return_value(subtype, context) + else: + extra_info = [] # type: List[str] + if subtype_label is not None or supertype_label is not None: + subtype_str, supertype_str = self.msg.format_distinctly(subtype, supertype) + if subtype_label is not None: + extra_info.append(subtype_label + ' ' + subtype_str) + if supertype_label is not None: + extra_info.append(supertype_label + ' ' + supertype_str) + if extra_info: + msg += ' (' + ', '.join(extra_info) + ')' + self.fail(msg, context) + + def named_type(self, name: str) -> Instance: + """Return an instance type with type given by the name and no + type arguments. For example, named_type('builtins.object') + produces the object type. + """ + # Assume that the name refers to a type. + sym = self.lookup_qualified(name) + return Instance(cast(TypeInfo, sym.node), []) + + def named_generic_type(self, name: str, args: List[Type]) -> Instance: + """Return an instance with the given name and type arguments. + + Assume that the number of arguments is correct. Assume that + the name refers to a compatible generic type. + """ + return Instance(self.lookup_typeinfo(name), args) + + def lookup_typeinfo(self, fullname: str) -> TypeInfo: + # Assume that the name refers to a class. + sym = self.lookup_qualified(fullname) + return cast(TypeInfo, sym.node) + + def type_type(self) -> Instance: + """Return instance type 'type'.""" + return self.named_type('builtins.type') + + def object_type(self) -> Instance: + """Return instance type 'object'.""" + return self.named_type('builtins.object') + + def bool_type(self) -> Instance: + """Return instance type 'bool'.""" + return self.named_type('builtins.bool') + + def str_type(self) -> Instance: + """Return instance type 'str'.""" + return self.named_type('builtins.str') + + def check_type_equivalency(self, t1: Type, t2: Type, node: Context, + msg: str = messages.INCOMPATIBLE_TYPES) -> None: + """Generate an error if the types are not equivalent. The + dynamic type is equivalent with all types. + """ + if not is_equivalent(t1, t2): + self.fail(msg, node) + + def store_type(self, node: Node, typ: Type) -> None: + """Store the type of a node in the type map.""" + self.type_map[node] = typ + + def typing_mode_none(self) -> bool: + if self.is_dynamic_function() and not self.options.check_untyped_defs: + return not self.weak_opts + elif self.function_stack: + return False + else: + return False + + def typing_mode_weak(self) -> bool: + if self.is_dynamic_function() and not self.options.check_untyped_defs: + return bool(self.weak_opts) + elif self.function_stack: + return False + else: + return 'global' in self.weak_opts + + def typing_mode_full(self) -> bool: + if self.is_dynamic_function() and not self.options.check_untyped_defs: + return False + elif self.function_stack: + return True + else: + return 'global' not in self.weak_opts + + def is_dynamic_function(self) -> bool: + return len(self.dynamic_funcs) > 0 and self.dynamic_funcs[-1] + + def lookup(self, name: str, kind: int) -> SymbolTableNode: + """Look up a definition from the symbol table with the given name. + TODO remove kind argument + """ + if name in self.globals: + return self.globals[name] + else: + b = self.globals.get('__builtins__', None) + if b: + table = cast(MypyFile, b.node).names + if name in table: + return table[name] + raise KeyError('Failed lookup: {}'.format(name)) + + def lookup_qualified(self, name: str) -> SymbolTableNode: + if '.' not in name: + return self.lookup(name, GDEF) # FIX kind + else: + parts = name.split('.') + n = self.modules[parts[0]] + for i in range(1, len(parts) - 1): + n = cast(MypyFile, n.names.get(parts[i], None).node) + return n.names[parts[-1]] + + def enter_partial_types(self) -> None: + """Push a new scope for collecting partial types.""" + self.partial_types.append({}) + + def leave_partial_types(self) -> None: + """Pop partial type scope. + + Also report errors for variables which still have partial + types, i.e. we couldn't infer a complete type. + """ + partial_types = self.partial_types.pop() + if not self.current_node_deferred: + for var, context in partial_types.items(): + if experiments.STRICT_OPTIONAL and cast(PartialType, var.type).type is None: + # None partial type: assume variable is intended to have type None + var.type = NoneTyp() + else: + self.msg.fail(messages.NEED_ANNOTATION_FOR_VAR, context) + var.type = AnyType() + + def find_partial_types(self, var: Var) -> Optional[Dict[Var, Context]]: + for partial_types in reversed(self.partial_types): + if var in partial_types: + return partial_types + return None + + def is_within_function(self) -> bool: + """Are we currently type checking within a function? + + I.e. not at class body or at the top level. + """ + return self.return_types != [] + + def check_not_void(self, typ: Type, context: Context) -> None: + """Generate an error if the type is Void.""" + if isinstance(typ, Void): + self.msg.does_not_return_value(typ, context) + + def temp_node(self, t: Type, context: Context = None) -> Node: + """Create a temporary node with the given, fixed type.""" + temp = TempNode(t) + if context: + temp.set_line(context.get_line()) + return temp + + def fail(self, msg: str, context: Context) -> None: + """Produce an error message.""" + self.msg.fail(msg, context) + + def iterable_item_type(self, instance: Instance) -> Type: + iterable = map_instance_to_supertype( + instance, + self.lookup_typeinfo('typing.Iterable')) + return iterable.args[0] + + def function_type(self, func: FuncBase) -> FunctionLike: + return function_type(func, self.named_type('builtins.function')) + + def method_type(self, func: FuncBase) -> FunctionLike: + return method_type_with_fallback(func, self.named_type('builtins.function')) + + +# Data structure returned by find_isinstance_check representing +# information learned from the truth or falsehood of a condition. The +# dict maps nodes representing expressions like 'a[0].x' to their +# refined types under the assumption that the condition has a +# particular truth value. A value of None means that the condition can +# never have that truth value. + +# NB: The keys of this dict are nodes in the original source program, +# which are compared by reference equality--effectively, being *the +# same* expression of the program, not just two identical expressions +# (such as two references to the same variable). TODO: it would +# probably be better to have the dict keyed by the nodes' literal_hash +# field instead. + +# NB: This should be `TypeMap = Optional[Dict[Node, Type]]`! +# But see https://github.com/python/mypy/issues/1637 +TypeMap = Dict[Node, Type] + + +def conditional_type_map(expr: Node, + current_type: Optional[Type], + proposed_type: Optional[Type], + *, + weak: bool = False + ) -> Tuple[TypeMap, TypeMap]: + """Takes in an expression, the current type of the expression, and a + proposed type of that expression. + + Returns a 2-tuple: The first element is a map from the expression to + the proposed type, if the expression can be the proposed type. The + second element is a map from the expression to the type it would hold + if it was not the proposed type, if any.""" + if proposed_type: + if current_type: + if is_proper_subtype(current_type, proposed_type): + return {expr: proposed_type}, None + elif not is_overlapping_types(current_type, proposed_type): + return None, {expr: current_type} + else: + remaining_type = restrict_subtype_away(current_type, proposed_type) + return {expr: proposed_type}, {expr: remaining_type} + else: + return {expr: proposed_type}, {} + else: + # An isinstance check, but we don't understand the type + if weak: + return {expr: AnyType()}, {expr: current_type} + else: + return {}, {} + + +def is_literal_none(n: Node) -> bool: + return isinstance(n, NameExpr) and n.fullname == 'builtins.None' + + +def and_conditional_maps(m1: TypeMap, m2: TypeMap) -> TypeMap: + """Calculate what information we can learn from the truth of (e1 and e2) + in terms of the information that we can learn from the truth of e1 and + the truth of e2. + """ + + if m1 is None or m2 is None: + # One of the conditions can never be true. + return None + # Both conditions can be true; combine the information. Anything + # we learn from either conditions's truth is valid. If the same + # expression's type is refined by both conditions, we somewhat + # arbitrarily give precedence to m2. (In the future, we could use + # an intersection type.) + result = m2.copy() + m2_keys = set(n2.literal_hash for n2 in m2) + for n1 in m1: + if n1.literal_hash not in m2_keys: + result[n1] = m1[n1] + return result + + +def or_conditional_maps(m1: TypeMap, m2: TypeMap) -> TypeMap: + """Calculate what information we can learn from the truth of (e1 or e2) + in terms of the information that we can learn from the truth of e1 and + the truth of e2. + """ + + if m1 is None: + return m2 + if m2 is None: + return m1 + # Both conditions can be true. Combine information about + # expressions whose type is refined by both conditions. (We do not + # learn anything about expressions whose type is refined by only + # one condition.) + result = {} + for n1 in m1: + for n2 in m2: + if n1.literal_hash == n2.literal_hash: + result[n1] = UnionType.make_simplified_union([m1[n1], m2[n2]]) + return result + + +def find_isinstance_check(node: Node, + type_map: Dict[Node, Type], + weak: bool=False + ) -> Tuple[TypeMap, TypeMap]: + """Find any isinstance checks (within a chain of ands). Includes + implicit and explicit checks for None. + + Return value is a map of variables to their types if the condition + is true and a map of variables to their types if the condition is false. + + If either of the values in the tuple is None, then that particular + branch can never occur. + + Guaranteed to not return None, None. (But may return {}, {}) + """ + if isinstance(node, CallExpr): + if refers_to_fullname(node.callee, 'builtins.isinstance'): + expr = node.args[0] + if expr.literal == LITERAL_TYPE: + vartype = type_map[expr] + type = get_isinstance_type(node.args[1], type_map) + return conditional_type_map(expr, vartype, type, weak=weak) + elif (isinstance(node, ComparisonExpr) and any(is_literal_none(n) for n in node.operands) and + experiments.STRICT_OPTIONAL): + # Check for `x is None` and `x is not None`. + is_not = node.operators == ['is not'] + if is_not or node.operators == ['is']: + if_vars = {} # type: Dict[Node, Type] + else_vars = {} # type: Dict[Node, Type] + for expr in node.operands: + if expr.literal == LITERAL_TYPE and not is_literal_none(expr) and expr in type_map: + # This should only be true at most once: there should be + # two elements in node.operands, and at least one of them + # should represent a None. + vartype = type_map[expr] + if_vars, else_vars = conditional_type_map(expr, vartype, NoneTyp(), weak=weak) + break + + if is_not: + if_vars, else_vars = else_vars, if_vars + return if_vars, else_vars + elif isinstance(node, RefExpr) and experiments.STRICT_OPTIONAL: + # The type could be falsy, so we can't deduce anything new about the else branch + vartype = type_map[node] + _, if_vars = conditional_type_map(node, vartype, NoneTyp(), weak=weak) + return if_vars, {} + elif isinstance(node, OpExpr) and node.op == 'and': + left_if_vars, left_else_vars = find_isinstance_check( + node.left, + type_map, + weak, + ) + + right_if_vars, right_else_vars = find_isinstance_check( + node.right, + type_map, + weak, + ) + + # (e1 and e2) is true if both e1 and e2 are true, + # and false if at least one of e1 and e2 is false. + return (and_conditional_maps(left_if_vars, right_if_vars), + or_conditional_maps(left_else_vars, right_else_vars)) + elif isinstance(node, OpExpr) and node.op == 'or': + left_if_vars, left_else_vars = find_isinstance_check( + node.left, + type_map, + weak, + ) + + right_if_vars, right_else_vars = find_isinstance_check( + node.right, + type_map, + weak, + ) + + # (e1 or e2) is true if at least one of e1 or e2 is true, + # and false if both e1 and e2 are false. + return (or_conditional_maps(left_if_vars, right_if_vars), + and_conditional_maps(left_else_vars, right_else_vars)) + elif isinstance(node, UnaryExpr) and node.op == 'not': + left, right = find_isinstance_check(node.expr, type_map, weak) + return right, left + + # Not a supported isinstance check + return {}, {} + + +def get_isinstance_type(node: Node, type_map: Dict[Node, Type]) -> Type: + type = type_map[node] + + if isinstance(type, TupleType): + all_types = type.items + else: + all_types = [type] + + types = [] # type: List[Type] + + for type in all_types: + if isinstance(type, FunctionLike): + if type.is_type_obj(): + # Type variables may be present -- erase them, which is the best + # we can do (outside disallowing them here). + type = erase_typevars(type.items()[0].ret_type) + + types.append(type) + + if len(types) == 0: + return None + elif len(types) == 1: + return types[0] + else: + return UnionType(types) + + +def expand_node(defn: Node, map: Dict[TypeVarId, Type]) -> Node: + visitor = TypeTransformVisitor(map) + return defn.accept(visitor) + + +def expand_func(defn: FuncItem, map: Dict[TypeVarId, Type]) -> FuncItem: + return cast(FuncItem, expand_node(defn, map)) + + +class TypeTransformVisitor(TransformVisitor): + def __init__(self, map: Dict[TypeVarId, Type]) -> None: + super().__init__() + self.map = map + + def type(self, type: Type) -> Type: + return expand_type(type, self.map) + + +def is_unsafe_overlapping_signatures(signature: Type, other: Type) -> bool: + """Check if two signatures may be unsafely overlapping. + + Two signatures s and t are overlapping if both can be valid for the same + statically typed values and the return types are incompatible. + + Assume calls are first checked against 'signature', then against 'other'. + Thus if 'signature' is more general than 'other', there is no unsafe + overlapping. + + TODO If argument types vary covariantly, the return type may vary + covariantly as well. + """ + if isinstance(signature, CallableType): + if isinstance(other, CallableType): + # TODO varargs + # TODO keyword args + # TODO erasure + # TODO allow to vary covariantly + # Check if the argument counts are overlapping. + min_args = max(signature.min_args, other.min_args) + max_args = min(len(signature.arg_types), len(other.arg_types)) + if min_args > max_args: + # Argument counts are not overlapping. + return False + # Signatures are overlapping iff if they are overlapping for the + # smallest common argument count. + for i in range(min_args): + t1 = signature.arg_types[i] + t2 = other.arg_types[i] + if not is_overlapping_types(t1, t2): + return False + # All arguments types for the smallest common argument count are + # overlapping => the signature is overlapping. The overlapping is + # safe if the return types are identical. + if is_same_type(signature.ret_type, other.ret_type): + return False + # If the first signature has more general argument types, the + # latter will never be called + if is_more_general_arg_prefix(signature, other): + return False + return not is_more_precise_signature(signature, other) + return True + + +def is_more_general_arg_prefix(t: FunctionLike, s: FunctionLike) -> bool: + """Does t have wider arguments than s?""" + # TODO should an overload with additional items be allowed to be more + # general than one with fewer items (or just one item)? + # TODO check argument kinds + if isinstance(t, CallableType): + if isinstance(s, CallableType): + return all(is_proper_subtype(args, argt) + for argt, args in zip(t.arg_types, s.arg_types)) + elif isinstance(t, FunctionLike): + if isinstance(s, FunctionLike): + if len(t.items()) == len(s.items()): + return all(is_same_arg_prefix(items, itemt) + for items, itemt in zip(t.items(), s.items())) + return False + + +def is_same_arg_prefix(t: CallableType, s: CallableType) -> bool: + # TODO check argument kinds + return all(is_same_type(argt, args) + for argt, args in zip(t.arg_types, s.arg_types)) + + +def is_more_precise_signature(t: CallableType, s: CallableType) -> bool: + """Is t more precise than s? + + A signature t is more precise than s if all argument types and the return + type of t are more precise than the corresponding types in s. + + Assume that the argument kinds and names are compatible, and that the + argument counts are overlapping. + """ + # TODO generic function types + # Only consider the common prefix of argument types. + for argt, args in zip(t.arg_types, s.arg_types): + if not is_more_precise(argt, args): + return False + return is_more_precise(t.ret_type, s.ret_type) + + +def infer_operator_assignment_method(type: Type, operator: str) -> str: + """Return the method used for operator assignment for given value type. + + For example, if operator is '+', return '__iadd__' or '__add__' depending + on which method is supported by the type. + """ + method = nodes.op_methods[operator] + if isinstance(type, Instance): + if operator in nodes.ops_with_inplace_method: + inplace = '__i' + method[2:] + if type.type.has_readable_member(inplace): + method = inplace + return method + + +def is_valid_inferred_type(typ: Type) -> bool: + """Is an inferred type valid? + + Examples of invalid types include the None type or a type with a None component. + """ + if is_same_type(typ, NoneTyp()): + # With strict Optional checking, we *may* eventually infer NoneTyp, but + # we only do that if we can't infer a specific Optional type. This + # resolution happens in leave_partial_types when we pop a partial types + # scope. + return False + if is_same_type(typ, UninhabitedType()): + return False + elif isinstance(typ, Instance): + for arg in typ.args: + if not is_valid_inferred_type(arg): + return False + elif isinstance(typ, TupleType): + for item in typ.items: + if not is_valid_inferred_type(item): + return False + return True diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 97a50142901eb..2fee197d9c86d 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1,1822 +1,1822 @@ -"""Expression type checker. This file is conceptually part of TypeChecker.""" - -from typing import cast, Dict, List, Tuple, Callable, Union, Optional - -from mypy.types import ( - Type, AnyType, CallableType, Overloaded, NoneTyp, Void, TypeVarDef, - TupleType, Instance, TypeVarId, TypeVarType, ErasedType, UnionType, - PartialType, DeletedType, UnboundType, UninhabitedType, TypeType -) -from mypy.nodes import ( - NameExpr, RefExpr, Var, FuncDef, OverloadedFuncDef, TypeInfo, CallExpr, - Node, MemberExpr, IntExpr, StrExpr, BytesExpr, UnicodeExpr, FloatExpr, - OpExpr, UnaryExpr, IndexExpr, CastExpr, RevealTypeExpr, TypeApplication, ListExpr, - TupleExpr, DictExpr, FuncExpr, SuperExpr, SliceExpr, Context, - ListComprehension, GeneratorExpr, SetExpr, MypyFile, Decorator, - ConditionalExpr, ComparisonExpr, TempNode, SetComprehension, - DictionaryComprehension, ComplexExpr, EllipsisExpr, - TypeAliasExpr, BackquoteExpr, ARG_POS, ARG_NAMED, ARG_STAR2 -) -from mypy.nodes import function_type -from mypy import nodes -import mypy.checker -from mypy import types -from mypy.sametypes import is_same_type -from mypy.erasetype import replace_meta_vars -from mypy.messages import MessageBuilder -from mypy import messages -from mypy.infer import infer_type_arguments, infer_function_type_arguments -from mypy import join -from mypy.subtypes import is_subtype, is_equivalent -from mypy import applytype -from mypy import erasetype -from mypy.checkmember import analyze_member_access, type_object_type -from mypy.semanal import self_type -from mypy.constraints import get_actual_type -from mypy.checkstrformat import StringFormatterChecker -from mypy.expandtype import expand_type -import mypy.checkexpr - -from mypy import experiments - -# Type of callback user for checking individual function arguments. See -# check_args() below for details. -ArgChecker = Callable[[Type, Type, int, Type, int, int, CallableType, Context, MessageBuilder], - None] - - -class Finished(Exception): - """Raised if we can terminate overload argument check early (no match).""" - - -class ExpressionChecker: - """Expression type checker. - - This class works closely together with checker.TypeChecker. - """ - - # Some services are provided by a TypeChecker instance. - chk = None # type: mypy.checker.TypeChecker - # This is shared with TypeChecker, but stored also here for convenience. - msg = None # type: MessageBuilder - - strfrm_checker = None # type: mypy.checkstrformat.StringFormatterChecker - - def __init__(self, - chk: 'mypy.checker.TypeChecker', - msg: MessageBuilder) -> None: - """Construct an expression type checker.""" - self.chk = chk - self.msg = msg - self.strfrm_checker = mypy.checkexpr.StringFormatterChecker(self, self.chk, self.msg) - - def visit_name_expr(self, e: NameExpr) -> Type: - """Type check a name expression. - - It can be of any kind: local, member or global. - """ - result = self.analyze_ref_expr(e) - return self.chk.narrow_type_from_binder(e, result) - - def analyze_ref_expr(self, e: RefExpr, lvalue: bool = False) -> Type: - result = None # type: Type - node = e.node - if isinstance(node, Var): - # Variable reference. - result = self.analyze_var_ref(node, e) - if isinstance(result, PartialType): - if result.type is None: - # 'None' partial type. It has a well-defined type. In an lvalue context - # we want to preserve the knowledge of it being a partial type. - if not lvalue: - result = NoneTyp() - else: - partial_types = self.chk.find_partial_types(node) - if partial_types is not None and not self.chk.current_node_deferred: - context = partial_types[node] - self.msg.fail(messages.NEED_ANNOTATION_FOR_VAR, context) - result = AnyType() - elif isinstance(node, FuncDef): - # Reference to a global function. - result = function_type(node, self.named_type('builtins.function')) - elif isinstance(node, OverloadedFuncDef): - result = node.type - elif isinstance(node, TypeInfo): - # Reference to a type object. - result = type_object_type(node, self.named_type) - elif isinstance(node, MypyFile): - # Reference to a module object. - result = self.named_type('builtins.module') - elif isinstance(node, Decorator): - result = self.analyze_var_ref(node.var, e) - else: - # Unknown reference; use any type implicitly to avoid - # generating extra type errors. - result = AnyType() - return result - - def analyze_var_ref(self, var: Var, context: Context) -> Type: - if not var.type: - if not var.is_ready and self.chk.typing_mode_full(): - self.chk.handle_cannot_determine_type(var.name(), context) - # Implicit 'Any' type. - return AnyType() - else: - # Look up local type of variable with type (inferred or explicit). - val = self.chk.binder.get(var) - if val is None: - return var.type - else: - return val - - def visit_call_expr(self, e: CallExpr) -> Type: - """Type check a call expression.""" - if e.analyzed: - # It's really a special form that only looks like a call. - return self.accept(e.analyzed, self.chk.type_context[-1]) - self.try_infer_partial_type(e) - callee_type = self.accept(e.callee) - if (self.chk.options.disallow_untyped_calls and - self.chk.typing_mode_full() and - isinstance(callee_type, CallableType) - and callee_type.implicit): - return self.msg.untyped_function_call(callee_type, e) - return self.check_call_expr_with_callee_type(callee_type, e) - - # Types and methods that can be used to infer partial types. - item_args = {'builtins.list': ['append'], - 'builtins.set': ['add', 'discard'], - } - container_args = {'builtins.list': {'extend': ['builtins.list']}, - 'builtins.dict': {'update': ['builtins.dict']}, - 'builtins.set': {'update': ['builtins.set', 'builtins.list']}, - } - - def try_infer_partial_type(self, e: CallExpr) -> None: - if isinstance(e.callee, MemberExpr) and isinstance(e.callee.expr, RefExpr): - var = cast(Var, e.callee.expr.node) - partial_types = self.chk.find_partial_types(var) - if partial_types is not None and not self.chk.current_node_deferred: - partial_type = cast(PartialType, var.type) - if partial_type is None or partial_type.type is None: - # A partial None type -> can't infer anything. - return - typename = partial_type.type.fullname() - methodname = e.callee.name - # Sometimes we can infer a full type for a partial List, Dict or Set type. - # TODO: Don't infer argument expression twice. - if (typename in self.item_args and methodname in self.item_args[typename] - and e.arg_kinds == [ARG_POS]): - item_type = self.accept(e.args[0]) - full_item_type = UnionType.make_simplified_union( - [item_type, partial_type.inner_types[0]]) - if mypy.checker.is_valid_inferred_type(full_item_type): - var.type = self.chk.named_generic_type(typename, [full_item_type]) - del partial_types[var] - elif (typename in self.container_args - and methodname in self.container_args[typename] - and e.arg_kinds == [ARG_POS]): - arg_type = self.accept(e.args[0]) - if isinstance(arg_type, Instance): - arg_typename = arg_type.type.fullname() - if arg_typename in self.container_args[typename][methodname]: - full_item_types = [ - UnionType.make_simplified_union([item_type, prev_type]) - for item_type, prev_type - in zip(arg_type.args, partial_type.inner_types) - ] - if all(mypy.checker.is_valid_inferred_type(item_type) - for item_type in full_item_types): - var.type = self.chk.named_generic_type(typename, - list(full_item_types)) - del partial_types[var] - - def check_call_expr_with_callee_type(self, callee_type: Type, - e: CallExpr) -> Type: - """Type check call expression. - - The given callee type overrides the type of the callee - expression. - """ - return self.check_call(callee_type, e.args, e.arg_kinds, e, - e.arg_names, callable_node=e.callee)[0] - - def check_call(self, callee: Type, args: List[Node], - arg_kinds: List[int], context: Context, - arg_names: List[str] = None, - callable_node: Node = None, - arg_messages: MessageBuilder = None) -> Tuple[Type, Type]: - """Type check a call. - - Also infer type arguments if the callee is a generic function. - - Return (result type, inferred callee type). - - Arguments: - callee: type of the called value - args: actual argument expressions - arg_kinds: contains nodes.ARG_* constant for each argument in args - describing whether the argument is positional, *arg, etc. - arg_names: names of arguments (optional) - callable_node: associate the inferred callable type to this node, - if specified - arg_messages: TODO - """ - arg_messages = arg_messages or self.msg - if isinstance(callee, CallableType): - if callee.is_concrete_type_obj() and callee.type_object().is_abstract: - type = callee.type_object() - self.msg.cannot_instantiate_abstract_class( - callee.type_object().name(), type.abstract_attributes, - context) - - formal_to_actual = map_actuals_to_formals( - arg_kinds, arg_names, - callee.arg_kinds, callee.arg_names, - lambda i: self.accept(args[i])) - - if callee.is_generic(): - callee = freshen_generic_callable(callee) - callee = self.infer_function_type_arguments_using_context( - callee, context) - callee = self.infer_function_type_arguments( - callee, args, arg_kinds, formal_to_actual, context) - - arg_types = self.infer_arg_types_in_context2( - callee, args, arg_kinds, formal_to_actual) - - self.check_argument_count(callee, arg_types, arg_kinds, - arg_names, formal_to_actual, context, self.msg) - - self.check_argument_types(arg_types, arg_kinds, callee, - formal_to_actual, context, - messages=arg_messages) - - if (callee.is_type_obj() and (len(arg_types) == 1) - and is_equivalent(callee.ret_type, self.named_type('builtins.type'))): - callee = callee.copy_modified(ret_type=TypeType(arg_types[0])) - - if callable_node: - # Store the inferred callable type. - self.chk.store_type(callable_node, callee) - return callee.ret_type, callee - elif isinstance(callee, Overloaded): - # Type check arguments in empty context. They will be checked again - # later in a context derived from the signature; these types are - # only used to pick a signature variant. - self.msg.disable_errors() - arg_types = self.infer_arg_types_in_context(None, args) - self.msg.enable_errors() - - target = self.overload_call_target(arg_types, arg_kinds, arg_names, - callee, context, - messages=arg_messages) - return self.check_call(target, args, arg_kinds, context, arg_names, - arg_messages=arg_messages) - elif isinstance(callee, AnyType) or self.chk.typing_mode_none(): - self.infer_arg_types_in_context(None, args) - return AnyType(), AnyType() - elif isinstance(callee, UnionType): - self.msg.disable_type_names += 1 - results = [self.check_call(subtype, args, arg_kinds, context, arg_names, - arg_messages=arg_messages) - for subtype in callee.items] - self.msg.disable_type_names -= 1 - return (UnionType.make_simplified_union([res[0] for res in results]), - callee) - elif isinstance(callee, Instance): - call_function = analyze_member_access('__call__', callee, context, - False, False, self.named_type, self.not_ready_callback, - self.msg) - return self.check_call(call_function, args, arg_kinds, context, arg_names, - callable_node, arg_messages) - elif isinstance(callee, TypeVarType): - return self.check_call(callee.upper_bound, args, arg_kinds, context, arg_names, - callable_node, arg_messages) - elif isinstance(callee, TypeType): - # Pass the original Type[] as context since that's where errors should go. - item = self.analyze_type_type_callee(callee.item, callee) - return self.check_call(item, args, arg_kinds, context, arg_names, - callable_node, arg_messages) - else: - return self.msg.not_callable(callee, context), AnyType() - - def analyze_type_type_callee(self, item: Type, context: Context) -> Type: - """Analyze the callee X in X(...) where X is Type[item]. - - Return a Y that we can pass to check_call(Y, ...). - """ - if isinstance(item, AnyType): - return AnyType() - if isinstance(item, Instance): - return type_object_type(item.type, self.named_type) - if isinstance(item, UnionType): - return UnionType([self.analyze_type_type_callee(item, context) - for item in item.items], item.line) - if isinstance(item, TypeVarType): - # Pretend we're calling the typevar's upper bound, - # i.e. its constructor (a poor approximation for reality, - # but better than AnyType...), but replace the return type - # with typevar. - callee = self.analyze_type_type_callee(item.upper_bound, context) - if isinstance(callee, CallableType): - if callee.is_generic(): - callee = None - else: - callee = callee.copy_modified(ret_type=item) - elif isinstance(callee, Overloaded): - if callee.items()[0].is_generic(): - callee = None - else: - callee = Overloaded([c.copy_modified(ret_type=item) - for c in callee.items()]) - if callee: - return callee - - self.msg.unsupported_type_type(item, context) - return AnyType() - - def infer_arg_types_in_context(self, callee: Optional[CallableType], - args: List[Node]) -> List[Type]: - """Infer argument expression types using a callable type as context. - - For example, if callee argument 2 has type List[int], infer the - argument expression with List[int] type context. - """ - # TODO Always called with callee as None, i.e. empty context. - res = [] # type: List[Type] - - fixed = len(args) - if callee: - fixed = min(fixed, callee.max_fixed_args()) - - arg_type = None # type: Type - ctx = None # type: Type - for i, arg in enumerate(args): - if i < fixed: - if callee and i < len(callee.arg_types): - ctx = callee.arg_types[i] - arg_type = self.accept(arg, ctx) - else: - if callee and callee.is_var_arg: - arg_type = self.accept(arg, callee.arg_types[-1]) - else: - arg_type = self.accept(arg) - if has_erased_component(arg_type): - res.append(NoneTyp()) - else: - res.append(arg_type) - return res - - def infer_arg_types_in_context2( - self, callee: CallableType, args: List[Node], arg_kinds: List[int], - formal_to_actual: List[List[int]]) -> List[Type]: - """Infer argument expression types using a callable type as context. - - For example, if callee argument 2 has type List[int], infer the - argument expression with List[int] type context. - - Returns the inferred types of *actual arguments*. - """ - res = [None] * len(args) # type: List[Type] - - for i, actuals in enumerate(formal_to_actual): - for ai in actuals: - if arg_kinds[ai] not in (nodes.ARG_STAR, nodes.ARG_STAR2): - res[ai] = self.accept(args[ai], callee.arg_types[i]) - - # Fill in the rest of the argument types. - for i, t in enumerate(res): - if not t: - res[i] = self.accept(args[i]) - return res - - def infer_function_type_arguments_using_context( - self, callable: CallableType, error_context: Context) -> CallableType: - """Unify callable return type to type context to infer type vars. - - For example, if the return type is set[t] where 't' is a type variable - of callable, and if the context is set[int], return callable modified - by substituting 't' with 'int'. - """ - ctx = self.chk.type_context[-1] - if not ctx: - return callable - # The return type may have references to type metavariables that - # we are inferring right now. We must consider them as indeterminate - # and they are not potential results; thus we replace them with the - # special ErasedType type. On the other hand, class type variables are - # valid results. - erased_ctx = replace_meta_vars(ctx, ErasedType()) - ret_type = callable.ret_type - if isinstance(ret_type, TypeVarType): - if ret_type.values or (not isinstance(ctx, Instance) or - not cast(Instance, ctx).args): - # The return type is a type variable. If it has values, we can't easily restrict - # type inference to conform to the valid values. If it's unrestricted, we could - # infer a too general type for the type variable if we use context, and this could - # result in confusing and spurious type errors elsewhere. - # - # Give up and just use function arguments for type inference. As an exception, - # if the context is a generic instance type, actually use it as context, as - # this *seems* to usually be the reasonable thing to do. - # - # See also github issues #462 and #360. - ret_type = NoneTyp() - args = infer_type_arguments(callable.type_var_ids(), ret_type, erased_ctx) - # Only substitute non-None and non-erased types. - new_args = [] # type: List[Type] - for arg in args: - if isinstance(arg, (NoneTyp, UninhabitedType)) or has_erased_component(arg): - new_args.append(None) - else: - new_args.append(arg) - return cast(CallableType, self.apply_generic_arguments(callable, new_args, - error_context)) - - def infer_function_type_arguments(self, callee_type: CallableType, - args: List[Node], - arg_kinds: List[int], - formal_to_actual: List[List[int]], - context: Context) -> CallableType: - """Infer the type arguments for a generic callee type. - - Infer based on the types of arguments. - - Return a derived callable type that has the arguments applied. - """ - if not self.chk.typing_mode_none(): - # Disable type errors during type inference. There may be errors - # due to partial available context information at this time, but - # these errors can be safely ignored as the arguments will be - # inferred again later. - self.msg.disable_errors() - - arg_types = self.infer_arg_types_in_context2( - callee_type, args, arg_kinds, formal_to_actual) - - self.msg.enable_errors() - - arg_pass_nums = self.get_arg_infer_passes( - callee_type.arg_types, formal_to_actual, len(args)) - - pass1_args = [] # type: List[Optional[Type]] - for i, arg in enumerate(arg_types): - if arg_pass_nums[i] > 1: - pass1_args.append(None) - else: - pass1_args.append(arg) - - inferred_args = infer_function_type_arguments( - callee_type, pass1_args, arg_kinds, formal_to_actual, - strict=self.chk.typing_mode_full()) # type: List[Type] - - if 2 in arg_pass_nums: - # Second pass of type inference. - (callee_type, - inferred_args) = self.infer_function_type_arguments_pass2( - callee_type, args, arg_kinds, formal_to_actual, - inferred_args, context) - - if callee_type.special_sig == 'dict' and len(inferred_args) == 2 and ( - ARG_NAMED in arg_kinds or ARG_STAR2 in arg_kinds): - # HACK: Infer str key type for dict(...) with keyword args. The type system - # can't represent this so we special case it, as this is a pretty common - # thing. This doesn't quite work with all possible subclasses of dict - # if they shuffle type variables around, as we assume that there is a 1-1 - # correspondence with dict type variables. This is a marginal issue and - # a little tricky to fix so it's left unfixed for now. - if isinstance(inferred_args[0], (NoneTyp, UninhabitedType)): - inferred_args[0] = self.named_type('builtins.str') - elif not is_subtype(self.named_type('builtins.str'), inferred_args[0]): - self.msg.fail(messages.KEYWORD_ARGUMENT_REQUIRES_STR_KEY_TYPE, - context) - else: - # In dynamically typed functions use implicit 'Any' types for - # type variables. - inferred_args = [AnyType()] * len(callee_type.variables) - return self.apply_inferred_arguments(callee_type, inferred_args, - context) - - def infer_function_type_arguments_pass2( - self, callee_type: CallableType, - args: List[Node], - arg_kinds: List[int], - formal_to_actual: List[List[int]], - inferred_args: List[Type], - context: Context) -> Tuple[CallableType, List[Type]]: - """Perform second pass of generic function type argument inference. - - The second pass is needed for arguments with types such as Callable[[T], S], - where both T and S are type variables, when the actual argument is a - lambda with inferred types. The idea is to infer the type variable T - in the first pass (based on the types of other arguments). This lets - us infer the argument and return type of the lambda expression and - thus also the type variable S in this second pass. - - Return (the callee with type vars applied, inferred actual arg types). - """ - # None or erased types in inferred types mean that there was not enough - # information to infer the argument. Replace them with None values so - # that they are not applied yet below. - for i, arg in enumerate(inferred_args): - if isinstance(arg, (NoneTyp, UninhabitedType)) or has_erased_component(arg): - inferred_args[i] = None - - callee_type = cast(CallableType, self.apply_generic_arguments( - callee_type, inferred_args, context)) - arg_types = self.infer_arg_types_in_context2( - callee_type, args, arg_kinds, formal_to_actual) - - inferred_args = infer_function_type_arguments( - callee_type, arg_types, arg_kinds, formal_to_actual) - - return callee_type, inferred_args - - def get_arg_infer_passes(self, arg_types: List[Type], - formal_to_actual: List[List[int]], - num_actuals: int) -> List[int]: - """Return pass numbers for args for two-pass argument type inference. - - For each actual, the pass number is either 1 (first pass) or 2 (second - pass). - - Two-pass argument type inference primarily lets us infer types of - lambdas more effectively. - """ - res = [1] * num_actuals - for i, arg in enumerate(arg_types): - if arg.accept(ArgInferSecondPassQuery()): - for j in formal_to_actual[i]: - res[j] = 2 - return res - - def apply_inferred_arguments(self, callee_type: CallableType, - inferred_args: List[Type], - context: Context) -> CallableType: - """Apply inferred values of type arguments to a generic function. - - Inferred_args contains the values of function type arguments. - """ - # Report error if some of the variables could not be solved. In that - # case assume that all variables have type Any to avoid extra - # bogus error messages. - for i, inferred_type in enumerate(inferred_args): - if not inferred_type or has_erased_component(inferred_type): - # Could not infer a non-trivial type for a type variable. - self.msg.could_not_infer_type_arguments( - callee_type, i + 1, context) - inferred_args = [AnyType()] * len(inferred_args) - # Apply the inferred types to the function type. In this case the - # return type must be CallableType, since we give the right number of type - # arguments. - return cast(CallableType, self.apply_generic_arguments(callee_type, - inferred_args, context)) - - def check_argument_count(self, callee: CallableType, actual_types: List[Type], - actual_kinds: List[int], actual_names: List[str], - formal_to_actual: List[List[int]], - context: Context, - messages: Optional[MessageBuilder]) -> bool: - """Check that there is a value for all required arguments to a function. - - Also check that there are no duplicate values for arguments. Report found errors - using 'messages' if it's not None. - - Return False if there were any errors. Otherwise return True - """ - # TODO(jukka): We could return as soon as we find an error if messages is None. - formal_kinds = callee.arg_kinds - - # Collect list of all actual arguments matched to formal arguments. - all_actuals = [] # type: List[int] - for actuals in formal_to_actual: - all_actuals.extend(actuals) - - is_unexpected_arg_error = False # Keep track of errors to avoid duplicate errors. - ok = True # False if we've found any error. - for i, kind in enumerate(actual_kinds): - if i not in all_actuals and ( - kind != nodes.ARG_STAR or - not is_empty_tuple(actual_types[i])): - # Extra actual: not matched by a formal argument. - ok = False - if kind != nodes.ARG_NAMED: - if messages: - messages.too_many_arguments(callee, context) - else: - if messages: - messages.unexpected_keyword_argument( - callee, actual_names[i], context) - is_unexpected_arg_error = True - elif kind == nodes.ARG_STAR and ( - nodes.ARG_STAR not in formal_kinds): - actual_type = actual_types[i] - if isinstance(actual_type, TupleType): - if all_actuals.count(i) < len(actual_type.items): - # Too many tuple items as some did not match. - if messages: - messages.too_many_arguments(callee, context) - ok = False - # *args can be applied even if the function takes a fixed - # number of positional arguments. This may succeed at runtime. - - for i, kind in enumerate(formal_kinds): - if kind == nodes.ARG_POS and (not formal_to_actual[i] and - not is_unexpected_arg_error): - # No actual for a mandatory positional formal. - if messages: - messages.too_few_arguments(callee, context, actual_names) - ok = False - elif kind in [nodes.ARG_POS, nodes.ARG_OPT, - nodes.ARG_NAMED] and is_duplicate_mapping( - formal_to_actual[i], actual_kinds): - if (self.chk.typing_mode_full() or - isinstance(actual_types[formal_to_actual[i][0]], TupleType)): - if messages: - messages.duplicate_argument_value(callee, i, context) - ok = False - elif (kind == nodes.ARG_NAMED and formal_to_actual[i] and - actual_kinds[formal_to_actual[i][0]] != nodes.ARG_NAMED): - # Positional argument when expecting a keyword argument. - if messages: - messages.too_many_positional_arguments(callee, context) - ok = False - return ok - - def check_argument_types(self, arg_types: List[Type], arg_kinds: List[int], - callee: CallableType, - formal_to_actual: List[List[int]], - context: Context, - messages: MessageBuilder = None, - check_arg: ArgChecker = None) -> None: - """Check argument types against a callable type. - - Report errors if the argument types are not compatible. - """ - messages = messages or self.msg - check_arg = check_arg or self.check_arg - # Keep track of consumed tuple *arg items. - tuple_counter = [0] - for i, actuals in enumerate(formal_to_actual): - for actual in actuals: - arg_type = arg_types[actual] - # Check that a *arg is valid as varargs. - if (arg_kinds[actual] == nodes.ARG_STAR and - not self.is_valid_var_arg(arg_type)): - messages.invalid_var_arg(arg_type, context) - if (arg_kinds[actual] == nodes.ARG_STAR2 and - not self.is_valid_keyword_var_arg(arg_type)): - messages.invalid_keyword_var_arg(arg_type, context) - # Get the type of an individual actual argument (for *args - # and **args this is the item type, not the collection type). - actual_type = get_actual_type(arg_type, arg_kinds[actual], - tuple_counter) - check_arg(actual_type, arg_type, arg_kinds[actual], - callee.arg_types[i], - actual + 1, i + 1, callee, context, messages) - - # There may be some remaining tuple varargs items that haven't - # been checked yet. Handle them. - if (callee.arg_kinds[i] == nodes.ARG_STAR and - arg_kinds[actual] == nodes.ARG_STAR and - isinstance(arg_types[actual], TupleType)): - tuplet = cast(TupleType, arg_types[actual]) - while tuple_counter[0] < len(tuplet.items): - actual_type = get_actual_type(arg_type, - arg_kinds[actual], - tuple_counter) - check_arg(actual_type, arg_type, arg_kinds[actual], - callee.arg_types[i], - actual + 1, i + 1, callee, context, messages) - - def check_arg(self, caller_type: Type, original_caller_type: Type, - caller_kind: int, - callee_type: Type, n: int, m: int, callee: CallableType, - context: Context, messages: MessageBuilder) -> None: - """Check the type of a single argument in a call.""" - if isinstance(caller_type, Void): - messages.does_not_return_value(caller_type, context) - elif isinstance(caller_type, DeletedType): - messages.deleted_as_rvalue(caller_type, context) - elif not is_subtype(caller_type, callee_type): - messages.incompatible_argument(n, m, callee, original_caller_type, - caller_kind, context) - - def overload_call_target(self, arg_types: List[Type], arg_kinds: List[int], - arg_names: List[str], - overload: Overloaded, context: Context, - messages: MessageBuilder = None) -> Type: - """Infer the correct overload item to call with given argument types. - - The return value may be CallableType or AnyType (if an unique item - could not be determined). - """ - messages = messages or self.msg - # TODO: For overlapping signatures we should try to get a more precise - # result than 'Any'. - match = [] # type: List[CallableType] - best_match = 0 - for typ in overload.items(): - similarity = self.erased_signature_similarity(arg_types, arg_kinds, arg_names, - typ, context=context) - if similarity > 0 and similarity >= best_match: - if (match and not is_same_type(match[-1].ret_type, - typ.ret_type) and - not mypy.checker.is_more_precise_signature( - match[-1], typ)): - # Ambiguous return type. Either the function overload is - # overlapping (which we don't handle very well here) or the - # caller has provided some Any argument types; in either - # case we'll fall back to Any. It's okay to use Any types - # in calls. - # - # Overlapping overload items are generally fine if the - # overlapping is only possible when there is multiple - # inheritance, as this is rare. See docstring of - # mypy.meet.is_overlapping_types for more about this. - # - # Note that there is no ambiguity if the items are - # covariant in both argument types and return types with - # respect to type precision. We'll pick the best/closest - # match. - # - # TODO: Consider returning a union type instead if the - # overlapping is NOT due to Any types? - return AnyType() - else: - match.append(typ) - best_match = max(best_match, similarity) - if not match: - messages.no_variant_matches_arguments(overload, arg_types, context) - return AnyType() - else: - if len(match) == 1: - return match[0] - else: - # More than one signature matches. Pick the first *non-erased* - # matching signature, or default to the first one if none - # match. - for m in match: - if self.match_signature_types(arg_types, arg_kinds, arg_names, m, - context=context): - return m - return match[0] - - def erased_signature_similarity(self, arg_types: List[Type], arg_kinds: List[int], - arg_names: List[str], callee: CallableType, - context: Context) -> int: - """Determine whether arguments could match the signature at runtime. - - Return similarity level (0 = no match, 1 = can match, 2 = non-promotion match). See - overload_arg_similarity for a discussion of similarity levels. - """ - formal_to_actual = map_actuals_to_formals(arg_kinds, - arg_names, - callee.arg_kinds, - callee.arg_names, - lambda i: arg_types[i]) - - if not self.check_argument_count(callee, arg_types, arg_kinds, arg_names, - formal_to_actual, None, None): - # Too few or many arguments -> no match. - return 0 - - similarity = 2 - - def check_arg(caller_type: Type, original_caller_type: Type, caller_kind: int, - callee_type: Type, n: int, m: int, callee: CallableType, - context: Context, messages: MessageBuilder) -> None: - nonlocal similarity - similarity = min(similarity, - overload_arg_similarity(caller_type, callee_type)) - if similarity == 0: - # No match -- exit early since none of the remaining work can change - # the result. - raise Finished - - try: - self.check_argument_types(arg_types, arg_kinds, callee, formal_to_actual, - context=context, check_arg=check_arg) - except Finished: - pass - - return similarity - - def match_signature_types(self, arg_types: List[Type], arg_kinds: List[int], - arg_names: List[str], callee: CallableType, - context: Context) -> bool: - """Determine whether arguments types match the signature. - - Assume that argument counts are compatible. - - Return True if arguments match. - """ - formal_to_actual = map_actuals_to_formals(arg_kinds, - arg_names, - callee.arg_kinds, - callee.arg_names, - lambda i: arg_types[i]) - ok = True - - def check_arg(caller_type: Type, original_caller_type: Type, caller_kind: int, - callee_type: Type, n: int, m: int, callee: CallableType, - context: Context, messages: MessageBuilder) -> None: - nonlocal ok - if not is_subtype(caller_type, callee_type): - ok = False - - self.check_argument_types(arg_types, arg_kinds, callee, formal_to_actual, - context=context, check_arg=check_arg) - return ok - - def apply_generic_arguments(self, callable: CallableType, types: List[Type], - context: Context) -> Type: - """Simple wrapper around mypy.applytype.apply_generic_arguments.""" - return applytype.apply_generic_arguments(callable, types, self.msg, context) - - def apply_generic_arguments2(self, overload: Overloaded, types: List[Type], - context: Context) -> Type: - items = [] # type: List[CallableType] - for item in overload.items(): - applied = self.apply_generic_arguments(item, types, context) - if isinstance(applied, CallableType): - items.append(applied) - else: - # There was an error. - return AnyType() - return Overloaded(items) - - def visit_member_expr(self, e: MemberExpr) -> Type: - """Visit member expression (of form e.id).""" - result = self.analyze_ordinary_member_access(e, False) - return self.chk.narrow_type_from_binder(e, result) - - def analyze_ordinary_member_access(self, e: MemberExpr, - is_lvalue: bool) -> Type: - """Analyse member expression or member lvalue.""" - if e.kind is not None: - # This is a reference to a module attribute. - return self.analyze_ref_expr(e) - else: - # This is a reference to a non-module attribute. - return analyze_member_access(e.name, self.accept(e.expr), e, - is_lvalue, False, - self.named_type, self.not_ready_callback, self.msg) - - def analyze_external_member_access(self, member: str, base_type: Type, - context: Context) -> Type: - """Analyse member access that is external, i.e. it cannot - refer to private definitions. Return the result type. - """ - # TODO remove; no private definitions in mypy - return analyze_member_access(member, base_type, context, False, False, - self.named_type, self.not_ready_callback, self.msg) - - def visit_int_expr(self, e: IntExpr) -> Type: - """Type check an integer literal (trivial).""" - return self.named_type('builtins.int') - - def visit_str_expr(self, e: StrExpr) -> Type: - """Type check a string literal (trivial).""" - return self.named_type('builtins.str') - - def visit_bytes_expr(self, e: BytesExpr) -> Type: - """Type check a bytes literal (trivial).""" - return self.named_type('builtins.bytes') - - def visit_unicode_expr(self, e: UnicodeExpr) -> Type: - """Type check a unicode literal (trivial).""" - return self.named_type('builtins.unicode') - - def visit_float_expr(self, e: FloatExpr) -> Type: - """Type check a float literal (trivial).""" - return self.named_type('builtins.float') - - def visit_complex_expr(self, e: ComplexExpr) -> Type: - """Type check a complex literal.""" - return self.named_type('builtins.complex') - - def visit_ellipsis(self, e: EllipsisExpr) -> Type: - """Type check '...'.""" - if self.chk.options.python_version[0] >= 3: - return self.named_type('builtins.ellipsis') - else: - # '...' is not valid in normal Python 2 code, but it can - # be used in stubs. The parser makes sure that we only - # get this far if we are in a stub, and we can safely - # return 'object' as ellipsis is special cased elsewhere. - # The builtins.ellipsis type does not exist in Python 2. - return self.named_type('builtins.object') - - def visit_op_expr(self, e: OpExpr) -> Type: - """Type check a binary operator expression.""" - if e.op == 'and' or e.op == 'or': - return self.check_boolean_op(e, e) - if e.op == '*' and isinstance(e.left, ListExpr): - # Expressions of form [...] * e get special type inference. - return self.check_list_multiply(e) - if e.op == '%' and isinstance(e.left, (StrExpr, BytesExpr)): - return self.strfrm_checker.check_str_interpolation(cast(StrExpr, e.left), e.right) - left_type = self.accept(e.left) - - if e.op in nodes.op_methods: - method = self.get_operator_method(e.op) - result, method_type = self.check_op(method, left_type, e.right, e, - allow_reverse=True) - e.method_type = method_type - return result - else: - raise RuntimeError('Unknown operator {}'.format(e.op)) - - def visit_comparison_expr(self, e: ComparisonExpr) -> Type: - """Type check a comparison expression. - - Comparison expressions are type checked consecutive-pair-wise - That is, 'a < b > c == d' is check as 'a < b and b > c and c == d' - """ - result = None # type: mypy.types.Type - - # Check each consecutive operand pair and their operator - for left, right, operator in zip(e.operands, e.operands[1:], e.operators): - left_type = self.accept(left) - - method_type = None # type: mypy.types.Type - - if operator == 'in' or operator == 'not in': - right_type = self.accept(right) # TODO only evaluate if needed - - # Keep track of whether we get type check errors (these won't be reported, they - # are just to verify whether something is valid typing wise). - local_errors = self.msg.copy() - local_errors.disable_count = 0 - sub_result, method_type = self.check_op_local('__contains__', right_type, - left, e, local_errors) - if isinstance(right_type, PartialType): - # We don't really know if this is an error or not, so just shut up. - pass - elif (local_errors.is_errors() and - # is_valid_var_arg is True for any Iterable - self.is_valid_var_arg(right_type)): - itertype = self.chk.analyze_iterable_item_type(right) - method_type = CallableType( - [left_type], - [nodes.ARG_POS], - [None], - self.chk.bool_type(), - self.named_type('builtins.function')) - sub_result = self.chk.bool_type() - if not is_subtype(left_type, itertype): - self.msg.unsupported_operand_types('in', left_type, right_type, e) - else: - self.msg.add_errors(local_errors) - if operator == 'not in': - sub_result = self.chk.bool_type() - elif operator in nodes.op_methods: - method = self.get_operator_method(operator) - sub_result, method_type = self.check_op(method, left_type, right, e, - allow_reverse=True) - - elif operator == 'is' or operator == 'is not': - sub_result = self.chk.bool_type() - method_type = None - else: - raise RuntimeError('Unknown comparison operator {}'.format(operator)) - - e.method_types.append(method_type) - - # Determine type of boolean-and of result and sub_result - if result is None: - result = sub_result - else: - # TODO: check on void needed? - self.check_not_void(sub_result, e) - result = join.join_types(result, sub_result) - - return result - - def get_operator_method(self, op: str) -> str: - if op == '/' and self.chk.options.python_version[0] == 2: - # TODO also check for "from __future__ import division" - return '__div__' - else: - return nodes.op_methods[op] - - def check_op_local(self, method: str, base_type: Type, arg: Node, - context: Context, local_errors: MessageBuilder) -> Tuple[Type, Type]: - """Type check a binary operation which maps to a method call. - - Return tuple (result type, inferred operator method type). - """ - method_type = analyze_member_access(method, base_type, context, False, False, - self.named_type, self.not_ready_callback, local_errors) - return self.check_call(method_type, [arg], [nodes.ARG_POS], - context, arg_messages=local_errors) - - def check_op(self, method: str, base_type: Type, arg: Node, - context: Context, - allow_reverse: bool = False) -> Tuple[Type, Type]: - """Type check a binary operation which maps to a method call. - - Return tuple (result type, inferred operator method type). - """ - # Use a local error storage for errors related to invalid argument - # type (but NOT other errors). This error may need to be suppressed - # for operators which support __rX methods. - local_errors = self.msg.copy() - local_errors.disable_count = 0 - if not allow_reverse or self.has_member(base_type, method): - result = self.check_op_local(method, base_type, arg, context, - local_errors) - if allow_reverse: - arg_type = self.chk.type_map[arg] - if isinstance(arg_type, AnyType): - # If the right operand has type Any, we can't make any - # conjectures about the type of the result, since the - # operand could have a __r method that returns anything. - - # However, in weak mode, we do make conjectures. - if not self.chk.typing_mode_weak(): - result = AnyType(), result[1] - success = not local_errors.is_errors() - else: - result = AnyType(), AnyType() - success = False - if success or not allow_reverse or isinstance(base_type, AnyType): - # We were able to call the normal variant of the operator method, - # or there was some problem not related to argument type - # validity, or the operator has no __rX method. In any case, we - # don't need to consider the __rX method. - self.msg.add_errors(local_errors) - return result - else: - # Calling the operator method was unsuccessful. Try the __rX - # method of the other operand instead. - rmethod = self.get_reverse_op_method(method) - arg_type = self.accept(arg) - if self.has_member(arg_type, rmethod): - method_type = self.analyze_external_member_access( - rmethod, arg_type, context) - temp = TempNode(base_type) - return self.check_call(method_type, [temp], [nodes.ARG_POS], - context) - else: - # No __rX method either. Do deferred type checking to produce - # error message that we may have missed previously. - # TODO Fix type checking an expression more than once. - return self.check_op_local(method, base_type, arg, context, - self.msg) - - def get_reverse_op_method(self, method: str) -> str: - if method == '__div__' and self.chk.options.python_version[0] == 2: - return '__rdiv__' - else: - return nodes.reverse_op_methods[method] - - def check_boolean_op(self, e: OpExpr, context: Context) -> Type: - """Type check a boolean operation ('and' or 'or').""" - - # A boolean operation can evaluate to either of the operands. - - # We use the current type context to guide the type inference of of - # the left operand. We also use the left operand type to guide the type - # inference of the right operand so that expressions such as - # '[1] or []' are inferred correctly. - ctx = self.chk.type_context[-1] - left_type = self.accept(e.left, ctx) - - if e.op == 'and': - right_map, _ = \ - mypy.checker.find_isinstance_check(e.left, self.chk.type_map, - self.chk.typing_mode_weak()) - elif e.op == 'or': - _, right_map = \ - mypy.checker.find_isinstance_check(e.left, self.chk.type_map, - self.chk.typing_mode_weak()) - else: - right_map = None - - with self.chk.binder.frame_context(): - if right_map: - for var, type in right_map.items(): - self.chk.binder.push(var, type) - - right_type = self.accept(e.right, left_type) - - self.check_not_void(left_type, context) - self.check_not_void(right_type, context) - return UnionType.make_simplified_union([left_type, right_type]) - - def check_list_multiply(self, e: OpExpr) -> Type: - """Type check an expression of form '[...] * e'. - - Type inference is special-cased for this common construct. - """ - right_type = self.accept(e.right) - if is_subtype(right_type, self.named_type('builtins.int')): - # Special case: [...] * . Use the type context of the - # OpExpr, since the multiplication does not affect the type. - left_type = self.accept(e.left, context=self.chk.type_context[-1]) - else: - left_type = self.accept(e.left) - result, method_type = self.check_op('__mul__', left_type, e.right, e) - e.method_type = method_type - return result - - def visit_unary_expr(self, e: UnaryExpr) -> Type: - """Type check an unary operation ('not', '-', '+' or '~').""" - operand_type = self.accept(e.expr) - op = e.op - if op == 'not': - self.check_not_void(operand_type, e) - result = self.chk.bool_type() # type: Type - elif op == '-': - method_type = self.analyze_external_member_access('__neg__', - operand_type, e) - result, method_type = self.check_call(method_type, [], [], e) - e.method_type = method_type - elif op == '+': - method_type = self.analyze_external_member_access('__pos__', - operand_type, e) - result, method_type = self.check_call(method_type, [], [], e) - e.method_type = method_type - else: - assert op == '~', "unhandled unary operator" - method_type = self.analyze_external_member_access('__invert__', - operand_type, e) - result, method_type = self.check_call(method_type, [], [], e) - e.method_type = method_type - return result - - def visit_index_expr(self, e: IndexExpr) -> Type: - """Type check an index expression (base[index]). - - It may also represent type application. - """ - result = self.visit_index_expr_helper(e) - return self.chk.narrow_type_from_binder(e, result) - - def visit_index_expr_helper(self, e: IndexExpr) -> Type: - if e.analyzed: - # It's actually a type application. - return self.accept(e.analyzed) - left_type = self.accept(e.base) - if isinstance(left_type, TupleType) and self.chk.typing_mode_full(): - # Special case for tuples. They support indexing only by integer - # literals. (Except in weak type checking mode.) - index = e.index - if isinstance(index, SliceExpr): - return self.visit_tuple_slice_helper(left_type, index) - - ok = False - if isinstance(index, IntExpr): - n = index.value - ok = True - elif isinstance(index, UnaryExpr): - if index.op == '-': - operand = index.expr - if isinstance(operand, IntExpr): - n = len(left_type.items) - operand.value - ok = True - if ok: - if n >= 0 and n < len(left_type.items): - return left_type.items[n] - else: - self.chk.fail(messages.TUPLE_INDEX_OUT_OF_RANGE, e) - return AnyType() - else: - self.chk.fail(messages.TUPLE_INDEX_MUST_BE_AN_INT_LITERAL, e) - return AnyType() - else: - result, method_type = self.check_op('__getitem__', left_type, e.index, e) - e.method_type = method_type - return result - - def visit_tuple_slice_helper(self, left_type: TupleType, slic: SliceExpr): - begin = None # type: int - end = None # type: int - stride = None # type:int - - if slic.begin_index: - begin = self._get_value(slic.begin_index) - if begin is None: - self.chk.fail( - messages.TUPLE_SLICE_MUST_BE_AN_INT_LITERAL, - slic.begin_index) - return AnyType() - - if slic.end_index: - end = self._get_value(slic.end_index) - if end is None: - self.chk.fail( - messages.TUPLE_SLICE_MUST_BE_AN_INT_LITERAL, - slic.end_index) - return AnyType() - - if slic.stride: - stride = self._get_value(slic.stride) - if stride is None: - self.chk.fail( - messages.TUPLE_SLICE_MUST_BE_AN_INT_LITERAL, - slic.stride) - return AnyType() - - return left_type.slice(begin, stride, end) - - def _get_value(self, index: Node) -> Optional[int]: - if isinstance(index, IntExpr): - return index.value - elif isinstance(index, UnaryExpr): - if index.op == '-': - operand = index.expr - if isinstance(operand, IntExpr): - return -1 * operand.value - return None - - def visit_cast_expr(self, expr: CastExpr) -> Type: - """Type check a cast expression.""" - source_type = self.accept(expr.expr, context=AnyType()) - target_type = expr.type - if self.chk.options.warn_redundant_casts and is_same_type(source_type, target_type): - self.msg.redundant_cast(target_type, expr) - if not self.is_valid_cast(source_type, target_type): - self.msg.invalid_cast(target_type, source_type, expr) - return target_type - - def is_valid_cast(self, source_type: Type, target_type: Type) -> bool: - """Is a cast from source_type to target_type meaningful?""" - return (isinstance(target_type, AnyType) or - (not isinstance(source_type, Void) and - not isinstance(target_type, Void))) - - def visit_reveal_type_expr(self, expr: RevealTypeExpr) -> Type: - """Type check a reveal_type expression.""" - revealed_type = self.accept(expr.expr) - self.msg.reveal_type(revealed_type, expr) - return revealed_type - - def visit_type_application(self, tapp: TypeApplication) -> Type: - """Type check a type application (expr[type, ...]).""" - self.chk.fail(messages.GENERIC_TYPE_NOT_VALID_AS_EXPRESSION, tapp) - return AnyType() - - def visit_type_alias_expr(self, alias: TypeAliasExpr) -> Type: - return AnyType() - - def visit_list_expr(self, e: ListExpr) -> Type: - """Type check a list expression [...].""" - return self.check_list_or_set_expr(e.items, 'builtins.list', '', - e) - - def visit_set_expr(self, e: SetExpr) -> Type: - return self.check_list_or_set_expr(e.items, 'builtins.set', '', e) - - def check_list_or_set_expr(self, items: List[Node], fullname: str, - tag: str, context: Context) -> Type: - # Translate into type checking a generic function call. - tvdef = TypeVarDef('T', -1, [], self.chk.object_type()) - tv = TypeVarType(tvdef) - constructor = CallableType( - [tv], - [nodes.ARG_STAR], - [None], - self.chk.named_generic_type(fullname, [tv]), - self.named_type('builtins.function'), - name=tag, - variables=[tvdef]) - return self.check_call(constructor, - items, - [nodes.ARG_POS] * len(items), context)[0] - - def visit_tuple_expr(self, e: TupleExpr) -> Type: - """Type check a tuple expression.""" - ctx = None # type: TupleType - # Try to determine type context for type inference. - if isinstance(self.chk.type_context[-1], TupleType): - t = self.chk.type_context[-1] - if len(t.items) == len(e.items): - ctx = t - # Infer item types. - items = [] # type: List[Type] - for i in range(len(e.items)): - item = e.items[i] - tt = None # type: Type - if not ctx: - tt = self.accept(item) - else: - tt = self.accept(item, ctx.items[i]) - self.check_not_void(tt, e) - items.append(tt) - fallback_item = join.join_type_list(items) - return TupleType(items, self.chk.named_generic_type('builtins.tuple', [fallback_item])) - - def visit_dict_expr(self, e: DictExpr) -> Type: - # Translate into type checking a generic function call. - ktdef = TypeVarDef('KT', -1, [], self.chk.object_type()) - vtdef = TypeVarDef('VT', -2, [], self.chk.object_type()) - kt = TypeVarType(ktdef) - vt = TypeVarType(vtdef) - # The callable type represents a function like this: - # - # def (*v: Tuple[kt, vt]) -> Dict[kt, vt]: ... - constructor = CallableType( - [TupleType([kt, vt], self.named_type('builtins.tuple'))], - [nodes.ARG_STAR], - [None], - self.chk.named_generic_type('builtins.dict', [kt, vt]), - self.named_type('builtins.function'), - name='', - variables=[ktdef, vtdef]) - # Synthesize function arguments. - args = [] # type: List[Node] - for key, value in e.items: - args.append(TupleExpr([key, value])) - return self.check_call(constructor, - args, - [nodes.ARG_POS] * len(args), e)[0] - - def visit_func_expr(self, e: FuncExpr) -> Type: - """Type check lambda expression.""" - inferred_type = self.infer_lambda_type_using_context(e) - if not inferred_type: - # No useful type context. - ret_type = e.expr().accept(self.chk) - if not e.arguments: - # Form 'lambda: e'; just use the inferred return type. - return CallableType([], [], [], ret_type, self.named_type('builtins.function')) - else: - # TODO: Consider reporting an error. However, this is fine if - # we are just doing the first pass in contextual type - # inference. - return AnyType() - else: - # Type context available. - self.chk.check_func_item(e, type_override=inferred_type) - if e.expr() not in self.chk.type_map: - self.accept(e.expr()) - ret_type = self.chk.type_map[e.expr()] - if isinstance(ret_type, NoneTyp): - # For "lambda ...: None", just use type from the context. - # Important when the context is Callable[..., None] which - # really means Void. See #1425. - return inferred_type - return replace_callable_return_type(inferred_type, ret_type) - - def infer_lambda_type_using_context(self, e: FuncExpr) -> CallableType: - """Try to infer lambda expression type using context. - - Return None if could not infer type. - """ - # TODO also accept 'Any' context - ctx = self.chk.type_context[-1] - if not ctx or not isinstance(ctx, CallableType): - return None - - # The context may have function type variables in it. We replace them - # since these are the type variables we are ultimately trying to infer; - # they must be considered as indeterminate. We use ErasedType since it - # does not affect type inference results (it is for purposes like this - # only). - ctx = replace_meta_vars(ctx, ErasedType()) - - callable_ctx = cast(CallableType, ctx) - - arg_kinds = [arg.kind for arg in e.arguments] - - if callable_ctx.is_ellipsis_args: - # Fill in Any arguments to match the arguments of the lambda. - callable_ctx = callable_ctx.copy_modified( - is_ellipsis_args=False, - arg_types=[AnyType()] * len(arg_kinds), - arg_kinds=arg_kinds - ) - - if callable_ctx.arg_kinds != arg_kinds: - # Incompatible context; cannot use it to infer types. - self.chk.fail(messages.CANNOT_INFER_LAMBDA_TYPE, e) - return None - - return callable_ctx - - def visit_super_expr(self, e: SuperExpr) -> Type: - """Type check a super expression (non-lvalue).""" - t = self.analyze_super(e, False) - return t - - def analyze_super(self, e: SuperExpr, is_lvalue: bool) -> Type: - """Type check a super expression.""" - if e.info and e.info.bases: - # TODO fix multiple inheritance etc - if len(e.info.mro) < 2: - self.chk.fail('Internal error: unexpected mro for {}: {}'.format( - e.info.name(), e.info.mro), e) - return AnyType() - for base in e.info.mro[1:]: - if e.name in base.names or base == e.info.mro[-1]: - if e.info.fallback_to_any and base == e.info.mro[-1]: - # There's an undefined base class, and we're - # at the end of the chain. That's not an error. - return AnyType() - if not self.chk.typing_mode_full(): - return AnyType() - return analyze_member_access(e.name, self_type(e.info), e, - is_lvalue, True, - self.named_type, self.not_ready_callback, - self.msg, base) - else: - # Invalid super. This has been reported by the semantic analyzer. - return AnyType() - - def visit_slice_expr(self, e: SliceExpr) -> Type: - for index in [e.begin_index, e.end_index, e.stride]: - if index: - t = self.accept(index) - self.chk.check_subtype(t, self.named_type('builtins.int'), - index, messages.INVALID_SLICE_INDEX) - return self.named_type('builtins.slice') - - def visit_list_comprehension(self, e: ListComprehension) -> Type: - return self.check_generator_or_comprehension( - e.generator, 'builtins.list', '') - - def visit_set_comprehension(self, e: SetComprehension) -> Type: - return self.check_generator_or_comprehension( - e.generator, 'builtins.set', '') - - def visit_generator_expr(self, e: GeneratorExpr) -> Type: - return self.check_generator_or_comprehension(e, 'typing.Iterator', - '') - - def check_generator_or_comprehension(self, gen: GeneratorExpr, - type_name: str, - id_for_messages: str) -> Type: - """Type check a generator expression or a list comprehension.""" - self.check_for_comp(gen) - - # Infer the type of the list comprehension by using a synthetic generic - # callable type. - tvdef = TypeVarDef('T', -1, [], self.chk.object_type()) - tv = TypeVarType(tvdef) - constructor = CallableType( - [tv], - [nodes.ARG_POS], - [None], - self.chk.named_generic_type(type_name, [tv]), - self.chk.named_type('builtins.function'), - name=id_for_messages, - variables=[tvdef]) - return self.check_call(constructor, - [gen.left_expr], [nodes.ARG_POS], gen)[0] - - def visit_dictionary_comprehension(self, e: DictionaryComprehension): - """Type check a dictionary comprehension.""" - self.check_for_comp(e) - - # Infer the type of the list comprehension by using a synthetic generic - # callable type. - ktdef = TypeVarDef('KT', -1, [], self.chk.object_type()) - vtdef = TypeVarDef('VT', -2, [], self.chk.object_type()) - kt = TypeVarType(ktdef) - vt = TypeVarType(vtdef) - constructor = CallableType( - [kt, vt], - [nodes.ARG_POS, nodes.ARG_POS], - [None, None], - self.chk.named_generic_type('builtins.dict', [kt, vt]), - self.chk.named_type('builtins.function'), - name='', - variables=[ktdef, vtdef]) - return self.check_call(constructor, - [e.key, e.value], [nodes.ARG_POS, nodes.ARG_POS], e)[0] - - def check_for_comp(self, e: Union[GeneratorExpr, DictionaryComprehension]) -> None: - """Check the for_comp part of comprehensions. That is the part from 'for': - ... for x in y if z - """ - with self.chk.binder.frame_context(): - for index, sequence, conditions in zip(e.indices, e.sequences, - e.condlists): - sequence_type = self.chk.analyze_iterable_item_type(sequence) - self.chk.analyze_index_variables(index, sequence_type, e) - for condition in conditions: - self.accept(condition) - - def visit_conditional_expr(self, e: ConditionalExpr) -> Type: - cond_type = self.accept(e.cond) - self.check_not_void(cond_type, e) - ctx = self.chk.type_context[-1] - - # Gain type information from isinstance if it is there - # but only for the current expression - if_map, else_map = mypy.checker.find_isinstance_check( - e.cond, - self.chk.type_map, - self.chk.typing_mode_weak()) - - if_type = self.analyze_cond_branch(if_map, e.if_expr, context=ctx) - - if not mypy.checker.is_valid_inferred_type(if_type): - # Analyze the right branch disregarding the left branch. - else_type = self.analyze_cond_branch(else_map, e.else_expr, context=ctx) - - # If it would make a difference, re-analyze the left - # branch using the right branch's type as context. - if ctx is None or not is_equivalent(else_type, ctx): - # TODO: If it's possible that the previous analysis of - # the left branch produced errors that are avoided - # using this context, suppress those errors. - if_type = self.analyze_cond_branch(if_map, e.if_expr, context=else_type) - - else: - # Analyze the right branch in the context of the left - # branch's type. - else_type = self.analyze_cond_branch(else_map, e.else_expr, context=if_type) - - res = join.join_types(if_type, else_type) - - return res - - def analyze_cond_branch(self, map: Optional[Dict[Node, Type]], - node: Node, context: Optional[Type]) -> Type: - with self.chk.binder.frame_context(): - if map: - for var, type in map.items(): - self.chk.binder.push(var, type) - return self.accept(node, context=context) - - def visit_backquote_expr(self, e: BackquoteExpr) -> Type: - self.accept(e.expr) - return self.named_type('builtins.str') - - # - # Helpers - # - - def accept(self, node: Node, context: Type = None) -> Type: - """Type check a node. Alias for TypeChecker.accept.""" - return self.chk.accept(node, context) - - def check_not_void(self, typ: Type, context: Context) -> None: - """Generate an error if type is Void.""" - self.chk.check_not_void(typ, context) - - def is_boolean(self, typ: Type) -> bool: - """Is type compatible with bool?""" - return is_subtype(typ, self.chk.bool_type()) - - def named_type(self, name: str) -> Instance: - """Return an instance type with type given by the name and no type - arguments. Alias for TypeChecker.named_type. - """ - return self.chk.named_type(name) - - def is_valid_var_arg(self, typ: Type) -> bool: - """Is a type valid as a *args argument?""" - return (isinstance(typ, TupleType) or - is_subtype(typ, self.chk.named_generic_type('typing.Iterable', - [AnyType()])) or - isinstance(typ, AnyType)) - - def is_valid_keyword_var_arg(self, typ: Type) -> bool: - """Is a type valid as a **kwargs argument?""" - return is_subtype(typ, self.chk.named_generic_type( - 'builtins.dict', [self.named_type('builtins.str'), AnyType()])) - - def has_non_method(self, typ: Type, member: str) -> bool: - """Does type have a member variable / property with the given name?""" - if isinstance(typ, Instance): - return (not typ.type.has_method(member) and - typ.type.has_readable_member(member)) - else: - return False - - def has_member(self, typ: Type, member: str) -> bool: - """Does type have member with the given name?""" - # TODO TupleType => also consider tuple attributes - if isinstance(typ, Instance): - return typ.type.has_readable_member(member) - elif isinstance(typ, AnyType): - return True - elif isinstance(typ, UnionType): - result = all(self.has_member(x, member) for x in typ.items) - return result - elif isinstance(typ, TupleType): - return self.has_member(typ.fallback, member) - else: - return False - - def not_ready_callback(self, name: str, context: Context) -> None: - """Called when we can't infer the type of a variable because it's not ready yet. - - Either defer type checking of the enclosing function to the next - pass or report an error. - """ - self.chk.handle_cannot_determine_type(name, context) - - -def map_actuals_to_formals(caller_kinds: List[int], - caller_names: List[str], - callee_kinds: List[int], - callee_names: List[str], - caller_arg_type: Callable[[int], - Type]) -> List[List[int]]: - """Calculate mapping between actual (caller) args and formals. - - The result contains a list of caller argument indexes mapping to each - callee argument index, indexed by callee index. - - The caller_arg_type argument should evaluate to the type of the actual - argument type with the given index. - """ - ncallee = len(callee_kinds) - map = [None] * ncallee # type: List[List[int]] - for i in range(ncallee): - map[i] = [] - j = 0 - for i, kind in enumerate(caller_kinds): - if kind == nodes.ARG_POS: - if j < ncallee: - if callee_kinds[j] in [nodes.ARG_POS, nodes.ARG_OPT, - nodes.ARG_NAMED]: - map[j].append(i) - j += 1 - elif callee_kinds[j] == nodes.ARG_STAR: - map[j].append(i) - elif kind == nodes.ARG_STAR: - # We need to know the actual type to map varargs. - argt = caller_arg_type(i) - if isinstance(argt, TupleType): - # A tuple actual maps to a fixed number of formals. - for _ in range(len(argt.items)): - if j < ncallee: - if callee_kinds[j] != nodes.ARG_STAR2: - map[j].append(i) - else: - raise NotImplementedError() - j += 1 - else: - # Assume that it is an iterable (if it isn't, there will be - # an error later). - while j < ncallee: - if callee_kinds[j] in (nodes.ARG_NAMED, nodes.ARG_STAR2): - break - else: - map[j].append(i) - j += 1 - elif kind == nodes.ARG_NAMED: - name = caller_names[i] - if name in callee_names: - map[callee_names.index(name)].append(i) - elif nodes.ARG_STAR2 in callee_kinds: - map[callee_kinds.index(nodes.ARG_STAR2)].append(i) - else: - assert kind == nodes.ARG_STAR2 - for j in range(ncallee): - # TODO tuple varargs complicate this - no_certain_match = ( - not map[j] or caller_kinds[map[j][0]] == nodes.ARG_STAR) - if ((callee_names[j] and no_certain_match) - or callee_kinds[j] == nodes.ARG_STAR2): - map[j].append(i) - return map - - -def is_empty_tuple(t: Type) -> bool: - return isinstance(t, TupleType) and not t.items - - -def is_duplicate_mapping(mapping: List[int], actual_kinds: List[int]) -> bool: - # Multiple actuals can map to the same formal only if they both come from - # varargs (*args and **kwargs); in this case at runtime it is possible that - # there are no duplicates. We need to allow this, as the convention - # f(..., *args, **kwargs) is common enough. - return len(mapping) > 1 and not ( - len(mapping) == 2 and - actual_kinds[mapping[0]] == nodes.ARG_STAR and - actual_kinds[mapping[1]] == nodes.ARG_STAR2) - - -def replace_callable_return_type(c: CallableType, new_ret_type: Type) -> CallableType: - """Return a copy of a callable type with a different return type.""" - return c.copy_modified(ret_type=new_ret_type) - - -class ArgInferSecondPassQuery(types.TypeQuery): - """Query whether an argument type should be inferred in the second pass. - - The result is True if the type has a type variable in a callable return - type anywhere. For example, the result for Callable[[], T] is True if t is - a type variable. - """ - def __init__(self) -> None: - super().__init__(False, types.ANY_TYPE_STRATEGY) - - def visit_callable_type(self, t: CallableType) -> bool: - return self.query_types(t.arg_types) or t.accept(HasTypeVarQuery()) - - -class HasTypeVarQuery(types.TypeQuery): - """Visitor for querying whether a type has a type variable component.""" - def __init__(self) -> None: - super().__init__(False, types.ANY_TYPE_STRATEGY) - - def visit_type_var(self, t: TypeVarType) -> bool: - return True - - -def has_erased_component(t: Type) -> bool: - return t is not None and t.accept(HasErasedComponentsQuery()) - - -class HasErasedComponentsQuery(types.TypeQuery): - """Visitor for querying whether a type has an erased component.""" - def __init__(self) -> None: - super().__init__(False, types.ANY_TYPE_STRATEGY) - - def visit_erased_type(self, t: ErasedType) -> bool: - return True - - -def overload_arg_similarity(actual: Type, formal: Type) -> int: - """Return if caller argument (actual) is compatible with overloaded signature arg (formal). - - Return a similarity level: - 0: no match - 1: actual is compatible, but only using type promotions (e.g. int vs float) - 2: actual is compatible without type promotions (e.g. int vs object) - - The distinction is important in cases where multiple overload items match. We want - give priority to higher similarity matches. - """ - # Replace type variables with their upper bounds. Overloading - # resolution is based on runtime behavior which erases type - # parameters, so no need to handle type variables occurring within - # a type. - if isinstance(actual, TypeVarType): - actual = actual.erase_to_union_or_bound() - if isinstance(formal, TypeVarType): - formal = formal.erase_to_union_or_bound() - if (isinstance(actual, UninhabitedType) or isinstance(actual, AnyType) or - isinstance(formal, AnyType) or isinstance(formal, CallableType) or - (isinstance(actual, Instance) and actual.type.fallback_to_any)): - # These could match anything at runtime. - return 2 - if not experiments.STRICT_OPTIONAL and isinstance(actual, NoneTyp): - # NoneTyp matches anything if we're not doing strict Optional checking - return 2 - if isinstance(actual, UnionType): - return max(overload_arg_similarity(item, formal) - for item in actual.items) - if isinstance(formal, UnionType): - return max(overload_arg_similarity(actual, item) - for item in formal.items) - if isinstance(formal, TypeType): - if isinstance(actual, TypeType): - # Since Type[T] is covariant, check if actual = Type[A] is - # a subtype of formal = Type[F]. - return overload_arg_similarity(actual.item, formal.item) - elif isinstance(actual, CallableType) and actual.is_type_obj(): - # Check if the actual is a constructor of some sort. - # Note that this is this unsound, since we don't check the __init__ signature. - return overload_arg_similarity(actual.ret_type, formal.item) - else: - return 0 - if isinstance(formal, Instance): - if isinstance(actual, CallableType): - actual = actual.fallback - if isinstance(actual, Overloaded): - actual = actual.items()[0].fallback - if isinstance(actual, TupleType): - actual = actual.fallback - if isinstance(actual, Instance): - # First perform a quick check (as an optimization) and fall back to generic - # subtyping algorithm if type promotions are possible (e.g., int vs. float). - if formal.type in actual.type.mro: - return 2 - elif actual.type._promote and is_subtype(actual, formal): - return 1 - else: - return 0 - elif isinstance(actual, TypeType): - if formal.type.fullname() in {"builtins.object", "builtins.type"}: - return 2 - else: - return 0 - else: - return 0 - if isinstance(actual, UnboundType) or isinstance(formal, UnboundType): - # Either actual or formal is the result of an error; shut up. - return 2 - # Fall back to a conservative equality check for the remaining kinds of type. - return 2 if is_same_type(erasetype.erase_type(actual), erasetype.erase_type(formal)) else 0 - - -def freshen_generic_callable(callee: CallableType) -> CallableType: - tvdefs = [] - tvmap = {} # type: Dict[TypeVarId, Type] - for v in callee.variables: - tvdef = TypeVarDef.new_unification_variable(v) - tvdefs.append(tvdef) - tvmap[v.id] = TypeVarType(tvdef) - - return cast(CallableType, expand_type(callee, tvmap)).copy_modified(variables=tvdefs) +"""Expression type checker. This file is conceptually part of TypeChecker.""" + +from typing import cast, Dict, List, Tuple, Callable, Union, Optional + +from mypy.types import ( + Type, AnyType, CallableType, Overloaded, NoneTyp, Void, TypeVarDef, + TupleType, Instance, TypeVarId, TypeVarType, ErasedType, UnionType, + PartialType, DeletedType, UnboundType, UninhabitedType, TypeType +) +from mypy.nodes import ( + NameExpr, RefExpr, Var, FuncDef, OverloadedFuncDef, TypeInfo, CallExpr, + Node, MemberExpr, IntExpr, StrExpr, BytesExpr, UnicodeExpr, FloatExpr, + OpExpr, UnaryExpr, IndexExpr, CastExpr, RevealTypeExpr, TypeApplication, ListExpr, + TupleExpr, DictExpr, FuncExpr, SuperExpr, SliceExpr, Context, + ListComprehension, GeneratorExpr, SetExpr, MypyFile, Decorator, + ConditionalExpr, ComparisonExpr, TempNode, SetComprehension, + DictionaryComprehension, ComplexExpr, EllipsisExpr, + TypeAliasExpr, BackquoteExpr, ARG_POS, ARG_NAMED, ARG_STAR2 +) +from mypy.nodes import function_type +from mypy import nodes +import mypy.checker +from mypy import types +from mypy.sametypes import is_same_type +from mypy.erasetype import replace_meta_vars +from mypy.messages import MessageBuilder +from mypy import messages +from mypy.infer import infer_type_arguments, infer_function_type_arguments +from mypy import join +from mypy.subtypes import is_subtype, is_equivalent +from mypy import applytype +from mypy import erasetype +from mypy.checkmember import analyze_member_access, type_object_type +from mypy.semanal import self_type +from mypy.constraints import get_actual_type +from mypy.checkstrformat import StringFormatterChecker +from mypy.expandtype import expand_type +import mypy.checkexpr + +from mypy import experiments + +# Type of callback user for checking individual function arguments. See +# check_args() below for details. +ArgChecker = Callable[[Type, Type, int, Type, int, int, CallableType, Context, MessageBuilder], + None] + + +class Finished(Exception): + """Raised if we can terminate overload argument check early (no match).""" + + +class ExpressionChecker: + """Expression type checker. + + This class works closely together with checker.TypeChecker. + """ + + # Some services are provided by a TypeChecker instance. + chk = None # type: mypy.checker.TypeChecker + # This is shared with TypeChecker, but stored also here for convenience. + msg = None # type: MessageBuilder + + strfrm_checker = None # type: mypy.checkstrformat.StringFormatterChecker + + def __init__(self, + chk: 'mypy.checker.TypeChecker', + msg: MessageBuilder) -> None: + """Construct an expression type checker.""" + self.chk = chk + self.msg = msg + self.strfrm_checker = mypy.checkexpr.StringFormatterChecker(self, self.chk, self.msg) + + def visit_name_expr(self, e: NameExpr) -> Type: + """Type check a name expression. + + It can be of any kind: local, member or global. + """ + result = self.analyze_ref_expr(e) + return self.chk.narrow_type_from_binder(e, result) + + def analyze_ref_expr(self, e: RefExpr, lvalue: bool = False) -> Type: + result = None # type: Type + node = e.node + if isinstance(node, Var): + # Variable reference. + result = self.analyze_var_ref(node, e) + if isinstance(result, PartialType): + if result.type is None: + # 'None' partial type. It has a well-defined type. In an lvalue context + # we want to preserve the knowledge of it being a partial type. + if not lvalue: + result = NoneTyp() + else: + partial_types = self.chk.find_partial_types(node) + if partial_types is not None and not self.chk.current_node_deferred: + context = partial_types[node] + self.msg.fail(messages.NEED_ANNOTATION_FOR_VAR, context) + result = AnyType() + elif isinstance(node, FuncDef): + # Reference to a global function. + result = function_type(node, self.named_type('builtins.function')) + elif isinstance(node, OverloadedFuncDef): + result = node.type + elif isinstance(node, TypeInfo): + # Reference to a type object. + result = type_object_type(node, self.named_type) + elif isinstance(node, MypyFile): + # Reference to a module object. + result = self.named_type('builtins.module') + elif isinstance(node, Decorator): + result = self.analyze_var_ref(node.var, e) + else: + # Unknown reference; use any type implicitly to avoid + # generating extra type errors. + result = AnyType() + return result + + def analyze_var_ref(self, var: Var, context: Context) -> Type: + if not var.type: + if not var.is_ready and self.chk.typing_mode_full(): + self.chk.handle_cannot_determine_type(var.name(), context) + # Implicit 'Any' type. + return AnyType() + else: + # Look up local type of variable with type (inferred or explicit). + val = self.chk.binder.get(var) + if val is None: + return var.type + else: + return val + + def visit_call_expr(self, e: CallExpr) -> Type: + """Type check a call expression.""" + if e.analyzed: + # It's really a special form that only looks like a call. + return self.accept(e.analyzed, self.chk.type_context[-1]) + self.try_infer_partial_type(e) + callee_type = self.accept(e.callee) + if (self.chk.options.disallow_untyped_calls and + self.chk.typing_mode_full() and + isinstance(callee_type, CallableType) + and callee_type.implicit): + return self.msg.untyped_function_call(callee_type, e) + return self.check_call_expr_with_callee_type(callee_type, e) + + # Types and methods that can be used to infer partial types. + item_args = {'builtins.list': ['append'], + 'builtins.set': ['add', 'discard'], + } + container_args = {'builtins.list': {'extend': ['builtins.list']}, + 'builtins.dict': {'update': ['builtins.dict']}, + 'builtins.set': {'update': ['builtins.set', 'builtins.list']}, + } + + def try_infer_partial_type(self, e: CallExpr) -> None: + if isinstance(e.callee, MemberExpr) and isinstance(e.callee.expr, RefExpr): + var = cast(Var, e.callee.expr.node) + partial_types = self.chk.find_partial_types(var) + if partial_types is not None and not self.chk.current_node_deferred: + partial_type = cast(PartialType, var.type) + if partial_type is None or partial_type.type is None: + # A partial None type -> can't infer anything. + return + typename = partial_type.type.fullname() + methodname = e.callee.name + # Sometimes we can infer a full type for a partial List, Dict or Set type. + # TODO: Don't infer argument expression twice. + if (typename in self.item_args and methodname in self.item_args[typename] + and e.arg_kinds == [ARG_POS]): + item_type = self.accept(e.args[0]) + full_item_type = UnionType.make_simplified_union( + [item_type, partial_type.inner_types[0]]) + if mypy.checker.is_valid_inferred_type(full_item_type): + var.type = self.chk.named_generic_type(typename, [full_item_type]) + del partial_types[var] + elif (typename in self.container_args + and methodname in self.container_args[typename] + and e.arg_kinds == [ARG_POS]): + arg_type = self.accept(e.args[0]) + if isinstance(arg_type, Instance): + arg_typename = arg_type.type.fullname() + if arg_typename in self.container_args[typename][methodname]: + full_item_types = [ + UnionType.make_simplified_union([item_type, prev_type]) + for item_type, prev_type + in zip(arg_type.args, partial_type.inner_types) + ] + if all(mypy.checker.is_valid_inferred_type(item_type) + for item_type in full_item_types): + var.type = self.chk.named_generic_type(typename, + list(full_item_types)) + del partial_types[var] + + def check_call_expr_with_callee_type(self, callee_type: Type, + e: CallExpr) -> Type: + """Type check call expression. + + The given callee type overrides the type of the callee + expression. + """ + return self.check_call(callee_type, e.args, e.arg_kinds, e, + e.arg_names, callable_node=e.callee)[0] + + def check_call(self, callee: Type, args: List[Node], + arg_kinds: List[int], context: Context, + arg_names: List[str] = None, + callable_node: Node = None, + arg_messages: MessageBuilder = None) -> Tuple[Type, Type]: + """Type check a call. + + Also infer type arguments if the callee is a generic function. + + Return (result type, inferred callee type). + + Arguments: + callee: type of the called value + args: actual argument expressions + arg_kinds: contains nodes.ARG_* constant for each argument in args + describing whether the argument is positional, *arg, etc. + arg_names: names of arguments (optional) + callable_node: associate the inferred callable type to this node, + if specified + arg_messages: TODO + """ + arg_messages = arg_messages or self.msg + if isinstance(callee, CallableType): + if callee.is_concrete_type_obj() and callee.type_object().is_abstract: + type = callee.type_object() + self.msg.cannot_instantiate_abstract_class( + callee.type_object().name(), type.abstract_attributes, + context) + + formal_to_actual = map_actuals_to_formals( + arg_kinds, arg_names, + callee.arg_kinds, callee.arg_names, + lambda i: self.accept(args[i])) + + if callee.is_generic(): + callee = freshen_generic_callable(callee) + callee = self.infer_function_type_arguments_using_context( + callee, context) + callee = self.infer_function_type_arguments( + callee, args, arg_kinds, formal_to_actual, context) + + arg_types = self.infer_arg_types_in_context2( + callee, args, arg_kinds, formal_to_actual) + + self.check_argument_count(callee, arg_types, arg_kinds, + arg_names, formal_to_actual, context, self.msg) + + self.check_argument_types(arg_types, arg_kinds, callee, + formal_to_actual, context, + messages=arg_messages) + + if (callee.is_type_obj() and (len(arg_types) == 1) + and is_equivalent(callee.ret_type, self.named_type('builtins.type'))): + callee = callee.copy_modified(ret_type=TypeType(arg_types[0])) + + if callable_node: + # Store the inferred callable type. + self.chk.store_type(callable_node, callee) + return callee.ret_type, callee + elif isinstance(callee, Overloaded): + # Type check arguments in empty context. They will be checked again + # later in a context derived from the signature; these types are + # only used to pick a signature variant. + self.msg.disable_errors() + arg_types = self.infer_arg_types_in_context(None, args) + self.msg.enable_errors() + + target = self.overload_call_target(arg_types, arg_kinds, arg_names, + callee, context, + messages=arg_messages) + return self.check_call(target, args, arg_kinds, context, arg_names, + arg_messages=arg_messages) + elif isinstance(callee, AnyType) or self.chk.typing_mode_none(): + self.infer_arg_types_in_context(None, args) + return AnyType(), AnyType() + elif isinstance(callee, UnionType): + self.msg.disable_type_names += 1 + results = [self.check_call(subtype, args, arg_kinds, context, arg_names, + arg_messages=arg_messages) + for subtype in callee.items] + self.msg.disable_type_names -= 1 + return (UnionType.make_simplified_union([res[0] for res in results]), + callee) + elif isinstance(callee, Instance): + call_function = analyze_member_access('__call__', callee, context, + False, False, self.named_type, self.not_ready_callback, + self.msg) + return self.check_call(call_function, args, arg_kinds, context, arg_names, + callable_node, arg_messages) + elif isinstance(callee, TypeVarType): + return self.check_call(callee.upper_bound, args, arg_kinds, context, arg_names, + callable_node, arg_messages) + elif isinstance(callee, TypeType): + # Pass the original Type[] as context since that's where errors should go. + item = self.analyze_type_type_callee(callee.item, callee) + return self.check_call(item, args, arg_kinds, context, arg_names, + callable_node, arg_messages) + else: + return self.msg.not_callable(callee, context), AnyType() + + def analyze_type_type_callee(self, item: Type, context: Context) -> Type: + """Analyze the callee X in X(...) where X is Type[item]. + + Return a Y that we can pass to check_call(Y, ...). + """ + if isinstance(item, AnyType): + return AnyType() + if isinstance(item, Instance): + return type_object_type(item.type, self.named_type) + if isinstance(item, UnionType): + return UnionType([self.analyze_type_type_callee(item, context) + for item in item.items], item.line) + if isinstance(item, TypeVarType): + # Pretend we're calling the typevar's upper bound, + # i.e. its constructor (a poor approximation for reality, + # but better than AnyType...), but replace the return type + # with typevar. + callee = self.analyze_type_type_callee(item.upper_bound, context) + if isinstance(callee, CallableType): + if callee.is_generic(): + callee = None + else: + callee = callee.copy_modified(ret_type=item) + elif isinstance(callee, Overloaded): + if callee.items()[0].is_generic(): + callee = None + else: + callee = Overloaded([c.copy_modified(ret_type=item) + for c in callee.items()]) + if callee: + return callee + + self.msg.unsupported_type_type(item, context) + return AnyType() + + def infer_arg_types_in_context(self, callee: Optional[CallableType], + args: List[Node]) -> List[Type]: + """Infer argument expression types using a callable type as context. + + For example, if callee argument 2 has type List[int], infer the + argument expression with List[int] type context. + """ + # TODO Always called with callee as None, i.e. empty context. + res = [] # type: List[Type] + + fixed = len(args) + if callee: + fixed = min(fixed, callee.max_fixed_args()) + + arg_type = None # type: Type + ctx = None # type: Type + for i, arg in enumerate(args): + if i < fixed: + if callee and i < len(callee.arg_types): + ctx = callee.arg_types[i] + arg_type = self.accept(arg, ctx) + else: + if callee and callee.is_var_arg: + arg_type = self.accept(arg, callee.arg_types[-1]) + else: + arg_type = self.accept(arg) + if has_erased_component(arg_type): + res.append(NoneTyp()) + else: + res.append(arg_type) + return res + + def infer_arg_types_in_context2( + self, callee: CallableType, args: List[Node], arg_kinds: List[int], + formal_to_actual: List[List[int]]) -> List[Type]: + """Infer argument expression types using a callable type as context. + + For example, if callee argument 2 has type List[int], infer the + argument expression with List[int] type context. + + Returns the inferred types of *actual arguments*. + """ + res = [None] * len(args) # type: List[Type] + + for i, actuals in enumerate(formal_to_actual): + for ai in actuals: + if arg_kinds[ai] not in (nodes.ARG_STAR, nodes.ARG_STAR2): + res[ai] = self.accept(args[ai], callee.arg_types[i]) + + # Fill in the rest of the argument types. + for i, t in enumerate(res): + if not t: + res[i] = self.accept(args[i]) + return res + + def infer_function_type_arguments_using_context( + self, callable: CallableType, error_context: Context) -> CallableType: + """Unify callable return type to type context to infer type vars. + + For example, if the return type is set[t] where 't' is a type variable + of callable, and if the context is set[int], return callable modified + by substituting 't' with 'int'. + """ + ctx = self.chk.type_context[-1] + if not ctx: + return callable + # The return type may have references to type metavariables that + # we are inferring right now. We must consider them as indeterminate + # and they are not potential results; thus we replace them with the + # special ErasedType type. On the other hand, class type variables are + # valid results. + erased_ctx = replace_meta_vars(ctx, ErasedType()) + ret_type = callable.ret_type + if isinstance(ret_type, TypeVarType): + if ret_type.values or (not isinstance(ctx, Instance) or + not cast(Instance, ctx).args): + # The return type is a type variable. If it has values, we can't easily restrict + # type inference to conform to the valid values. If it's unrestricted, we could + # infer a too general type for the type variable if we use context, and this could + # result in confusing and spurious type errors elsewhere. + # + # Give up and just use function arguments for type inference. As an exception, + # if the context is a generic instance type, actually use it as context, as + # this *seems* to usually be the reasonable thing to do. + # + # See also github issues #462 and #360. + ret_type = NoneTyp() + args = infer_type_arguments(callable.type_var_ids(), ret_type, erased_ctx) + # Only substitute non-None and non-erased types. + new_args = [] # type: List[Type] + for arg in args: + if isinstance(arg, (NoneTyp, UninhabitedType)) or has_erased_component(arg): + new_args.append(None) + else: + new_args.append(arg) + return cast(CallableType, self.apply_generic_arguments(callable, new_args, + error_context)) + + def infer_function_type_arguments(self, callee_type: CallableType, + args: List[Node], + arg_kinds: List[int], + formal_to_actual: List[List[int]], + context: Context) -> CallableType: + """Infer the type arguments for a generic callee type. + + Infer based on the types of arguments. + + Return a derived callable type that has the arguments applied. + """ + if not self.chk.typing_mode_none(): + # Disable type errors during type inference. There may be errors + # due to partial available context information at this time, but + # these errors can be safely ignored as the arguments will be + # inferred again later. + self.msg.disable_errors() + + arg_types = self.infer_arg_types_in_context2( + callee_type, args, arg_kinds, formal_to_actual) + + self.msg.enable_errors() + + arg_pass_nums = self.get_arg_infer_passes( + callee_type.arg_types, formal_to_actual, len(args)) + + pass1_args = [] # type: List[Optional[Type]] + for i, arg in enumerate(arg_types): + if arg_pass_nums[i] > 1: + pass1_args.append(None) + else: + pass1_args.append(arg) + + inferred_args = infer_function_type_arguments( + callee_type, pass1_args, arg_kinds, formal_to_actual, + strict=self.chk.typing_mode_full()) # type: List[Type] + + if 2 in arg_pass_nums: + # Second pass of type inference. + (callee_type, + inferred_args) = self.infer_function_type_arguments_pass2( + callee_type, args, arg_kinds, formal_to_actual, + inferred_args, context) + + if callee_type.special_sig == 'dict' and len(inferred_args) == 2 and ( + ARG_NAMED in arg_kinds or ARG_STAR2 in arg_kinds): + # HACK: Infer str key type for dict(...) with keyword args. The type system + # can't represent this so we special case it, as this is a pretty common + # thing. This doesn't quite work with all possible subclasses of dict + # if they shuffle type variables around, as we assume that there is a 1-1 + # correspondence with dict type variables. This is a marginal issue and + # a little tricky to fix so it's left unfixed for now. + if isinstance(inferred_args[0], (NoneTyp, UninhabitedType)): + inferred_args[0] = self.named_type('builtins.str') + elif not is_subtype(self.named_type('builtins.str'), inferred_args[0]): + self.msg.fail(messages.KEYWORD_ARGUMENT_REQUIRES_STR_KEY_TYPE, + context) + else: + # In dynamically typed functions use implicit 'Any' types for + # type variables. + inferred_args = [AnyType()] * len(callee_type.variables) + return self.apply_inferred_arguments(callee_type, inferred_args, + context) + + def infer_function_type_arguments_pass2( + self, callee_type: CallableType, + args: List[Node], + arg_kinds: List[int], + formal_to_actual: List[List[int]], + inferred_args: List[Type], + context: Context) -> Tuple[CallableType, List[Type]]: + """Perform second pass of generic function type argument inference. + + The second pass is needed for arguments with types such as Callable[[T], S], + where both T and S are type variables, when the actual argument is a + lambda with inferred types. The idea is to infer the type variable T + in the first pass (based on the types of other arguments). This lets + us infer the argument and return type of the lambda expression and + thus also the type variable S in this second pass. + + Return (the callee with type vars applied, inferred actual arg types). + """ + # None or erased types in inferred types mean that there was not enough + # information to infer the argument. Replace them with None values so + # that they are not applied yet below. + for i, arg in enumerate(inferred_args): + if isinstance(arg, (NoneTyp, UninhabitedType)) or has_erased_component(arg): + inferred_args[i] = None + + callee_type = cast(CallableType, self.apply_generic_arguments( + callee_type, inferred_args, context)) + arg_types = self.infer_arg_types_in_context2( + callee_type, args, arg_kinds, formal_to_actual) + + inferred_args = infer_function_type_arguments( + callee_type, arg_types, arg_kinds, formal_to_actual) + + return callee_type, inferred_args + + def get_arg_infer_passes(self, arg_types: List[Type], + formal_to_actual: List[List[int]], + num_actuals: int) -> List[int]: + """Return pass numbers for args for two-pass argument type inference. + + For each actual, the pass number is either 1 (first pass) or 2 (second + pass). + + Two-pass argument type inference primarily lets us infer types of + lambdas more effectively. + """ + res = [1] * num_actuals + for i, arg in enumerate(arg_types): + if arg.accept(ArgInferSecondPassQuery()): + for j in formal_to_actual[i]: + res[j] = 2 + return res + + def apply_inferred_arguments(self, callee_type: CallableType, + inferred_args: List[Type], + context: Context) -> CallableType: + """Apply inferred values of type arguments to a generic function. + + Inferred_args contains the values of function type arguments. + """ + # Report error if some of the variables could not be solved. In that + # case assume that all variables have type Any to avoid extra + # bogus error messages. + for i, inferred_type in enumerate(inferred_args): + if not inferred_type or has_erased_component(inferred_type): + # Could not infer a non-trivial type for a type variable. + self.msg.could_not_infer_type_arguments( + callee_type, i + 1, context) + inferred_args = [AnyType()] * len(inferred_args) + # Apply the inferred types to the function type. In this case the + # return type must be CallableType, since we give the right number of type + # arguments. + return cast(CallableType, self.apply_generic_arguments(callee_type, + inferred_args, context)) + + def check_argument_count(self, callee: CallableType, actual_types: List[Type], + actual_kinds: List[int], actual_names: List[str], + formal_to_actual: List[List[int]], + context: Context, + messages: Optional[MessageBuilder]) -> bool: + """Check that there is a value for all required arguments to a function. + + Also check that there are no duplicate values for arguments. Report found errors + using 'messages' if it's not None. + + Return False if there were any errors. Otherwise return True + """ + # TODO(jukka): We could return as soon as we find an error if messages is None. + formal_kinds = callee.arg_kinds + + # Collect list of all actual arguments matched to formal arguments. + all_actuals = [] # type: List[int] + for actuals in formal_to_actual: + all_actuals.extend(actuals) + + is_unexpected_arg_error = False # Keep track of errors to avoid duplicate errors. + ok = True # False if we've found any error. + for i, kind in enumerate(actual_kinds): + if i not in all_actuals and ( + kind != nodes.ARG_STAR or + not is_empty_tuple(actual_types[i])): + # Extra actual: not matched by a formal argument. + ok = False + if kind != nodes.ARG_NAMED: + if messages: + messages.too_many_arguments(callee, context) + else: + if messages: + messages.unexpected_keyword_argument( + callee, actual_names[i], context) + is_unexpected_arg_error = True + elif kind == nodes.ARG_STAR and ( + nodes.ARG_STAR not in formal_kinds): + actual_type = actual_types[i] + if isinstance(actual_type, TupleType): + if all_actuals.count(i) < len(actual_type.items): + # Too many tuple items as some did not match. + if messages: + messages.too_many_arguments(callee, context) + ok = False + # *args can be applied even if the function takes a fixed + # number of positional arguments. This may succeed at runtime. + + for i, kind in enumerate(formal_kinds): + if kind == nodes.ARG_POS and (not formal_to_actual[i] and + not is_unexpected_arg_error): + # No actual for a mandatory positional formal. + if messages: + messages.too_few_arguments(callee, context, actual_names) + ok = False + elif kind in [nodes.ARG_POS, nodes.ARG_OPT, + nodes.ARG_NAMED] and is_duplicate_mapping( + formal_to_actual[i], actual_kinds): + if (self.chk.typing_mode_full() or + isinstance(actual_types[formal_to_actual[i][0]], TupleType)): + if messages: + messages.duplicate_argument_value(callee, i, context) + ok = False + elif (kind == nodes.ARG_NAMED and formal_to_actual[i] and + actual_kinds[formal_to_actual[i][0]] != nodes.ARG_NAMED): + # Positional argument when expecting a keyword argument. + if messages: + messages.too_many_positional_arguments(callee, context) + ok = False + return ok + + def check_argument_types(self, arg_types: List[Type], arg_kinds: List[int], + callee: CallableType, + formal_to_actual: List[List[int]], + context: Context, + messages: MessageBuilder = None, + check_arg: ArgChecker = None) -> None: + """Check argument types against a callable type. + + Report errors if the argument types are not compatible. + """ + messages = messages or self.msg + check_arg = check_arg or self.check_arg + # Keep track of consumed tuple *arg items. + tuple_counter = [0] + for i, actuals in enumerate(formal_to_actual): + for actual in actuals: + arg_type = arg_types[actual] + # Check that a *arg is valid as varargs. + if (arg_kinds[actual] == nodes.ARG_STAR and + not self.is_valid_var_arg(arg_type)): + messages.invalid_var_arg(arg_type, context) + if (arg_kinds[actual] == nodes.ARG_STAR2 and + not self.is_valid_keyword_var_arg(arg_type)): + messages.invalid_keyword_var_arg(arg_type, context) + # Get the type of an individual actual argument (for *args + # and **args this is the item type, not the collection type). + actual_type = get_actual_type(arg_type, arg_kinds[actual], + tuple_counter) + check_arg(actual_type, arg_type, arg_kinds[actual], + callee.arg_types[i], + actual + 1, i + 1, callee, context, messages) + + # There may be some remaining tuple varargs items that haven't + # been checked yet. Handle them. + if (callee.arg_kinds[i] == nodes.ARG_STAR and + arg_kinds[actual] == nodes.ARG_STAR and + isinstance(arg_types[actual], TupleType)): + tuplet = cast(TupleType, arg_types[actual]) + while tuple_counter[0] < len(tuplet.items): + actual_type = get_actual_type(arg_type, + arg_kinds[actual], + tuple_counter) + check_arg(actual_type, arg_type, arg_kinds[actual], + callee.arg_types[i], + actual + 1, i + 1, callee, context, messages) + + def check_arg(self, caller_type: Type, original_caller_type: Type, + caller_kind: int, + callee_type: Type, n: int, m: int, callee: CallableType, + context: Context, messages: MessageBuilder) -> None: + """Check the type of a single argument in a call.""" + if isinstance(caller_type, Void): + messages.does_not_return_value(caller_type, context) + elif isinstance(caller_type, DeletedType): + messages.deleted_as_rvalue(caller_type, context) + elif not is_subtype(caller_type, callee_type): + messages.incompatible_argument(n, m, callee, original_caller_type, + caller_kind, context) + + def overload_call_target(self, arg_types: List[Type], arg_kinds: List[int], + arg_names: List[str], + overload: Overloaded, context: Context, + messages: MessageBuilder = None) -> Type: + """Infer the correct overload item to call with given argument types. + + The return value may be CallableType or AnyType (if an unique item + could not be determined). + """ + messages = messages or self.msg + # TODO: For overlapping signatures we should try to get a more precise + # result than 'Any'. + match = [] # type: List[CallableType] + best_match = 0 + for typ in overload.items(): + similarity = self.erased_signature_similarity(arg_types, arg_kinds, arg_names, + typ, context=context) + if similarity > 0 and similarity >= best_match: + if (match and not is_same_type(match[-1].ret_type, + typ.ret_type) and + not mypy.checker.is_more_precise_signature( + match[-1], typ)): + # Ambiguous return type. Either the function overload is + # overlapping (which we don't handle very well here) or the + # caller has provided some Any argument types; in either + # case we'll fall back to Any. It's okay to use Any types + # in calls. + # + # Overlapping overload items are generally fine if the + # overlapping is only possible when there is multiple + # inheritance, as this is rare. See docstring of + # mypy.meet.is_overlapping_types for more about this. + # + # Note that there is no ambiguity if the items are + # covariant in both argument types and return types with + # respect to type precision. We'll pick the best/closest + # match. + # + # TODO: Consider returning a union type instead if the + # overlapping is NOT due to Any types? + return AnyType() + else: + match.append(typ) + best_match = max(best_match, similarity) + if not match: + messages.no_variant_matches_arguments(overload, arg_types, context) + return AnyType() + else: + if len(match) == 1: + return match[0] + else: + # More than one signature matches. Pick the first *non-erased* + # matching signature, or default to the first one if none + # match. + for m in match: + if self.match_signature_types(arg_types, arg_kinds, arg_names, m, + context=context): + return m + return match[0] + + def erased_signature_similarity(self, arg_types: List[Type], arg_kinds: List[int], + arg_names: List[str], callee: CallableType, + context: Context) -> int: + """Determine whether arguments could match the signature at runtime. + + Return similarity level (0 = no match, 1 = can match, 2 = non-promotion match). See + overload_arg_similarity for a discussion of similarity levels. + """ + formal_to_actual = map_actuals_to_formals(arg_kinds, + arg_names, + callee.arg_kinds, + callee.arg_names, + lambda i: arg_types[i]) + + if not self.check_argument_count(callee, arg_types, arg_kinds, arg_names, + formal_to_actual, None, None): + # Too few or many arguments -> no match. + return 0 + + similarity = 2 + + def check_arg(caller_type: Type, original_caller_type: Type, caller_kind: int, + callee_type: Type, n: int, m: int, callee: CallableType, + context: Context, messages: MessageBuilder) -> None: + nonlocal similarity + similarity = min(similarity, + overload_arg_similarity(caller_type, callee_type)) + if similarity == 0: + # No match -- exit early since none of the remaining work can change + # the result. + raise Finished + + try: + self.check_argument_types(arg_types, arg_kinds, callee, formal_to_actual, + context=context, check_arg=check_arg) + except Finished: + pass + + return similarity + + def match_signature_types(self, arg_types: List[Type], arg_kinds: List[int], + arg_names: List[str], callee: CallableType, + context: Context) -> bool: + """Determine whether arguments types match the signature. + + Assume that argument counts are compatible. + + Return True if arguments match. + """ + formal_to_actual = map_actuals_to_formals(arg_kinds, + arg_names, + callee.arg_kinds, + callee.arg_names, + lambda i: arg_types[i]) + ok = True + + def check_arg(caller_type: Type, original_caller_type: Type, caller_kind: int, + callee_type: Type, n: int, m: int, callee: CallableType, + context: Context, messages: MessageBuilder) -> None: + nonlocal ok + if not is_subtype(caller_type, callee_type): + ok = False + + self.check_argument_types(arg_types, arg_kinds, callee, formal_to_actual, + context=context, check_arg=check_arg) + return ok + + def apply_generic_arguments(self, callable: CallableType, types: List[Type], + context: Context) -> Type: + """Simple wrapper around mypy.applytype.apply_generic_arguments.""" + return applytype.apply_generic_arguments(callable, types, self.msg, context) + + def apply_generic_arguments2(self, overload: Overloaded, types: List[Type], + context: Context) -> Type: + items = [] # type: List[CallableType] + for item in overload.items(): + applied = self.apply_generic_arguments(item, types, context) + if isinstance(applied, CallableType): + items.append(applied) + else: + # There was an error. + return AnyType() + return Overloaded(items) + + def visit_member_expr(self, e: MemberExpr) -> Type: + """Visit member expression (of form e.id).""" + result = self.analyze_ordinary_member_access(e, False) + return self.chk.narrow_type_from_binder(e, result) + + def analyze_ordinary_member_access(self, e: MemberExpr, + is_lvalue: bool) -> Type: + """Analyse member expression or member lvalue.""" + if e.kind is not None: + # This is a reference to a module attribute. + return self.analyze_ref_expr(e) + else: + # This is a reference to a non-module attribute. + return analyze_member_access(e.name, self.accept(e.expr), e, + is_lvalue, False, + self.named_type, self.not_ready_callback, self.msg) + + def analyze_external_member_access(self, member: str, base_type: Type, + context: Context) -> Type: + """Analyse member access that is external, i.e. it cannot + refer to private definitions. Return the result type. + """ + # TODO remove; no private definitions in mypy + return analyze_member_access(member, base_type, context, False, False, + self.named_type, self.not_ready_callback, self.msg) + + def visit_int_expr(self, e: IntExpr) -> Type: + """Type check an integer literal (trivial).""" + return self.named_type('builtins.int') + + def visit_str_expr(self, e: StrExpr) -> Type: + """Type check a string literal (trivial).""" + return self.named_type('builtins.str') + + def visit_bytes_expr(self, e: BytesExpr) -> Type: + """Type check a bytes literal (trivial).""" + return self.named_type('builtins.bytes') + + def visit_unicode_expr(self, e: UnicodeExpr) -> Type: + """Type check a unicode literal (trivial).""" + return self.named_type('builtins.unicode') + + def visit_float_expr(self, e: FloatExpr) -> Type: + """Type check a float literal (trivial).""" + return self.named_type('builtins.float') + + def visit_complex_expr(self, e: ComplexExpr) -> Type: + """Type check a complex literal.""" + return self.named_type('builtins.complex') + + def visit_ellipsis(self, e: EllipsisExpr) -> Type: + """Type check '...'.""" + if self.chk.options.python_version[0] >= 3: + return self.named_type('builtins.ellipsis') + else: + # '...' is not valid in normal Python 2 code, but it can + # be used in stubs. The parser makes sure that we only + # get this far if we are in a stub, and we can safely + # return 'object' as ellipsis is special cased elsewhere. + # The builtins.ellipsis type does not exist in Python 2. + return self.named_type('builtins.object') + + def visit_op_expr(self, e: OpExpr) -> Type: + """Type check a binary operator expression.""" + if e.op == 'and' or e.op == 'or': + return self.check_boolean_op(e, e) + if e.op == '*' and isinstance(e.left, ListExpr): + # Expressions of form [...] * e get special type inference. + return self.check_list_multiply(e) + if e.op == '%' and isinstance(e.left, (StrExpr, BytesExpr)): + return self.strfrm_checker.check_str_interpolation(cast(StrExpr, e.left), e.right) + left_type = self.accept(e.left) + + if e.op in nodes.op_methods: + method = self.get_operator_method(e.op) + result, method_type = self.check_op(method, left_type, e.right, e, + allow_reverse=True) + e.method_type = method_type + return result + else: + raise RuntimeError('Unknown operator {}'.format(e.op)) + + def visit_comparison_expr(self, e: ComparisonExpr) -> Type: + """Type check a comparison expression. + + Comparison expressions are type checked consecutive-pair-wise + That is, 'a < b > c == d' is check as 'a < b and b > c and c == d' + """ + result = None # type: mypy.types.Type + + # Check each consecutive operand pair and their operator + for left, right, operator in zip(e.operands, e.operands[1:], e.operators): + left_type = self.accept(left) + + method_type = None # type: mypy.types.Type + + if operator == 'in' or operator == 'not in': + right_type = self.accept(right) # TODO only evaluate if needed + + # Keep track of whether we get type check errors (these won't be reported, they + # are just to verify whether something is valid typing wise). + local_errors = self.msg.copy() + local_errors.disable_count = 0 + sub_result, method_type = self.check_op_local('__contains__', right_type, + left, e, local_errors) + if isinstance(right_type, PartialType): + # We don't really know if this is an error or not, so just shut up. + pass + elif (local_errors.is_errors() and + # is_valid_var_arg is True for any Iterable + self.is_valid_var_arg(right_type)): + itertype = self.chk.analyze_iterable_item_type(right) + method_type = CallableType( + [left_type], + [nodes.ARG_POS], + [None], + self.chk.bool_type(), + self.named_type('builtins.function')) + sub_result = self.chk.bool_type() + if not is_subtype(left_type, itertype): + self.msg.unsupported_operand_types('in', left_type, right_type, e) + else: + self.msg.add_errors(local_errors) + if operator == 'not in': + sub_result = self.chk.bool_type() + elif operator in nodes.op_methods: + method = self.get_operator_method(operator) + sub_result, method_type = self.check_op(method, left_type, right, e, + allow_reverse=True) + + elif operator == 'is' or operator == 'is not': + sub_result = self.chk.bool_type() + method_type = None + else: + raise RuntimeError('Unknown comparison operator {}'.format(operator)) + + e.method_types.append(method_type) + + # Determine type of boolean-and of result and sub_result + if result is None: + result = sub_result + else: + # TODO: check on void needed? + self.check_not_void(sub_result, e) + result = join.join_types(result, sub_result) + + return result + + def get_operator_method(self, op: str) -> str: + if op == '/' and self.chk.options.python_version[0] == 2: + # TODO also check for "from __future__ import division" + return '__div__' + else: + return nodes.op_methods[op] + + def check_op_local(self, method: str, base_type: Type, arg: Node, + context: Context, local_errors: MessageBuilder) -> Tuple[Type, Type]: + """Type check a binary operation which maps to a method call. + + Return tuple (result type, inferred operator method type). + """ + method_type = analyze_member_access(method, base_type, context, False, False, + self.named_type, self.not_ready_callback, local_errors) + return self.check_call(method_type, [arg], [nodes.ARG_POS], + context, arg_messages=local_errors) + + def check_op(self, method: str, base_type: Type, arg: Node, + context: Context, + allow_reverse: bool = False) -> Tuple[Type, Type]: + """Type check a binary operation which maps to a method call. + + Return tuple (result type, inferred operator method type). + """ + # Use a local error storage for errors related to invalid argument + # type (but NOT other errors). This error may need to be suppressed + # for operators which support __rX methods. + local_errors = self.msg.copy() + local_errors.disable_count = 0 + if not allow_reverse or self.has_member(base_type, method): + result = self.check_op_local(method, base_type, arg, context, + local_errors) + if allow_reverse: + arg_type = self.chk.type_map[arg] + if isinstance(arg_type, AnyType): + # If the right operand has type Any, we can't make any + # conjectures about the type of the result, since the + # operand could have a __r method that returns anything. + + # However, in weak mode, we do make conjectures. + if not self.chk.typing_mode_weak(): + result = AnyType(), result[1] + success = not local_errors.is_errors() + else: + result = AnyType(), AnyType() + success = False + if success or not allow_reverse or isinstance(base_type, AnyType): + # We were able to call the normal variant of the operator method, + # or there was some problem not related to argument type + # validity, or the operator has no __rX method. In any case, we + # don't need to consider the __rX method. + self.msg.add_errors(local_errors) + return result + else: + # Calling the operator method was unsuccessful. Try the __rX + # method of the other operand instead. + rmethod = self.get_reverse_op_method(method) + arg_type = self.accept(arg) + if self.has_member(arg_type, rmethod): + method_type = self.analyze_external_member_access( + rmethod, arg_type, context) + temp = TempNode(base_type) + return self.check_call(method_type, [temp], [nodes.ARG_POS], + context) + else: + # No __rX method either. Do deferred type checking to produce + # error message that we may have missed previously. + # TODO Fix type checking an expression more than once. + return self.check_op_local(method, base_type, arg, context, + self.msg) + + def get_reverse_op_method(self, method: str) -> str: + if method == '__div__' and self.chk.options.python_version[0] == 2: + return '__rdiv__' + else: + return nodes.reverse_op_methods[method] + + def check_boolean_op(self, e: OpExpr, context: Context) -> Type: + """Type check a boolean operation ('and' or 'or').""" + + # A boolean operation can evaluate to either of the operands. + + # We use the current type context to guide the type inference of of + # the left operand. We also use the left operand type to guide the type + # inference of the right operand so that expressions such as + # '[1] or []' are inferred correctly. + ctx = self.chk.type_context[-1] + left_type = self.accept(e.left, ctx) + + if e.op == 'and': + right_map, _ = \ + mypy.checker.find_isinstance_check(e.left, self.chk.type_map, + self.chk.typing_mode_weak()) + elif e.op == 'or': + _, right_map = \ + mypy.checker.find_isinstance_check(e.left, self.chk.type_map, + self.chk.typing_mode_weak()) + else: + right_map = None + + with self.chk.binder.frame_context(): + if right_map: + for var, type in right_map.items(): + self.chk.binder.push(var, type) + + right_type = self.accept(e.right, left_type) + + self.check_not_void(left_type, context) + self.check_not_void(right_type, context) + return UnionType.make_simplified_union([left_type, right_type]) + + def check_list_multiply(self, e: OpExpr) -> Type: + """Type check an expression of form '[...] * e'. + + Type inference is special-cased for this common construct. + """ + right_type = self.accept(e.right) + if is_subtype(right_type, self.named_type('builtins.int')): + # Special case: [...] * . Use the type context of the + # OpExpr, since the multiplication does not affect the type. + left_type = self.accept(e.left, context=self.chk.type_context[-1]) + else: + left_type = self.accept(e.left) + result, method_type = self.check_op('__mul__', left_type, e.right, e) + e.method_type = method_type + return result + + def visit_unary_expr(self, e: UnaryExpr) -> Type: + """Type check an unary operation ('not', '-', '+' or '~').""" + operand_type = self.accept(e.expr) + op = e.op + if op == 'not': + self.check_not_void(operand_type, e) + result = self.chk.bool_type() # type: Type + elif op == '-': + method_type = self.analyze_external_member_access('__neg__', + operand_type, e) + result, method_type = self.check_call(method_type, [], [], e) + e.method_type = method_type + elif op == '+': + method_type = self.analyze_external_member_access('__pos__', + operand_type, e) + result, method_type = self.check_call(method_type, [], [], e) + e.method_type = method_type + else: + assert op == '~', "unhandled unary operator" + method_type = self.analyze_external_member_access('__invert__', + operand_type, e) + result, method_type = self.check_call(method_type, [], [], e) + e.method_type = method_type + return result + + def visit_index_expr(self, e: IndexExpr) -> Type: + """Type check an index expression (base[index]). + + It may also represent type application. + """ + result = self.visit_index_expr_helper(e) + return self.chk.narrow_type_from_binder(e, result) + + def visit_index_expr_helper(self, e: IndexExpr) -> Type: + if e.analyzed: + # It's actually a type application. + return self.accept(e.analyzed) + left_type = self.accept(e.base) + if isinstance(left_type, TupleType) and self.chk.typing_mode_full(): + # Special case for tuples. They support indexing only by integer + # literals. (Except in weak type checking mode.) + index = e.index + if isinstance(index, SliceExpr): + return self.visit_tuple_slice_helper(left_type, index) + + ok = False + if isinstance(index, IntExpr): + n = index.value + ok = True + elif isinstance(index, UnaryExpr): + if index.op == '-': + operand = index.expr + if isinstance(operand, IntExpr): + n = len(left_type.items) - operand.value + ok = True + if ok: + if n >= 0 and n < len(left_type.items): + return left_type.items[n] + else: + self.chk.fail(messages.TUPLE_INDEX_OUT_OF_RANGE, e) + return AnyType() + else: + self.chk.fail(messages.TUPLE_INDEX_MUST_BE_AN_INT_LITERAL, e) + return AnyType() + else: + result, method_type = self.check_op('__getitem__', left_type, e.index, e) + e.method_type = method_type + return result + + def visit_tuple_slice_helper(self, left_type: TupleType, slic: SliceExpr): + begin = None # type: int + end = None # type: int + stride = None # type:int + + if slic.begin_index: + begin = self._get_value(slic.begin_index) + if begin is None: + self.chk.fail( + messages.TUPLE_SLICE_MUST_BE_AN_INT_LITERAL, + slic.begin_index) + return AnyType() + + if slic.end_index: + end = self._get_value(slic.end_index) + if end is None: + self.chk.fail( + messages.TUPLE_SLICE_MUST_BE_AN_INT_LITERAL, + slic.end_index) + return AnyType() + + if slic.stride: + stride = self._get_value(slic.stride) + if stride is None: + self.chk.fail( + messages.TUPLE_SLICE_MUST_BE_AN_INT_LITERAL, + slic.stride) + return AnyType() + + return left_type.slice(begin, stride, end) + + def _get_value(self, index: Node) -> Optional[int]: + if isinstance(index, IntExpr): + return index.value + elif isinstance(index, UnaryExpr): + if index.op == '-': + operand = index.expr + if isinstance(operand, IntExpr): + return -1 * operand.value + return None + + def visit_cast_expr(self, expr: CastExpr) -> Type: + """Type check a cast expression.""" + source_type = self.accept(expr.expr, context=AnyType()) + target_type = expr.type + if self.chk.options.warn_redundant_casts and is_same_type(source_type, target_type): + self.msg.redundant_cast(target_type, expr) + if not self.is_valid_cast(source_type, target_type): + self.msg.invalid_cast(target_type, source_type, expr) + return target_type + + def is_valid_cast(self, source_type: Type, target_type: Type) -> bool: + """Is a cast from source_type to target_type meaningful?""" + return (isinstance(target_type, AnyType) or + (not isinstance(source_type, Void) and + not isinstance(target_type, Void))) + + def visit_reveal_type_expr(self, expr: RevealTypeExpr) -> Type: + """Type check a reveal_type expression.""" + revealed_type = self.accept(expr.expr) + self.msg.reveal_type(revealed_type, expr) + return revealed_type + + def visit_type_application(self, tapp: TypeApplication) -> Type: + """Type check a type application (expr[type, ...]).""" + self.chk.fail(messages.GENERIC_TYPE_NOT_VALID_AS_EXPRESSION, tapp) + return AnyType() + + def visit_type_alias_expr(self, alias: TypeAliasExpr) -> Type: + return AnyType() + + def visit_list_expr(self, e: ListExpr) -> Type: + """Type check a list expression [...].""" + return self.check_list_or_set_expr(e.items, 'builtins.list', '', + e) + + def visit_set_expr(self, e: SetExpr) -> Type: + return self.check_list_or_set_expr(e.items, 'builtins.set', '', e) + + def check_list_or_set_expr(self, items: List[Node], fullname: str, + tag: str, context: Context) -> Type: + # Translate into type checking a generic function call. + tvdef = TypeVarDef('T', -1, [], self.chk.object_type()) + tv = TypeVarType(tvdef) + constructor = CallableType( + [tv], + [nodes.ARG_STAR], + [None], + self.chk.named_generic_type(fullname, [tv]), + self.named_type('builtins.function'), + name=tag, + variables=[tvdef]) + return self.check_call(constructor, + items, + [nodes.ARG_POS] * len(items), context)[0] + + def visit_tuple_expr(self, e: TupleExpr) -> Type: + """Type check a tuple expression.""" + ctx = None # type: TupleType + # Try to determine type context for type inference. + if isinstance(self.chk.type_context[-1], TupleType): + t = self.chk.type_context[-1] + if len(t.items) == len(e.items): + ctx = t + # Infer item types. + items = [] # type: List[Type] + for i in range(len(e.items)): + item = e.items[i] + tt = None # type: Type + if not ctx: + tt = self.accept(item) + else: + tt = self.accept(item, ctx.items[i]) + self.check_not_void(tt, e) + items.append(tt) + fallback_item = join.join_type_list(items) + return TupleType(items, self.chk.named_generic_type('builtins.tuple', [fallback_item])) + + def visit_dict_expr(self, e: DictExpr) -> Type: + # Translate into type checking a generic function call. + ktdef = TypeVarDef('KT', -1, [], self.chk.object_type()) + vtdef = TypeVarDef('VT', -2, [], self.chk.object_type()) + kt = TypeVarType(ktdef) + vt = TypeVarType(vtdef) + # The callable type represents a function like this: + # + # def (*v: Tuple[kt, vt]) -> Dict[kt, vt]: ... + constructor = CallableType( + [TupleType([kt, vt], self.named_type('builtins.tuple'))], + [nodes.ARG_STAR], + [None], + self.chk.named_generic_type('builtins.dict', [kt, vt]), + self.named_type('builtins.function'), + name='', + variables=[ktdef, vtdef]) + # Synthesize function arguments. + args = [] # type: List[Node] + for key, value in e.items: + args.append(TupleExpr([key, value])) + return self.check_call(constructor, + args, + [nodes.ARG_POS] * len(args), e)[0] + + def visit_func_expr(self, e: FuncExpr) -> Type: + """Type check lambda expression.""" + inferred_type = self.infer_lambda_type_using_context(e) + if not inferred_type: + # No useful type context. + ret_type = e.expr().accept(self.chk) + if not e.arguments: + # Form 'lambda: e'; just use the inferred return type. + return CallableType([], [], [], ret_type, self.named_type('builtins.function')) + else: + # TODO: Consider reporting an error. However, this is fine if + # we are just doing the first pass in contextual type + # inference. + return AnyType() + else: + # Type context available. + self.chk.check_func_item(e, type_override=inferred_type) + if e.expr() not in self.chk.type_map: + self.accept(e.expr()) + ret_type = self.chk.type_map[e.expr()] + if isinstance(ret_type, NoneTyp): + # For "lambda ...: None", just use type from the context. + # Important when the context is Callable[..., None] which + # really means Void. See #1425. + return inferred_type + return replace_callable_return_type(inferred_type, ret_type) + + def infer_lambda_type_using_context(self, e: FuncExpr) -> CallableType: + """Try to infer lambda expression type using context. + + Return None if could not infer type. + """ + # TODO also accept 'Any' context + ctx = self.chk.type_context[-1] + if not ctx or not isinstance(ctx, CallableType): + return None + + # The context may have function type variables in it. We replace them + # since these are the type variables we are ultimately trying to infer; + # they must be considered as indeterminate. We use ErasedType since it + # does not affect type inference results (it is for purposes like this + # only). + ctx = replace_meta_vars(ctx, ErasedType()) + + callable_ctx = cast(CallableType, ctx) + + arg_kinds = [arg.kind for arg in e.arguments] + + if callable_ctx.is_ellipsis_args: + # Fill in Any arguments to match the arguments of the lambda. + callable_ctx = callable_ctx.copy_modified( + is_ellipsis_args=False, + arg_types=[AnyType()] * len(arg_kinds), + arg_kinds=arg_kinds + ) + + if callable_ctx.arg_kinds != arg_kinds: + # Incompatible context; cannot use it to infer types. + self.chk.fail(messages.CANNOT_INFER_LAMBDA_TYPE, e) + return None + + return callable_ctx + + def visit_super_expr(self, e: SuperExpr) -> Type: + """Type check a super expression (non-lvalue).""" + t = self.analyze_super(e, False) + return t + + def analyze_super(self, e: SuperExpr, is_lvalue: bool) -> Type: + """Type check a super expression.""" + if e.info and e.info.bases: + # TODO fix multiple inheritance etc + if len(e.info.mro) < 2: + self.chk.fail('Internal error: unexpected mro for {}: {}'.format( + e.info.name(), e.info.mro), e) + return AnyType() + for base in e.info.mro[1:]: + if e.name in base.names or base == e.info.mro[-1]: + if e.info.fallback_to_any and base == e.info.mro[-1]: + # There's an undefined base class, and we're + # at the end of the chain. That's not an error. + return AnyType() + if not self.chk.typing_mode_full(): + return AnyType() + return analyze_member_access(e.name, self_type(e.info), e, + is_lvalue, True, + self.named_type, self.not_ready_callback, + self.msg, base) + else: + # Invalid super. This has been reported by the semantic analyzer. + return AnyType() + + def visit_slice_expr(self, e: SliceExpr) -> Type: + for index in [e.begin_index, e.end_index, e.stride]: + if index: + t = self.accept(index) + self.chk.check_subtype(t, self.named_type('builtins.int'), + index, messages.INVALID_SLICE_INDEX) + return self.named_type('builtins.slice') + + def visit_list_comprehension(self, e: ListComprehension) -> Type: + return self.check_generator_or_comprehension( + e.generator, 'builtins.list', '') + + def visit_set_comprehension(self, e: SetComprehension) -> Type: + return self.check_generator_or_comprehension( + e.generator, 'builtins.set', '') + + def visit_generator_expr(self, e: GeneratorExpr) -> Type: + return self.check_generator_or_comprehension(e, 'typing.Iterator', + '') + + def check_generator_or_comprehension(self, gen: GeneratorExpr, + type_name: str, + id_for_messages: str) -> Type: + """Type check a generator expression or a list comprehension.""" + self.check_for_comp(gen) + + # Infer the type of the list comprehension by using a synthetic generic + # callable type. + tvdef = TypeVarDef('T', -1, [], self.chk.object_type()) + tv = TypeVarType(tvdef) + constructor = CallableType( + [tv], + [nodes.ARG_POS], + [None], + self.chk.named_generic_type(type_name, [tv]), + self.chk.named_type('builtins.function'), + name=id_for_messages, + variables=[tvdef]) + return self.check_call(constructor, + [gen.left_expr], [nodes.ARG_POS], gen)[0] + + def visit_dictionary_comprehension(self, e: DictionaryComprehension): + """Type check a dictionary comprehension.""" + self.check_for_comp(e) + + # Infer the type of the list comprehension by using a synthetic generic + # callable type. + ktdef = TypeVarDef('KT', -1, [], self.chk.object_type()) + vtdef = TypeVarDef('VT', -2, [], self.chk.object_type()) + kt = TypeVarType(ktdef) + vt = TypeVarType(vtdef) + constructor = CallableType( + [kt, vt], + [nodes.ARG_POS, nodes.ARG_POS], + [None, None], + self.chk.named_generic_type('builtins.dict', [kt, vt]), + self.chk.named_type('builtins.function'), + name='', + variables=[ktdef, vtdef]) + return self.check_call(constructor, + [e.key, e.value], [nodes.ARG_POS, nodes.ARG_POS], e)[0] + + def check_for_comp(self, e: Union[GeneratorExpr, DictionaryComprehension]) -> None: + """Check the for_comp part of comprehensions. That is the part from 'for': + ... for x in y if z + """ + with self.chk.binder.frame_context(): + for index, sequence, conditions in zip(e.indices, e.sequences, + e.condlists): + sequence_type = self.chk.analyze_iterable_item_type(sequence) + self.chk.analyze_index_variables(index, sequence_type, e) + for condition in conditions: + self.accept(condition) + + def visit_conditional_expr(self, e: ConditionalExpr) -> Type: + cond_type = self.accept(e.cond) + self.check_not_void(cond_type, e) + ctx = self.chk.type_context[-1] + + # Gain type information from isinstance if it is there + # but only for the current expression + if_map, else_map = mypy.checker.find_isinstance_check( + e.cond, + self.chk.type_map, + self.chk.typing_mode_weak()) + + if_type = self.analyze_cond_branch(if_map, e.if_expr, context=ctx) + + if not mypy.checker.is_valid_inferred_type(if_type): + # Analyze the right branch disregarding the left branch. + else_type = self.analyze_cond_branch(else_map, e.else_expr, context=ctx) + + # If it would make a difference, re-analyze the left + # branch using the right branch's type as context. + if ctx is None or not is_equivalent(else_type, ctx): + # TODO: If it's possible that the previous analysis of + # the left branch produced errors that are avoided + # using this context, suppress those errors. + if_type = self.analyze_cond_branch(if_map, e.if_expr, context=else_type) + + else: + # Analyze the right branch in the context of the left + # branch's type. + else_type = self.analyze_cond_branch(else_map, e.else_expr, context=if_type) + + res = join.join_types(if_type, else_type) + + return res + + def analyze_cond_branch(self, map: Optional[Dict[Node, Type]], + node: Node, context: Optional[Type]) -> Type: + with self.chk.binder.frame_context(): + if map: + for var, type in map.items(): + self.chk.binder.push(var, type) + return self.accept(node, context=context) + + def visit_backquote_expr(self, e: BackquoteExpr) -> Type: + self.accept(e.expr) + return self.named_type('builtins.str') + + # + # Helpers + # + + def accept(self, node: Node, context: Type = None) -> Type: + """Type check a node. Alias for TypeChecker.accept.""" + return self.chk.accept(node, context) + + def check_not_void(self, typ: Type, context: Context) -> None: + """Generate an error if type is Void.""" + self.chk.check_not_void(typ, context) + + def is_boolean(self, typ: Type) -> bool: + """Is type compatible with bool?""" + return is_subtype(typ, self.chk.bool_type()) + + def named_type(self, name: str) -> Instance: + """Return an instance type with type given by the name and no type + arguments. Alias for TypeChecker.named_type. + """ + return self.chk.named_type(name) + + def is_valid_var_arg(self, typ: Type) -> bool: + """Is a type valid as a *args argument?""" + return (isinstance(typ, TupleType) or + is_subtype(typ, self.chk.named_generic_type('typing.Iterable', + [AnyType()])) or + isinstance(typ, AnyType)) + + def is_valid_keyword_var_arg(self, typ: Type) -> bool: + """Is a type valid as a **kwargs argument?""" + return is_subtype(typ, self.chk.named_generic_type( + 'builtins.dict', [self.named_type('builtins.str'), AnyType()])) + + def has_non_method(self, typ: Type, member: str) -> bool: + """Does type have a member variable / property with the given name?""" + if isinstance(typ, Instance): + return (not typ.type.has_method(member) and + typ.type.has_readable_member(member)) + else: + return False + + def has_member(self, typ: Type, member: str) -> bool: + """Does type have member with the given name?""" + # TODO TupleType => also consider tuple attributes + if isinstance(typ, Instance): + return typ.type.has_readable_member(member) + elif isinstance(typ, AnyType): + return True + elif isinstance(typ, UnionType): + result = all(self.has_member(x, member) for x in typ.items) + return result + elif isinstance(typ, TupleType): + return self.has_member(typ.fallback, member) + else: + return False + + def not_ready_callback(self, name: str, context: Context) -> None: + """Called when we can't infer the type of a variable because it's not ready yet. + + Either defer type checking of the enclosing function to the next + pass or report an error. + """ + self.chk.handle_cannot_determine_type(name, context) + + +def map_actuals_to_formals(caller_kinds: List[int], + caller_names: List[str], + callee_kinds: List[int], + callee_names: List[str], + caller_arg_type: Callable[[int], + Type]) -> List[List[int]]: + """Calculate mapping between actual (caller) args and formals. + + The result contains a list of caller argument indexes mapping to each + callee argument index, indexed by callee index. + + The caller_arg_type argument should evaluate to the type of the actual + argument type with the given index. + """ + ncallee = len(callee_kinds) + map = [None] * ncallee # type: List[List[int]] + for i in range(ncallee): + map[i] = [] + j = 0 + for i, kind in enumerate(caller_kinds): + if kind == nodes.ARG_POS: + if j < ncallee: + if callee_kinds[j] in [nodes.ARG_POS, nodes.ARG_OPT, + nodes.ARG_NAMED]: + map[j].append(i) + j += 1 + elif callee_kinds[j] == nodes.ARG_STAR: + map[j].append(i) + elif kind == nodes.ARG_STAR: + # We need to know the actual type to map varargs. + argt = caller_arg_type(i) + if isinstance(argt, TupleType): + # A tuple actual maps to a fixed number of formals. + for _ in range(len(argt.items)): + if j < ncallee: + if callee_kinds[j] != nodes.ARG_STAR2: + map[j].append(i) + else: + raise NotImplementedError() + j += 1 + else: + # Assume that it is an iterable (if it isn't, there will be + # an error later). + while j < ncallee: + if callee_kinds[j] in (nodes.ARG_NAMED, nodes.ARG_STAR2): + break + else: + map[j].append(i) + j += 1 + elif kind == nodes.ARG_NAMED: + name = caller_names[i] + if name in callee_names: + map[callee_names.index(name)].append(i) + elif nodes.ARG_STAR2 in callee_kinds: + map[callee_kinds.index(nodes.ARG_STAR2)].append(i) + else: + assert kind == nodes.ARG_STAR2 + for j in range(ncallee): + # TODO tuple varargs complicate this + no_certain_match = ( + not map[j] or caller_kinds[map[j][0]] == nodes.ARG_STAR) + if ((callee_names[j] and no_certain_match) + or callee_kinds[j] == nodes.ARG_STAR2): + map[j].append(i) + return map + + +def is_empty_tuple(t: Type) -> bool: + return isinstance(t, TupleType) and not t.items + + +def is_duplicate_mapping(mapping: List[int], actual_kinds: List[int]) -> bool: + # Multiple actuals can map to the same formal only if they both come from + # varargs (*args and **kwargs); in this case at runtime it is possible that + # there are no duplicates. We need to allow this, as the convention + # f(..., *args, **kwargs) is common enough. + return len(mapping) > 1 and not ( + len(mapping) == 2 and + actual_kinds[mapping[0]] == nodes.ARG_STAR and + actual_kinds[mapping[1]] == nodes.ARG_STAR2) + + +def replace_callable_return_type(c: CallableType, new_ret_type: Type) -> CallableType: + """Return a copy of a callable type with a different return type.""" + return c.copy_modified(ret_type=new_ret_type) + + +class ArgInferSecondPassQuery(types.TypeQuery): + """Query whether an argument type should be inferred in the second pass. + + The result is True if the type has a type variable in a callable return + type anywhere. For example, the result for Callable[[], T] is True if t is + a type variable. + """ + def __init__(self) -> None: + super().__init__(False, types.ANY_TYPE_STRATEGY) + + def visit_callable_type(self, t: CallableType) -> bool: + return self.query_types(t.arg_types) or t.accept(HasTypeVarQuery()) + + +class HasTypeVarQuery(types.TypeQuery): + """Visitor for querying whether a type has a type variable component.""" + def __init__(self) -> None: + super().__init__(False, types.ANY_TYPE_STRATEGY) + + def visit_type_var(self, t: TypeVarType) -> bool: + return True + + +def has_erased_component(t: Type) -> bool: + return t is not None and t.accept(HasErasedComponentsQuery()) + + +class HasErasedComponentsQuery(types.TypeQuery): + """Visitor for querying whether a type has an erased component.""" + def __init__(self) -> None: + super().__init__(False, types.ANY_TYPE_STRATEGY) + + def visit_erased_type(self, t: ErasedType) -> bool: + return True + + +def overload_arg_similarity(actual: Type, formal: Type) -> int: + """Return if caller argument (actual) is compatible with overloaded signature arg (formal). + + Return a similarity level: + 0: no match + 1: actual is compatible, but only using type promotions (e.g. int vs float) + 2: actual is compatible without type promotions (e.g. int vs object) + + The distinction is important in cases where multiple overload items match. We want + give priority to higher similarity matches. + """ + # Replace type variables with their upper bounds. Overloading + # resolution is based on runtime behavior which erases type + # parameters, so no need to handle type variables occurring within + # a type. + if isinstance(actual, TypeVarType): + actual = actual.erase_to_union_or_bound() + if isinstance(formal, TypeVarType): + formal = formal.erase_to_union_or_bound() + if (isinstance(actual, UninhabitedType) or isinstance(actual, AnyType) or + isinstance(formal, AnyType) or isinstance(formal, CallableType) or + (isinstance(actual, Instance) and actual.type.fallback_to_any)): + # These could match anything at runtime. + return 2 + if not experiments.STRICT_OPTIONAL and isinstance(actual, NoneTyp): + # NoneTyp matches anything if we're not doing strict Optional checking + return 2 + if isinstance(actual, UnionType): + return max(overload_arg_similarity(item, formal) + for item in actual.items) + if isinstance(formal, UnionType): + return max(overload_arg_similarity(actual, item) + for item in formal.items) + if isinstance(formal, TypeType): + if isinstance(actual, TypeType): + # Since Type[T] is covariant, check if actual = Type[A] is + # a subtype of formal = Type[F]. + return overload_arg_similarity(actual.item, formal.item) + elif isinstance(actual, CallableType) and actual.is_type_obj(): + # Check if the actual is a constructor of some sort. + # Note that this is this unsound, since we don't check the __init__ signature. + return overload_arg_similarity(actual.ret_type, formal.item) + else: + return 0 + if isinstance(formal, Instance): + if isinstance(actual, CallableType): + actual = actual.fallback + if isinstance(actual, Overloaded): + actual = actual.items()[0].fallback + if isinstance(actual, TupleType): + actual = actual.fallback + if isinstance(actual, Instance): + # First perform a quick check (as an optimization) and fall back to generic + # subtyping algorithm if type promotions are possible (e.g., int vs. float). + if formal.type in actual.type.mro: + return 2 + elif actual.type._promote and is_subtype(actual, formal): + return 1 + else: + return 0 + elif isinstance(actual, TypeType): + if formal.type.fullname() in {"builtins.object", "builtins.type"}: + return 2 + else: + return 0 + else: + return 0 + if isinstance(actual, UnboundType) or isinstance(formal, UnboundType): + # Either actual or formal is the result of an error; shut up. + return 2 + # Fall back to a conservative equality check for the remaining kinds of type. + return 2 if is_same_type(erasetype.erase_type(actual), erasetype.erase_type(formal)) else 0 + + +def freshen_generic_callable(callee: CallableType) -> CallableType: + tvdefs = [] + tvmap = {} # type: Dict[TypeVarId, Type] + for v in callee.variables: + tvdef = TypeVarDef.new_unification_variable(v) + tvdefs.append(tvdef) + tvmap[v.id] = TypeVarType(tvdef) + + return cast(CallableType, expand_type(callee, tvmap)).copy_modified(variables=tvdefs) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index ce6d5190fac6a..b74c29f83cd9d 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -1,112 +1,112 @@ -from typing import Dict, Tuple, List, cast - -from mypy.types import ( - Type, Instance, CallableType, TypeVisitor, UnboundType, ErrorType, AnyType, - Void, NoneTyp, TypeVarType, Overloaded, TupleType, UnionType, ErasedType, TypeList, - PartialType, DeletedType, UninhabitedType, TypeType, TypeVarId -) - - -def expand_type(typ: Type, env: Dict[TypeVarId, Type]) -> Type: - """Substitute any type variable references in a type given by a type - environment. - """ - - return typ.accept(ExpandTypeVisitor(env)) - - -def expand_type_by_instance(typ: Type, instance: Instance) -> Type: - """Substitute type variables in type using values from an Instance. - Type variables are considered to be bound by the class declaration.""" - - if instance.args == []: - return typ - else: - variables = {} # type: Dict[TypeVarId, Type] - for binder, arg in zip(instance.type.defn.type_vars, instance.args): - variables[binder.id] = arg - return expand_type(typ, variables) - - -class ExpandTypeVisitor(TypeVisitor[Type]): - """Visitor that substitutes type variables with values.""" - - variables = None # type: Dict[TypeVarId, Type] # TypeVar id -> TypeVar value - - def __init__(self, variables: Dict[TypeVarId, Type]) -> None: - self.variables = variables - - def visit_unbound_type(self, t: UnboundType) -> Type: - return t - - def visit_error_type(self, t: ErrorType) -> Type: - return t - - def visit_type_list(self, t: TypeList) -> Type: - assert False, 'Not supported' - - def visit_any(self, t: AnyType) -> Type: - return t - - def visit_void(self, t: Void) -> Type: - return t - - def visit_none_type(self, t: NoneTyp) -> Type: - return t - - def visit_uninhabited_type(self, t: UninhabitedType) -> Type: - return t - - def visit_deleted_type(self, t: DeletedType) -> Type: - return t - - def visit_erased_type(self, t: ErasedType) -> Type: - # Should not get here. - raise RuntimeError() - - def visit_instance(self, t: Instance) -> Type: - args = self.expand_types(t.args) - return Instance(t.type, args, t.line) - - def visit_type_var(self, t: TypeVarType) -> Type: - repl = self.variables.get(t.id, t) - if isinstance(repl, Instance): - inst = repl - # Return copy of instance with type erasure flag on. - return Instance(inst.type, inst.args, inst.line, True) - else: - return repl - - def visit_callable_type(self, t: CallableType) -> Type: - return t.copy_modified(arg_types=self.expand_types(t.arg_types), - ret_type=t.ret_type.accept(self)) - - def visit_overloaded(self, t: Overloaded) -> Type: - items = [] # type: List[CallableType] - for item in t.items(): - items.append(cast(CallableType, item.accept(self))) - return Overloaded(items) - - def visit_tuple_type(self, t: TupleType) -> Type: - return t.with_types(self.expand_types(t.items)) - - def visit_union_type(self, t: UnionType) -> Type: - # After substituting for type variables in t.items, - # some of the resulting types might be subtypes of others. - return UnionType.make_simplified_union(self.expand_types(t.items), t.line) - - def visit_partial_type(self, t: PartialType) -> Type: - return t - - def visit_type_type(self, t: TypeType) -> Type: - # TODO: Verify that the new item type is valid (instance or - # union of instances or Any). Sadly we can't report errors - # here yet. - item = t.item.accept(self) - return TypeType(item) - - def expand_types(self, types: List[Type]) -> List[Type]: - a = [] # type: List[Type] - for t in types: - a.append(t.accept(self)) - return a +from typing import Dict, Tuple, List, cast + +from mypy.types import ( + Type, Instance, CallableType, TypeVisitor, UnboundType, ErrorType, AnyType, + Void, NoneTyp, TypeVarType, Overloaded, TupleType, UnionType, ErasedType, TypeList, + PartialType, DeletedType, UninhabitedType, TypeType, TypeVarId +) + + +def expand_type(typ: Type, env: Dict[TypeVarId, Type]) -> Type: + """Substitute any type variable references in a type given by a type + environment. + """ + + return typ.accept(ExpandTypeVisitor(env)) + + +def expand_type_by_instance(typ: Type, instance: Instance) -> Type: + """Substitute type variables in type using values from an Instance. + Type variables are considered to be bound by the class declaration.""" + + if instance.args == []: + return typ + else: + variables = {} # type: Dict[TypeVarId, Type] + for binder, arg in zip(instance.type.defn.type_vars, instance.args): + variables[binder.id] = arg + return expand_type(typ, variables) + + +class ExpandTypeVisitor(TypeVisitor[Type]): + """Visitor that substitutes type variables with values.""" + + variables = None # type: Dict[TypeVarId, Type] # TypeVar id -> TypeVar value + + def __init__(self, variables: Dict[TypeVarId, Type]) -> None: + self.variables = variables + + def visit_unbound_type(self, t: UnboundType) -> Type: + return t + + def visit_error_type(self, t: ErrorType) -> Type: + return t + + def visit_type_list(self, t: TypeList) -> Type: + assert False, 'Not supported' + + def visit_any(self, t: AnyType) -> Type: + return t + + def visit_void(self, t: Void) -> Type: + return t + + def visit_none_type(self, t: NoneTyp) -> Type: + return t + + def visit_uninhabited_type(self, t: UninhabitedType) -> Type: + return t + + def visit_deleted_type(self, t: DeletedType) -> Type: + return t + + def visit_erased_type(self, t: ErasedType) -> Type: + # Should not get here. + raise RuntimeError() + + def visit_instance(self, t: Instance) -> Type: + args = self.expand_types(t.args) + return Instance(t.type, args, t.line) + + def visit_type_var(self, t: TypeVarType) -> Type: + repl = self.variables.get(t.id, t) + if isinstance(repl, Instance): + inst = repl + # Return copy of instance with type erasure flag on. + return Instance(inst.type, inst.args, inst.line, True) + else: + return repl + + def visit_callable_type(self, t: CallableType) -> Type: + return t.copy_modified(arg_types=self.expand_types(t.arg_types), + ret_type=t.ret_type.accept(self)) + + def visit_overloaded(self, t: Overloaded) -> Type: + items = [] # type: List[CallableType] + for item in t.items(): + items.append(cast(CallableType, item.accept(self))) + return Overloaded(items) + + def visit_tuple_type(self, t: TupleType) -> Type: + return t.with_types(self.expand_types(t.items)) + + def visit_union_type(self, t: UnionType) -> Type: + # After substituting for type variables in t.items, + # some of the resulting types might be subtypes of others. + return UnionType.make_simplified_union(self.expand_types(t.items), t.line) + + def visit_partial_type(self, t: PartialType) -> Type: + return t + + def visit_type_type(self, t: TypeType) -> Type: + # TODO: Verify that the new item type is valid (instance or + # union of instances or Any). Sadly we can't report errors + # here yet. + item = t.item.accept(self) + return TypeType(item) + + def expand_types(self, types: List[Type]) -> List[Type]: + a = [] # type: List[Type] + for t in types: + a.append(t.accept(self)) + return a diff --git a/mypy/join.py b/mypy/join.py index 684d62cfeaebd..e0709c2717911 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -1,367 +1,367 @@ -"""Calculation of the least upper bound types (joins).""" - -from typing import List - -from mypy.types import ( - Type, AnyType, NoneTyp, Void, TypeVisitor, Instance, UnboundType, - ErrorType, TypeVarType, CallableType, TupleType, ErasedType, TypeList, - UnionType, FunctionLike, Overloaded, PartialType, DeletedType, - UninhabitedType, TypeType, NamedTupleType -) -from mypy.maptype import map_instance_to_supertype -from mypy.subtypes import is_subtype, is_equivalent, is_subtype_ignoring_tvars - -from mypy import experiments - - -def join_simple(declaration: Type, s: Type, t: Type) -> Type: - """Return a simple least upper bound given the declared type.""" - - if isinstance(s, AnyType): - return s - - if isinstance(s, ErasedType): - return t - - if is_subtype(s, t): - return t - - if is_subtype(t, s): - return s - - if isinstance(declaration, UnionType): - return UnionType.make_simplified_union([s, t]) - - if isinstance(s, NoneTyp) and not isinstance(t, NoneTyp): - s, t = t, s - - if isinstance(s, UninhabitedType) and not isinstance(t, UninhabitedType): - s, t = t, s - - value = t.accept(TypeJoinVisitor(s)) - - if value is None: - # XXX this code path probably should be avoided. - # It seems to happen when a line (x = y) is a type error, and - # it's not clear that assuming that x is arbitrary afterward - # is a good idea. - return declaration - - if declaration is None or is_subtype(value, declaration): - return value - - return declaration - - -def join_types(s: Type, t: Type) -> Type: - """Return the least upper bound of s and t. - - For example, the join of 'int' and 'object' is 'object'. - - If the join does not exist, return an ErrorType instance. - """ - if isinstance(s, AnyType): - return s - - if isinstance(s, ErasedType): - return t - - if isinstance(s, NoneTyp) and not isinstance(t, NoneTyp): - s, t = t, s - - # Use a visitor to handle non-trivial cases. - return t.accept(TypeJoinVisitor(s)) - - -class TypeJoinVisitor(TypeVisitor[Type]): - """Implementation of the least upper bound algorithm. - - Attributes: - s: The other (left) type operand. - """ - - def __init__(self, s: Type) -> None: - self.s = s - - def visit_unbound_type(self, t: UnboundType) -> Type: - if isinstance(self.s, Void) or isinstance(self.s, ErrorType): - return ErrorType() - else: - return AnyType() - - def visit_union_type(self, t: UnionType) -> Type: - if is_subtype(self.s, t): - return t - else: - return UnionType.make_simplified_union([self.s, t]) - - def visit_error_type(self, t: ErrorType) -> Type: - return t - - def visit_type_list(self, t: TypeList) -> Type: - assert False, 'Not supported' - - def visit_any(self, t: AnyType) -> Type: - return t - - def visit_void(self, t: Void) -> Type: - if isinstance(self.s, Void): - return t - else: - return ErrorType() - - def visit_none_type(self, t: NoneTyp) -> Type: - if experiments.STRICT_OPTIONAL: - if isinstance(self.s, (NoneTyp, UninhabitedType)): - return t - else: - return self.default(self.s) - else: - if not isinstance(self.s, Void): - return self.s - else: - return self.default(self.s) - - def visit_uninhabited_type(self, t: UninhabitedType) -> Type: - if not isinstance(self.s, Void): - return self.s - else: - return self.default(self.s) - - def visit_deleted_type(self, t: DeletedType) -> Type: - if not isinstance(self.s, Void): - return self.s - else: - return self.default(self.s) - - def visit_erased_type(self, t: ErasedType) -> Type: - return self.s - - def visit_type_var(self, t: TypeVarType) -> Type: - if isinstance(self.s, TypeVarType) and self.s.id == t.id: - return self.s - else: - return self.default(self.s) - - def visit_instance(self, t: Instance) -> Type: - if isinstance(self.s, Instance): - return join_instances(t, self.s) - elif isinstance(self.s, FunctionLike): - return join_types(t, self.s.fallback) - elif isinstance(self.s, TypeType): - return join_types(t, self.s) - else: - return self.default(self.s) - - def visit_callable_type(self, t: CallableType) -> Type: - # TODO: Consider subtyping instead of just similarity. - if isinstance(self.s, CallableType) and is_similar_callables(t, self.s): - return combine_similar_callables(t, self.s) - elif isinstance(self.s, Overloaded): - # Switch the order of arguments to that we'll get to visit_overloaded. - return join_types(t, self.s) - else: - return join_types(t.fallback, self.s) - - def visit_overloaded(self, t: Overloaded) -> Type: - # This is more complex than most other cases. Here are some - # examples that illustrate how this works. - # - # First let's define a concise notation: - # - Cn are callable types (for n in 1, 2, ...) - # - Ov(C1, C2, ...) is an overloaded type with items C1, C2, ... - # - Callable[[T, ...], S] is written as [T, ...] -> S. - # - # We want some basic properties to hold (assume Cn are all - # unrelated via Any-similarity): - # - # join(Ov(C1, C2), C1) == C1 - # join(Ov(C1, C2), Ov(C1, C2)) == Ov(C1, C2) - # join(Ov(C1, C2), Ov(C1, C3)) == C1 - # join(Ov(C2, C2), C3) == join of fallback types - # - # The presence of Any types makes things more interesting. The join is the - # most general type we can get with respect to Any: - # - # join(Ov([int] -> int, [str] -> str), [Any] -> str) == Any -> str - # - # We could use a simplification step that removes redundancies, but that's not - # implemented right now. Consider this example, where we get a redundancy: - # - # join(Ov([int, Any] -> Any, [str, Any] -> Any), [Any, int] -> Any) == - # Ov([Any, int] -> Any, [Any, int] -> Any) - # - # TODO: Use callable subtyping instead of just similarity. - result = [] # type: List[CallableType] - s = self.s - if isinstance(s, FunctionLike): - # The interesting case where both types are function types. - for t_item in t.items(): - for s_item in s.items(): - if is_similar_callables(t_item, s_item): - result.append(combine_similar_callables(t_item, s_item)) - if result: - # TODO: Simplify redundancies from the result. - if len(result) == 1: - return result[0] - else: - return Overloaded(result) - return join_types(t.fallback, s.fallback) - return join_types(t.fallback, s) - - def visit_tuple_type(self, t: TupleType) -> Type: - if self.s == t: - return self.s - if isinstance(self.s, TupleType) and self.s.length() == t.length(): - items = [] # type: List[Type] - for i in range(t.length()): - items.append(self.join(t.items[i], self.s.items[i])) - if (not (isinstance(t, NamedTupleType) and isinstance(self.s, NamedTupleType)) - or self.s.attrs == t.attrs): - return self.s.with_types(items) - # TODO: What if the fallback types are different? - return TupleType(items, t.fallback) - else: - return self.default(self.s) - - def visit_partial_type(self, t: PartialType) -> Type: - # We only have partial information so we can't decide the join result. We should - # never get here. - assert False, "Internal error" - - def visit_type_type(self, t: TypeType) -> Type: - if isinstance(self.s, TypeType): - return TypeType(self.join(t.item, self.s.item), line=t.line) - elif isinstance(self.s, Instance) and self.s.type.fullname() == 'builtins.type': - return self.s - else: - return self.default(self.s) - - def join(self, s: Type, t: Type) -> Type: - return join_types(s, t) - - def default(self, typ: Type) -> Type: - if isinstance(typ, Instance): - return object_from_instance(typ) - elif isinstance(typ, UnboundType): - return AnyType() - elif isinstance(typ, Void) or isinstance(typ, ErrorType): - return ErrorType() - elif isinstance(typ, TupleType): - return self.default(typ.fallback) - elif isinstance(typ, FunctionLike): - return self.default(typ.fallback) - elif isinstance(typ, TypeVarType): - return self.default(typ.upper_bound) - else: - return AnyType() - - -def join_instances(t: Instance, s: Instance) -> Type: - """Calculate the join of two instance types. - - If allow_interfaces is True, also consider interface-type results for - non-interface types. - - Return ErrorType if the result is ambiguous. - """ - if t.type == s.type: - # Simplest case: join two types with the same base type (but - # potentially different arguments). - if is_subtype(t, s) or is_subtype(s, t): - # Compatible; combine type arguments. - args = [] # type: List[Type] - for i in range(len(t.args)): - args.append(join_types(t.args[i], s.args[i])) - return Instance(t.type, args) - else: - # Incompatible; return trivial result object. - return object_from_instance(t) - elif t.type.bases and is_subtype_ignoring_tvars(t, s): - return join_instances_via_supertype(t, s) - else: - # Now t is not a subtype of s, and t != s. Now s could be a subtype - # of t; alternatively, we need to find a common supertype. This works - # in of the both cases. - return join_instances_via_supertype(s, t) - - -def join_instances_via_supertype(t: Instance, s: Instance) -> Type: - # Give preference to joins via duck typing relationship, so that - # join(int, float) == float, for example. - if t.type._promote and is_subtype(t.type._promote, s): - return join_types(t.type._promote, s) - elif s.type._promote and is_subtype(s.type._promote, t): - return join_types(t, s.type._promote) - # Compute the "best" supertype of t when joined with s. - # The definition of "best" may evolve; for now it is the one with - # the longest MRO. Ties are broken by using the earlier base. - best = None # type: Type - for base in t.type.bases: - mapped = map_instance_to_supertype(t, base.type) - res = join_instances(mapped, s) - if best is None or is_better(res, best): - best = res - assert best is not None - return best - - -def is_better(t: Type, s: Type) -> bool: - # Given two possible results from join_instances_via_supertype(), - # indicate whether t is the better one. - if isinstance(t, Instance): - if not isinstance(s, Instance): - return True - # Use len(mro) as a proxy for the better choice. - if len(t.type.mro) > len(s.type.mro): - return True - return False - - -def is_similar_callables(t: CallableType, s: CallableType) -> bool: - """Return True if t and s are equivalent and have identical numbers of - arguments, default arguments and varargs. - """ - - return (len(t.arg_types) == len(s.arg_types) and t.min_args == s.min_args - and t.is_var_arg == s.is_var_arg and is_equivalent(t, s)) - - -def combine_similar_callables(t: CallableType, s: CallableType) -> CallableType: - arg_types = [] # type: List[Type] - for i in range(len(t.arg_types)): - arg_types.append(join_types(t.arg_types[i], s.arg_types[i])) - # TODO kinds and argument names - # The fallback type can be either 'function' or 'type'. The result should have 'type' as - # fallback only if both operands have it as 'type'. - if t.fallback.type.fullname() != 'builtins.type': - fallback = t.fallback - else: - fallback = s.fallback - return t.copy_modified(arg_types=arg_types, - ret_type=join_types(t.ret_type, s.ret_type), - fallback=fallback, - name=None) - - -def object_from_instance(instance: Instance) -> Instance: - """Construct the type 'builtins.object' from an instance type.""" - # Use the fact that 'object' is always the last class in the mro. - res = Instance(instance.type.mro[-1], []) - return res - - -def join_type_list(types: List[Type]) -> Type: - if not types: - # This is a little arbitrary but reasonable. Any empty tuple should be compatible - # with all variable length tuples, and this makes it possible. A better approach - # would be to use a special bottom type, which we do when strict Optional - # checking is enabled. - if experiments.STRICT_OPTIONAL: - return UninhabitedType() - else: - return NoneTyp() - joined = types[0] - for t in types[1:]: - joined = join_types(joined, t) - return joined +"""Calculation of the least upper bound types (joins).""" + +from typing import List + +from mypy.types import ( + Type, AnyType, NoneTyp, Void, TypeVisitor, Instance, UnboundType, + ErrorType, TypeVarType, CallableType, TupleType, ErasedType, TypeList, + UnionType, FunctionLike, Overloaded, PartialType, DeletedType, + UninhabitedType, TypeType, NamedTupleType +) +from mypy.maptype import map_instance_to_supertype +from mypy.subtypes import is_subtype, is_equivalent, is_subtype_ignoring_tvars + +from mypy import experiments + + +def join_simple(declaration: Type, s: Type, t: Type) -> Type: + """Return a simple least upper bound given the declared type.""" + + if isinstance(s, AnyType): + return s + + if isinstance(s, ErasedType): + return t + + if is_subtype(s, t): + return t + + if is_subtype(t, s): + return s + + if isinstance(declaration, UnionType): + return UnionType.make_simplified_union([s, t]) + + if isinstance(s, NoneTyp) and not isinstance(t, NoneTyp): + s, t = t, s + + if isinstance(s, UninhabitedType) and not isinstance(t, UninhabitedType): + s, t = t, s + + value = t.accept(TypeJoinVisitor(s)) + + if value is None: + # XXX this code path probably should be avoided. + # It seems to happen when a line (x = y) is a type error, and + # it's not clear that assuming that x is arbitrary afterward + # is a good idea. + return declaration + + if declaration is None or is_subtype(value, declaration): + return value + + return declaration + + +def join_types(s: Type, t: Type) -> Type: + """Return the least upper bound of s and t. + + For example, the join of 'int' and 'object' is 'object'. + + If the join does not exist, return an ErrorType instance. + """ + if isinstance(s, AnyType): + return s + + if isinstance(s, ErasedType): + return t + + if isinstance(s, NoneTyp) and not isinstance(t, NoneTyp): + s, t = t, s + + # Use a visitor to handle non-trivial cases. + return t.accept(TypeJoinVisitor(s)) + + +class TypeJoinVisitor(TypeVisitor[Type]): + """Implementation of the least upper bound algorithm. + + Attributes: + s: The other (left) type operand. + """ + + def __init__(self, s: Type) -> None: + self.s = s + + def visit_unbound_type(self, t: UnboundType) -> Type: + if isinstance(self.s, Void) or isinstance(self.s, ErrorType): + return ErrorType() + else: + return AnyType() + + def visit_union_type(self, t: UnionType) -> Type: + if is_subtype(self.s, t): + return t + else: + return UnionType.make_simplified_union([self.s, t]) + + def visit_error_type(self, t: ErrorType) -> Type: + return t + + def visit_type_list(self, t: TypeList) -> Type: + assert False, 'Not supported' + + def visit_any(self, t: AnyType) -> Type: + return t + + def visit_void(self, t: Void) -> Type: + if isinstance(self.s, Void): + return t + else: + return ErrorType() + + def visit_none_type(self, t: NoneTyp) -> Type: + if experiments.STRICT_OPTIONAL: + if isinstance(self.s, (NoneTyp, UninhabitedType)): + return t + else: + return self.default(self.s) + else: + if not isinstance(self.s, Void): + return self.s + else: + return self.default(self.s) + + def visit_uninhabited_type(self, t: UninhabitedType) -> Type: + if not isinstance(self.s, Void): + return self.s + else: + return self.default(self.s) + + def visit_deleted_type(self, t: DeletedType) -> Type: + if not isinstance(self.s, Void): + return self.s + else: + return self.default(self.s) + + def visit_erased_type(self, t: ErasedType) -> Type: + return self.s + + def visit_type_var(self, t: TypeVarType) -> Type: + if isinstance(self.s, TypeVarType) and self.s.id == t.id: + return self.s + else: + return self.default(self.s) + + def visit_instance(self, t: Instance) -> Type: + if isinstance(self.s, Instance): + return join_instances(t, self.s) + elif isinstance(self.s, FunctionLike): + return join_types(t, self.s.fallback) + elif isinstance(self.s, TypeType): + return join_types(t, self.s) + else: + return self.default(self.s) + + def visit_callable_type(self, t: CallableType) -> Type: + # TODO: Consider subtyping instead of just similarity. + if isinstance(self.s, CallableType) and is_similar_callables(t, self.s): + return combine_similar_callables(t, self.s) + elif isinstance(self.s, Overloaded): + # Switch the order of arguments to that we'll get to visit_overloaded. + return join_types(t, self.s) + else: + return join_types(t.fallback, self.s) + + def visit_overloaded(self, t: Overloaded) -> Type: + # This is more complex than most other cases. Here are some + # examples that illustrate how this works. + # + # First let's define a concise notation: + # - Cn are callable types (for n in 1, 2, ...) + # - Ov(C1, C2, ...) is an overloaded type with items C1, C2, ... + # - Callable[[T, ...], S] is written as [T, ...] -> S. + # + # We want some basic properties to hold (assume Cn are all + # unrelated via Any-similarity): + # + # join(Ov(C1, C2), C1) == C1 + # join(Ov(C1, C2), Ov(C1, C2)) == Ov(C1, C2) + # join(Ov(C1, C2), Ov(C1, C3)) == C1 + # join(Ov(C2, C2), C3) == join of fallback types + # + # The presence of Any types makes things more interesting. The join is the + # most general type we can get with respect to Any: + # + # join(Ov([int] -> int, [str] -> str), [Any] -> str) == Any -> str + # + # We could use a simplification step that removes redundancies, but that's not + # implemented right now. Consider this example, where we get a redundancy: + # + # join(Ov([int, Any] -> Any, [str, Any] -> Any), [Any, int] -> Any) == + # Ov([Any, int] -> Any, [Any, int] -> Any) + # + # TODO: Use callable subtyping instead of just similarity. + result = [] # type: List[CallableType] + s = self.s + if isinstance(s, FunctionLike): + # The interesting case where both types are function types. + for t_item in t.items(): + for s_item in s.items(): + if is_similar_callables(t_item, s_item): + result.append(combine_similar_callables(t_item, s_item)) + if result: + # TODO: Simplify redundancies from the result. + if len(result) == 1: + return result[0] + else: + return Overloaded(result) + return join_types(t.fallback, s.fallback) + return join_types(t.fallback, s) + + def visit_tuple_type(self, t: TupleType) -> Type: + if self.s == t: + return self.s + if isinstance(self.s, TupleType) and self.s.length() == t.length(): + items = [] # type: List[Type] + for i in range(t.length()): + items.append(self.join(t.items[i], self.s.items[i])) + if (not (isinstance(t, NamedTupleType) and isinstance(self.s, NamedTupleType)) + or self.s.attrs == t.attrs): + return self.s.with_types(items) + # TODO: What if the fallback types are different? + return TupleType(items, t.fallback) + else: + return self.default(self.s) + + def visit_partial_type(self, t: PartialType) -> Type: + # We only have partial information so we can't decide the join result. We should + # never get here. + assert False, "Internal error" + + def visit_type_type(self, t: TypeType) -> Type: + if isinstance(self.s, TypeType): + return TypeType(self.join(t.item, self.s.item), line=t.line) + elif isinstance(self.s, Instance) and self.s.type.fullname() == 'builtins.type': + return self.s + else: + return self.default(self.s) + + def join(self, s: Type, t: Type) -> Type: + return join_types(s, t) + + def default(self, typ: Type) -> Type: + if isinstance(typ, Instance): + return object_from_instance(typ) + elif isinstance(typ, UnboundType): + return AnyType() + elif isinstance(typ, Void) or isinstance(typ, ErrorType): + return ErrorType() + elif isinstance(typ, TupleType): + return self.default(typ.fallback) + elif isinstance(typ, FunctionLike): + return self.default(typ.fallback) + elif isinstance(typ, TypeVarType): + return self.default(typ.upper_bound) + else: + return AnyType() + + +def join_instances(t: Instance, s: Instance) -> Type: + """Calculate the join of two instance types. + + If allow_interfaces is True, also consider interface-type results for + non-interface types. + + Return ErrorType if the result is ambiguous. + """ + if t.type == s.type: + # Simplest case: join two types with the same base type (but + # potentially different arguments). + if is_subtype(t, s) or is_subtype(s, t): + # Compatible; combine type arguments. + args = [] # type: List[Type] + for i in range(len(t.args)): + args.append(join_types(t.args[i], s.args[i])) + return Instance(t.type, args) + else: + # Incompatible; return trivial result object. + return object_from_instance(t) + elif t.type.bases and is_subtype_ignoring_tvars(t, s): + return join_instances_via_supertype(t, s) + else: + # Now t is not a subtype of s, and t != s. Now s could be a subtype + # of t; alternatively, we need to find a common supertype. This works + # in of the both cases. + return join_instances_via_supertype(s, t) + + +def join_instances_via_supertype(t: Instance, s: Instance) -> Type: + # Give preference to joins via duck typing relationship, so that + # join(int, float) == float, for example. + if t.type._promote and is_subtype(t.type._promote, s): + return join_types(t.type._promote, s) + elif s.type._promote and is_subtype(s.type._promote, t): + return join_types(t, s.type._promote) + # Compute the "best" supertype of t when joined with s. + # The definition of "best" may evolve; for now it is the one with + # the longest MRO. Ties are broken by using the earlier base. + best = None # type: Type + for base in t.type.bases: + mapped = map_instance_to_supertype(t, base.type) + res = join_instances(mapped, s) + if best is None or is_better(res, best): + best = res + assert best is not None + return best + + +def is_better(t: Type, s: Type) -> bool: + # Given two possible results from join_instances_via_supertype(), + # indicate whether t is the better one. + if isinstance(t, Instance): + if not isinstance(s, Instance): + return True + # Use len(mro) as a proxy for the better choice. + if len(t.type.mro) > len(s.type.mro): + return True + return False + + +def is_similar_callables(t: CallableType, s: CallableType) -> bool: + """Return True if t and s are equivalent and have identical numbers of + arguments, default arguments and varargs. + """ + + return (len(t.arg_types) == len(s.arg_types) and t.min_args == s.min_args + and t.is_var_arg == s.is_var_arg and is_equivalent(t, s)) + + +def combine_similar_callables(t: CallableType, s: CallableType) -> CallableType: + arg_types = [] # type: List[Type] + for i in range(len(t.arg_types)): + arg_types.append(join_types(t.arg_types[i], s.arg_types[i])) + # TODO kinds and argument names + # The fallback type can be either 'function' or 'type'. The result should have 'type' as + # fallback only if both operands have it as 'type'. + if t.fallback.type.fullname() != 'builtins.type': + fallback = t.fallback + else: + fallback = s.fallback + return t.copy_modified(arg_types=arg_types, + ret_type=join_types(t.ret_type, s.ret_type), + fallback=fallback, + name=None) + + +def object_from_instance(instance: Instance) -> Instance: + """Construct the type 'builtins.object' from an instance type.""" + # Use the fact that 'object' is always the last class in the mro. + res = Instance(instance.type.mro[-1], []) + return res + + +def join_type_list(types: List[Type]) -> Type: + if not types: + # This is a little arbitrary but reasonable. Any empty tuple should be compatible + # with all variable length tuples, and this makes it possible. A better approach + # would be to use a special bottom type, which we do when strict Optional + # checking is enabled. + if experiments.STRICT_OPTIONAL: + return UninhabitedType() + else: + return NoneTyp() + joined = types[0] + for t in types[1:]: + joined = join_types(joined, t) + return joined diff --git a/mypy/meet.py b/mypy/meet.py index 5c98385c7f1f2..c4e3831e6f5f7 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -1,285 +1,285 @@ -from typing import cast, List - -from mypy.join import is_similar_callables, combine_similar_callables -from mypy.types import ( - Type, AnyType, TypeVisitor, UnboundType, Void, ErrorType, NoneTyp, TypeVarType, - Instance, CallableType, TupleType, ErasedType, TypeList, UnionType, PartialType, - DeletedType, UninhabitedType, TypeType -) -from mypy.subtypes import is_subtype -from mypy.nodes import TypeInfo - -from mypy import experiments - -# TODO Describe this module. - - -def meet_types(s: Type, t: Type) -> Type: - """Return the greatest lower bound of two types.""" - if isinstance(s, ErasedType): - return s - if isinstance(s, AnyType): - return t - if isinstance(s, UnionType) and not isinstance(t, UnionType): - s, t = t, s - return t.accept(TypeMeetVisitor(s)) - - -def meet_simple(s: Type, t: Type, default_right: bool = True) -> Type: - if s == t: - return s - if isinstance(s, UnionType): - return UnionType.make_simplified_union([meet_types(x, t) for x in s.items]) - elif not is_overlapping_types(s, t, use_promotions=True): - if experiments.STRICT_OPTIONAL: - return UninhabitedType() - else: - return NoneTyp() - else: - if default_right: - return t - else: - return s - - -def is_overlapping_types(t: Type, s: Type, use_promotions: bool = False) -> bool: - """Can a value of type t be a value of type s, or vice versa? - - Note that this effectively checks against erased types, since type - variables are erased at runtime and the overlapping check is based - on runtime behavior. - - If use_promotions is True, also consider type promotions (int and - float would only be overlapping if it's True). - - This does not consider multiple inheritance. For example, A and B in - the following example are not considered overlapping, even though - via C they can be overlapping: - - class A: ... - class B: ... - class C(A, B): ... - - The rationale is that this case is usually very unlikely as multiple - inheritance is rare. Also, we can't reliably determine whether - multiple inheritance actually occurs somewhere in a program, due to - stub files hiding implementation details, dynamic loading etc. - - TODO: Don't consider tuples always overlapping. - TODO: Don't consider callables always overlapping. - TODO: Don't consider type variables with values always overlapping. - """ - # Since we are effectively working with the erased types, we only - # need to handle occurrences of TypeVarType at the top level. - if isinstance(t, TypeVarType): - t = t.erase_to_union_or_bound() - if isinstance(s, TypeVarType): - s = s.erase_to_union_or_bound() - if isinstance(t, Instance): - if isinstance(s, Instance): - # Consider two classes non-disjoint if one is included in the mro - # of another. - if use_promotions: - # Consider cases like int vs float to be overlapping where - # there is only a type promotion relationship but not proper - # subclassing. - if t.type._promote and is_overlapping_types(t.type._promote, s): - return True - if s.type._promote and is_overlapping_types(s.type._promote, t): - return True - return t.type in s.type.mro or s.type in t.type.mro - if isinstance(t, UnionType): - return any(is_overlapping_types(item, s) - for item in t.items) - if isinstance(s, UnionType): - return any(is_overlapping_types(t, item) - for item in s.items) - if isinstance(t, TypeType) and isinstance(s, TypeType): - # If both types are TypeType, compare their inner types. - return is_overlapping_types(t.item, s.item, use_promotions) - elif isinstance(t, TypeType) or isinstance(s, TypeType): - # If exactly only one of t or s is a TypeType, check if one of them - # is an `object` or a `type` and otherwise assume no overlap. - other = s if isinstance(t, TypeType) else t - if isinstance(other, Instance): - return other.type.fullname() in {'builtins.object', 'builtins.type'} - else: - return False - if experiments.STRICT_OPTIONAL: - if isinstance(t, NoneTyp) != isinstance(s, NoneTyp): - # NoneTyp does not overlap with other non-Union types under strict Optional checking - return False - # We conservatively assume that non-instance, non-union, and non-TypeType types can overlap - # any other types. - return True - - -def nearest_builtin_ancestor(type: TypeInfo) -> TypeInfo: - for base in type.mro: - if base.defn.is_builtinclass: - return base - else: - return None - - -class TypeMeetVisitor(TypeVisitor[Type]): - def __init__(self, s: Type) -> None: - self.s = s - - def visit_unbound_type(self, t: UnboundType) -> Type: - if isinstance(self.s, Void) or isinstance(self.s, ErrorType): - return ErrorType() - elif isinstance(self.s, NoneTyp): - if experiments.STRICT_OPTIONAL: - return AnyType() - else: - return self.s - elif isinstance(self.s, UninhabitedType): - return self.s - else: - return AnyType() - - def visit_error_type(self, t: ErrorType) -> Type: - return t - - def visit_type_list(self, t: TypeList) -> Type: - assert False, 'Not supported' - - def visit_any(self, t: AnyType) -> Type: - return self.s - - def visit_union_type(self, t: UnionType) -> Type: - if isinstance(self.s, UnionType): - meets = [] # type: List[Type] - for x in t.items: - for y in self.s.items: - meets.append(meet_types(x, y)) - else: - meets = [meet_types(x, self.s) - for x in t.items] - return UnionType.make_simplified_union(meets) - - def visit_void(self, t: Void) -> Type: - if isinstance(self.s, Void): - return t - else: - return ErrorType() - - def visit_none_type(self, t: NoneTyp) -> Type: - if experiments.STRICT_OPTIONAL: - if isinstance(self.s, NoneTyp) or (isinstance(self.s, Instance) and - self.s.type.fullname() == 'builtins.object'): - return t - else: - return UninhabitedType() - else: - if not isinstance(self.s, Void) and not isinstance(self.s, ErrorType): - return t - else: - return ErrorType() - - def visit_uninhabited_type(self, t: UninhabitedType) -> Type: - if not isinstance(self.s, Void) and not isinstance(self.s, ErrorType): - return t - else: - return ErrorType() - - def visit_deleted_type(self, t: DeletedType) -> Type: - if not isinstance(self.s, Void) and not isinstance(self.s, ErrorType): - if isinstance(self.s, NoneTyp): - if experiments.STRICT_OPTIONAL: - return t - else: - return self.s - elif isinstance(self.s, UninhabitedType): - return self.s - else: - return t - else: - return ErrorType() - - def visit_erased_type(self, t: ErasedType) -> Type: - return self.s - - def visit_type_var(self, t: TypeVarType) -> Type: - if isinstance(self.s, TypeVarType) and self.s.id == t.id: - return self.s - else: - return self.default(self.s) - - def visit_instance(self, t: Instance) -> Type: - if isinstance(self.s, Instance): - si = self.s - if t.type == si.type: - if is_subtype(t, self.s) or is_subtype(self.s, t): - # Combine type arguments. We could have used join below - # equivalently. - args = [] # type: List[Type] - for i in range(len(t.args)): - args.append(self.meet(t.args[i], si.args[i])) - return Instance(t.type, args) - else: - if experiments.STRICT_OPTIONAL: - return UninhabitedType() - else: - return NoneTyp() - else: - if is_subtype(t, self.s): - return t - elif is_subtype(self.s, t): - # See also above comment. - return self.s - else: - if experiments.STRICT_OPTIONAL: - return UninhabitedType() - else: - return NoneTyp() - elif isinstance(self.s, TypeType): - return meet_types(t, self.s) - else: - return self.default(self.s) - - def visit_callable_type(self, t: CallableType) -> Type: - if isinstance(self.s, CallableType) and is_similar_callables(t, self.s): - return combine_similar_callables(t, self.s) - else: - return self.default(self.s) - - def visit_tuple_type(self, t: TupleType) -> Type: - if isinstance(self.s, TupleType) and self.s.length() == t.length(): - items = [] # type: List[Type] - for i in range(t.length()): - items.append(self.meet(t.items[i], self.s.items[i])) - # TODO: What if the fallbacks are different? - return TupleType(items, t.fallback) - else: - return self.default(self.s) - - def visit_partial_type(self, t: PartialType) -> Type: - # We can't determine the meet of partial types. We should never get here. - assert False, 'Internal error' - - def visit_type_type(self, t: TypeType) -> Type: - if isinstance(self.s, TypeType): - typ = self.meet(t.item, self.s.item) - if not isinstance(typ, NoneTyp): - typ = TypeType(typ, line=t.line) - return typ - elif isinstance(self.s, Instance) and self.s.type.fullname() == 'builtins.type': - return t - else: - return self.default(self.s) - - def meet(self, s, t): - return meet_types(s, t) - - def default(self, typ): - if isinstance(typ, UnboundType): - return AnyType() - elif isinstance(typ, Void) or isinstance(typ, ErrorType): - return ErrorType() - else: - if experiments.STRICT_OPTIONAL: - return UninhabitedType() - else: - return NoneTyp() +from typing import cast, List + +from mypy.join import is_similar_callables, combine_similar_callables +from mypy.types import ( + Type, AnyType, TypeVisitor, UnboundType, Void, ErrorType, NoneTyp, TypeVarType, + Instance, CallableType, TupleType, ErasedType, TypeList, UnionType, PartialType, + DeletedType, UninhabitedType, TypeType +) +from mypy.subtypes import is_subtype +from mypy.nodes import TypeInfo + +from mypy import experiments + +# TODO Describe this module. + + +def meet_types(s: Type, t: Type) -> Type: + """Return the greatest lower bound of two types.""" + if isinstance(s, ErasedType): + return s + if isinstance(s, AnyType): + return t + if isinstance(s, UnionType) and not isinstance(t, UnionType): + s, t = t, s + return t.accept(TypeMeetVisitor(s)) + + +def meet_simple(s: Type, t: Type, default_right: bool = True) -> Type: + if s == t: + return s + if isinstance(s, UnionType): + return UnionType.make_simplified_union([meet_types(x, t) for x in s.items]) + elif not is_overlapping_types(s, t, use_promotions=True): + if experiments.STRICT_OPTIONAL: + return UninhabitedType() + else: + return NoneTyp() + else: + if default_right: + return t + else: + return s + + +def is_overlapping_types(t: Type, s: Type, use_promotions: bool = False) -> bool: + """Can a value of type t be a value of type s, or vice versa? + + Note that this effectively checks against erased types, since type + variables are erased at runtime and the overlapping check is based + on runtime behavior. + + If use_promotions is True, also consider type promotions (int and + float would only be overlapping if it's True). + + This does not consider multiple inheritance. For example, A and B in + the following example are not considered overlapping, even though + via C they can be overlapping: + + class A: ... + class B: ... + class C(A, B): ... + + The rationale is that this case is usually very unlikely as multiple + inheritance is rare. Also, we can't reliably determine whether + multiple inheritance actually occurs somewhere in a program, due to + stub files hiding implementation details, dynamic loading etc. + + TODO: Don't consider tuples always overlapping. + TODO: Don't consider callables always overlapping. + TODO: Don't consider type variables with values always overlapping. + """ + # Since we are effectively working with the erased types, we only + # need to handle occurrences of TypeVarType at the top level. + if isinstance(t, TypeVarType): + t = t.erase_to_union_or_bound() + if isinstance(s, TypeVarType): + s = s.erase_to_union_or_bound() + if isinstance(t, Instance): + if isinstance(s, Instance): + # Consider two classes non-disjoint if one is included in the mro + # of another. + if use_promotions: + # Consider cases like int vs float to be overlapping where + # there is only a type promotion relationship but not proper + # subclassing. + if t.type._promote and is_overlapping_types(t.type._promote, s): + return True + if s.type._promote and is_overlapping_types(s.type._promote, t): + return True + return t.type in s.type.mro or s.type in t.type.mro + if isinstance(t, UnionType): + return any(is_overlapping_types(item, s) + for item in t.items) + if isinstance(s, UnionType): + return any(is_overlapping_types(t, item) + for item in s.items) + if isinstance(t, TypeType) and isinstance(s, TypeType): + # If both types are TypeType, compare their inner types. + return is_overlapping_types(t.item, s.item, use_promotions) + elif isinstance(t, TypeType) or isinstance(s, TypeType): + # If exactly only one of t or s is a TypeType, check if one of them + # is an `object` or a `type` and otherwise assume no overlap. + other = s if isinstance(t, TypeType) else t + if isinstance(other, Instance): + return other.type.fullname() in {'builtins.object', 'builtins.type'} + else: + return False + if experiments.STRICT_OPTIONAL: + if isinstance(t, NoneTyp) != isinstance(s, NoneTyp): + # NoneTyp does not overlap with other non-Union types under strict Optional checking + return False + # We conservatively assume that non-instance, non-union, and non-TypeType types can overlap + # any other types. + return True + + +def nearest_builtin_ancestor(type: TypeInfo) -> TypeInfo: + for base in type.mro: + if base.defn.is_builtinclass: + return base + else: + return None + + +class TypeMeetVisitor(TypeVisitor[Type]): + def __init__(self, s: Type) -> None: + self.s = s + + def visit_unbound_type(self, t: UnboundType) -> Type: + if isinstance(self.s, Void) or isinstance(self.s, ErrorType): + return ErrorType() + elif isinstance(self.s, NoneTyp): + if experiments.STRICT_OPTIONAL: + return AnyType() + else: + return self.s + elif isinstance(self.s, UninhabitedType): + return self.s + else: + return AnyType() + + def visit_error_type(self, t: ErrorType) -> Type: + return t + + def visit_type_list(self, t: TypeList) -> Type: + assert False, 'Not supported' + + def visit_any(self, t: AnyType) -> Type: + return self.s + + def visit_union_type(self, t: UnionType) -> Type: + if isinstance(self.s, UnionType): + meets = [] # type: List[Type] + for x in t.items: + for y in self.s.items: + meets.append(meet_types(x, y)) + else: + meets = [meet_types(x, self.s) + for x in t.items] + return UnionType.make_simplified_union(meets) + + def visit_void(self, t: Void) -> Type: + if isinstance(self.s, Void): + return t + else: + return ErrorType() + + def visit_none_type(self, t: NoneTyp) -> Type: + if experiments.STRICT_OPTIONAL: + if isinstance(self.s, NoneTyp) or (isinstance(self.s, Instance) and + self.s.type.fullname() == 'builtins.object'): + return t + else: + return UninhabitedType() + else: + if not isinstance(self.s, Void) and not isinstance(self.s, ErrorType): + return t + else: + return ErrorType() + + def visit_uninhabited_type(self, t: UninhabitedType) -> Type: + if not isinstance(self.s, Void) and not isinstance(self.s, ErrorType): + return t + else: + return ErrorType() + + def visit_deleted_type(self, t: DeletedType) -> Type: + if not isinstance(self.s, Void) and not isinstance(self.s, ErrorType): + if isinstance(self.s, NoneTyp): + if experiments.STRICT_OPTIONAL: + return t + else: + return self.s + elif isinstance(self.s, UninhabitedType): + return self.s + else: + return t + else: + return ErrorType() + + def visit_erased_type(self, t: ErasedType) -> Type: + return self.s + + def visit_type_var(self, t: TypeVarType) -> Type: + if isinstance(self.s, TypeVarType) and self.s.id == t.id: + return self.s + else: + return self.default(self.s) + + def visit_instance(self, t: Instance) -> Type: + if isinstance(self.s, Instance): + si = self.s + if t.type == si.type: + if is_subtype(t, self.s) or is_subtype(self.s, t): + # Combine type arguments. We could have used join below + # equivalently. + args = [] # type: List[Type] + for i in range(len(t.args)): + args.append(self.meet(t.args[i], si.args[i])) + return Instance(t.type, args) + else: + if experiments.STRICT_OPTIONAL: + return UninhabitedType() + else: + return NoneTyp() + else: + if is_subtype(t, self.s): + return t + elif is_subtype(self.s, t): + # See also above comment. + return self.s + else: + if experiments.STRICT_OPTIONAL: + return UninhabitedType() + else: + return NoneTyp() + elif isinstance(self.s, TypeType): + return meet_types(t, self.s) + else: + return self.default(self.s) + + def visit_callable_type(self, t: CallableType) -> Type: + if isinstance(self.s, CallableType) and is_similar_callables(t, self.s): + return combine_similar_callables(t, self.s) + else: + return self.default(self.s) + + def visit_tuple_type(self, t: TupleType) -> Type: + if isinstance(self.s, TupleType) and self.s.length() == t.length(): + items = [] # type: List[Type] + for i in range(t.length()): + items.append(self.meet(t.items[i], self.s.items[i])) + # TODO: What if the fallbacks are different? + return TupleType(items, t.fallback) + else: + return self.default(self.s) + + def visit_partial_type(self, t: PartialType) -> Type: + # We can't determine the meet of partial types. We should never get here. + assert False, 'Internal error' + + def visit_type_type(self, t: TypeType) -> Type: + if isinstance(self.s, TypeType): + typ = self.meet(t.item, self.s.item) + if not isinstance(typ, NoneTyp): + typ = TypeType(typ, line=t.line) + return typ + elif isinstance(self.s, Instance) and self.s.type.fullname() == 'builtins.type': + return t + else: + return self.default(self.s) + + def meet(self, s, t): + return meet_types(s, t) + + def default(self, typ): + if isinstance(typ, UnboundType): + return AnyType() + elif isinstance(typ, Void) or isinstance(typ, ErrorType): + return ErrorType() + else: + if experiments.STRICT_OPTIONAL: + return UninhabitedType() + else: + return NoneTyp() diff --git a/mypy/messages.py b/mypy/messages.py index 5d82c4845782a..cc9b4761b703a 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -1,920 +1,920 @@ -"""Facilities and constants for generating error messages during type checking. - -The type checker itself does not deal with message string literals to -improve code clarity and to simplify localization (in the future).""" - -import re -import difflib - -from typing import cast, List, Dict, Any, Sequence, Iterable, Tuple - -from mypy.errors import Errors -from mypy.types import ( - Type, CallableType, Instance, TypeVarType, TupleType, UnionType, Void, NoneTyp, AnyType, - Overloaded, FunctionLike, DeletedType, TypeType -) -from mypy.nodes import ( - TypeInfo, Context, MypyFile, op_methods, FuncDef, reverse_type_aliases, - ARG_STAR, ARG_STAR2 -) - - -# Constants that represent simple type checker error message, i.e. messages -# that do not have any parameters. - -NO_RETURN_VALUE_EXPECTED = 'No return value expected' -INCOMPATIBLE_RETURN_VALUE_TYPE = 'Incompatible return value type' -RETURN_VALUE_EXPECTED = 'Return value expected' -BOOLEAN_VALUE_EXPECTED = 'Boolean value expected' -BOOLEAN_EXPECTED_FOR_IF = 'Boolean value expected for if condition' -BOOLEAN_EXPECTED_FOR_WHILE = 'Boolean value expected for while condition' -BOOLEAN_EXPECTED_FOR_UNTIL = 'Boolean value expected for until condition' -BOOLEAN_EXPECTED_FOR_NOT = 'Boolean value expected for not operand' -INVALID_EXCEPTION = 'Exception must be derived from BaseException' -INVALID_EXCEPTION_TYPE = 'Exception type must be derived from BaseException' -INVALID_RETURN_TYPE_FOR_GENERATOR = \ - 'The return type of a generator function should be "Generator" or one of its supertypes' -INVALID_GENERATOR_RETURN_ITEM_TYPE = \ - 'The return type of a generator function must be None in its third type parameter in Python 2' -YIELD_VALUE_EXPECTED = 'Yield value expected' -INCOMPATIBLE_TYPES = 'Incompatible types' -INCOMPATIBLE_TYPES_IN_ASSIGNMENT = 'Incompatible types in assignment' -INCOMPATIBLE_REDEFINITION = 'Incompatible redefinition' -INCOMPATIBLE_TYPES_IN_YIELD = 'Incompatible types in yield' -INCOMPATIBLE_TYPES_IN_YIELD_FROM = 'Incompatible types in "yield from"' -INCOMPATIBLE_TYPES_IN_STR_INTERPOLATION = 'Incompatible types in string interpolation' -INIT_MUST_HAVE_NONE_RETURN_TYPE = 'The return type of "__init__" must be None' -GETTER_TYPE_INCOMPATIBLE_WITH_SETTER = \ - 'Type of getter incompatible with setter' -TUPLE_INDEX_MUST_BE_AN_INT_LITERAL = 'Tuple index must be an integer literal' -TUPLE_SLICE_MUST_BE_AN_INT_LITERAL = 'Tuple slice must be an integer literal' -TUPLE_INDEX_OUT_OF_RANGE = 'Tuple index out of range' -TYPE_CONSTANT_EXPECTED = 'Type "Constant" or initializer expected' -INCOMPATIBLE_PAIR_ITEM_TYPE = 'Incompatible Pair item type' -INVALID_TYPE_APPLICATION_TARGET_TYPE = 'Invalid type application target type' -INCOMPATIBLE_TUPLE_ITEM_TYPE = 'Incompatible tuple item type' -INCOMPATIBLE_KEY_TYPE = 'Incompatible dictionary key type' -INCOMPATIBLE_VALUE_TYPE = 'Incompatible dictionary value type' -NEED_ANNOTATION_FOR_VAR = 'Need type annotation for variable' -ITERABLE_EXPECTED = 'Iterable expected' -INCOMPATIBLE_TYPES_IN_FOR = 'Incompatible types in for statement' -INCOMPATIBLE_ARRAY_VAR_ARGS = 'Incompatible variable arguments in call' -INVALID_SLICE_INDEX = 'Slice index must be an integer or None' -CANNOT_INFER_LAMBDA_TYPE = 'Cannot infer type of lambda' -CANNOT_INFER_ITEM_TYPE = 'Cannot infer iterable item type' -CANNOT_ACCESS_INIT = 'Cannot access "__init__" directly' -CANNOT_ASSIGN_TO_METHOD = 'Cannot assign to a method' -CANNOT_ASSIGN_TO_TYPE = 'Cannot assign to a type' -INCONSISTENT_ABSTRACT_OVERLOAD = \ - 'Overloaded method has both abstract and non-abstract variants' -READ_ONLY_PROPERTY_OVERRIDES_READ_WRITE = \ - 'Read-only property cannot override read-write property' -INSTANCE_LAYOUT_CONFLICT = 'Instance layout conflict in multiple inheritance' -FORMAT_REQUIRES_MAPPING = 'Format requires a mapping' -GENERIC_TYPE_NOT_VALID_AS_EXPRESSION = \ - "Generic type is prohibited as a runtime expression (use a type alias or '# type:' comment)" -RETURN_TYPE_CANNOT_BE_CONTRAVARIANT = "Cannot use a contravariant type variable as return type" -FUNCTION_PARAMETER_CANNOT_BE_COVARIANT = "Cannot use a covariant type variable as a parameter" -INCOMPATIBLE_IMPORT_OF = "Incompatible import of" -FUNCTION_TYPE_EXPECTED = "Function is missing a type annotation" -RETURN_TYPE_EXPECTED = "Function is missing a return type annotation" -ARGUMENT_TYPE_EXPECTED = "Function is missing a type annotation for one or more arguments" -KEYWORD_ARGUMENT_REQUIRES_STR_KEY_TYPE = \ - 'Keyword argument only valid with "str" key type in call to "dict"' -ALL_MUST_BE_SEQ_STR = 'Type of __all__ must be {}, not {}' - - -class MessageBuilder: - """Helper class for reporting type checker error messages with parameters. - - The methods of this class need to be provided with the context within a - file; the errors member manages the wider context. - - IDEA: Support a 'verbose mode' that includes full information about types - in error messages and that may otherwise produce more detailed error - messages. - """ - - # Report errors using this instance. It knows about the current file and - # import context. - errors = None # type: Errors - - modules = None # type: Dict[str, MypyFile] - - # Number of times errors have been disabled. - disable_count = 0 - - # Hack to deduplicate error messages from union types - disable_type_names = 0 - - def __init__(self, errors: Errors, modules: Dict[str, MypyFile]) -> None: - self.errors = errors - self.modules = modules - self.disable_count = 0 - self.disable_type_names = 0 - - # - # Helpers - # - - def copy(self) -> 'MessageBuilder': - new = MessageBuilder(self.errors.copy(), self.modules) - new.disable_count = self.disable_count - new.disable_type_names = self.disable_type_names - return new - - def add_errors(self, messages: 'MessageBuilder') -> None: - """Add errors in messages to this builder.""" - if self.disable_count <= 0: - for info in messages.errors.error_info: - self.errors.add_error_info(info) - - def disable_errors(self) -> None: - self.disable_count += 1 - - def enable_errors(self) -> None: - self.disable_count -= 1 - - def is_errors(self) -> bool: - return self.errors.is_errors() - - def report(self, msg: str, context: Context, severity: str, file: str = None) -> None: - """Report an error or note (unless disabled).""" - if self.disable_count <= 0: - self.errors.report(context.get_line() if context else -1, - msg.strip(), severity=severity, file=file) - - def fail(self, msg: str, context: Context, file: str = None) -> None: - """Report an error message (unless disabled).""" - # import pdb; pdb.set_trace() - self.report(msg, context, 'error', file=file) - - def note(self, msg: str, context: Context, file: str = None) -> None: - """Report an error message (unless disabled).""" - self.report(msg, context, 'note', file=file) - - def format(self, typ: Type, verbosity: int = 0) -> str: - """Convert a type to a relatively short string that is suitable for error messages. - - Mostly behave like format_simple below, but never return an empty string. - """ - s = self.format_simple(typ, verbosity) - if s != '': - # If format_simple returns a non-trivial result, use that. - return s - elif isinstance(typ, FunctionLike): - func = typ - if func.is_type_obj(): - # The type of a type object type can be derived from the - # return type (this always works). - itype = cast(Instance, func.items()[0].ret_type) - result = self.format(itype) - if verbosity >= 1: - # In some contexts we want to be explicit about the distinction - # between type X and the type of type object X. - result += ' (type object)' - return result - elif isinstance(func, CallableType): - return_type = strip_quotes(self.format(func.ret_type)) - if func.is_ellipsis_args: - return 'Callable[..., {}]'.format(return_type) - arg_types = [strip_quotes(self.format(t)) for t in func.arg_types] - return 'Callable[[{}], {}]'.format(", ".join(arg_types), return_type) - else: - # Use a simple representation for function types; proper - # function types may result in long and difficult-to-read - # error messages. - return 'overloaded function' - else: - # Default case; we simply have to return something meaningful here. - return 'object' - - def format_simple(self, typ: Type, verbosity: int = 0) -> str: - """Convert simple types to string that is suitable for error messages. - - Return "" for complex types. Try to keep the length of the result - relatively short to avoid overly long error messages. - - Examples: - builtins.int -> 'int' - Any type -> 'Any' - None -> None - callable type -> "" (empty string) - """ - if isinstance(typ, Instance): - itype = typ - # Get the short name of the type. - if verbosity >= 2: - base_str = itype.type.fullname() - else: - base_str = itype.type.name() - if itype.args == []: - # No type arguments. Place the type name in quotes to avoid - # potential for confusion: otherwise, the type name could be - # interpreted as a normal word. - return '"{}"'.format(base_str) - elif itype.type.fullname() == 'builtins.tuple': - item_type = strip_quotes(self.format(itype.args[0])) - return 'Tuple[{}, ...]'.format(item_type) - elif itype.type.fullname() in reverse_type_aliases: - alias = reverse_type_aliases[itype.type.fullname()] - alias = alias.split('.')[-1] - items = [strip_quotes(self.format(arg)) for arg in itype.args] - return '{}[{}]'.format(alias, ', '.join(items)) - else: - # There are type arguments. Convert the arguments to strings - # (using format() instead of format_simple() to avoid empty - # strings). If the result is too long, replace arguments - # with [...]. - a = [] # type: List[str] - for arg in itype.args: - a.append(strip_quotes(self.format(arg))) - s = ', '.join(a) - if len((base_str + s)) < 25: - return '{}[{}]'.format(base_str, s) - else: - return '{}[...]'.format(base_str) - elif isinstance(typ, TypeVarType): - # This is similar to non-generic instance types. - return '"{}"'.format(typ.name) - elif isinstance(typ, TupleType): - # Prefer the name of the fallback class (if not tuple), as it's more informative. - if typ.fallback.type.fullname() != 'builtins.tuple': - return self.format_simple(typ.fallback) - items = [] - for t in typ.items: - items.append(strip_quotes(self.format(t))) - s = '"Tuple[{}]"'.format(', '.join(items)) - if len(s) < 40: - return s - else: - return 'tuple(length {})'.format(len(items)) - elif isinstance(typ, UnionType): - # Only print Unions as Optionals if the Optional wouldn't have to contain another Union - print_as_optional = (len(typ.items) - - sum(isinstance(t, NoneTyp) for t in typ.items) == 1) - if print_as_optional: - rest = [t for t in typ.items if not isinstance(t, NoneTyp)] - return '"Optional[{}]"'.format(strip_quotes(self.format(rest[0]))) - else: - items = [] - for t in typ.items: - items.append(strip_quotes(self.format(t))) - s = '"Union[{}]"'.format(', '.join(items)) - if len(s) < 40: - return s - else: - return 'union type ({} items)'.format(len(items)) - elif isinstance(typ, Void): - return 'None' - elif isinstance(typ, NoneTyp): - return 'None' - elif isinstance(typ, AnyType): - return '"Any"' - elif isinstance(typ, DeletedType): - return '' - elif isinstance(typ, TypeType): - return 'Type[{}]'.format( - strip_quotes(self.format_simple(typ.item, verbosity))) - elif typ is None: - raise RuntimeError('Type is None') - else: - # No simple representation for this type that would convey very - # useful information. No need to mention the type explicitly in a - # message. - return '' - - def format_distinctly(self, type1: Type, type2: Type) -> Tuple[str, str]: - """Jointly format a pair of types to distinct strings. - - Increase the verbosity of the type strings until they become distinct. - """ - verbosity = 0 - for verbosity in range(3): - str1 = self.format(type1, verbosity=verbosity) - str2 = self.format(type2, verbosity=verbosity) - if str1 != str2: - return (str1, str2) - return (str1, str2) - - # - # Specific operations - # - - # The following operations are for generating specific error messages. They - # get some information as arguments, and they build an error message based - # on them. - - def has_no_attr(self, typ: Type, member: str, context: Context) -> Type: - """Report a missing or non-accessible member. - - The type argument is the base type. If member corresponds to - an operator, use the corresponding operator name in the - messages. Return type Any. - """ - if (isinstance(typ, Instance) and - typ.type.has_readable_member(member)): - self.fail('Member "{}" is not assignable'.format(member), context) - elif isinstance(typ, Void): - self.check_void(typ, context) - elif member == '__contains__': - self.fail('Unsupported right operand type for in ({})'.format( - self.format(typ)), context) - elif member in op_methods.values(): - # Access to a binary operator member (e.g. _add). This case does - # not handle indexing operations. - for op, method in op_methods.items(): - if method == member: - self.unsupported_left_operand(op, typ, context) - break - elif member == '__neg__': - self.fail('Unsupported operand type for unary - ({})'.format( - self.format(typ)), context) - elif member == '__pos__': - self.fail('Unsupported operand type for unary + ({})'.format( - self.format(typ)), context) - elif member == '__invert__': - self.fail('Unsupported operand type for ~ ({})'.format( - self.format(typ)), context) - elif member == '__getitem__': - # Indexed get. - self.fail('Value of type {} is not indexable'.format( - self.format(typ)), context) - elif member == '__setitem__': - # Indexed set. - self.fail('Unsupported target for indexed assignment', context) - elif member == '__call__': - self.fail('{} not callable'.format(self.format(typ)), context) - else: - # The non-special case: a missing ordinary attribute. - if not self.disable_type_names: - failed = False - if isinstance(typ, Instance) and typ.type.names: - alternatives = set(typ.type.names.keys()) - matches = [m for m in COMMON_MISTAKES.get(member, []) if m in alternatives] - matches.extend(best_matches(member, alternatives)[:3]) - if matches: - self.fail('{} has no attribute "{}"; maybe {}?'.format( - self.format(typ), member, pretty_or(matches)), context) - failed = True - if not failed: - self.fail('{} has no attribute "{}"'.format(self.format(typ), - member), context) - else: - self.fail('Some element of union has no attribute "{}"'.format( - member), context) - return AnyType() - - def unsupported_operand_types(self, op: str, left_type: Any, - right_type: Any, context: Context) -> None: - """Report unsupported operand types for a binary operation. - - Types can be Type objects or strings. - """ - if isinstance(left_type, Void) or isinstance(right_type, Void): - self.check_void(left_type, context) - self.check_void(right_type, context) - return - left_str = '' - if isinstance(left_type, str): - left_str = left_type - else: - left_str = self.format(left_type) - - right_str = '' - if isinstance(right_type, str): - right_str = right_type - else: - right_str = self.format(right_type) - - if self.disable_type_names: - msg = 'Unsupported operand types for {} (likely involving Union)'.format(op) - else: - msg = 'Unsupported operand types for {} ({} and {})'.format( - op, left_str, right_str) - self.fail(msg, context) - - def unsupported_left_operand(self, op: str, typ: Type, - context: Context) -> None: - if not self.check_void(typ, context): - if self.disable_type_names: - msg = 'Unsupported left operand type for {} (some union)'.format(op) - else: - msg = 'Unsupported left operand type for {} ({})'.format( - op, self.format(typ)) - self.fail(msg, context) - - def type_expected_as_right_operand_of_is(self, context: Context) -> None: - self.fail('Type expected as right operand of "is"', context) - - def not_callable(self, typ: Type, context: Context) -> Type: - self.fail('{} not callable'.format(self.format(typ)), context) - return AnyType() - - def untyped_function_call(self, callee: CallableType, context: Context) -> Type: - name = callee.name if callee.name is not None else '(unknown)' - self.fail('call to untyped function {} in typed context'.format(name), context) - return AnyType() - - def incompatible_argument(self, n: int, m: int, callee: CallableType, arg_type: Type, - arg_kind: int, context: Context) -> None: - """Report an error about an incompatible argument type. - - The argument type is arg_type, argument number is n and the - callee type is 'callee'. If the callee represents a method - that corresponds to an operator, use the corresponding - operator name in the messages. - """ - target = '' - if callee.name: - name = callee.name - base = extract_type(name) - - for op, method in op_methods.items(): - for variant in method, '__r' + method[2:]: - if name.startswith('"{}" of'.format(variant)): - if op == 'in' or variant != method: - # Reversed order of base/argument. - self.unsupported_operand_types(op, arg_type, base, - context) - else: - self.unsupported_operand_types(op, base, arg_type, - context) - return - - if name.startswith('"__getitem__" of'): - self.invalid_index_type(arg_type, base, context) - return - - if name.startswith('"__setitem__" of'): - if n == 1: - self.invalid_index_type(arg_type, base, context) - else: - msg = '{} (expression has type {}, target has type {})' - arg_type_str, callee_type_str = self.format_distinctly(arg_type, - callee.arg_types[n - 1]) - self.fail(msg.format(INCOMPATIBLE_TYPES_IN_ASSIGNMENT, - arg_type_str, callee_type_str), - context) - return - - target = 'to {} '.format(name) - - msg = '' - if callee.name == '': - name = callee.name[1:-1] - n -= 1 - msg = '{} item {} has incompatible type {}'.format( - name[0].upper() + name[1:], n, self.format_simple(arg_type)) - elif callee.name == '': - msg = 'List comprehension has incompatible type List[{}]'.format( - strip_quotes(self.format(arg_type))) - elif callee.name == '': - msg = 'Set comprehension has incompatible type Set[{}]'.format( - strip_quotes(self.format(arg_type))) - elif callee.name == '': - msg = ('{} expression in dictionary comprehension has incompatible type {}; ' - 'expected type {}').format( - 'Key' if n == 1 else 'Value', - self.format(arg_type), - self.format(callee.arg_types[n - 1])) - elif callee.name == '': - msg = 'Generator has incompatible item type {}'.format( - self.format_simple(arg_type)) - else: - try: - expected_type = callee.arg_types[m - 1] - except IndexError: # Varargs callees - expected_type = callee.arg_types[-1] - arg_type_str, expected_type_str = self.format_distinctly(arg_type, expected_type) - #import pdb; pdb.set_trace() - if arg_kind == ARG_STAR: - arg_type_str = '*' + arg_type_str - elif arg_kind == ARG_STAR2: - arg_type_str = '**' + arg_type_str - msg = 'Argument {} {}has incompatible type {}; expected {}'.format( - n, target, arg_type_str, expected_type_str) - self.fail(msg, context) - - def invalid_index_type(self, index_type: Type, base_str: str, - context: Context) -> None: - self.fail('Invalid index type {} for {}'.format( - self.format(index_type), base_str), context) - - def too_few_arguments(self, callee: CallableType, context: Context, - argument_names: List[str]) -> None: - if (argument_names is not None and not all(k is None for k in argument_names) - and len(argument_names) >= 1): - diff = [k for k in callee.arg_names if k not in argument_names] - if len(diff) == 1: - msg = 'Missing positional argument' - else: - msg = 'Missing positional arguments' - if callee.name and diff: - msg += ' "{}" in call to {}'.format('", "'.join(diff), callee.name) - else: - msg = 'Too few arguments' - if callee.name: - msg += ' for {}'.format(callee.name) - self.fail(msg, context) - - def too_many_arguments(self, callee: CallableType, context: Context) -> None: - msg = 'Too many arguments' - if callee.name: - msg += ' for {}'.format(callee.name) - self.fail(msg, context) - - def too_many_positional_arguments(self, callee: CallableType, - context: Context) -> None: - msg = 'Too many positional arguments' - if callee.name: - msg += ' for {}'.format(callee.name) - self.fail(msg, context) - - def unexpected_keyword_argument(self, callee: CallableType, name: str, - context: Context) -> None: - msg = 'Unexpected keyword argument "{}"'.format(name) - if callee.name: - msg += ' for {}'.format(callee.name) - self.fail(msg, context) - if callee.definition: - fullname = callee.definition.fullname() - if fullname is not None and '.' in fullname: - module_name = fullname.rsplit('.', 1)[0] - path = self.modules[module_name].path - self.note('{} defined here'.format(callee.name), callee.definition, file=path) - - def duplicate_argument_value(self, callee: CallableType, index: int, - context: Context) -> None: - self.fail('{} gets multiple values for keyword argument "{}"'. - format(capitalize(callable_name(callee)), - callee.arg_names[index]), context) - - def does_not_return_value(self, void_type: Type, context: Context) -> None: - """Report an error about a void type in a non-void context. - - The first argument must be a void type. If the void type has a - source in it, report it in the error message. This allows - giving messages such as 'Foo does not return a value'. - """ - if (cast(Void, void_type)).source is None: - self.fail('Function does not return a value', context) - else: - self.fail('{} does not return a value'.format( - capitalize((cast(Void, void_type)).source)), context) - - def deleted_as_rvalue(self, typ: DeletedType, context: Context) -> None: - """Report an error about using an deleted type as an rvalue.""" - if typ.source is None: - s = "" - else: - s = " '{}'".format(typ.source) - self.fail('Trying to read deleted variable{}'.format(s), context) - - def deleted_as_lvalue(self, typ: DeletedType, context: Context) -> None: - """Report an error about using an deleted type as an lvalue. - - Currently, this only occurs when trying to assign to an - exception variable outside the local except: blocks. - """ - if typ.source is None: - s = "" - else: - s = " '{}'".format(typ.source) - self.fail('Assignment to variable{} outside except: block'.format(s), context) - - def no_variant_matches_arguments(self, overload: Overloaded, arg_types: List[Type], - context: Context) -> None: - if overload.name(): - self.fail('No overload variant of {} matches argument types {}' - .format(overload.name(), arg_types), context) - else: - self.fail('No overload variant matches argument types {}'.format(arg_types), context) - - def function_variants_overlap(self, n1: int, n2: int, - context: Context) -> None: - self.fail('Function signature variants {} and {} overlap'.format( - n1 + 1, n2 + 1), context) - - def invalid_cast(self, target_type: Type, source_type: Type, - context: Context) -> None: - if not self.check_void(source_type, context): - self.fail('Cannot cast from {} to {}'.format( - self.format(source_type), self.format(target_type)), context) - - def wrong_number_values_to_unpack(self, provided: int, expected: int, - context: Context) -> None: - if provided < expected: - if provided == 1: - self.fail('Need more than 1 value to unpack ({} expected)'.format(expected), - context) - else: - self.fail('Need more than {} values to unpack ({} expected)'.format( - provided, expected), context) - elif provided > expected: - self.fail('Too many values to unpack ({} expected, {} provided)'.format( - expected, provided), context) - - def type_not_iterable(self, type: Type, context: Context) -> None: - self.fail('\'{}\' object is not iterable'.format(type), context) - - def incompatible_operator_assignment(self, op: str, - context: Context) -> None: - self.fail('Result type of {} incompatible in assignment'.format(op), - context) - - def incompatible_value_count_in_assignment(self, lvalue_count: int, - rvalue_count: int, - context: Context) -> None: - if rvalue_count < lvalue_count: - self.fail('Need {} values to assign'.format(lvalue_count), context) - elif rvalue_count > lvalue_count: - self.fail('Too many values to assign', context) - - def type_incompatible_with_supertype(self, name: str, supertype: TypeInfo, - context: Context) -> None: - self.fail('Type of "{}" incompatible with supertype "{}"'.format( - name, supertype.name), context) - - def signature_incompatible_with_supertype( - self, name: str, name_in_super: str, supertype: str, - context: Context) -> None: - target = self.override_target(name, name_in_super, supertype) - self.fail('Signature of "{}" incompatible with {}'.format( - name, target), context) - - def argument_incompatible_with_supertype( - self, arg_num: int, name: str, name_in_supertype: str, - supertype: str, context: Context) -> None: - target = self.override_target(name, name_in_supertype, supertype) - self.fail('Argument {} of "{}" incompatible with {}' - .format(arg_num, name, target), context) - - def return_type_incompatible_with_supertype( - self, name: str, name_in_supertype: str, supertype: str, - context: Context) -> None: - target = self.override_target(name, name_in_supertype, supertype) - self.fail('Return type of "{}" incompatible with {}' - .format(name, target), context) - - def override_target(self, name: str, name_in_super: str, - supertype: str) -> str: - target = 'supertype "{}"'.format(supertype) - if name_in_super != name: - target = '"{}" of {}'.format(name_in_super, target) - return target - - def boolean_return_value_expected(self, method: str, - context: Context) -> None: - self.fail('Boolean return value expected for method "{}"'.format( - method), context) - - def incompatible_type_application(self, expected_arg_count: int, - actual_arg_count: int, - context: Context) -> None: - if expected_arg_count == 0: - self.fail('Type application targets a non-generic function', - context) - elif actual_arg_count > expected_arg_count: - self.fail('Type application has too many types ({} expected)' - .format(expected_arg_count), context) - else: - self.fail('Type application has too few types ({} expected)' - .format(expected_arg_count), context) - - def incompatible_array_item_type(self, typ: Type, index: int, - context: Context) -> None: - self.fail('Array item {} has incompatible type {}'.format( - index, self.format(typ)), context) - - def could_not_infer_type_arguments(self, callee_type: CallableType, n: int, - context: Context) -> None: - if callee_type.name and n > 0: - self.fail('Cannot infer type argument {} of {}'.format( - n, callee_type.name), context) - else: - self.fail('Cannot infer function type argument', context) - - def invalid_var_arg(self, typ: Type, context: Context) -> None: - self.fail('List or tuple expected as variable arguments', context) - - def invalid_keyword_var_arg(self, typ: Type, context: Context) -> None: - if isinstance(typ, Instance) and (typ.type.fullname() == 'builtins.dict'): - self.fail('Keywords must be strings', context) - else: - self.fail('Argument after ** must be a dictionary', - context) - - def incomplete_type_var_match(self, member: str, context: Context) -> None: - self.fail('"{}" has incomplete match to supertype type variable' - .format(member), context) - - def not_implemented(self, msg: str, context: Context) -> Type: - self.fail('Feature not implemented yet ({})'.format(msg), context) - return AnyType() - - def undefined_in_superclass(self, member: str, context: Context) -> None: - self.fail('"{}" undefined in superclass'.format(member), context) - - def check_void(self, typ: Type, context: Context) -> bool: - """If type is void, report an error such as '.. does not - return a value' and return True. Otherwise, return False. - """ - if isinstance(typ, Void): - self.does_not_return_value(typ, context) - return True - else: - return False - - def too_few_string_formatting_arguments(self, context: Context) -> None: - self.fail('Not enough arguments for format string', context) - - def too_many_string_formatting_arguments(self, context: Context) -> None: - self.fail('Not all arguments converted during string formatting', context) - - def incomplete_conversion_specifier_format(self, context: Context) -> None: - self.fail('Incomplete format', context) - - def unsupported_placeholder(self, placeholder: str, context: Context) -> None: - self.fail('Unsupported format character \'%s\'' % placeholder, context) - - def string_interpolation_with_star_and_key(self, context: Context) -> None: - self.fail('String interpolation contains both stars and mapping keys', context) - - def requires_int_or_char(self, context: Context) -> None: - self.fail('%c requires int or char', context) - - def key_not_in_mapping(self, key: str, context: Context) -> None: - self.fail('Key \'%s\' not found in mapping' % key, context) - - def string_interpolation_mixing_key_and_non_keys(self, context: Context) -> None: - self.fail('String interpolation mixes specifier with and without mapping keys', context) - - def cannot_determine_type(self, name: str, context: Context) -> None: - self.fail("Cannot determine type of '%s'" % name, context) - - def cannot_determine_type_in_base(self, name: str, base: str, context: Context) -> None: - self.fail("Cannot determine type of '%s' in base class '%s'" % (name, base), context) - - def invalid_method_type(self, sig: CallableType, context: Context) -> None: - self.fail('Invalid method type', context) - - def invalid_class_method_type(self, sig: CallableType, context: Context) -> None: - self.fail('Invalid class method type', context) - - def incompatible_conditional_function_def(self, defn: FuncDef) -> None: - self.fail('All conditional function variants must have identical ' - 'signatures', defn) - - def cannot_instantiate_abstract_class(self, class_name: str, - abstract_attributes: List[str], - context: Context) -> None: - attrs = format_string_list("'%s'" % a for a in abstract_attributes[:5]) - self.fail("Cannot instantiate abstract class '%s' with abstract " - "attribute%s %s" % (class_name, plural_s(abstract_attributes), - attrs), - context) - - def base_class_definitions_incompatible(self, name: str, base1: TypeInfo, - base2: TypeInfo, - context: Context) -> None: - self.fail('Definition of "{}" in base class "{}" is incompatible ' - 'with definition in base class "{}"'.format( - name, base1.name(), base2.name()), context) - - def cant_assign_to_method(self, context: Context) -> None: - self.fail(CANNOT_ASSIGN_TO_METHOD, context) - - def read_only_property(self, name: str, type: TypeInfo, - context: Context) -> None: - self.fail('Property "{}" defined in "{}" is read-only'.format( - name, type.name()), context) - - def incompatible_typevar_value(self, callee: CallableType, index: int, - type: Type, context: Context) -> None: - self.fail('Type argument {} of {} has incompatible value {}'.format( - index, callable_name(callee), self.format(type)), context) - - def overloaded_signatures_overlap(self, index1: int, index2: int, - context: Context) -> None: - self.fail('Overloaded function signatures {} and {} overlap with ' - 'incompatible return types'.format(index1, index2), context) - - def invalid_reverse_operator_signature(self, reverse: str, other: str, - context: Context) -> None: - self.fail('"Any" return type expected since argument to {} ' - 'does not support {}'.format(reverse, other), context) - - def reverse_operator_method_with_any_arg_must_return_any( - self, method: str, context: Context) -> None: - self.fail('"Any" return type expected since argument to {} ' - 'has type "Any"'.format(method), context) - - def operator_method_signatures_overlap( - self, reverse_class: str, reverse_method: str, forward_class: str, - forward_method: str, context: Context) -> None: - self.fail('Signatures of "{}" of "{}" and "{}" of "{}" ' - 'are unsafely overlapping'.format( - reverse_method, reverse_class, - forward_method, forward_class), - context) - - def forward_operator_not_callable( - self, forward_method: str, context: Context) -> None: - self.fail('Forward operator "{}" is not callable'.format( - forward_method), context) - - def signatures_incompatible(self, method: str, other_method: str, - context: Context) -> None: - self.fail('Signatures of "{}" and "{}" are incompatible'.format( - method, other_method), context) - - def yield_from_invalid_operand_type(self, expr: Type, context: Context) -> Type: - text = self.format(expr) if self.format(expr) != 'object' else expr - self.fail('"yield from" can\'t be applied to {}'.format(text), context) - return AnyType() - - def invalid_signature(self, func_type: Type, context: Context) -> None: - self.fail('Invalid signature "{}"'.format(func_type), context) - - def reveal_type(self, typ: Type, context: Context) -> None: - self.fail('Revealed type is \'{}\''.format(typ), context) - - def unsupported_type_type(self, item: Type, context: Context) -> None: - self.fail('Unsupported type Type[{}]'.format(self.format(item)), context) - - def redundant_cast(self, typ: Type, context: Context) -> None: - self.note('Redundant cast to {}'.format(self.format(typ)), context) - - -def capitalize(s: str) -> str: - """Capitalize the first character of a string.""" - if s == '': - return '' - else: - return s[0].upper() + s[1:] - - -def extract_type(name: str) -> str: - """If the argument is the name of a method (of form C.m), return - the type portion in quotes (e.g. "y"). Otherwise, return the string - unmodified. - """ - name = re.sub('^"[a-zA-Z0-9_]+" of ', '', name) - return name - - -def strip_quotes(s: str) -> str: - """Strip a double quote at the beginning and end of the string, if any.""" - s = re.sub('^"', '', s) - s = re.sub('"$', '', s) - return s - - -def plural_s(s: Sequence[Any]) -> str: - if len(s) > 1: - return 's' - else: - return '' - - -def format_string_list(s: Iterable[str]) -> str: - l = list(s) - assert len(l) > 0 - if len(l) == 1: - return l[0] - else: - return '%s and %s' % (', '.join(l[:-1]), l[-1]) - - -def callable_name(type: CallableType) -> str: - if type.name: - return type.name - else: - return 'function' - - -def temp_message_builder() -> MessageBuilder: - """Return a message builder usable for throwaway errors (which may not format properly).""" - return MessageBuilder(Errors(), {}) - - -# For hard-coding suggested missing member alternatives. -COMMON_MISTAKES = { - 'add': ('append', 'extend'), -} # type: Dict[str, Sequence[str]] - - -def best_matches(current: str, options: Iterable[str]) -> List[str]: - ratios = {v: difflib.SequenceMatcher(a=current, b=v).ratio() for v in options} - return sorted((o for o in options if ratios[o] > 0.75), - reverse=True, key=lambda v: (ratios[v], v)) - - -def pretty_or(args: List[str]) -> str: - quoted = ['"' + a + '"' for a in args] - if len(quoted) == 1: - return quoted[0] - if len(quoted) == 2: - return "{} or {}".format(quoted[0], quoted[1]) - return ", ".join(quoted[:-1]) + ", or " + quoted[-1] +"""Facilities and constants for generating error messages during type checking. + +The type checker itself does not deal with message string literals to +improve code clarity and to simplify localization (in the future).""" + +import re +import difflib + +from typing import cast, List, Dict, Any, Sequence, Iterable, Tuple + +from mypy.errors import Errors +from mypy.types import ( + Type, CallableType, Instance, TypeVarType, TupleType, UnionType, Void, NoneTyp, AnyType, + Overloaded, FunctionLike, DeletedType, TypeType +) +from mypy.nodes import ( + TypeInfo, Context, MypyFile, op_methods, FuncDef, reverse_type_aliases, + ARG_STAR, ARG_STAR2 +) + + +# Constants that represent simple type checker error message, i.e. messages +# that do not have any parameters. + +NO_RETURN_VALUE_EXPECTED = 'No return value expected' +INCOMPATIBLE_RETURN_VALUE_TYPE = 'Incompatible return value type' +RETURN_VALUE_EXPECTED = 'Return value expected' +BOOLEAN_VALUE_EXPECTED = 'Boolean value expected' +BOOLEAN_EXPECTED_FOR_IF = 'Boolean value expected for if condition' +BOOLEAN_EXPECTED_FOR_WHILE = 'Boolean value expected for while condition' +BOOLEAN_EXPECTED_FOR_UNTIL = 'Boolean value expected for until condition' +BOOLEAN_EXPECTED_FOR_NOT = 'Boolean value expected for not operand' +INVALID_EXCEPTION = 'Exception must be derived from BaseException' +INVALID_EXCEPTION_TYPE = 'Exception type must be derived from BaseException' +INVALID_RETURN_TYPE_FOR_GENERATOR = \ + 'The return type of a generator function should be "Generator" or one of its supertypes' +INVALID_GENERATOR_RETURN_ITEM_TYPE = \ + 'The return type of a generator function must be None in its third type parameter in Python 2' +YIELD_VALUE_EXPECTED = 'Yield value expected' +INCOMPATIBLE_TYPES = 'Incompatible types' +INCOMPATIBLE_TYPES_IN_ASSIGNMENT = 'Incompatible types in assignment' +INCOMPATIBLE_REDEFINITION = 'Incompatible redefinition' +INCOMPATIBLE_TYPES_IN_YIELD = 'Incompatible types in yield' +INCOMPATIBLE_TYPES_IN_YIELD_FROM = 'Incompatible types in "yield from"' +INCOMPATIBLE_TYPES_IN_STR_INTERPOLATION = 'Incompatible types in string interpolation' +INIT_MUST_HAVE_NONE_RETURN_TYPE = 'The return type of "__init__" must be None' +GETTER_TYPE_INCOMPATIBLE_WITH_SETTER = \ + 'Type of getter incompatible with setter' +TUPLE_INDEX_MUST_BE_AN_INT_LITERAL = 'Tuple index must be an integer literal' +TUPLE_SLICE_MUST_BE_AN_INT_LITERAL = 'Tuple slice must be an integer literal' +TUPLE_INDEX_OUT_OF_RANGE = 'Tuple index out of range' +TYPE_CONSTANT_EXPECTED = 'Type "Constant" or initializer expected' +INCOMPATIBLE_PAIR_ITEM_TYPE = 'Incompatible Pair item type' +INVALID_TYPE_APPLICATION_TARGET_TYPE = 'Invalid type application target type' +INCOMPATIBLE_TUPLE_ITEM_TYPE = 'Incompatible tuple item type' +INCOMPATIBLE_KEY_TYPE = 'Incompatible dictionary key type' +INCOMPATIBLE_VALUE_TYPE = 'Incompatible dictionary value type' +NEED_ANNOTATION_FOR_VAR = 'Need type annotation for variable' +ITERABLE_EXPECTED = 'Iterable expected' +INCOMPATIBLE_TYPES_IN_FOR = 'Incompatible types in for statement' +INCOMPATIBLE_ARRAY_VAR_ARGS = 'Incompatible variable arguments in call' +INVALID_SLICE_INDEX = 'Slice index must be an integer or None' +CANNOT_INFER_LAMBDA_TYPE = 'Cannot infer type of lambda' +CANNOT_INFER_ITEM_TYPE = 'Cannot infer iterable item type' +CANNOT_ACCESS_INIT = 'Cannot access "__init__" directly' +CANNOT_ASSIGN_TO_METHOD = 'Cannot assign to a method' +CANNOT_ASSIGN_TO_TYPE = 'Cannot assign to a type' +INCONSISTENT_ABSTRACT_OVERLOAD = \ + 'Overloaded method has both abstract and non-abstract variants' +READ_ONLY_PROPERTY_OVERRIDES_READ_WRITE = \ + 'Read-only property cannot override read-write property' +INSTANCE_LAYOUT_CONFLICT = 'Instance layout conflict in multiple inheritance' +FORMAT_REQUIRES_MAPPING = 'Format requires a mapping' +GENERIC_TYPE_NOT_VALID_AS_EXPRESSION = \ + "Generic type is prohibited as a runtime expression (use a type alias or '# type:' comment)" +RETURN_TYPE_CANNOT_BE_CONTRAVARIANT = "Cannot use a contravariant type variable as return type" +FUNCTION_PARAMETER_CANNOT_BE_COVARIANT = "Cannot use a covariant type variable as a parameter" +INCOMPATIBLE_IMPORT_OF = "Incompatible import of" +FUNCTION_TYPE_EXPECTED = "Function is missing a type annotation" +RETURN_TYPE_EXPECTED = "Function is missing a return type annotation" +ARGUMENT_TYPE_EXPECTED = "Function is missing a type annotation for one or more arguments" +KEYWORD_ARGUMENT_REQUIRES_STR_KEY_TYPE = \ + 'Keyword argument only valid with "str" key type in call to "dict"' +ALL_MUST_BE_SEQ_STR = 'Type of __all__ must be {}, not {}' + + +class MessageBuilder: + """Helper class for reporting type checker error messages with parameters. + + The methods of this class need to be provided with the context within a + file; the errors member manages the wider context. + + IDEA: Support a 'verbose mode' that includes full information about types + in error messages and that may otherwise produce more detailed error + messages. + """ + + # Report errors using this instance. It knows about the current file and + # import context. + errors = None # type: Errors + + modules = None # type: Dict[str, MypyFile] + + # Number of times errors have been disabled. + disable_count = 0 + + # Hack to deduplicate error messages from union types + disable_type_names = 0 + + def __init__(self, errors: Errors, modules: Dict[str, MypyFile]) -> None: + self.errors = errors + self.modules = modules + self.disable_count = 0 + self.disable_type_names = 0 + + # + # Helpers + # + + def copy(self) -> 'MessageBuilder': + new = MessageBuilder(self.errors.copy(), self.modules) + new.disable_count = self.disable_count + new.disable_type_names = self.disable_type_names + return new + + def add_errors(self, messages: 'MessageBuilder') -> None: + """Add errors in messages to this builder.""" + if self.disable_count <= 0: + for info in messages.errors.error_info: + self.errors.add_error_info(info) + + def disable_errors(self) -> None: + self.disable_count += 1 + + def enable_errors(self) -> None: + self.disable_count -= 1 + + def is_errors(self) -> bool: + return self.errors.is_errors() + + def report(self, msg: str, context: Context, severity: str, file: str = None) -> None: + """Report an error or note (unless disabled).""" + if self.disable_count <= 0: + self.errors.report(context.get_line() if context else -1, + msg.strip(), severity=severity, file=file) + + def fail(self, msg: str, context: Context, file: str = None) -> None: + """Report an error message (unless disabled).""" + # import pdb; pdb.set_trace() + self.report(msg, context, 'error', file=file) + + def note(self, msg: str, context: Context, file: str = None) -> None: + """Report an error message (unless disabled).""" + self.report(msg, context, 'note', file=file) + + def format(self, typ: Type, verbosity: int = 0) -> str: + """Convert a type to a relatively short string that is suitable for error messages. + + Mostly behave like format_simple below, but never return an empty string. + """ + s = self.format_simple(typ, verbosity) + if s != '': + # If format_simple returns a non-trivial result, use that. + return s + elif isinstance(typ, FunctionLike): + func = typ + if func.is_type_obj(): + # The type of a type object type can be derived from the + # return type (this always works). + itype = cast(Instance, func.items()[0].ret_type) + result = self.format(itype) + if verbosity >= 1: + # In some contexts we want to be explicit about the distinction + # between type X and the type of type object X. + result += ' (type object)' + return result + elif isinstance(func, CallableType): + return_type = strip_quotes(self.format(func.ret_type)) + if func.is_ellipsis_args: + return 'Callable[..., {}]'.format(return_type) + arg_types = [strip_quotes(self.format(t)) for t in func.arg_types] + return 'Callable[[{}], {}]'.format(", ".join(arg_types), return_type) + else: + # Use a simple representation for function types; proper + # function types may result in long and difficult-to-read + # error messages. + return 'overloaded function' + else: + # Default case; we simply have to return something meaningful here. + return 'object' + + def format_simple(self, typ: Type, verbosity: int = 0) -> str: + """Convert simple types to string that is suitable for error messages. + + Return "" for complex types. Try to keep the length of the result + relatively short to avoid overly long error messages. + + Examples: + builtins.int -> 'int' + Any type -> 'Any' + None -> None + callable type -> "" (empty string) + """ + if isinstance(typ, Instance): + itype = typ + # Get the short name of the type. + if verbosity >= 2: + base_str = itype.type.fullname() + else: + base_str = itype.type.name() + if itype.args == []: + # No type arguments. Place the type name in quotes to avoid + # potential for confusion: otherwise, the type name could be + # interpreted as a normal word. + return '"{}"'.format(base_str) + elif itype.type.fullname() == 'builtins.tuple': + item_type = strip_quotes(self.format(itype.args[0])) + return 'Tuple[{}, ...]'.format(item_type) + elif itype.type.fullname() in reverse_type_aliases: + alias = reverse_type_aliases[itype.type.fullname()] + alias = alias.split('.')[-1] + items = [strip_quotes(self.format(arg)) for arg in itype.args] + return '{}[{}]'.format(alias, ', '.join(items)) + else: + # There are type arguments. Convert the arguments to strings + # (using format() instead of format_simple() to avoid empty + # strings). If the result is too long, replace arguments + # with [...]. + a = [] # type: List[str] + for arg in itype.args: + a.append(strip_quotes(self.format(arg))) + s = ', '.join(a) + if len((base_str + s)) < 25: + return '{}[{}]'.format(base_str, s) + else: + return '{}[...]'.format(base_str) + elif isinstance(typ, TypeVarType): + # This is similar to non-generic instance types. + return '"{}"'.format(typ.name) + elif isinstance(typ, TupleType): + # Prefer the name of the fallback class (if not tuple), as it's more informative. + if typ.fallback.type.fullname() != 'builtins.tuple': + return self.format_simple(typ.fallback) + items = [] + for t in typ.items: + items.append(strip_quotes(self.format(t))) + s = '"Tuple[{}]"'.format(', '.join(items)) + if len(s) < 40: + return s + else: + return 'tuple(length {})'.format(len(items)) + elif isinstance(typ, UnionType): + # Only print Unions as Optionals if the Optional wouldn't have to contain another Union + print_as_optional = (len(typ.items) - + sum(isinstance(t, NoneTyp) for t in typ.items) == 1) + if print_as_optional: + rest = [t for t in typ.items if not isinstance(t, NoneTyp)] + return '"Optional[{}]"'.format(strip_quotes(self.format(rest[0]))) + else: + items = [] + for t in typ.items: + items.append(strip_quotes(self.format(t))) + s = '"Union[{}]"'.format(', '.join(items)) + if len(s) < 40: + return s + else: + return 'union type ({} items)'.format(len(items)) + elif isinstance(typ, Void): + return 'None' + elif isinstance(typ, NoneTyp): + return 'None' + elif isinstance(typ, AnyType): + return '"Any"' + elif isinstance(typ, DeletedType): + return '' + elif isinstance(typ, TypeType): + return 'Type[{}]'.format( + strip_quotes(self.format_simple(typ.item, verbosity))) + elif typ is None: + raise RuntimeError('Type is None') + else: + # No simple representation for this type that would convey very + # useful information. No need to mention the type explicitly in a + # message. + return '' + + def format_distinctly(self, type1: Type, type2: Type) -> Tuple[str, str]: + """Jointly format a pair of types to distinct strings. + + Increase the verbosity of the type strings until they become distinct. + """ + verbosity = 0 + for verbosity in range(3): + str1 = self.format(type1, verbosity=verbosity) + str2 = self.format(type2, verbosity=verbosity) + if str1 != str2: + return (str1, str2) + return (str1, str2) + + # + # Specific operations + # + + # The following operations are for generating specific error messages. They + # get some information as arguments, and they build an error message based + # on them. + + def has_no_attr(self, typ: Type, member: str, context: Context) -> Type: + """Report a missing or non-accessible member. + + The type argument is the base type. If member corresponds to + an operator, use the corresponding operator name in the + messages. Return type Any. + """ + if (isinstance(typ, Instance) and + typ.type.has_readable_member(member)): + self.fail('Member "{}" is not assignable'.format(member), context) + elif isinstance(typ, Void): + self.check_void(typ, context) + elif member == '__contains__': + self.fail('Unsupported right operand type for in ({})'.format( + self.format(typ)), context) + elif member in op_methods.values(): + # Access to a binary operator member (e.g. _add). This case does + # not handle indexing operations. + for op, method in op_methods.items(): + if method == member: + self.unsupported_left_operand(op, typ, context) + break + elif member == '__neg__': + self.fail('Unsupported operand type for unary - ({})'.format( + self.format(typ)), context) + elif member == '__pos__': + self.fail('Unsupported operand type for unary + ({})'.format( + self.format(typ)), context) + elif member == '__invert__': + self.fail('Unsupported operand type for ~ ({})'.format( + self.format(typ)), context) + elif member == '__getitem__': + # Indexed get. + self.fail('Value of type {} is not indexable'.format( + self.format(typ)), context) + elif member == '__setitem__': + # Indexed set. + self.fail('Unsupported target for indexed assignment', context) + elif member == '__call__': + self.fail('{} not callable'.format(self.format(typ)), context) + else: + # The non-special case: a missing ordinary attribute. + if not self.disable_type_names: + failed = False + if isinstance(typ, Instance) and typ.type.names: + alternatives = set(typ.type.names.keys()) + matches = [m for m in COMMON_MISTAKES.get(member, []) if m in alternatives] + matches.extend(best_matches(member, alternatives)[:3]) + if matches: + self.fail('{} has no attribute "{}"; maybe {}?'.format( + self.format(typ), member, pretty_or(matches)), context) + failed = True + if not failed: + self.fail('{} has no attribute "{}"'.format(self.format(typ), + member), context) + else: + self.fail('Some element of union has no attribute "{}"'.format( + member), context) + return AnyType() + + def unsupported_operand_types(self, op: str, left_type: Any, + right_type: Any, context: Context) -> None: + """Report unsupported operand types for a binary operation. + + Types can be Type objects or strings. + """ + if isinstance(left_type, Void) or isinstance(right_type, Void): + self.check_void(left_type, context) + self.check_void(right_type, context) + return + left_str = '' + if isinstance(left_type, str): + left_str = left_type + else: + left_str = self.format(left_type) + + right_str = '' + if isinstance(right_type, str): + right_str = right_type + else: + right_str = self.format(right_type) + + if self.disable_type_names: + msg = 'Unsupported operand types for {} (likely involving Union)'.format(op) + else: + msg = 'Unsupported operand types for {} ({} and {})'.format( + op, left_str, right_str) + self.fail(msg, context) + + def unsupported_left_operand(self, op: str, typ: Type, + context: Context) -> None: + if not self.check_void(typ, context): + if self.disable_type_names: + msg = 'Unsupported left operand type for {} (some union)'.format(op) + else: + msg = 'Unsupported left operand type for {} ({})'.format( + op, self.format(typ)) + self.fail(msg, context) + + def type_expected_as_right_operand_of_is(self, context: Context) -> None: + self.fail('Type expected as right operand of "is"', context) + + def not_callable(self, typ: Type, context: Context) -> Type: + self.fail('{} not callable'.format(self.format(typ)), context) + return AnyType() + + def untyped_function_call(self, callee: CallableType, context: Context) -> Type: + name = callee.name if callee.name is not None else '(unknown)' + self.fail('call to untyped function {} in typed context'.format(name), context) + return AnyType() + + def incompatible_argument(self, n: int, m: int, callee: CallableType, arg_type: Type, + arg_kind: int, context: Context) -> None: + """Report an error about an incompatible argument type. + + The argument type is arg_type, argument number is n and the + callee type is 'callee'. If the callee represents a method + that corresponds to an operator, use the corresponding + operator name in the messages. + """ + target = '' + if callee.name: + name = callee.name + base = extract_type(name) + + for op, method in op_methods.items(): + for variant in method, '__r' + method[2:]: + if name.startswith('"{}" of'.format(variant)): + if op == 'in' or variant != method: + # Reversed order of base/argument. + self.unsupported_operand_types(op, arg_type, base, + context) + else: + self.unsupported_operand_types(op, base, arg_type, + context) + return + + if name.startswith('"__getitem__" of'): + self.invalid_index_type(arg_type, base, context) + return + + if name.startswith('"__setitem__" of'): + if n == 1: + self.invalid_index_type(arg_type, base, context) + else: + msg = '{} (expression has type {}, target has type {})' + arg_type_str, callee_type_str = self.format_distinctly(arg_type, + callee.arg_types[n - 1]) + self.fail(msg.format(INCOMPATIBLE_TYPES_IN_ASSIGNMENT, + arg_type_str, callee_type_str), + context) + return + + target = 'to {} '.format(name) + + msg = '' + if callee.name == '': + name = callee.name[1:-1] + n -= 1 + msg = '{} item {} has incompatible type {}'.format( + name[0].upper() + name[1:], n, self.format_simple(arg_type)) + elif callee.name == '': + msg = 'List comprehension has incompatible type List[{}]'.format( + strip_quotes(self.format(arg_type))) + elif callee.name == '': + msg = 'Set comprehension has incompatible type Set[{}]'.format( + strip_quotes(self.format(arg_type))) + elif callee.name == '': + msg = ('{} expression in dictionary comprehension has incompatible type {}; ' + 'expected type {}').format( + 'Key' if n == 1 else 'Value', + self.format(arg_type), + self.format(callee.arg_types[n - 1])) + elif callee.name == '': + msg = 'Generator has incompatible item type {}'.format( + self.format_simple(arg_type)) + else: + try: + expected_type = callee.arg_types[m - 1] + except IndexError: # Varargs callees + expected_type = callee.arg_types[-1] + arg_type_str, expected_type_str = self.format_distinctly(arg_type, expected_type) + #import pdb; pdb.set_trace() + if arg_kind == ARG_STAR: + arg_type_str = '*' + arg_type_str + elif arg_kind == ARG_STAR2: + arg_type_str = '**' + arg_type_str + msg = 'Argument {} {}has incompatible type {}; expected {}'.format( + n, target, arg_type_str, expected_type_str) + self.fail(msg, context) + + def invalid_index_type(self, index_type: Type, base_str: str, + context: Context) -> None: + self.fail('Invalid index type {} for {}'.format( + self.format(index_type), base_str), context) + + def too_few_arguments(self, callee: CallableType, context: Context, + argument_names: List[str]) -> None: + if (argument_names is not None and not all(k is None for k in argument_names) + and len(argument_names) >= 1): + diff = [k for k in callee.arg_names if k not in argument_names] + if len(diff) == 1: + msg = 'Missing positional argument' + else: + msg = 'Missing positional arguments' + if callee.name and diff: + msg += ' "{}" in call to {}'.format('", "'.join(diff), callee.name) + else: + msg = 'Too few arguments' + if callee.name: + msg += ' for {}'.format(callee.name) + self.fail(msg, context) + + def too_many_arguments(self, callee: CallableType, context: Context) -> None: + msg = 'Too many arguments' + if callee.name: + msg += ' for {}'.format(callee.name) + self.fail(msg, context) + + def too_many_positional_arguments(self, callee: CallableType, + context: Context) -> None: + msg = 'Too many positional arguments' + if callee.name: + msg += ' for {}'.format(callee.name) + self.fail(msg, context) + + def unexpected_keyword_argument(self, callee: CallableType, name: str, + context: Context) -> None: + msg = 'Unexpected keyword argument "{}"'.format(name) + if callee.name: + msg += ' for {}'.format(callee.name) + self.fail(msg, context) + if callee.definition: + fullname = callee.definition.fullname() + if fullname is not None and '.' in fullname: + module_name = fullname.rsplit('.', 1)[0] + path = self.modules[module_name].path + self.note('{} defined here'.format(callee.name), callee.definition, file=path) + + def duplicate_argument_value(self, callee: CallableType, index: int, + context: Context) -> None: + self.fail('{} gets multiple values for keyword argument "{}"'. + format(capitalize(callable_name(callee)), + callee.arg_names[index]), context) + + def does_not_return_value(self, void_type: Type, context: Context) -> None: + """Report an error about a void type in a non-void context. + + The first argument must be a void type. If the void type has a + source in it, report it in the error message. This allows + giving messages such as 'Foo does not return a value'. + """ + if (cast(Void, void_type)).source is None: + self.fail('Function does not return a value', context) + else: + self.fail('{} does not return a value'.format( + capitalize((cast(Void, void_type)).source)), context) + + def deleted_as_rvalue(self, typ: DeletedType, context: Context) -> None: + """Report an error about using an deleted type as an rvalue.""" + if typ.source is None: + s = "" + else: + s = " '{}'".format(typ.source) + self.fail('Trying to read deleted variable{}'.format(s), context) + + def deleted_as_lvalue(self, typ: DeletedType, context: Context) -> None: + """Report an error about using an deleted type as an lvalue. + + Currently, this only occurs when trying to assign to an + exception variable outside the local except: blocks. + """ + if typ.source is None: + s = "" + else: + s = " '{}'".format(typ.source) + self.fail('Assignment to variable{} outside except: block'.format(s), context) + + def no_variant_matches_arguments(self, overload: Overloaded, arg_types: List[Type], + context: Context) -> None: + if overload.name(): + self.fail('No overload variant of {} matches argument types {}' + .format(overload.name(), arg_types), context) + else: + self.fail('No overload variant matches argument types {}'.format(arg_types), context) + + def function_variants_overlap(self, n1: int, n2: int, + context: Context) -> None: + self.fail('Function signature variants {} and {} overlap'.format( + n1 + 1, n2 + 1), context) + + def invalid_cast(self, target_type: Type, source_type: Type, + context: Context) -> None: + if not self.check_void(source_type, context): + self.fail('Cannot cast from {} to {}'.format( + self.format(source_type), self.format(target_type)), context) + + def wrong_number_values_to_unpack(self, provided: int, expected: int, + context: Context) -> None: + if provided < expected: + if provided == 1: + self.fail('Need more than 1 value to unpack ({} expected)'.format(expected), + context) + else: + self.fail('Need more than {} values to unpack ({} expected)'.format( + provided, expected), context) + elif provided > expected: + self.fail('Too many values to unpack ({} expected, {} provided)'.format( + expected, provided), context) + + def type_not_iterable(self, type: Type, context: Context) -> None: + self.fail('\'{}\' object is not iterable'.format(type), context) + + def incompatible_operator_assignment(self, op: str, + context: Context) -> None: + self.fail('Result type of {} incompatible in assignment'.format(op), + context) + + def incompatible_value_count_in_assignment(self, lvalue_count: int, + rvalue_count: int, + context: Context) -> None: + if rvalue_count < lvalue_count: + self.fail('Need {} values to assign'.format(lvalue_count), context) + elif rvalue_count > lvalue_count: + self.fail('Too many values to assign', context) + + def type_incompatible_with_supertype(self, name: str, supertype: TypeInfo, + context: Context) -> None: + self.fail('Type of "{}" incompatible with supertype "{}"'.format( + name, supertype.name), context) + + def signature_incompatible_with_supertype( + self, name: str, name_in_super: str, supertype: str, + context: Context) -> None: + target = self.override_target(name, name_in_super, supertype) + self.fail('Signature of "{}" incompatible with {}'.format( + name, target), context) + + def argument_incompatible_with_supertype( + self, arg_num: int, name: str, name_in_supertype: str, + supertype: str, context: Context) -> None: + target = self.override_target(name, name_in_supertype, supertype) + self.fail('Argument {} of "{}" incompatible with {}' + .format(arg_num, name, target), context) + + def return_type_incompatible_with_supertype( + self, name: str, name_in_supertype: str, supertype: str, + context: Context) -> None: + target = self.override_target(name, name_in_supertype, supertype) + self.fail('Return type of "{}" incompatible with {}' + .format(name, target), context) + + def override_target(self, name: str, name_in_super: str, + supertype: str) -> str: + target = 'supertype "{}"'.format(supertype) + if name_in_super != name: + target = '"{}" of {}'.format(name_in_super, target) + return target + + def boolean_return_value_expected(self, method: str, + context: Context) -> None: + self.fail('Boolean return value expected for method "{}"'.format( + method), context) + + def incompatible_type_application(self, expected_arg_count: int, + actual_arg_count: int, + context: Context) -> None: + if expected_arg_count == 0: + self.fail('Type application targets a non-generic function', + context) + elif actual_arg_count > expected_arg_count: + self.fail('Type application has too many types ({} expected)' + .format(expected_arg_count), context) + else: + self.fail('Type application has too few types ({} expected)' + .format(expected_arg_count), context) + + def incompatible_array_item_type(self, typ: Type, index: int, + context: Context) -> None: + self.fail('Array item {} has incompatible type {}'.format( + index, self.format(typ)), context) + + def could_not_infer_type_arguments(self, callee_type: CallableType, n: int, + context: Context) -> None: + if callee_type.name and n > 0: + self.fail('Cannot infer type argument {} of {}'.format( + n, callee_type.name), context) + else: + self.fail('Cannot infer function type argument', context) + + def invalid_var_arg(self, typ: Type, context: Context) -> None: + self.fail('List or tuple expected as variable arguments', context) + + def invalid_keyword_var_arg(self, typ: Type, context: Context) -> None: + if isinstance(typ, Instance) and (typ.type.fullname() == 'builtins.dict'): + self.fail('Keywords must be strings', context) + else: + self.fail('Argument after ** must be a dictionary', + context) + + def incomplete_type_var_match(self, member: str, context: Context) -> None: + self.fail('"{}" has incomplete match to supertype type variable' + .format(member), context) + + def not_implemented(self, msg: str, context: Context) -> Type: + self.fail('Feature not implemented yet ({})'.format(msg), context) + return AnyType() + + def undefined_in_superclass(self, member: str, context: Context) -> None: + self.fail('"{}" undefined in superclass'.format(member), context) + + def check_void(self, typ: Type, context: Context) -> bool: + """If type is void, report an error such as '.. does not + return a value' and return True. Otherwise, return False. + """ + if isinstance(typ, Void): + self.does_not_return_value(typ, context) + return True + else: + return False + + def too_few_string_formatting_arguments(self, context: Context) -> None: + self.fail('Not enough arguments for format string', context) + + def too_many_string_formatting_arguments(self, context: Context) -> None: + self.fail('Not all arguments converted during string formatting', context) + + def incomplete_conversion_specifier_format(self, context: Context) -> None: + self.fail('Incomplete format', context) + + def unsupported_placeholder(self, placeholder: str, context: Context) -> None: + self.fail('Unsupported format character \'%s\'' % placeholder, context) + + def string_interpolation_with_star_and_key(self, context: Context) -> None: + self.fail('String interpolation contains both stars and mapping keys', context) + + def requires_int_or_char(self, context: Context) -> None: + self.fail('%c requires int or char', context) + + def key_not_in_mapping(self, key: str, context: Context) -> None: + self.fail('Key \'%s\' not found in mapping' % key, context) + + def string_interpolation_mixing_key_and_non_keys(self, context: Context) -> None: + self.fail('String interpolation mixes specifier with and without mapping keys', context) + + def cannot_determine_type(self, name: str, context: Context) -> None: + self.fail("Cannot determine type of '%s'" % name, context) + + def cannot_determine_type_in_base(self, name: str, base: str, context: Context) -> None: + self.fail("Cannot determine type of '%s' in base class '%s'" % (name, base), context) + + def invalid_method_type(self, sig: CallableType, context: Context) -> None: + self.fail('Invalid method type', context) + + def invalid_class_method_type(self, sig: CallableType, context: Context) -> None: + self.fail('Invalid class method type', context) + + def incompatible_conditional_function_def(self, defn: FuncDef) -> None: + self.fail('All conditional function variants must have identical ' + 'signatures', defn) + + def cannot_instantiate_abstract_class(self, class_name: str, + abstract_attributes: List[str], + context: Context) -> None: + attrs = format_string_list("'%s'" % a for a in abstract_attributes[:5]) + self.fail("Cannot instantiate abstract class '%s' with abstract " + "attribute%s %s" % (class_name, plural_s(abstract_attributes), + attrs), + context) + + def base_class_definitions_incompatible(self, name: str, base1: TypeInfo, + base2: TypeInfo, + context: Context) -> None: + self.fail('Definition of "{}" in base class "{}" is incompatible ' + 'with definition in base class "{}"'.format( + name, base1.name(), base2.name()), context) + + def cant_assign_to_method(self, context: Context) -> None: + self.fail(CANNOT_ASSIGN_TO_METHOD, context) + + def read_only_property(self, name: str, type: TypeInfo, + context: Context) -> None: + self.fail('Property "{}" defined in "{}" is read-only'.format( + name, type.name()), context) + + def incompatible_typevar_value(self, callee: CallableType, index: int, + type: Type, context: Context) -> None: + self.fail('Type argument {} of {} has incompatible value {}'.format( + index, callable_name(callee), self.format(type)), context) + + def overloaded_signatures_overlap(self, index1: int, index2: int, + context: Context) -> None: + self.fail('Overloaded function signatures {} and {} overlap with ' + 'incompatible return types'.format(index1, index2), context) + + def invalid_reverse_operator_signature(self, reverse: str, other: str, + context: Context) -> None: + self.fail('"Any" return type expected since argument to {} ' + 'does not support {}'.format(reverse, other), context) + + def reverse_operator_method_with_any_arg_must_return_any( + self, method: str, context: Context) -> None: + self.fail('"Any" return type expected since argument to {} ' + 'has type "Any"'.format(method), context) + + def operator_method_signatures_overlap( + self, reverse_class: str, reverse_method: str, forward_class: str, + forward_method: str, context: Context) -> None: + self.fail('Signatures of "{}" of "{}" and "{}" of "{}" ' + 'are unsafely overlapping'.format( + reverse_method, reverse_class, + forward_method, forward_class), + context) + + def forward_operator_not_callable( + self, forward_method: str, context: Context) -> None: + self.fail('Forward operator "{}" is not callable'.format( + forward_method), context) + + def signatures_incompatible(self, method: str, other_method: str, + context: Context) -> None: + self.fail('Signatures of "{}" and "{}" are incompatible'.format( + method, other_method), context) + + def yield_from_invalid_operand_type(self, expr: Type, context: Context) -> Type: + text = self.format(expr) if self.format(expr) != 'object' else expr + self.fail('"yield from" can\'t be applied to {}'.format(text), context) + return AnyType() + + def invalid_signature(self, func_type: Type, context: Context) -> None: + self.fail('Invalid signature "{}"'.format(func_type), context) + + def reveal_type(self, typ: Type, context: Context) -> None: + self.fail('Revealed type is \'{}\''.format(typ), context) + + def unsupported_type_type(self, item: Type, context: Context) -> None: + self.fail('Unsupported type Type[{}]'.format(self.format(item)), context) + + def redundant_cast(self, typ: Type, context: Context) -> None: + self.note('Redundant cast to {}'.format(self.format(typ)), context) + + +def capitalize(s: str) -> str: + """Capitalize the first character of a string.""" + if s == '': + return '' + else: + return s[0].upper() + s[1:] + + +def extract_type(name: str) -> str: + """If the argument is the name of a method (of form C.m), return + the type portion in quotes (e.g. "y"). Otherwise, return the string + unmodified. + """ + name = re.sub('^"[a-zA-Z0-9_]+" of ', '', name) + return name + + +def strip_quotes(s: str) -> str: + """Strip a double quote at the beginning and end of the string, if any.""" + s = re.sub('^"', '', s) + s = re.sub('"$', '', s) + return s + + +def plural_s(s: Sequence[Any]) -> str: + if len(s) > 1: + return 's' + else: + return '' + + +def format_string_list(s: Iterable[str]) -> str: + l = list(s) + assert len(l) > 0 + if len(l) == 1: + return l[0] + else: + return '%s and %s' % (', '.join(l[:-1]), l[-1]) + + +def callable_name(type: CallableType) -> str: + if type.name: + return type.name + else: + return 'function' + + +def temp_message_builder() -> MessageBuilder: + """Return a message builder usable for throwaway errors (which may not format properly).""" + return MessageBuilder(Errors(), {}) + + +# For hard-coding suggested missing member alternatives. +COMMON_MISTAKES = { + 'add': ('append', 'extend'), +} # type: Dict[str, Sequence[str]] + + +def best_matches(current: str, options: Iterable[str]) -> List[str]: + ratios = {v: difflib.SequenceMatcher(a=current, b=v).ratio() for v in options} + return sorted((o for o in options if ratios[o] > 0.75), + reverse=True, key=lambda v: (ratios[v], v)) + + +def pretty_or(args: List[str]) -> str: + quoted = ['"' + a + '"' for a in args] + if len(quoted) == 1: + return quoted[0] + if len(quoted) == 2: + return "{} or {}".format(quoted[0], quoted[1]) + return ", ".join(quoted[:-1]) + ", or " + quoted[-1] diff --git a/mypy/nodes.py b/mypy/nodes.py index 60d7b5d18944f..02f86b1537b93 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1,2242 +1,2242 @@ -"""Abstract syntax tree node classes (i.e. parse tree).""" - -import os -import re -from abc import abstractmethod, ABCMeta - -from typing import ( - Any, TypeVar, List, Tuple, cast, Set, Dict, Union, Optional -) - -from mypy.lex import Token -import mypy.strconv -from mypy.visitor import NodeVisitor -from mypy.util import dump_tagged, short_type - - -class Context: - """Base type for objects that are valid as error message locations.""" - @abstractmethod - def get_line(self) -> int: pass - - -if False: - # break import cycle only needed for mypy - import mypy.types - - -T = TypeVar('T') - -JsonDict = Dict[str, Any] - - -# Symbol table node kinds -# -# TODO rename to use more descriptive names - -LDEF = 0 # type: int -GDEF = 1 # type: int -MDEF = 2 # type: int -MODULE_REF = 3 # type: int -# Type variable declared using TypeVar(...) has kind UNBOUND_TVAR. It's not -# valid as a type. A type variable is valid as a type (kind BOUND_TVAR) within -# (1) a generic class that uses the type variable as a type argument or -# (2) a generic function that refers to the type variable in its signature. -UNBOUND_TVAR = 4 # type: int -BOUND_TVAR = 5 # type: int -TYPE_ALIAS = 6 # type: int -# Placeholder for a name imported via 'from ... import'. Second phase of -# semantic will replace this the actual imported reference. This is -# needed so that we can detect whether a name has been imported during -# XXX what? -UNBOUND_IMPORTED = 7 # type: int - - -LITERAL_YES = 2 -LITERAL_TYPE = 1 -LITERAL_NO = 0 - -# Hard coded name of Enum baseclass. -ENUM_BASECLASS = "enum.Enum" - -node_kinds = { - LDEF: 'Ldef', - GDEF: 'Gdef', - MDEF: 'Mdef', - MODULE_REF: 'ModuleRef', - UNBOUND_TVAR: 'UnboundTvar', - BOUND_TVAR: 'Tvar', - TYPE_ALIAS: 'TypeAlias', - UNBOUND_IMPORTED: 'UnboundImported', -} -inverse_node_kinds = {_kind: _name for _name, _kind in node_kinds.items()} - - -implicit_module_attrs = {'__name__': '__builtins__.str', - '__doc__': '__builtins__.str', - '__file__': '__builtins__.str', - '__package__': '__builtins__.str'} - - -type_aliases = { - 'typing.List': '__builtins__.list', - 'typing.Dict': '__builtins__.dict', - 'typing.Set': '__builtins__.set', -} - -reverse_type_aliases = dict((name.replace('__builtins__', 'builtins'), alias) - for alias, name in type_aliases.items()) # type: Dict[str, str] - - -class Node(Context): - """Common base class for all non-type parse tree nodes.""" - - line = -1 - - literal = LITERAL_NO - literal_hash = None # type: Any - - def __str__(self) -> str: - ans = self.accept(mypy.strconv.StrConv()) - if ans is None: - return repr(self) - return ans - - def set_line(self, target: Union[Token, 'Node', int]) -> 'Node': - if isinstance(target, int): - self.line = target - else: - self.line = target.line - return self - - def get_line(self) -> int: - # TODO this should be just 'line' - return self.line - - def accept(self, visitor: NodeVisitor[T]) -> T: - raise RuntimeError('Not implemented') - - -# These are placeholders for a future refactoring; see #1783. -# For now they serve as (unchecked) documentation of what various -# fields of Node subtypes are expected to contain. -Statement = Node -Expression = Node - - -class SymbolNode(Node): - # Nodes that can be stored in a symbol table. - - # TODO do not use methods for these - - @abstractmethod - def name(self) -> str: pass - - @abstractmethod - def fullname(self) -> str: pass - - # NOTE: Can't use @abstractmethod, since many subclasses of Node - # don't implement serialize(). - def serialize(self) -> Any: - raise NotImplementedError('Cannot serialize {} instance'.format(self.__class__.__name__)) - - @classmethod - def deserialize(cls, data: JsonDict) -> 'SymbolNode': - classname = data['.class'] - glo = globals() - if classname in glo: - cl = glo[classname] - if issubclass(cl, cls) and 'deserialize' in cl.__dict__: - return cl.deserialize(data) - raise NotImplementedError('unexpected .class {}'.format(classname)) - - -class MypyFile(SymbolNode, Statement): - """The abstract syntax tree of a single source file.""" - - # Module name ('__main__' for initial file) - _name = None # type: str - # Fully qualified module name - _fullname = None # type: str - # Path to the file (None if not known) - path = '' - # Top-level definitions and statements - defs = None # type: List[Statement] - # Is there a UTF-8 BOM at the start? - is_bom = False - names = None # type: SymbolTable - # All import nodes within the file (also ones within functions etc.) - imports = None # type: List[ImportBase] - # Lines to ignore when checking - ignored_lines = None # type: Set[int] - # Is this file represented by a stub file (.pyi)? - is_stub = False - # Do weak typing globally in the file? - weak_opts = None # type: Set[str] - - def __init__(self, - defs: List[Statement], - imports: List['ImportBase'], - is_bom: bool = False, - ignored_lines: Set[int] = None, - weak_opts: Set[str] = None) -> None: - self.defs = defs - self.line = 1 # Dummy line number - self.imports = imports - self.is_bom = is_bom - self.weak_opts = weak_opts - if ignored_lines: - self.ignored_lines = ignored_lines - else: - self.ignored_lines = set() - - def name(self) -> str: - return self._name - - def fullname(self) -> str: - return self._fullname - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_mypy_file(self) - - def is_package_init_file(self) -> bool: - return not (self.path is None) and len(self.path) != 0 \ - and os.path.basename(self.path).startswith('__init__.') - - def serialize(self) -> JsonDict: - return {'.class': 'MypyFile', - '_name': self._name, - '_fullname': self._fullname, - 'names': self.names.serialize(self._fullname), - 'is_stub': self.is_stub, - 'path': self.path, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'MypyFile': - assert data['.class'] == 'MypyFile', data - tree = MypyFile([], []) - tree._name = data['_name'] - tree._fullname = data['_fullname'] - tree.names = SymbolTable.deserialize(data['names']) - tree.is_stub = data['is_stub'] - tree.path = data['path'] - return tree - - -class ImportBase(Statement): - """Base class for all import statements.""" - is_unreachable = False - is_top_level = False # Set by semanal.FirstPass - # If an import replaces existing definitions, we construct dummy assignment - # statements that assign the imported names to the names in the current scope, - # for type checking purposes. Example: - # - # x = 1 - # from m import x <-- add assignment representing "x = m.x" - assignments = None # type: List[AssignmentStmt] - - def __init__(self) -> None: - self.assignments = [] - - -class Import(ImportBase): - """import m [as n]""" - - ids = None # type: List[Tuple[str, Optional[str]]] # (module id, as id) - - def __init__(self, ids: List[Tuple[str, Optional[str]]]) -> None: - super().__init__() - self.ids = ids - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_import(self) - - -class ImportFrom(ImportBase): - """from m import x [as y], ...""" - - names = None # type: List[Tuple[str, Optional[str]]] # Tuples (name, as name) - - def __init__(self, id: str, relative: int, names: List[Tuple[str, Optional[str]]]) -> None: - super().__init__() - self.id = id - self.names = names - self.relative = relative - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_import_from(self) - - -class ImportAll(ImportBase): - """from m import *""" - - def __init__(self, id: str, relative: int) -> None: - super().__init__() - self.id = id - self.relative = relative - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_import_all(self) - - -class FuncBase(SymbolNode): - """Abstract base class for function-like nodes""" - - # Type signature. This is usually CallableType or Overloaded, but it can be something else for - # decorated functions/ - type = None # type: mypy.types.Type - # If method, reference to TypeInfo - info = None # type: TypeInfo - is_property = False - _fullname = None # type: str # Name with module prefix - - @abstractmethod - def name(self) -> str: pass - - def fullname(self) -> str: - return self._fullname - - def is_method(self) -> bool: - return bool(self.info) - - -class OverloadedFuncDef(FuncBase, Statement): - """A logical node representing all the variants of an overloaded function. - - This node has no explicit representation in the source program. - Overloaded variants must be consecutive in the source file. - """ - - items = None # type: List[Decorator] - - def __init__(self, items: List['Decorator']) -> None: - self.items = items - self.set_line(items[0].line) - - def name(self) -> str: - return self.items[0].func.name() - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_overloaded_func_def(self) - - def serialize(self) -> JsonDict: - return {'.class': 'OverloadedFuncDef', - 'items': [i.serialize() for i in self.items], - 'type': None if self.type is None else self.type.serialize(), - 'fullname': self._fullname, - 'is_property': self.is_property, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'OverloadedFuncDef': - assert data['.class'] == 'OverloadedFuncDef' - res = OverloadedFuncDef([Decorator.deserialize(d) for d in data['items']]) - if data.get('type') is not None: - res.type = mypy.types.Type.deserialize(data['type']) - res._fullname = data['fullname'] - res.is_property = data['is_property'] - # NOTE: res.info will be set in the fixup phase. - return res - - -class Argument(Node): - """A single argument in a FuncItem.""" - - def __init__(self, variable: 'Var', type_annotation: 'Optional[mypy.types.Type]', - initializer: Optional[Expression], kind: int, - initialization_statement: Optional['AssignmentStmt'] = None) -> None: - self.variable = variable - - self.type_annotation = type_annotation - self.initializer = initializer - - self.initialization_statement = initialization_statement - if not self.initialization_statement: - self.initialization_statement = self._initialization_statement() - - self.kind = kind - - def _initialization_statement(self) -> Optional['AssignmentStmt']: - """Convert the initializer into an assignment statement. - """ - if not self.initializer: - return None - - rvalue = self.initializer - lvalue = NameExpr(self.variable.name()) - assign = AssignmentStmt([lvalue], rvalue) - return assign - - def set_line(self, target: Union[Token, Node, int]) -> Node: - super().set_line(target) - - if self.initializer: - self.initializer.set_line(self.line) - - self.variable.set_line(self.line) - - if self.initialization_statement: - self.initialization_statement.set_line(self.line) - self.initialization_statement.lvalues[0].set_line(self.line) - - def serialize(self) -> JsonDict: - data = {'.class': 'Argument', - 'kind': self.kind, - 'variable': self.variable.serialize(), - 'type_annotation': (None if self.type_annotation is None - else self.type_annotation.serialize()), - } # type: JsonDict - # TODO: initializer? - return data - - @classmethod - def deserialize(cls, data: JsonDict) -> 'Argument': - assert data['.class'] == 'Argument' - return Argument(Var.deserialize(data['variable']), - (None if data.get('type_annotation') is None - else mypy.types.Type.deserialize(data['type_annotation'])), - None, # TODO: initializer? - kind=data['kind']) - - -class FuncItem(FuncBase): - arguments = [] # type: List[Argument] - # Minimum number of arguments - min_args = 0 - # Maximum number of positional arguments, -1 if no explicit limit (*args not included) - max_pos = 0 - body = None # type: Block - # Is this an overload variant of function with more than one overload variant? - is_overload = False - is_generator = False # Contains a yield statement? - is_static = False # Uses @staticmethod? - is_class = False # Uses @classmethod? - # Variants of function with type variables with values expanded - expanded = None # type: List[FuncItem] - - def __init__(self, arguments: List[Argument], body: 'Block', - typ: 'mypy.types.FunctionLike' = None) -> None: - self.arguments = arguments - arg_kinds = [arg.kind for arg in self.arguments] - self.max_pos = arg_kinds.count(ARG_POS) + arg_kinds.count(ARG_OPT) - self.body = body - self.type = typ - self.expanded = [] - - self.min_args = 0 - for i in range(len(self.arguments)): - if self.arguments[i] is None and i < self.max_fixed_argc(): - self.min_args = i + 1 - - def max_fixed_argc(self) -> int: - return self.max_pos - - def set_line(self, target: Union[Token, Node, int]) -> Node: - super().set_line(target) - for arg in self.arguments: - arg.set_line(self.line) - return self - - def is_dynamic(self): - return self.type is None - - -class FuncDef(FuncItem, Statement): - """Function definition. - - This is a non-lambda function defined using 'def'. - """ - - is_decorated = False - is_conditional = False # Defined conditionally (within block)? - is_abstract = False - is_property = False - original_def = None # type: Union[None, FuncDef, Var] # Original conditional definition - - def __init__(self, - name: str, # Function name - arguments: List[Argument], - body: 'Block', - typ: 'mypy.types.FunctionLike' = None) -> None: - super().__init__(arguments, body, typ) - self._name = name - - def name(self) -> str: - return self._name - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_func_def(self) - - def is_constructor(self) -> bool: - return self.info is not None and self._name == '__init__' - - def serialize(self) -> JsonDict: - return {'.class': 'FuncDef', - 'name': self._name, - 'fullname': self._fullname, - 'arguments': [a.serialize() for a in self.arguments], - 'type': None if self.type is None else self.type.serialize(), - 'is_property': self.is_property, - 'is_overload': self.is_overload, - 'is_generator': self.is_generator, - 'is_static': self.is_static, - 'is_class': self.is_class, - 'is_decorated': self.is_decorated, - 'is_conditional': self.is_conditional, - 'is_abstract': self.is_abstract, - # TODO: Do we need expanded, original_def? - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'FuncDef': - assert data['.class'] == 'FuncDef' - body = Block([]) - ret = FuncDef(data['name'], - [Argument.deserialize(a) for a in data['arguments']], - body, - (None if data['type'] is None - else mypy.types.FunctionLike.deserialize(data['type']))) - ret._fullname = data['fullname'] - ret.is_property = data['is_property'] - ret.is_overload = data['is_overload'] - ret.is_generator = data['is_generator'] - ret.is_static = data['is_static'] - ret.is_class = data['is_class'] - ret.is_decorated = data['is_decorated'] - ret.is_conditional = data['is_conditional'] - ret.is_abstract = data['is_abstract'] - # NOTE: ret.info is set in the fixup phase. - return ret - - -class Decorator(SymbolNode, Statement): - """A decorated function. - - A single Decorator object can include any number of function decorators. - """ - - func = None # type: FuncDef # Decorated function - decorators = None # type: List[Expression] # Decorators, at least one # XXX Not true - var = None # type: Var # Represents the decorated function obj - is_overload = False - - def __init__(self, func: FuncDef, decorators: List[Expression], - var: 'Var') -> None: - self.func = func - self.decorators = decorators - self.var = var - self.is_overload = False - - def name(self) -> str: - return self.func.name() - - def fullname(self) -> str: - return self.func.fullname() - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_decorator(self) - - def serialize(self) -> JsonDict: - return {'.class': 'Decorator', - 'func': self.func.serialize(), - 'var': self.var.serialize(), - 'is_overload': self.is_overload, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'Decorator': - assert data['.class'] == 'Decorator' - dec = Decorator(FuncDef.deserialize(data['func']), - [], - Var.deserialize(data['var'])) - dec.is_overload = data['is_overload'] - return dec - - -class Var(SymbolNode, Statement): - """A variable. - - It can refer to global/local variable or a data attribute. - """ - - _name = None # type: str # Name without module prefix - _fullname = None # type: str # Name with module prefix - info = None # type: TypeInfo # Defining class (for member variables) - type = None # type: mypy.types.Type # Declared or inferred type, or None - # Is this the first argument to an ordinary method (usually "self")? - is_self = False - is_ready = False # If inferred, is the inferred type available? - # Is this initialized explicitly to a non-None value in class body? - is_initialized_in_class = False - is_staticmethod = False - is_classmethod = False - is_property = False - is_settable_property = False - - def __init__(self, name: str, type: 'mypy.types.Type' = None) -> None: - self._name = name - self.type = type - self.is_self = False - self.is_ready = True - self.is_initialized_in_class = False - - def name(self) -> str: - return self._name - - def fullname(self) -> str: - return self._fullname - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_var(self) - - def serialize(self) -> JsonDict: - # TODO: Leave default values out? - # NOTE: Sometimes self.is_ready is False here, but we don't care. - data = {'.class': 'Var', - 'name': self._name, - 'fullname': self._fullname, - 'type': None if self.type is None else self.type.serialize(), - 'is_self': self.is_self, - 'is_initialized_in_class': self.is_initialized_in_class, - 'is_staticmethod': self.is_staticmethod, - 'is_classmethod': self.is_classmethod, - 'is_property': self.is_property, - 'is_settable_property': self.is_settable_property, - } # type: JsonDict - return data - - @classmethod - def deserialize(cls, data: JsonDict) -> 'Var': - assert data['.class'] == 'Var' - name = data['name'] - type = None if data['type'] is None else mypy.types.Type.deserialize(data['type']) - v = Var(name, type) - v._fullname = data['fullname'] - v.is_self = data['is_self'] - v.is_initialized_in_class = data['is_initialized_in_class'] - v.is_staticmethod = data['is_staticmethod'] - v.is_classmethod = data['is_classmethod'] - v.is_property = data['is_property'] - v.is_settable_property = data['is_settable_property'] - return v - - -class ClassDef(Statement): - """Class definition""" - - name = None # type: str # Name of the class without module prefix - fullname = None # type: str # Fully qualified name of the class - defs = None # type: Block - type_vars = None # type: List[mypy.types.TypeVarDef] - # Base class expressions (not semantically analyzed -- can be arbitrary expressions) - base_type_exprs = None # type: List[Expression] - info = None # type: TypeInfo # Related TypeInfo - metaclass = '' - decorators = None # type: List[Expression] - # Built-in/extension class? (single implementation inheritance only) - is_builtinclass = False - - def __init__(self, - name: str, - defs: 'Block', - type_vars: List['mypy.types.TypeVarDef'] = None, - base_type_exprs: List[Expression] = None, - metaclass: str = None) -> None: - self.name = name - self.defs = defs - self.type_vars = type_vars or [] - self.base_type_exprs = base_type_exprs or [] - self.metaclass = metaclass - self.decorators = [] - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_class_def(self) - - def is_generic(self) -> bool: - return self.info.is_generic() - - def serialize(self) -> JsonDict: - # Not serialized: defs, base_type_exprs, decorators - return {'.class': 'ClassDef', - 'name': self.name, - 'fullname': self.fullname, - 'type_vars': [v.serialize() for v in self.type_vars], - 'metaclass': self.metaclass, - 'is_builtinclass': self.is_builtinclass, - } - - @classmethod - def deserialize(self, data: JsonDict) -> 'ClassDef': - assert data['.class'] == 'ClassDef' - res = ClassDef(data['name'], - Block([]), - [mypy.types.TypeVarDef.deserialize(v) for v in data['type_vars']], - metaclass=data['metaclass'], - ) - res.fullname = data['fullname'] - res.is_builtinclass = data['is_builtinclass'] - return res - - -class GlobalDecl(Statement): - """Declaration global x, y, ...""" - - names = None # type: List[str] - - def __init__(self, names: List[str]) -> None: - self.names = names - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_global_decl(self) - - -class NonlocalDecl(Statement): - """Declaration nonlocal x, y, ...""" - - names = None # type: List[str] - - def __init__(self, names: List[str]) -> None: - self.names = names - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_nonlocal_decl(self) - - -class Block(Statement): - body = None # type: List[Statement] - # True if we can determine that this block is not executed. For example, - # this applies to blocks that are protected by something like "if PY3:" - # when using Python 2. - is_unreachable = False - - def __init__(self, body: List[Statement]) -> None: - self.body = body - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_block(self) - - -# Statements - - -class ExpressionStmt(Statement): - """An expression as a statement, such as print(s).""" - expr = None # type: Expression - - def __init__(self, expr: Expression) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_expression_stmt(self) - - -class AssignmentStmt(Statement): - """Assignment statement - The same node class is used for single assignment, multiple assignment - (e.g. x, y = z) and chained assignment (e.g. x = y = z), assignments - that define new names, and assignments with explicit types (# type). - - An lvalue can be NameExpr, TupleExpr, ListExpr, MemberExpr, IndexExpr. - """ - - lvalues = None # type: List[Expression] - rvalue = None # type: Expression - # Declared type in a comment, may be None. - type = None # type: mypy.types.Type - - def __init__(self, lvalues: List[Expression], rvalue: Expression, - type: 'mypy.types.Type' = None) -> None: - self.lvalues = lvalues - self.rvalue = rvalue - self.type = type - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_assignment_stmt(self) - - -class OperatorAssignmentStmt(Statement): - """Operator assignment statement such as x += 1""" - - op = '' - lvalue = None # type: Expression - rvalue = None # type: Expression - - def __init__(self, op: str, lvalue: Expression, rvalue: Expression) -> None: - self.op = op - self.lvalue = lvalue - self.rvalue = rvalue - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_operator_assignment_stmt(self) - - -class WhileStmt(Statement): - expr = None # type: Expression - body = None # type: Block - else_body = None # type: Block - - def __init__(self, expr: Expression, body: Block, else_body: Block) -> None: - self.expr = expr - self.body = body - self.else_body = else_body - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_while_stmt(self) - - -class ForStmt(Statement): - # Index variables - index = None # type: Expression - # Expression to iterate - expr = None # type: Expression - body = None # type: Block - else_body = None # type: Block - - def __init__(self, index: Expression, expr: Expression, body: Block, - else_body: Block) -> None: - self.index = index - self.expr = expr - self.body = body - self.else_body = else_body - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_for_stmt(self) - - -class ReturnStmt(Statement): - expr = None # type: Optional[Expression] - - def __init__(self, expr: Optional[Expression]) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_return_stmt(self) - - -class AssertStmt(Statement): - expr = None # type: Expression - - def __init__(self, expr: Expression) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_assert_stmt(self) - - -class DelStmt(Statement): - expr = None # type: Expression - - def __init__(self, expr: Expression) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_del_stmt(self) - - -class BreakStmt(Statement): - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_break_stmt(self) - - -class ContinueStmt(Statement): - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_continue_stmt(self) - - -class PassStmt(Statement): - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_pass_stmt(self) - - -class IfStmt(Statement): - expr = None # type: List[Expression] - body = None # type: List[Block] - else_body = None # type: Block - - def __init__(self, expr: List[Expression], body: List[Block], - else_body: Block) -> None: - self.expr = expr - self.body = body - self.else_body = else_body - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_if_stmt(self) - - -class RaiseStmt(Statement): - expr = None # type: Expression - from_expr = None # type: Expression - - def __init__(self, expr: Expression, from_expr: Expression = None) -> None: - self.expr = expr - self.from_expr = from_expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_raise_stmt(self) - - -class TryStmt(Statement): - body = None # type: Block # Try body - types = None # type: List[Expression] # Except type expressions - vars = None # type: List[NameExpr] # Except variable names - handlers = None # type: List[Block] # Except bodies - else_body = None # type: Block - finally_body = None # type: Block - - def __init__(self, body: Block, vars: List['NameExpr'], types: List[Expression], - handlers: List[Block], else_body: Block, - finally_body: Block) -> None: - self.body = body - self.vars = vars - self.types = types - self.handlers = handlers - self.else_body = else_body - self.finally_body = finally_body - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_try_stmt(self) - - -class WithStmt(Statement): - expr = None # type: List[Expression] - target = None # type: List[Expression] - body = None # type: Block - - def __init__(self, expr: List[Expression], target: List[Expression], - body: Block) -> None: - self.expr = expr - self.target = target - self.body = body - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_with_stmt(self) - - -class PrintStmt(Statement): - """Python 2 print statement""" - - args = None # type: List[Expression] - newline = False - # The file-like target object (given using >>). - target = None # type: Optional[Expression] - - def __init__(self, args: List[Expression], newline: bool, target: Expression = None) -> None: - self.args = args - self.newline = newline - self.target = target - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_print_stmt(self) - - -class ExecStmt(Statement): - """Python 2 exec statement""" - - expr = None # type: Expression - variables1 = None # type: Optional[Expression] - variables2 = None # type: Optional[Expression] - - def __init__(self, expr: Expression, - variables1: Optional[Expression], - variables2: Optional[Expression]) -> None: - self.expr = expr - self.variables1 = variables1 - self.variables2 = variables2 - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_exec_stmt(self) - - -# Expressions - - -class IntExpr(Expression): - """Integer literal""" - - value = 0 - literal = LITERAL_YES - - def __init__(self, value: int) -> None: - self.value = value - self.literal_hash = value - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_int_expr(self) - - -# How mypy uses StrExpr, BytesExpr, and UnicodeExpr: -# In Python 2 mode: -# b'x', 'x' -> StrExpr -# u'x' -> UnicodeExpr -# BytesExpr is unused -# -# In Python 3 mode: -# b'x' -> BytesExpr -# 'x', u'x' -> StrExpr -# UnicodeExpr is unused - -class StrExpr(Expression): - """String literal""" - - value = '' - literal = LITERAL_YES - - def __init__(self, value: str) -> None: - self.value = value - self.literal_hash = value - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_str_expr(self) - - -class BytesExpr(Expression): - """Bytes literal""" - - value = '' # TODO use bytes - literal = LITERAL_YES - - def __init__(self, value: str) -> None: - self.value = value - self.literal_hash = value - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_bytes_expr(self) - - -class UnicodeExpr(Expression): - """Unicode literal (Python 2.x)""" - - value = '' # TODO use bytes - literal = LITERAL_YES - - def __init__(self, value: str) -> None: - self.value = value - self.literal_hash = value - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_unicode_expr(self) - - -class FloatExpr(Expression): - """Float literal""" - - value = 0.0 - literal = LITERAL_YES - - def __init__(self, value: float) -> None: - self.value = value - self.literal_hash = value - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_float_expr(self) - - -class ComplexExpr(Expression): - """Complex literal""" - - value = 0.0j - literal = LITERAL_YES - - def __init__(self, value: complex) -> None: - self.value = value - self.literal_hash = value - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_complex_expr(self) - - -class EllipsisExpr(Expression): - """Ellipsis (...)""" - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_ellipsis(self) - - -class StarExpr(Expression): - """Star expression""" - - expr = None # type: Expression - - def __init__(self, expr: Expression) -> None: - self.expr = expr - self.literal = self.expr.literal - self.literal_hash = ('Star', expr.literal_hash,) - - # Whether this starred expression is used in a tuple/list and as lvalue - self.valid = False - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_star_expr(self) - - -class RefExpr(Expression): - """Abstract base class for name-like constructs""" - - kind = None # type: int # LDEF/GDEF/MDEF/... (None if not available) - node = None # type: SymbolNode # Var, FuncDef or TypeInfo that describes this - fullname = None # type: str # Fully qualified name (or name if not global) - - # Does this define a new name with inferred type? - # - # For members, after semantic analysis, this does not take base - # classes into consideration at all; the type checker deals with these. - is_def = False - - -class NameExpr(RefExpr): - """Name expression - - This refers to a local name, global name or a module. - """ - - name = None # type: str # Name referred to (may be qualified) - # TypeInfo of class surrounding expression (may be None) - info = None # type: TypeInfo - - literal = LITERAL_TYPE - - def __init__(self, name: str) -> None: - self.name = name - self.literal_hash = ('Var', name,) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_name_expr(self) - - def serialize(self) -> JsonDict: - # TODO: Find out where and why NameExpr is being serialized (if at all). - assert False, "Serializing NameExpr: %s" % (self,) - return {'.class': 'NameExpr', - 'kind': self.kind, - 'node': None if self.node is None else self.node.serialize(), - 'fullname': self.fullname, - 'is_def': self.is_def, - 'name': self.name, - 'literal': self.literal, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'NameExpr': - assert data['.class'] == 'NameExpr' - ret = NameExpr(data['name']) - ret.kind = data['kind'] - ret.node = None if data['node'] is None else SymbolNode.deserialize(data['node']) - ret.fullname = data['fullname'] - ret.is_def = data['is_def'] - ret.literal = data['literal'] - return ret - - -class MemberExpr(RefExpr): - """Member access expression x.y""" - - expr = None # type: Expression - name = None # type: str - # The variable node related to a definition. - def_var = None # type: Var - - def __init__(self, expr: Expression, name: str) -> None: - self.expr = expr - self.name = name - self.literal = self.expr.literal - self.literal_hash = ('Member', expr.literal_hash, name) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_member_expr(self) - - -# Kinds of arguments - -# Positional argument -ARG_POS = 0 # type: int -# Positional, optional argument (functions only, not calls) -ARG_OPT = 1 # type: int -# *arg argument -ARG_STAR = 2 # type: int -# Keyword argument x=y in call, or keyword-only function arg -ARG_NAMED = 3 # type: int -# **arg argument -ARG_STAR2 = 4 # type: int - - -class CallExpr(Expression): - """Call expression. - - This can also represent several special forms that are syntactically calls - such as cast(...) and None # type: .... - """ - - callee = None # type: Expression - args = None # type: List[Expression] - arg_kinds = None # type: List[int] # ARG_ constants - # Each name can be None if not a keyword argument. - arg_names = None # type: List[str] - # If not None, the node that represents the meaning of the CallExpr. For - # cast(...) this is a CastExpr. - analyzed = None # type: Optional[Expression] - - def __init__(self, callee: Expression, args: List[Expression], arg_kinds: List[int], - arg_names: List[str] = None, analyzed: Expression = None) -> None: - if not arg_names: - arg_names = [None] * len(args) - - self.callee = callee - self.args = args - self.arg_kinds = arg_kinds - self.arg_names = arg_names - self.analyzed = analyzed - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_call_expr(self) - - -class YieldFromExpr(Expression): - expr = None # type: Expression - - def __init__(self, expr: Expression) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_yield_from_expr(self) - - -class YieldExpr(Expression): - expr = None # type: Optional[Expression] - - def __init__(self, expr: Optional[Expression]) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_yield_expr(self) - - -class IndexExpr(Expression): - """Index expression x[y]. - - Also wraps type application such as List[int] as a special form. - """ - - base = None # type: Expression - index = None # type: Expression - # Inferred __getitem__ method type - method_type = None # type: mypy.types.Type - # If not None, this is actually semantically a type application - # Class[type, ...] or a type alias initializer. - analyzed = None # type: Union[TypeApplication, TypeAliasExpr] - - def __init__(self, base: Expression, index: Expression) -> None: - self.base = base - self.index = index - self.analyzed = None - if self.index.literal == LITERAL_YES: - self.literal = self.base.literal - self.literal_hash = ('Member', base.literal_hash, - index.literal_hash) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_index_expr(self) - - -class UnaryExpr(Expression): - """Unary operation""" - - op = '' - expr = None # type: Expression - # Inferred operator method type - method_type = None # type: mypy.types.Type - - def __init__(self, op: str, expr: Expression) -> None: - self.op = op - self.expr = expr - self.literal = self.expr.literal - self.literal_hash = ('Unary', op, expr.literal_hash) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_unary_expr(self) - - -# Map from binary operator id to related method name (in Python 3). -op_methods = { - '+': '__add__', - '-': '__sub__', - '*': '__mul__', - '/': '__truediv__', - '%': '__mod__', - '//': '__floordiv__', - '**': '__pow__', - '&': '__and__', - '|': '__or__', - '^': '__xor__', - '<<': '__lshift__', - '>>': '__rshift__', - '==': '__eq__', - '!=': '__ne__', - '<': '__lt__', - '>=': '__ge__', - '>': '__gt__', - '<=': '__le__', - 'in': '__contains__', -} # type: Dict[str, str] - -ops_with_inplace_method = { - '+', '-', '*', '/', '%', '//', '**', '&', '|', '^', '<<', '>>'} - -inplace_operator_methods = set( - '__i' + op_methods[op][2:] for op in ops_with_inplace_method) - -reverse_op_methods = { - '__add__': '__radd__', - '__sub__': '__rsub__', - '__mul__': '__rmul__', - '__truediv__': '__rtruediv__', - '__mod__': '__rmod__', - '__floordiv__': '__rfloordiv__', - '__pow__': '__rpow__', - '__and__': '__rand__', - '__or__': '__ror__', - '__xor__': '__rxor__', - '__lshift__': '__rlshift__', - '__rshift__': '__rrshift__', - '__eq__': '__eq__', - '__ne__': '__ne__', - '__lt__': '__gt__', - '__ge__': '__le__', - '__gt__': '__lt__', - '__le__': '__ge__', -} - -normal_from_reverse_op = dict((m, n) for n, m in reverse_op_methods.items()) -reverse_op_method_set = set(reverse_op_methods.values()) - - -class OpExpr(Expression): - """Binary operation (other than . or [] or comparison operators, - which have specific nodes).""" - - op = '' - left = None # type: Expression - right = None # type: Expression - # Inferred type for the operator method type (when relevant). - method_type = None # type: mypy.types.Type - - def __init__(self, op: str, left: Expression, right: Expression) -> None: - self.op = op - self.left = left - self.right = right - self.literal = min(self.left.literal, self.right.literal) - self.literal_hash = ('Binary', op, left.literal_hash, right.literal_hash) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_op_expr(self) - - -class ComparisonExpr(Expression): - """Comparison expression (e.g. a < b > c < d).""" - - operators = None # type: List[str] - operands = None # type: List[Expression] - # Inferred type for the operator methods (when relevant; None for 'is'). - method_types = None # type: List[mypy.types.Type] - - def __init__(self, operators: List[str], operands: List[Expression]) -> None: - self.operators = operators - self.operands = operands - self.method_types = [] - self.literal = min(o.literal for o in self.operands) - self.literal_hash = (('Comparison',) + tuple(operators) + - tuple(o.literal_hash for o in operands)) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_comparison_expr(self) - - -class SliceExpr(Expression): - """Slice expression (e.g. 'x:y', 'x:', '::2' or ':'). - - This is only valid as index in index expressions. - """ - - begin_index = None # type: Optional[Expression] - end_index = None # type: Optional[Expression] - stride = None # type: Optional[Expression] - - def __init__(self, begin_index: Optional[Expression], - end_index: Optional[Expression], - stride: Optional[Expression]) -> None: - self.begin_index = begin_index - self.end_index = end_index - self.stride = stride - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_slice_expr(self) - - -class CastExpr(Expression): - """Cast expression cast(type, expr).""" - - expr = None # type: Expression - type = None # type: mypy.types.Type - - def __init__(self, expr: Expression, typ: 'mypy.types.Type') -> None: - self.expr = expr - self.type = typ - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_cast_expr(self) - - -class RevealTypeExpr(Expression): - """Reveal type expression reveal_type(expr).""" - - expr = None # type: Expression - - def __init__(self, expr: Expression) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_reveal_type_expr(self) - - -class SuperExpr(Expression): - """Expression super().name""" - - name = '' - info = None # type: TypeInfo # Type that contains this super expression - - def __init__(self, name: str) -> None: - self.name = name - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_super_expr(self) - - -class FuncExpr(FuncItem, Expression): - """Lambda expression""" - - def name(self) -> str: - return '' - - def expr(self) -> Expression: - """Return the expression (the body) of the lambda.""" - ret = cast(ReturnStmt, self.body.body[-1]) - return ret.expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_func_expr(self) - - -class ListExpr(Expression): - """List literal expression [...].""" - - items = None # type: List[Expression] - - def __init__(self, items: List[Expression]) -> None: - self.items = items - if all(x.literal == LITERAL_YES for x in items): - self.literal = LITERAL_YES - self.literal_hash = ('List',) + tuple(x.literal_hash for x in items) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_list_expr(self) - - -class DictExpr(Expression): - """Dictionary literal expression {key: value, ...}.""" - - items = None # type: List[Tuple[Expression, Expression]] - - def __init__(self, items: List[Tuple[Expression, Expression]]) -> None: - self.items = items - if all(x[0].literal == LITERAL_YES and x[1].literal == LITERAL_YES - for x in items): - self.literal = LITERAL_YES - self.literal_hash = ('Dict',) + tuple( - (x[0].literal_hash, x[1].literal_hash) for x in items) # type: ignore - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_dict_expr(self) - - -class TupleExpr(Expression): - """Tuple literal expression (..., ...)""" - - items = None # type: List[Expression] - - def __init__(self, items: List[Expression]) -> None: - self.items = items - if all(x.literal == LITERAL_YES for x in items): - self.literal = LITERAL_YES - self.literal_hash = ('Tuple',) + tuple(x.literal_hash for x in items) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_tuple_expr(self) - - -class SetExpr(Expression): - """Set literal expression {value, ...}.""" - - items = None # type: List[Expression] - - def __init__(self, items: List[Expression]) -> None: - self.items = items - if all(x.literal == LITERAL_YES for x in items): - self.literal = LITERAL_YES - self.literal_hash = ('Set',) + tuple(x.literal_hash for x in items) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_set_expr(self) - - -class GeneratorExpr(Expression): - """Generator expression ... for ... in ... [ for ... in ... ] [ if ... ].""" - - left_expr = None # type: Expression - sequences_expr = None # type: List[Expression] - condlists = None # type: List[List[Expression]] - indices = None # type: List[Expression] - - def __init__(self, left_expr: Expression, indices: List[Expression], - sequences: List[Expression], condlists: List[List[Expression]]) -> None: - self.left_expr = left_expr - self.sequences = sequences - self.condlists = condlists - self.indices = indices - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_generator_expr(self) - - -class ListComprehension(Expression): - """List comprehension (e.g. [x + 1 for x in a])""" - - generator = None # type: GeneratorExpr - - def __init__(self, generator: GeneratorExpr) -> None: - self.generator = generator - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_list_comprehension(self) - - -class SetComprehension(Expression): - """Set comprehension (e.g. {x + 1 for x in a})""" - - generator = None # type: GeneratorExpr - - def __init__(self, generator: GeneratorExpr) -> None: - self.generator = generator - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_set_comprehension(self) - - -class DictionaryComprehension(Expression): - """Dictionary comprehension (e.g. {k: v for k, v in a}""" - - key = None # type: Expression - value = None # type: Expression - sequences_expr = None # type: List[Expression] - condlists = None # type: List[List[Expression]] - indices = None # type: List[Expression] - - def __init__(self, key: Expression, value: Expression, indices: List[Expression], - sequences: List[Expression], condlists: List[List[Expression]]) -> None: - self.key = key - self.value = value - self.sequences = sequences - self.condlists = condlists - self.indices = indices - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_dictionary_comprehension(self) - - -class ConditionalExpr(Expression): - """Conditional expression (e.g. x if y else z)""" - - cond = None # type: Expression - if_expr = None # type: Expression - else_expr = None # type: Expression - - def __init__(self, cond: Expression, if_expr: Expression, else_expr: Expression) -> None: - self.cond = cond - self.if_expr = if_expr - self.else_expr = else_expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_conditional_expr(self) - - -class BackquoteExpr(Expression): - """Python 2 expression `...`.""" - - expr = None # type: Expression - - def __init__(self, expr: Expression) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_backquote_expr(self) - - -class TypeApplication(Expression): - """Type application expr[type, ...]""" - - expr = None # type: Expression - types = None # type: List[mypy.types.Type] - - def __init__(self, expr: Expression, types: List['mypy.types.Type']) -> None: - self.expr = expr - self.types = types - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_type_application(self) - - -# Variance of a type variable. For example, T in the definition of -# List[T] is invariant, so List[int] is not a subtype of List[object], -# and also List[object] is not a subtype of List[int]. -# -# The T in Iterable[T] is covariant, so Iterable[int] is a subtype of -# Iterable[object], but not vice versa. -# -# If T is contravariant in Foo[T], Foo[object] is a subtype of -# Foo[int], but not vice versa. -INVARIANT = 0 # type: int -COVARIANT = 1 # type: int -CONTRAVARIANT = 2 # type: int - - -class TypeVarExpr(SymbolNode, Expression): - """Type variable expression TypeVar(...).""" - - _name = '' - _fullname = '' - # Value restriction: only types in the list are valid as values. If the - # list is empty, there is no restriction. - values = None # type: List[mypy.types.Type] - # Upper bound: only subtypes of upper_bound are valid as values. By default - # this is 'object', meaning no restriction. - upper_bound = None # type: mypy.types.Type - # Variance of the type variable. Invariant is the default. - # TypeVar(..., covariant=True) defines a covariant type variable. - # TypeVar(..., contravariant=True) defines a contravariant type - # variable. - variance = INVARIANT - - def __init__(self, name: str, fullname: str, - values: List['mypy.types.Type'], - upper_bound: 'mypy.types.Type', - variance: int=INVARIANT) -> None: - self._name = name - self._fullname = fullname - self.values = values - self.upper_bound = upper_bound - self.variance = variance - - def name(self) -> str: - return self._name - - def fullname(self) -> str: - return self._fullname - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_type_var_expr(self) - - def serialize(self) -> JsonDict: - return {'.class': 'TypeVarExpr', - 'name': self._name, - 'fullname': self._fullname, - 'values': [t.serialize() for t in self.values], - 'upper_bound': self.upper_bound.serialize(), - 'variance': self.variance, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'TypeVarExpr': - assert data['.class'] == 'TypeVarExpr' - return TypeVarExpr(data['name'], - data['fullname'], - [mypy.types.Type.deserialize(v) for v in data['values']], - mypy.types.Type.deserialize(data['upper_bound']), - data['variance']) - - -class TypeAliasExpr(Expression): - """Type alias expression (rvalue).""" - - type = None # type: mypy.types.Type - - def __init__(self, type: 'mypy.types.Type') -> None: - self.type = type - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_type_alias_expr(self) - - -class NamedTupleExpr(Expression): - """Named tuple expression namedtuple(...).""" - - # The class representation of this named tuple (its tuple_type attribute contains - # the tuple item types) - info = None # type: TypeInfo - - def __init__(self, info: 'TypeInfo') -> None: - self.info = info - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_namedtuple_expr(self) - - -class PromoteExpr(Expression): - """Ducktype class decorator expression _promote(...).""" - - type = None # type: mypy.types.Type - - def __init__(self, type: 'mypy.types.Type') -> None: - self.type = type - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit__promote_expr(self) - - -# Constants - - -class TempNode(Expression): - """Temporary dummy node used during type checking. - - This node is not present in the original program; it is just an artifact - of the type checker implementation. It only represents an opaque node with - some fixed type. - """ - - type = None # type: mypy.types.Type - - def __init__(self, typ: 'mypy.types.Type') -> None: - self.type = typ - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_temp_node(self) - - -class TypeInfo(SymbolNode): - """The type structure of a single class. - - Each TypeInfo corresponds one-to-one to a ClassDef, which - represents the AST of the class. - - In type-theory terms, this is a "type constructor", and if the - class is generic then it will be a type constructor of higher kind. - Where the class is used in an actual type, it's in the form of an - Instance, which amounts to a type application of the tycon to - the appropriate number of arguments. - """ - - _fullname = None # type: str # Fully qualified name - defn = None # type: ClassDef # Corresponding ClassDef - # Method Resolution Order: the order of looking up attributes. The first - # value always to refers to this class. - mro = None # type: List[TypeInfo] - subtypes = None # type: Set[TypeInfo] # Direct subclasses encountered so far - names = None # type: SymbolTable # Names defined directly in this type - is_abstract = False # Does the class have any abstract attributes? - abstract_attributes = None # type: List[str] - # Classes inheriting from Enum shadow their true members with a __getattr__, so we - # have to treat them as a special case. - is_enum = False - # If true, any unknown attributes should have type 'Any' instead - # of generating a type error. This would be true if there is a - # base class with type 'Any', but other use cases may be - # possible. This is similar to having __getattr__ that returns Any - # (and __setattr__), but without the __getattr__ method. - fallback_to_any = False - - # Information related to type annotations. - - # Generic type variable names - type_vars = None # type: List[str] - - # Direct base classes. - bases = None # type: List[mypy.types.Instance] - - # Another type which this type will be treated as a subtype of, - # even though it's not a subclass in Python. The non-standard - # `@_promote` decorator introduces this, and there are also - # several builtin examples, in particular `int` -> `float`. - _promote = None # type: mypy.types.Type - - # Representation of a Tuple[...] base class, if the class has any - # (e.g., for named tuples). If this is not None, the actual Type - # object used for this class is not an Instance but a TupleType; - # the corresponding Instance is set as the fallback type of the - # tuple type. - tuple_type = None # type: mypy.types.TupleType - - # Is this a named tuple type? - is_named_tuple = False - - # Is this a dummy from deserialization? - is_dummy = False - - # Alternative to fullname() for 'anonymous' classes. - alt_fullname = None # type: Optional[str] - - def __init__(self, names: 'SymbolTable', defn: ClassDef) -> None: - """Initialize a TypeInfo.""" - self.names = names - self.defn = defn - self.subtypes = set() - self.type_vars = [] - self.bases = [] - # Leave self.mro uninitialized until we compute it for real, - # so we don't accidentally try to use it prematurely. - self._fullname = defn.fullname - self.is_abstract = False - self.abstract_attributes = [] - if defn.type_vars: - for vd in defn.type_vars: - self.type_vars.append(vd.name) - - def name(self) -> str: - """Short name.""" - return self.defn.name - - def fullname(self) -> str: - return self._fullname - - def is_generic(self) -> bool: - """Is the type generic (i.e. does it have type variables)?""" - return len(self.type_vars) > 0 - - def get(self, name: str) -> 'SymbolTableNode': - for cls in self.mro: - n = cls.names.get(name) - if n: - return n - return None - - def __getitem__(self, name: str) -> 'SymbolTableNode': - n = self.get(name) - if n: - return n - else: - raise KeyError(name) - - def __repr__(self) -> str: - return '' % self.fullname() - - # IDEA: Refactor the has* methods to be more consistent and document - # them. - - def has_readable_member(self, name: str) -> bool: - return self.get(name) is not None - - def has_writable_member(self, name: str) -> bool: - return self.has_var(name) - - def has_var(self, name: str) -> bool: - return self.get_var(name) is not None - - def has_method(self, name: str) -> bool: - return self.get_method(name) is not None - - def get_var(self, name: str) -> Var: - for cls in self.mro: - if name in cls.names: - node = cls.names[name].node - if isinstance(node, Var): - return node - else: - return None - return None - - def get_var_or_getter(self, name: str) -> SymbolNode: - # TODO getter - return self.get_var(name) - - def get_var_or_setter(self, name: str) -> SymbolNode: - # TODO setter - return self.get_var(name) - - def get_method(self, name: str) -> FuncBase: - if self.mro is None: # Might be because of a previous error. - return None - for cls in self.mro: - if name in cls.names: - node = cls.names[name].node - if isinstance(node, FuncBase): - return node - else: - return None - return None - - def calculate_mro(self) -> None: - """Calculate and set mro (method resolution order). - - Raise MroError if cannot determine mro. - """ - mro = linearize_hierarchy(self) - assert mro, "Could not produce a MRO at all for %s" % (self,) - self.mro = mro - self.is_enum = self._calculate_is_enum() - - def _calculate_is_enum(self) -> bool: - """ - If this is "enum.Enum" itself, then yes, it's an enum. - If the flag .is_enum has been set on anything in the MRO, it's an enum. - """ - if self.fullname() == ENUM_BASECLASS: - return True - if self.mro: - return any(type_info.is_enum for type_info in self.mro) - return False - - def has_base(self, fullname: str) -> bool: - """Return True if type has a base type with the specified name. - - This can be either via extension or via implementation. - """ - if self.mro: - for cls in self.mro: - if cls.fullname() == fullname: - return True - return False - - def all_subtypes(self) -> 'Set[TypeInfo]': - """Return TypeInfos of all subtypes, including this type, as a set.""" - subtypes = set([self]) - for subt in self.subtypes: - for t in subt.all_subtypes(): - subtypes.add(t) - return subtypes - - def all_base_classes(self) -> 'List[TypeInfo]': - """Return a list of base classes, including indirect bases.""" - assert False - - def direct_base_classes(self) -> 'List[TypeInfo]': - """Return a direct base classes. - - Omit base classes of other base classes. - """ - return [base.type for base in self.bases] - - def __str__(self) -> str: - """Return a string representation of the type. - - This includes the most important information about the type. - """ - base = None # type: str - if self.bases: - base = 'Bases({})'.format(', '.join(str(base) - for base in self.bases)) - return dump_tagged(['Name({})'.format(self.fullname()), - base, - ('Names', sorted(self.names.keys()))], - 'TypeInfo') - - def serialize(self) -> Union[str, JsonDict]: - # NOTE: This is where all ClassDefs originate, so there shouldn't be duplicates. - data = {'.class': 'TypeInfo', - 'fullname': self.fullname(), - 'alt_fullname': self.alt_fullname, - 'names': self.names.serialize(self.alt_fullname or self.fullname()), - 'defn': self.defn.serialize(), - 'is_abstract': self.is_abstract, - 'abstract_attributes': self.abstract_attributes, - 'is_enum': self.is_enum, - 'fallback_to_any': self.fallback_to_any, - 'type_vars': self.type_vars, - 'bases': [b.serialize() for b in self.bases], - '_promote': None if self._promote is None else self._promote.serialize(), - 'tuple_type': None if self.tuple_type is None else self.tuple_type.serialize(), - 'is_named_tuple': self.is_named_tuple, - } - return data - - @classmethod - def deserialize(cls, data: JsonDict) -> 'TypeInfo': - names = SymbolTable.deserialize(data['names']) - defn = ClassDef.deserialize(data['defn']) - ti = TypeInfo(names, defn) - ti._fullname = data['fullname'] - ti.alt_fullname = data['alt_fullname'] - # TODO: Is there a reason to reconstruct ti.subtypes? - ti.is_abstract = data['is_abstract'] - ti.abstract_attributes = data['abstract_attributes'] - ti.is_enum = data['is_enum'] - ti.fallback_to_any = data['fallback_to_any'] - ti.type_vars = data['type_vars'] - ti.bases = [mypy.types.Instance.deserialize(b) for b in data['bases']] - ti._promote = (None if data['_promote'] is None - else mypy.types.Type.deserialize(data['_promote'])) - ti.tuple_type = (None if data['tuple_type'] is None - else mypy.types.TupleType.deserialize(data['tuple_type'])) - ti.is_named_tuple = data['is_named_tuple'] - return ti - - -def NamedTupleTypeInfo(tup: 'mypy.types.NamedTupleType', *args) -> TypeInfo: - info = TypeInfo(*args) - info.tuple_type = tup - info.bases = [tup.fallback] - info.is_named_tuple = True - info.mro = [info] + tup.fallback.type.mro - return info - -class SymbolTableNode: - # Kind of node. Possible values: - # - LDEF: local definition (of any kind) - # - GDEF: global (module-level) definition - # - MDEF: class member definition - # - UNBOUND_TVAR: TypeVar(...) definition, not bound - # - TVAR: type variable in a bound scope (generic function / generic clas) - # - MODULE_REF: reference to a module - # - TYPE_ALIAS: type alias - # - UNBOUND_IMPORTED: temporary kind for imported names - kind = None # type: int - # AST node of definition (FuncDef/Var/TypeInfo/Decorator/TypeVarExpr, - # or None for a bound type variable). - node = None # type: Optional[SymbolNode] - # Type variable definition (for bound type variables only) - tvar_def = None # type: Optional[mypy.types.TypeVarDef] - # Module id (e.g. "foo.bar") or None - mod_id = '' - # If this not None, override the type of the 'node' attribute. - type_override = None # type: Optional[mypy.types.Type] - # If False, this name won't be imported via 'from import *'. - # This has no effect on names within classes. - module_public = True - # For deserialized MODULE_REF nodes, the referenced module name; - # for other nodes, optionally the name of the referenced object. - cross_ref = None # type: Optional[str] - - def __init__(self, kind: int, node: Optional[SymbolNode], mod_id: str = None, - typ: 'mypy.types.Type' = None, - tvar_def: 'mypy.types.TypeVarDef' = None, - module_public: bool = True) -> None: - self.kind = kind - self.node = node - self.type_override = typ - self.mod_id = mod_id - self.tvar_def = tvar_def - self.module_public = module_public - - @property - def fullname(self) -> str: - if self.node is not None: - return self.node.fullname() - else: - return None - - @property - def type(self) -> 'mypy.types.Type': - # IDEA: Get rid of the Any type. - node = self.node # type: Any - if self.type_override is not None: - return self.type_override - elif ((isinstance(node, Var) or isinstance(node, FuncDef)) - and node.type is not None): - return node.type - elif isinstance(node, Decorator): - return node.var.type - else: - return None - - def __str__(self) -> str: - s = '{}/{}'.format(node_kinds[self.kind], short_type(self.node)) - if self.mod_id is not None: - s += ' ({})'.format(self.mod_id) - # Include declared type of variables and functions. - if self.type is not None: - s += ' : {}'.format(self.type) - return s - - def serialize(self, prefix: str, name: str) -> JsonDict: - """Serialize a SymbolTableNode. - - Args: - prefix: full name of the containing module or class; or None - name: name of this object relative to the containing object - """ - data = {'.class': 'SymbolTableNode', - 'kind': node_kinds[self.kind], - } # type: JsonDict - if self.tvar_def: - data['tvar_def'] = self.tvar_def.serialize() - if not self.module_public: - data['module_public'] = False - if self.kind == MODULE_REF: - assert self.node is not None, "Missing module cross ref in %s for %s" % (prefix, name) - data['cross_ref'] = self.node.fullname() - else: - if self.node is not None: - if prefix is not None: - # Check whether this is an alias for another object. - # If the object's canonical full name differs from - # the full name computed from prefix and name, - # it's an alias, and we serialize it as a cross ref. - if isinstance(self.node, TypeInfo): - fullname = self.node.alt_fullname or self.node.fullname() - else: - fullname = self.node.fullname() - if (fullname is not None and '.' in fullname and - fullname != prefix + '.' + name): - data['cross_ref'] = fullname - return data - data['node'] = self.node.serialize() - if self.type_override is not None: - data['type_override'] = self.type_override.serialize() - return data - - @classmethod - def deserialize(cls, data: JsonDict) -> 'SymbolTableNode': - assert data['.class'] == 'SymbolTableNode' - kind = inverse_node_kinds[data['kind']] - if 'cross_ref' in data: - # This will be fixed up later. - stnode = SymbolTableNode(kind, None) - stnode.cross_ref = data['cross_ref'] - else: - node = None - if 'node' in data: - node = SymbolNode.deserialize(data['node']) - typ = None - if 'type_override' in data: - typ = mypy.types.Type.deserialize(data['type_override']) - stnode = SymbolTableNode(kind, node, typ=typ) - if 'tvar_def' in data: - stnode.tvar_def = mypy.types.TypeVarDef.deserialize(data['tvar_def']) - if 'module_public' in data: - stnode.module_public = data['module_public'] - return stnode - - -class SymbolTable(Dict[str, SymbolTableNode]): - def __str__(self) -> str: - a = [] # type: List[str] - for key, value in self.items(): - # Filter out the implicit import of builtins. - if isinstance(value, SymbolTableNode): - if (value.fullname != 'builtins' and - (value.fullname or '').split('.')[-1] not in - implicit_module_attrs): - a.append(' ' + str(key) + ' : ' + str(value)) - else: - a.append(' ') - a = sorted(a) - a.insert(0, 'SymbolTable(') - a[-1] += ')' - return '\n'.join(a) - - def serialize(self, fullname: str) -> JsonDict: - data = {'.class': 'SymbolTable'} # type: JsonDict - for key, value in self.items(): - # Skip __builtins__: it's a reference to the builtins - # module that gets added to every module by - # SemanticAnalyzer.visit_file(), but it shouldn't be - # accessed by users of the module. - if key == '__builtins__': - continue - data[key] = value.serialize(fullname, key) - return data - - @classmethod - def deserialize(cls, data: JsonDict) -> 'SymbolTable': - assert data['.class'] == 'SymbolTable' - st = SymbolTable() - for key, value in data.items(): - if key != '.class': - st[key] = SymbolTableNode.deserialize(value) - return st - - -def function_type(func: FuncBase, fallback: 'mypy.types.Instance') -> 'mypy.types.FunctionLike': - if func.type: - assert isinstance(func.type, mypy.types.FunctionLike) - return func.type - else: - # Implicit type signature with dynamic types. - # Overloaded functions always have a signature, so func must be an ordinary function. - fdef = cast(FuncDef, func) - name = func.name() - if name: - name = '"{}"'.format(name) - names = [] # type: List[str] - for arg in fdef.arguments: - names.append(arg.variable.name()) - - return mypy.types.CallableType( - [mypy.types.AnyType()] * len(fdef.arguments), - [arg.kind for arg in fdef.arguments], - names, - mypy.types.AnyType(), - fallback, - name, - implicit=True, - ) - - -def method_type_with_fallback(func: FuncBase, - fallback: 'mypy.types.Instance') -> 'mypy.types.FunctionLike': - """Return the signature of a method (omit self).""" - return method_type(function_type(func, fallback)) - - -def method_type(sig: 'mypy.types.FunctionLike') -> 'mypy.types.FunctionLike': - if isinstance(sig, mypy.types.CallableType): - return method_callable(sig) - else: - sig = cast(mypy.types.Overloaded, sig) - items = [] # type: List[mypy.types.CallableType] - for c in sig.items(): - items.append(method_callable(c)) - return mypy.types.Overloaded(items) - - -def method_callable(c: 'mypy.types.CallableType') -> 'mypy.types.CallableType': - if c.arg_kinds and c.arg_kinds[0] == ARG_STAR: - # The signature is of the form 'def foo(*args, ...)'. - # In this case we shouldn't drop the first arg, - # since self will be absorbed by the *args. - return c - return c.copy_modified(arg_types=c.arg_types[1:], - arg_kinds=c.arg_kinds[1:], - arg_names=c.arg_names[1:]) - - -class MroError(Exception): - """Raised if a consistent mro cannot be determined for a class.""" - - -def linearize_hierarchy(info: TypeInfo) -> List[TypeInfo]: - # TODO describe - if info.mro: - return info.mro - bases = info.direct_base_classes() - lin_bases = [] - for base in bases: - assert base is not None, "Cannot linearize bases for %s %s" % (info.fullname(), bases) - lin_bases.append(linearize_hierarchy(base)) - lin_bases.append(bases) - return [info] + merge(lin_bases) - - -def merge(seqs: List[List[TypeInfo]]) -> List[TypeInfo]: - seqs = [s[:] for s in seqs] - result = [] # type: List[TypeInfo] - while True: - seqs = [s for s in seqs if s] - if not seqs: - return result - for seq in seqs: - head = seq[0] - if not [s for s in seqs if head in s[1:]]: - break - else: - raise MroError() - result.append(head) - for s in seqs: - if s[0] is head: - del s[0] +"""Abstract syntax tree node classes (i.e. parse tree).""" + +import os +import re +from abc import abstractmethod, ABCMeta + +from typing import ( + Any, TypeVar, List, Tuple, cast, Set, Dict, Union, Optional +) + +from mypy.lex import Token +import mypy.strconv +from mypy.visitor import NodeVisitor +from mypy.util import dump_tagged, short_type + + +class Context: + """Base type for objects that are valid as error message locations.""" + @abstractmethod + def get_line(self) -> int: pass + + +if False: + # break import cycle only needed for mypy + import mypy.types + + +T = TypeVar('T') + +JsonDict = Dict[str, Any] + + +# Symbol table node kinds +# +# TODO rename to use more descriptive names + +LDEF = 0 # type: int +GDEF = 1 # type: int +MDEF = 2 # type: int +MODULE_REF = 3 # type: int +# Type variable declared using TypeVar(...) has kind UNBOUND_TVAR. It's not +# valid as a type. A type variable is valid as a type (kind BOUND_TVAR) within +# (1) a generic class that uses the type variable as a type argument or +# (2) a generic function that refers to the type variable in its signature. +UNBOUND_TVAR = 4 # type: int +BOUND_TVAR = 5 # type: int +TYPE_ALIAS = 6 # type: int +# Placeholder for a name imported via 'from ... import'. Second phase of +# semantic will replace this the actual imported reference. This is +# needed so that we can detect whether a name has been imported during +# XXX what? +UNBOUND_IMPORTED = 7 # type: int + + +LITERAL_YES = 2 +LITERAL_TYPE = 1 +LITERAL_NO = 0 + +# Hard coded name of Enum baseclass. +ENUM_BASECLASS = "enum.Enum" + +node_kinds = { + LDEF: 'Ldef', + GDEF: 'Gdef', + MDEF: 'Mdef', + MODULE_REF: 'ModuleRef', + UNBOUND_TVAR: 'UnboundTvar', + BOUND_TVAR: 'Tvar', + TYPE_ALIAS: 'TypeAlias', + UNBOUND_IMPORTED: 'UnboundImported', +} +inverse_node_kinds = {_kind: _name for _name, _kind in node_kinds.items()} + + +implicit_module_attrs = {'__name__': '__builtins__.str', + '__doc__': '__builtins__.str', + '__file__': '__builtins__.str', + '__package__': '__builtins__.str'} + + +type_aliases = { + 'typing.List': '__builtins__.list', + 'typing.Dict': '__builtins__.dict', + 'typing.Set': '__builtins__.set', +} + +reverse_type_aliases = dict((name.replace('__builtins__', 'builtins'), alias) + for alias, name in type_aliases.items()) # type: Dict[str, str] + + +class Node(Context): + """Common base class for all non-type parse tree nodes.""" + + line = -1 + + literal = LITERAL_NO + literal_hash = None # type: Any + + def __str__(self) -> str: + ans = self.accept(mypy.strconv.StrConv()) + if ans is None: + return repr(self) + return ans + + def set_line(self, target: Union[Token, 'Node', int]) -> 'Node': + if isinstance(target, int): + self.line = target + else: + self.line = target.line + return self + + def get_line(self) -> int: + # TODO this should be just 'line' + return self.line + + def accept(self, visitor: NodeVisitor[T]) -> T: + raise RuntimeError('Not implemented') + + +# These are placeholders for a future refactoring; see #1783. +# For now they serve as (unchecked) documentation of what various +# fields of Node subtypes are expected to contain. +Statement = Node +Expression = Node + + +class SymbolNode(Node): + # Nodes that can be stored in a symbol table. + + # TODO do not use methods for these + + @abstractmethod + def name(self) -> str: pass + + @abstractmethod + def fullname(self) -> str: pass + + # NOTE: Can't use @abstractmethod, since many subclasses of Node + # don't implement serialize(). + def serialize(self) -> Any: + raise NotImplementedError('Cannot serialize {} instance'.format(self.__class__.__name__)) + + @classmethod + def deserialize(cls, data: JsonDict) -> 'SymbolNode': + classname = data['.class'] + glo = globals() + if classname in glo: + cl = glo[classname] + if issubclass(cl, cls) and 'deserialize' in cl.__dict__: + return cl.deserialize(data) + raise NotImplementedError('unexpected .class {}'.format(classname)) + + +class MypyFile(SymbolNode, Statement): + """The abstract syntax tree of a single source file.""" + + # Module name ('__main__' for initial file) + _name = None # type: str + # Fully qualified module name + _fullname = None # type: str + # Path to the file (None if not known) + path = '' + # Top-level definitions and statements + defs = None # type: List[Statement] + # Is there a UTF-8 BOM at the start? + is_bom = False + names = None # type: SymbolTable + # All import nodes within the file (also ones within functions etc.) + imports = None # type: List[ImportBase] + # Lines to ignore when checking + ignored_lines = None # type: Set[int] + # Is this file represented by a stub file (.pyi)? + is_stub = False + # Do weak typing globally in the file? + weak_opts = None # type: Set[str] + + def __init__(self, + defs: List[Statement], + imports: List['ImportBase'], + is_bom: bool = False, + ignored_lines: Set[int] = None, + weak_opts: Set[str] = None) -> None: + self.defs = defs + self.line = 1 # Dummy line number + self.imports = imports + self.is_bom = is_bom + self.weak_opts = weak_opts + if ignored_lines: + self.ignored_lines = ignored_lines + else: + self.ignored_lines = set() + + def name(self) -> str: + return self._name + + def fullname(self) -> str: + return self._fullname + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_mypy_file(self) + + def is_package_init_file(self) -> bool: + return not (self.path is None) and len(self.path) != 0 \ + and os.path.basename(self.path).startswith('__init__.') + + def serialize(self) -> JsonDict: + return {'.class': 'MypyFile', + '_name': self._name, + '_fullname': self._fullname, + 'names': self.names.serialize(self._fullname), + 'is_stub': self.is_stub, + 'path': self.path, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'MypyFile': + assert data['.class'] == 'MypyFile', data + tree = MypyFile([], []) + tree._name = data['_name'] + tree._fullname = data['_fullname'] + tree.names = SymbolTable.deserialize(data['names']) + tree.is_stub = data['is_stub'] + tree.path = data['path'] + return tree + + +class ImportBase(Statement): + """Base class for all import statements.""" + is_unreachable = False + is_top_level = False # Set by semanal.FirstPass + # If an import replaces existing definitions, we construct dummy assignment + # statements that assign the imported names to the names in the current scope, + # for type checking purposes. Example: + # + # x = 1 + # from m import x <-- add assignment representing "x = m.x" + assignments = None # type: List[AssignmentStmt] + + def __init__(self) -> None: + self.assignments = [] + + +class Import(ImportBase): + """import m [as n]""" + + ids = None # type: List[Tuple[str, Optional[str]]] # (module id, as id) + + def __init__(self, ids: List[Tuple[str, Optional[str]]]) -> None: + super().__init__() + self.ids = ids + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_import(self) + + +class ImportFrom(ImportBase): + """from m import x [as y], ...""" + + names = None # type: List[Tuple[str, Optional[str]]] # Tuples (name, as name) + + def __init__(self, id: str, relative: int, names: List[Tuple[str, Optional[str]]]) -> None: + super().__init__() + self.id = id + self.names = names + self.relative = relative + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_import_from(self) + + +class ImportAll(ImportBase): + """from m import *""" + + def __init__(self, id: str, relative: int) -> None: + super().__init__() + self.id = id + self.relative = relative + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_import_all(self) + + +class FuncBase(SymbolNode): + """Abstract base class for function-like nodes""" + + # Type signature. This is usually CallableType or Overloaded, but it can be something else for + # decorated functions/ + type = None # type: mypy.types.Type + # If method, reference to TypeInfo + info = None # type: TypeInfo + is_property = False + _fullname = None # type: str # Name with module prefix + + @abstractmethod + def name(self) -> str: pass + + def fullname(self) -> str: + return self._fullname + + def is_method(self) -> bool: + return bool(self.info) + + +class OverloadedFuncDef(FuncBase, Statement): + """A logical node representing all the variants of an overloaded function. + + This node has no explicit representation in the source program. + Overloaded variants must be consecutive in the source file. + """ + + items = None # type: List[Decorator] + + def __init__(self, items: List['Decorator']) -> None: + self.items = items + self.set_line(items[0].line) + + def name(self) -> str: + return self.items[0].func.name() + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_overloaded_func_def(self) + + def serialize(self) -> JsonDict: + return {'.class': 'OverloadedFuncDef', + 'items': [i.serialize() for i in self.items], + 'type': None if self.type is None else self.type.serialize(), + 'fullname': self._fullname, + 'is_property': self.is_property, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'OverloadedFuncDef': + assert data['.class'] == 'OverloadedFuncDef' + res = OverloadedFuncDef([Decorator.deserialize(d) for d in data['items']]) + if data.get('type') is not None: + res.type = mypy.types.Type.deserialize(data['type']) + res._fullname = data['fullname'] + res.is_property = data['is_property'] + # NOTE: res.info will be set in the fixup phase. + return res + + +class Argument(Node): + """A single argument in a FuncItem.""" + + def __init__(self, variable: 'Var', type_annotation: 'Optional[mypy.types.Type]', + initializer: Optional[Expression], kind: int, + initialization_statement: Optional['AssignmentStmt'] = None) -> None: + self.variable = variable + + self.type_annotation = type_annotation + self.initializer = initializer + + self.initialization_statement = initialization_statement + if not self.initialization_statement: + self.initialization_statement = self._initialization_statement() + + self.kind = kind + + def _initialization_statement(self) -> Optional['AssignmentStmt']: + """Convert the initializer into an assignment statement. + """ + if not self.initializer: + return None + + rvalue = self.initializer + lvalue = NameExpr(self.variable.name()) + assign = AssignmentStmt([lvalue], rvalue) + return assign + + def set_line(self, target: Union[Token, Node, int]) -> Node: + super().set_line(target) + + if self.initializer: + self.initializer.set_line(self.line) + + self.variable.set_line(self.line) + + if self.initialization_statement: + self.initialization_statement.set_line(self.line) + self.initialization_statement.lvalues[0].set_line(self.line) + + def serialize(self) -> JsonDict: + data = {'.class': 'Argument', + 'kind': self.kind, + 'variable': self.variable.serialize(), + 'type_annotation': (None if self.type_annotation is None + else self.type_annotation.serialize()), + } # type: JsonDict + # TODO: initializer? + return data + + @classmethod + def deserialize(cls, data: JsonDict) -> 'Argument': + assert data['.class'] == 'Argument' + return Argument(Var.deserialize(data['variable']), + (None if data.get('type_annotation') is None + else mypy.types.Type.deserialize(data['type_annotation'])), + None, # TODO: initializer? + kind=data['kind']) + + +class FuncItem(FuncBase): + arguments = [] # type: List[Argument] + # Minimum number of arguments + min_args = 0 + # Maximum number of positional arguments, -1 if no explicit limit (*args not included) + max_pos = 0 + body = None # type: Block + # Is this an overload variant of function with more than one overload variant? + is_overload = False + is_generator = False # Contains a yield statement? + is_static = False # Uses @staticmethod? + is_class = False # Uses @classmethod? + # Variants of function with type variables with values expanded + expanded = None # type: List[FuncItem] + + def __init__(self, arguments: List[Argument], body: 'Block', + typ: 'mypy.types.FunctionLike' = None) -> None: + self.arguments = arguments + arg_kinds = [arg.kind for arg in self.arguments] + self.max_pos = arg_kinds.count(ARG_POS) + arg_kinds.count(ARG_OPT) + self.body = body + self.type = typ + self.expanded = [] + + self.min_args = 0 + for i in range(len(self.arguments)): + if self.arguments[i] is None and i < self.max_fixed_argc(): + self.min_args = i + 1 + + def max_fixed_argc(self) -> int: + return self.max_pos + + def set_line(self, target: Union[Token, Node, int]) -> Node: + super().set_line(target) + for arg in self.arguments: + arg.set_line(self.line) + return self + + def is_dynamic(self): + return self.type is None + + +class FuncDef(FuncItem, Statement): + """Function definition. + + This is a non-lambda function defined using 'def'. + """ + + is_decorated = False + is_conditional = False # Defined conditionally (within block)? + is_abstract = False + is_property = False + original_def = None # type: Union[None, FuncDef, Var] # Original conditional definition + + def __init__(self, + name: str, # Function name + arguments: List[Argument], + body: 'Block', + typ: 'mypy.types.FunctionLike' = None) -> None: + super().__init__(arguments, body, typ) + self._name = name + + def name(self) -> str: + return self._name + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_func_def(self) + + def is_constructor(self) -> bool: + return self.info is not None and self._name == '__init__' + + def serialize(self) -> JsonDict: + return {'.class': 'FuncDef', + 'name': self._name, + 'fullname': self._fullname, + 'arguments': [a.serialize() for a in self.arguments], + 'type': None if self.type is None else self.type.serialize(), + 'is_property': self.is_property, + 'is_overload': self.is_overload, + 'is_generator': self.is_generator, + 'is_static': self.is_static, + 'is_class': self.is_class, + 'is_decorated': self.is_decorated, + 'is_conditional': self.is_conditional, + 'is_abstract': self.is_abstract, + # TODO: Do we need expanded, original_def? + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'FuncDef': + assert data['.class'] == 'FuncDef' + body = Block([]) + ret = FuncDef(data['name'], + [Argument.deserialize(a) for a in data['arguments']], + body, + (None if data['type'] is None + else mypy.types.FunctionLike.deserialize(data['type']))) + ret._fullname = data['fullname'] + ret.is_property = data['is_property'] + ret.is_overload = data['is_overload'] + ret.is_generator = data['is_generator'] + ret.is_static = data['is_static'] + ret.is_class = data['is_class'] + ret.is_decorated = data['is_decorated'] + ret.is_conditional = data['is_conditional'] + ret.is_abstract = data['is_abstract'] + # NOTE: ret.info is set in the fixup phase. + return ret + + +class Decorator(SymbolNode, Statement): + """A decorated function. + + A single Decorator object can include any number of function decorators. + """ + + func = None # type: FuncDef # Decorated function + decorators = None # type: List[Expression] # Decorators, at least one # XXX Not true + var = None # type: Var # Represents the decorated function obj + is_overload = False + + def __init__(self, func: FuncDef, decorators: List[Expression], + var: 'Var') -> None: + self.func = func + self.decorators = decorators + self.var = var + self.is_overload = False + + def name(self) -> str: + return self.func.name() + + def fullname(self) -> str: + return self.func.fullname() + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_decorator(self) + + def serialize(self) -> JsonDict: + return {'.class': 'Decorator', + 'func': self.func.serialize(), + 'var': self.var.serialize(), + 'is_overload': self.is_overload, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'Decorator': + assert data['.class'] == 'Decorator' + dec = Decorator(FuncDef.deserialize(data['func']), + [], + Var.deserialize(data['var'])) + dec.is_overload = data['is_overload'] + return dec + + +class Var(SymbolNode, Statement): + """A variable. + + It can refer to global/local variable or a data attribute. + """ + + _name = None # type: str # Name without module prefix + _fullname = None # type: str # Name with module prefix + info = None # type: TypeInfo # Defining class (for member variables) + type = None # type: mypy.types.Type # Declared or inferred type, or None + # Is this the first argument to an ordinary method (usually "self")? + is_self = False + is_ready = False # If inferred, is the inferred type available? + # Is this initialized explicitly to a non-None value in class body? + is_initialized_in_class = False + is_staticmethod = False + is_classmethod = False + is_property = False + is_settable_property = False + + def __init__(self, name: str, type: 'mypy.types.Type' = None) -> None: + self._name = name + self.type = type + self.is_self = False + self.is_ready = True + self.is_initialized_in_class = False + + def name(self) -> str: + return self._name + + def fullname(self) -> str: + return self._fullname + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_var(self) + + def serialize(self) -> JsonDict: + # TODO: Leave default values out? + # NOTE: Sometimes self.is_ready is False here, but we don't care. + data = {'.class': 'Var', + 'name': self._name, + 'fullname': self._fullname, + 'type': None if self.type is None else self.type.serialize(), + 'is_self': self.is_self, + 'is_initialized_in_class': self.is_initialized_in_class, + 'is_staticmethod': self.is_staticmethod, + 'is_classmethod': self.is_classmethod, + 'is_property': self.is_property, + 'is_settable_property': self.is_settable_property, + } # type: JsonDict + return data + + @classmethod + def deserialize(cls, data: JsonDict) -> 'Var': + assert data['.class'] == 'Var' + name = data['name'] + type = None if data['type'] is None else mypy.types.Type.deserialize(data['type']) + v = Var(name, type) + v._fullname = data['fullname'] + v.is_self = data['is_self'] + v.is_initialized_in_class = data['is_initialized_in_class'] + v.is_staticmethod = data['is_staticmethod'] + v.is_classmethod = data['is_classmethod'] + v.is_property = data['is_property'] + v.is_settable_property = data['is_settable_property'] + return v + + +class ClassDef(Statement): + """Class definition""" + + name = None # type: str # Name of the class without module prefix + fullname = None # type: str # Fully qualified name of the class + defs = None # type: Block + type_vars = None # type: List[mypy.types.TypeVarDef] + # Base class expressions (not semantically analyzed -- can be arbitrary expressions) + base_type_exprs = None # type: List[Expression] + info = None # type: TypeInfo # Related TypeInfo + metaclass = '' + decorators = None # type: List[Expression] + # Built-in/extension class? (single implementation inheritance only) + is_builtinclass = False + + def __init__(self, + name: str, + defs: 'Block', + type_vars: List['mypy.types.TypeVarDef'] = None, + base_type_exprs: List[Expression] = None, + metaclass: str = None) -> None: + self.name = name + self.defs = defs + self.type_vars = type_vars or [] + self.base_type_exprs = base_type_exprs or [] + self.metaclass = metaclass + self.decorators = [] + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_class_def(self) + + def is_generic(self) -> bool: + return self.info.is_generic() + + def serialize(self) -> JsonDict: + # Not serialized: defs, base_type_exprs, decorators + return {'.class': 'ClassDef', + 'name': self.name, + 'fullname': self.fullname, + 'type_vars': [v.serialize() for v in self.type_vars], + 'metaclass': self.metaclass, + 'is_builtinclass': self.is_builtinclass, + } + + @classmethod + def deserialize(self, data: JsonDict) -> 'ClassDef': + assert data['.class'] == 'ClassDef' + res = ClassDef(data['name'], + Block([]), + [mypy.types.TypeVarDef.deserialize(v) for v in data['type_vars']], + metaclass=data['metaclass'], + ) + res.fullname = data['fullname'] + res.is_builtinclass = data['is_builtinclass'] + return res + + +class GlobalDecl(Statement): + """Declaration global x, y, ...""" + + names = None # type: List[str] + + def __init__(self, names: List[str]) -> None: + self.names = names + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_global_decl(self) + + +class NonlocalDecl(Statement): + """Declaration nonlocal x, y, ...""" + + names = None # type: List[str] + + def __init__(self, names: List[str]) -> None: + self.names = names + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_nonlocal_decl(self) + + +class Block(Statement): + body = None # type: List[Statement] + # True if we can determine that this block is not executed. For example, + # this applies to blocks that are protected by something like "if PY3:" + # when using Python 2. + is_unreachable = False + + def __init__(self, body: List[Statement]) -> None: + self.body = body + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_block(self) + + +# Statements + + +class ExpressionStmt(Statement): + """An expression as a statement, such as print(s).""" + expr = None # type: Expression + + def __init__(self, expr: Expression) -> None: + self.expr = expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_expression_stmt(self) + + +class AssignmentStmt(Statement): + """Assignment statement + The same node class is used for single assignment, multiple assignment + (e.g. x, y = z) and chained assignment (e.g. x = y = z), assignments + that define new names, and assignments with explicit types (# type). + + An lvalue can be NameExpr, TupleExpr, ListExpr, MemberExpr, IndexExpr. + """ + + lvalues = None # type: List[Expression] + rvalue = None # type: Expression + # Declared type in a comment, may be None. + type = None # type: mypy.types.Type + + def __init__(self, lvalues: List[Expression], rvalue: Expression, + type: 'mypy.types.Type' = None) -> None: + self.lvalues = lvalues + self.rvalue = rvalue + self.type = type + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_assignment_stmt(self) + + +class OperatorAssignmentStmt(Statement): + """Operator assignment statement such as x += 1""" + + op = '' + lvalue = None # type: Expression + rvalue = None # type: Expression + + def __init__(self, op: str, lvalue: Expression, rvalue: Expression) -> None: + self.op = op + self.lvalue = lvalue + self.rvalue = rvalue + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_operator_assignment_stmt(self) + + +class WhileStmt(Statement): + expr = None # type: Expression + body = None # type: Block + else_body = None # type: Block + + def __init__(self, expr: Expression, body: Block, else_body: Block) -> None: + self.expr = expr + self.body = body + self.else_body = else_body + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_while_stmt(self) + + +class ForStmt(Statement): + # Index variables + index = None # type: Expression + # Expression to iterate + expr = None # type: Expression + body = None # type: Block + else_body = None # type: Block + + def __init__(self, index: Expression, expr: Expression, body: Block, + else_body: Block) -> None: + self.index = index + self.expr = expr + self.body = body + self.else_body = else_body + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_for_stmt(self) + + +class ReturnStmt(Statement): + expr = None # type: Optional[Expression] + + def __init__(self, expr: Optional[Expression]) -> None: + self.expr = expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_return_stmt(self) + + +class AssertStmt(Statement): + expr = None # type: Expression + + def __init__(self, expr: Expression) -> None: + self.expr = expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_assert_stmt(self) + + +class DelStmt(Statement): + expr = None # type: Expression + + def __init__(self, expr: Expression) -> None: + self.expr = expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_del_stmt(self) + + +class BreakStmt(Statement): + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_break_stmt(self) + + +class ContinueStmt(Statement): + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_continue_stmt(self) + + +class PassStmt(Statement): + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_pass_stmt(self) + + +class IfStmt(Statement): + expr = None # type: List[Expression] + body = None # type: List[Block] + else_body = None # type: Block + + def __init__(self, expr: List[Expression], body: List[Block], + else_body: Block) -> None: + self.expr = expr + self.body = body + self.else_body = else_body + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_if_stmt(self) + + +class RaiseStmt(Statement): + expr = None # type: Expression + from_expr = None # type: Expression + + def __init__(self, expr: Expression, from_expr: Expression = None) -> None: + self.expr = expr + self.from_expr = from_expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_raise_stmt(self) + + +class TryStmt(Statement): + body = None # type: Block # Try body + types = None # type: List[Expression] # Except type expressions + vars = None # type: List[NameExpr] # Except variable names + handlers = None # type: List[Block] # Except bodies + else_body = None # type: Block + finally_body = None # type: Block + + def __init__(self, body: Block, vars: List['NameExpr'], types: List[Expression], + handlers: List[Block], else_body: Block, + finally_body: Block) -> None: + self.body = body + self.vars = vars + self.types = types + self.handlers = handlers + self.else_body = else_body + self.finally_body = finally_body + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_try_stmt(self) + + +class WithStmt(Statement): + expr = None # type: List[Expression] + target = None # type: List[Expression] + body = None # type: Block + + def __init__(self, expr: List[Expression], target: List[Expression], + body: Block) -> None: + self.expr = expr + self.target = target + self.body = body + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_with_stmt(self) + + +class PrintStmt(Statement): + """Python 2 print statement""" + + args = None # type: List[Expression] + newline = False + # The file-like target object (given using >>). + target = None # type: Optional[Expression] + + def __init__(self, args: List[Expression], newline: bool, target: Expression = None) -> None: + self.args = args + self.newline = newline + self.target = target + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_print_stmt(self) + + +class ExecStmt(Statement): + """Python 2 exec statement""" + + expr = None # type: Expression + variables1 = None # type: Optional[Expression] + variables2 = None # type: Optional[Expression] + + def __init__(self, expr: Expression, + variables1: Optional[Expression], + variables2: Optional[Expression]) -> None: + self.expr = expr + self.variables1 = variables1 + self.variables2 = variables2 + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_exec_stmt(self) + + +# Expressions + + +class IntExpr(Expression): + """Integer literal""" + + value = 0 + literal = LITERAL_YES + + def __init__(self, value: int) -> None: + self.value = value + self.literal_hash = value + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_int_expr(self) + + +# How mypy uses StrExpr, BytesExpr, and UnicodeExpr: +# In Python 2 mode: +# b'x', 'x' -> StrExpr +# u'x' -> UnicodeExpr +# BytesExpr is unused +# +# In Python 3 mode: +# b'x' -> BytesExpr +# 'x', u'x' -> StrExpr +# UnicodeExpr is unused + +class StrExpr(Expression): + """String literal""" + + value = '' + literal = LITERAL_YES + + def __init__(self, value: str) -> None: + self.value = value + self.literal_hash = value + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_str_expr(self) + + +class BytesExpr(Expression): + """Bytes literal""" + + value = '' # TODO use bytes + literal = LITERAL_YES + + def __init__(self, value: str) -> None: + self.value = value + self.literal_hash = value + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_bytes_expr(self) + + +class UnicodeExpr(Expression): + """Unicode literal (Python 2.x)""" + + value = '' # TODO use bytes + literal = LITERAL_YES + + def __init__(self, value: str) -> None: + self.value = value + self.literal_hash = value + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_unicode_expr(self) + + +class FloatExpr(Expression): + """Float literal""" + + value = 0.0 + literal = LITERAL_YES + + def __init__(self, value: float) -> None: + self.value = value + self.literal_hash = value + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_float_expr(self) + + +class ComplexExpr(Expression): + """Complex literal""" + + value = 0.0j + literal = LITERAL_YES + + def __init__(self, value: complex) -> None: + self.value = value + self.literal_hash = value + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_complex_expr(self) + + +class EllipsisExpr(Expression): + """Ellipsis (...)""" + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_ellipsis(self) + + +class StarExpr(Expression): + """Star expression""" + + expr = None # type: Expression + + def __init__(self, expr: Expression) -> None: + self.expr = expr + self.literal = self.expr.literal + self.literal_hash = ('Star', expr.literal_hash,) + + # Whether this starred expression is used in a tuple/list and as lvalue + self.valid = False + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_star_expr(self) + + +class RefExpr(Expression): + """Abstract base class for name-like constructs""" + + kind = None # type: int # LDEF/GDEF/MDEF/... (None if not available) + node = None # type: SymbolNode # Var, FuncDef or TypeInfo that describes this + fullname = None # type: str # Fully qualified name (or name if not global) + + # Does this define a new name with inferred type? + # + # For members, after semantic analysis, this does not take base + # classes into consideration at all; the type checker deals with these. + is_def = False + + +class NameExpr(RefExpr): + """Name expression + + This refers to a local name, global name or a module. + """ + + name = None # type: str # Name referred to (may be qualified) + # TypeInfo of class surrounding expression (may be None) + info = None # type: TypeInfo + + literal = LITERAL_TYPE + + def __init__(self, name: str) -> None: + self.name = name + self.literal_hash = ('Var', name,) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_name_expr(self) + + def serialize(self) -> JsonDict: + # TODO: Find out where and why NameExpr is being serialized (if at all). + assert False, "Serializing NameExpr: %s" % (self,) + return {'.class': 'NameExpr', + 'kind': self.kind, + 'node': None if self.node is None else self.node.serialize(), + 'fullname': self.fullname, + 'is_def': self.is_def, + 'name': self.name, + 'literal': self.literal, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'NameExpr': + assert data['.class'] == 'NameExpr' + ret = NameExpr(data['name']) + ret.kind = data['kind'] + ret.node = None if data['node'] is None else SymbolNode.deserialize(data['node']) + ret.fullname = data['fullname'] + ret.is_def = data['is_def'] + ret.literal = data['literal'] + return ret + + +class MemberExpr(RefExpr): + """Member access expression x.y""" + + expr = None # type: Expression + name = None # type: str + # The variable node related to a definition. + def_var = None # type: Var + + def __init__(self, expr: Expression, name: str) -> None: + self.expr = expr + self.name = name + self.literal = self.expr.literal + self.literal_hash = ('Member', expr.literal_hash, name) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_member_expr(self) + + +# Kinds of arguments + +# Positional argument +ARG_POS = 0 # type: int +# Positional, optional argument (functions only, not calls) +ARG_OPT = 1 # type: int +# *arg argument +ARG_STAR = 2 # type: int +# Keyword argument x=y in call, or keyword-only function arg +ARG_NAMED = 3 # type: int +# **arg argument +ARG_STAR2 = 4 # type: int + + +class CallExpr(Expression): + """Call expression. + + This can also represent several special forms that are syntactically calls + such as cast(...) and None # type: .... + """ + + callee = None # type: Expression + args = None # type: List[Expression] + arg_kinds = None # type: List[int] # ARG_ constants + # Each name can be None if not a keyword argument. + arg_names = None # type: List[str] + # If not None, the node that represents the meaning of the CallExpr. For + # cast(...) this is a CastExpr. + analyzed = None # type: Optional[Expression] + + def __init__(self, callee: Expression, args: List[Expression], arg_kinds: List[int], + arg_names: List[str] = None, analyzed: Expression = None) -> None: + if not arg_names: + arg_names = [None] * len(args) + + self.callee = callee + self.args = args + self.arg_kinds = arg_kinds + self.arg_names = arg_names + self.analyzed = analyzed + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_call_expr(self) + + +class YieldFromExpr(Expression): + expr = None # type: Expression + + def __init__(self, expr: Expression) -> None: + self.expr = expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_yield_from_expr(self) + + +class YieldExpr(Expression): + expr = None # type: Optional[Expression] + + def __init__(self, expr: Optional[Expression]) -> None: + self.expr = expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_yield_expr(self) + + +class IndexExpr(Expression): + """Index expression x[y]. + + Also wraps type application such as List[int] as a special form. + """ + + base = None # type: Expression + index = None # type: Expression + # Inferred __getitem__ method type + method_type = None # type: mypy.types.Type + # If not None, this is actually semantically a type application + # Class[type, ...] or a type alias initializer. + analyzed = None # type: Union[TypeApplication, TypeAliasExpr] + + def __init__(self, base: Expression, index: Expression) -> None: + self.base = base + self.index = index + self.analyzed = None + if self.index.literal == LITERAL_YES: + self.literal = self.base.literal + self.literal_hash = ('Member', base.literal_hash, + index.literal_hash) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_index_expr(self) + + +class UnaryExpr(Expression): + """Unary operation""" + + op = '' + expr = None # type: Expression + # Inferred operator method type + method_type = None # type: mypy.types.Type + + def __init__(self, op: str, expr: Expression) -> None: + self.op = op + self.expr = expr + self.literal = self.expr.literal + self.literal_hash = ('Unary', op, expr.literal_hash) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_unary_expr(self) + + +# Map from binary operator id to related method name (in Python 3). +op_methods = { + '+': '__add__', + '-': '__sub__', + '*': '__mul__', + '/': '__truediv__', + '%': '__mod__', + '//': '__floordiv__', + '**': '__pow__', + '&': '__and__', + '|': '__or__', + '^': '__xor__', + '<<': '__lshift__', + '>>': '__rshift__', + '==': '__eq__', + '!=': '__ne__', + '<': '__lt__', + '>=': '__ge__', + '>': '__gt__', + '<=': '__le__', + 'in': '__contains__', +} # type: Dict[str, str] + +ops_with_inplace_method = { + '+', '-', '*', '/', '%', '//', '**', '&', '|', '^', '<<', '>>'} + +inplace_operator_methods = set( + '__i' + op_methods[op][2:] for op in ops_with_inplace_method) + +reverse_op_methods = { + '__add__': '__radd__', + '__sub__': '__rsub__', + '__mul__': '__rmul__', + '__truediv__': '__rtruediv__', + '__mod__': '__rmod__', + '__floordiv__': '__rfloordiv__', + '__pow__': '__rpow__', + '__and__': '__rand__', + '__or__': '__ror__', + '__xor__': '__rxor__', + '__lshift__': '__rlshift__', + '__rshift__': '__rrshift__', + '__eq__': '__eq__', + '__ne__': '__ne__', + '__lt__': '__gt__', + '__ge__': '__le__', + '__gt__': '__lt__', + '__le__': '__ge__', +} + +normal_from_reverse_op = dict((m, n) for n, m in reverse_op_methods.items()) +reverse_op_method_set = set(reverse_op_methods.values()) + + +class OpExpr(Expression): + """Binary operation (other than . or [] or comparison operators, + which have specific nodes).""" + + op = '' + left = None # type: Expression + right = None # type: Expression + # Inferred type for the operator method type (when relevant). + method_type = None # type: mypy.types.Type + + def __init__(self, op: str, left: Expression, right: Expression) -> None: + self.op = op + self.left = left + self.right = right + self.literal = min(self.left.literal, self.right.literal) + self.literal_hash = ('Binary', op, left.literal_hash, right.literal_hash) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_op_expr(self) + + +class ComparisonExpr(Expression): + """Comparison expression (e.g. a < b > c < d).""" + + operators = None # type: List[str] + operands = None # type: List[Expression] + # Inferred type for the operator methods (when relevant; None for 'is'). + method_types = None # type: List[mypy.types.Type] + + def __init__(self, operators: List[str], operands: List[Expression]) -> None: + self.operators = operators + self.operands = operands + self.method_types = [] + self.literal = min(o.literal for o in self.operands) + self.literal_hash = (('Comparison',) + tuple(operators) + + tuple(o.literal_hash for o in operands)) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_comparison_expr(self) + + +class SliceExpr(Expression): + """Slice expression (e.g. 'x:y', 'x:', '::2' or ':'). + + This is only valid as index in index expressions. + """ + + begin_index = None # type: Optional[Expression] + end_index = None # type: Optional[Expression] + stride = None # type: Optional[Expression] + + def __init__(self, begin_index: Optional[Expression], + end_index: Optional[Expression], + stride: Optional[Expression]) -> None: + self.begin_index = begin_index + self.end_index = end_index + self.stride = stride + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_slice_expr(self) + + +class CastExpr(Expression): + """Cast expression cast(type, expr).""" + + expr = None # type: Expression + type = None # type: mypy.types.Type + + def __init__(self, expr: Expression, typ: 'mypy.types.Type') -> None: + self.expr = expr + self.type = typ + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_cast_expr(self) + + +class RevealTypeExpr(Expression): + """Reveal type expression reveal_type(expr).""" + + expr = None # type: Expression + + def __init__(self, expr: Expression) -> None: + self.expr = expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_reveal_type_expr(self) + + +class SuperExpr(Expression): + """Expression super().name""" + + name = '' + info = None # type: TypeInfo # Type that contains this super expression + + def __init__(self, name: str) -> None: + self.name = name + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_super_expr(self) + + +class FuncExpr(FuncItem, Expression): + """Lambda expression""" + + def name(self) -> str: + return '' + + def expr(self) -> Expression: + """Return the expression (the body) of the lambda.""" + ret = cast(ReturnStmt, self.body.body[-1]) + return ret.expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_func_expr(self) + + +class ListExpr(Expression): + """List literal expression [...].""" + + items = None # type: List[Expression] + + def __init__(self, items: List[Expression]) -> None: + self.items = items + if all(x.literal == LITERAL_YES for x in items): + self.literal = LITERAL_YES + self.literal_hash = ('List',) + tuple(x.literal_hash for x in items) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_list_expr(self) + + +class DictExpr(Expression): + """Dictionary literal expression {key: value, ...}.""" + + items = None # type: List[Tuple[Expression, Expression]] + + def __init__(self, items: List[Tuple[Expression, Expression]]) -> None: + self.items = items + if all(x[0].literal == LITERAL_YES and x[1].literal == LITERAL_YES + for x in items): + self.literal = LITERAL_YES + self.literal_hash = ('Dict',) + tuple( + (x[0].literal_hash, x[1].literal_hash) for x in items) # type: ignore + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_dict_expr(self) + + +class TupleExpr(Expression): + """Tuple literal expression (..., ...)""" + + items = None # type: List[Expression] + + def __init__(self, items: List[Expression]) -> None: + self.items = items + if all(x.literal == LITERAL_YES for x in items): + self.literal = LITERAL_YES + self.literal_hash = ('Tuple',) + tuple(x.literal_hash for x in items) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_tuple_expr(self) + + +class SetExpr(Expression): + """Set literal expression {value, ...}.""" + + items = None # type: List[Expression] + + def __init__(self, items: List[Expression]) -> None: + self.items = items + if all(x.literal == LITERAL_YES for x in items): + self.literal = LITERAL_YES + self.literal_hash = ('Set',) + tuple(x.literal_hash for x in items) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_set_expr(self) + + +class GeneratorExpr(Expression): + """Generator expression ... for ... in ... [ for ... in ... ] [ if ... ].""" + + left_expr = None # type: Expression + sequences_expr = None # type: List[Expression] + condlists = None # type: List[List[Expression]] + indices = None # type: List[Expression] + + def __init__(self, left_expr: Expression, indices: List[Expression], + sequences: List[Expression], condlists: List[List[Expression]]) -> None: + self.left_expr = left_expr + self.sequences = sequences + self.condlists = condlists + self.indices = indices + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_generator_expr(self) + + +class ListComprehension(Expression): + """List comprehension (e.g. [x + 1 for x in a])""" + + generator = None # type: GeneratorExpr + + def __init__(self, generator: GeneratorExpr) -> None: + self.generator = generator + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_list_comprehension(self) + + +class SetComprehension(Expression): + """Set comprehension (e.g. {x + 1 for x in a})""" + + generator = None # type: GeneratorExpr + + def __init__(self, generator: GeneratorExpr) -> None: + self.generator = generator + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_set_comprehension(self) + + +class DictionaryComprehension(Expression): + """Dictionary comprehension (e.g. {k: v for k, v in a}""" + + key = None # type: Expression + value = None # type: Expression + sequences_expr = None # type: List[Expression] + condlists = None # type: List[List[Expression]] + indices = None # type: List[Expression] + + def __init__(self, key: Expression, value: Expression, indices: List[Expression], + sequences: List[Expression], condlists: List[List[Expression]]) -> None: + self.key = key + self.value = value + self.sequences = sequences + self.condlists = condlists + self.indices = indices + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_dictionary_comprehension(self) + + +class ConditionalExpr(Expression): + """Conditional expression (e.g. x if y else z)""" + + cond = None # type: Expression + if_expr = None # type: Expression + else_expr = None # type: Expression + + def __init__(self, cond: Expression, if_expr: Expression, else_expr: Expression) -> None: + self.cond = cond + self.if_expr = if_expr + self.else_expr = else_expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_conditional_expr(self) + + +class BackquoteExpr(Expression): + """Python 2 expression `...`.""" + + expr = None # type: Expression + + def __init__(self, expr: Expression) -> None: + self.expr = expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_backquote_expr(self) + + +class TypeApplication(Expression): + """Type application expr[type, ...]""" + + expr = None # type: Expression + types = None # type: List[mypy.types.Type] + + def __init__(self, expr: Expression, types: List['mypy.types.Type']) -> None: + self.expr = expr + self.types = types + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_type_application(self) + + +# Variance of a type variable. For example, T in the definition of +# List[T] is invariant, so List[int] is not a subtype of List[object], +# and also List[object] is not a subtype of List[int]. +# +# The T in Iterable[T] is covariant, so Iterable[int] is a subtype of +# Iterable[object], but not vice versa. +# +# If T is contravariant in Foo[T], Foo[object] is a subtype of +# Foo[int], but not vice versa. +INVARIANT = 0 # type: int +COVARIANT = 1 # type: int +CONTRAVARIANT = 2 # type: int + + +class TypeVarExpr(SymbolNode, Expression): + """Type variable expression TypeVar(...).""" + + _name = '' + _fullname = '' + # Value restriction: only types in the list are valid as values. If the + # list is empty, there is no restriction. + values = None # type: List[mypy.types.Type] + # Upper bound: only subtypes of upper_bound are valid as values. By default + # this is 'object', meaning no restriction. + upper_bound = None # type: mypy.types.Type + # Variance of the type variable. Invariant is the default. + # TypeVar(..., covariant=True) defines a covariant type variable. + # TypeVar(..., contravariant=True) defines a contravariant type + # variable. + variance = INVARIANT + + def __init__(self, name: str, fullname: str, + values: List['mypy.types.Type'], + upper_bound: 'mypy.types.Type', + variance: int=INVARIANT) -> None: + self._name = name + self._fullname = fullname + self.values = values + self.upper_bound = upper_bound + self.variance = variance + + def name(self) -> str: + return self._name + + def fullname(self) -> str: + return self._fullname + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_type_var_expr(self) + + def serialize(self) -> JsonDict: + return {'.class': 'TypeVarExpr', + 'name': self._name, + 'fullname': self._fullname, + 'values': [t.serialize() for t in self.values], + 'upper_bound': self.upper_bound.serialize(), + 'variance': self.variance, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'TypeVarExpr': + assert data['.class'] == 'TypeVarExpr' + return TypeVarExpr(data['name'], + data['fullname'], + [mypy.types.Type.deserialize(v) for v in data['values']], + mypy.types.Type.deserialize(data['upper_bound']), + data['variance']) + + +class TypeAliasExpr(Expression): + """Type alias expression (rvalue).""" + + type = None # type: mypy.types.Type + + def __init__(self, type: 'mypy.types.Type') -> None: + self.type = type + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_type_alias_expr(self) + + +class NamedTupleExpr(Expression): + """Named tuple expression namedtuple(...).""" + + # The class representation of this named tuple (its tuple_type attribute contains + # the tuple item types) + info = None # type: TypeInfo + + def __init__(self, info: 'TypeInfo') -> None: + self.info = info + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_namedtuple_expr(self) + + +class PromoteExpr(Expression): + """Ducktype class decorator expression _promote(...).""" + + type = None # type: mypy.types.Type + + def __init__(self, type: 'mypy.types.Type') -> None: + self.type = type + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit__promote_expr(self) + + +# Constants + + +class TempNode(Expression): + """Temporary dummy node used during type checking. + + This node is not present in the original program; it is just an artifact + of the type checker implementation. It only represents an opaque node with + some fixed type. + """ + + type = None # type: mypy.types.Type + + def __init__(self, typ: 'mypy.types.Type') -> None: + self.type = typ + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_temp_node(self) + + +class TypeInfo(SymbolNode): + """The type structure of a single class. + + Each TypeInfo corresponds one-to-one to a ClassDef, which + represents the AST of the class. + + In type-theory terms, this is a "type constructor", and if the + class is generic then it will be a type constructor of higher kind. + Where the class is used in an actual type, it's in the form of an + Instance, which amounts to a type application of the tycon to + the appropriate number of arguments. + """ + + _fullname = None # type: str # Fully qualified name + defn = None # type: ClassDef # Corresponding ClassDef + # Method Resolution Order: the order of looking up attributes. The first + # value always to refers to this class. + mro = None # type: List[TypeInfo] + subtypes = None # type: Set[TypeInfo] # Direct subclasses encountered so far + names = None # type: SymbolTable # Names defined directly in this type + is_abstract = False # Does the class have any abstract attributes? + abstract_attributes = None # type: List[str] + # Classes inheriting from Enum shadow their true members with a __getattr__, so we + # have to treat them as a special case. + is_enum = False + # If true, any unknown attributes should have type 'Any' instead + # of generating a type error. This would be true if there is a + # base class with type 'Any', but other use cases may be + # possible. This is similar to having __getattr__ that returns Any + # (and __setattr__), but without the __getattr__ method. + fallback_to_any = False + + # Information related to type annotations. + + # Generic type variable names + type_vars = None # type: List[str] + + # Direct base classes. + bases = None # type: List[mypy.types.Instance] + + # Another type which this type will be treated as a subtype of, + # even though it's not a subclass in Python. The non-standard + # `@_promote` decorator introduces this, and there are also + # several builtin examples, in particular `int` -> `float`. + _promote = None # type: mypy.types.Type + + # Representation of a Tuple[...] base class, if the class has any + # (e.g., for named tuples). If this is not None, the actual Type + # object used for this class is not an Instance but a TupleType; + # the corresponding Instance is set as the fallback type of the + # tuple type. + tuple_type = None # type: mypy.types.TupleType + + # Is this a named tuple type? + is_named_tuple = False + + # Is this a dummy from deserialization? + is_dummy = False + + # Alternative to fullname() for 'anonymous' classes. + alt_fullname = None # type: Optional[str] + + def __init__(self, names: 'SymbolTable', defn: ClassDef) -> None: + """Initialize a TypeInfo.""" + self.names = names + self.defn = defn + self.subtypes = set() + self.type_vars = [] + self.bases = [] + # Leave self.mro uninitialized until we compute it for real, + # so we don't accidentally try to use it prematurely. + self._fullname = defn.fullname + self.is_abstract = False + self.abstract_attributes = [] + if defn.type_vars: + for vd in defn.type_vars: + self.type_vars.append(vd.name) + + def name(self) -> str: + """Short name.""" + return self.defn.name + + def fullname(self) -> str: + return self._fullname + + def is_generic(self) -> bool: + """Is the type generic (i.e. does it have type variables)?""" + return len(self.type_vars) > 0 + + def get(self, name: str) -> 'SymbolTableNode': + for cls in self.mro: + n = cls.names.get(name) + if n: + return n + return None + + def __getitem__(self, name: str) -> 'SymbolTableNode': + n = self.get(name) + if n: + return n + else: + raise KeyError(name) + + def __repr__(self) -> str: + return '' % self.fullname() + + # IDEA: Refactor the has* methods to be more consistent and document + # them. + + def has_readable_member(self, name: str) -> bool: + return self.get(name) is not None + + def has_writable_member(self, name: str) -> bool: + return self.has_var(name) + + def has_var(self, name: str) -> bool: + return self.get_var(name) is not None + + def has_method(self, name: str) -> bool: + return self.get_method(name) is not None + + def get_var(self, name: str) -> Var: + for cls in self.mro: + if name in cls.names: + node = cls.names[name].node + if isinstance(node, Var): + return node + else: + return None + return None + + def get_var_or_getter(self, name: str) -> SymbolNode: + # TODO getter + return self.get_var(name) + + def get_var_or_setter(self, name: str) -> SymbolNode: + # TODO setter + return self.get_var(name) + + def get_method(self, name: str) -> FuncBase: + if self.mro is None: # Might be because of a previous error. + return None + for cls in self.mro: + if name in cls.names: + node = cls.names[name].node + if isinstance(node, FuncBase): + return node + else: + return None + return None + + def calculate_mro(self) -> None: + """Calculate and set mro (method resolution order). + + Raise MroError if cannot determine mro. + """ + mro = linearize_hierarchy(self) + assert mro, "Could not produce a MRO at all for %s" % (self,) + self.mro = mro + self.is_enum = self._calculate_is_enum() + + def _calculate_is_enum(self) -> bool: + """ + If this is "enum.Enum" itself, then yes, it's an enum. + If the flag .is_enum has been set on anything in the MRO, it's an enum. + """ + if self.fullname() == ENUM_BASECLASS: + return True + if self.mro: + return any(type_info.is_enum for type_info in self.mro) + return False + + def has_base(self, fullname: str) -> bool: + """Return True if type has a base type with the specified name. + + This can be either via extension or via implementation. + """ + if self.mro: + for cls in self.mro: + if cls.fullname() == fullname: + return True + return False + + def all_subtypes(self) -> 'Set[TypeInfo]': + """Return TypeInfos of all subtypes, including this type, as a set.""" + subtypes = set([self]) + for subt in self.subtypes: + for t in subt.all_subtypes(): + subtypes.add(t) + return subtypes + + def all_base_classes(self) -> 'List[TypeInfo]': + """Return a list of base classes, including indirect bases.""" + assert False + + def direct_base_classes(self) -> 'List[TypeInfo]': + """Return a direct base classes. + + Omit base classes of other base classes. + """ + return [base.type for base in self.bases] + + def __str__(self) -> str: + """Return a string representation of the type. + + This includes the most important information about the type. + """ + base = None # type: str + if self.bases: + base = 'Bases({})'.format(', '.join(str(base) + for base in self.bases)) + return dump_tagged(['Name({})'.format(self.fullname()), + base, + ('Names', sorted(self.names.keys()))], + 'TypeInfo') + + def serialize(self) -> Union[str, JsonDict]: + # NOTE: This is where all ClassDefs originate, so there shouldn't be duplicates. + data = {'.class': 'TypeInfo', + 'fullname': self.fullname(), + 'alt_fullname': self.alt_fullname, + 'names': self.names.serialize(self.alt_fullname or self.fullname()), + 'defn': self.defn.serialize(), + 'is_abstract': self.is_abstract, + 'abstract_attributes': self.abstract_attributes, + 'is_enum': self.is_enum, + 'fallback_to_any': self.fallback_to_any, + 'type_vars': self.type_vars, + 'bases': [b.serialize() for b in self.bases], + '_promote': None if self._promote is None else self._promote.serialize(), + 'tuple_type': None if self.tuple_type is None else self.tuple_type.serialize(), + 'is_named_tuple': self.is_named_tuple, + } + return data + + @classmethod + def deserialize(cls, data: JsonDict) -> 'TypeInfo': + names = SymbolTable.deserialize(data['names']) + defn = ClassDef.deserialize(data['defn']) + ti = TypeInfo(names, defn) + ti._fullname = data['fullname'] + ti.alt_fullname = data['alt_fullname'] + # TODO: Is there a reason to reconstruct ti.subtypes? + ti.is_abstract = data['is_abstract'] + ti.abstract_attributes = data['abstract_attributes'] + ti.is_enum = data['is_enum'] + ti.fallback_to_any = data['fallback_to_any'] + ti.type_vars = data['type_vars'] + ti.bases = [mypy.types.Instance.deserialize(b) for b in data['bases']] + ti._promote = (None if data['_promote'] is None + else mypy.types.Type.deserialize(data['_promote'])) + ti.tuple_type = (None if data['tuple_type'] is None + else mypy.types.TupleType.deserialize(data['tuple_type'])) + ti.is_named_tuple = data['is_named_tuple'] + return ti + + +def NamedTupleTypeInfo(tup: 'mypy.types.NamedTupleType', *args) -> TypeInfo: + info = TypeInfo(*args) + info.tuple_type = tup + info.bases = [tup.fallback] + info.is_named_tuple = True + info.mro = [info] + tup.fallback.type.mro + return info + +class SymbolTableNode: + # Kind of node. Possible values: + # - LDEF: local definition (of any kind) + # - GDEF: global (module-level) definition + # - MDEF: class member definition + # - UNBOUND_TVAR: TypeVar(...) definition, not bound + # - TVAR: type variable in a bound scope (generic function / generic clas) + # - MODULE_REF: reference to a module + # - TYPE_ALIAS: type alias + # - UNBOUND_IMPORTED: temporary kind for imported names + kind = None # type: int + # AST node of definition (FuncDef/Var/TypeInfo/Decorator/TypeVarExpr, + # or None for a bound type variable). + node = None # type: Optional[SymbolNode] + # Type variable definition (for bound type variables only) + tvar_def = None # type: Optional[mypy.types.TypeVarDef] + # Module id (e.g. "foo.bar") or None + mod_id = '' + # If this not None, override the type of the 'node' attribute. + type_override = None # type: Optional[mypy.types.Type] + # If False, this name won't be imported via 'from import *'. + # This has no effect on names within classes. + module_public = True + # For deserialized MODULE_REF nodes, the referenced module name; + # for other nodes, optionally the name of the referenced object. + cross_ref = None # type: Optional[str] + + def __init__(self, kind: int, node: Optional[SymbolNode], mod_id: str = None, + typ: 'mypy.types.Type' = None, + tvar_def: 'mypy.types.TypeVarDef' = None, + module_public: bool = True) -> None: + self.kind = kind + self.node = node + self.type_override = typ + self.mod_id = mod_id + self.tvar_def = tvar_def + self.module_public = module_public + + @property + def fullname(self) -> str: + if self.node is not None: + return self.node.fullname() + else: + return None + + @property + def type(self) -> 'mypy.types.Type': + # IDEA: Get rid of the Any type. + node = self.node # type: Any + if self.type_override is not None: + return self.type_override + elif ((isinstance(node, Var) or isinstance(node, FuncDef)) + and node.type is not None): + return node.type + elif isinstance(node, Decorator): + return node.var.type + else: + return None + + def __str__(self) -> str: + s = '{}/{}'.format(node_kinds[self.kind], short_type(self.node)) + if self.mod_id is not None: + s += ' ({})'.format(self.mod_id) + # Include declared type of variables and functions. + if self.type is not None: + s += ' : {}'.format(self.type) + return s + + def serialize(self, prefix: str, name: str) -> JsonDict: + """Serialize a SymbolTableNode. + + Args: + prefix: full name of the containing module or class; or None + name: name of this object relative to the containing object + """ + data = {'.class': 'SymbolTableNode', + 'kind': node_kinds[self.kind], + } # type: JsonDict + if self.tvar_def: + data['tvar_def'] = self.tvar_def.serialize() + if not self.module_public: + data['module_public'] = False + if self.kind == MODULE_REF: + assert self.node is not None, "Missing module cross ref in %s for %s" % (prefix, name) + data['cross_ref'] = self.node.fullname() + else: + if self.node is not None: + if prefix is not None: + # Check whether this is an alias for another object. + # If the object's canonical full name differs from + # the full name computed from prefix and name, + # it's an alias, and we serialize it as a cross ref. + if isinstance(self.node, TypeInfo): + fullname = self.node.alt_fullname or self.node.fullname() + else: + fullname = self.node.fullname() + if (fullname is not None and '.' in fullname and + fullname != prefix + '.' + name): + data['cross_ref'] = fullname + return data + data['node'] = self.node.serialize() + if self.type_override is not None: + data['type_override'] = self.type_override.serialize() + return data + + @classmethod + def deserialize(cls, data: JsonDict) -> 'SymbolTableNode': + assert data['.class'] == 'SymbolTableNode' + kind = inverse_node_kinds[data['kind']] + if 'cross_ref' in data: + # This will be fixed up later. + stnode = SymbolTableNode(kind, None) + stnode.cross_ref = data['cross_ref'] + else: + node = None + if 'node' in data: + node = SymbolNode.deserialize(data['node']) + typ = None + if 'type_override' in data: + typ = mypy.types.Type.deserialize(data['type_override']) + stnode = SymbolTableNode(kind, node, typ=typ) + if 'tvar_def' in data: + stnode.tvar_def = mypy.types.TypeVarDef.deserialize(data['tvar_def']) + if 'module_public' in data: + stnode.module_public = data['module_public'] + return stnode + + +class SymbolTable(Dict[str, SymbolTableNode]): + def __str__(self) -> str: + a = [] # type: List[str] + for key, value in self.items(): + # Filter out the implicit import of builtins. + if isinstance(value, SymbolTableNode): + if (value.fullname != 'builtins' and + (value.fullname or '').split('.')[-1] not in + implicit_module_attrs): + a.append(' ' + str(key) + ' : ' + str(value)) + else: + a.append(' ') + a = sorted(a) + a.insert(0, 'SymbolTable(') + a[-1] += ')' + return '\n'.join(a) + + def serialize(self, fullname: str) -> JsonDict: + data = {'.class': 'SymbolTable'} # type: JsonDict + for key, value in self.items(): + # Skip __builtins__: it's a reference to the builtins + # module that gets added to every module by + # SemanticAnalyzer.visit_file(), but it shouldn't be + # accessed by users of the module. + if key == '__builtins__': + continue + data[key] = value.serialize(fullname, key) + return data + + @classmethod + def deserialize(cls, data: JsonDict) -> 'SymbolTable': + assert data['.class'] == 'SymbolTable' + st = SymbolTable() + for key, value in data.items(): + if key != '.class': + st[key] = SymbolTableNode.deserialize(value) + return st + + +def function_type(func: FuncBase, fallback: 'mypy.types.Instance') -> 'mypy.types.FunctionLike': + if func.type: + assert isinstance(func.type, mypy.types.FunctionLike) + return func.type + else: + # Implicit type signature with dynamic types. + # Overloaded functions always have a signature, so func must be an ordinary function. + fdef = cast(FuncDef, func) + name = func.name() + if name: + name = '"{}"'.format(name) + names = [] # type: List[str] + for arg in fdef.arguments: + names.append(arg.variable.name()) + + return mypy.types.CallableType( + [mypy.types.AnyType()] * len(fdef.arguments), + [arg.kind for arg in fdef.arguments], + names, + mypy.types.AnyType(), + fallback, + name, + implicit=True, + ) + + +def method_type_with_fallback(func: FuncBase, + fallback: 'mypy.types.Instance') -> 'mypy.types.FunctionLike': + """Return the signature of a method (omit self).""" + return method_type(function_type(func, fallback)) + + +def method_type(sig: 'mypy.types.FunctionLike') -> 'mypy.types.FunctionLike': + if isinstance(sig, mypy.types.CallableType): + return method_callable(sig) + else: + sig = cast(mypy.types.Overloaded, sig) + items = [] # type: List[mypy.types.CallableType] + for c in sig.items(): + items.append(method_callable(c)) + return mypy.types.Overloaded(items) + + +def method_callable(c: 'mypy.types.CallableType') -> 'mypy.types.CallableType': + if c.arg_kinds and c.arg_kinds[0] == ARG_STAR: + # The signature is of the form 'def foo(*args, ...)'. + # In this case we shouldn't drop the first arg, + # since self will be absorbed by the *args. + return c + return c.copy_modified(arg_types=c.arg_types[1:], + arg_kinds=c.arg_kinds[1:], + arg_names=c.arg_names[1:]) + + +class MroError(Exception): + """Raised if a consistent mro cannot be determined for a class.""" + + +def linearize_hierarchy(info: TypeInfo) -> List[TypeInfo]: + # TODO describe + if info.mro: + return info.mro + bases = info.direct_base_classes() + lin_bases = [] + for base in bases: + assert base is not None, "Cannot linearize bases for %s %s" % (info.fullname(), bases) + lin_bases.append(linearize_hierarchy(base)) + lin_bases.append(bases) + return [info] + merge(lin_bases) + + +def merge(seqs: List[List[TypeInfo]]) -> List[TypeInfo]: + seqs = [s[:] for s in seqs] + result = [] # type: List[TypeInfo] + while True: + seqs = [s for s in seqs if s] + if not seqs: + return result + for seq in seqs: + head = seq[0] + if not [s for s in seqs if head in s[1:]]: + break + else: + raise MroError() + result.append(head) + for s in seqs: + if s[0] is head: + del s[0] diff --git a/mypy/parse.py b/mypy/parse.py index a496f32bebcd3..579f03c57525f 100644 --- a/mypy/parse.py +++ b/mypy/parse.py @@ -1,2034 +1,2034 @@ -"""Mypy parser. - -Constructs a parse tree (abstract syntax tree) based on a string -representing a source file. Performs only minimal semantic checks. -""" - -import re - -from typing import List, Tuple, Any, Set, cast, Union, Optional - -from mypy import lex, docstring -from mypy.lex import ( - Token, Eof, Bom, Break, Name, Colon, Dedent, IntLit, StrLit, BytesLit, - UnicodeLit, FloatLit, Op, Indent, Keyword, Punct, LexError, ComplexLit, - EllipsisToken -) -import mypy.types -from mypy.nodes import ( - MypyFile, Import, Node, ImportAll, ImportFrom, FuncDef, OverloadedFuncDef, - ClassDef, Decorator, Block, Var, OperatorAssignmentStmt, - ExpressionStmt, AssignmentStmt, ReturnStmt, RaiseStmt, AssertStmt, - DelStmt, BreakStmt, ContinueStmt, PassStmt, GlobalDecl, - WhileStmt, ForStmt, IfStmt, TryStmt, WithStmt, - TupleExpr, GeneratorExpr, ListComprehension, ListExpr, ConditionalExpr, - DictExpr, SetExpr, NameExpr, IntExpr, StrExpr, BytesExpr, UnicodeExpr, - FloatExpr, CallExpr, SuperExpr, MemberExpr, IndexExpr, SliceExpr, OpExpr, - UnaryExpr, FuncExpr, PrintStmt, ImportBase, ComparisonExpr, - StarExpr, YieldFromExpr, NonlocalDecl, DictionaryComprehension, - SetComprehension, ComplexExpr, EllipsisExpr, YieldExpr, ExecStmt, Argument, - BackquoteExpr -) -from mypy import defaults -from mypy import nodes -from mypy.errors import Errors, CompileError -from mypy.types import Type, CallableType, AnyType, UnboundType -from mypy.parsetype import ( - parse_type, parse_types, parse_signature, TypeParseError, parse_str_as_signature -) -from mypy.options import Options - -from mypy import experiments - - -precedence = { - '**': 16, - '-u': 15, '+u': 15, '~': 15, # unary operators (-, + and ~) - '': 14, - '*': 13, '/': 13, '//': 13, '%': 13, - '+': 12, '-': 12, - '>>': 11, '<<': 11, - '&': 10, - '^': 9, - '|': 8, - '==': 7, '!=': 7, '<': 7, '>': 7, '<=': 7, '>=': 7, 'is': 7, 'in': 7, - '*u': 7, # unary * for star expressions - 'not': 6, - 'and': 5, - 'or': 4, - '': 3, # conditional expression - '': 2, # list comprehension - ',': 1} - - -op_assign = set([ - '+=', '-=', '*=', '/=', '//=', '%=', '**=', '|=', '&=', '^=', '>>=', - '<<=']) - -op_comp = set([ - '>', '<', '==', '>=', '<=', '<>', '!=', 'is', 'is', 'in', 'not']) - -none = Token('') # Empty token - - -def parse(source: Union[str, bytes], - fnam: str, - errors: Errors, - options: Options) -> MypyFile: - """Parse a source file, without doing any semantic analysis. - - Return the parse tree. If errors is not provided, raise ParseError - on failure. Otherwise, use the errors object to report parse errors. - - The python_version (major, minor) option determines the Python syntax variant. - """ - if options.fast_parser: - import mypy.fastparse - return mypy.fastparse.parse(source, - fnam=fnam, - errors=errors, - pyversion=options.python_version, - custom_typing_module=options.custom_typing_module) - - is_stub_file = bool(fnam) and fnam.endswith('.pyi') - parser = Parser(fnam, - errors, - options.python_version, - options.custom_typing_module, - is_stub_file=is_stub_file) - tree = parser.parse(source) - tree.path = fnam - tree.is_stub = is_stub_file - return tree - - -class Parser: - """Mypy parser that parses a string into an AST. - - Parses type annotations in addition to basic Python syntax. It supports both Python 2 and 3 - (though Python 2 support is incomplete). - - The AST classes are defined in mypy.nodes and mypy.types. - """ - - tok = None # type: List[Token] - ind = 0 - errors = None # type: Errors - # If True, raise an exception on any parse error. Otherwise, errors are reported via 'errors'. - raise_on_error = False - - # Are we currently parsing the body of a class definition? - is_class_body = False - # All import nodes encountered so far in this parse unit. - imports = None # type: List[ImportBase] - # Names imported from __future__. - future_options = None # type: List[str] - # Lines to ignore (using # type: ignore). - ignored_lines = None # type: Set[int] - - def __init__(self, fnam: str, errors: Errors, pyversion: Tuple[int, int], - custom_typing_module: str = None, is_stub_file: bool = False) -> None: - self.raise_on_error = errors is None - self.pyversion = pyversion - self.custom_typing_module = custom_typing_module - self.is_stub_file = is_stub_file - if errors is not None: - self.errors = errors - else: - self.errors = Errors() - if fnam is not None: - self.errors.set_file(fnam) - else: - self.errors.set_file('') - - def parse(self, s: Union[str, bytes]) -> MypyFile: - self.tok, self.ignored_lines = lex.lex(s, pyversion=self.pyversion, - is_stub_file=self.is_stub_file) - self.ind = 0 - self.imports = [] - self.future_options = [] - file = self.parse_file() - if self.raise_on_error and self.errors.is_errors(): - self.errors.raise_error() - return file - - def weak_opts(self) -> Set[str]: - """Do weak typing if any of the first ten tokens is a comment saying so. - - The comment can be one of: - # mypy: weak=global - # mypy: weak=local - # mypy: weak <- defaults to local - """ - regexp = re.compile(r'^[\s]*# *mypy: *weak(=?)([^\s]*)', re.M) - for t in self.tok[:10]: - for s in [t.string, t.pre]: - m = regexp.search(s) - if m: - opts = set(x for x in m.group(2).split(',') if x) - if not opts: - opts.add('local') - return opts - return set() - - def parse_file(self) -> MypyFile: - """Parse a mypy source file.""" - is_bom = self.parse_bom() - defs = self.parse_defs() - weak_opts = self.weak_opts() - self.expect_type(Eof) - node = MypyFile(defs, self.imports, is_bom, self.ignored_lines, - weak_opts=weak_opts) - return node - - # Parse the initial part - - def parse_bom(self) -> bool: - """Parse the optional byte order mark at the beginning of a file.""" - if isinstance(self.current(), Bom): - self.expect_type(Bom) - if isinstance(self.current(), Break): - self.expect_break() - return True - else: - return False - - def parse_import(self) -> Import: - self.expect('import') - ids = [] - while True: - id = self.parse_qualified_name() - translated = self.translate_module_id(id) - as_id = None - if self.current_str() == 'as': - self.expect('as') - name_tok = self.expect_type(Name) - as_id = name_tok.string - elif translated != id: - as_id = id - ids.append((translated, as_id)) - if self.current_str() != ',': - break - self.expect(',') - node = Import(ids) - self.imports.append(node) - return node - - def translate_module_id(self, id: str) -> str: - """Return the actual, internal module id for a source text id. - - For example, translate '__builtin__' in Python 2 to 'builtins'. - """ - if id == self.custom_typing_module: - return 'typing' - elif id == '__builtin__' and self.pyversion[0] == 2: - # HACK: __builtin__ in Python 2 is aliases to builtins. However, the implementation - # is named __builtin__.py (there is another layer of translation elsewhere). - return 'builtins' - return id - - def parse_import_from(self) -> Node: - self.expect('from') - - # Build the list of beginning relative tokens. - relative = 0 - while self.current_str() in (".", "..."): - relative += len(self.current_str()) - self.skip() - - # Parse qualified name to actually import from. - if self.current_str() == "import": - # Empty/default values. - name = "" - else: - name = self.parse_qualified_name() - - name = self.translate_module_id(name) - - # Parse import list - self.expect('import') - node = None # type: ImportBase - if self.current_str() == '*': - if name == '__future__': - self.parse_error() - # An import all from a module node: - self.skip() - node = ImportAll(name, relative) - else: - is_paren = self.current_str() == '(' - if is_paren: - self.expect('(') - targets = [] # type: List[Tuple[str, str]] - while True: - id, as_id = self.parse_import_name() - if '%s.%s' % (name, id) == self.custom_typing_module: - if targets or self.current_str() == ',': - self.fail('You cannot import any other modules when you ' - 'import a custom typing module', - self.current().line) - node = Import([('typing', as_id)]) - self.skip_until_break() - break - targets.append((id, as_id)) - if self.current_str() != ',': - break - self.expect(',') - if is_paren and self.current_str() == ')': - break - if is_paren: - self.expect(')') - if node is None: - node = ImportFrom(name, relative, targets) - self.imports.append(node) - if name == '__future__': - self.future_options.extend(target[0] for target in targets) - return node - - def parse_import_name(self) -> Tuple[str, Optional[str]]: - tok = self.expect_type(Name) - name = tok.string - if self.current_str() == 'as': - self.skip() - as_name = self.expect_type(Name) - return name, as_name.string - else: - return name, None - - def parse_qualified_name(self) -> str: - """Parse a name with an optional module qualifier. - - Return a tuple with the name as a string and a token array - containing all the components of the name. - """ - tok = self.expect_type(Name) - n = tok.string - while self.current_str() == '.': - self.expect('.') - tok = self.expect_type(Name) - n += '.' + tok.string - return n - - # Parsing global definitions - - def parse_defs(self) -> List[Node]: - defs = [] # type: List[Node] - while not self.eof(): - try: - defn, is_simple = self.parse_statement() - if is_simple: - self.expect_break() - if defn is not None: - if not self.try_combine_overloads(defn, defs): - defs.append(defn) - except ParseError: - pass - return defs - - def parse_class_def(self) -> ClassDef: - old_is_class_body = self.is_class_body - self.is_class_body = True - - self.expect('class') - metaclass = None - - try: - base_types = [] # type: List[Node] - try: - name_tok = self.expect_type(Name) - name = name_tok.string - - self.errors.push_type(name) - - if self.current_str() == '(': - self.skip() - while True: - if self.current_str() == ')': - break - if self.peek().string == '=': - metaclass = self.parse_class_keywords() - break - base_types.append(self.parse_super_type()) - if self.current_str() != ',': - break - self.skip() - self.expect(')') - except ParseError: - pass - - defs, _ = self.parse_block() - - node = ClassDef(name, defs, None, base_types, metaclass=metaclass) - return node - finally: - self.errors.pop_type() - self.is_class_body = old_is_class_body - - def parse_class_keywords(self) -> Optional[str]: - """Parse the class keyword arguments, keeping the metaclass but - ignoring all others. Returns None if the metaclass isn't found. - """ - metaclass = None - while True: - key = self.expect_type(Name) - self.expect('=') - if key.string == 'metaclass': - metaclass = self.parse_qualified_name() - else: - # skip the class value - self.parse_expression(precedence[',']) - if self.current_str() != ',': - break - self.skip() - if self.current_str() == ')': - break - return metaclass - - def parse_super_type(self) -> Node: - return self.parse_expression(precedence[',']) - - def parse_decorated_function_or_class(self) -> Node: - decorators = [] - no_type_checks = False - while self.current_str() == '@': - self.expect('@') - d_exp = self.parse_expression() - if self.is_no_type_check_decorator(d_exp): - no_type_checks = True - decorators.append(d_exp) - self.expect_break() - if self.current_str() != 'class': - func = self.parse_function(no_type_checks) - func.is_decorated = True - var = Var(func.name()) - # Types of decorated functions must always be inferred. - var.is_ready = False - var.set_line(decorators[0].line) - node = Decorator(func, decorators, var) - return node - else: - cls = self.parse_class_def() - cls.decorators = decorators - return cls - - def is_no_type_check_decorator(self, expr: Node) -> bool: - if isinstance(expr, NameExpr): - return expr.name == 'no_type_check' - elif isinstance(expr, MemberExpr): - if isinstance(expr.expr, NameExpr): - return expr.expr.name == 'typing' and expr.name == 'no_type_check' - else: - return False - - def parse_function(self, no_type_checks: bool=False) -> FuncDef: - def_tok = self.expect('def') - is_method = self.is_class_body - self.is_class_body = False - try: - (name, args, typ, is_error, extra_stmts) = self.parse_function_header(no_type_checks) - - arg_kinds = [arg.kind for arg in args] - arg_names = [arg.variable.name() for arg in args] - - body, comment_type = self.parse_block(allow_type=True) - # Potentially insert extra assignment statements to the beginning of the - # body, used to decompose Python 2 tuple arguments. - body.body[:0] = extra_stmts - if comment_type: - # The function has a # type: ... signature. - if typ: - self.errors.report( - def_tok.line, 'Function has duplicate type signatures') - sig = cast(CallableType, comment_type) - if sig.is_ellipsis_args: - # When we encounter an ellipsis, fill in the arg_types with - # a bunch of AnyTypes, emulating Callable[..., T] - arg_types = [AnyType()] * len(arg_kinds) # type: List[Type] - typ = CallableType( - arg_types, - arg_kinds, - arg_names, - sig.ret_type, - None, - line=def_tok.line) - elif is_method and len(sig.arg_kinds) < len(arg_kinds): - self.check_argument_kinds(arg_kinds, - [nodes.ARG_POS] + sig.arg_kinds, - def_tok.line) - # Add implicit 'self' argument to signature. - first_arg = [AnyType()] # type: List[Type] - typ = CallableType( - first_arg + sig.arg_types, - arg_kinds, - arg_names, - sig.ret_type, - None, - line=def_tok.line) - else: - self.check_argument_kinds(arg_kinds, sig.arg_kinds, - def_tok.line) - typ = CallableType( - sig.arg_types, - arg_kinds, - arg_names, - sig.ret_type, - None, - line=def_tok.line) - - # If there was a serious error, we really cannot build a parse tree - # node. - if is_error: - return None - - if typ and isinstance(typ.ret_type, UnboundType): - typ.ret_type.is_ret_type = True - - node = FuncDef(name, args, body, typ) - node.set_line(def_tok) - if typ is not None: - typ.definition = node - return node - finally: - self.errors.pop_function() - self.is_class_body = is_method - - def check_argument_kinds(self, funckinds: List[int], sigkinds: List[int], - line: int) -> None: - """Check that arguments are consistent. - - This verifies that they have the same number and the kinds correspond. - - Arguments: - funckinds: kinds of arguments in function definition - sigkinds: kinds of arguments in signature (after # type:) - """ - if len(funckinds) != len(sigkinds): - if len(funckinds) > len(sigkinds): - self.fail("Type signature has too few arguments", line) - else: - self.fail("Type signature has too many arguments", line) - return - for kind, token in [(nodes.ARG_STAR, '*'), - (nodes.ARG_STAR2, '**')]: - if ((funckinds.count(kind) != sigkinds.count(kind)) or - (kind in funckinds and sigkinds.index(kind) != funckinds.index(kind))): - self.fail( - "Inconsistent use of '{}' in function " - "signature".format(token), line) - - def parse_function_header( - self, no_type_checks: bool=False) -> Tuple[str, - List[Argument], - CallableType, - bool, - List[AssignmentStmt]]: - """Parse function header (a name followed by arguments) - - Return a 5-tuple with the following items: - name - arguments - signature (annotation) - error flag (True if error) - extra statements needed to decompose arguments (usually empty) - - See parse_arg_list for an explanation of the final tuple item. - """ - name = '' - - try: - name_tok = self.expect_type(Name) - name = name_tok.string - - self.errors.push_function(name) - - args, typ, extra_stmts = self.parse_args(no_type_checks) - except ParseError: - if not isinstance(self.current(), Break): - self.ind -= 1 # Kludge: go back to the Break token - # Resynchronise parsing by going back over :, if present. - if isinstance(self.tok[self.ind - 1], Colon): - self.ind -= 1 - return (name, [], None, True, []) - - return (name, args, typ, False, extra_stmts) - - def parse_args(self, no_type_checks: bool=False) -> Tuple[List[Argument], - CallableType, - List[AssignmentStmt]]: - """Parse a function signature (...) [-> t]. - - See parse_arg_list for an explanation of the final tuple item. - """ - lparen = self.expect('(') - - # Parse the argument list (everything within '(' and ')'). - args, extra_stmts = self.parse_arg_list(no_type_checks=no_type_checks) - - self.expect(')') - - if self.current_str() == '->': - self.skip() - if no_type_checks: - self.parse_expression() - ret_type = None - else: - ret_type = self.parse_type() - else: - ret_type = None - - arg_kinds = [arg.kind for arg in args] - self.verify_argument_kinds(arg_kinds, lparen.line) - - annotation = self.build_func_annotation( - ret_type, args, lparen.line) - - return args, annotation, extra_stmts - - def build_func_annotation(self, ret_type: Type, args: List[Argument], - line: int, is_default_ret: bool = False) -> CallableType: - arg_types = [arg.type_annotation for arg in args] - # Are there any type annotations? - if ((ret_type and not is_default_ret) - or arg_types != [None] * len(arg_types)): - # Yes. Construct a type for the function signature. - return self.construct_function_type(args, ret_type, line) - else: - return None - - def parse_arg_list(self, allow_signature: bool = True, - no_type_checks: bool=False) -> Tuple[List[Argument], - List[AssignmentStmt]]: - """Parse function definition argument list. - - This includes everything between '(' and ')' (but not the - parentheses). - - Return tuple (arguments, - extra statements for decomposing arguments). - - The final argument is only used for Python 2 argument lists with - tuples; they contain destructuring assignment statements used to - decompose tuple arguments. For example, consider a header like this: - - . def f((x, y)) - - The actual (sole) argument will be __tuple_arg_1 (a generated - name), whereas the extra statement list will contain a single - assignment statement corresponding to this assignment: - - x, y = __tuple_arg_1 - """ - args = [] # type: List[Argument] - extra_stmts = [] - # This is for checking duplicate argument names. - arg_names = [] # type: List[str] - has_tuple_arg = False - - require_named = False - bare_asterisk_before = -1 - - if self.current_str() != ')' and self.current_str() != ':': - while self.current_str() != ')': - if self.current_str() == '*' and self.peek().string == ',': - self.expect('*') - require_named = True - bare_asterisk_before = len(args) - elif self.current_str() in ['*', '**']: - if bare_asterisk_before == len(args): - # named arguments must follow bare * - self.parse_error() - - arg = self.parse_asterisk_arg( - allow_signature, - no_type_checks, - ) - args.append(arg) - require_named = True - elif self.current_str() == '(': - arg, extra_stmt, names = self.parse_tuple_arg(len(args)) - args.append(arg) - if extra_stmt is not None: - extra_stmts.append(extra_stmt) - has_tuple_arg = True - arg_names.extend(names) - else: - arg, require_named = self.parse_normal_arg( - require_named, - allow_signature, - no_type_checks, - ) - args.append(arg) - arg_names.append(arg.variable.name()) - - if self.current().string != ',': - break - - self.expect(',') - - # Non-tuple argument dupes will be checked elsewhere. Avoid - # generating duplicate errors. - if has_tuple_arg: - self.check_duplicate_argument_names(arg_names) - - return args, extra_stmts - - def check_duplicate_argument_names(self, names: List[str]) -> None: - found = set() # type: Set[str] - for name in names: - if name in found: - self.fail('Duplicate argument name "{}"'.format(name), - self.current().line) - found.add(name) - - def parse_asterisk_arg(self, - allow_signature: bool, - no_type_checks: bool) -> Argument: - asterisk = self.skip() - name = self.expect_type(Name) - variable = Var(name.string) - if asterisk.string == '*': - kind = nodes.ARG_STAR - else: - kind = nodes.ARG_STAR2 - - type = None - if no_type_checks: - self.parse_parameter_annotation() - else: - type = self.parse_arg_type(allow_signature) - - return Argument(variable, type, None, kind) - - def parse_tuple_arg(self, index: int) -> Tuple[Argument, AssignmentStmt, List[str]]: - """Parse a single Python 2 tuple argument. - - Example: def f(x, (y, z)): ... - - The tuple arguments gets transformed into an assignment in the - function body (the second return value). - - Return tuple (argument, decomposing assignment, list of names defined). - - Special case: if the argument is just (x) then it's not a tuple; - we indicate this by returning (argument, None, ['x']). - However, if the argument is (x,) then it *is* a (singleton) tuple. - """ - line = self.current().line - # Generate a new argument name that is very unlikely to clash with anything. - arg_name = '__tuple_arg_{}'.format(index + 1) - if self.pyversion[0] >= 3: - self.fail('Tuples in argument lists only supported in Python 2 mode', line) - paren_arg = self.parse_parentheses() - self.verify_tuple_arg(paren_arg) - if isinstance(paren_arg, NameExpr): - # This isn't a tuple. Revert to a normal argument. - arg_name = paren_arg.name - decompose = None - else: - rvalue = NameExpr(arg_name) - rvalue.set_line(line) - decompose = AssignmentStmt([paren_arg], rvalue) - decompose.set_line(line) - kind = nodes.ARG_POS - initializer = None - if self.current_str() == '=': - self.expect('=') - initializer = self.parse_expression(precedence[',']) - kind = nodes.ARG_OPT - var = Var(arg_name) - arg_names = self.find_tuple_arg_argument_names(paren_arg) - return Argument(var, None, initializer, kind), decompose, arg_names - - def verify_tuple_arg(self, paren_arg: Node) -> None: - if isinstance(paren_arg, TupleExpr): - if not paren_arg.items: - self.fail('Empty tuple not valid as an argument', paren_arg.line) - for item in paren_arg.items: - self.verify_tuple_arg(item) - elif not isinstance(paren_arg, NameExpr): - self.fail('Invalid item in tuple argument', paren_arg.line) - - def find_tuple_arg_argument_names(self, node: Node) -> List[str]: - result = [] # type: List[str] - if isinstance(node, TupleExpr): - for item in node.items: - result.extend(self.find_tuple_arg_argument_names(item)) - elif isinstance(node, NameExpr): - result.append(node.name) - return result - - def parse_normal_arg(self, require_named: bool, - allow_signature: bool, - no_type_checks: bool) -> Tuple[Argument, bool]: - name = self.expect_type(Name) - variable = Var(name.string) - - type = None - if no_type_checks: - self.parse_parameter_annotation() - else: - type = self.parse_arg_type(allow_signature) - - initializer = None # type: Node - if self.current_str() == '=': - self.expect('=') - initializer = self.parse_expression(precedence[',']) - if require_named: - kind = nodes.ARG_NAMED - else: - kind = nodes.ARG_OPT - else: - if require_named: - kind = nodes.ARG_NAMED - else: - kind = nodes.ARG_POS - - self.set_type_optional(type, initializer) - - return Argument(variable, type, initializer, kind), require_named - - def set_type_optional(self, type: Type, initializer: Node) -> None: - if not experiments.STRICT_OPTIONAL: - return - # Indicate that type should be wrapped in an Optional if arg is initialized to None. - optional = isinstance(initializer, NameExpr) and initializer.name == 'None' - if isinstance(type, UnboundType): - type.optional = optional - - def parse_parameter_annotation(self) -> Node: - if self.current_str() == ':': - self.skip() - return self.parse_expression(precedence[',']) - - def parse_arg_type(self, allow_signature: bool) -> Type: - if self.current_str() == ':' and allow_signature: - self.skip() - return self.parse_type() - else: - return None - - def verify_argument_kinds(self, kinds: List[int], line: int) -> None: - found = set() # type: Set[int] - for i, kind in enumerate(kinds): - if kind == nodes.ARG_POS and found & set([nodes.ARG_OPT, - nodes.ARG_STAR, - nodes.ARG_STAR2]): - self.fail('Invalid argument list', line) - elif kind == nodes.ARG_STAR and nodes.ARG_STAR in found: - self.fail('Invalid argument list', line) - elif kind == nodes.ARG_STAR2 and i != len(kinds) - 1: - self.fail('Invalid argument list', line) - found.add(kind) - - def construct_function_type(self, args: List[Argument], ret_type: Type, - line: int) -> CallableType: - # Complete the type annotation by replacing omitted types with 'Any'. - arg_types = [arg.type_annotation for arg in args] - for i in range(len(arg_types)): - if arg_types[i] is None: - arg_types[i] = AnyType(implicit=True) - if ret_type is None: - ret_type = AnyType(implicit=True) - arg_kinds = [arg.kind for arg in args] - arg_names = [arg.variable.name() for arg in args] - return CallableType(arg_types, arg_kinds, arg_names, ret_type, None, name=None, - variables=None, line=line) - - # Parsing statements - - def parse_block(self, allow_type: bool = False) -> Tuple[Block, Type]: - colon = self.expect(':') - if not isinstance(self.current(), Break): - # Block immediately after ':'. - nodes = [] - while True: - ind = self.ind - stmt, is_simple = self.parse_statement() - if not is_simple: - self.parse_error_at(self.tok[ind]) - break - nodes.append(stmt) - brk = self.expect_break() - if brk.string != ';': - break - node = Block(nodes) - node.set_line(colon) - return node, None - else: - # Indented block. - brk = self.expect_break() - type = self.parse_type_comment(brk, signature=True) - self.expect_indent() - stmt_list = [] # type: List[Node] - if allow_type: - cur = self.current() - if type is None and isinstance(cur, StrLit): - ds = docstring.parse_docstring(cur.parsed()) - if ds and False: # TODO: Enable when this is working. - try: - type = parse_str_as_signature(ds.as_type_str(), cur.line) - except TypeParseError: - # We don't require docstrings to be actually correct. - # TODO: Report something here. - type = None - while (not isinstance(self.current(), Dedent) and - not isinstance(self.current(), Eof)): - try: - stmt, is_simple = self.parse_statement() - if is_simple: - self.expect_break() - if stmt is not None: - if not self.try_combine_overloads(stmt, stmt_list): - stmt_list.append(stmt) - except ParseError: - pass - if isinstance(self.current(), Dedent): - self.skip() - node = Block(stmt_list) - node.set_line(colon) - return node, type - - def try_combine_overloads(self, s: Node, stmt: List[Node]) -> bool: - if isinstance(s, Decorator) and stmt: - fdef = s - n = fdef.func.name() - if isinstance(stmt[-1], Decorator) and stmt[-1].func.name() == n: - stmt[-1] = OverloadedFuncDef([stmt[-1], fdef]) - return True - elif isinstance(stmt[-1], OverloadedFuncDef) and stmt[-1].name() == n: - stmt[-1].items.append(fdef) - return True - return False - - def parse_statement(self) -> Tuple[Node, bool]: - stmt = None # type: Node - t = self.current() - ts = self.current_str() - is_simple = True # Is this a non-block statement? - if ts == 'if': - stmt = self.parse_if_stmt() - is_simple = False - elif ts == 'def': - stmt = self.parse_function() - is_simple = False - elif ts == 'while': - stmt = self.parse_while_stmt() - is_simple = False - elif ts == 'return': - stmt = self.parse_return_stmt() - elif ts == 'for': - stmt = self.parse_for_stmt() - is_simple = False - elif ts == 'try': - stmt = self.parse_try_stmt() - is_simple = False - elif ts == 'break': - stmt = self.parse_break_stmt() - elif ts == 'continue': - stmt = self.parse_continue_stmt() - elif ts == 'pass': - stmt = self.parse_pass_stmt() - elif ts == 'raise': - stmt = self.parse_raise_stmt() - elif ts == 'import': - stmt = self.parse_import() - elif ts == 'from': - stmt = self.parse_import_from() - elif ts == 'class': - stmt = self.parse_class_def() - is_simple = False - elif ts == 'global': - stmt = self.parse_global_decl() - elif ts == 'nonlocal' and self.pyversion[0] >= 3: - stmt = self.parse_nonlocal_decl() - elif ts == 'assert': - stmt = self.parse_assert_stmt() - elif ts == 'del': - stmt = self.parse_del_stmt() - elif ts == 'with': - stmt = self.parse_with_stmt() - is_simple = False - elif ts == '@': - stmt = self.parse_decorated_function_or_class() - is_simple = False - elif ts == 'print' and (self.pyversion[0] == 2 and - 'print_function' not in self.future_options): - stmt = self.parse_print_stmt() - elif ts == 'exec' and self.pyversion[0] == 2: - stmt = self.parse_exec_stmt() - else: - stmt = self.parse_expression_or_assignment() - if stmt is not None: - stmt.set_line(t) - return stmt, is_simple - - def parse_expression_or_assignment(self) -> Node: - expr = self.parse_expression(star_expr_allowed=True) - if self.current_str() == '=': - return self.parse_assignment(expr) - elif self.current_str() in op_assign: - # Operator assignment statement. - op = self.current_str()[:-1] - self.skip() - rvalue = self.parse_expression() - return OperatorAssignmentStmt(op, expr, rvalue) - else: - # Expression statement. - return ExpressionStmt(expr) - - def parse_assignment(self, lvalue: Any) -> Node: - """Parse an assignment statement. - - Assume that lvalue has been parsed already, and the current token is '='. - Also parse an optional '# type:' comment. - """ - self.expect('=') - lvalues = [lvalue] - expr = self.parse_expression(star_expr_allowed=True) - while self.current_str() == '=': - self.skip() - lvalues.append(expr) - expr = self.parse_expression(star_expr_allowed=True) - cur = self.current() - if isinstance(cur, Break): - type = self.parse_type_comment(cur, signature=False) - else: - type = None - return AssignmentStmt(lvalues, expr, type) - - def parse_return_stmt(self) -> ReturnStmt: - self.expect('return') - expr = None - current = self.current() - if current.string == 'yield': - self.parse_error() - if not isinstance(current, Break): - expr = self.parse_expression() - node = ReturnStmt(expr) - return node - - def parse_raise_stmt(self) -> RaiseStmt: - self.expect('raise') - expr = None - from_expr = None - if not isinstance(self.current(), Break): - expr = self.parse_expression() - if self.current_str() == 'from': - self.expect('from') - from_expr = self.parse_expression() - node = RaiseStmt(expr, from_expr) - return node - - def parse_assert_stmt(self) -> AssertStmt: - self.expect('assert') - expr = self.parse_expression() - node = AssertStmt(expr) - return node - - def parse_yield_or_yield_from_expr(self) -> Union[YieldFromExpr, YieldExpr]: - self.expect("yield") - expr = None - node = YieldExpr(expr) # type: Union[YieldFromExpr, YieldExpr] - if not isinstance(self.current(), Break): - if self.current_str() == "from": - self.expect("from") - expr = self.parse_expression() # when yield from is assigned to a variable - node = YieldFromExpr(expr) - else: - if self.current_str() == ')': - node = YieldExpr(None) - else: - expr = self.parse_expression() - node = YieldExpr(expr) - return node - - def parse_ellipsis(self) -> EllipsisExpr: - self.expect('...') - node = EllipsisExpr() - return node - - def parse_del_stmt(self) -> DelStmt: - self.expect('del') - expr = self.parse_expression() - node = DelStmt(expr) - return node - - def parse_break_stmt(self) -> BreakStmt: - self.expect('break') - node = BreakStmt() - return node - - def parse_continue_stmt(self) -> ContinueStmt: - self.expect('continue') - node = ContinueStmt() - return node - - def parse_pass_stmt(self) -> PassStmt: - self.expect('pass') - node = PassStmt() - return node - - def parse_global_decl(self) -> GlobalDecl: - self.expect('global') - names = self.parse_identifier_list() - node = GlobalDecl(names) - return node - - def parse_nonlocal_decl(self) -> NonlocalDecl: - self.expect('nonlocal') - names = self.parse_identifier_list() - node = NonlocalDecl(names) - return node - - def parse_identifier_list(self) -> List[str]: - names = [] - while True: - n = self.expect_type(Name) - names.append(n.string) - if self.current_str() != ',': - break - self.skip() - return names - - def parse_while_stmt(self) -> WhileStmt: - is_error = False - self.expect('while') - try: - expr = self.parse_expression() - except ParseError: - is_error = True - body, _ = self.parse_block() - if self.current_str() == 'else': - self.expect('else') - else_body, _ = self.parse_block() - else: - else_body = None - if is_error is not None: - node = WhileStmt(expr, body, else_body) - return node - else: - return None - - def parse_for_stmt(self) -> ForStmt: - self.expect('for') - index = self.parse_for_index_variables() - self.expect('in') - expr = self.parse_expression() - - body, _ = self.parse_block() - - if self.current_str() == 'else': - self.expect('else') - else_body, _ = self.parse_block() - else: - else_body = None - - node = ForStmt(index, expr, body, else_body) - return node - - def parse_for_index_variables(self) -> Node: - # Parse index variables of a 'for' statement. - index_items = [] - force_tuple = False - - while True: - v = self.parse_expression(precedence['in'], - star_expr_allowed=True) # Prevent parsing of for stmt 'in' - index_items.append(v) - if self.current_str() != ',': - break - self.skip() - if self.current_str() == 'in': - force_tuple = True - break - - if len(index_items) == 1 and not force_tuple: - index = index_items[0] - else: - index = TupleExpr(index_items) - index.set_line(index_items[0].get_line()) - - return index - - def parse_if_stmt(self) -> IfStmt: - is_error = False - - self.expect('if') - expr = [] - try: - expr.append(self.parse_expression()) - except ParseError: - is_error = True - - body = [self.parse_block()[0]] - - while self.current_str() == 'elif': - self.expect('elif') - try: - expr.append(self.parse_expression()) - except ParseError: - is_error = True - body.append(self.parse_block()[0]) - - if self.current_str() == 'else': - self.expect('else') - else_body, _ = self.parse_block() - else: - else_body = None - - if not is_error: - node = IfStmt(expr, body, else_body) - return node - else: - return None - - def parse_try_stmt(self) -> Node: - self.expect('try') - body, _ = self.parse_block() - is_error = False - vars = [] # type: List[NameExpr] - types = [] # type: List[Node] - handlers = [] # type: List[Block] - while self.current_str() == 'except': - self.expect('except') - if not isinstance(self.current(), Colon): - try: - t = self.current() - types.append(self.parse_expression(precedence[',']).set_line(t)) - if self.current_str() == 'as': - self.expect('as') - vars.append(self.parse_name_expr()) - elif self.pyversion[0] == 2 and self.current_str() == ',': - self.expect(',') - vars.append(self.parse_name_expr()) - else: - vars.append(None) - except ParseError: - is_error = True - else: - types.append(None) - vars.append(None) - handlers.append(self.parse_block()[0]) - if not is_error: - if self.current_str() == 'else': - self.skip() - else_body, _ = self.parse_block() - else: - else_body = None - if self.current_str() == 'finally': - self.expect('finally') - finally_body, _ = self.parse_block() - else: - finally_body = None - node = TryStmt(body, vars, types, handlers, else_body, - finally_body) - return node - else: - return None - - def parse_with_stmt(self) -> WithStmt: - self.expect('with') - exprs = [] - targets = [] - while True: - expr = self.parse_expression(precedence[',']) - if self.current_str() == 'as': - self.expect('as') - target = self.parse_expression(precedence[',']) - else: - target = None - exprs.append(expr) - targets.append(target) - if self.current_str() != ',': - break - self.expect(',') - body, _ = self.parse_block() - return WithStmt(exprs, targets, body) - - def parse_print_stmt(self) -> PrintStmt: - self.expect('print') - args = [] - target = None - if self.current_str() == '>>': - self.skip() - target = self.parse_expression(precedence[',']) - if self.current_str() == ',': - self.skip() - if isinstance(self.current(), Break): - self.parse_error() - else: - if not isinstance(self.current(), Break): - self.parse_error() - comma = False - while not isinstance(self.current(), Break): - args.append(self.parse_expression(precedence[','])) - if self.current_str() == ',': - comma = True - self.skip() - else: - comma = False - break - return PrintStmt(args, newline=not comma, target=target) - - def parse_exec_stmt(self) -> ExecStmt: - self.expect('exec') - expr = self.parse_expression(precedence['in']) - variables1 = None - variables2 = None - if self.current_str() == 'in': - self.skip() - variables1 = self.parse_expression(precedence[',']) - if self.current_str() == ',': - self.skip() - variables2 = self.parse_expression(precedence[',']) - return ExecStmt(expr, variables1, variables2) - - # Parsing expressions - - def parse_expression(self, prec: int = 0, star_expr_allowed: bool = False) -> Node: - """Parse a subexpression within a specific precedence context.""" - expr = None # type: Node - current = self.current() # Remember token for setting the line number. - - # Parse a "value" expression or unary operator expression and store - # that in expr. - s = self.current_str() - if s == '(': - # Parenthesised expression or cast. - expr = self.parse_parentheses() - elif s == '[': - expr = self.parse_list_expr() - elif s in ['-', '+', 'not', '~']: - # Unary operation. - expr = self.parse_unary_expr() - elif s == 'lambda': - expr = self.parse_lambda_expr() - elif s == '{': - expr = self.parse_dict_or_set_expr() - elif s == '*' and star_expr_allowed: - expr = self.parse_star_expr() - elif s == '`' and self.pyversion[0] == 2: - expr = self.parse_backquote_expr() - else: - if isinstance(current, Name): - # Name expression. - expr = self.parse_name_expr() - elif isinstance(current, IntLit): - expr = self.parse_int_expr() - elif isinstance(current, StrLit): - expr = self.parse_str_expr() - elif isinstance(current, BytesLit): - expr = self.parse_bytes_literal() - elif isinstance(current, UnicodeLit): - expr = self.parse_unicode_literal() - elif isinstance(current, FloatLit): - expr = self.parse_float_expr() - elif isinstance(current, ComplexLit): - expr = self.parse_complex_expr() - elif isinstance(current, Keyword) and s == "yield": - # The expression yield from and yield to assign - expr = self.parse_yield_or_yield_from_expr() - elif isinstance(current, EllipsisToken) and (self.pyversion[0] >= 3 - or self.is_stub_file): - expr = self.parse_ellipsis() - else: - # Invalid expression. - self.parse_error() - - # Set the line of the expression node, if not specified. This - # simplifies recording the line number as not every node type needs to - # deal with it separately. - if expr.line < 0: - expr.set_line(current) - - # Parse operations that require a left argument (stored in expr). - while True: - current = self.current() - s = self.current_str() - if s == '(': - # Call expression. - expr = self.parse_call_expr(expr) - elif s == '.': - # Member access expression. - expr = self.parse_member_expr(expr) - elif s == '[': - # Indexing expression. - expr = self.parse_index_expr(expr) - elif s == ',': - # The comma operator is used to build tuples. Comma also - # separates array items and function arguments, but in this - # case the precedence is too low to build a tuple. - if precedence[','] > prec: - expr = self.parse_tuple_expr(expr) - else: - break - elif s == 'for': - if precedence[''] > prec: - # List comprehension or generator expression. Parse as - # generator expression; it will be converted to list - # comprehension if needed elsewhere. - expr = self.parse_generator_expr(expr) - else: - break - elif s == 'if': - # Conditional expression. - if precedence[''] > prec: - expr = self.parse_conditional_expr(expr) - else: - break - else: - # Binary operation or a special case. - if isinstance(current, Op): - op = self.current_str() - op_prec = precedence[op] - if op == 'not': - # Either "not in" or an error. - op_prec = precedence['in'] - if op_prec > prec: - if op in op_comp: - expr = self.parse_comparison_expr(expr, op_prec) - else: - expr = self.parse_bin_op_expr(expr, op_prec) - else: - # The operation cannot be associated with the - # current left operand due to the precedence - # context; let the caller handle it. - break - else: - # Not an operation that accepts a left argument; let the - # caller handle the rest. - break - - # Set the line of the expression node, if not specified. This - # simplifies recording the line number as not every node type - # needs to deal with it separately. - if expr.line < 0: - expr.set_line(current) - - return expr - - def parse_parentheses(self) -> Node: - self.skip() - if self.current_str() == ')': - # Empty tuple (). - expr = self.parse_empty_tuple_expr() # type: Node - else: - # Parenthesised expression. - expr = self.parse_expression(0, star_expr_allowed=True) - self.expect(')') - return expr - - def parse_star_expr(self) -> Node: - star = self.expect('*') - expr = self.parse_expression(precedence['*u']) - expr = StarExpr(expr) - if expr.line < 0: - expr.set_line(star) - return expr - - def parse_empty_tuple_expr(self) -> TupleExpr: - self.expect(')') - node = TupleExpr([]) - return node - - def parse_list_expr(self) -> Node: - """Parse list literal or list comprehension.""" - items = [] - self.expect('[') - while self.current_str() != ']' and not self.eol(): - items.append(self.parse_expression(precedence[''], star_expr_allowed=True)) - if self.current_str() != ',': - break - self.expect(',') - if self.current_str() == 'for' and len(items) == 1: - items[0] = self.parse_generator_expr(items[0]) - self.expect(']') - if len(items) == 1 and isinstance(items[0], GeneratorExpr): - return ListComprehension(items[0]) - else: - expr = ListExpr(items) - return expr - - def parse_generator_expr(self, left_expr: Node) -> GeneratorExpr: - tok = self.current() - indices, sequences, condlists = self.parse_comp_for() - - gen = GeneratorExpr(left_expr, indices, sequences, condlists) - gen.set_line(tok) - return gen - - def parse_comp_for(self) -> Tuple[List[Node], List[Node], List[List[Node]]]: - indices = [] - sequences = [] - condlists = [] # type: List[List[Node]] - while self.current_str() == 'for': - conds = [] - self.expect('for') - index = self.parse_for_index_variables() - indices.append(index) - self.expect('in') - if self.pyversion[0] >= 3: - sequence = self.parse_expression(precedence['']) - else: - sequence = self.parse_expression_list() - sequences.append(sequence) - while self.current_str() == 'if': - self.skip() - conds.append(self.parse_expression(precedence[''])) - condlists.append(conds) - - return indices, sequences, condlists - - def parse_expression_list(self) -> Node: - prec = precedence[''] - expr = self.parse_expression(prec) - if self.current_str() != ',': - return expr - else: - t = self.current() - return self.parse_tuple_expr(expr, prec).set_line(t) - - def parse_conditional_expr(self, left_expr: Node) -> ConditionalExpr: - self.expect('if') - cond = self.parse_expression(precedence['']) - self.expect('else') - else_expr = self.parse_expression(precedence['']) - return ConditionalExpr(cond, left_expr, else_expr) - - def parse_dict_or_set_expr(self) -> Node: - items = [] # type: List[Tuple[Node, Node]] - self.expect('{') - while self.current_str() != '}' and not self.eol(): - key = self.parse_expression(precedence['']) - if self.current_str() in [',', '}'] and items == []: - return self.parse_set_expr(key) - elif self.current_str() == 'for' and items == []: - return self.parse_set_comprehension(key) - elif self.current_str() != ':': - self.parse_error() - colon = self.expect(':') - value = self.parse_expression(precedence['']) - if self.current_str() == 'for' and items == []: - return self.parse_dict_comprehension(key, value, colon) - items.append((key, value)) - if self.current_str() != ',': - break - self.expect(',') - self.expect('}') - node = DictExpr(items) - return node - - def parse_set_expr(self, first: Node) -> SetExpr: - items = [first] - while self.current_str() != '}' and not self.eol(): - self.expect(',') - if self.current_str() == '}': - break - items.append(self.parse_expression(precedence[','])) - self.expect('}') - expr = SetExpr(items) - return expr - - def parse_set_comprehension(self, expr: Node): - gen = self.parse_generator_expr(expr) - self.expect('}') - set_comp = SetComprehension(gen) - return set_comp - - def parse_dict_comprehension(self, key: Node, value: Node, - colon: Token) -> DictionaryComprehension: - indices, sequences, condlists = self.parse_comp_for() - dic = DictionaryComprehension(key, value, indices, sequences, condlists) - dic.set_line(colon) - self.expect('}') - return dic - - def parse_tuple_expr(self, expr: Node, - prec: int = precedence[',']) -> TupleExpr: - items = [expr] - while True: - self.expect(',') - if (self.current_str() in [')', ']', '=', ':'] or - isinstance(self.current(), Break)): - break - items.append(self.parse_expression(prec, star_expr_allowed=True)) - if self.current_str() != ',': break - node = TupleExpr(items) - return node - - def parse_backquote_expr(self) -> BackquoteExpr: - self.expect('`') - expr = self.parse_expression() - self.expect('`') - return BackquoteExpr(expr) - - def parse_name_expr(self) -> NameExpr: - tok = self.expect_type(Name) - node = NameExpr(tok.string) - node.set_line(tok) - return node - - octal_int = re.compile('0+[1-9]') - - def parse_int_expr(self) -> IntExpr: - tok = self.expect_type(IntLit) - string = tok.string.rstrip('lL') # Strip L prefix (Python 2 long literals) - if self.octal_int.match(string): - value = int(string, 8) - else: - value = int(string, 0) - node = IntExpr(value) - return node - - def parse_str_expr(self) -> Node: - # XXX \uxxxx literals - token = self.expect_type(StrLit) - value = cast(StrLit, token).parsed() - is_unicode = False - while isinstance(self.current(), (StrLit, UnicodeLit)): - token = self.skip() - if isinstance(token, StrLit): - value += token.parsed() - elif isinstance(token, UnicodeLit): - value += token.parsed() - is_unicode = True - if is_unicode or (self.pyversion[0] == 2 and 'unicode_literals' in self.future_options): - node = UnicodeExpr(value) # type: Node - else: - node = StrExpr(value) - return node - - def parse_bytes_literal(self) -> Node: - # XXX \uxxxx literals - tok = [self.expect_type(BytesLit)] - value = (cast(BytesLit, tok[0])).parsed() - while isinstance(self.current(), BytesLit): - t = cast(BytesLit, self.skip()) - value += t.parsed() - if self.pyversion[0] >= 3: - node = BytesExpr(value) # type: Node - else: - node = StrExpr(value) - return node - - def parse_unicode_literal(self) -> Node: - # XXX \uxxxx literals - token = self.expect_type(UnicodeLit) - value = cast(UnicodeLit, token).parsed() - while isinstance(self.current(), (UnicodeLit, StrLit)): - token = cast(Union[UnicodeLit, StrLit], self.skip()) - value += token.parsed() - if self.pyversion[0] >= 3: - # Python 3.3 supports u'...' as an alias of '...'. - node = StrExpr(value) # type: Node - else: - node = UnicodeExpr(value) - return node - - def parse_float_expr(self) -> FloatExpr: - tok = self.expect_type(FloatLit) - node = FloatExpr(float(tok.string)) - return node - - def parse_complex_expr(self) -> ComplexExpr: - tok = self.expect_type(ComplexLit) - node = ComplexExpr(complex(tok.string)) - return node - - def parse_call_expr(self, callee: Any) -> CallExpr: - self.expect('(') - args, kinds, names = self.parse_arg_expr() - self.expect(')') - node = CallExpr(callee, args, kinds, names) - return node - - def parse_arg_expr(self) -> Tuple[List[Node], List[int], List[str]]: - """Parse arguments in a call expression (within '(' and ')'). - - Return a tuple with these items: - argument expressions - argument kinds - argument names (for named arguments; None for ordinary args) - """ - args = [] # type: List[Node] - kinds = [] # type: List[int] - names = [] # type: List[str] - var_arg = False - dict_arg = False - named_args = False - while self.current_str() != ')' and not self.eol() and not dict_arg: - if isinstance(self.current(), Name) and self.peek().string == '=': - # Named argument - name = self.expect_type(Name) - self.expect('=') - kinds.append(nodes.ARG_NAMED) - names.append(name.string) - named_args = True - elif (self.current_str() == '*' and not var_arg and not dict_arg): - # *args - var_arg = True - self.expect('*') - kinds.append(nodes.ARG_STAR) - names.append(None) - elif self.current_str() == '**': - # **kwargs - self.expect('**') - dict_arg = True - kinds.append(nodes.ARG_STAR2) - names.append(None) - elif not var_arg and not named_args: - # Ordinary argument - kinds.append(nodes.ARG_POS) - names.append(None) - else: - self.parse_error() - args.append(self.parse_expression(precedence[','])) - if self.current_str() != ',': - break - self.expect(',') - return args, kinds, names - - def parse_member_expr(self, expr: Any) -> Node: - self.expect('.') - name = self.expect_type(Name) - if (isinstance(expr, CallExpr) and isinstance(expr.callee, NameExpr) - and expr.callee.name == 'super'): - # super() expression - node = SuperExpr(name.string) # type: Node - else: - node = MemberExpr(expr, name.string) - return node - - def parse_index_expr(self, base: Any) -> IndexExpr: - self.expect('[') - index = self.parse_slice_item() - if self.current_str() == ',': - # Extended slicing such as x[1:, :2]. - items = [index] - while self.current_str() == ',': - self.skip() - if self.current_str() == ']' or isinstance(self.current(), Break): - break - items.append(self.parse_slice_item()) - index = TupleExpr(items) - index.set_line(items[0].line) - self.expect(']') - node = IndexExpr(base, index) - return node - - def parse_slice_item(self) -> Node: - if self.current_str() != ':': - if self.current_str() == '...': - # Ellipsis is valid here even in Python 2 (but not elsewhere). - ellipsis = EllipsisExpr() - token = self.skip() - ellipsis.set_line(token) - return ellipsis - else: - item = self.parse_expression(precedence[',']) - else: - item = None - if self.current_str() == ':': - # Slice. - index = item - colon = self.expect(':') - if self.current_str() not in (']', ':', ','): - end_index = self.parse_expression(precedence[',']) - else: - end_index = None - stride = None - if self.current_str() == ':': - self.expect(':') - if self.current_str() not in (']', ','): - stride = self.parse_expression(precedence[',']) - item = SliceExpr(index, end_index, stride).set_line(colon.line) - return item - - def parse_bin_op_expr(self, left: Node, prec: int) -> OpExpr: - op = self.expect_type(Op) - op_str = op.string - if op_str == '~': - self.ind -= 1 - self.parse_error() - right = self.parse_expression(prec) - node = OpExpr(op_str, left, right) - return node - - def parse_comparison_expr(self, left: Node, prec: int) -> ComparisonExpr: - operators_str = [] - operands = [left] - - while True: - op = self.expect_type(Op) - op_str = op.string - if op_str == 'not': - if self.current_str() == 'in': - op_str = 'not in' - self.skip() - else: - self.parse_error() - elif op_str == 'is' and self.current_str() == 'not': - op_str = 'is not' - self.skip() - - operators_str.append(op_str) - operand = self.parse_expression(prec) - operands.append(operand) - - # Continue if next token is a comparison operator - self.current() - s = self.current_str() - if s not in op_comp: - break - - node = ComparisonExpr(operators_str, operands) - return node - - def parse_unary_expr(self) -> UnaryExpr: - op_tok = self.skip() - op = op_tok.string - if op == '-' or op == '+': - prec = precedence['-u'] - else: - prec = precedence[op] - expr = self.parse_expression(prec) - node = UnaryExpr(op, expr) - return node - - def parse_lambda_expr(self) -> FuncExpr: - lambda_tok = self.expect('lambda') - - args, extra_stmts = self.parse_arg_list(allow_signature=False) - - # Use 'object' as the placeholder return type; it will be inferred - # later. We can't use 'Any' since it could make type inference results - # less precise. - ret_type = UnboundType('__builtins__.object') - typ = self.build_func_annotation(ret_type, args, - lambda_tok.line, is_default_ret=True) - - colon = self.expect(':') - - expr = self.parse_expression(precedence[',']) - - nodes = [ReturnStmt(expr).set_line(lambda_tok)] - # Potentially insert extra assignment statements to the beginning of the - # body, used to decompose Python 2 tuple arguments. - nodes[:0] = extra_stmts - body = Block(nodes) - body.set_line(colon) - - return FuncExpr(args, body, typ) - - # Helper methods - - def skip(self) -> Token: - self.ind += 1 - return self.tok[self.ind - 1] - - def expect(self, string: str) -> Token: - if self.current_str() == string: - self.ind += 1 - return self.tok[self.ind - 1] - else: - self.parse_error() - - def expect_indent(self) -> Token: - if isinstance(self.current(), Indent): - return self.expect_type(Indent) - else: - self.fail('Expected an indented block', self.current().line) - return none - - def fail(self, msg: str, line: int) -> None: - self.errors.report(line, msg) - - def expect_type(self, typ: type) -> Token: - current = self.current() - if isinstance(current, typ): - self.ind += 1 - return current - else: - self.parse_error() - - def expect_colon_and_break(self) -> Tuple[Token, Token]: - return self.expect_type(Colon), self.expect_type(Break) - - def expect_break(self) -> Token: - return self.expect_type(Break) - - def current(self) -> Token: - return self.tok[self.ind] - - def current_str(self) -> str: - return self.current().string - - def peek(self) -> Token: - return self.tok[self.ind + 1] - - def parse_error(self) -> None: - self.parse_error_at(self.current()) - raise ParseError() - - def parse_error_at(self, tok: Token, skip: bool = True, reason: Optional[str] = None) -> None: - msg = '' - if isinstance(tok, LexError): - msg = token_repr(tok) - msg = msg[0].upper() + msg[1:] - elif isinstance(tok, Indent) or isinstance(tok, Dedent): - msg = 'Inconsistent indentation' - else: - formatted_reason = ": {}".format(reason) if reason else "" - msg = 'Parse error before {}{}'.format(token_repr(tok), formatted_reason) - - self.errors.report(tok.line, msg) - - if skip: - self.skip_until_next_line() - - def skip_until_break(self) -> None: - n = 0 - while (not isinstance(self.current(), Break) - and not isinstance(self.current(), Eof)): - self.skip() - n += 1 - if isinstance(self.tok[self.ind - 1], Colon) and n > 1: - self.ind -= 1 - - def skip_until_next_line(self) -> None: - self.skip_until_break() - if isinstance(self.current(), Break): - self.skip() - - def eol(self) -> bool: - return isinstance(self.current(), Break) or self.eof() - - def eof(self) -> bool: - return isinstance(self.current(), Eof) - - # Type annotation related functionality - - def parse_type(self) -> Type: - try: - typ, self.ind = parse_type(self.tok, self.ind) - except TypeParseError as e: - self.parse_error_at(e.token, reason=e.message) - raise ParseError() - return typ - - annotation_prefix_re = re.compile(r'#\s*type:') - ignore_prefix_re = re.compile(r'ignore\b') - - def parse_type_comment(self, token: Token, signature: bool) -> Type: - """Parse a '# type: ...' annotation. - - Return None if no annotation found. If signature is True, expect - a type signature of form (...) -> t. - """ - whitespace_or_comments = token.rep().strip() - if self.annotation_prefix_re.match(whitespace_or_comments): - type_as_str = whitespace_or_comments.split(':', 1)[1].strip() - if self.ignore_prefix_re.match(type_as_str): - # Actually a "# type: ignore" annotation -> not a type. - return None - tokens = lex.lex(type_as_str, token.line)[0] - if len(tokens) < 2: - # Empty annotation (only Eof token) - self.errors.report(token.line, 'Empty type annotation') - return None - try: - if not signature: - type, index = parse_types(tokens, 0) - else: - type, index = parse_signature(tokens) - except TypeParseError as e: - self.parse_error_at(e.token, skip=False, reason=e.message) - return None - if index < len(tokens) - 2: - self.parse_error_at(tokens[index], skip=False) - return None - return type - else: - return None - - -class ParseError(Exception): pass - - -def token_repr(tok: Token) -> str: - """Return a representation of a token for use in parse error messages.""" - if isinstance(tok, Break): - return 'end of line' - elif isinstance(tok, Eof): - return 'end of file' - elif isinstance(tok, Keyword) or isinstance(tok, Name): - return '"{}"'.format(tok.string) - elif isinstance(tok, IntLit) or isinstance(tok, FloatLit) or isinstance(tok, ComplexLit): - return 'numeric literal' - elif isinstance(tok, StrLit) or isinstance(tok, UnicodeLit): - return 'string literal' - elif (isinstance(tok, Punct) or isinstance(tok, Op) - or isinstance(tok, Colon)): - return tok.string - elif isinstance(tok, Bom): - return 'byte order mark' - elif isinstance(tok, Indent): - return 'indent' - elif isinstance(tok, Dedent): - return 'dedent' - elif isinstance(tok, EllipsisToken): - return '...' - else: - if isinstance(tok, LexError): - t = tok.type - if t == lex.NUMERIC_LITERAL_ERROR: - return 'invalid numeric literal' - elif t == lex.UNTERMINATED_STRING_LITERAL: - return 'unterminated string literal' - elif t == lex.INVALID_CHARACTER: - msg = 'unrecognized character' - if ord(tok.string) in range(33, 127): - msg += ' ' + tok.string - return msg - elif t == lex.INVALID_DEDENT: - return 'inconsistent indentation' - elif t == lex.DECODE_ERROR: - return tok.message - raise ValueError('Unknown token {}'.format(repr(tok))) - - -if __name__ == '__main__': - # Parse a file and dump the AST (or display errors). - import sys - - def usage(): - print('Usage: parse.py [--py2] [--quiet] FILE [...]', file=sys.stderr) - sys.exit(2) - - args = sys.argv[1:] - pyversion = defaults.PYTHON3_VERSION - quiet = False - while args and args[0].startswith('--'): - if args[0] == '--py2': - pyversion = defaults.PYTHON2_VERSION - elif args[0] == '--quiet': - quiet = True - else: - usage() - args = args[1:] - if len(args) < 1: - usage() - status = 0 - for fnam in args: - s = open(fnam, 'rb').read() - errors = Errors() - try: - options = Options() - options.python_version = pyversion - tree = parse(s, fnam, None, options=options) - if not quiet: - print(tree) - except CompileError as e: - for msg in e.messages: - sys.stderr.write('%s\n' % msg) - status = 1 - sys.exit(status) +"""Mypy parser. + +Constructs a parse tree (abstract syntax tree) based on a string +representing a source file. Performs only minimal semantic checks. +""" + +import re + +from typing import List, Tuple, Any, Set, cast, Union, Optional + +from mypy import lex, docstring +from mypy.lex import ( + Token, Eof, Bom, Break, Name, Colon, Dedent, IntLit, StrLit, BytesLit, + UnicodeLit, FloatLit, Op, Indent, Keyword, Punct, LexError, ComplexLit, + EllipsisToken +) +import mypy.types +from mypy.nodes import ( + MypyFile, Import, Node, ImportAll, ImportFrom, FuncDef, OverloadedFuncDef, + ClassDef, Decorator, Block, Var, OperatorAssignmentStmt, + ExpressionStmt, AssignmentStmt, ReturnStmt, RaiseStmt, AssertStmt, + DelStmt, BreakStmt, ContinueStmt, PassStmt, GlobalDecl, + WhileStmt, ForStmt, IfStmt, TryStmt, WithStmt, + TupleExpr, GeneratorExpr, ListComprehension, ListExpr, ConditionalExpr, + DictExpr, SetExpr, NameExpr, IntExpr, StrExpr, BytesExpr, UnicodeExpr, + FloatExpr, CallExpr, SuperExpr, MemberExpr, IndexExpr, SliceExpr, OpExpr, + UnaryExpr, FuncExpr, PrintStmt, ImportBase, ComparisonExpr, + StarExpr, YieldFromExpr, NonlocalDecl, DictionaryComprehension, + SetComprehension, ComplexExpr, EllipsisExpr, YieldExpr, ExecStmt, Argument, + BackquoteExpr +) +from mypy import defaults +from mypy import nodes +from mypy.errors import Errors, CompileError +from mypy.types import Type, CallableType, AnyType, UnboundType +from mypy.parsetype import ( + parse_type, parse_types, parse_signature, TypeParseError, parse_str_as_signature +) +from mypy.options import Options + +from mypy import experiments + + +precedence = { + '**': 16, + '-u': 15, '+u': 15, '~': 15, # unary operators (-, + and ~) + '': 14, + '*': 13, '/': 13, '//': 13, '%': 13, + '+': 12, '-': 12, + '>>': 11, '<<': 11, + '&': 10, + '^': 9, + '|': 8, + '==': 7, '!=': 7, '<': 7, '>': 7, '<=': 7, '>=': 7, 'is': 7, 'in': 7, + '*u': 7, # unary * for star expressions + 'not': 6, + 'and': 5, + 'or': 4, + '': 3, # conditional expression + '': 2, # list comprehension + ',': 1} + + +op_assign = set([ + '+=', '-=', '*=', '/=', '//=', '%=', '**=', '|=', '&=', '^=', '>>=', + '<<=']) + +op_comp = set([ + '>', '<', '==', '>=', '<=', '<>', '!=', 'is', 'is', 'in', 'not']) + +none = Token('') # Empty token + + +def parse(source: Union[str, bytes], + fnam: str, + errors: Errors, + options: Options) -> MypyFile: + """Parse a source file, without doing any semantic analysis. + + Return the parse tree. If errors is not provided, raise ParseError + on failure. Otherwise, use the errors object to report parse errors. + + The python_version (major, minor) option determines the Python syntax variant. + """ + if options.fast_parser: + import mypy.fastparse + return mypy.fastparse.parse(source, + fnam=fnam, + errors=errors, + pyversion=options.python_version, + custom_typing_module=options.custom_typing_module) + + is_stub_file = bool(fnam) and fnam.endswith('.pyi') + parser = Parser(fnam, + errors, + options.python_version, + options.custom_typing_module, + is_stub_file=is_stub_file) + tree = parser.parse(source) + tree.path = fnam + tree.is_stub = is_stub_file + return tree + + +class Parser: + """Mypy parser that parses a string into an AST. + + Parses type annotations in addition to basic Python syntax. It supports both Python 2 and 3 + (though Python 2 support is incomplete). + + The AST classes are defined in mypy.nodes and mypy.types. + """ + + tok = None # type: List[Token] + ind = 0 + errors = None # type: Errors + # If True, raise an exception on any parse error. Otherwise, errors are reported via 'errors'. + raise_on_error = False + + # Are we currently parsing the body of a class definition? + is_class_body = False + # All import nodes encountered so far in this parse unit. + imports = None # type: List[ImportBase] + # Names imported from __future__. + future_options = None # type: List[str] + # Lines to ignore (using # type: ignore). + ignored_lines = None # type: Set[int] + + def __init__(self, fnam: str, errors: Errors, pyversion: Tuple[int, int], + custom_typing_module: str = None, is_stub_file: bool = False) -> None: + self.raise_on_error = errors is None + self.pyversion = pyversion + self.custom_typing_module = custom_typing_module + self.is_stub_file = is_stub_file + if errors is not None: + self.errors = errors + else: + self.errors = Errors() + if fnam is not None: + self.errors.set_file(fnam) + else: + self.errors.set_file('') + + def parse(self, s: Union[str, bytes]) -> MypyFile: + self.tok, self.ignored_lines = lex.lex(s, pyversion=self.pyversion, + is_stub_file=self.is_stub_file) + self.ind = 0 + self.imports = [] + self.future_options = [] + file = self.parse_file() + if self.raise_on_error and self.errors.is_errors(): + self.errors.raise_error() + return file + + def weak_opts(self) -> Set[str]: + """Do weak typing if any of the first ten tokens is a comment saying so. + + The comment can be one of: + # mypy: weak=global + # mypy: weak=local + # mypy: weak <- defaults to local + """ + regexp = re.compile(r'^[\s]*# *mypy: *weak(=?)([^\s]*)', re.M) + for t in self.tok[:10]: + for s in [t.string, t.pre]: + m = regexp.search(s) + if m: + opts = set(x for x in m.group(2).split(',') if x) + if not opts: + opts.add('local') + return opts + return set() + + def parse_file(self) -> MypyFile: + """Parse a mypy source file.""" + is_bom = self.parse_bom() + defs = self.parse_defs() + weak_opts = self.weak_opts() + self.expect_type(Eof) + node = MypyFile(defs, self.imports, is_bom, self.ignored_lines, + weak_opts=weak_opts) + return node + + # Parse the initial part + + def parse_bom(self) -> bool: + """Parse the optional byte order mark at the beginning of a file.""" + if isinstance(self.current(), Bom): + self.expect_type(Bom) + if isinstance(self.current(), Break): + self.expect_break() + return True + else: + return False + + def parse_import(self) -> Import: + self.expect('import') + ids = [] + while True: + id = self.parse_qualified_name() + translated = self.translate_module_id(id) + as_id = None + if self.current_str() == 'as': + self.expect('as') + name_tok = self.expect_type(Name) + as_id = name_tok.string + elif translated != id: + as_id = id + ids.append((translated, as_id)) + if self.current_str() != ',': + break + self.expect(',') + node = Import(ids) + self.imports.append(node) + return node + + def translate_module_id(self, id: str) -> str: + """Return the actual, internal module id for a source text id. + + For example, translate '__builtin__' in Python 2 to 'builtins'. + """ + if id == self.custom_typing_module: + return 'typing' + elif id == '__builtin__' and self.pyversion[0] == 2: + # HACK: __builtin__ in Python 2 is aliases to builtins. However, the implementation + # is named __builtin__.py (there is another layer of translation elsewhere). + return 'builtins' + return id + + def parse_import_from(self) -> Node: + self.expect('from') + + # Build the list of beginning relative tokens. + relative = 0 + while self.current_str() in (".", "..."): + relative += len(self.current_str()) + self.skip() + + # Parse qualified name to actually import from. + if self.current_str() == "import": + # Empty/default values. + name = "" + else: + name = self.parse_qualified_name() + + name = self.translate_module_id(name) + + # Parse import list + self.expect('import') + node = None # type: ImportBase + if self.current_str() == '*': + if name == '__future__': + self.parse_error() + # An import all from a module node: + self.skip() + node = ImportAll(name, relative) + else: + is_paren = self.current_str() == '(' + if is_paren: + self.expect('(') + targets = [] # type: List[Tuple[str, str]] + while True: + id, as_id = self.parse_import_name() + if '%s.%s' % (name, id) == self.custom_typing_module: + if targets or self.current_str() == ',': + self.fail('You cannot import any other modules when you ' + 'import a custom typing module', + self.current().line) + node = Import([('typing', as_id)]) + self.skip_until_break() + break + targets.append((id, as_id)) + if self.current_str() != ',': + break + self.expect(',') + if is_paren and self.current_str() == ')': + break + if is_paren: + self.expect(')') + if node is None: + node = ImportFrom(name, relative, targets) + self.imports.append(node) + if name == '__future__': + self.future_options.extend(target[0] for target in targets) + return node + + def parse_import_name(self) -> Tuple[str, Optional[str]]: + tok = self.expect_type(Name) + name = tok.string + if self.current_str() == 'as': + self.skip() + as_name = self.expect_type(Name) + return name, as_name.string + else: + return name, None + + def parse_qualified_name(self) -> str: + """Parse a name with an optional module qualifier. + + Return a tuple with the name as a string and a token array + containing all the components of the name. + """ + tok = self.expect_type(Name) + n = tok.string + while self.current_str() == '.': + self.expect('.') + tok = self.expect_type(Name) + n += '.' + tok.string + return n + + # Parsing global definitions + + def parse_defs(self) -> List[Node]: + defs = [] # type: List[Node] + while not self.eof(): + try: + defn, is_simple = self.parse_statement() + if is_simple: + self.expect_break() + if defn is not None: + if not self.try_combine_overloads(defn, defs): + defs.append(defn) + except ParseError: + pass + return defs + + def parse_class_def(self) -> ClassDef: + old_is_class_body = self.is_class_body + self.is_class_body = True + + self.expect('class') + metaclass = None + + try: + base_types = [] # type: List[Node] + try: + name_tok = self.expect_type(Name) + name = name_tok.string + + self.errors.push_type(name) + + if self.current_str() == '(': + self.skip() + while True: + if self.current_str() == ')': + break + if self.peek().string == '=': + metaclass = self.parse_class_keywords() + break + base_types.append(self.parse_super_type()) + if self.current_str() != ',': + break + self.skip() + self.expect(')') + except ParseError: + pass + + defs, _ = self.parse_block() + + node = ClassDef(name, defs, None, base_types, metaclass=metaclass) + return node + finally: + self.errors.pop_type() + self.is_class_body = old_is_class_body + + def parse_class_keywords(self) -> Optional[str]: + """Parse the class keyword arguments, keeping the metaclass but + ignoring all others. Returns None if the metaclass isn't found. + """ + metaclass = None + while True: + key = self.expect_type(Name) + self.expect('=') + if key.string == 'metaclass': + metaclass = self.parse_qualified_name() + else: + # skip the class value + self.parse_expression(precedence[',']) + if self.current_str() != ',': + break + self.skip() + if self.current_str() == ')': + break + return metaclass + + def parse_super_type(self) -> Node: + return self.parse_expression(precedence[',']) + + def parse_decorated_function_or_class(self) -> Node: + decorators = [] + no_type_checks = False + while self.current_str() == '@': + self.expect('@') + d_exp = self.parse_expression() + if self.is_no_type_check_decorator(d_exp): + no_type_checks = True + decorators.append(d_exp) + self.expect_break() + if self.current_str() != 'class': + func = self.parse_function(no_type_checks) + func.is_decorated = True + var = Var(func.name()) + # Types of decorated functions must always be inferred. + var.is_ready = False + var.set_line(decorators[0].line) + node = Decorator(func, decorators, var) + return node + else: + cls = self.parse_class_def() + cls.decorators = decorators + return cls + + def is_no_type_check_decorator(self, expr: Node) -> bool: + if isinstance(expr, NameExpr): + return expr.name == 'no_type_check' + elif isinstance(expr, MemberExpr): + if isinstance(expr.expr, NameExpr): + return expr.expr.name == 'typing' and expr.name == 'no_type_check' + else: + return False + + def parse_function(self, no_type_checks: bool=False) -> FuncDef: + def_tok = self.expect('def') + is_method = self.is_class_body + self.is_class_body = False + try: + (name, args, typ, is_error, extra_stmts) = self.parse_function_header(no_type_checks) + + arg_kinds = [arg.kind for arg in args] + arg_names = [arg.variable.name() for arg in args] + + body, comment_type = self.parse_block(allow_type=True) + # Potentially insert extra assignment statements to the beginning of the + # body, used to decompose Python 2 tuple arguments. + body.body[:0] = extra_stmts + if comment_type: + # The function has a # type: ... signature. + if typ: + self.errors.report( + def_tok.line, 'Function has duplicate type signatures') + sig = cast(CallableType, comment_type) + if sig.is_ellipsis_args: + # When we encounter an ellipsis, fill in the arg_types with + # a bunch of AnyTypes, emulating Callable[..., T] + arg_types = [AnyType()] * len(arg_kinds) # type: List[Type] + typ = CallableType( + arg_types, + arg_kinds, + arg_names, + sig.ret_type, + None, + line=def_tok.line) + elif is_method and len(sig.arg_kinds) < len(arg_kinds): + self.check_argument_kinds(arg_kinds, + [nodes.ARG_POS] + sig.arg_kinds, + def_tok.line) + # Add implicit 'self' argument to signature. + first_arg = [AnyType()] # type: List[Type] + typ = CallableType( + first_arg + sig.arg_types, + arg_kinds, + arg_names, + sig.ret_type, + None, + line=def_tok.line) + else: + self.check_argument_kinds(arg_kinds, sig.arg_kinds, + def_tok.line) + typ = CallableType( + sig.arg_types, + arg_kinds, + arg_names, + sig.ret_type, + None, + line=def_tok.line) + + # If there was a serious error, we really cannot build a parse tree + # node. + if is_error: + return None + + if typ and isinstance(typ.ret_type, UnboundType): + typ.ret_type.is_ret_type = True + + node = FuncDef(name, args, body, typ) + node.set_line(def_tok) + if typ is not None: + typ.definition = node + return node + finally: + self.errors.pop_function() + self.is_class_body = is_method + + def check_argument_kinds(self, funckinds: List[int], sigkinds: List[int], + line: int) -> None: + """Check that arguments are consistent. + + This verifies that they have the same number and the kinds correspond. + + Arguments: + funckinds: kinds of arguments in function definition + sigkinds: kinds of arguments in signature (after # type:) + """ + if len(funckinds) != len(sigkinds): + if len(funckinds) > len(sigkinds): + self.fail("Type signature has too few arguments", line) + else: + self.fail("Type signature has too many arguments", line) + return + for kind, token in [(nodes.ARG_STAR, '*'), + (nodes.ARG_STAR2, '**')]: + if ((funckinds.count(kind) != sigkinds.count(kind)) or + (kind in funckinds and sigkinds.index(kind) != funckinds.index(kind))): + self.fail( + "Inconsistent use of '{}' in function " + "signature".format(token), line) + + def parse_function_header( + self, no_type_checks: bool=False) -> Tuple[str, + List[Argument], + CallableType, + bool, + List[AssignmentStmt]]: + """Parse function header (a name followed by arguments) + + Return a 5-tuple with the following items: + name + arguments + signature (annotation) + error flag (True if error) + extra statements needed to decompose arguments (usually empty) + + See parse_arg_list for an explanation of the final tuple item. + """ + name = '' + + try: + name_tok = self.expect_type(Name) + name = name_tok.string + + self.errors.push_function(name) + + args, typ, extra_stmts = self.parse_args(no_type_checks) + except ParseError: + if not isinstance(self.current(), Break): + self.ind -= 1 # Kludge: go back to the Break token + # Resynchronise parsing by going back over :, if present. + if isinstance(self.tok[self.ind - 1], Colon): + self.ind -= 1 + return (name, [], None, True, []) + + return (name, args, typ, False, extra_stmts) + + def parse_args(self, no_type_checks: bool=False) -> Tuple[List[Argument], + CallableType, + List[AssignmentStmt]]: + """Parse a function signature (...) [-> t]. + + See parse_arg_list for an explanation of the final tuple item. + """ + lparen = self.expect('(') + + # Parse the argument list (everything within '(' and ')'). + args, extra_stmts = self.parse_arg_list(no_type_checks=no_type_checks) + + self.expect(')') + + if self.current_str() == '->': + self.skip() + if no_type_checks: + self.parse_expression() + ret_type = None + else: + ret_type = self.parse_type() + else: + ret_type = None + + arg_kinds = [arg.kind for arg in args] + self.verify_argument_kinds(arg_kinds, lparen.line) + + annotation = self.build_func_annotation( + ret_type, args, lparen.line) + + return args, annotation, extra_stmts + + def build_func_annotation(self, ret_type: Type, args: List[Argument], + line: int, is_default_ret: bool = False) -> CallableType: + arg_types = [arg.type_annotation for arg in args] + # Are there any type annotations? + if ((ret_type and not is_default_ret) + or arg_types != [None] * len(arg_types)): + # Yes. Construct a type for the function signature. + return self.construct_function_type(args, ret_type, line) + else: + return None + + def parse_arg_list(self, allow_signature: bool = True, + no_type_checks: bool=False) -> Tuple[List[Argument], + List[AssignmentStmt]]: + """Parse function definition argument list. + + This includes everything between '(' and ')' (but not the + parentheses). + + Return tuple (arguments, + extra statements for decomposing arguments). + + The final argument is only used for Python 2 argument lists with + tuples; they contain destructuring assignment statements used to + decompose tuple arguments. For example, consider a header like this: + + . def f((x, y)) + + The actual (sole) argument will be __tuple_arg_1 (a generated + name), whereas the extra statement list will contain a single + assignment statement corresponding to this assignment: + + x, y = __tuple_arg_1 + """ + args = [] # type: List[Argument] + extra_stmts = [] + # This is for checking duplicate argument names. + arg_names = [] # type: List[str] + has_tuple_arg = False + + require_named = False + bare_asterisk_before = -1 + + if self.current_str() != ')' and self.current_str() != ':': + while self.current_str() != ')': + if self.current_str() == '*' and self.peek().string == ',': + self.expect('*') + require_named = True + bare_asterisk_before = len(args) + elif self.current_str() in ['*', '**']: + if bare_asterisk_before == len(args): + # named arguments must follow bare * + self.parse_error() + + arg = self.parse_asterisk_arg( + allow_signature, + no_type_checks, + ) + args.append(arg) + require_named = True + elif self.current_str() == '(': + arg, extra_stmt, names = self.parse_tuple_arg(len(args)) + args.append(arg) + if extra_stmt is not None: + extra_stmts.append(extra_stmt) + has_tuple_arg = True + arg_names.extend(names) + else: + arg, require_named = self.parse_normal_arg( + require_named, + allow_signature, + no_type_checks, + ) + args.append(arg) + arg_names.append(arg.variable.name()) + + if self.current().string != ',': + break + + self.expect(',') + + # Non-tuple argument dupes will be checked elsewhere. Avoid + # generating duplicate errors. + if has_tuple_arg: + self.check_duplicate_argument_names(arg_names) + + return args, extra_stmts + + def check_duplicate_argument_names(self, names: List[str]) -> None: + found = set() # type: Set[str] + for name in names: + if name in found: + self.fail('Duplicate argument name "{}"'.format(name), + self.current().line) + found.add(name) + + def parse_asterisk_arg(self, + allow_signature: bool, + no_type_checks: bool) -> Argument: + asterisk = self.skip() + name = self.expect_type(Name) + variable = Var(name.string) + if asterisk.string == '*': + kind = nodes.ARG_STAR + else: + kind = nodes.ARG_STAR2 + + type = None + if no_type_checks: + self.parse_parameter_annotation() + else: + type = self.parse_arg_type(allow_signature) + + return Argument(variable, type, None, kind) + + def parse_tuple_arg(self, index: int) -> Tuple[Argument, AssignmentStmt, List[str]]: + """Parse a single Python 2 tuple argument. + + Example: def f(x, (y, z)): ... + + The tuple arguments gets transformed into an assignment in the + function body (the second return value). + + Return tuple (argument, decomposing assignment, list of names defined). + + Special case: if the argument is just (x) then it's not a tuple; + we indicate this by returning (argument, None, ['x']). + However, if the argument is (x,) then it *is* a (singleton) tuple. + """ + line = self.current().line + # Generate a new argument name that is very unlikely to clash with anything. + arg_name = '__tuple_arg_{}'.format(index + 1) + if self.pyversion[0] >= 3: + self.fail('Tuples in argument lists only supported in Python 2 mode', line) + paren_arg = self.parse_parentheses() + self.verify_tuple_arg(paren_arg) + if isinstance(paren_arg, NameExpr): + # This isn't a tuple. Revert to a normal argument. + arg_name = paren_arg.name + decompose = None + else: + rvalue = NameExpr(arg_name) + rvalue.set_line(line) + decompose = AssignmentStmt([paren_arg], rvalue) + decompose.set_line(line) + kind = nodes.ARG_POS + initializer = None + if self.current_str() == '=': + self.expect('=') + initializer = self.parse_expression(precedence[',']) + kind = nodes.ARG_OPT + var = Var(arg_name) + arg_names = self.find_tuple_arg_argument_names(paren_arg) + return Argument(var, None, initializer, kind), decompose, arg_names + + def verify_tuple_arg(self, paren_arg: Node) -> None: + if isinstance(paren_arg, TupleExpr): + if not paren_arg.items: + self.fail('Empty tuple not valid as an argument', paren_arg.line) + for item in paren_arg.items: + self.verify_tuple_arg(item) + elif not isinstance(paren_arg, NameExpr): + self.fail('Invalid item in tuple argument', paren_arg.line) + + def find_tuple_arg_argument_names(self, node: Node) -> List[str]: + result = [] # type: List[str] + if isinstance(node, TupleExpr): + for item in node.items: + result.extend(self.find_tuple_arg_argument_names(item)) + elif isinstance(node, NameExpr): + result.append(node.name) + return result + + def parse_normal_arg(self, require_named: bool, + allow_signature: bool, + no_type_checks: bool) -> Tuple[Argument, bool]: + name = self.expect_type(Name) + variable = Var(name.string) + + type = None + if no_type_checks: + self.parse_parameter_annotation() + else: + type = self.parse_arg_type(allow_signature) + + initializer = None # type: Node + if self.current_str() == '=': + self.expect('=') + initializer = self.parse_expression(precedence[',']) + if require_named: + kind = nodes.ARG_NAMED + else: + kind = nodes.ARG_OPT + else: + if require_named: + kind = nodes.ARG_NAMED + else: + kind = nodes.ARG_POS + + self.set_type_optional(type, initializer) + + return Argument(variable, type, initializer, kind), require_named + + def set_type_optional(self, type: Type, initializer: Node) -> None: + if not experiments.STRICT_OPTIONAL: + return + # Indicate that type should be wrapped in an Optional if arg is initialized to None. + optional = isinstance(initializer, NameExpr) and initializer.name == 'None' + if isinstance(type, UnboundType): + type.optional = optional + + def parse_parameter_annotation(self) -> Node: + if self.current_str() == ':': + self.skip() + return self.parse_expression(precedence[',']) + + def parse_arg_type(self, allow_signature: bool) -> Type: + if self.current_str() == ':' and allow_signature: + self.skip() + return self.parse_type() + else: + return None + + def verify_argument_kinds(self, kinds: List[int], line: int) -> None: + found = set() # type: Set[int] + for i, kind in enumerate(kinds): + if kind == nodes.ARG_POS and found & set([nodes.ARG_OPT, + nodes.ARG_STAR, + nodes.ARG_STAR2]): + self.fail('Invalid argument list', line) + elif kind == nodes.ARG_STAR and nodes.ARG_STAR in found: + self.fail('Invalid argument list', line) + elif kind == nodes.ARG_STAR2 and i != len(kinds) - 1: + self.fail('Invalid argument list', line) + found.add(kind) + + def construct_function_type(self, args: List[Argument], ret_type: Type, + line: int) -> CallableType: + # Complete the type annotation by replacing omitted types with 'Any'. + arg_types = [arg.type_annotation for arg in args] + for i in range(len(arg_types)): + if arg_types[i] is None: + arg_types[i] = AnyType(implicit=True) + if ret_type is None: + ret_type = AnyType(implicit=True) + arg_kinds = [arg.kind for arg in args] + arg_names = [arg.variable.name() for arg in args] + return CallableType(arg_types, arg_kinds, arg_names, ret_type, None, name=None, + variables=None, line=line) + + # Parsing statements + + def parse_block(self, allow_type: bool = False) -> Tuple[Block, Type]: + colon = self.expect(':') + if not isinstance(self.current(), Break): + # Block immediately after ':'. + nodes = [] + while True: + ind = self.ind + stmt, is_simple = self.parse_statement() + if not is_simple: + self.parse_error_at(self.tok[ind]) + break + nodes.append(stmt) + brk = self.expect_break() + if brk.string != ';': + break + node = Block(nodes) + node.set_line(colon) + return node, None + else: + # Indented block. + brk = self.expect_break() + type = self.parse_type_comment(brk, signature=True) + self.expect_indent() + stmt_list = [] # type: List[Node] + if allow_type: + cur = self.current() + if type is None and isinstance(cur, StrLit): + ds = docstring.parse_docstring(cur.parsed()) + if ds and False: # TODO: Enable when this is working. + try: + type = parse_str_as_signature(ds.as_type_str(), cur.line) + except TypeParseError: + # We don't require docstrings to be actually correct. + # TODO: Report something here. + type = None + while (not isinstance(self.current(), Dedent) and + not isinstance(self.current(), Eof)): + try: + stmt, is_simple = self.parse_statement() + if is_simple: + self.expect_break() + if stmt is not None: + if not self.try_combine_overloads(stmt, stmt_list): + stmt_list.append(stmt) + except ParseError: + pass + if isinstance(self.current(), Dedent): + self.skip() + node = Block(stmt_list) + node.set_line(colon) + return node, type + + def try_combine_overloads(self, s: Node, stmt: List[Node]) -> bool: + if isinstance(s, Decorator) and stmt: + fdef = s + n = fdef.func.name() + if isinstance(stmt[-1], Decorator) and stmt[-1].func.name() == n: + stmt[-1] = OverloadedFuncDef([stmt[-1], fdef]) + return True + elif isinstance(stmt[-1], OverloadedFuncDef) and stmt[-1].name() == n: + stmt[-1].items.append(fdef) + return True + return False + + def parse_statement(self) -> Tuple[Node, bool]: + stmt = None # type: Node + t = self.current() + ts = self.current_str() + is_simple = True # Is this a non-block statement? + if ts == 'if': + stmt = self.parse_if_stmt() + is_simple = False + elif ts == 'def': + stmt = self.parse_function() + is_simple = False + elif ts == 'while': + stmt = self.parse_while_stmt() + is_simple = False + elif ts == 'return': + stmt = self.parse_return_stmt() + elif ts == 'for': + stmt = self.parse_for_stmt() + is_simple = False + elif ts == 'try': + stmt = self.parse_try_stmt() + is_simple = False + elif ts == 'break': + stmt = self.parse_break_stmt() + elif ts == 'continue': + stmt = self.parse_continue_stmt() + elif ts == 'pass': + stmt = self.parse_pass_stmt() + elif ts == 'raise': + stmt = self.parse_raise_stmt() + elif ts == 'import': + stmt = self.parse_import() + elif ts == 'from': + stmt = self.parse_import_from() + elif ts == 'class': + stmt = self.parse_class_def() + is_simple = False + elif ts == 'global': + stmt = self.parse_global_decl() + elif ts == 'nonlocal' and self.pyversion[0] >= 3: + stmt = self.parse_nonlocal_decl() + elif ts == 'assert': + stmt = self.parse_assert_stmt() + elif ts == 'del': + stmt = self.parse_del_stmt() + elif ts == 'with': + stmt = self.parse_with_stmt() + is_simple = False + elif ts == '@': + stmt = self.parse_decorated_function_or_class() + is_simple = False + elif ts == 'print' and (self.pyversion[0] == 2 and + 'print_function' not in self.future_options): + stmt = self.parse_print_stmt() + elif ts == 'exec' and self.pyversion[0] == 2: + stmt = self.parse_exec_stmt() + else: + stmt = self.parse_expression_or_assignment() + if stmt is not None: + stmt.set_line(t) + return stmt, is_simple + + def parse_expression_or_assignment(self) -> Node: + expr = self.parse_expression(star_expr_allowed=True) + if self.current_str() == '=': + return self.parse_assignment(expr) + elif self.current_str() in op_assign: + # Operator assignment statement. + op = self.current_str()[:-1] + self.skip() + rvalue = self.parse_expression() + return OperatorAssignmentStmt(op, expr, rvalue) + else: + # Expression statement. + return ExpressionStmt(expr) + + def parse_assignment(self, lvalue: Any) -> Node: + """Parse an assignment statement. + + Assume that lvalue has been parsed already, and the current token is '='. + Also parse an optional '# type:' comment. + """ + self.expect('=') + lvalues = [lvalue] + expr = self.parse_expression(star_expr_allowed=True) + while self.current_str() == '=': + self.skip() + lvalues.append(expr) + expr = self.parse_expression(star_expr_allowed=True) + cur = self.current() + if isinstance(cur, Break): + type = self.parse_type_comment(cur, signature=False) + else: + type = None + return AssignmentStmt(lvalues, expr, type) + + def parse_return_stmt(self) -> ReturnStmt: + self.expect('return') + expr = None + current = self.current() + if current.string == 'yield': + self.parse_error() + if not isinstance(current, Break): + expr = self.parse_expression() + node = ReturnStmt(expr) + return node + + def parse_raise_stmt(self) -> RaiseStmt: + self.expect('raise') + expr = None + from_expr = None + if not isinstance(self.current(), Break): + expr = self.parse_expression() + if self.current_str() == 'from': + self.expect('from') + from_expr = self.parse_expression() + node = RaiseStmt(expr, from_expr) + return node + + def parse_assert_stmt(self) -> AssertStmt: + self.expect('assert') + expr = self.parse_expression() + node = AssertStmt(expr) + return node + + def parse_yield_or_yield_from_expr(self) -> Union[YieldFromExpr, YieldExpr]: + self.expect("yield") + expr = None + node = YieldExpr(expr) # type: Union[YieldFromExpr, YieldExpr] + if not isinstance(self.current(), Break): + if self.current_str() == "from": + self.expect("from") + expr = self.parse_expression() # when yield from is assigned to a variable + node = YieldFromExpr(expr) + else: + if self.current_str() == ')': + node = YieldExpr(None) + else: + expr = self.parse_expression() + node = YieldExpr(expr) + return node + + def parse_ellipsis(self) -> EllipsisExpr: + self.expect('...') + node = EllipsisExpr() + return node + + def parse_del_stmt(self) -> DelStmt: + self.expect('del') + expr = self.parse_expression() + node = DelStmt(expr) + return node + + def parse_break_stmt(self) -> BreakStmt: + self.expect('break') + node = BreakStmt() + return node + + def parse_continue_stmt(self) -> ContinueStmt: + self.expect('continue') + node = ContinueStmt() + return node + + def parse_pass_stmt(self) -> PassStmt: + self.expect('pass') + node = PassStmt() + return node + + def parse_global_decl(self) -> GlobalDecl: + self.expect('global') + names = self.parse_identifier_list() + node = GlobalDecl(names) + return node + + def parse_nonlocal_decl(self) -> NonlocalDecl: + self.expect('nonlocal') + names = self.parse_identifier_list() + node = NonlocalDecl(names) + return node + + def parse_identifier_list(self) -> List[str]: + names = [] + while True: + n = self.expect_type(Name) + names.append(n.string) + if self.current_str() != ',': + break + self.skip() + return names + + def parse_while_stmt(self) -> WhileStmt: + is_error = False + self.expect('while') + try: + expr = self.parse_expression() + except ParseError: + is_error = True + body, _ = self.parse_block() + if self.current_str() == 'else': + self.expect('else') + else_body, _ = self.parse_block() + else: + else_body = None + if is_error is not None: + node = WhileStmt(expr, body, else_body) + return node + else: + return None + + def parse_for_stmt(self) -> ForStmt: + self.expect('for') + index = self.parse_for_index_variables() + self.expect('in') + expr = self.parse_expression() + + body, _ = self.parse_block() + + if self.current_str() == 'else': + self.expect('else') + else_body, _ = self.parse_block() + else: + else_body = None + + node = ForStmt(index, expr, body, else_body) + return node + + def parse_for_index_variables(self) -> Node: + # Parse index variables of a 'for' statement. + index_items = [] + force_tuple = False + + while True: + v = self.parse_expression(precedence['in'], + star_expr_allowed=True) # Prevent parsing of for stmt 'in' + index_items.append(v) + if self.current_str() != ',': + break + self.skip() + if self.current_str() == 'in': + force_tuple = True + break + + if len(index_items) == 1 and not force_tuple: + index = index_items[0] + else: + index = TupleExpr(index_items) + index.set_line(index_items[0].get_line()) + + return index + + def parse_if_stmt(self) -> IfStmt: + is_error = False + + self.expect('if') + expr = [] + try: + expr.append(self.parse_expression()) + except ParseError: + is_error = True + + body = [self.parse_block()[0]] + + while self.current_str() == 'elif': + self.expect('elif') + try: + expr.append(self.parse_expression()) + except ParseError: + is_error = True + body.append(self.parse_block()[0]) + + if self.current_str() == 'else': + self.expect('else') + else_body, _ = self.parse_block() + else: + else_body = None + + if not is_error: + node = IfStmt(expr, body, else_body) + return node + else: + return None + + def parse_try_stmt(self) -> Node: + self.expect('try') + body, _ = self.parse_block() + is_error = False + vars = [] # type: List[NameExpr] + types = [] # type: List[Node] + handlers = [] # type: List[Block] + while self.current_str() == 'except': + self.expect('except') + if not isinstance(self.current(), Colon): + try: + t = self.current() + types.append(self.parse_expression(precedence[',']).set_line(t)) + if self.current_str() == 'as': + self.expect('as') + vars.append(self.parse_name_expr()) + elif self.pyversion[0] == 2 and self.current_str() == ',': + self.expect(',') + vars.append(self.parse_name_expr()) + else: + vars.append(None) + except ParseError: + is_error = True + else: + types.append(None) + vars.append(None) + handlers.append(self.parse_block()[0]) + if not is_error: + if self.current_str() == 'else': + self.skip() + else_body, _ = self.parse_block() + else: + else_body = None + if self.current_str() == 'finally': + self.expect('finally') + finally_body, _ = self.parse_block() + else: + finally_body = None + node = TryStmt(body, vars, types, handlers, else_body, + finally_body) + return node + else: + return None + + def parse_with_stmt(self) -> WithStmt: + self.expect('with') + exprs = [] + targets = [] + while True: + expr = self.parse_expression(precedence[',']) + if self.current_str() == 'as': + self.expect('as') + target = self.parse_expression(precedence[',']) + else: + target = None + exprs.append(expr) + targets.append(target) + if self.current_str() != ',': + break + self.expect(',') + body, _ = self.parse_block() + return WithStmt(exprs, targets, body) + + def parse_print_stmt(self) -> PrintStmt: + self.expect('print') + args = [] + target = None + if self.current_str() == '>>': + self.skip() + target = self.parse_expression(precedence[',']) + if self.current_str() == ',': + self.skip() + if isinstance(self.current(), Break): + self.parse_error() + else: + if not isinstance(self.current(), Break): + self.parse_error() + comma = False + while not isinstance(self.current(), Break): + args.append(self.parse_expression(precedence[','])) + if self.current_str() == ',': + comma = True + self.skip() + else: + comma = False + break + return PrintStmt(args, newline=not comma, target=target) + + def parse_exec_stmt(self) -> ExecStmt: + self.expect('exec') + expr = self.parse_expression(precedence['in']) + variables1 = None + variables2 = None + if self.current_str() == 'in': + self.skip() + variables1 = self.parse_expression(precedence[',']) + if self.current_str() == ',': + self.skip() + variables2 = self.parse_expression(precedence[',']) + return ExecStmt(expr, variables1, variables2) + + # Parsing expressions + + def parse_expression(self, prec: int = 0, star_expr_allowed: bool = False) -> Node: + """Parse a subexpression within a specific precedence context.""" + expr = None # type: Node + current = self.current() # Remember token for setting the line number. + + # Parse a "value" expression or unary operator expression and store + # that in expr. + s = self.current_str() + if s == '(': + # Parenthesised expression or cast. + expr = self.parse_parentheses() + elif s == '[': + expr = self.parse_list_expr() + elif s in ['-', '+', 'not', '~']: + # Unary operation. + expr = self.parse_unary_expr() + elif s == 'lambda': + expr = self.parse_lambda_expr() + elif s == '{': + expr = self.parse_dict_or_set_expr() + elif s == '*' and star_expr_allowed: + expr = self.parse_star_expr() + elif s == '`' and self.pyversion[0] == 2: + expr = self.parse_backquote_expr() + else: + if isinstance(current, Name): + # Name expression. + expr = self.parse_name_expr() + elif isinstance(current, IntLit): + expr = self.parse_int_expr() + elif isinstance(current, StrLit): + expr = self.parse_str_expr() + elif isinstance(current, BytesLit): + expr = self.parse_bytes_literal() + elif isinstance(current, UnicodeLit): + expr = self.parse_unicode_literal() + elif isinstance(current, FloatLit): + expr = self.parse_float_expr() + elif isinstance(current, ComplexLit): + expr = self.parse_complex_expr() + elif isinstance(current, Keyword) and s == "yield": + # The expression yield from and yield to assign + expr = self.parse_yield_or_yield_from_expr() + elif isinstance(current, EllipsisToken) and (self.pyversion[0] >= 3 + or self.is_stub_file): + expr = self.parse_ellipsis() + else: + # Invalid expression. + self.parse_error() + + # Set the line of the expression node, if not specified. This + # simplifies recording the line number as not every node type needs to + # deal with it separately. + if expr.line < 0: + expr.set_line(current) + + # Parse operations that require a left argument (stored in expr). + while True: + current = self.current() + s = self.current_str() + if s == '(': + # Call expression. + expr = self.parse_call_expr(expr) + elif s == '.': + # Member access expression. + expr = self.parse_member_expr(expr) + elif s == '[': + # Indexing expression. + expr = self.parse_index_expr(expr) + elif s == ',': + # The comma operator is used to build tuples. Comma also + # separates array items and function arguments, but in this + # case the precedence is too low to build a tuple. + if precedence[','] > prec: + expr = self.parse_tuple_expr(expr) + else: + break + elif s == 'for': + if precedence[''] > prec: + # List comprehension or generator expression. Parse as + # generator expression; it will be converted to list + # comprehension if needed elsewhere. + expr = self.parse_generator_expr(expr) + else: + break + elif s == 'if': + # Conditional expression. + if precedence[''] > prec: + expr = self.parse_conditional_expr(expr) + else: + break + else: + # Binary operation or a special case. + if isinstance(current, Op): + op = self.current_str() + op_prec = precedence[op] + if op == 'not': + # Either "not in" or an error. + op_prec = precedence['in'] + if op_prec > prec: + if op in op_comp: + expr = self.parse_comparison_expr(expr, op_prec) + else: + expr = self.parse_bin_op_expr(expr, op_prec) + else: + # The operation cannot be associated with the + # current left operand due to the precedence + # context; let the caller handle it. + break + else: + # Not an operation that accepts a left argument; let the + # caller handle the rest. + break + + # Set the line of the expression node, if not specified. This + # simplifies recording the line number as not every node type + # needs to deal with it separately. + if expr.line < 0: + expr.set_line(current) + + return expr + + def parse_parentheses(self) -> Node: + self.skip() + if self.current_str() == ')': + # Empty tuple (). + expr = self.parse_empty_tuple_expr() # type: Node + else: + # Parenthesised expression. + expr = self.parse_expression(0, star_expr_allowed=True) + self.expect(')') + return expr + + def parse_star_expr(self) -> Node: + star = self.expect('*') + expr = self.parse_expression(precedence['*u']) + expr = StarExpr(expr) + if expr.line < 0: + expr.set_line(star) + return expr + + def parse_empty_tuple_expr(self) -> TupleExpr: + self.expect(')') + node = TupleExpr([]) + return node + + def parse_list_expr(self) -> Node: + """Parse list literal or list comprehension.""" + items = [] + self.expect('[') + while self.current_str() != ']' and not self.eol(): + items.append(self.parse_expression(precedence[''], star_expr_allowed=True)) + if self.current_str() != ',': + break + self.expect(',') + if self.current_str() == 'for' and len(items) == 1: + items[0] = self.parse_generator_expr(items[0]) + self.expect(']') + if len(items) == 1 and isinstance(items[0], GeneratorExpr): + return ListComprehension(items[0]) + else: + expr = ListExpr(items) + return expr + + def parse_generator_expr(self, left_expr: Node) -> GeneratorExpr: + tok = self.current() + indices, sequences, condlists = self.parse_comp_for() + + gen = GeneratorExpr(left_expr, indices, sequences, condlists) + gen.set_line(tok) + return gen + + def parse_comp_for(self) -> Tuple[List[Node], List[Node], List[List[Node]]]: + indices = [] + sequences = [] + condlists = [] # type: List[List[Node]] + while self.current_str() == 'for': + conds = [] + self.expect('for') + index = self.parse_for_index_variables() + indices.append(index) + self.expect('in') + if self.pyversion[0] >= 3: + sequence = self.parse_expression(precedence['']) + else: + sequence = self.parse_expression_list() + sequences.append(sequence) + while self.current_str() == 'if': + self.skip() + conds.append(self.parse_expression(precedence[''])) + condlists.append(conds) + + return indices, sequences, condlists + + def parse_expression_list(self) -> Node: + prec = precedence[''] + expr = self.parse_expression(prec) + if self.current_str() != ',': + return expr + else: + t = self.current() + return self.parse_tuple_expr(expr, prec).set_line(t) + + def parse_conditional_expr(self, left_expr: Node) -> ConditionalExpr: + self.expect('if') + cond = self.parse_expression(precedence['']) + self.expect('else') + else_expr = self.parse_expression(precedence['']) + return ConditionalExpr(cond, left_expr, else_expr) + + def parse_dict_or_set_expr(self) -> Node: + items = [] # type: List[Tuple[Node, Node]] + self.expect('{') + while self.current_str() != '}' and not self.eol(): + key = self.parse_expression(precedence['']) + if self.current_str() in [',', '}'] and items == []: + return self.parse_set_expr(key) + elif self.current_str() == 'for' and items == []: + return self.parse_set_comprehension(key) + elif self.current_str() != ':': + self.parse_error() + colon = self.expect(':') + value = self.parse_expression(precedence['']) + if self.current_str() == 'for' and items == []: + return self.parse_dict_comprehension(key, value, colon) + items.append((key, value)) + if self.current_str() != ',': + break + self.expect(',') + self.expect('}') + node = DictExpr(items) + return node + + def parse_set_expr(self, first: Node) -> SetExpr: + items = [first] + while self.current_str() != '}' and not self.eol(): + self.expect(',') + if self.current_str() == '}': + break + items.append(self.parse_expression(precedence[','])) + self.expect('}') + expr = SetExpr(items) + return expr + + def parse_set_comprehension(self, expr: Node): + gen = self.parse_generator_expr(expr) + self.expect('}') + set_comp = SetComprehension(gen) + return set_comp + + def parse_dict_comprehension(self, key: Node, value: Node, + colon: Token) -> DictionaryComprehension: + indices, sequences, condlists = self.parse_comp_for() + dic = DictionaryComprehension(key, value, indices, sequences, condlists) + dic.set_line(colon) + self.expect('}') + return dic + + def parse_tuple_expr(self, expr: Node, + prec: int = precedence[',']) -> TupleExpr: + items = [expr] + while True: + self.expect(',') + if (self.current_str() in [')', ']', '=', ':'] or + isinstance(self.current(), Break)): + break + items.append(self.parse_expression(prec, star_expr_allowed=True)) + if self.current_str() != ',': break + node = TupleExpr(items) + return node + + def parse_backquote_expr(self) -> BackquoteExpr: + self.expect('`') + expr = self.parse_expression() + self.expect('`') + return BackquoteExpr(expr) + + def parse_name_expr(self) -> NameExpr: + tok = self.expect_type(Name) + node = NameExpr(tok.string) + node.set_line(tok) + return node + + octal_int = re.compile('0+[1-9]') + + def parse_int_expr(self) -> IntExpr: + tok = self.expect_type(IntLit) + string = tok.string.rstrip('lL') # Strip L prefix (Python 2 long literals) + if self.octal_int.match(string): + value = int(string, 8) + else: + value = int(string, 0) + node = IntExpr(value) + return node + + def parse_str_expr(self) -> Node: + # XXX \uxxxx literals + token = self.expect_type(StrLit) + value = cast(StrLit, token).parsed() + is_unicode = False + while isinstance(self.current(), (StrLit, UnicodeLit)): + token = self.skip() + if isinstance(token, StrLit): + value += token.parsed() + elif isinstance(token, UnicodeLit): + value += token.parsed() + is_unicode = True + if is_unicode or (self.pyversion[0] == 2 and 'unicode_literals' in self.future_options): + node = UnicodeExpr(value) # type: Node + else: + node = StrExpr(value) + return node + + def parse_bytes_literal(self) -> Node: + # XXX \uxxxx literals + tok = [self.expect_type(BytesLit)] + value = (cast(BytesLit, tok[0])).parsed() + while isinstance(self.current(), BytesLit): + t = cast(BytesLit, self.skip()) + value += t.parsed() + if self.pyversion[0] >= 3: + node = BytesExpr(value) # type: Node + else: + node = StrExpr(value) + return node + + def parse_unicode_literal(self) -> Node: + # XXX \uxxxx literals + token = self.expect_type(UnicodeLit) + value = cast(UnicodeLit, token).parsed() + while isinstance(self.current(), (UnicodeLit, StrLit)): + token = cast(Union[UnicodeLit, StrLit], self.skip()) + value += token.parsed() + if self.pyversion[0] >= 3: + # Python 3.3 supports u'...' as an alias of '...'. + node = StrExpr(value) # type: Node + else: + node = UnicodeExpr(value) + return node + + def parse_float_expr(self) -> FloatExpr: + tok = self.expect_type(FloatLit) + node = FloatExpr(float(tok.string)) + return node + + def parse_complex_expr(self) -> ComplexExpr: + tok = self.expect_type(ComplexLit) + node = ComplexExpr(complex(tok.string)) + return node + + def parse_call_expr(self, callee: Any) -> CallExpr: + self.expect('(') + args, kinds, names = self.parse_arg_expr() + self.expect(')') + node = CallExpr(callee, args, kinds, names) + return node + + def parse_arg_expr(self) -> Tuple[List[Node], List[int], List[str]]: + """Parse arguments in a call expression (within '(' and ')'). + + Return a tuple with these items: + argument expressions + argument kinds + argument names (for named arguments; None for ordinary args) + """ + args = [] # type: List[Node] + kinds = [] # type: List[int] + names = [] # type: List[str] + var_arg = False + dict_arg = False + named_args = False + while self.current_str() != ')' and not self.eol() and not dict_arg: + if isinstance(self.current(), Name) and self.peek().string == '=': + # Named argument + name = self.expect_type(Name) + self.expect('=') + kinds.append(nodes.ARG_NAMED) + names.append(name.string) + named_args = True + elif (self.current_str() == '*' and not var_arg and not dict_arg): + # *args + var_arg = True + self.expect('*') + kinds.append(nodes.ARG_STAR) + names.append(None) + elif self.current_str() == '**': + # **kwargs + self.expect('**') + dict_arg = True + kinds.append(nodes.ARG_STAR2) + names.append(None) + elif not var_arg and not named_args: + # Ordinary argument + kinds.append(nodes.ARG_POS) + names.append(None) + else: + self.parse_error() + args.append(self.parse_expression(precedence[','])) + if self.current_str() != ',': + break + self.expect(',') + return args, kinds, names + + def parse_member_expr(self, expr: Any) -> Node: + self.expect('.') + name = self.expect_type(Name) + if (isinstance(expr, CallExpr) and isinstance(expr.callee, NameExpr) + and expr.callee.name == 'super'): + # super() expression + node = SuperExpr(name.string) # type: Node + else: + node = MemberExpr(expr, name.string) + return node + + def parse_index_expr(self, base: Any) -> IndexExpr: + self.expect('[') + index = self.parse_slice_item() + if self.current_str() == ',': + # Extended slicing such as x[1:, :2]. + items = [index] + while self.current_str() == ',': + self.skip() + if self.current_str() == ']' or isinstance(self.current(), Break): + break + items.append(self.parse_slice_item()) + index = TupleExpr(items) + index.set_line(items[0].line) + self.expect(']') + node = IndexExpr(base, index) + return node + + def parse_slice_item(self) -> Node: + if self.current_str() != ':': + if self.current_str() == '...': + # Ellipsis is valid here even in Python 2 (but not elsewhere). + ellipsis = EllipsisExpr() + token = self.skip() + ellipsis.set_line(token) + return ellipsis + else: + item = self.parse_expression(precedence[',']) + else: + item = None + if self.current_str() == ':': + # Slice. + index = item + colon = self.expect(':') + if self.current_str() not in (']', ':', ','): + end_index = self.parse_expression(precedence[',']) + else: + end_index = None + stride = None + if self.current_str() == ':': + self.expect(':') + if self.current_str() not in (']', ','): + stride = self.parse_expression(precedence[',']) + item = SliceExpr(index, end_index, stride).set_line(colon.line) + return item + + def parse_bin_op_expr(self, left: Node, prec: int) -> OpExpr: + op = self.expect_type(Op) + op_str = op.string + if op_str == '~': + self.ind -= 1 + self.parse_error() + right = self.parse_expression(prec) + node = OpExpr(op_str, left, right) + return node + + def parse_comparison_expr(self, left: Node, prec: int) -> ComparisonExpr: + operators_str = [] + operands = [left] + + while True: + op = self.expect_type(Op) + op_str = op.string + if op_str == 'not': + if self.current_str() == 'in': + op_str = 'not in' + self.skip() + else: + self.parse_error() + elif op_str == 'is' and self.current_str() == 'not': + op_str = 'is not' + self.skip() + + operators_str.append(op_str) + operand = self.parse_expression(prec) + operands.append(operand) + + # Continue if next token is a comparison operator + self.current() + s = self.current_str() + if s not in op_comp: + break + + node = ComparisonExpr(operators_str, operands) + return node + + def parse_unary_expr(self) -> UnaryExpr: + op_tok = self.skip() + op = op_tok.string + if op == '-' or op == '+': + prec = precedence['-u'] + else: + prec = precedence[op] + expr = self.parse_expression(prec) + node = UnaryExpr(op, expr) + return node + + def parse_lambda_expr(self) -> FuncExpr: + lambda_tok = self.expect('lambda') + + args, extra_stmts = self.parse_arg_list(allow_signature=False) + + # Use 'object' as the placeholder return type; it will be inferred + # later. We can't use 'Any' since it could make type inference results + # less precise. + ret_type = UnboundType('__builtins__.object') + typ = self.build_func_annotation(ret_type, args, + lambda_tok.line, is_default_ret=True) + + colon = self.expect(':') + + expr = self.parse_expression(precedence[',']) + + nodes = [ReturnStmt(expr).set_line(lambda_tok)] + # Potentially insert extra assignment statements to the beginning of the + # body, used to decompose Python 2 tuple arguments. + nodes[:0] = extra_stmts + body = Block(nodes) + body.set_line(colon) + + return FuncExpr(args, body, typ) + + # Helper methods + + def skip(self) -> Token: + self.ind += 1 + return self.tok[self.ind - 1] + + def expect(self, string: str) -> Token: + if self.current_str() == string: + self.ind += 1 + return self.tok[self.ind - 1] + else: + self.parse_error() + + def expect_indent(self) -> Token: + if isinstance(self.current(), Indent): + return self.expect_type(Indent) + else: + self.fail('Expected an indented block', self.current().line) + return none + + def fail(self, msg: str, line: int) -> None: + self.errors.report(line, msg) + + def expect_type(self, typ: type) -> Token: + current = self.current() + if isinstance(current, typ): + self.ind += 1 + return current + else: + self.parse_error() + + def expect_colon_and_break(self) -> Tuple[Token, Token]: + return self.expect_type(Colon), self.expect_type(Break) + + def expect_break(self) -> Token: + return self.expect_type(Break) + + def current(self) -> Token: + return self.tok[self.ind] + + def current_str(self) -> str: + return self.current().string + + def peek(self) -> Token: + return self.tok[self.ind + 1] + + def parse_error(self) -> None: + self.parse_error_at(self.current()) + raise ParseError() + + def parse_error_at(self, tok: Token, skip: bool = True, reason: Optional[str] = None) -> None: + msg = '' + if isinstance(tok, LexError): + msg = token_repr(tok) + msg = msg[0].upper() + msg[1:] + elif isinstance(tok, Indent) or isinstance(tok, Dedent): + msg = 'Inconsistent indentation' + else: + formatted_reason = ": {}".format(reason) if reason else "" + msg = 'Parse error before {}{}'.format(token_repr(tok), formatted_reason) + + self.errors.report(tok.line, msg) + + if skip: + self.skip_until_next_line() + + def skip_until_break(self) -> None: + n = 0 + while (not isinstance(self.current(), Break) + and not isinstance(self.current(), Eof)): + self.skip() + n += 1 + if isinstance(self.tok[self.ind - 1], Colon) and n > 1: + self.ind -= 1 + + def skip_until_next_line(self) -> None: + self.skip_until_break() + if isinstance(self.current(), Break): + self.skip() + + def eol(self) -> bool: + return isinstance(self.current(), Break) or self.eof() + + def eof(self) -> bool: + return isinstance(self.current(), Eof) + + # Type annotation related functionality + + def parse_type(self) -> Type: + try: + typ, self.ind = parse_type(self.tok, self.ind) + except TypeParseError as e: + self.parse_error_at(e.token, reason=e.message) + raise ParseError() + return typ + + annotation_prefix_re = re.compile(r'#\s*type:') + ignore_prefix_re = re.compile(r'ignore\b') + + def parse_type_comment(self, token: Token, signature: bool) -> Type: + """Parse a '# type: ...' annotation. + + Return None if no annotation found. If signature is True, expect + a type signature of form (...) -> t. + """ + whitespace_or_comments = token.rep().strip() + if self.annotation_prefix_re.match(whitespace_or_comments): + type_as_str = whitespace_or_comments.split(':', 1)[1].strip() + if self.ignore_prefix_re.match(type_as_str): + # Actually a "# type: ignore" annotation -> not a type. + return None + tokens = lex.lex(type_as_str, token.line)[0] + if len(tokens) < 2: + # Empty annotation (only Eof token) + self.errors.report(token.line, 'Empty type annotation') + return None + try: + if not signature: + type, index = parse_types(tokens, 0) + else: + type, index = parse_signature(tokens) + except TypeParseError as e: + self.parse_error_at(e.token, skip=False, reason=e.message) + return None + if index < len(tokens) - 2: + self.parse_error_at(tokens[index], skip=False) + return None + return type + else: + return None + + +class ParseError(Exception): pass + + +def token_repr(tok: Token) -> str: + """Return a representation of a token for use in parse error messages.""" + if isinstance(tok, Break): + return 'end of line' + elif isinstance(tok, Eof): + return 'end of file' + elif isinstance(tok, Keyword) or isinstance(tok, Name): + return '"{}"'.format(tok.string) + elif isinstance(tok, IntLit) or isinstance(tok, FloatLit) or isinstance(tok, ComplexLit): + return 'numeric literal' + elif isinstance(tok, StrLit) or isinstance(tok, UnicodeLit): + return 'string literal' + elif (isinstance(tok, Punct) or isinstance(tok, Op) + or isinstance(tok, Colon)): + return tok.string + elif isinstance(tok, Bom): + return 'byte order mark' + elif isinstance(tok, Indent): + return 'indent' + elif isinstance(tok, Dedent): + return 'dedent' + elif isinstance(tok, EllipsisToken): + return '...' + else: + if isinstance(tok, LexError): + t = tok.type + if t == lex.NUMERIC_LITERAL_ERROR: + return 'invalid numeric literal' + elif t == lex.UNTERMINATED_STRING_LITERAL: + return 'unterminated string literal' + elif t == lex.INVALID_CHARACTER: + msg = 'unrecognized character' + if ord(tok.string) in range(33, 127): + msg += ' ' + tok.string + return msg + elif t == lex.INVALID_DEDENT: + return 'inconsistent indentation' + elif t == lex.DECODE_ERROR: + return tok.message + raise ValueError('Unknown token {}'.format(repr(tok))) + + +if __name__ == '__main__': + # Parse a file and dump the AST (or display errors). + import sys + + def usage(): + print('Usage: parse.py [--py2] [--quiet] FILE [...]', file=sys.stderr) + sys.exit(2) + + args = sys.argv[1:] + pyversion = defaults.PYTHON3_VERSION + quiet = False + while args and args[0].startswith('--'): + if args[0] == '--py2': + pyversion = defaults.PYTHON2_VERSION + elif args[0] == '--quiet': + quiet = True + else: + usage() + args = args[1:] + if len(args) < 1: + usage() + status = 0 + for fnam in args: + s = open(fnam, 'rb').read() + errors = Errors() + try: + options = Options() + options.python_version = pyversion + tree = parse(s, fnam, None, options=options) + if not quiet: + print(tree) + except CompileError as e: + for msg in e.messages: + sys.stderr.write('%s\n' % msg) + status = 1 + sys.exit(status) diff --git a/mypy/semanal.py b/mypy/semanal.py index 0f4331499d7cc..8ab29f560a4e7 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1,2839 +1,2839 @@ -"""The semantic analyzer. - -Bind names to definitions and do various other simple consistency -checks. For example, consider this program: - - x = 1 - y = x - -Here semantic analysis would detect that the assignment 'x = 1' -defines a new variable, the type of which is to be inferred (in a -later pass; type inference or type checking is not part of semantic -analysis). Also, it would bind both references to 'x' to the same -module-level variable node. The second assignment would also be -analyzed, and the type of 'y' marked as being inferred. - -Semantic analysis is the first analysis pass after parsing, and it is -subdivided into three passes: - - * FirstPass looks up externally visible names defined in a module but - ignores imports and local definitions. It helps enable (some) - cyclic references between modules, such as module 'a' that imports - module 'b' and used names defined in b *and* vice versa. The first - pass can be performed before dependent modules have been processed. - - * SemanticAnalyzer is the second pass. It does the bulk of the work. - It assumes that dependent modules have been semantically analyzed, - up to the second pass, unless there is a import cycle. - - * ThirdPass checks that type argument counts are valid; for example, - it will reject Dict[int]. We don't do this in the second pass, - since we infer the type argument counts of classes during this - pass, and it is possible to refer to classes defined later in a - file, which would not have the type argument count set yet. This - pass also recomputes the method resolution order of each class, in - case one of its bases belongs to a module involved in an import - loop. - -Semantic analysis of types is implemented in module mypy.typeanal. - -TODO: Check if the third pass slows down type checking significantly. - We could probably get rid of it -- for example, we could collect all - analyzed types in a collection and check them without having to - traverse the entire AST. -""" - -from typing import ( - List, Dict, Set, Tuple, cast, Any, overload, TypeVar, Union, Optional, Callable -) - -from mypy.nodes import ( - MypyFile, TypeInfo, Node, AssignmentStmt, FuncDef, OverloadedFuncDef, - ClassDef, Var, GDEF, MODULE_REF, FuncItem, Import, - ImportFrom, ImportAll, Block, LDEF, NameExpr, MemberExpr, - IndexExpr, TupleExpr, ListExpr, ExpressionStmt, ReturnStmt, - RaiseStmt, AssertStmt, OperatorAssignmentStmt, WhileStmt, - ForStmt, BreakStmt, ContinueStmt, IfStmt, TryStmt, WithStmt, DelStmt, - GlobalDecl, SuperExpr, DictExpr, CallExpr, RefExpr, OpExpr, UnaryExpr, - SliceExpr, CastExpr, RevealTypeExpr, TypeApplication, Context, SymbolTable, - SymbolTableNode, BOUND_TVAR, UNBOUND_TVAR, ListComprehension, GeneratorExpr, - FuncExpr, MDEF, FuncBase, Decorator, SetExpr, TypeVarExpr, - StrExpr, BytesExpr, PrintStmt, ConditionalExpr, PromoteExpr, - ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, MroError, type_aliases, - YieldFromExpr, NamedTupleExpr, NonlocalDecl, - SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr, - YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, COVARIANT, CONTRAVARIANT, - IntExpr, FloatExpr, UnicodeExpr, - INVARIANT, UNBOUND_IMPORTED, Expression, EllipsisExpr, NamedTupleTypeInfo -) -from mypy.visitor import NodeVisitor -from mypy.traverser import TraverserVisitor -from mypy.errors import Errors, report_internal_error -from mypy.types import ( - NoneTyp, CallableType, Overloaded, Instance, Type, TypeVarType, AnyType, - FunctionLike, UnboundType, TypeList, ErrorType, TypeVarDef, - replace_leading_arg_type, TupleType, UnionType, StarType, EllipsisType, - NamedTupleType -) -from mypy.nodes import function_type, implicit_module_attrs -from mypy.typeanal import TypeAnalyser, TypeAnalyserPass3, analyze_type_alias -from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError -from mypy.lex import lex -from mypy.parsetype import parse_type -from mypy.sametypes import is_same_type -from mypy.erasetype import erase_typevars -from mypy import defaults -from mypy.options import Options - - -T = TypeVar('T') - - -# Inferred value of an expression. -ALWAYS_TRUE = 0 -ALWAYS_FALSE = 1 -TRUTH_VALUE_UNKNOWN = 2 - -# Map from obsolete name to the current spelling. -obsolete_name_mapping = { - 'typing.Function': 'typing.Callable', - 'typing.typevar': 'typing.TypeVar', -} - -# Hard coded type promotions (shared between all Python versions). -# These add extra ad-hoc edges to the subtyping relation. For example, -# int is considered a subtype of float, even though there is no -# subclass relationship. -TYPE_PROMOTIONS = { - 'builtins.int': 'builtins.float', - 'builtins.float': 'builtins.complex', -} - -# Hard coded type promotions for Python 3. -# -# Note that the bytearray -> bytes promotion is a little unsafe -# as some functions only accept bytes objects. Here convenience -# trumps safety. -TYPE_PROMOTIONS_PYTHON3 = TYPE_PROMOTIONS.copy() -TYPE_PROMOTIONS_PYTHON3.update({ - 'builtins.bytearray': 'builtins.bytes', -}) - -# Hard coded type promotions for Python 2. -# -# These promotions are unsafe, but we are doing them anyway -# for convenience and also for Python 3 compatibility -# (bytearray -> str). -TYPE_PROMOTIONS_PYTHON2 = TYPE_PROMOTIONS.copy() -TYPE_PROMOTIONS_PYTHON2.update({ - 'builtins.str': 'builtins.unicode', - 'builtins.bytearray': 'builtins.str', -}) - -# When analyzing a function, should we analyze the whole function in one go, or -# should we only perform one phase of the analysis? The latter is used for -# nested functions. In the first phase we add the function to the symbol table -# but don't process body. In the second phase we process function body. This -# way we can have mutually recursive nested functions. -FUNCTION_BOTH_PHASES = 0 # Everthing in one go -FUNCTION_FIRST_PHASE_POSTPONE_SECOND = 1 # Add to symbol table but postpone body -FUNCTION_SECOND_PHASE = 2 # Only analyze body - - -class SemanticAnalyzer(NodeVisitor): - """Semantically analyze parsed mypy files. - - The analyzer binds names and does various consistency checks for a - parse tree. Note that type checking is performed as a separate - pass. - - This is the second phase of semantic analysis. - """ - - # Library search paths - lib_path = None # type: List[str] - # Module name space - modules = None # type: Dict[str, MypyFile] - # Global name space for current module - globals = None # type: SymbolTable - # Names declared using "global" (separate set for each scope) - global_decls = None # type: List[Set[str]] - # Names declated using "nonlocal" (separate set for each scope) - nonlocal_decls = None # type: List[Set[str]] - # Local names of function scopes; None for non-function scopes. - locals = None # type: List[SymbolTable] - # Nested block depths of scopes - block_depth = None # type: List[int] - # TypeInfo of directly enclosing class (or None) - type = None # type: TypeInfo - # Stack of outer classes (the second tuple item contains tvars). - type_stack = None # type: List[TypeInfo] - # Type variables that are bound by the directly enclosing class - bound_tvars = None # type: List[SymbolTableNode] - # Stack of type variables that were bound by outer classess - tvar_stack = None # type: List[List[SymbolTableNode]] - # Do weak type checking in this file - weak_opts = set() # type: Set[str] - - # Stack of functions being analyzed - function_stack = None # type: List[FuncItem] - # Stack of next available function type variable ids - next_function_tvar_id_stack = None # type: List[int] - - # Status of postponing analysis of nested function bodies. By using this we - # can have mutually recursive nested functions. Values are FUNCTION_x - # constants. Note that separate phasea are not used for methods. - postpone_nested_functions_stack = None # type: List[int] - # Postponed functions collected if - # postpone_nested_functions_stack[-1] == FUNCTION_FIRST_PHASE_POSTPONE_SECOND. - postponed_functions_stack = None # type: List[List[Node]] - - loop_depth = 0 # Depth of breakable loops - cur_mod_id = '' # Current module id (or None) (phase 2) - is_stub_file = False # Are we analyzing a stub file? - imports = None # type: Set[str] # Imported modules (during phase 2 analysis) - errors = None # type: Errors # Keeps track of generated errors - - def __init__(self, - lib_path: List[str], - errors: Errors, - options: Options) -> None: - """Construct semantic analyzer. - - Use lib_path to search for modules, and report analysis errors - using the Errors instance. - """ - self.locals = [None] - self.imports = set() - self.type = None - self.type_stack = [] - self.bound_tvars = None - self.tvar_stack = [] - self.function_stack = [] - self.next_function_tvar_id_stack = [-1] - self.block_depth = [0] - self.loop_depth = 0 - self.lib_path = lib_path - self.errors = errors - self.modules = {} - self.options = options - self.postpone_nested_functions_stack = [FUNCTION_BOTH_PHASES] - self.postponed_functions_stack = [] - self.all_exports = set() # type: Set[str] - - def visit_file(self, file_node: MypyFile, fnam: str) -> None: - self.errors.set_file(fnam) - self.cur_mod_node = file_node - self.cur_mod_id = file_node.fullname() - self.is_stub_file = fnam.lower().endswith('.pyi') - self.globals = file_node.names - self.weak_opts = file_node.weak_opts - - if 'builtins' in self.modules: - self.globals['__builtins__'] = SymbolTableNode( - MODULE_REF, self.modules['builtins'], self.cur_mod_id) - - for name in implicit_module_attrs: - v = self.globals[name].node - if isinstance(v, Var): - v.type = self.anal_type(v.type) - v.is_ready = True - - defs = file_node.defs - for d in defs: - self.accept(d) - - if self.cur_mod_id == 'builtins': - remove_imported_names_from_symtable(self.globals, 'builtins') - - if '__all__' in self.globals: - for name, g in self.globals.items(): - if name not in self.all_exports: - g.module_public = False - - def visit_func_def(self, defn: FuncDef) -> None: - phase_info = self.postpone_nested_functions_stack[-1] - if phase_info != FUNCTION_SECOND_PHASE: - self.function_stack.append(defn) - # First phase of analysis for function. - self.errors.push_function(defn.name()) - self.update_function_type_variables(defn) - self.errors.pop_function() - self.function_stack.pop() - - defn.is_conditional = self.block_depth[-1] > 0 - - # TODO(jukka): Figure out how to share the various cases. It doesn't - # make sense to have (almost) duplicate code (here and elsewhere) for - # 3 cases: module-level, class-level and local names. Maybe implement - # a common stack of namespaces. As the 3 kinds of namespaces have - # different semantics, this wouldn't always work, but it might still - # be a win. - if self.is_class_scope(): - # Method definition - defn.info = self.type - if not defn.is_decorated and not defn.is_overload: - if defn.name() in self.type.names: - # Redefinition. Conditional redefinition is okay. - n = self.type.names[defn.name()].node - if self.is_conditional_func(n, defn): - defn.original_def = cast(FuncDef, n) - else: - self.name_already_defined(defn.name(), defn) - self.type.names[defn.name()] = SymbolTableNode(MDEF, defn) - self.prepare_method_signature(defn) - elif self.is_func_scope(): - # Nested function - if not defn.is_decorated and not defn.is_overload: - if defn.name() in self.locals[-1]: - # Redefinition. Conditional redefinition is okay. - n = self.locals[-1][defn.name()].node - if self.is_conditional_func(n, defn): - defn.original_def = cast(FuncDef, n) - else: - self.name_already_defined(defn.name(), defn) - else: - self.add_local(defn, defn) - else: - # Top-level function - if not defn.is_decorated and not defn.is_overload: - symbol = self.globals.get(defn.name()) - if isinstance(symbol.node, FuncDef) and symbol.node != defn: - # This is redefinition. Conditional redefinition is okay. - original_def = symbol.node - if self.is_conditional_func(original_def, defn): - # Conditional function definition -- multiple defs are ok. - defn.original_def = original_def - else: - # Report error. - self.check_no_global(defn.name(), defn, True) - if phase_info == FUNCTION_FIRST_PHASE_POSTPONE_SECOND: - # Postpone this function (for the second phase). - self.postponed_functions_stack[-1].append(defn) - return - if phase_info != FUNCTION_FIRST_PHASE_POSTPONE_SECOND: - # Second phase of analysis for function. - self.errors.push_function(defn.name()) - self.analyze_function(defn) - self.errors.pop_function() - - def prepare_method_signature(self, func: FuncDef) -> None: - """Check basic signature validity and tweak annotation of self/cls argument.""" - # Only non-static methods are special. - if not func.is_static: - if not func.arguments: - self.fail('Method must have at least one argument', func) - elif func.type: - sig = cast(FunctionLike, func.type) - if func.is_class: - leading_type = self.class_type(self.type) - else: - leading_type = self_type(self.type) - func.type = replace_implicit_first_type(sig, leading_type) - - def is_conditional_func(self, previous: Node, new: FuncDef) -> bool: - """Does 'new' conditionally redefine 'previous'? - - We reject straight redefinitions of functions, as they are usually - a programming error. For example: - - . def f(): ... - . def f(): ... # Error: 'f' redefined - """ - return isinstance(previous, (FuncDef, Var)) and new.is_conditional - - def update_function_type_variables(self, defn: FuncDef) -> None: - """Make any type variables in the signature of defn explicit. - - Update the signature of defn to contain type variable definitions - if defn is generic. - """ - if defn.type: - functype = cast(CallableType, defn.type) - typevars = self.infer_type_variables(functype) - # Do not define a new type variable if already defined in scope. - typevars = [(name, tvar) for name, tvar in typevars - if not self.is_defined_type_var(name, defn)] - if typevars: - next_tvar_id = self.next_function_tvar_id() - defs = [TypeVarDef(tvar[0], next_tvar_id - i, - tvar[1].values, tvar[1].upper_bound, - tvar[1].variance) - for i, tvar in enumerate(typevars)] - functype.variables = defs - - def infer_type_variables(self, - type: CallableType) -> List[Tuple[str, TypeVarExpr]]: - """Return list of unique type variables referred to in a callable.""" - names = [] # type: List[str] - tvars = [] # type: List[TypeVarExpr] - for arg in type.arg_types + [type.ret_type]: - for name, tvar_expr in self.find_type_variables_in_type(arg): - if name not in names: - names.append(name) - tvars.append(tvar_expr) - return list(zip(names, tvars)) - - def find_type_variables_in_type( - self, type: Type) -> List[Tuple[str, TypeVarExpr]]: - """Return a list of all unique type variable references in type. - - This effectively does partial name binding, results of which are mostly thrown away. - """ - result = [] # type: List[Tuple[str, TypeVarExpr]] - if isinstance(type, UnboundType): - name = type.name - node = self.lookup_qualified(name, type) - if node and node.kind == UNBOUND_TVAR: - result.append((name, cast(TypeVarExpr, node.node))) - for arg in type.args: - result.extend(self.find_type_variables_in_type(arg)) - elif isinstance(type, TypeList): - for item in type.items: - result.extend(self.find_type_variables_in_type(item)) - elif isinstance(type, UnionType): - for item in type.items: - result.extend(self.find_type_variables_in_type(item)) - elif isinstance(type, AnyType): - pass - elif isinstance(type, EllipsisType) or isinstance(type, TupleType): - pass - else: - assert False, 'Unsupported type %s' % type - return result - - def is_defined_type_var(self, tvar: str, context: Node) -> bool: - return self.lookup_qualified(tvar, context).kind == BOUND_TVAR - - def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: - t = [] # type: List[CallableType] - for i, item in enumerate(defn.items): - # TODO support decorated overloaded functions properly - item.is_overload = True - item.func.is_overload = True - item.accept(self) - t.append(cast(CallableType, function_type(item.func, - self.builtin_type('builtins.function')))) - if item.func.is_property and i == 0: - # This defines a property, probably with a setter and/or deleter. - self.analyze_property_with_multi_part_definition(defn) - break - if not [dec for dec in item.decorators - if refers_to_fullname(dec, 'typing.overload')]: - self.fail("'overload' decorator expected", item) - - defn.type = Overloaded(t) - defn.type.line = defn.line - - if self.is_class_scope(): - self.type.names[defn.name()] = SymbolTableNode(MDEF, defn, - typ=defn.type) - defn.info = self.type - elif self.is_func_scope(): - self.add_local(defn, defn) - - def analyze_property_with_multi_part_definition(self, defn: OverloadedFuncDef) -> None: - """Analyze a property defined using multiple methods (e.g., using @x.setter). - - Assume that the first method (@property) has already been analyzed. - """ - defn.is_property = True - items = defn.items - for item in items[1:]: - if len(item.decorators) == 1: - node = item.decorators[0] - if isinstance(node, MemberExpr): - if node.name == 'setter': - # The first item represents the entire property. - defn.items[0].var.is_settable_property = True - # Get abstractness from the original definition. - item.func.is_abstract = items[0].func.is_abstract - else: - self.fail("Decorated property not supported", item) - item.func.accept(self) - - def next_function_tvar_id(self) -> int: - return self.next_function_tvar_id_stack[-1] - - def analyze_function(self, defn: FuncItem) -> None: - is_method = self.is_class_scope() - - tvarnodes = self.add_func_type_variables_to_symbol_table(defn) - next_function_tvar_id = min([self.next_function_tvar_id()] + - [n.tvar_def.id.raw_id - 1 for n in tvarnodes]) - self.next_function_tvar_id_stack.append(next_function_tvar_id) - - if defn.type: - # Signature must be analyzed in the surrounding scope so that - # class-level imported names and type variables are in scope. - defn.type = self.anal_type(defn.type) - self.check_function_signature(defn) - if isinstance(defn, FuncDef): - defn.type = set_callable_name(defn.type, defn) - for arg in defn.arguments: - if arg.initializer: - arg.initializer.accept(self) - self.function_stack.append(defn) - self.enter() - for arg in defn.arguments: - self.add_local(arg.variable, defn) - for arg in defn.arguments: - if arg.initialization_statement: - lvalue = arg.initialization_statement.lvalues[0] - lvalue.accept(self) - - # The first argument of a non-static, non-class method is like 'self' - # (though the name could be different), having the enclosing class's - # instance type. - if is_method and not defn.is_static and not defn.is_class and defn.arguments: - defn.arguments[0].variable.is_self = True - - # First analyze body of the function but ignore nested functions. - self.postpone_nested_functions_stack.append(FUNCTION_FIRST_PHASE_POSTPONE_SECOND) - self.postponed_functions_stack.append([]) - defn.body.accept(self) - - # Analyze nested functions (if any) as a second phase. - self.postpone_nested_functions_stack[-1] = FUNCTION_SECOND_PHASE - for postponed in self.postponed_functions_stack[-1]: - postponed.accept(self) - self.postpone_nested_functions_stack.pop() - self.postponed_functions_stack.pop() - - self.next_function_tvar_id_stack.pop() - disable_typevars(tvarnodes) - - self.leave() - self.function_stack.pop() - - def add_func_type_variables_to_symbol_table( - self, defn: FuncItem) -> List[SymbolTableNode]: - nodes = [] # type: List[SymbolTableNode] - if defn.type: - tt = defn.type - names = self.type_var_names() - items = cast(CallableType, tt).variables - for item in items: - name = item.name - if name in names: - self.name_already_defined(name, defn) - node = self.bind_type_var(name, item, defn) - nodes.append(node) - names.add(name) - return nodes - - def type_var_names(self) -> Set[str]: - if not self.type: - return set() - else: - return set(self.type.type_vars) - - def bind_type_var(self, fullname: str, tvar_def: TypeVarDef, - context: Context) -> SymbolTableNode: - node = self.lookup_qualified(fullname, context) - node.kind = BOUND_TVAR - node.tvar_def = tvar_def - return node - - def check_function_signature(self, fdef: FuncItem) -> None: - sig = cast(CallableType, fdef.type) - if len(sig.arg_types) < len(fdef.arguments): - self.fail('Type signature has too few arguments', fdef) - # Add dummy Any arguments to prevent crashes later. - extra_anys = [AnyType()] * (len(fdef.arguments) - len(sig.arg_types)) - sig.arg_types.extend(extra_anys) - elif len(sig.arg_types) > len(fdef.arguments): - self.fail('Type signature has too many arguments', fdef) - - def visit_class_def(self, defn: ClassDef) -> None: - self.clean_up_bases_and_infer_type_variables(defn) - self.setup_class_def_analysis(defn) - - self.bind_class_type_vars(defn) - - self.analyze_base_classes(defn) - self.analyze_metaclass(defn) - - for decorator in defn.decorators: - self.analyze_class_decorator(defn, decorator) - - self.enter_class(defn) - - self.setup_is_builtinclass(defn) - - # Analyze class body. - defn.defs.accept(self) - - self.calculate_abstract_status(defn.info) - self.setup_type_promotion(defn) - - self.leave_class() - self.unbind_class_type_vars() - - def enter_class(self, defn: ClassDef) -> None: - # Remember previous active class - self.type_stack.append(self.type) - self.locals.append(None) # Add class scope - self.block_depth.append(-1) # The class body increments this to 0 - self.postpone_nested_functions_stack.append(FUNCTION_BOTH_PHASES) - self.type = defn.info - - def leave_class(self) -> None: - """ Restore analyzer state. """ - self.postpone_nested_functions_stack.pop() - self.block_depth.pop() - self.locals.pop() - self.type = self.type_stack.pop() - - def bind_class_type_vars(self, defn: ClassDef) -> None: - """ Unbind type variables of previously active class and bind - the type variables for the active class. - """ - if self.bound_tvars: - disable_typevars(self.bound_tvars) - self.tvar_stack.append(self.bound_tvars) - self.bound_tvars = self.bind_class_type_variables_in_symbol_table(defn.info) - - def unbind_class_type_vars(self) -> None: - """ Unbind the active class' type vars and rebind the - type vars of the previously active class. - """ - disable_typevars(self.bound_tvars) - self.bound_tvars = self.tvar_stack.pop() - if self.bound_tvars: - enable_typevars(self.bound_tvars) - - def analyze_class_decorator(self, defn: ClassDef, decorator: Node) -> None: - decorator.accept(self) - - def setup_is_builtinclass(self, defn: ClassDef): - for decorator in defn.decorators: - if refers_to_fullname(decorator, 'typing.builtinclass'): - defn.is_builtinclass = True - if defn.fullname == 'builtins.object': - # Only 'object' is marked as a built-in class, as otherwise things elsewhere - # would break. We need a better way of dealing with built-in classes. - defn.is_builtinclass = True - - def calculate_abstract_status(self, typ: TypeInfo) -> None: - """Calculate abstract status of a class. - - Set is_abstract of the type to True if the type has an unimplemented - abstract attribute. Also compute a list of abstract attributes. - """ - concrete = set() # type: Set[str] - abstract = [] # type: List[str] - for base in typ.mro: - for name, symnode in base.names.items(): - node = symnode.node - if isinstance(node, OverloadedFuncDef): - # Unwrap an overloaded function definition. We can just - # check arbitrarily the first overload item. If the - # different items have a different abstract status, there - # should be an error reported elsewhere. - func = node.items[0] # type: Node - else: - func = node - if isinstance(func, Decorator): - fdef = func.func - if fdef.is_abstract and name not in concrete: - typ.is_abstract = True - abstract.append(name) - concrete.add(name) - typ.abstract_attributes = sorted(abstract) - - def setup_type_promotion(self, defn: ClassDef) -> None: - """Setup extra, ad-hoc subtyping relationships between classes (promotion). - - This includes things like 'int' being compatible with 'float'. - """ - promote_target = None # type: Type - for decorator in defn.decorators: - if isinstance(decorator, CallExpr): - analyzed = decorator.analyzed - if isinstance(analyzed, PromoteExpr): - # _promote class decorator (undocumented faeture). - promote_target = analyzed.type - if not promote_target: - promotions = (TYPE_PROMOTIONS_PYTHON3 if self.options.python_version[0] >= 3 - else TYPE_PROMOTIONS_PYTHON2) - if defn.fullname in promotions: - promote_target = self.named_type_or_none(promotions[defn.fullname]) - defn.info._promote = promote_target - - def clean_up_bases_and_infer_type_variables(self, defn: ClassDef) -> None: - """Remove extra base classes such as Generic and infer type vars. - - For example, consider this class: - - . class Foo(Bar, Generic[T]): ... - - Now we will remove Generic[T] from bases of Foo and infer that the - type variable 'T' is a type argument of Foo. - - Note that this is performed *before* semantic analysis. - """ - removed = [] # type: List[int] - type_vars = [] # type: List[TypeVarDef] - for i, base_expr in enumerate(defn.base_type_exprs): - try: - base = expr_to_unanalyzed_type(base_expr) - except TypeTranslationError: - # This error will be caught later. - continue - tvars = self.analyze_typevar_declaration(base) - if tvars is not None: - if type_vars: - self.fail('Duplicate Generic in bases', defn) - removed.append(i) - for j, (name, tvar_expr) in enumerate(tvars): - type_vars.append(TypeVarDef(name, j + 1, tvar_expr.values, - tvar_expr.upper_bound, tvar_expr.variance)) - if type_vars: - defn.type_vars = type_vars - if defn.info: - defn.info.type_vars = [tv.name for tv in type_vars] - for i in reversed(removed): - del defn.base_type_exprs[i] - - def analyze_typevar_declaration(self, t: Type) -> Optional[List[Tuple[str, TypeVarExpr]]]: - if not isinstance(t, UnboundType): - return None - unbound = t - sym = self.lookup_qualified(unbound.name, unbound) - if sym is None or sym.node is None: - return None - if sym.node.fullname() == 'typing.Generic': - tvars = [] # type: List[Tuple[str, TypeVarExpr]] - for arg in unbound.args: - tvar = self.analyze_unbound_tvar(arg) - if tvar: - tvars.append(tvar) - else: - self.fail('Free type variable expected in %s[...]' % - sym.node.name(), t) - return tvars - return None - - def analyze_unbound_tvar(self, t: Type) -> Tuple[str, TypeVarExpr]: - if not isinstance(t, UnboundType): - return None - unbound = t - sym = self.lookup_qualified(unbound.name, unbound) - if sym is not None and sym.kind == UNBOUND_TVAR: - return unbound.name, cast(TypeVarExpr, sym.node) - return None - - def setup_class_def_analysis(self, defn: ClassDef) -> None: - """Prepare for the analysis of a class definition.""" - if not defn.info: - defn.info = TypeInfo(SymbolTable(), defn) - defn.info._fullname = defn.info.name() - if self.is_func_scope() or self.type: - kind = MDEF - if self.is_func_scope(): - kind = LDEF - self.add_symbol(defn.name, SymbolTableNode(kind, defn.info), defn) - - def analyze_base_classes(self, defn: ClassDef) -> None: - """Analyze and set up base classes. - - This computes several attributes on the corresponding TypeInfo defn.info - related to the base classes: defn.info.bases, defn.info.mro, and - miscellaneous others (at least tuple_type, fallback_to_any, and is_enum.) - """ - - base_types = [] # type: List[Instance] - info = defn.info - for base_expr in defn.base_type_exprs: - try: - base = self.expr_to_analyzed_type(base_expr) - except TypeTranslationError: - self.fail('Invalid base class', base_expr) - info.fallback_to_any = True - continue - - if isinstance(base, TupleType): - if info.tuple_type: - self.fail("Class has two incompatible bases derived from tuple", defn) - defn.incompatible_baseclass = True - if (not self.is_stub_file - and not info.is_named_tuple - and base.fallback.type.fullname() == 'builtins.tuple'): - self.fail("Tuple[...] not supported as a base class outside a stub file", defn) - info.tuple_type = base - base_types.append(base.fallback) - elif isinstance(base, Instance): - base_types.append(base) - elif isinstance(base, AnyType): - info.fallback_to_any = True - else: - self.fail('Invalid base class', base_expr) - info.fallback_to_any = True - - # Add 'object' as implicit base if there is no other base class. - if (not base_types and defn.fullname != 'builtins.object'): - base_types.append(self.object_type()) - - info.bases = base_types - - # Calculate the MRO. It might be incomplete at this point if - # the bases of defn include classes imported from other - # modules in an import loop. We'll recompute it in ThirdPass. - if not self.verify_base_classes(defn): - info.mro = [] - return - calculate_class_mro(defn, self.fail) - # If there are cyclic imports, we may be missing 'object' in - # the MRO. Fix MRO if needed. - if info.mro and info.mro[-1].fullname() != 'builtins.object': - info.mro.append(self.object_type().type) - - def expr_to_analyzed_type(self, expr: Node) -> Type: - if isinstance(expr, CallExpr): - expr.accept(self) - info = self.check_namedtuple(expr) - if info is None: - # Some form of namedtuple is the only valid type that looks like a call - # expression. This isn't a valid type. - raise TypeTranslationError() - fallback = Instance(info, []) - return TupleType(info.tuple_type.items, fallback=fallback) - typ = expr_to_unanalyzed_type(expr) - return self.anal_type(typ) - - def verify_base_classes(self, defn: ClassDef) -> bool: - info = defn.info - for base in info.bases: - baseinfo = base.type - if self.is_base_class(info, baseinfo): - self.fail('Cycle in inheritance hierarchy', defn) - # Clear bases to forcefully get rid of the cycle. - info.bases = [] - if baseinfo.fullname() == 'builtins.bool': - self.fail("'%s' is not a valid base class" % - baseinfo.name(), defn) - return False - dup = find_duplicate(info.direct_base_classes()) - if dup: - self.fail('Duplicate base class "%s"' % dup.name(), defn) - return False - return True - - def is_base_class(self, t: TypeInfo, s: TypeInfo) -> bool: - """Determine if t is a base class of s (but do not use mro).""" - # Search the base class graph for t, starting from s. - worklist = [s] - visited = {s} - while worklist: - nxt = worklist.pop() - if nxt == t: - return True - for base in nxt.bases: - if base.type not in visited: - worklist.append(base.type) - visited.add(base.type) - return False - - def analyze_metaclass(self, defn: ClassDef) -> None: - if defn.metaclass: - sym = self.lookup_qualified(defn.metaclass, defn) - if sym is not None and not isinstance(sym.node, TypeInfo): - self.fail("Invalid metaclass '%s'" % defn.metaclass, defn) - - def object_type(self) -> Instance: - return self.named_type('__builtins__.object') - - def class_type(self, info: TypeInfo) -> Type: - # Construct a function type whose fallback is cls. - from mypy import checkmember # To avoid import cycle. - leading_type = checkmember.type_object_type(info, self.builtin_type) - if isinstance(leading_type, Overloaded): - # Overloaded __init__ is too complex to handle. Plus it's stubs only. - return AnyType() - else: - return leading_type - - def named_type(self, qualified_name: str, args: List[Type] = None) -> Instance: - sym = self.lookup_qualified(qualified_name, None) - return Instance(cast(TypeInfo, sym.node), args or []) - - def named_type_or_none(self, qualified_name: str) -> Instance: - sym = self.lookup_fully_qualified_or_none(qualified_name) - if not sym: - return None - return Instance(cast(TypeInfo, sym.node), []) - - def is_instance_type(self, t: Type) -> bool: - return isinstance(t, Instance) - - def bind_class_type_variables_in_symbol_table( - self, info: TypeInfo) -> List[SymbolTableNode]: - nodes = [] # type: List[SymbolTableNode] - for var, binder in zip(info.type_vars, info.defn.type_vars): - node = self.bind_type_var(var, binder, info) - nodes.append(node) - return nodes - - def visit_import(self, i: Import) -> None: - for id, as_id in i.ids: - if as_id is not None: - self.add_module_symbol(id, as_id, module_public=True, context=i) - else: - # Modules imported in a stub file without using 'as x' won't get exported when - # doing 'from m import *'. - module_public = not self.is_stub_file - base = id.split('.')[0] - self.add_module_symbol(base, base, module_public=module_public, - context=i) - - def add_module_symbol(self, id: str, as_id: str, module_public: bool, - context: Context) -> None: - if id in self.modules: - m = self.modules[id] - self.add_symbol(as_id, SymbolTableNode(MODULE_REF, m, self.cur_mod_id, - module_public=module_public), context) - else: - self.add_unknown_symbol(as_id, context) - - def visit_import_from(self, imp: ImportFrom) -> None: - import_id = self.correct_relative_import(imp) - if import_id in self.modules: - module = self.modules[import_id] - for id, as_id in imp.names: - node = module.names.get(id) - if node and node.kind != UNBOUND_IMPORTED: - node = self.normalize_type_alias(node, imp) - if not node: - return - imported_id = as_id or id - existing_symbol = self.globals.get(imported_id) - if existing_symbol: - # Import can redefine a variable. They get special treatment. - if self.process_import_over_existing_name( - imported_id, existing_symbol, node, imp): - continue - # 'from m import x as x' exports x in a stub file. - module_public = not self.is_stub_file or as_id is not None - symbol = SymbolTableNode(node.kind, node.node, - self.cur_mod_id, - node.type_override, - module_public=module_public) - self.add_symbol(imported_id, symbol, imp) - else: - message = "Module has no attribute '{}'".format(id) - extra = self.undefined_name_extra_info('{}.{}'.format(import_id, id)) - if extra: - message += " {}".format(extra) - self.fail(message, imp) - else: - # Missing module. - for id, as_id in imp.names: - self.add_unknown_symbol(as_id or id, imp) - - def process_import_over_existing_name(self, - imported_id: str, existing_symbol: SymbolTableNode, - module_symbol: SymbolTableNode, - import_node: ImportBase) -> bool: - if (existing_symbol.kind in (LDEF, GDEF, MDEF) and - isinstance(existing_symbol.node, (Var, FuncDef))): - # This is a valid import over an existing definition in the file. Construct a dummy - # assignment that we'll use to type check the import. - lvalue = NameExpr(imported_id) - lvalue.kind = existing_symbol.kind - lvalue.node = existing_symbol.node - rvalue = NameExpr(imported_id) - rvalue.kind = module_symbol.kind - rvalue.node = module_symbol.node - assignment = AssignmentStmt([lvalue], rvalue) - for node in assignment, lvalue, rvalue: - node.set_line(import_node) - import_node.assignments.append(assignment) - return True - return False - - def normalize_type_alias(self, node: SymbolTableNode, - ctx: Context) -> SymbolTableNode: - if node.fullname in type_aliases: - # Node refers to an aliased type such as typing.List; normalize. - node = self.lookup_qualified(type_aliases[node.fullname], ctx) - if node.fullname == 'typing.DefaultDict': - self.add_module_symbol('collections', '__mypy_collections__', False, ctx) - node = self.lookup_qualified('__mypy_collections__.defaultdict', ctx) - return node - - def correct_relative_import(self, node: Union[ImportFrom, ImportAll]) -> str: - if node.relative == 0: - return node.id - - parts = self.cur_mod_id.split(".") - cur_mod_id = self.cur_mod_id - - rel = node.relative - if self.cur_mod_node.is_package_init_file(): - rel -= 1 - if len(parts) < rel: - self.fail("Relative import climbs too many namespaces", node) - if rel != 0: - cur_mod_id = ".".join(parts[:-rel]) - - return cur_mod_id + (("." + node.id) if node.id else "") - - def visit_import_all(self, i: ImportAll) -> None: - i_id = self.correct_relative_import(i) - if i_id in self.modules: - m = self.modules[i_id] - for name, node in m.names.items(): - node = self.normalize_type_alias(node, i) - if not name.startswith('_') and node.module_public: - existing_symbol = self.globals.get(name) - if existing_symbol: - # Import can redefine a variable. They get special treatment. - if self.process_import_over_existing_name( - name, existing_symbol, node, i): - continue - self.add_symbol(name, SymbolTableNode(node.kind, node.node, - self.cur_mod_id), i) - else: - # Don't add any dummy symbols for 'from x import *' if 'x' is unknown. - pass - - def add_unknown_symbol(self, name: str, context: Context) -> None: - var = Var(name) - var._fullname = self.qualified_name(name) - var.is_ready = True - var.type = AnyType() - self.add_symbol(name, SymbolTableNode(GDEF, var, self.cur_mod_id), context) - - # - # Statements - # - - def visit_block(self, b: Block) -> None: - if b.is_unreachable: - return - self.block_depth[-1] += 1 - for s in b.body: - self.accept(s) - self.block_depth[-1] -= 1 - - def visit_block_maybe(self, b: Block) -> None: - if b: - self.visit_block(b) - - def anal_type(self, t: Type, allow_tuple_literal: bool = False) -> Type: - if t: - if allow_tuple_literal: - # Types such as (t1, t2, ...) only allowed in assignment statements. They'll - # generate errors elsewhere, and Tuple[t1, t2, ...] must be used instead. - if isinstance(t, TupleType): - # Unlike TypeAnalyser, also allow implicit tuple types (without Tuple[...]). - star_count = sum(1 for item in t.items if isinstance(item, StarType)) - if star_count > 1: - self.fail('At most one star type allowed in a tuple', t) - return TupleType([AnyType() for _ in t.items], - self.builtin_type('builtins.tuple'), t.line) - items = [self.anal_type(item, True) - for item in t.items] - return TupleType(items, self.builtin_type('builtins.tuple'), t.line) - a = TypeAnalyser(self.lookup_qualified, - self.lookup_fully_qualified, - self.fail) - return t.accept(a) - else: - return None - - def visit_assignment_stmt(self, s: AssignmentStmt) -> None: - for lval in s.lvalues: - self.analyze_lvalue(lval, explicit_type=s.type is not None) - s.rvalue.accept(self) - if s.type: - allow_tuple_literal = isinstance(s.lvalues[-1], (TupleExpr, ListExpr)) - s.type = self.anal_type(s.type, allow_tuple_literal) - else: - # For simple assignments, allow binding type aliases. - # Also set the type if the rvalue is a simple literal. - if (s.type is None and len(s.lvalues) == 1 and - isinstance(s.lvalues[0], NameExpr)): - if s.lvalues[0].is_def: - s.type = self.analyze_simple_literal_type(s.rvalue) - res = analyze_type_alias(s.rvalue, - self.lookup_qualified, - self.lookup_fully_qualified, - self.fail) - if res and (not isinstance(res, Instance) or cast(Instance, res).args): - # TODO: What if this gets reassigned? - name = s.lvalues[0] - node = self.lookup(name.name, name) - node.kind = TYPE_ALIAS - node.type_override = res - if isinstance(s.rvalue, IndexExpr): - s.rvalue.analyzed = TypeAliasExpr(res) - if s.type: - # Store type into nodes. - for lvalue in s.lvalues: - self.store_declared_types(lvalue, s.type) - self.check_and_set_up_type_alias(s) - self.process_typevar_declaration(s) - self.process_namedtuple_definition(s) - - if (len(s.lvalues) == 1 and isinstance(s.lvalues[0], NameExpr) and - s.lvalues[0].name == '__all__' and s.lvalues[0].kind == GDEF and - isinstance(s.rvalue, (ListExpr, TupleExpr))): - self.add_exports(*s.rvalue.items) - - def analyze_simple_literal_type(self, rvalue: Node) -> Optional[Type]: - """Return builtins.int if rvalue is an int literal, etc.""" - if self.weak_opts or self.options.semantic_analysis_only or self.function_stack: - # Skip this if any weak options are set. - # Also skip if we're only doing the semantic analysis pass. - # This is mostly to avoid breaking unit tests. - # Also skip inside a function; this is to avoid confusing - # the code that handles dead code due to isinstance() - # inside type variables with value restrictions (like - # AnyStr). - return None - if isinstance(rvalue, IntExpr): - return self.named_type_or_none('builtins.int') - if isinstance(rvalue, FloatExpr): - return self.named_type_or_none('builtins.float') - if isinstance(rvalue, StrExpr): - return self.named_type_or_none('builtins.str') - if isinstance(rvalue, BytesExpr): - return self.named_type_or_none('builtins.bytes') - if isinstance(rvalue, UnicodeExpr): - return self.named_type_or_none('builtins.unicode') - return None - - def check_and_set_up_type_alias(self, s: AssignmentStmt) -> None: - """Check if assignment creates a type alias and set it up as needed.""" - # For now, type aliases only work at the top level of a module. - if (len(s.lvalues) == 1 and not self.is_func_scope() and not self.type - and not s.type): - lvalue = s.lvalues[0] - if isinstance(lvalue, NameExpr): - if not lvalue.is_def: - # Only a definition can create a type alias, not regular assignment. - return - rvalue = s.rvalue - if isinstance(rvalue, RefExpr): - node = rvalue.node - if isinstance(node, TypeInfo): - # TODO: We should record the fact that this is a variable - # that refers to a type, rather than making this - # just an alias for the type. - self.globals[lvalue.name].node = node - - def analyze_lvalue(self, lval: Node, nested: bool = False, - add_global: bool = False, - explicit_type: bool = False) -> None: - """Analyze an lvalue or assignment target. - - Only if add_global is True, add name to globals table. If nested - is true, the lvalue is within a tuple or list lvalue expression. - """ - - if isinstance(lval, NameExpr): - # Top-level definitions within some statements (at least while) are - # not handled in the first pass, so they have to be added now. - nested_global = (not self.is_func_scope() and - self.block_depth[-1] > 0 and - not self.type) - if (add_global or nested_global) and lval.name not in self.globals: - # Define new global name. - v = Var(lval.name) - v._fullname = self.qualified_name(lval.name) - v.is_ready = False # Type not inferred yet - lval.node = v - lval.is_def = True - lval.kind = GDEF - lval.fullname = v._fullname - self.globals[lval.name] = SymbolTableNode(GDEF, v, - self.cur_mod_id) - elif isinstance(lval.node, Var) and lval.is_def: - # Since the is_def flag is set, this must have been analyzed - # already in the first pass and added to the symbol table. - assert lval.node.name() in self.globals - elif (self.is_func_scope() and lval.name not in self.locals[-1] and - lval.name not in self.global_decls[-1] and - lval.name not in self.nonlocal_decls[-1]): - # Define new local name. - v = Var(lval.name) - lval.node = v - lval.is_def = True - lval.kind = LDEF - lval.fullname = lval.name - self.add_local(v, lval) - elif not self.is_func_scope() and (self.type and - lval.name not in self.type.names): - # Define a new attribute within class body. - v = Var(lval.name) - v.info = self.type - v.is_initialized_in_class = True - v.set_line(lval) - lval.node = v - lval.is_def = True - lval.kind = MDEF - lval.fullname = lval.name - self.type.names[lval.name] = SymbolTableNode(MDEF, v) - else: - # Bind to an existing name. - if explicit_type: - self.name_already_defined(lval.name, lval) - lval.accept(self) - self.check_lvalue_validity(lval.node, lval) - elif isinstance(lval, MemberExpr): - if not add_global: - self.analyze_member_lvalue(lval) - if explicit_type and not self.is_self_member_ref(lval): - self.fail('Type cannot be declared in assignment to non-self ' - 'attribute', lval) - elif isinstance(lval, IndexExpr): - if explicit_type: - self.fail('Unexpected type declaration', lval) - if not add_global: - lval.accept(self) - elif (isinstance(lval, TupleExpr) or - isinstance(lval, ListExpr)): - items = cast(Any, lval).items - if len(items) == 0 and isinstance(lval, TupleExpr): - self.fail("Can't assign to ()", lval) - self.analyze_tuple_or_list_lvalue(cast(Union[ListExpr, TupleExpr], lval), - add_global, explicit_type) - elif isinstance(lval, StarExpr): - if nested: - self.analyze_lvalue(lval.expr, nested, add_global, explicit_type) - else: - self.fail('Starred assignment target must be in a list or tuple', lval) - else: - self.fail('Invalid assignment target', lval) - - def analyze_tuple_or_list_lvalue(self, lval: Union[ListExpr, TupleExpr], - add_global: bool = False, - explicit_type: bool = False) -> None: - """Analyze an lvalue or assignment target that is a list or tuple.""" - items = lval.items - star_exprs = [cast(StarExpr, item) - for item in items - if isinstance(item, StarExpr)] - - if len(star_exprs) > 1: - self.fail('Two starred expressions in assignment', lval) - else: - if len(star_exprs) == 1: - star_exprs[0].valid = True - for i in items: - self.analyze_lvalue(i, nested=True, add_global=add_global, - explicit_type = explicit_type) - - def analyze_member_lvalue(self, lval: MemberExpr) -> None: - lval.accept(self) - if (self.is_self_member_ref(lval) and - self.type.get(lval.name) is None): - # Implicit attribute definition in __init__. - lval.is_def = True - v = Var(lval.name) - v.info = self.type - v.is_ready = False - lval.def_var = v - lval.node = v - self.type.names[lval.name] = SymbolTableNode(MDEF, v) - self.check_lvalue_validity(lval.node, lval) - - def is_self_member_ref(self, memberexpr: MemberExpr) -> bool: - """Does memberexpr to refer to an attribute of self?""" - if not isinstance(memberexpr.expr, NameExpr): - return False - node = memberexpr.expr.node - return isinstance(node, Var) and node.is_self - - def check_lvalue_validity(self, node: Node, ctx: Context) -> None: - if isinstance(node, (TypeInfo, TypeVarExpr)): - self.fail('Invalid assignment target', ctx) - - def store_declared_types(self, lvalue: Node, typ: Type) -> None: - if isinstance(typ, StarType) and not isinstance(lvalue, StarExpr): - self.fail('Star type only allowed for starred expressions', lvalue) - if isinstance(lvalue, RefExpr): - lvalue.is_def = False - if isinstance(lvalue.node, Var): - var = lvalue.node - var.type = typ - var.is_ready = True - # If node is not a variable, we'll catch it elsewhere. - elif isinstance(lvalue, TupleExpr): - if isinstance(typ, TupleType): - if len(lvalue.items) != len(typ.items): - self.fail('Incompatible number of tuple items', lvalue) - return - for item, itemtype in zip(lvalue.items, typ.items): - self.store_declared_types(item, itemtype) - else: - self.fail('Tuple type expected for multiple variables', - lvalue) - elif isinstance(lvalue, StarExpr): - if isinstance(typ, StarType): - self.store_declared_types(lvalue.expr, typ.type) - else: - self.fail('Star type expected for starred expression', lvalue) - else: - # This has been flagged elsewhere as an error, so just ignore here. - pass - - def process_typevar_declaration(self, s: AssignmentStmt) -> None: - """Check if s declares a TypeVar; it yes, store it in symbol table.""" - call = self.get_typevar_declaration(s) - if not call: - return - - lvalue = cast(NameExpr, s.lvalues[0]) - name = lvalue.name - if not lvalue.is_def: - if s.type: - self.fail("Cannot declare the type of a type variable", s) - else: - self.fail("Cannot redefine '%s' as a type variable" % name, s) - return - - if not self.check_typevar_name(call, name, s): - return - - # Constraining types - n_values = call.arg_kinds[1:].count(ARG_POS) - values = self.analyze_types(call.args[1:1 + n_values]) - - res = self.process_typevar_parameters(call.args[1 + n_values:], - call.arg_names[1 + n_values:], - call.arg_kinds[1 + n_values:], - bool(values), - s) - if res is None: - return - variance, upper_bound = res - - # Yes, it's a valid type variable definition! Add it to the symbol table. - node = self.lookup(name, s) - node.kind = UNBOUND_TVAR - TypeVar = TypeVarExpr(name, node.fullname, values, upper_bound, variance) - TypeVar.line = call.line - call.analyzed = TypeVar - node.node = TypeVar - - def check_typevar_name(self, call: CallExpr, name: str, context: Context) -> bool: - if len(call.args) < 1: - self.fail("Too few arguments for TypeVar()", context) - return False - if not isinstance(call.args[0], (StrExpr, BytesExpr)) or not call.arg_kinds[0] == ARG_POS: - self.fail("TypeVar() expects a string literal as first argument", context) - return False - if cast(StrExpr, call.args[0]).value != name: - self.fail("Unexpected TypeVar() argument value", context) - return False - return True - - def get_typevar_declaration(self, s: AssignmentStmt) -> Optional[CallExpr]: - """Returns the TypeVar() call expression if `s` is a type var declaration - or None otherwise. - """ - if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): - return None - if not isinstance(s.rvalue, CallExpr): - return None - call = s.rvalue - if not isinstance(call.callee, RefExpr): - return None - callee = call.callee - if callee.fullname != 'typing.TypeVar': - return None - return call - - def process_typevar_parameters(self, args: List[Node], - names: List[Optional[str]], - kinds: List[int], - has_values: bool, - context: Context) -> Optional[Tuple[int, Type]]: - covariant = False - contravariant = False - upper_bound = self.object_type() # type: Type - for param_value, param_name, param_kind in zip(args, names, kinds): - if not param_kind == ARG_NAMED: - self.fail("Unexpected argument to TypeVar()", context) - return None - if param_name == 'covariant': - if isinstance(param_value, NameExpr): - if param_value.name == 'True': - covariant = True - else: - self.fail("TypeVar 'covariant' may only be 'True'", context) - return None - else: - self.fail("TypeVar 'covariant' may only be 'True'", context) - return None - elif param_name == 'contravariant': - if isinstance(param_value, NameExpr): - if param_value.name == 'True': - contravariant = True - else: - self.fail("TypeVar 'contravariant' may only be 'True'", context) - return None - else: - self.fail("TypeVar 'contravariant' may only be 'True'", context) - return None - elif param_name == 'bound': - if has_values: - self.fail("TypeVar cannot have both values and an upper bound", context) - return None - try: - upper_bound = self.expr_to_analyzed_type(param_value) - except TypeTranslationError: - self.fail("TypeVar 'bound' must be a type", param_value) - return None - elif param_name == 'values': - # Probably using obsolete syntax with values=(...). Explain the current syntax. - self.fail("TypeVar 'values' argument not supported", context) - self.fail("Use TypeVar('T', t, ...) instead of TypeVar('T', values=(t, ...))", - context) - return None - else: - self.fail("Unexpected argument to TypeVar(): {}".format(param_name), context) - return None - - if covariant and contravariant: - self.fail("TypeVar cannot be both covariant and contravariant", context) - return None - elif covariant: - variance = COVARIANT - elif contravariant: - variance = CONTRAVARIANT - else: - variance = INVARIANT - return (variance, upper_bound) - - def process_namedtuple_definition(self, s: AssignmentStmt) -> None: - """Check if s defines a namedtuple; if yes, store the definition in symbol table.""" - if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): - return - lvalue = cast(NameExpr, s.lvalues[0]) - name = lvalue.name - named_tuple = self.check_namedtuple(s.rvalue, name) - if named_tuple is None: - return - # Yes, it's a valid namedtuple definition. Add it to the symbol table. - node = self.lookup(name, s) - node.kind = GDEF # TODO locally defined namedtuple - # TODO call.analyzed - node.node = named_tuple - - def check_namedtuple(self, node: Node, var_name: str = None) -> TypeInfo: - """Check if a call defines a namedtuple. - - The optional var_name argument is the name of the variable to - which this is assigned, if any. - - If it does, return the corresponding TypeInfo. Return None otherwise. - - If the definition is invalid but looks like a namedtuple, - report errors but return (some) TypeInfo. - """ - if not isinstance(node, CallExpr): - return None - call = node - if not isinstance(call.callee, RefExpr): - return None - callee = call.callee - fullname = callee.fullname - if fullname not in ('collections.namedtuple', 'typing.NamedTuple'): - return None - items, types, ok = self.parse_namedtuple_args(call, fullname) - if not ok: - # Error. Construct dummy return value. - return self.build_namedtuple_typeinfo('namedtuple', [], []) - else: - # Give it a unique name derived from the line number. - name = cast(StrExpr, call.args[0]).value - if name != var_name: - name += '@' + str(call.line) - info = self.build_namedtuple_typeinfo(name, items, types) - # Store it as a global just in case it would remain anonymous. - self.globals[name] = SymbolTableNode(GDEF, info, self.cur_mod_id) - call.analyzed = NamedTupleExpr(info).set_line(call.line) - return info - - def parse_namedtuple_args(self, call: CallExpr, - fullname: str) -> Tuple[List[str], List[Type], bool]: - # TODO Share code with check_argument_count in checkexpr.py? - args = call.args - if len(args) < 2: - return self.fail_namedtuple_arg("Too few arguments for namedtuple()", call) - if len(args) > 2: - return self.fail_namedtuple_arg("Too many arguments for namedtuple()", call) - if call.arg_kinds != [ARG_POS, ARG_POS]: - return self.fail_namedtuple_arg("Unexpected arguments to namedtuple()", call) - if not isinstance(args[0], (StrExpr, BytesExpr)): - return self.fail_namedtuple_arg( - "namedtuple() expects a string literal as the first argument", call) - types = [] # type: List[Type] - ok = True - if not isinstance(args[1], ListExpr): - if fullname == 'collections.namedtuple' and isinstance(args[1], (StrExpr, BytesExpr)): - str_expr = cast(StrExpr, args[1]) - items = str_expr.value.split() - else: - return self.fail_namedtuple_arg( - "List literal expected as the second argument to namedtuple()", call) - else: - listexpr = args[1] - if fullname == 'collections.namedtuple': - # The fields argument contains just names, with implicit Any types. - if any(not isinstance(item, (StrExpr, BytesExpr)) for item in listexpr.items): - return self.fail_namedtuple_arg("String literal expected as namedtuple() item", - call) - items = [cast(StrExpr, item).value for item in listexpr.items] - else: - # The fields argument contains (name, type) tuples. - items, types, ok = self.parse_namedtuple_fields_with_types(listexpr.items, call) - if not types: - types = [AnyType() for _ in items] - return items, types, ok - - def parse_namedtuple_fields_with_types(self, nodes: List[Node], - context: Context) -> Tuple[List[str], List[Type], bool]: - items = [] # type: List[str] - types = [] # type: List[Type] - for item in nodes: - if isinstance(item, TupleExpr): - if len(item.items) != 2: - return self.fail_namedtuple_arg("Invalid NamedTuple field definition", - item) - name, type_node = item.items - if isinstance(name, (StrExpr, BytesExpr)): - items.append(name.value) - else: - return self.fail_namedtuple_arg("Invalid NamedTuple() field name", item) - try: - type = expr_to_unanalyzed_type(type_node) - except TypeTranslationError: - return self.fail_namedtuple_arg('Invalid field type', type_node) - types.append(self.anal_type(type)) - else: - return self.fail_namedtuple_arg("Tuple expected as NamedTuple() field", item) - return items, types, True - - def fail_namedtuple_arg(self, message: str, - context: Context) -> Tuple[List[str], List[Type], bool]: - self.fail(message, context) - return [], [], False - - def build_namedtuple_typeinfo(self, name: str, items: List[str], - types: List[Type]) -> TypeInfo: - symbols = SymbolTable() - tup = NamedTupleType(name, items, types, self.named_type('__builtins__.tuple', types)) - class_def = ClassDef(name, Block([])) - class_def.fullname = self.qualified_name(name) - info = NamedTupleTypeInfo(tup, symbols, class_def) - info.attrs = items - vars = [Var(item, typ) for item, typ in zip(items, types)] - # Add named tuple items as attributes. - # TODO: Make them read-only. - for var in vars: - var.info = info - symbols[var.name()] = SymbolTableNode(MDEF, var) - this_type = self_type(info) - # Add __init__, _replace, _asdict methods and _fields static field - self.add_namedtuple_method('_replace', symbols, info, this_type, this_type, - self.make_factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) - self.add_namedtuple_method('__init__', symbols, info, this_type, NoneTyp(), - self.make_factory_args(vars, ARG_POS), info.name()) - # TODO: refine to OrderedDict[Union[types]] - self.add_namedtuple_method('_asdict', symbols, info, this_type, AnyType(), []) - return info - - def make_factory_args(self, vars: List[Var], kind: int, - initializer: Expression = None) -> List[Argument]: - return [Argument(var, var.type, initializer, kind) for var in vars] - - def add_namedtuple_method(self, funcname: str, symbols: SymbolTable, info: TypeInfo, - this_type: Type, ret: Type, args: List[Argument], name=None) -> None: - args = [Argument(Var('self'), this_type, None, ARG_POS)] + args - types = [arg.type_annotation for arg in args] - items = [arg.variable.name() for arg in args] - arg_kinds = [arg.kind for arg in args] - # TODO: Make sure that the self argument name is not visible? - signature = CallableType(types, arg_kinds, items, ret, - self.named_type('__builtins__.function'), - name=name or info.name() + '.' + funcname) - func = FuncDef(funcname, - args, - Block([]), - typ=signature) - func.info = info - symbols[funcname] = SymbolTableNode(MDEF, func) - - def analyze_types(self, items: List[Node]) -> List[Type]: - result = [] # type: List[Type] - for node in items: - try: - result.append(self.anal_type(expr_to_unanalyzed_type(node))) - except TypeTranslationError: - self.fail('Type expected', node) - result.append(AnyType()) - return result - - def visit_decorator(self, dec: Decorator) -> None: - for d in dec.decorators: - d.accept(self) - removed = [] # type: List[int] - no_type_check = False - for i, d in enumerate(dec.decorators): - # A bunch of decorators are special cased here. - if refers_to_fullname(d, 'abc.abstractmethod'): - removed.append(i) - dec.func.is_abstract = True - self.check_decorated_function_is_method('abstractmethod', dec) - elif refers_to_fullname(d, 'asyncio.tasks.coroutine'): - removed.append(i) - elif refers_to_fullname(d, 'builtins.staticmethod'): - removed.append(i) - dec.func.is_static = True - dec.var.is_staticmethod = True - self.check_decorated_function_is_method('staticmethod', dec) - elif refers_to_fullname(d, 'builtins.classmethod'): - removed.append(i) - dec.func.is_class = True - dec.var.is_classmethod = True - self.check_decorated_function_is_method('classmethod', dec) - elif (refers_to_fullname(d, 'builtins.property') or - refers_to_fullname(d, 'abc.abstractproperty')): - removed.append(i) - dec.func.is_property = True - dec.var.is_property = True - if refers_to_fullname(d, 'abc.abstractproperty'): - dec.func.is_abstract = True - self.check_decorated_function_is_method('property', dec) - if len(dec.func.arguments) > 1: - self.fail('Too many arguments', dec.func) - elif refers_to_fullname(d, 'typing.no_type_check'): - dec.var.type = AnyType() - no_type_check = True - for i in reversed(removed): - del dec.decorators[i] - if not dec.is_overload or dec.var.is_property: - if self.is_func_scope(): - self.add_symbol(dec.var.name(), SymbolTableNode(LDEF, dec), - dec) - elif self.type: - dec.var.info = self.type - dec.var.is_initialized_in_class = True - self.add_symbol(dec.var.name(), SymbolTableNode(MDEF, dec), - dec) - if not no_type_check: - dec.func.accept(self) - if dec.decorators and dec.var.is_property: - self.fail('Decorated property not supported', dec) - - def check_decorated_function_is_method(self, decorator: str, - context: Context) -> None: - if not self.type or self.is_func_scope(): - self.fail("'%s' used with a non-method" % decorator, context) - - def visit_expression_stmt(self, s: ExpressionStmt) -> None: - s.expr.accept(self) - - def visit_return_stmt(self, s: ReturnStmt) -> None: - if not self.is_func_scope(): - self.fail("'return' outside function", s) - if s.expr: - s.expr.accept(self) - - def visit_raise_stmt(self, s: RaiseStmt) -> None: - if s.expr: - s.expr.accept(self) - if s.from_expr: - s.from_expr.accept(self) - - def visit_assert_stmt(self, s: AssertStmt) -> None: - if s.expr: - s.expr.accept(self) - - def visit_operator_assignment_stmt(self, - s: OperatorAssignmentStmt) -> None: - s.lvalue.accept(self) - s.rvalue.accept(self) - if (isinstance(s.lvalue, NameExpr) and s.lvalue.name == '__all__' and - s.lvalue.kind == GDEF and isinstance(s.rvalue, (ListExpr, TupleExpr))): - self.add_exports(*s.rvalue.items) - - def visit_while_stmt(self, s: WhileStmt) -> None: - s.expr.accept(self) - self.loop_depth += 1 - s.body.accept(self) - self.loop_depth -= 1 - self.visit_block_maybe(s.else_body) - - def visit_for_stmt(self, s: ForStmt) -> None: - s.expr.accept(self) - - # Bind index variables and check if they define new names. - self.analyze_lvalue(s.index) - - self.loop_depth += 1 - self.visit_block(s.body) - self.loop_depth -= 1 - - self.visit_block_maybe(s.else_body) - - def visit_break_stmt(self, s: BreakStmt) -> None: - if self.loop_depth == 0: - self.fail("'break' outside loop", s, True, blocker=True) - - def visit_continue_stmt(self, s: ContinueStmt) -> None: - if self.loop_depth == 0: - self.fail("'continue' outside loop", s, True, blocker=True) - - def visit_if_stmt(self, s: IfStmt) -> None: - infer_reachability_of_if_statement(s, pyversion=self.options.python_version) - for i in range(len(s.expr)): - s.expr[i].accept(self) - self.visit_block(s.body[i]) - self.visit_block_maybe(s.else_body) - - def visit_try_stmt(self, s: TryStmt) -> None: - self.analyze_try_stmt(s, self) - - def analyze_try_stmt(self, s: TryStmt, visitor: NodeVisitor, - add_global: bool = False) -> None: - s.body.accept(visitor) - for type, var, handler in zip(s.types, s.vars, s.handlers): - if type: - type.accept(visitor) - if var: - self.analyze_lvalue(var, add_global=add_global) - handler.accept(visitor) - if s.else_body: - s.else_body.accept(visitor) - if s.finally_body: - s.finally_body.accept(visitor) - - def visit_with_stmt(self, s: WithStmt) -> None: - for e, n in zip(s.expr, s.target): - e.accept(self) - if n: - self.analyze_lvalue(n) - self.visit_block(s.body) - - def visit_del_stmt(self, s: DelStmt) -> None: - s.expr.accept(self) - if not self.is_valid_del_target(s.expr): - self.fail('Invalid delete target', s) - - def is_valid_del_target(self, s: Node) -> bool: - if isinstance(s, (IndexExpr, NameExpr, MemberExpr)): - return True - elif isinstance(s, TupleExpr): - return all(self.is_valid_del_target(item) for item in s.items) - - def visit_global_decl(self, g: GlobalDecl) -> None: - for name in g.names: - if name in self.nonlocal_decls[-1]: - self.fail("Name '{}' is nonlocal and global".format(name), g) - self.global_decls[-1].add(name) - - def visit_nonlocal_decl(self, d: NonlocalDecl) -> None: - if not self.is_func_scope(): - self.fail("nonlocal declaration not allowed at module level", d) - else: - for name in d.names: - for table in reversed(self.locals[:-1]): - if table is not None and name in table: - break - else: - self.fail("No binding for nonlocal '{}' found".format(name), d) - - if self.locals[-1] is not None and name in self.locals[-1]: - self.fail("Name '{}' is already defined in local " - "scope before nonlocal declaration".format(name), d) - - if name in self.global_decls[-1]: - self.fail("Name '{}' is nonlocal and global".format(name), d) - self.nonlocal_decls[-1].add(name) - - def visit_print_stmt(self, s: PrintStmt) -> None: - for arg in s.args: - arg.accept(self) - if s.target: - s.target.accept(self) - - def visit_exec_stmt(self, s: ExecStmt) -> None: - s.expr.accept(self) - if s.variables1: - s.variables1.accept(self) - if s.variables2: - s.variables2.accept(self) - - # - # Expressions - # - - def visit_name_expr(self, expr: NameExpr) -> None: - n = self.lookup(expr.name, expr) - if n: - if n.kind == BOUND_TVAR: - self.fail("'{}' is a type variable and only valid in type " - "context".format(expr.name), expr) - else: - expr.kind = n.kind - expr.node = n.node - expr.fullname = n.fullname - - def visit_super_expr(self, expr: SuperExpr) -> None: - if not self.type: - self.fail('"super" used outside class', expr) - return - expr.info = self.type - - def visit_tuple_expr(self, expr: TupleExpr) -> None: - for item in expr.items: - item.accept(self) - - def visit_list_expr(self, expr: ListExpr) -> None: - for item in expr.items: - item.accept(self) - - def visit_set_expr(self, expr: SetExpr) -> None: - for item in expr.items: - item.accept(self) - - def visit_dict_expr(self, expr: DictExpr) -> None: - for key, value in expr.items: - key.accept(self) - value.accept(self) - - def visit_star_expr(self, expr: StarExpr) -> None: - if not expr.valid: - self.fail('Can use starred expression only as assignment target', expr) - else: - expr.expr.accept(self) - - def visit_yield_from_expr(self, e: YieldFromExpr) -> None: - if not self.is_func_scope(): # not sure - self.fail("'yield from' outside function", e, True, blocker=True) - else: - self.function_stack[-1].is_generator = True - if e.expr: - e.expr.accept(self) - - def visit_call_expr(self, expr: CallExpr) -> None: - """Analyze a call expression. - - Some call expressions are recognized as special forms, including - cast(...) and Any(...). - """ - expr.callee.accept(self) - if refers_to_fullname(expr.callee, 'typing.cast'): - # Special form cast(...). - if not self.check_fixed_args(expr, 2, 'cast'): - return - # Translate first argument to an unanalyzed type. - try: - target = expr_to_unanalyzed_type(expr.args[0]) - except TypeTranslationError: - self.fail('Cast target is not a type', expr) - return - # Piggyback CastExpr object to the CallExpr object; it takes - # precedence over the CallExpr semantics. - expr.analyzed = CastExpr(expr.args[1], target) - expr.analyzed.line = expr.line - expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'builtins.reveal_type'): - if not self.check_fixed_args(expr, 1, 'reveal_type'): - return - expr.analyzed = RevealTypeExpr(expr.args[0]) - expr.analyzed.line = expr.line - expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'typing.Any'): - # Special form Any(...). - if not self.check_fixed_args(expr, 1, 'Any'): - return - expr.analyzed = CastExpr(expr.args[0], AnyType()) - expr.analyzed.line = expr.line - expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'typing._promote'): - # Special form _promote(...). - if not self.check_fixed_args(expr, 1, '_promote'): - return - # Translate first argument to an unanalyzed type. - try: - target = expr_to_unanalyzed_type(expr.args[0]) - except TypeTranslationError: - self.fail('Argument 1 to _promote is not a type', expr) - return - expr.analyzed = PromoteExpr(target) - expr.analyzed.line = expr.line - expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'builtins.dict'): - expr.analyzed = self.translate_dict_call(expr) - else: - # Normal call expression. - for a in expr.args: - a.accept(self) - - if (isinstance(expr.callee, MemberExpr) and - isinstance(expr.callee.expr, NameExpr) and - expr.callee.expr.name == '__all__' and - expr.callee.expr.kind == GDEF and - expr.callee.name in ('append', 'extend')): - if expr.callee.name == 'append' and expr.args: - self.add_exports(expr.args[0]) - elif (expr.callee.name == 'extend' and expr.args and - isinstance(expr.args[0], (ListExpr, TupleExpr))): - self.add_exports(*expr.args[0].items) - - def translate_dict_call(self, call: CallExpr) -> Optional[DictExpr]: - """Translate 'dict(x=y, ...)' to {'x': y, ...}. - - For other variants of dict(...), return None. - """ - if not call.args: - return None - if not all(kind == ARG_NAMED for kind in call.arg_kinds): - # Must still accept those args. - for a in call.args: - a.accept(self) - return None - expr = DictExpr([(StrExpr(key), value) - for key, value in zip(call.arg_names, call.args)]) - expr.set_line(call) - expr.accept(self) - return expr - - def check_fixed_args(self, expr: CallExpr, numargs: int, - name: str) -> bool: - """Verify that expr has specified number of positional args. - - Return True if the arguments are valid. - """ - s = 's' - if numargs == 1: - s = '' - if len(expr.args) != numargs: - self.fail("'%s' expects %d argument%s" % (name, numargs, s), - expr) - return False - if expr.arg_kinds != [ARG_POS] * numargs: - self.fail("'%s' must be called with %s positional argument%s" % - (name, numargs, s), expr) - return False - return True - - def visit_member_expr(self, expr: MemberExpr) -> None: - base = expr.expr - base.accept(self) - # Bind references to module attributes. - if isinstance(base, RefExpr) and base.kind == MODULE_REF: - file = cast(MypyFile, base.node) - n = file.names.get(expr.name, None) if file is not None else None - if n: - n = self.normalize_type_alias(n, expr) - if not n: - return - expr.kind = n.kind - expr.fullname = n.fullname - expr.node = n.node - else: - # We only catch some errors here; the rest will be - # catched during type checking. - # - # This way we can report a larger number of errors in - # one type checker run. If we reported errors here, - # the build would terminate after semantic analysis - # and we wouldn't be able to report any type errors. - full_name = '%s.%s' % (file.fullname() if file is not None else None, expr.name) - if full_name in obsolete_name_mapping: - self.fail("Module has no attribute %r (it's now called %r)" % ( - expr.name, obsolete_name_mapping[full_name]), expr) - - def visit_op_expr(self, expr: OpExpr) -> None: - expr.left.accept(self) - expr.right.accept(self) - - def visit_comparison_expr(self, expr: ComparisonExpr) -> None: - for operand in expr.operands: - operand.accept(self) - - def visit_unary_expr(self, expr: UnaryExpr) -> None: - expr.expr.accept(self) - - def visit_index_expr(self, expr: IndexExpr) -> None: - expr.base.accept(self) - if refers_to_class_or_function(expr.base): - # Special form -- type application. - # Translate index to an unanalyzed type. - types = [] # type: List[Type] - if isinstance(expr.index, TupleExpr): - items = expr.index.items - else: - items = [expr.index] - for item in items: - try: - typearg = expr_to_unanalyzed_type(item) - except TypeTranslationError: - self.fail('Type expected within [...]', expr) - return - typearg = self.anal_type(typearg) - types.append(typearg) - expr.analyzed = TypeApplication(expr.base, types) - expr.analyzed.line = expr.line - else: - expr.index.accept(self) - - def visit_slice_expr(self, expr: SliceExpr) -> None: - if expr.begin_index: - expr.begin_index.accept(self) - if expr.end_index: - expr.end_index.accept(self) - if expr.stride: - expr.stride.accept(self) - - def visit_cast_expr(self, expr: CastExpr) -> None: - expr.expr.accept(self) - expr.type = self.anal_type(expr.type) - - def visit_reveal_type_expr(self, expr: RevealTypeExpr) -> None: - expr.expr.accept(self) - - def visit_type_application(self, expr: TypeApplication) -> None: - expr.expr.accept(self) - for i in range(len(expr.types)): - expr.types[i] = self.anal_type(expr.types[i]) - - def visit_list_comprehension(self, expr: ListComprehension) -> None: - expr.generator.accept(self) - - def visit_set_comprehension(self, expr: SetComprehension) -> None: - expr.generator.accept(self) - - def visit_dictionary_comprehension(self, expr: DictionaryComprehension) -> None: - self.enter() - self.analyze_comp_for(expr) - expr.key.accept(self) - expr.value.accept(self) - self.leave() - self.analyze_comp_for_2(expr) - - def visit_generator_expr(self, expr: GeneratorExpr) -> None: - self.enter() - self.analyze_comp_for(expr) - expr.left_expr.accept(self) - self.leave() - self.analyze_comp_for_2(expr) - - def analyze_comp_for(self, expr: Union[GeneratorExpr, - DictionaryComprehension]) -> None: - """Analyses the 'comp_for' part of comprehensions (part 1). - - That is the part after 'for' in (x for x in l if p). This analyzes - variables and conditions which are analyzed in a local scope. - """ - for i, (index, sequence, conditions) in enumerate(zip(expr.indices, - expr.sequences, - expr.condlists)): - if i > 0: - sequence.accept(self) - # Bind index variables. - self.analyze_lvalue(index) - for cond in conditions: - cond.accept(self) - - def analyze_comp_for_2(self, expr: Union[GeneratorExpr, - DictionaryComprehension]) -> None: - """Analyses the 'comp_for' part of comprehensions (part 2). - - That is the part after 'for' in (x for x in l if p). This analyzes - the 'l' part which is analyzed in the surrounding scope. - """ - expr.sequences[0].accept(self) - - def visit_func_expr(self, expr: FuncExpr) -> None: - self.analyze_function(expr) - - def visit_conditional_expr(self, expr: ConditionalExpr) -> None: - expr.if_expr.accept(self) - expr.cond.accept(self) - expr.else_expr.accept(self) - - def visit_backquote_expr(self, expr: BackquoteExpr) -> None: - expr.expr.accept(self) - - def visit__promote_expr(self, expr: PromoteExpr) -> None: - expr.type = self.anal_type(expr.type) - - def visit_yield_expr(self, expr: YieldExpr) -> None: - if not self.is_func_scope(): - self.fail("'yield' outside function", expr, True, blocker=True) - else: - self.function_stack[-1].is_generator = True - if expr.expr: - expr.expr.accept(self) - - # - # Helpers - # - - def lookup(self, name: str, ctx: Context) -> SymbolTableNode: - """Look up an unqualified name in all active namespaces.""" - # 1a. Name declared using 'global x' takes precedence - if name in self.global_decls[-1]: - if name in self.globals: - return self.globals[name] - else: - self.name_not_defined(name, ctx) - return None - # 1b. Name declared using 'nonlocal x' takes precedence - if name in self.nonlocal_decls[-1]: - for table in reversed(self.locals[:-1]): - if table is not None and name in table: - return table[name] - else: - self.name_not_defined(name, ctx) - return None - # 2. Class attributes (if within class definition) - if self.is_class_scope() and name in self.type.names: - return self.type[name] - # 3. Local (function) scopes - for table in reversed(self.locals): - if table is not None and name in table: - return table[name] - # 4. Current file global scope - if name in self.globals: - return self.globals[name] - # 5. Builtins - b = self.globals.get('__builtins__', None) - if b: - table = cast(MypyFile, b.node).names - if name in table: - if name[0] == "_" and name[1] != "_": - self.name_not_defined(name, ctx) - return None - node = table[name] - return node - # Give up. - self.name_not_defined(name, ctx) - self.check_for_obsolete_short_name(name, ctx) - return None - - def check_for_obsolete_short_name(self, name: str, ctx: Context) -> None: - matches = [obsolete_name - for obsolete_name in obsolete_name_mapping - if obsolete_name.rsplit('.', 1)[-1] == name] - if len(matches) == 1: - self.note("(Did you mean '{}'?)".format(obsolete_name_mapping[matches[0]]), ctx) - - def lookup_qualified(self, name: str, ctx: Context) -> SymbolTableNode: - if '.' not in name: - return self.lookup(name, ctx) - else: - parts = name.split('.') - n = self.lookup(parts[0], ctx) # type: SymbolTableNode - if n: - for i in range(1, len(parts)): - if isinstance(n.node, TypeInfo): - if n.node.mro is None: - # We haven't yet analyzed the class `n.node`. Fall back to direct - # lookup in the names declared directly under it, without its base - # classes. This can happen when we have a forward reference to a - # nested class, and the reference is bound before the outer class - # has been fully semantically analyzed. - # - # A better approach would be to introduce a new analysis pass or - # to move things around between passes, but this unblocks a common - # use case even though this is a little limited in case there is - # inheritance involved. - result = n.node.names.get(parts[i]) - else: - result = n.node.get(parts[i]) - n = result - elif isinstance(n.node, MypyFile): - n = n.node.names.get(parts[i], None) - # TODO: What if node is Var or FuncDef? - if not n: - self.name_not_defined(name, ctx) - break - if n: - n = self.normalize_type_alias(n, ctx) - return n - - def builtin_type(self, fully_qualified_name: str) -> Instance: - node = self.lookup_fully_qualified(fully_qualified_name) - info = cast(TypeInfo, node.node) - return Instance(info, []) - - def lookup_fully_qualified(self, name: str) -> SymbolTableNode: - """Lookup a fully qualified name. - - Assume that the name is defined. This happens in the global namespace -- the local - module namespace is ignored. - """ - assert '.' in name - parts = name.split('.') - n = self.modules[parts[0]] - for i in range(1, len(parts) - 1): - n = cast(MypyFile, n.names[parts[i]].node) - return n.names[parts[-1]] - - def lookup_fully_qualified_or_none(self, name: str) -> SymbolTableNode: - """Lookup a fully qualified name. - - Assume that the name is defined. This happens in the global namespace -- the local - module namespace is ignored. - """ - assert '.' in name - parts = name.split('.') - n = self.modules[parts[0]] - for i in range(1, len(parts) - 1): - next_sym = n.names.get(parts[i]) - if not next_sym: - return None - n = cast(MypyFile, next_sym.node) - return n.names.get(parts[-1]) - - def qualified_name(self, n: str) -> str: - return self.cur_mod_id + '.' + n - - def enter(self) -> None: - self.locals.append(SymbolTable()) - self.global_decls.append(set()) - self.nonlocal_decls.append(set()) - # -1 since entering block will increment this to 0. - self.block_depth.append(-1) - - def leave(self) -> None: - self.locals.pop() - self.global_decls.pop() - self.nonlocal_decls.pop() - self.block_depth.pop() - - def is_func_scope(self) -> bool: - return self.locals[-1] is not None - - def is_class_scope(self) -> bool: - return self.type is not None and not self.is_func_scope() - - def add_symbol(self, name: str, node: SymbolTableNode, - context: Context) -> None: - if self.is_func_scope(): - if name in self.locals[-1]: - # Flag redefinition unless this is a reimport of a module. - if not (node.kind == MODULE_REF and - self.locals[-1][name].node == node.node): - self.name_already_defined(name, context) - self.locals[-1][name] = node - elif self.type: - self.type.names[name] = node - else: - existing = self.globals.get(name) - if existing and (not isinstance(node.node, MypyFile) or - existing.node != node.node) and existing.kind != UNBOUND_IMPORTED: - # Modules can be imported multiple times to support import - # of multiple submodules of a package (e.g. a.x and a.y). - ok = False - # Only report an error if the symbol collision provides a different type. - if existing.type and node.type and is_same_type(existing.type, node.type): - ok = True - if not ok: - self.name_already_defined(name, context) - self.globals[name] = node - - def add_var(self, v: Var, ctx: Context) -> None: - if self.is_func_scope(): - self.add_local(v, ctx) - else: - self.globals[v.name()] = SymbolTableNode(GDEF, v, self.cur_mod_id) - v._fullname = self.qualified_name(v.name()) - - def add_local(self, node: Union[Var, FuncBase], ctx: Context) -> None: - name = node.name() - if name in self.locals[-1]: - self.name_already_defined(name, ctx) - node._fullname = name - self.locals[-1][name] = SymbolTableNode(LDEF, node) - - def add_exports(self, *exps: Node) -> None: - for exp in exps: - if isinstance(exp, StrExpr): - self.all_exports.add(exp.value) - - def check_no_global(self, n: str, ctx: Context, - is_overloaded_func: bool = False) -> None: - if n in self.globals: - prev_is_overloaded = isinstance(self.globals[n], OverloadedFuncDef) - if is_overloaded_func and prev_is_overloaded: - self.fail("Nonconsecutive overload {} found".format(n), ctx) - elif prev_is_overloaded: - self.fail("Definition of '{}' missing 'overload'".format(n), ctx) - else: - self.name_already_defined(n, ctx) - - def name_not_defined(self, name: str, ctx: Context) -> None: - message = "Name '{}' is not defined".format(name) - extra = self.undefined_name_extra_info(name) - if extra: - message += ' {}'.format(extra) - self.fail(message, ctx) - - def name_already_defined(self, name: str, ctx: Context) -> None: - self.fail("Name '{}' already defined".format(name), ctx) - - def fail(self, msg: str, ctx: Context, serious: bool = False, *, - blocker: bool = False) -> None: - if (not serious and - not self.options.check_untyped_defs and - self.function_stack and - self.function_stack[-1].is_dynamic()): - return - self.errors.report(ctx.get_line(), msg, blocker=blocker) - - def note(self, msg: str, ctx: Context) -> None: - if (not self.options.check_untyped_defs and - self.function_stack and - self.function_stack[-1].is_dynamic()): - return - self.errors.report(ctx.get_line(), msg, severity='note') - - def undefined_name_extra_info(self, fullname: str) -> Optional[str]: - if fullname in obsolete_name_mapping: - return "(it's now called '{}')".format(obsolete_name_mapping[fullname]) - else: - return None - - def accept(self, node: Node) -> None: - try: - node.accept(self) - except Exception as err: - report_internal_error(err, self.errors.file, node.line) - - -class FirstPass(NodeVisitor): - """First phase of semantic analysis. - - See docstring of 'analyze()' below for a description of what this does. - """ - - def __init__(self, sem: SemanticAnalyzer) -> None: - self.sem = sem - self.pyversion = sem.options.python_version - - def analyze(self, file: MypyFile, fnam: str, mod_id: str) -> None: - """Perform the first analysis pass. - - Populate module global table. Resolve the full names of - definitions not nested within functions and construct type - info structures, but do not resolve inter-definition - references such as base classes. - - Also add implicit definitions such as __name__. - - In this phase we don't resolve imports. For 'from ... import', - we generate dummy symbol table nodes for the imported names, - and these will get resolved in later phases of semantic - analysis. - """ - sem = self.sem - sem.cur_mod_id = mod_id - sem.errors.set_file(fnam) - sem.globals = SymbolTable() - sem.global_decls = [set()] - sem.nonlocal_decls = [set()] - sem.block_depth = [0] - - defs = file.defs - - # Add implicit definitions of module '__name__' etc. - for name, t in implicit_module_attrs.items(): - v = Var(name, UnboundType(t)) - v._fullname = self.sem.qualified_name(name) - self.sem.globals[name] = SymbolTableNode(GDEF, v, self.sem.cur_mod_id) - - for d in defs: - d.accept(self) - - # Add implicit definition of literals/keywords to builtins, as we - # cannot define a variable with them explicitly. - if mod_id == 'builtins': - literal_types = [ - ('None', NoneTyp()), - # reveal_type is a mypy-only function that gives an error with the type of its arg - ('reveal_type', AnyType()), - ] # type: List[Tuple[str, Type]] - - # TODO(ddfisher): This guard is only needed because mypy defines - # fake builtins for its tests which often don't define bool. If - # mypy is fast enough that we no longer need those, this - # conditional check should be removed. - if 'bool' in self.sem.globals: - bool_type = self.sem.named_type('bool') - literal_types.extend([ - ('True', bool_type), - ('False', bool_type), - ('__debug__', bool_type), - ]) - - for name, typ in literal_types: - v = Var(name, typ) - v._fullname = self.sem.qualified_name(name) - self.sem.globals[name] = SymbolTableNode(GDEF, v, self.sem.cur_mod_id) - - def visit_block(self, b: Block) -> None: - if b.is_unreachable: - return - self.sem.block_depth[-1] += 1 - for node in b.body: - node.accept(self) - self.sem.block_depth[-1] -= 1 - - def visit_assignment_stmt(self, s: AssignmentStmt) -> None: - for lval in s.lvalues: - self.analyze_lvalue(lval, explicit_type=s.type is not None) - - def visit_func_def(self, func: FuncDef) -> None: - sem = self.sem - func.is_conditional = sem.block_depth[-1] > 0 - func._fullname = sem.qualified_name(func.name()) - if func.name() in sem.globals: - # Already defined in this module. - original_sym = sem.globals[func.name()] - if original_sym.kind == UNBOUND_IMPORTED: - # Ah this is an imported name. We can't resolve them now, so we'll postpone - # this until the main phase of semantic analysis. - return - original_def = original_sym.node - if sem.is_conditional_func(original_def, func): - # Conditional function definition -- multiple defs are ok. - func.original_def = cast(FuncDef, original_def) - else: - # Report error. - sem.check_no_global(func.name(), func) - else: - sem.globals[func.name()] = SymbolTableNode(GDEF, func, sem.cur_mod_id) - - def visit_overloaded_func_def(self, func: OverloadedFuncDef) -> None: - self.sem.check_no_global(func.name(), func, True) - func._fullname = self.sem.qualified_name(func.name()) - self.sem.globals[func.name()] = SymbolTableNode(GDEF, func, - self.sem.cur_mod_id) - - def visit_class_def(self, cdef: ClassDef) -> None: - self.sem.check_no_global(cdef.name, cdef) - cdef.fullname = self.sem.qualified_name(cdef.name) - info = TypeInfo(SymbolTable(), cdef) - info.set_line(cdef.line) - cdef.info = info - self.sem.globals[cdef.name] = SymbolTableNode(GDEF, info, - self.sem.cur_mod_id) - self.process_nested_classes(cdef) - - def process_nested_classes(self, outer_def: ClassDef) -> None: - for node in outer_def.defs.body: - if isinstance(node, ClassDef): - node.info = TypeInfo(SymbolTable(), node) - if outer_def.fullname: - node.info._fullname = outer_def.fullname + '.' + node.info.name() - else: - node.info._fullname = node.info.name() - symbol = SymbolTableNode(MDEF, node.info) - outer_def.info.names[node.name] = symbol - self.process_nested_classes(node) - - def visit_import_from(self, node: ImportFrom) -> None: - # We can't bind module names during the first pass, as the target module might be - # unprocessed. However, we add dummy unbound imported names to the symbol table so - # that we at least know that the name refers to a module. - node.is_top_level = True - for name, as_name in node.names: - imported_name = as_name or name - if imported_name not in self.sem.globals: - self.sem.add_symbol(imported_name, SymbolTableNode(UNBOUND_IMPORTED, None), node) - - def visit_import(self, node: Import) -> None: - node.is_top_level = True - # This is similar to visit_import_from -- see the comment there. - for id, as_id in node.ids: - imported_id = as_id or id - if imported_id not in self.sem.globals: - self.sem.add_symbol(imported_id, SymbolTableNode(UNBOUND_IMPORTED, None), node) - else: - # If the previous symbol is a variable, this should take precedence. - self.sem.globals[imported_id] = SymbolTableNode(UNBOUND_IMPORTED, None) - - def visit_import_all(self, node: ImportAll) -> None: - node.is_top_level = True - - def visit_while_stmt(self, s: WhileStmt) -> None: - s.body.accept(self) - if s.else_body: - s.else_body.accept(self) - - def visit_for_stmt(self, s: ForStmt) -> None: - self.analyze_lvalue(s.index) - s.body.accept(self) - if s.else_body: - s.else_body.accept(self) - - def visit_with_stmt(self, s: WithStmt) -> None: - for n in s.target: - if n: - self.analyze_lvalue(n) - s.body.accept(self) - - def visit_decorator(self, d: Decorator) -> None: - d.var._fullname = self.sem.qualified_name(d.var.name()) - self.sem.add_symbol(d.var.name(), SymbolTableNode(GDEF, d.var), d) - - def visit_if_stmt(self, s: IfStmt) -> None: - infer_reachability_of_if_statement(s, pyversion=self.pyversion) - for node in s.body: - node.accept(self) - if s.else_body: - s.else_body.accept(self) - - def visit_try_stmt(self, s: TryStmt) -> None: - self.sem.analyze_try_stmt(s, self, add_global=True) - - def analyze_lvalue(self, lvalue: Node, explicit_type: bool = False) -> None: - self.sem.analyze_lvalue(lvalue, add_global=True, explicit_type=explicit_type) - - -class ThirdPass(TraverserVisitor): - """The third and final pass of semantic analysis. - - Check type argument counts and values of generic types, and perform some - straightforward type inference. - """ - - def __init__(self, modules: Dict[str, MypyFile], errors: Errors) -> None: - self.modules = modules - self.errors = errors - - def visit_file(self, file_node: MypyFile, fnam: str) -> None: - self.errors.set_file(fnam) - self.accept(file_node) - - def accept(self, node: Node) -> None: - try: - node.accept(self) - except Exception as err: - report_internal_error(err, self.errors.file, node.line) - - def visit_block(self, b: Block) -> None: - if b.is_unreachable: - return - super().visit_block(b) - - def visit_func_def(self, fdef: FuncDef) -> None: - self.errors.push_function(fdef.name()) - self.analyze(fdef.type) - super().visit_func_def(fdef) - self.errors.pop_function() - - def visit_class_def(self, tdef: ClassDef) -> None: - for type in tdef.info.bases: - self.analyze(type) - # Recompute MRO now that we have analyzed all modules, to pick - # up superclasses of bases imported from other modules in an - # import loop. (Only do so if we succeeded the first time.) - if tdef.info.mro: - tdef.info.mro = [] # Force recomputation - calculate_class_mro(tdef, self.fail) - super().visit_class_def(tdef) - - def visit_decorator(self, dec: Decorator) -> None: - """Try to infer the type of the decorated function. - - This lets us resolve references to decorated functions during - type checking when there are cyclic imports, as otherwise the - type might not be available when we need it. - - This basically uses a simple special-purpose type inference - engine just for decorators. - """ - super().visit_decorator(dec) - if dec.var.is_property: - # Decorators are expected to have a callable type (it's a little odd). - if dec.func.type is None: - dec.var.type = CallableType( - [AnyType()], - [ARG_POS], - [None], - AnyType(), - self.builtin_type('function'), - name=dec.var.name()) - elif isinstance(dec.func.type, CallableType): - dec.var.type = dec.func.type - return - decorator_preserves_type = True - for expr in dec.decorators: - preserve_type = False - if isinstance(expr, RefExpr) and isinstance(expr.node, FuncDef): - if is_identity_signature(expr.node.type): - preserve_type = True - if not preserve_type: - decorator_preserves_type = False - break - if decorator_preserves_type: - # No non-identity decorators left. We can trivially infer the type - # of the function here. - dec.var.type = function_type(dec.func, self.builtin_type('function')) - if dec.decorators: - if returns_any_if_called(dec.decorators[0]): - # The outermost decorator will return Any so we know the type of the - # decorated function. - dec.var.type = AnyType() - sig = find_fixed_callable_return(dec.decorators[0]) - if sig: - # The outermost decorator always returns the same kind of function, - # so we know that this is the type of the decoratored function. - orig_sig = function_type(dec.func, self.builtin_type('function')) - sig.name = orig_sig.items()[0].name - dec.var.type = sig - - def visit_assignment_stmt(self, s: AssignmentStmt) -> None: - self.analyze(s.type) - super().visit_assignment_stmt(s) - - def visit_cast_expr(self, e: CastExpr) -> None: - self.analyze(e.type) - super().visit_cast_expr(e) - - def visit_reveal_type_expr(self, e: RevealTypeExpr) -> None: - super().visit_reveal_type_expr(e) - - def visit_type_application(self, e: TypeApplication) -> None: - for type in e.types: - self.analyze(type) - super().visit_type_application(e) - - # Helpers - - def analyze(self, type: Type) -> None: - if type: - analyzer = TypeAnalyserPass3(self.fail) - type.accept(analyzer) - - def fail(self, msg: str, ctx: Context) -> None: - self.errors.report(ctx.get_line(), msg) - - def builtin_type(self, name: str, args: List[Type] = None) -> Instance: - names = self.modules['builtins'] - sym = names.names[name] - assert isinstance(sym.node, TypeInfo) - return Instance(sym.node, args or []) - - -def self_type(typ: TypeInfo) -> Union[Instance, TupleType]: - """For a non-generic type, return instance type representing the type. - For a generic G type with parameters T1, .., Tn, return G[T1, ..., Tn]. - """ - tv = [] # type: List[Type] - for i in range(len(typ.type_vars)): - tv.append(TypeVarType(typ.defn.type_vars[i])) - inst = Instance(typ, tv) - if typ.tuple_type is None: - return inst - return typ.tuple_type.self_type(inst) - - -def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: - if isinstance(sig, CallableType): - return replace_leading_arg_type(sig, new) - else: - sig = cast(Overloaded, sig) - return Overloaded([cast(CallableType, replace_implicit_first_type(i, new)) - for i in sig.items()]) - - -def set_callable_name(sig: Type, fdef: FuncDef) -> Type: - if isinstance(sig, FunctionLike): - if fdef.info: - return sig.with_name( - '"{}" of "{}"'.format(fdef.name(), fdef.info.name())) - else: - return sig.with_name('"{}"'.format(fdef.name())) - else: - return sig - - -def refers_to_fullname(node: Node, fullname: str) -> bool: - """Is node a name or member expression with the given full name?""" - return isinstance(node, RefExpr) and node.fullname == fullname - - -def refers_to_class_or_function(node: Node) -> bool: - """Does semantically analyzed node refer to a class?""" - return (isinstance(node, RefExpr) and - isinstance(node.node, (TypeInfo, FuncDef, OverloadedFuncDef))) - - -def calculate_class_mro(defn: ClassDef, fail: Callable[[str, Context], None]) -> None: - try: - defn.info.calculate_mro() - except MroError: - fail("Cannot determine consistent method resolution order " - '(MRO) for "%s"' % defn.name, defn) - defn.info.mro = [] - # The property of falling back to Any is inherited. - defn.info.fallback_to_any = any(baseinfo.fallback_to_any for baseinfo in defn.info.mro) - - -def find_duplicate(list: List[T]) -> T: - """If the list has duplicates, return one of the duplicates. - - Otherwise, return None. - """ - for i in range(1, len(list)): - if list[i] in list[:i]: - return list[i] - return None - - -def disable_typevars(nodes: List[SymbolTableNode]) -> None: - for node in nodes: - assert node.kind in (BOUND_TVAR, UNBOUND_TVAR) - node.kind = UNBOUND_TVAR - - -def enable_typevars(nodes: List[SymbolTableNode]) -> None: - for node in nodes: - assert node.kind in (BOUND_TVAR, UNBOUND_TVAR) - node.kind = BOUND_TVAR - - -def remove_imported_names_from_symtable(names: SymbolTable, - module: str) -> None: - """Remove all imported names from the symbol table of a module.""" - removed = [] # type: List[str] - for name, node in names.items(): - if node.node is None: - continue - fullname = node.node.fullname() - prefix = fullname[:fullname.rfind('.')] - if prefix != module: - removed.append(name) - for name in removed: - del names[name] - - -def infer_reachability_of_if_statement(s: IfStmt, - pyversion: Tuple[int, int]) -> None: - for i in range(len(s.expr)): - result = infer_if_condition_value(s.expr[i], pyversion) - if result == ALWAYS_FALSE: - # The condition is always false, so we skip the if/elif body. - mark_block_unreachable(s.body[i]) - elif result == ALWAYS_TRUE: - # This condition is always true, so all of the remaining - # elif/else bodies will never be executed. - for body in s.body[i + 1:]: - mark_block_unreachable(body) - if s.else_body: - mark_block_unreachable(s.else_body) - break - - -def infer_if_condition_value(expr: Node, pyversion: Tuple[int, int]) -> int: - """Infer whether if condition is always true/false. - - Return ALWAYS_TRUE if always true, ALWAYS_FALSE if always false, - and TRUTH_VALUE_UNKNOWN otherwise. - """ - name = '' - negated = False - alias = expr - if isinstance(alias, UnaryExpr): - if alias.op == 'not': - expr = alias.expr - negated = True - if isinstance(expr, NameExpr): - name = expr.name - elif isinstance(expr, MemberExpr): - name = expr.name - result = TRUTH_VALUE_UNKNOWN - if name == 'PY2': - result = ALWAYS_TRUE if pyversion[0] == 2 else ALWAYS_FALSE - elif name == 'PY3': - result = ALWAYS_TRUE if pyversion[0] == 3 else ALWAYS_FALSE - elif name == 'MYPY': - result = ALWAYS_TRUE - if negated: - if result == ALWAYS_TRUE: - result = ALWAYS_FALSE - elif result == ALWAYS_FALSE: - result = ALWAYS_TRUE - return result - - -def mark_block_unreachable(block: Block) -> None: - block.is_unreachable = True - block.accept(MarkImportsUnreachableVisitor()) - - -class MarkImportsUnreachableVisitor(TraverserVisitor): - """Visitor that flags all imports nested within a node as unreachable.""" - - def visit_import(self, node: Import) -> None: - node.is_unreachable = True - - def visit_import_from(self, node: ImportFrom) -> None: - node.is_unreachable = True - - def visit_import_all(self, node: ImportAll) -> None: - node.is_unreachable = True - - -def is_identity_signature(sig: Type) -> bool: - """Is type a callable of form T -> T (where T is a type variable)?""" - if isinstance(sig, CallableType) and sig.arg_kinds == [ARG_POS]: - if isinstance(sig.arg_types[0], TypeVarType) and isinstance(sig.ret_type, TypeVarType): - return sig.arg_types[0].id == sig.ret_type.id - return False - - -def returns_any_if_called(expr: Node) -> bool: - """Return True if we can predict that expr will return Any if called. - - This only uses information available during semantic analysis so this - will sometimes return False because of insufficient information (as - type inference hasn't run yet). - """ - if isinstance(expr, RefExpr): - if isinstance(expr.node, FuncDef): - typ = expr.node.type - if typ is None: - # No signature -> default to Any. - return True - # Explicit Any return? - return isinstance(typ, CallableType) and isinstance(typ.ret_type, AnyType) - elif isinstance(expr.node, Var): - typ = expr.node.type - return typ is None or isinstance(typ, AnyType) - elif isinstance(expr, CallExpr): - return returns_any_if_called(expr.callee) - return False - - -def find_fixed_callable_return(expr: Node) -> Optional[CallableType]: - if isinstance(expr, RefExpr): - if isinstance(expr.node, FuncDef): - typ = expr.node.type - if typ: - if isinstance(typ, CallableType) and has_no_typevars(typ.ret_type): - if isinstance(typ.ret_type, CallableType): - return typ.ret_type - elif isinstance(expr, CallExpr): - t = find_fixed_callable_return(expr.callee) - if t: - if isinstance(t.ret_type, CallableType): - return t.ret_type - return None - - -def has_no_typevars(typ: Type) -> bool: - return is_same_type(typ, erase_typevars(typ)) +"""The semantic analyzer. + +Bind names to definitions and do various other simple consistency +checks. For example, consider this program: + + x = 1 + y = x + +Here semantic analysis would detect that the assignment 'x = 1' +defines a new variable, the type of which is to be inferred (in a +later pass; type inference or type checking is not part of semantic +analysis). Also, it would bind both references to 'x' to the same +module-level variable node. The second assignment would also be +analyzed, and the type of 'y' marked as being inferred. + +Semantic analysis is the first analysis pass after parsing, and it is +subdivided into three passes: + + * FirstPass looks up externally visible names defined in a module but + ignores imports and local definitions. It helps enable (some) + cyclic references between modules, such as module 'a' that imports + module 'b' and used names defined in b *and* vice versa. The first + pass can be performed before dependent modules have been processed. + + * SemanticAnalyzer is the second pass. It does the bulk of the work. + It assumes that dependent modules have been semantically analyzed, + up to the second pass, unless there is a import cycle. + + * ThirdPass checks that type argument counts are valid; for example, + it will reject Dict[int]. We don't do this in the second pass, + since we infer the type argument counts of classes during this + pass, and it is possible to refer to classes defined later in a + file, which would not have the type argument count set yet. This + pass also recomputes the method resolution order of each class, in + case one of its bases belongs to a module involved in an import + loop. + +Semantic analysis of types is implemented in module mypy.typeanal. + +TODO: Check if the third pass slows down type checking significantly. + We could probably get rid of it -- for example, we could collect all + analyzed types in a collection and check them without having to + traverse the entire AST. +""" + +from typing import ( + List, Dict, Set, Tuple, cast, Any, overload, TypeVar, Union, Optional, Callable +) + +from mypy.nodes import ( + MypyFile, TypeInfo, Node, AssignmentStmt, FuncDef, OverloadedFuncDef, + ClassDef, Var, GDEF, MODULE_REF, FuncItem, Import, + ImportFrom, ImportAll, Block, LDEF, NameExpr, MemberExpr, + IndexExpr, TupleExpr, ListExpr, ExpressionStmt, ReturnStmt, + RaiseStmt, AssertStmt, OperatorAssignmentStmt, WhileStmt, + ForStmt, BreakStmt, ContinueStmt, IfStmt, TryStmt, WithStmt, DelStmt, + GlobalDecl, SuperExpr, DictExpr, CallExpr, RefExpr, OpExpr, UnaryExpr, + SliceExpr, CastExpr, RevealTypeExpr, TypeApplication, Context, SymbolTable, + SymbolTableNode, BOUND_TVAR, UNBOUND_TVAR, ListComprehension, GeneratorExpr, + FuncExpr, MDEF, FuncBase, Decorator, SetExpr, TypeVarExpr, + StrExpr, BytesExpr, PrintStmt, ConditionalExpr, PromoteExpr, + ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, MroError, type_aliases, + YieldFromExpr, NamedTupleExpr, NonlocalDecl, + SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr, + YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, COVARIANT, CONTRAVARIANT, + IntExpr, FloatExpr, UnicodeExpr, + INVARIANT, UNBOUND_IMPORTED, Expression, EllipsisExpr, NamedTupleTypeInfo +) +from mypy.visitor import NodeVisitor +from mypy.traverser import TraverserVisitor +from mypy.errors import Errors, report_internal_error +from mypy.types import ( + NoneTyp, CallableType, Overloaded, Instance, Type, TypeVarType, AnyType, + FunctionLike, UnboundType, TypeList, ErrorType, TypeVarDef, + replace_leading_arg_type, TupleType, UnionType, StarType, EllipsisType, + NamedTupleType +) +from mypy.nodes import function_type, implicit_module_attrs +from mypy.typeanal import TypeAnalyser, TypeAnalyserPass3, analyze_type_alias +from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError +from mypy.lex import lex +from mypy.parsetype import parse_type +from mypy.sametypes import is_same_type +from mypy.erasetype import erase_typevars +from mypy import defaults +from mypy.options import Options + + +T = TypeVar('T') + + +# Inferred value of an expression. +ALWAYS_TRUE = 0 +ALWAYS_FALSE = 1 +TRUTH_VALUE_UNKNOWN = 2 + +# Map from obsolete name to the current spelling. +obsolete_name_mapping = { + 'typing.Function': 'typing.Callable', + 'typing.typevar': 'typing.TypeVar', +} + +# Hard coded type promotions (shared between all Python versions). +# These add extra ad-hoc edges to the subtyping relation. For example, +# int is considered a subtype of float, even though there is no +# subclass relationship. +TYPE_PROMOTIONS = { + 'builtins.int': 'builtins.float', + 'builtins.float': 'builtins.complex', +} + +# Hard coded type promotions for Python 3. +# +# Note that the bytearray -> bytes promotion is a little unsafe +# as some functions only accept bytes objects. Here convenience +# trumps safety. +TYPE_PROMOTIONS_PYTHON3 = TYPE_PROMOTIONS.copy() +TYPE_PROMOTIONS_PYTHON3.update({ + 'builtins.bytearray': 'builtins.bytes', +}) + +# Hard coded type promotions for Python 2. +# +# These promotions are unsafe, but we are doing them anyway +# for convenience and also for Python 3 compatibility +# (bytearray -> str). +TYPE_PROMOTIONS_PYTHON2 = TYPE_PROMOTIONS.copy() +TYPE_PROMOTIONS_PYTHON2.update({ + 'builtins.str': 'builtins.unicode', + 'builtins.bytearray': 'builtins.str', +}) + +# When analyzing a function, should we analyze the whole function in one go, or +# should we only perform one phase of the analysis? The latter is used for +# nested functions. In the first phase we add the function to the symbol table +# but don't process body. In the second phase we process function body. This +# way we can have mutually recursive nested functions. +FUNCTION_BOTH_PHASES = 0 # Everthing in one go +FUNCTION_FIRST_PHASE_POSTPONE_SECOND = 1 # Add to symbol table but postpone body +FUNCTION_SECOND_PHASE = 2 # Only analyze body + + +class SemanticAnalyzer(NodeVisitor): + """Semantically analyze parsed mypy files. + + The analyzer binds names and does various consistency checks for a + parse tree. Note that type checking is performed as a separate + pass. + + This is the second phase of semantic analysis. + """ + + # Library search paths + lib_path = None # type: List[str] + # Module name space + modules = None # type: Dict[str, MypyFile] + # Global name space for current module + globals = None # type: SymbolTable + # Names declared using "global" (separate set for each scope) + global_decls = None # type: List[Set[str]] + # Names declated using "nonlocal" (separate set for each scope) + nonlocal_decls = None # type: List[Set[str]] + # Local names of function scopes; None for non-function scopes. + locals = None # type: List[SymbolTable] + # Nested block depths of scopes + block_depth = None # type: List[int] + # TypeInfo of directly enclosing class (or None) + type = None # type: TypeInfo + # Stack of outer classes (the second tuple item contains tvars). + type_stack = None # type: List[TypeInfo] + # Type variables that are bound by the directly enclosing class + bound_tvars = None # type: List[SymbolTableNode] + # Stack of type variables that were bound by outer classess + tvar_stack = None # type: List[List[SymbolTableNode]] + # Do weak type checking in this file + weak_opts = set() # type: Set[str] + + # Stack of functions being analyzed + function_stack = None # type: List[FuncItem] + # Stack of next available function type variable ids + next_function_tvar_id_stack = None # type: List[int] + + # Status of postponing analysis of nested function bodies. By using this we + # can have mutually recursive nested functions. Values are FUNCTION_x + # constants. Note that separate phasea are not used for methods. + postpone_nested_functions_stack = None # type: List[int] + # Postponed functions collected if + # postpone_nested_functions_stack[-1] == FUNCTION_FIRST_PHASE_POSTPONE_SECOND. + postponed_functions_stack = None # type: List[List[Node]] + + loop_depth = 0 # Depth of breakable loops + cur_mod_id = '' # Current module id (or None) (phase 2) + is_stub_file = False # Are we analyzing a stub file? + imports = None # type: Set[str] # Imported modules (during phase 2 analysis) + errors = None # type: Errors # Keeps track of generated errors + + def __init__(self, + lib_path: List[str], + errors: Errors, + options: Options) -> None: + """Construct semantic analyzer. + + Use lib_path to search for modules, and report analysis errors + using the Errors instance. + """ + self.locals = [None] + self.imports = set() + self.type = None + self.type_stack = [] + self.bound_tvars = None + self.tvar_stack = [] + self.function_stack = [] + self.next_function_tvar_id_stack = [-1] + self.block_depth = [0] + self.loop_depth = 0 + self.lib_path = lib_path + self.errors = errors + self.modules = {} + self.options = options + self.postpone_nested_functions_stack = [FUNCTION_BOTH_PHASES] + self.postponed_functions_stack = [] + self.all_exports = set() # type: Set[str] + + def visit_file(self, file_node: MypyFile, fnam: str) -> None: + self.errors.set_file(fnam) + self.cur_mod_node = file_node + self.cur_mod_id = file_node.fullname() + self.is_stub_file = fnam.lower().endswith('.pyi') + self.globals = file_node.names + self.weak_opts = file_node.weak_opts + + if 'builtins' in self.modules: + self.globals['__builtins__'] = SymbolTableNode( + MODULE_REF, self.modules['builtins'], self.cur_mod_id) + + for name in implicit_module_attrs: + v = self.globals[name].node + if isinstance(v, Var): + v.type = self.anal_type(v.type) + v.is_ready = True + + defs = file_node.defs + for d in defs: + self.accept(d) + + if self.cur_mod_id == 'builtins': + remove_imported_names_from_symtable(self.globals, 'builtins') + + if '__all__' in self.globals: + for name, g in self.globals.items(): + if name not in self.all_exports: + g.module_public = False + + def visit_func_def(self, defn: FuncDef) -> None: + phase_info = self.postpone_nested_functions_stack[-1] + if phase_info != FUNCTION_SECOND_PHASE: + self.function_stack.append(defn) + # First phase of analysis for function. + self.errors.push_function(defn.name()) + self.update_function_type_variables(defn) + self.errors.pop_function() + self.function_stack.pop() + + defn.is_conditional = self.block_depth[-1] > 0 + + # TODO(jukka): Figure out how to share the various cases. It doesn't + # make sense to have (almost) duplicate code (here and elsewhere) for + # 3 cases: module-level, class-level and local names. Maybe implement + # a common stack of namespaces. As the 3 kinds of namespaces have + # different semantics, this wouldn't always work, but it might still + # be a win. + if self.is_class_scope(): + # Method definition + defn.info = self.type + if not defn.is_decorated and not defn.is_overload: + if defn.name() in self.type.names: + # Redefinition. Conditional redefinition is okay. + n = self.type.names[defn.name()].node + if self.is_conditional_func(n, defn): + defn.original_def = cast(FuncDef, n) + else: + self.name_already_defined(defn.name(), defn) + self.type.names[defn.name()] = SymbolTableNode(MDEF, defn) + self.prepare_method_signature(defn) + elif self.is_func_scope(): + # Nested function + if not defn.is_decorated and not defn.is_overload: + if defn.name() in self.locals[-1]: + # Redefinition. Conditional redefinition is okay. + n = self.locals[-1][defn.name()].node + if self.is_conditional_func(n, defn): + defn.original_def = cast(FuncDef, n) + else: + self.name_already_defined(defn.name(), defn) + else: + self.add_local(defn, defn) + else: + # Top-level function + if not defn.is_decorated and not defn.is_overload: + symbol = self.globals.get(defn.name()) + if isinstance(symbol.node, FuncDef) and symbol.node != defn: + # This is redefinition. Conditional redefinition is okay. + original_def = symbol.node + if self.is_conditional_func(original_def, defn): + # Conditional function definition -- multiple defs are ok. + defn.original_def = original_def + else: + # Report error. + self.check_no_global(defn.name(), defn, True) + if phase_info == FUNCTION_FIRST_PHASE_POSTPONE_SECOND: + # Postpone this function (for the second phase). + self.postponed_functions_stack[-1].append(defn) + return + if phase_info != FUNCTION_FIRST_PHASE_POSTPONE_SECOND: + # Second phase of analysis for function. + self.errors.push_function(defn.name()) + self.analyze_function(defn) + self.errors.pop_function() + + def prepare_method_signature(self, func: FuncDef) -> None: + """Check basic signature validity and tweak annotation of self/cls argument.""" + # Only non-static methods are special. + if not func.is_static: + if not func.arguments: + self.fail('Method must have at least one argument', func) + elif func.type: + sig = cast(FunctionLike, func.type) + if func.is_class: + leading_type = self.class_type(self.type) + else: + leading_type = self_type(self.type) + func.type = replace_implicit_first_type(sig, leading_type) + + def is_conditional_func(self, previous: Node, new: FuncDef) -> bool: + """Does 'new' conditionally redefine 'previous'? + + We reject straight redefinitions of functions, as they are usually + a programming error. For example: + + . def f(): ... + . def f(): ... # Error: 'f' redefined + """ + return isinstance(previous, (FuncDef, Var)) and new.is_conditional + + def update_function_type_variables(self, defn: FuncDef) -> None: + """Make any type variables in the signature of defn explicit. + + Update the signature of defn to contain type variable definitions + if defn is generic. + """ + if defn.type: + functype = cast(CallableType, defn.type) + typevars = self.infer_type_variables(functype) + # Do not define a new type variable if already defined in scope. + typevars = [(name, tvar) for name, tvar in typevars + if not self.is_defined_type_var(name, defn)] + if typevars: + next_tvar_id = self.next_function_tvar_id() + defs = [TypeVarDef(tvar[0], next_tvar_id - i, + tvar[1].values, tvar[1].upper_bound, + tvar[1].variance) + for i, tvar in enumerate(typevars)] + functype.variables = defs + + def infer_type_variables(self, + type: CallableType) -> List[Tuple[str, TypeVarExpr]]: + """Return list of unique type variables referred to in a callable.""" + names = [] # type: List[str] + tvars = [] # type: List[TypeVarExpr] + for arg in type.arg_types + [type.ret_type]: + for name, tvar_expr in self.find_type_variables_in_type(arg): + if name not in names: + names.append(name) + tvars.append(tvar_expr) + return list(zip(names, tvars)) + + def find_type_variables_in_type( + self, type: Type) -> List[Tuple[str, TypeVarExpr]]: + """Return a list of all unique type variable references in type. + + This effectively does partial name binding, results of which are mostly thrown away. + """ + result = [] # type: List[Tuple[str, TypeVarExpr]] + if isinstance(type, UnboundType): + name = type.name + node = self.lookup_qualified(name, type) + if node and node.kind == UNBOUND_TVAR: + result.append((name, cast(TypeVarExpr, node.node))) + for arg in type.args: + result.extend(self.find_type_variables_in_type(arg)) + elif isinstance(type, TypeList): + for item in type.items: + result.extend(self.find_type_variables_in_type(item)) + elif isinstance(type, UnionType): + for item in type.items: + result.extend(self.find_type_variables_in_type(item)) + elif isinstance(type, AnyType): + pass + elif isinstance(type, EllipsisType) or isinstance(type, TupleType): + pass + else: + assert False, 'Unsupported type %s' % type + return result + + def is_defined_type_var(self, tvar: str, context: Node) -> bool: + return self.lookup_qualified(tvar, context).kind == BOUND_TVAR + + def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: + t = [] # type: List[CallableType] + for i, item in enumerate(defn.items): + # TODO support decorated overloaded functions properly + item.is_overload = True + item.func.is_overload = True + item.accept(self) + t.append(cast(CallableType, function_type(item.func, + self.builtin_type('builtins.function')))) + if item.func.is_property and i == 0: + # This defines a property, probably with a setter and/or deleter. + self.analyze_property_with_multi_part_definition(defn) + break + if not [dec for dec in item.decorators + if refers_to_fullname(dec, 'typing.overload')]: + self.fail("'overload' decorator expected", item) + + defn.type = Overloaded(t) + defn.type.line = defn.line + + if self.is_class_scope(): + self.type.names[defn.name()] = SymbolTableNode(MDEF, defn, + typ=defn.type) + defn.info = self.type + elif self.is_func_scope(): + self.add_local(defn, defn) + + def analyze_property_with_multi_part_definition(self, defn: OverloadedFuncDef) -> None: + """Analyze a property defined using multiple methods (e.g., using @x.setter). + + Assume that the first method (@property) has already been analyzed. + """ + defn.is_property = True + items = defn.items + for item in items[1:]: + if len(item.decorators) == 1: + node = item.decorators[0] + if isinstance(node, MemberExpr): + if node.name == 'setter': + # The first item represents the entire property. + defn.items[0].var.is_settable_property = True + # Get abstractness from the original definition. + item.func.is_abstract = items[0].func.is_abstract + else: + self.fail("Decorated property not supported", item) + item.func.accept(self) + + def next_function_tvar_id(self) -> int: + return self.next_function_tvar_id_stack[-1] + + def analyze_function(self, defn: FuncItem) -> None: + is_method = self.is_class_scope() + + tvarnodes = self.add_func_type_variables_to_symbol_table(defn) + next_function_tvar_id = min([self.next_function_tvar_id()] + + [n.tvar_def.id.raw_id - 1 for n in tvarnodes]) + self.next_function_tvar_id_stack.append(next_function_tvar_id) + + if defn.type: + # Signature must be analyzed in the surrounding scope so that + # class-level imported names and type variables are in scope. + defn.type = self.anal_type(defn.type) + self.check_function_signature(defn) + if isinstance(defn, FuncDef): + defn.type = set_callable_name(defn.type, defn) + for arg in defn.arguments: + if arg.initializer: + arg.initializer.accept(self) + self.function_stack.append(defn) + self.enter() + for arg in defn.arguments: + self.add_local(arg.variable, defn) + for arg in defn.arguments: + if arg.initialization_statement: + lvalue = arg.initialization_statement.lvalues[0] + lvalue.accept(self) + + # The first argument of a non-static, non-class method is like 'self' + # (though the name could be different), having the enclosing class's + # instance type. + if is_method and not defn.is_static and not defn.is_class and defn.arguments: + defn.arguments[0].variable.is_self = True + + # First analyze body of the function but ignore nested functions. + self.postpone_nested_functions_stack.append(FUNCTION_FIRST_PHASE_POSTPONE_SECOND) + self.postponed_functions_stack.append([]) + defn.body.accept(self) + + # Analyze nested functions (if any) as a second phase. + self.postpone_nested_functions_stack[-1] = FUNCTION_SECOND_PHASE + for postponed in self.postponed_functions_stack[-1]: + postponed.accept(self) + self.postpone_nested_functions_stack.pop() + self.postponed_functions_stack.pop() + + self.next_function_tvar_id_stack.pop() + disable_typevars(tvarnodes) + + self.leave() + self.function_stack.pop() + + def add_func_type_variables_to_symbol_table( + self, defn: FuncItem) -> List[SymbolTableNode]: + nodes = [] # type: List[SymbolTableNode] + if defn.type: + tt = defn.type + names = self.type_var_names() + items = cast(CallableType, tt).variables + for item in items: + name = item.name + if name in names: + self.name_already_defined(name, defn) + node = self.bind_type_var(name, item, defn) + nodes.append(node) + names.add(name) + return nodes + + def type_var_names(self) -> Set[str]: + if not self.type: + return set() + else: + return set(self.type.type_vars) + + def bind_type_var(self, fullname: str, tvar_def: TypeVarDef, + context: Context) -> SymbolTableNode: + node = self.lookup_qualified(fullname, context) + node.kind = BOUND_TVAR + node.tvar_def = tvar_def + return node + + def check_function_signature(self, fdef: FuncItem) -> None: + sig = cast(CallableType, fdef.type) + if len(sig.arg_types) < len(fdef.arguments): + self.fail('Type signature has too few arguments', fdef) + # Add dummy Any arguments to prevent crashes later. + extra_anys = [AnyType()] * (len(fdef.arguments) - len(sig.arg_types)) + sig.arg_types.extend(extra_anys) + elif len(sig.arg_types) > len(fdef.arguments): + self.fail('Type signature has too many arguments', fdef) + + def visit_class_def(self, defn: ClassDef) -> None: + self.clean_up_bases_and_infer_type_variables(defn) + self.setup_class_def_analysis(defn) + + self.bind_class_type_vars(defn) + + self.analyze_base_classes(defn) + self.analyze_metaclass(defn) + + for decorator in defn.decorators: + self.analyze_class_decorator(defn, decorator) + + self.enter_class(defn) + + self.setup_is_builtinclass(defn) + + # Analyze class body. + defn.defs.accept(self) + + self.calculate_abstract_status(defn.info) + self.setup_type_promotion(defn) + + self.leave_class() + self.unbind_class_type_vars() + + def enter_class(self, defn: ClassDef) -> None: + # Remember previous active class + self.type_stack.append(self.type) + self.locals.append(None) # Add class scope + self.block_depth.append(-1) # The class body increments this to 0 + self.postpone_nested_functions_stack.append(FUNCTION_BOTH_PHASES) + self.type = defn.info + + def leave_class(self) -> None: + """ Restore analyzer state. """ + self.postpone_nested_functions_stack.pop() + self.block_depth.pop() + self.locals.pop() + self.type = self.type_stack.pop() + + def bind_class_type_vars(self, defn: ClassDef) -> None: + """ Unbind type variables of previously active class and bind + the type variables for the active class. + """ + if self.bound_tvars: + disable_typevars(self.bound_tvars) + self.tvar_stack.append(self.bound_tvars) + self.bound_tvars = self.bind_class_type_variables_in_symbol_table(defn.info) + + def unbind_class_type_vars(self) -> None: + """ Unbind the active class' type vars and rebind the + type vars of the previously active class. + """ + disable_typevars(self.bound_tvars) + self.bound_tvars = self.tvar_stack.pop() + if self.bound_tvars: + enable_typevars(self.bound_tvars) + + def analyze_class_decorator(self, defn: ClassDef, decorator: Node) -> None: + decorator.accept(self) + + def setup_is_builtinclass(self, defn: ClassDef): + for decorator in defn.decorators: + if refers_to_fullname(decorator, 'typing.builtinclass'): + defn.is_builtinclass = True + if defn.fullname == 'builtins.object': + # Only 'object' is marked as a built-in class, as otherwise things elsewhere + # would break. We need a better way of dealing with built-in classes. + defn.is_builtinclass = True + + def calculate_abstract_status(self, typ: TypeInfo) -> None: + """Calculate abstract status of a class. + + Set is_abstract of the type to True if the type has an unimplemented + abstract attribute. Also compute a list of abstract attributes. + """ + concrete = set() # type: Set[str] + abstract = [] # type: List[str] + for base in typ.mro: + for name, symnode in base.names.items(): + node = symnode.node + if isinstance(node, OverloadedFuncDef): + # Unwrap an overloaded function definition. We can just + # check arbitrarily the first overload item. If the + # different items have a different abstract status, there + # should be an error reported elsewhere. + func = node.items[0] # type: Node + else: + func = node + if isinstance(func, Decorator): + fdef = func.func + if fdef.is_abstract and name not in concrete: + typ.is_abstract = True + abstract.append(name) + concrete.add(name) + typ.abstract_attributes = sorted(abstract) + + def setup_type_promotion(self, defn: ClassDef) -> None: + """Setup extra, ad-hoc subtyping relationships between classes (promotion). + + This includes things like 'int' being compatible with 'float'. + """ + promote_target = None # type: Type + for decorator in defn.decorators: + if isinstance(decorator, CallExpr): + analyzed = decorator.analyzed + if isinstance(analyzed, PromoteExpr): + # _promote class decorator (undocumented faeture). + promote_target = analyzed.type + if not promote_target: + promotions = (TYPE_PROMOTIONS_PYTHON3 if self.options.python_version[0] >= 3 + else TYPE_PROMOTIONS_PYTHON2) + if defn.fullname in promotions: + promote_target = self.named_type_or_none(promotions[defn.fullname]) + defn.info._promote = promote_target + + def clean_up_bases_and_infer_type_variables(self, defn: ClassDef) -> None: + """Remove extra base classes such as Generic and infer type vars. + + For example, consider this class: + + . class Foo(Bar, Generic[T]): ... + + Now we will remove Generic[T] from bases of Foo and infer that the + type variable 'T' is a type argument of Foo. + + Note that this is performed *before* semantic analysis. + """ + removed = [] # type: List[int] + type_vars = [] # type: List[TypeVarDef] + for i, base_expr in enumerate(defn.base_type_exprs): + try: + base = expr_to_unanalyzed_type(base_expr) + except TypeTranslationError: + # This error will be caught later. + continue + tvars = self.analyze_typevar_declaration(base) + if tvars is not None: + if type_vars: + self.fail('Duplicate Generic in bases', defn) + removed.append(i) + for j, (name, tvar_expr) in enumerate(tvars): + type_vars.append(TypeVarDef(name, j + 1, tvar_expr.values, + tvar_expr.upper_bound, tvar_expr.variance)) + if type_vars: + defn.type_vars = type_vars + if defn.info: + defn.info.type_vars = [tv.name for tv in type_vars] + for i in reversed(removed): + del defn.base_type_exprs[i] + + def analyze_typevar_declaration(self, t: Type) -> Optional[List[Tuple[str, TypeVarExpr]]]: + if not isinstance(t, UnboundType): + return None + unbound = t + sym = self.lookup_qualified(unbound.name, unbound) + if sym is None or sym.node is None: + return None + if sym.node.fullname() == 'typing.Generic': + tvars = [] # type: List[Tuple[str, TypeVarExpr]] + for arg in unbound.args: + tvar = self.analyze_unbound_tvar(arg) + if tvar: + tvars.append(tvar) + else: + self.fail('Free type variable expected in %s[...]' % + sym.node.name(), t) + return tvars + return None + + def analyze_unbound_tvar(self, t: Type) -> Tuple[str, TypeVarExpr]: + if not isinstance(t, UnboundType): + return None + unbound = t + sym = self.lookup_qualified(unbound.name, unbound) + if sym is not None and sym.kind == UNBOUND_TVAR: + return unbound.name, cast(TypeVarExpr, sym.node) + return None + + def setup_class_def_analysis(self, defn: ClassDef) -> None: + """Prepare for the analysis of a class definition.""" + if not defn.info: + defn.info = TypeInfo(SymbolTable(), defn) + defn.info._fullname = defn.info.name() + if self.is_func_scope() or self.type: + kind = MDEF + if self.is_func_scope(): + kind = LDEF + self.add_symbol(defn.name, SymbolTableNode(kind, defn.info), defn) + + def analyze_base_classes(self, defn: ClassDef) -> None: + """Analyze and set up base classes. + + This computes several attributes on the corresponding TypeInfo defn.info + related to the base classes: defn.info.bases, defn.info.mro, and + miscellaneous others (at least tuple_type, fallback_to_any, and is_enum.) + """ + + base_types = [] # type: List[Instance] + info = defn.info + for base_expr in defn.base_type_exprs: + try: + base = self.expr_to_analyzed_type(base_expr) + except TypeTranslationError: + self.fail('Invalid base class', base_expr) + info.fallback_to_any = True + continue + + if isinstance(base, TupleType): + if info.tuple_type: + self.fail("Class has two incompatible bases derived from tuple", defn) + defn.incompatible_baseclass = True + if (not self.is_stub_file + and not info.is_named_tuple + and base.fallback.type.fullname() == 'builtins.tuple'): + self.fail("Tuple[...] not supported as a base class outside a stub file", defn) + info.tuple_type = base + base_types.append(base.fallback) + elif isinstance(base, Instance): + base_types.append(base) + elif isinstance(base, AnyType): + info.fallback_to_any = True + else: + self.fail('Invalid base class', base_expr) + info.fallback_to_any = True + + # Add 'object' as implicit base if there is no other base class. + if (not base_types and defn.fullname != 'builtins.object'): + base_types.append(self.object_type()) + + info.bases = base_types + + # Calculate the MRO. It might be incomplete at this point if + # the bases of defn include classes imported from other + # modules in an import loop. We'll recompute it in ThirdPass. + if not self.verify_base_classes(defn): + info.mro = [] + return + calculate_class_mro(defn, self.fail) + # If there are cyclic imports, we may be missing 'object' in + # the MRO. Fix MRO if needed. + if info.mro and info.mro[-1].fullname() != 'builtins.object': + info.mro.append(self.object_type().type) + + def expr_to_analyzed_type(self, expr: Node) -> Type: + if isinstance(expr, CallExpr): + expr.accept(self) + info = self.check_namedtuple(expr) + if info is None: + # Some form of namedtuple is the only valid type that looks like a call + # expression. This isn't a valid type. + raise TypeTranslationError() + fallback = Instance(info, []) + return TupleType(info.tuple_type.items, fallback=fallback) + typ = expr_to_unanalyzed_type(expr) + return self.anal_type(typ) + + def verify_base_classes(self, defn: ClassDef) -> bool: + info = defn.info + for base in info.bases: + baseinfo = base.type + if self.is_base_class(info, baseinfo): + self.fail('Cycle in inheritance hierarchy', defn) + # Clear bases to forcefully get rid of the cycle. + info.bases = [] + if baseinfo.fullname() == 'builtins.bool': + self.fail("'%s' is not a valid base class" % + baseinfo.name(), defn) + return False + dup = find_duplicate(info.direct_base_classes()) + if dup: + self.fail('Duplicate base class "%s"' % dup.name(), defn) + return False + return True + + def is_base_class(self, t: TypeInfo, s: TypeInfo) -> bool: + """Determine if t is a base class of s (but do not use mro).""" + # Search the base class graph for t, starting from s. + worklist = [s] + visited = {s} + while worklist: + nxt = worklist.pop() + if nxt == t: + return True + for base in nxt.bases: + if base.type not in visited: + worklist.append(base.type) + visited.add(base.type) + return False + + def analyze_metaclass(self, defn: ClassDef) -> None: + if defn.metaclass: + sym = self.lookup_qualified(defn.metaclass, defn) + if sym is not None and not isinstance(sym.node, TypeInfo): + self.fail("Invalid metaclass '%s'" % defn.metaclass, defn) + + def object_type(self) -> Instance: + return self.named_type('__builtins__.object') + + def class_type(self, info: TypeInfo) -> Type: + # Construct a function type whose fallback is cls. + from mypy import checkmember # To avoid import cycle. + leading_type = checkmember.type_object_type(info, self.builtin_type) + if isinstance(leading_type, Overloaded): + # Overloaded __init__ is too complex to handle. Plus it's stubs only. + return AnyType() + else: + return leading_type + + def named_type(self, qualified_name: str, args: List[Type] = None) -> Instance: + sym = self.lookup_qualified(qualified_name, None) + return Instance(cast(TypeInfo, sym.node), args or []) + + def named_type_or_none(self, qualified_name: str) -> Instance: + sym = self.lookup_fully_qualified_or_none(qualified_name) + if not sym: + return None + return Instance(cast(TypeInfo, sym.node), []) + + def is_instance_type(self, t: Type) -> bool: + return isinstance(t, Instance) + + def bind_class_type_variables_in_symbol_table( + self, info: TypeInfo) -> List[SymbolTableNode]: + nodes = [] # type: List[SymbolTableNode] + for var, binder in zip(info.type_vars, info.defn.type_vars): + node = self.bind_type_var(var, binder, info) + nodes.append(node) + return nodes + + def visit_import(self, i: Import) -> None: + for id, as_id in i.ids: + if as_id is not None: + self.add_module_symbol(id, as_id, module_public=True, context=i) + else: + # Modules imported in a stub file without using 'as x' won't get exported when + # doing 'from m import *'. + module_public = not self.is_stub_file + base = id.split('.')[0] + self.add_module_symbol(base, base, module_public=module_public, + context=i) + + def add_module_symbol(self, id: str, as_id: str, module_public: bool, + context: Context) -> None: + if id in self.modules: + m = self.modules[id] + self.add_symbol(as_id, SymbolTableNode(MODULE_REF, m, self.cur_mod_id, + module_public=module_public), context) + else: + self.add_unknown_symbol(as_id, context) + + def visit_import_from(self, imp: ImportFrom) -> None: + import_id = self.correct_relative_import(imp) + if import_id in self.modules: + module = self.modules[import_id] + for id, as_id in imp.names: + node = module.names.get(id) + if node and node.kind != UNBOUND_IMPORTED: + node = self.normalize_type_alias(node, imp) + if not node: + return + imported_id = as_id or id + existing_symbol = self.globals.get(imported_id) + if existing_symbol: + # Import can redefine a variable. They get special treatment. + if self.process_import_over_existing_name( + imported_id, existing_symbol, node, imp): + continue + # 'from m import x as x' exports x in a stub file. + module_public = not self.is_stub_file or as_id is not None + symbol = SymbolTableNode(node.kind, node.node, + self.cur_mod_id, + node.type_override, + module_public=module_public) + self.add_symbol(imported_id, symbol, imp) + else: + message = "Module has no attribute '{}'".format(id) + extra = self.undefined_name_extra_info('{}.{}'.format(import_id, id)) + if extra: + message += " {}".format(extra) + self.fail(message, imp) + else: + # Missing module. + for id, as_id in imp.names: + self.add_unknown_symbol(as_id or id, imp) + + def process_import_over_existing_name(self, + imported_id: str, existing_symbol: SymbolTableNode, + module_symbol: SymbolTableNode, + import_node: ImportBase) -> bool: + if (existing_symbol.kind in (LDEF, GDEF, MDEF) and + isinstance(existing_symbol.node, (Var, FuncDef))): + # This is a valid import over an existing definition in the file. Construct a dummy + # assignment that we'll use to type check the import. + lvalue = NameExpr(imported_id) + lvalue.kind = existing_symbol.kind + lvalue.node = existing_symbol.node + rvalue = NameExpr(imported_id) + rvalue.kind = module_symbol.kind + rvalue.node = module_symbol.node + assignment = AssignmentStmt([lvalue], rvalue) + for node in assignment, lvalue, rvalue: + node.set_line(import_node) + import_node.assignments.append(assignment) + return True + return False + + def normalize_type_alias(self, node: SymbolTableNode, + ctx: Context) -> SymbolTableNode: + if node.fullname in type_aliases: + # Node refers to an aliased type such as typing.List; normalize. + node = self.lookup_qualified(type_aliases[node.fullname], ctx) + if node.fullname == 'typing.DefaultDict': + self.add_module_symbol('collections', '__mypy_collections__', False, ctx) + node = self.lookup_qualified('__mypy_collections__.defaultdict', ctx) + return node + + def correct_relative_import(self, node: Union[ImportFrom, ImportAll]) -> str: + if node.relative == 0: + return node.id + + parts = self.cur_mod_id.split(".") + cur_mod_id = self.cur_mod_id + + rel = node.relative + if self.cur_mod_node.is_package_init_file(): + rel -= 1 + if len(parts) < rel: + self.fail("Relative import climbs too many namespaces", node) + if rel != 0: + cur_mod_id = ".".join(parts[:-rel]) + + return cur_mod_id + (("." + node.id) if node.id else "") + + def visit_import_all(self, i: ImportAll) -> None: + i_id = self.correct_relative_import(i) + if i_id in self.modules: + m = self.modules[i_id] + for name, node in m.names.items(): + node = self.normalize_type_alias(node, i) + if not name.startswith('_') and node.module_public: + existing_symbol = self.globals.get(name) + if existing_symbol: + # Import can redefine a variable. They get special treatment. + if self.process_import_over_existing_name( + name, existing_symbol, node, i): + continue + self.add_symbol(name, SymbolTableNode(node.kind, node.node, + self.cur_mod_id), i) + else: + # Don't add any dummy symbols for 'from x import *' if 'x' is unknown. + pass + + def add_unknown_symbol(self, name: str, context: Context) -> None: + var = Var(name) + var._fullname = self.qualified_name(name) + var.is_ready = True + var.type = AnyType() + self.add_symbol(name, SymbolTableNode(GDEF, var, self.cur_mod_id), context) + + # + # Statements + # + + def visit_block(self, b: Block) -> None: + if b.is_unreachable: + return + self.block_depth[-1] += 1 + for s in b.body: + self.accept(s) + self.block_depth[-1] -= 1 + + def visit_block_maybe(self, b: Block) -> None: + if b: + self.visit_block(b) + + def anal_type(self, t: Type, allow_tuple_literal: bool = False) -> Type: + if t: + if allow_tuple_literal: + # Types such as (t1, t2, ...) only allowed in assignment statements. They'll + # generate errors elsewhere, and Tuple[t1, t2, ...] must be used instead. + if isinstance(t, TupleType): + # Unlike TypeAnalyser, also allow implicit tuple types (without Tuple[...]). + star_count = sum(1 for item in t.items if isinstance(item, StarType)) + if star_count > 1: + self.fail('At most one star type allowed in a tuple', t) + return TupleType([AnyType() for _ in t.items], + self.builtin_type('builtins.tuple'), t.line) + items = [self.anal_type(item, True) + for item in t.items] + return TupleType(items, self.builtin_type('builtins.tuple'), t.line) + a = TypeAnalyser(self.lookup_qualified, + self.lookup_fully_qualified, + self.fail) + return t.accept(a) + else: + return None + + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + for lval in s.lvalues: + self.analyze_lvalue(lval, explicit_type=s.type is not None) + s.rvalue.accept(self) + if s.type: + allow_tuple_literal = isinstance(s.lvalues[-1], (TupleExpr, ListExpr)) + s.type = self.anal_type(s.type, allow_tuple_literal) + else: + # For simple assignments, allow binding type aliases. + # Also set the type if the rvalue is a simple literal. + if (s.type is None and len(s.lvalues) == 1 and + isinstance(s.lvalues[0], NameExpr)): + if s.lvalues[0].is_def: + s.type = self.analyze_simple_literal_type(s.rvalue) + res = analyze_type_alias(s.rvalue, + self.lookup_qualified, + self.lookup_fully_qualified, + self.fail) + if res and (not isinstance(res, Instance) or cast(Instance, res).args): + # TODO: What if this gets reassigned? + name = s.lvalues[0] + node = self.lookup(name.name, name) + node.kind = TYPE_ALIAS + node.type_override = res + if isinstance(s.rvalue, IndexExpr): + s.rvalue.analyzed = TypeAliasExpr(res) + if s.type: + # Store type into nodes. + for lvalue in s.lvalues: + self.store_declared_types(lvalue, s.type) + self.check_and_set_up_type_alias(s) + self.process_typevar_declaration(s) + self.process_namedtuple_definition(s) + + if (len(s.lvalues) == 1 and isinstance(s.lvalues[0], NameExpr) and + s.lvalues[0].name == '__all__' and s.lvalues[0].kind == GDEF and + isinstance(s.rvalue, (ListExpr, TupleExpr))): + self.add_exports(*s.rvalue.items) + + def analyze_simple_literal_type(self, rvalue: Node) -> Optional[Type]: + """Return builtins.int if rvalue is an int literal, etc.""" + if self.weak_opts or self.options.semantic_analysis_only or self.function_stack: + # Skip this if any weak options are set. + # Also skip if we're only doing the semantic analysis pass. + # This is mostly to avoid breaking unit tests. + # Also skip inside a function; this is to avoid confusing + # the code that handles dead code due to isinstance() + # inside type variables with value restrictions (like + # AnyStr). + return None + if isinstance(rvalue, IntExpr): + return self.named_type_or_none('builtins.int') + if isinstance(rvalue, FloatExpr): + return self.named_type_or_none('builtins.float') + if isinstance(rvalue, StrExpr): + return self.named_type_or_none('builtins.str') + if isinstance(rvalue, BytesExpr): + return self.named_type_or_none('builtins.bytes') + if isinstance(rvalue, UnicodeExpr): + return self.named_type_or_none('builtins.unicode') + return None + + def check_and_set_up_type_alias(self, s: AssignmentStmt) -> None: + """Check if assignment creates a type alias and set it up as needed.""" + # For now, type aliases only work at the top level of a module. + if (len(s.lvalues) == 1 and not self.is_func_scope() and not self.type + and not s.type): + lvalue = s.lvalues[0] + if isinstance(lvalue, NameExpr): + if not lvalue.is_def: + # Only a definition can create a type alias, not regular assignment. + return + rvalue = s.rvalue + if isinstance(rvalue, RefExpr): + node = rvalue.node + if isinstance(node, TypeInfo): + # TODO: We should record the fact that this is a variable + # that refers to a type, rather than making this + # just an alias for the type. + self.globals[lvalue.name].node = node + + def analyze_lvalue(self, lval: Node, nested: bool = False, + add_global: bool = False, + explicit_type: bool = False) -> None: + """Analyze an lvalue or assignment target. + + Only if add_global is True, add name to globals table. If nested + is true, the lvalue is within a tuple or list lvalue expression. + """ + + if isinstance(lval, NameExpr): + # Top-level definitions within some statements (at least while) are + # not handled in the first pass, so they have to be added now. + nested_global = (not self.is_func_scope() and + self.block_depth[-1] > 0 and + not self.type) + if (add_global or nested_global) and lval.name not in self.globals: + # Define new global name. + v = Var(lval.name) + v._fullname = self.qualified_name(lval.name) + v.is_ready = False # Type not inferred yet + lval.node = v + lval.is_def = True + lval.kind = GDEF + lval.fullname = v._fullname + self.globals[lval.name] = SymbolTableNode(GDEF, v, + self.cur_mod_id) + elif isinstance(lval.node, Var) and lval.is_def: + # Since the is_def flag is set, this must have been analyzed + # already in the first pass and added to the symbol table. + assert lval.node.name() in self.globals + elif (self.is_func_scope() and lval.name not in self.locals[-1] and + lval.name not in self.global_decls[-1] and + lval.name not in self.nonlocal_decls[-1]): + # Define new local name. + v = Var(lval.name) + lval.node = v + lval.is_def = True + lval.kind = LDEF + lval.fullname = lval.name + self.add_local(v, lval) + elif not self.is_func_scope() and (self.type and + lval.name not in self.type.names): + # Define a new attribute within class body. + v = Var(lval.name) + v.info = self.type + v.is_initialized_in_class = True + v.set_line(lval) + lval.node = v + lval.is_def = True + lval.kind = MDEF + lval.fullname = lval.name + self.type.names[lval.name] = SymbolTableNode(MDEF, v) + else: + # Bind to an existing name. + if explicit_type: + self.name_already_defined(lval.name, lval) + lval.accept(self) + self.check_lvalue_validity(lval.node, lval) + elif isinstance(lval, MemberExpr): + if not add_global: + self.analyze_member_lvalue(lval) + if explicit_type and not self.is_self_member_ref(lval): + self.fail('Type cannot be declared in assignment to non-self ' + 'attribute', lval) + elif isinstance(lval, IndexExpr): + if explicit_type: + self.fail('Unexpected type declaration', lval) + if not add_global: + lval.accept(self) + elif (isinstance(lval, TupleExpr) or + isinstance(lval, ListExpr)): + items = cast(Any, lval).items + if len(items) == 0 and isinstance(lval, TupleExpr): + self.fail("Can't assign to ()", lval) + self.analyze_tuple_or_list_lvalue(cast(Union[ListExpr, TupleExpr], lval), + add_global, explicit_type) + elif isinstance(lval, StarExpr): + if nested: + self.analyze_lvalue(lval.expr, nested, add_global, explicit_type) + else: + self.fail('Starred assignment target must be in a list or tuple', lval) + else: + self.fail('Invalid assignment target', lval) + + def analyze_tuple_or_list_lvalue(self, lval: Union[ListExpr, TupleExpr], + add_global: bool = False, + explicit_type: bool = False) -> None: + """Analyze an lvalue or assignment target that is a list or tuple.""" + items = lval.items + star_exprs = [cast(StarExpr, item) + for item in items + if isinstance(item, StarExpr)] + + if len(star_exprs) > 1: + self.fail('Two starred expressions in assignment', lval) + else: + if len(star_exprs) == 1: + star_exprs[0].valid = True + for i in items: + self.analyze_lvalue(i, nested=True, add_global=add_global, + explicit_type = explicit_type) + + def analyze_member_lvalue(self, lval: MemberExpr) -> None: + lval.accept(self) + if (self.is_self_member_ref(lval) and + self.type.get(lval.name) is None): + # Implicit attribute definition in __init__. + lval.is_def = True + v = Var(lval.name) + v.info = self.type + v.is_ready = False + lval.def_var = v + lval.node = v + self.type.names[lval.name] = SymbolTableNode(MDEF, v) + self.check_lvalue_validity(lval.node, lval) + + def is_self_member_ref(self, memberexpr: MemberExpr) -> bool: + """Does memberexpr to refer to an attribute of self?""" + if not isinstance(memberexpr.expr, NameExpr): + return False + node = memberexpr.expr.node + return isinstance(node, Var) and node.is_self + + def check_lvalue_validity(self, node: Node, ctx: Context) -> None: + if isinstance(node, (TypeInfo, TypeVarExpr)): + self.fail('Invalid assignment target', ctx) + + def store_declared_types(self, lvalue: Node, typ: Type) -> None: + if isinstance(typ, StarType) and not isinstance(lvalue, StarExpr): + self.fail('Star type only allowed for starred expressions', lvalue) + if isinstance(lvalue, RefExpr): + lvalue.is_def = False + if isinstance(lvalue.node, Var): + var = lvalue.node + var.type = typ + var.is_ready = True + # If node is not a variable, we'll catch it elsewhere. + elif isinstance(lvalue, TupleExpr): + if isinstance(typ, TupleType): + if len(lvalue.items) != len(typ.items): + self.fail('Incompatible number of tuple items', lvalue) + return + for item, itemtype in zip(lvalue.items, typ.items): + self.store_declared_types(item, itemtype) + else: + self.fail('Tuple type expected for multiple variables', + lvalue) + elif isinstance(lvalue, StarExpr): + if isinstance(typ, StarType): + self.store_declared_types(lvalue.expr, typ.type) + else: + self.fail('Star type expected for starred expression', lvalue) + else: + # This has been flagged elsewhere as an error, so just ignore here. + pass + + def process_typevar_declaration(self, s: AssignmentStmt) -> None: + """Check if s declares a TypeVar; it yes, store it in symbol table.""" + call = self.get_typevar_declaration(s) + if not call: + return + + lvalue = cast(NameExpr, s.lvalues[0]) + name = lvalue.name + if not lvalue.is_def: + if s.type: + self.fail("Cannot declare the type of a type variable", s) + else: + self.fail("Cannot redefine '%s' as a type variable" % name, s) + return + + if not self.check_typevar_name(call, name, s): + return + + # Constraining types + n_values = call.arg_kinds[1:].count(ARG_POS) + values = self.analyze_types(call.args[1:1 + n_values]) + + res = self.process_typevar_parameters(call.args[1 + n_values:], + call.arg_names[1 + n_values:], + call.arg_kinds[1 + n_values:], + bool(values), + s) + if res is None: + return + variance, upper_bound = res + + # Yes, it's a valid type variable definition! Add it to the symbol table. + node = self.lookup(name, s) + node.kind = UNBOUND_TVAR + TypeVar = TypeVarExpr(name, node.fullname, values, upper_bound, variance) + TypeVar.line = call.line + call.analyzed = TypeVar + node.node = TypeVar + + def check_typevar_name(self, call: CallExpr, name: str, context: Context) -> bool: + if len(call.args) < 1: + self.fail("Too few arguments for TypeVar()", context) + return False + if not isinstance(call.args[0], (StrExpr, BytesExpr)) or not call.arg_kinds[0] == ARG_POS: + self.fail("TypeVar() expects a string literal as first argument", context) + return False + if cast(StrExpr, call.args[0]).value != name: + self.fail("Unexpected TypeVar() argument value", context) + return False + return True + + def get_typevar_declaration(self, s: AssignmentStmt) -> Optional[CallExpr]: + """Returns the TypeVar() call expression if `s` is a type var declaration + or None otherwise. + """ + if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): + return None + if not isinstance(s.rvalue, CallExpr): + return None + call = s.rvalue + if not isinstance(call.callee, RefExpr): + return None + callee = call.callee + if callee.fullname != 'typing.TypeVar': + return None + return call + + def process_typevar_parameters(self, args: List[Node], + names: List[Optional[str]], + kinds: List[int], + has_values: bool, + context: Context) -> Optional[Tuple[int, Type]]: + covariant = False + contravariant = False + upper_bound = self.object_type() # type: Type + for param_value, param_name, param_kind in zip(args, names, kinds): + if not param_kind == ARG_NAMED: + self.fail("Unexpected argument to TypeVar()", context) + return None + if param_name == 'covariant': + if isinstance(param_value, NameExpr): + if param_value.name == 'True': + covariant = True + else: + self.fail("TypeVar 'covariant' may only be 'True'", context) + return None + else: + self.fail("TypeVar 'covariant' may only be 'True'", context) + return None + elif param_name == 'contravariant': + if isinstance(param_value, NameExpr): + if param_value.name == 'True': + contravariant = True + else: + self.fail("TypeVar 'contravariant' may only be 'True'", context) + return None + else: + self.fail("TypeVar 'contravariant' may only be 'True'", context) + return None + elif param_name == 'bound': + if has_values: + self.fail("TypeVar cannot have both values and an upper bound", context) + return None + try: + upper_bound = self.expr_to_analyzed_type(param_value) + except TypeTranslationError: + self.fail("TypeVar 'bound' must be a type", param_value) + return None + elif param_name == 'values': + # Probably using obsolete syntax with values=(...). Explain the current syntax. + self.fail("TypeVar 'values' argument not supported", context) + self.fail("Use TypeVar('T', t, ...) instead of TypeVar('T', values=(t, ...))", + context) + return None + else: + self.fail("Unexpected argument to TypeVar(): {}".format(param_name), context) + return None + + if covariant and contravariant: + self.fail("TypeVar cannot be both covariant and contravariant", context) + return None + elif covariant: + variance = COVARIANT + elif contravariant: + variance = CONTRAVARIANT + else: + variance = INVARIANT + return (variance, upper_bound) + + def process_namedtuple_definition(self, s: AssignmentStmt) -> None: + """Check if s defines a namedtuple; if yes, store the definition in symbol table.""" + if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): + return + lvalue = cast(NameExpr, s.lvalues[0]) + name = lvalue.name + named_tuple = self.check_namedtuple(s.rvalue, name) + if named_tuple is None: + return + # Yes, it's a valid namedtuple definition. Add it to the symbol table. + node = self.lookup(name, s) + node.kind = GDEF # TODO locally defined namedtuple + # TODO call.analyzed + node.node = named_tuple + + def check_namedtuple(self, node: Node, var_name: str = None) -> TypeInfo: + """Check if a call defines a namedtuple. + + The optional var_name argument is the name of the variable to + which this is assigned, if any. + + If it does, return the corresponding TypeInfo. Return None otherwise. + + If the definition is invalid but looks like a namedtuple, + report errors but return (some) TypeInfo. + """ + if not isinstance(node, CallExpr): + return None + call = node + if not isinstance(call.callee, RefExpr): + return None + callee = call.callee + fullname = callee.fullname + if fullname not in ('collections.namedtuple', 'typing.NamedTuple'): + return None + items, types, ok = self.parse_namedtuple_args(call, fullname) + if not ok: + # Error. Construct dummy return value. + return self.build_namedtuple_typeinfo('namedtuple', [], []) + else: + # Give it a unique name derived from the line number. + name = cast(StrExpr, call.args[0]).value + if name != var_name: + name += '@' + str(call.line) + info = self.build_namedtuple_typeinfo(name, items, types) + # Store it as a global just in case it would remain anonymous. + self.globals[name] = SymbolTableNode(GDEF, info, self.cur_mod_id) + call.analyzed = NamedTupleExpr(info).set_line(call.line) + return info + + def parse_namedtuple_args(self, call: CallExpr, + fullname: str) -> Tuple[List[str], List[Type], bool]: + # TODO Share code with check_argument_count in checkexpr.py? + args = call.args + if len(args) < 2: + return self.fail_namedtuple_arg("Too few arguments for namedtuple()", call) + if len(args) > 2: + return self.fail_namedtuple_arg("Too many arguments for namedtuple()", call) + if call.arg_kinds != [ARG_POS, ARG_POS]: + return self.fail_namedtuple_arg("Unexpected arguments to namedtuple()", call) + if not isinstance(args[0], (StrExpr, BytesExpr)): + return self.fail_namedtuple_arg( + "namedtuple() expects a string literal as the first argument", call) + types = [] # type: List[Type] + ok = True + if not isinstance(args[1], ListExpr): + if fullname == 'collections.namedtuple' and isinstance(args[1], (StrExpr, BytesExpr)): + str_expr = cast(StrExpr, args[1]) + items = str_expr.value.split() + else: + return self.fail_namedtuple_arg( + "List literal expected as the second argument to namedtuple()", call) + else: + listexpr = args[1] + if fullname == 'collections.namedtuple': + # The fields argument contains just names, with implicit Any types. + if any(not isinstance(item, (StrExpr, BytesExpr)) for item in listexpr.items): + return self.fail_namedtuple_arg("String literal expected as namedtuple() item", + call) + items = [cast(StrExpr, item).value for item in listexpr.items] + else: + # The fields argument contains (name, type) tuples. + items, types, ok = self.parse_namedtuple_fields_with_types(listexpr.items, call) + if not types: + types = [AnyType() for _ in items] + return items, types, ok + + def parse_namedtuple_fields_with_types(self, nodes: List[Node], + context: Context) -> Tuple[List[str], List[Type], bool]: + items = [] # type: List[str] + types = [] # type: List[Type] + for item in nodes: + if isinstance(item, TupleExpr): + if len(item.items) != 2: + return self.fail_namedtuple_arg("Invalid NamedTuple field definition", + item) + name, type_node = item.items + if isinstance(name, (StrExpr, BytesExpr)): + items.append(name.value) + else: + return self.fail_namedtuple_arg("Invalid NamedTuple() field name", item) + try: + type = expr_to_unanalyzed_type(type_node) + except TypeTranslationError: + return self.fail_namedtuple_arg('Invalid field type', type_node) + types.append(self.anal_type(type)) + else: + return self.fail_namedtuple_arg("Tuple expected as NamedTuple() field", item) + return items, types, True + + def fail_namedtuple_arg(self, message: str, + context: Context) -> Tuple[List[str], List[Type], bool]: + self.fail(message, context) + return [], [], False + + def build_namedtuple_typeinfo(self, name: str, items: List[str], + types: List[Type]) -> TypeInfo: + symbols = SymbolTable() + tup = NamedTupleType(name, items, types, self.named_type('__builtins__.tuple', types)) + class_def = ClassDef(name, Block([])) + class_def.fullname = self.qualified_name(name) + info = NamedTupleTypeInfo(tup, symbols, class_def) + info.attrs = items + vars = [Var(item, typ) for item, typ in zip(items, types)] + # Add named tuple items as attributes. + # TODO: Make them read-only. + for var in vars: + var.info = info + symbols[var.name()] = SymbolTableNode(MDEF, var) + this_type = self_type(info) + # Add __init__, _replace, _asdict methods and _fields static field + self.add_namedtuple_method('_replace', symbols, info, this_type, this_type, + self.make_factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) + self.add_namedtuple_method('__init__', symbols, info, this_type, NoneTyp(), + self.make_factory_args(vars, ARG_POS), info.name()) + # TODO: refine to OrderedDict[Union[types]] + self.add_namedtuple_method('_asdict', symbols, info, this_type, AnyType(), []) + return info + + def make_factory_args(self, vars: List[Var], kind: int, + initializer: Expression = None) -> List[Argument]: + return [Argument(var, var.type, initializer, kind) for var in vars] + + def add_namedtuple_method(self, funcname: str, symbols: SymbolTable, info: TypeInfo, + this_type: Type, ret: Type, args: List[Argument], name=None) -> None: + args = [Argument(Var('self'), this_type, None, ARG_POS)] + args + types = [arg.type_annotation for arg in args] + items = [arg.variable.name() for arg in args] + arg_kinds = [arg.kind for arg in args] + # TODO: Make sure that the self argument name is not visible? + signature = CallableType(types, arg_kinds, items, ret, + self.named_type('__builtins__.function'), + name=name or info.name() + '.' + funcname) + func = FuncDef(funcname, + args, + Block([]), + typ=signature) + func.info = info + symbols[funcname] = SymbolTableNode(MDEF, func) + + def analyze_types(self, items: List[Node]) -> List[Type]: + result = [] # type: List[Type] + for node in items: + try: + result.append(self.anal_type(expr_to_unanalyzed_type(node))) + except TypeTranslationError: + self.fail('Type expected', node) + result.append(AnyType()) + return result + + def visit_decorator(self, dec: Decorator) -> None: + for d in dec.decorators: + d.accept(self) + removed = [] # type: List[int] + no_type_check = False + for i, d in enumerate(dec.decorators): + # A bunch of decorators are special cased here. + if refers_to_fullname(d, 'abc.abstractmethod'): + removed.append(i) + dec.func.is_abstract = True + self.check_decorated_function_is_method('abstractmethod', dec) + elif refers_to_fullname(d, 'asyncio.tasks.coroutine'): + removed.append(i) + elif refers_to_fullname(d, 'builtins.staticmethod'): + removed.append(i) + dec.func.is_static = True + dec.var.is_staticmethod = True + self.check_decorated_function_is_method('staticmethod', dec) + elif refers_to_fullname(d, 'builtins.classmethod'): + removed.append(i) + dec.func.is_class = True + dec.var.is_classmethod = True + self.check_decorated_function_is_method('classmethod', dec) + elif (refers_to_fullname(d, 'builtins.property') or + refers_to_fullname(d, 'abc.abstractproperty')): + removed.append(i) + dec.func.is_property = True + dec.var.is_property = True + if refers_to_fullname(d, 'abc.abstractproperty'): + dec.func.is_abstract = True + self.check_decorated_function_is_method('property', dec) + if len(dec.func.arguments) > 1: + self.fail('Too many arguments', dec.func) + elif refers_to_fullname(d, 'typing.no_type_check'): + dec.var.type = AnyType() + no_type_check = True + for i in reversed(removed): + del dec.decorators[i] + if not dec.is_overload or dec.var.is_property: + if self.is_func_scope(): + self.add_symbol(dec.var.name(), SymbolTableNode(LDEF, dec), + dec) + elif self.type: + dec.var.info = self.type + dec.var.is_initialized_in_class = True + self.add_symbol(dec.var.name(), SymbolTableNode(MDEF, dec), + dec) + if not no_type_check: + dec.func.accept(self) + if dec.decorators and dec.var.is_property: + self.fail('Decorated property not supported', dec) + + def check_decorated_function_is_method(self, decorator: str, + context: Context) -> None: + if not self.type or self.is_func_scope(): + self.fail("'%s' used with a non-method" % decorator, context) + + def visit_expression_stmt(self, s: ExpressionStmt) -> None: + s.expr.accept(self) + + def visit_return_stmt(self, s: ReturnStmt) -> None: + if not self.is_func_scope(): + self.fail("'return' outside function", s) + if s.expr: + s.expr.accept(self) + + def visit_raise_stmt(self, s: RaiseStmt) -> None: + if s.expr: + s.expr.accept(self) + if s.from_expr: + s.from_expr.accept(self) + + def visit_assert_stmt(self, s: AssertStmt) -> None: + if s.expr: + s.expr.accept(self) + + def visit_operator_assignment_stmt(self, + s: OperatorAssignmentStmt) -> None: + s.lvalue.accept(self) + s.rvalue.accept(self) + if (isinstance(s.lvalue, NameExpr) and s.lvalue.name == '__all__' and + s.lvalue.kind == GDEF and isinstance(s.rvalue, (ListExpr, TupleExpr))): + self.add_exports(*s.rvalue.items) + + def visit_while_stmt(self, s: WhileStmt) -> None: + s.expr.accept(self) + self.loop_depth += 1 + s.body.accept(self) + self.loop_depth -= 1 + self.visit_block_maybe(s.else_body) + + def visit_for_stmt(self, s: ForStmt) -> None: + s.expr.accept(self) + + # Bind index variables and check if they define new names. + self.analyze_lvalue(s.index) + + self.loop_depth += 1 + self.visit_block(s.body) + self.loop_depth -= 1 + + self.visit_block_maybe(s.else_body) + + def visit_break_stmt(self, s: BreakStmt) -> None: + if self.loop_depth == 0: + self.fail("'break' outside loop", s, True, blocker=True) + + def visit_continue_stmt(self, s: ContinueStmt) -> None: + if self.loop_depth == 0: + self.fail("'continue' outside loop", s, True, blocker=True) + + def visit_if_stmt(self, s: IfStmt) -> None: + infer_reachability_of_if_statement(s, pyversion=self.options.python_version) + for i in range(len(s.expr)): + s.expr[i].accept(self) + self.visit_block(s.body[i]) + self.visit_block_maybe(s.else_body) + + def visit_try_stmt(self, s: TryStmt) -> None: + self.analyze_try_stmt(s, self) + + def analyze_try_stmt(self, s: TryStmt, visitor: NodeVisitor, + add_global: bool = False) -> None: + s.body.accept(visitor) + for type, var, handler in zip(s.types, s.vars, s.handlers): + if type: + type.accept(visitor) + if var: + self.analyze_lvalue(var, add_global=add_global) + handler.accept(visitor) + if s.else_body: + s.else_body.accept(visitor) + if s.finally_body: + s.finally_body.accept(visitor) + + def visit_with_stmt(self, s: WithStmt) -> None: + for e, n in zip(s.expr, s.target): + e.accept(self) + if n: + self.analyze_lvalue(n) + self.visit_block(s.body) + + def visit_del_stmt(self, s: DelStmt) -> None: + s.expr.accept(self) + if not self.is_valid_del_target(s.expr): + self.fail('Invalid delete target', s) + + def is_valid_del_target(self, s: Node) -> bool: + if isinstance(s, (IndexExpr, NameExpr, MemberExpr)): + return True + elif isinstance(s, TupleExpr): + return all(self.is_valid_del_target(item) for item in s.items) + + def visit_global_decl(self, g: GlobalDecl) -> None: + for name in g.names: + if name in self.nonlocal_decls[-1]: + self.fail("Name '{}' is nonlocal and global".format(name), g) + self.global_decls[-1].add(name) + + def visit_nonlocal_decl(self, d: NonlocalDecl) -> None: + if not self.is_func_scope(): + self.fail("nonlocal declaration not allowed at module level", d) + else: + for name in d.names: + for table in reversed(self.locals[:-1]): + if table is not None and name in table: + break + else: + self.fail("No binding for nonlocal '{}' found".format(name), d) + + if self.locals[-1] is not None and name in self.locals[-1]: + self.fail("Name '{}' is already defined in local " + "scope before nonlocal declaration".format(name), d) + + if name in self.global_decls[-1]: + self.fail("Name '{}' is nonlocal and global".format(name), d) + self.nonlocal_decls[-1].add(name) + + def visit_print_stmt(self, s: PrintStmt) -> None: + for arg in s.args: + arg.accept(self) + if s.target: + s.target.accept(self) + + def visit_exec_stmt(self, s: ExecStmt) -> None: + s.expr.accept(self) + if s.variables1: + s.variables1.accept(self) + if s.variables2: + s.variables2.accept(self) + + # + # Expressions + # + + def visit_name_expr(self, expr: NameExpr) -> None: + n = self.lookup(expr.name, expr) + if n: + if n.kind == BOUND_TVAR: + self.fail("'{}' is a type variable and only valid in type " + "context".format(expr.name), expr) + else: + expr.kind = n.kind + expr.node = n.node + expr.fullname = n.fullname + + def visit_super_expr(self, expr: SuperExpr) -> None: + if not self.type: + self.fail('"super" used outside class', expr) + return + expr.info = self.type + + def visit_tuple_expr(self, expr: TupleExpr) -> None: + for item in expr.items: + item.accept(self) + + def visit_list_expr(self, expr: ListExpr) -> None: + for item in expr.items: + item.accept(self) + + def visit_set_expr(self, expr: SetExpr) -> None: + for item in expr.items: + item.accept(self) + + def visit_dict_expr(self, expr: DictExpr) -> None: + for key, value in expr.items: + key.accept(self) + value.accept(self) + + def visit_star_expr(self, expr: StarExpr) -> None: + if not expr.valid: + self.fail('Can use starred expression only as assignment target', expr) + else: + expr.expr.accept(self) + + def visit_yield_from_expr(self, e: YieldFromExpr) -> None: + if not self.is_func_scope(): # not sure + self.fail("'yield from' outside function", e, True, blocker=True) + else: + self.function_stack[-1].is_generator = True + if e.expr: + e.expr.accept(self) + + def visit_call_expr(self, expr: CallExpr) -> None: + """Analyze a call expression. + + Some call expressions are recognized as special forms, including + cast(...) and Any(...). + """ + expr.callee.accept(self) + if refers_to_fullname(expr.callee, 'typing.cast'): + # Special form cast(...). + if not self.check_fixed_args(expr, 2, 'cast'): + return + # Translate first argument to an unanalyzed type. + try: + target = expr_to_unanalyzed_type(expr.args[0]) + except TypeTranslationError: + self.fail('Cast target is not a type', expr) + return + # Piggyback CastExpr object to the CallExpr object; it takes + # precedence over the CallExpr semantics. + expr.analyzed = CastExpr(expr.args[1], target) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, 'builtins.reveal_type'): + if not self.check_fixed_args(expr, 1, 'reveal_type'): + return + expr.analyzed = RevealTypeExpr(expr.args[0]) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, 'typing.Any'): + # Special form Any(...). + if not self.check_fixed_args(expr, 1, 'Any'): + return + expr.analyzed = CastExpr(expr.args[0], AnyType()) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, 'typing._promote'): + # Special form _promote(...). + if not self.check_fixed_args(expr, 1, '_promote'): + return + # Translate first argument to an unanalyzed type. + try: + target = expr_to_unanalyzed_type(expr.args[0]) + except TypeTranslationError: + self.fail('Argument 1 to _promote is not a type', expr) + return + expr.analyzed = PromoteExpr(target) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, 'builtins.dict'): + expr.analyzed = self.translate_dict_call(expr) + else: + # Normal call expression. + for a in expr.args: + a.accept(self) + + if (isinstance(expr.callee, MemberExpr) and + isinstance(expr.callee.expr, NameExpr) and + expr.callee.expr.name == '__all__' and + expr.callee.expr.kind == GDEF and + expr.callee.name in ('append', 'extend')): + if expr.callee.name == 'append' and expr.args: + self.add_exports(expr.args[0]) + elif (expr.callee.name == 'extend' and expr.args and + isinstance(expr.args[0], (ListExpr, TupleExpr))): + self.add_exports(*expr.args[0].items) + + def translate_dict_call(self, call: CallExpr) -> Optional[DictExpr]: + """Translate 'dict(x=y, ...)' to {'x': y, ...}. + + For other variants of dict(...), return None. + """ + if not call.args: + return None + if not all(kind == ARG_NAMED for kind in call.arg_kinds): + # Must still accept those args. + for a in call.args: + a.accept(self) + return None + expr = DictExpr([(StrExpr(key), value) + for key, value in zip(call.arg_names, call.args)]) + expr.set_line(call) + expr.accept(self) + return expr + + def check_fixed_args(self, expr: CallExpr, numargs: int, + name: str) -> bool: + """Verify that expr has specified number of positional args. + + Return True if the arguments are valid. + """ + s = 's' + if numargs == 1: + s = '' + if len(expr.args) != numargs: + self.fail("'%s' expects %d argument%s" % (name, numargs, s), + expr) + return False + if expr.arg_kinds != [ARG_POS] * numargs: + self.fail("'%s' must be called with %s positional argument%s" % + (name, numargs, s), expr) + return False + return True + + def visit_member_expr(self, expr: MemberExpr) -> None: + base = expr.expr + base.accept(self) + # Bind references to module attributes. + if isinstance(base, RefExpr) and base.kind == MODULE_REF: + file = cast(MypyFile, base.node) + n = file.names.get(expr.name, None) if file is not None else None + if n: + n = self.normalize_type_alias(n, expr) + if not n: + return + expr.kind = n.kind + expr.fullname = n.fullname + expr.node = n.node + else: + # We only catch some errors here; the rest will be + # catched during type checking. + # + # This way we can report a larger number of errors in + # one type checker run. If we reported errors here, + # the build would terminate after semantic analysis + # and we wouldn't be able to report any type errors. + full_name = '%s.%s' % (file.fullname() if file is not None else None, expr.name) + if full_name in obsolete_name_mapping: + self.fail("Module has no attribute %r (it's now called %r)" % ( + expr.name, obsolete_name_mapping[full_name]), expr) + + def visit_op_expr(self, expr: OpExpr) -> None: + expr.left.accept(self) + expr.right.accept(self) + + def visit_comparison_expr(self, expr: ComparisonExpr) -> None: + for operand in expr.operands: + operand.accept(self) + + def visit_unary_expr(self, expr: UnaryExpr) -> None: + expr.expr.accept(self) + + def visit_index_expr(self, expr: IndexExpr) -> None: + expr.base.accept(self) + if refers_to_class_or_function(expr.base): + # Special form -- type application. + # Translate index to an unanalyzed type. + types = [] # type: List[Type] + if isinstance(expr.index, TupleExpr): + items = expr.index.items + else: + items = [expr.index] + for item in items: + try: + typearg = expr_to_unanalyzed_type(item) + except TypeTranslationError: + self.fail('Type expected within [...]', expr) + return + typearg = self.anal_type(typearg) + types.append(typearg) + expr.analyzed = TypeApplication(expr.base, types) + expr.analyzed.line = expr.line + else: + expr.index.accept(self) + + def visit_slice_expr(self, expr: SliceExpr) -> None: + if expr.begin_index: + expr.begin_index.accept(self) + if expr.end_index: + expr.end_index.accept(self) + if expr.stride: + expr.stride.accept(self) + + def visit_cast_expr(self, expr: CastExpr) -> None: + expr.expr.accept(self) + expr.type = self.anal_type(expr.type) + + def visit_reveal_type_expr(self, expr: RevealTypeExpr) -> None: + expr.expr.accept(self) + + def visit_type_application(self, expr: TypeApplication) -> None: + expr.expr.accept(self) + for i in range(len(expr.types)): + expr.types[i] = self.anal_type(expr.types[i]) + + def visit_list_comprehension(self, expr: ListComprehension) -> None: + expr.generator.accept(self) + + def visit_set_comprehension(self, expr: SetComprehension) -> None: + expr.generator.accept(self) + + def visit_dictionary_comprehension(self, expr: DictionaryComprehension) -> None: + self.enter() + self.analyze_comp_for(expr) + expr.key.accept(self) + expr.value.accept(self) + self.leave() + self.analyze_comp_for_2(expr) + + def visit_generator_expr(self, expr: GeneratorExpr) -> None: + self.enter() + self.analyze_comp_for(expr) + expr.left_expr.accept(self) + self.leave() + self.analyze_comp_for_2(expr) + + def analyze_comp_for(self, expr: Union[GeneratorExpr, + DictionaryComprehension]) -> None: + """Analyses the 'comp_for' part of comprehensions (part 1). + + That is the part after 'for' in (x for x in l if p). This analyzes + variables and conditions which are analyzed in a local scope. + """ + for i, (index, sequence, conditions) in enumerate(zip(expr.indices, + expr.sequences, + expr.condlists)): + if i > 0: + sequence.accept(self) + # Bind index variables. + self.analyze_lvalue(index) + for cond in conditions: + cond.accept(self) + + def analyze_comp_for_2(self, expr: Union[GeneratorExpr, + DictionaryComprehension]) -> None: + """Analyses the 'comp_for' part of comprehensions (part 2). + + That is the part after 'for' in (x for x in l if p). This analyzes + the 'l' part which is analyzed in the surrounding scope. + """ + expr.sequences[0].accept(self) + + def visit_func_expr(self, expr: FuncExpr) -> None: + self.analyze_function(expr) + + def visit_conditional_expr(self, expr: ConditionalExpr) -> None: + expr.if_expr.accept(self) + expr.cond.accept(self) + expr.else_expr.accept(self) + + def visit_backquote_expr(self, expr: BackquoteExpr) -> None: + expr.expr.accept(self) + + def visit__promote_expr(self, expr: PromoteExpr) -> None: + expr.type = self.anal_type(expr.type) + + def visit_yield_expr(self, expr: YieldExpr) -> None: + if not self.is_func_scope(): + self.fail("'yield' outside function", expr, True, blocker=True) + else: + self.function_stack[-1].is_generator = True + if expr.expr: + expr.expr.accept(self) + + # + # Helpers + # + + def lookup(self, name: str, ctx: Context) -> SymbolTableNode: + """Look up an unqualified name in all active namespaces.""" + # 1a. Name declared using 'global x' takes precedence + if name in self.global_decls[-1]: + if name in self.globals: + return self.globals[name] + else: + self.name_not_defined(name, ctx) + return None + # 1b. Name declared using 'nonlocal x' takes precedence + if name in self.nonlocal_decls[-1]: + for table in reversed(self.locals[:-1]): + if table is not None and name in table: + return table[name] + else: + self.name_not_defined(name, ctx) + return None + # 2. Class attributes (if within class definition) + if self.is_class_scope() and name in self.type.names: + return self.type[name] + # 3. Local (function) scopes + for table in reversed(self.locals): + if table is not None and name in table: + return table[name] + # 4. Current file global scope + if name in self.globals: + return self.globals[name] + # 5. Builtins + b = self.globals.get('__builtins__', None) + if b: + table = cast(MypyFile, b.node).names + if name in table: + if name[0] == "_" and name[1] != "_": + self.name_not_defined(name, ctx) + return None + node = table[name] + return node + # Give up. + self.name_not_defined(name, ctx) + self.check_for_obsolete_short_name(name, ctx) + return None + + def check_for_obsolete_short_name(self, name: str, ctx: Context) -> None: + matches = [obsolete_name + for obsolete_name in obsolete_name_mapping + if obsolete_name.rsplit('.', 1)[-1] == name] + if len(matches) == 1: + self.note("(Did you mean '{}'?)".format(obsolete_name_mapping[matches[0]]), ctx) + + def lookup_qualified(self, name: str, ctx: Context) -> SymbolTableNode: + if '.' not in name: + return self.lookup(name, ctx) + else: + parts = name.split('.') + n = self.lookup(parts[0], ctx) # type: SymbolTableNode + if n: + for i in range(1, len(parts)): + if isinstance(n.node, TypeInfo): + if n.node.mro is None: + # We haven't yet analyzed the class `n.node`. Fall back to direct + # lookup in the names declared directly under it, without its base + # classes. This can happen when we have a forward reference to a + # nested class, and the reference is bound before the outer class + # has been fully semantically analyzed. + # + # A better approach would be to introduce a new analysis pass or + # to move things around between passes, but this unblocks a common + # use case even though this is a little limited in case there is + # inheritance involved. + result = n.node.names.get(parts[i]) + else: + result = n.node.get(parts[i]) + n = result + elif isinstance(n.node, MypyFile): + n = n.node.names.get(parts[i], None) + # TODO: What if node is Var or FuncDef? + if not n: + self.name_not_defined(name, ctx) + break + if n: + n = self.normalize_type_alias(n, ctx) + return n + + def builtin_type(self, fully_qualified_name: str) -> Instance: + node = self.lookup_fully_qualified(fully_qualified_name) + info = cast(TypeInfo, node.node) + return Instance(info, []) + + def lookup_fully_qualified(self, name: str) -> SymbolTableNode: + """Lookup a fully qualified name. + + Assume that the name is defined. This happens in the global namespace -- the local + module namespace is ignored. + """ + assert '.' in name + parts = name.split('.') + n = self.modules[parts[0]] + for i in range(1, len(parts) - 1): + n = cast(MypyFile, n.names[parts[i]].node) + return n.names[parts[-1]] + + def lookup_fully_qualified_or_none(self, name: str) -> SymbolTableNode: + """Lookup a fully qualified name. + + Assume that the name is defined. This happens in the global namespace -- the local + module namespace is ignored. + """ + assert '.' in name + parts = name.split('.') + n = self.modules[parts[0]] + for i in range(1, len(parts) - 1): + next_sym = n.names.get(parts[i]) + if not next_sym: + return None + n = cast(MypyFile, next_sym.node) + return n.names.get(parts[-1]) + + def qualified_name(self, n: str) -> str: + return self.cur_mod_id + '.' + n + + def enter(self) -> None: + self.locals.append(SymbolTable()) + self.global_decls.append(set()) + self.nonlocal_decls.append(set()) + # -1 since entering block will increment this to 0. + self.block_depth.append(-1) + + def leave(self) -> None: + self.locals.pop() + self.global_decls.pop() + self.nonlocal_decls.pop() + self.block_depth.pop() + + def is_func_scope(self) -> bool: + return self.locals[-1] is not None + + def is_class_scope(self) -> bool: + return self.type is not None and not self.is_func_scope() + + def add_symbol(self, name: str, node: SymbolTableNode, + context: Context) -> None: + if self.is_func_scope(): + if name in self.locals[-1]: + # Flag redefinition unless this is a reimport of a module. + if not (node.kind == MODULE_REF and + self.locals[-1][name].node == node.node): + self.name_already_defined(name, context) + self.locals[-1][name] = node + elif self.type: + self.type.names[name] = node + else: + existing = self.globals.get(name) + if existing and (not isinstance(node.node, MypyFile) or + existing.node != node.node) and existing.kind != UNBOUND_IMPORTED: + # Modules can be imported multiple times to support import + # of multiple submodules of a package (e.g. a.x and a.y). + ok = False + # Only report an error if the symbol collision provides a different type. + if existing.type and node.type and is_same_type(existing.type, node.type): + ok = True + if not ok: + self.name_already_defined(name, context) + self.globals[name] = node + + def add_var(self, v: Var, ctx: Context) -> None: + if self.is_func_scope(): + self.add_local(v, ctx) + else: + self.globals[v.name()] = SymbolTableNode(GDEF, v, self.cur_mod_id) + v._fullname = self.qualified_name(v.name()) + + def add_local(self, node: Union[Var, FuncBase], ctx: Context) -> None: + name = node.name() + if name in self.locals[-1]: + self.name_already_defined(name, ctx) + node._fullname = name + self.locals[-1][name] = SymbolTableNode(LDEF, node) + + def add_exports(self, *exps: Node) -> None: + for exp in exps: + if isinstance(exp, StrExpr): + self.all_exports.add(exp.value) + + def check_no_global(self, n: str, ctx: Context, + is_overloaded_func: bool = False) -> None: + if n in self.globals: + prev_is_overloaded = isinstance(self.globals[n], OverloadedFuncDef) + if is_overloaded_func and prev_is_overloaded: + self.fail("Nonconsecutive overload {} found".format(n), ctx) + elif prev_is_overloaded: + self.fail("Definition of '{}' missing 'overload'".format(n), ctx) + else: + self.name_already_defined(n, ctx) + + def name_not_defined(self, name: str, ctx: Context) -> None: + message = "Name '{}' is not defined".format(name) + extra = self.undefined_name_extra_info(name) + if extra: + message += ' {}'.format(extra) + self.fail(message, ctx) + + def name_already_defined(self, name: str, ctx: Context) -> None: + self.fail("Name '{}' already defined".format(name), ctx) + + def fail(self, msg: str, ctx: Context, serious: bool = False, *, + blocker: bool = False) -> None: + if (not serious and + not self.options.check_untyped_defs and + self.function_stack and + self.function_stack[-1].is_dynamic()): + return + self.errors.report(ctx.get_line(), msg, blocker=blocker) + + def note(self, msg: str, ctx: Context) -> None: + if (not self.options.check_untyped_defs and + self.function_stack and + self.function_stack[-1].is_dynamic()): + return + self.errors.report(ctx.get_line(), msg, severity='note') + + def undefined_name_extra_info(self, fullname: str) -> Optional[str]: + if fullname in obsolete_name_mapping: + return "(it's now called '{}')".format(obsolete_name_mapping[fullname]) + else: + return None + + def accept(self, node: Node) -> None: + try: + node.accept(self) + except Exception as err: + report_internal_error(err, self.errors.file, node.line) + + +class FirstPass(NodeVisitor): + """First phase of semantic analysis. + + See docstring of 'analyze()' below for a description of what this does. + """ + + def __init__(self, sem: SemanticAnalyzer) -> None: + self.sem = sem + self.pyversion = sem.options.python_version + + def analyze(self, file: MypyFile, fnam: str, mod_id: str) -> None: + """Perform the first analysis pass. + + Populate module global table. Resolve the full names of + definitions not nested within functions and construct type + info structures, but do not resolve inter-definition + references such as base classes. + + Also add implicit definitions such as __name__. + + In this phase we don't resolve imports. For 'from ... import', + we generate dummy symbol table nodes for the imported names, + and these will get resolved in later phases of semantic + analysis. + """ + sem = self.sem + sem.cur_mod_id = mod_id + sem.errors.set_file(fnam) + sem.globals = SymbolTable() + sem.global_decls = [set()] + sem.nonlocal_decls = [set()] + sem.block_depth = [0] + + defs = file.defs + + # Add implicit definitions of module '__name__' etc. + for name, t in implicit_module_attrs.items(): + v = Var(name, UnboundType(t)) + v._fullname = self.sem.qualified_name(name) + self.sem.globals[name] = SymbolTableNode(GDEF, v, self.sem.cur_mod_id) + + for d in defs: + d.accept(self) + + # Add implicit definition of literals/keywords to builtins, as we + # cannot define a variable with them explicitly. + if mod_id == 'builtins': + literal_types = [ + ('None', NoneTyp()), + # reveal_type is a mypy-only function that gives an error with the type of its arg + ('reveal_type', AnyType()), + ] # type: List[Tuple[str, Type]] + + # TODO(ddfisher): This guard is only needed because mypy defines + # fake builtins for its tests which often don't define bool. If + # mypy is fast enough that we no longer need those, this + # conditional check should be removed. + if 'bool' in self.sem.globals: + bool_type = self.sem.named_type('bool') + literal_types.extend([ + ('True', bool_type), + ('False', bool_type), + ('__debug__', bool_type), + ]) + + for name, typ in literal_types: + v = Var(name, typ) + v._fullname = self.sem.qualified_name(name) + self.sem.globals[name] = SymbolTableNode(GDEF, v, self.sem.cur_mod_id) + + def visit_block(self, b: Block) -> None: + if b.is_unreachable: + return + self.sem.block_depth[-1] += 1 + for node in b.body: + node.accept(self) + self.sem.block_depth[-1] -= 1 + + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + for lval in s.lvalues: + self.analyze_lvalue(lval, explicit_type=s.type is not None) + + def visit_func_def(self, func: FuncDef) -> None: + sem = self.sem + func.is_conditional = sem.block_depth[-1] > 0 + func._fullname = sem.qualified_name(func.name()) + if func.name() in sem.globals: + # Already defined in this module. + original_sym = sem.globals[func.name()] + if original_sym.kind == UNBOUND_IMPORTED: + # Ah this is an imported name. We can't resolve them now, so we'll postpone + # this until the main phase of semantic analysis. + return + original_def = original_sym.node + if sem.is_conditional_func(original_def, func): + # Conditional function definition -- multiple defs are ok. + func.original_def = cast(FuncDef, original_def) + else: + # Report error. + sem.check_no_global(func.name(), func) + else: + sem.globals[func.name()] = SymbolTableNode(GDEF, func, sem.cur_mod_id) + + def visit_overloaded_func_def(self, func: OverloadedFuncDef) -> None: + self.sem.check_no_global(func.name(), func, True) + func._fullname = self.sem.qualified_name(func.name()) + self.sem.globals[func.name()] = SymbolTableNode(GDEF, func, + self.sem.cur_mod_id) + + def visit_class_def(self, cdef: ClassDef) -> None: + self.sem.check_no_global(cdef.name, cdef) + cdef.fullname = self.sem.qualified_name(cdef.name) + info = TypeInfo(SymbolTable(), cdef) + info.set_line(cdef.line) + cdef.info = info + self.sem.globals[cdef.name] = SymbolTableNode(GDEF, info, + self.sem.cur_mod_id) + self.process_nested_classes(cdef) + + def process_nested_classes(self, outer_def: ClassDef) -> None: + for node in outer_def.defs.body: + if isinstance(node, ClassDef): + node.info = TypeInfo(SymbolTable(), node) + if outer_def.fullname: + node.info._fullname = outer_def.fullname + '.' + node.info.name() + else: + node.info._fullname = node.info.name() + symbol = SymbolTableNode(MDEF, node.info) + outer_def.info.names[node.name] = symbol + self.process_nested_classes(node) + + def visit_import_from(self, node: ImportFrom) -> None: + # We can't bind module names during the first pass, as the target module might be + # unprocessed. However, we add dummy unbound imported names to the symbol table so + # that we at least know that the name refers to a module. + node.is_top_level = True + for name, as_name in node.names: + imported_name = as_name or name + if imported_name not in self.sem.globals: + self.sem.add_symbol(imported_name, SymbolTableNode(UNBOUND_IMPORTED, None), node) + + def visit_import(self, node: Import) -> None: + node.is_top_level = True + # This is similar to visit_import_from -- see the comment there. + for id, as_id in node.ids: + imported_id = as_id or id + if imported_id not in self.sem.globals: + self.sem.add_symbol(imported_id, SymbolTableNode(UNBOUND_IMPORTED, None), node) + else: + # If the previous symbol is a variable, this should take precedence. + self.sem.globals[imported_id] = SymbolTableNode(UNBOUND_IMPORTED, None) + + def visit_import_all(self, node: ImportAll) -> None: + node.is_top_level = True + + def visit_while_stmt(self, s: WhileStmt) -> None: + s.body.accept(self) + if s.else_body: + s.else_body.accept(self) + + def visit_for_stmt(self, s: ForStmt) -> None: + self.analyze_lvalue(s.index) + s.body.accept(self) + if s.else_body: + s.else_body.accept(self) + + def visit_with_stmt(self, s: WithStmt) -> None: + for n in s.target: + if n: + self.analyze_lvalue(n) + s.body.accept(self) + + def visit_decorator(self, d: Decorator) -> None: + d.var._fullname = self.sem.qualified_name(d.var.name()) + self.sem.add_symbol(d.var.name(), SymbolTableNode(GDEF, d.var), d) + + def visit_if_stmt(self, s: IfStmt) -> None: + infer_reachability_of_if_statement(s, pyversion=self.pyversion) + for node in s.body: + node.accept(self) + if s.else_body: + s.else_body.accept(self) + + def visit_try_stmt(self, s: TryStmt) -> None: + self.sem.analyze_try_stmt(s, self, add_global=True) + + def analyze_lvalue(self, lvalue: Node, explicit_type: bool = False) -> None: + self.sem.analyze_lvalue(lvalue, add_global=True, explicit_type=explicit_type) + + +class ThirdPass(TraverserVisitor): + """The third and final pass of semantic analysis. + + Check type argument counts and values of generic types, and perform some + straightforward type inference. + """ + + def __init__(self, modules: Dict[str, MypyFile], errors: Errors) -> None: + self.modules = modules + self.errors = errors + + def visit_file(self, file_node: MypyFile, fnam: str) -> None: + self.errors.set_file(fnam) + self.accept(file_node) + + def accept(self, node: Node) -> None: + try: + node.accept(self) + except Exception as err: + report_internal_error(err, self.errors.file, node.line) + + def visit_block(self, b: Block) -> None: + if b.is_unreachable: + return + super().visit_block(b) + + def visit_func_def(self, fdef: FuncDef) -> None: + self.errors.push_function(fdef.name()) + self.analyze(fdef.type) + super().visit_func_def(fdef) + self.errors.pop_function() + + def visit_class_def(self, tdef: ClassDef) -> None: + for type in tdef.info.bases: + self.analyze(type) + # Recompute MRO now that we have analyzed all modules, to pick + # up superclasses of bases imported from other modules in an + # import loop. (Only do so if we succeeded the first time.) + if tdef.info.mro: + tdef.info.mro = [] # Force recomputation + calculate_class_mro(tdef, self.fail) + super().visit_class_def(tdef) + + def visit_decorator(self, dec: Decorator) -> None: + """Try to infer the type of the decorated function. + + This lets us resolve references to decorated functions during + type checking when there are cyclic imports, as otherwise the + type might not be available when we need it. + + This basically uses a simple special-purpose type inference + engine just for decorators. + """ + super().visit_decorator(dec) + if dec.var.is_property: + # Decorators are expected to have a callable type (it's a little odd). + if dec.func.type is None: + dec.var.type = CallableType( + [AnyType()], + [ARG_POS], + [None], + AnyType(), + self.builtin_type('function'), + name=dec.var.name()) + elif isinstance(dec.func.type, CallableType): + dec.var.type = dec.func.type + return + decorator_preserves_type = True + for expr in dec.decorators: + preserve_type = False + if isinstance(expr, RefExpr) and isinstance(expr.node, FuncDef): + if is_identity_signature(expr.node.type): + preserve_type = True + if not preserve_type: + decorator_preserves_type = False + break + if decorator_preserves_type: + # No non-identity decorators left. We can trivially infer the type + # of the function here. + dec.var.type = function_type(dec.func, self.builtin_type('function')) + if dec.decorators: + if returns_any_if_called(dec.decorators[0]): + # The outermost decorator will return Any so we know the type of the + # decorated function. + dec.var.type = AnyType() + sig = find_fixed_callable_return(dec.decorators[0]) + if sig: + # The outermost decorator always returns the same kind of function, + # so we know that this is the type of the decoratored function. + orig_sig = function_type(dec.func, self.builtin_type('function')) + sig.name = orig_sig.items()[0].name + dec.var.type = sig + + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + self.analyze(s.type) + super().visit_assignment_stmt(s) + + def visit_cast_expr(self, e: CastExpr) -> None: + self.analyze(e.type) + super().visit_cast_expr(e) + + def visit_reveal_type_expr(self, e: RevealTypeExpr) -> None: + super().visit_reveal_type_expr(e) + + def visit_type_application(self, e: TypeApplication) -> None: + for type in e.types: + self.analyze(type) + super().visit_type_application(e) + + # Helpers + + def analyze(self, type: Type) -> None: + if type: + analyzer = TypeAnalyserPass3(self.fail) + type.accept(analyzer) + + def fail(self, msg: str, ctx: Context) -> None: + self.errors.report(ctx.get_line(), msg) + + def builtin_type(self, name: str, args: List[Type] = None) -> Instance: + names = self.modules['builtins'] + sym = names.names[name] + assert isinstance(sym.node, TypeInfo) + return Instance(sym.node, args or []) + + +def self_type(typ: TypeInfo) -> Union[Instance, TupleType]: + """For a non-generic type, return instance type representing the type. + For a generic G type with parameters T1, .., Tn, return G[T1, ..., Tn]. + """ + tv = [] # type: List[Type] + for i in range(len(typ.type_vars)): + tv.append(TypeVarType(typ.defn.type_vars[i])) + inst = Instance(typ, tv) + if typ.tuple_type is None: + return inst + return typ.tuple_type.self_type(inst) + + +def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: + if isinstance(sig, CallableType): + return replace_leading_arg_type(sig, new) + else: + sig = cast(Overloaded, sig) + return Overloaded([cast(CallableType, replace_implicit_first_type(i, new)) + for i in sig.items()]) + + +def set_callable_name(sig: Type, fdef: FuncDef) -> Type: + if isinstance(sig, FunctionLike): + if fdef.info: + return sig.with_name( + '"{}" of "{}"'.format(fdef.name(), fdef.info.name())) + else: + return sig.with_name('"{}"'.format(fdef.name())) + else: + return sig + + +def refers_to_fullname(node: Node, fullname: str) -> bool: + """Is node a name or member expression with the given full name?""" + return isinstance(node, RefExpr) and node.fullname == fullname + + +def refers_to_class_or_function(node: Node) -> bool: + """Does semantically analyzed node refer to a class?""" + return (isinstance(node, RefExpr) and + isinstance(node.node, (TypeInfo, FuncDef, OverloadedFuncDef))) + + +def calculate_class_mro(defn: ClassDef, fail: Callable[[str, Context], None]) -> None: + try: + defn.info.calculate_mro() + except MroError: + fail("Cannot determine consistent method resolution order " + '(MRO) for "%s"' % defn.name, defn) + defn.info.mro = [] + # The property of falling back to Any is inherited. + defn.info.fallback_to_any = any(baseinfo.fallback_to_any for baseinfo in defn.info.mro) + + +def find_duplicate(list: List[T]) -> T: + """If the list has duplicates, return one of the duplicates. + + Otherwise, return None. + """ + for i in range(1, len(list)): + if list[i] in list[:i]: + return list[i] + return None + + +def disable_typevars(nodes: List[SymbolTableNode]) -> None: + for node in nodes: + assert node.kind in (BOUND_TVAR, UNBOUND_TVAR) + node.kind = UNBOUND_TVAR + + +def enable_typevars(nodes: List[SymbolTableNode]) -> None: + for node in nodes: + assert node.kind in (BOUND_TVAR, UNBOUND_TVAR) + node.kind = BOUND_TVAR + + +def remove_imported_names_from_symtable(names: SymbolTable, + module: str) -> None: + """Remove all imported names from the symbol table of a module.""" + removed = [] # type: List[str] + for name, node in names.items(): + if node.node is None: + continue + fullname = node.node.fullname() + prefix = fullname[:fullname.rfind('.')] + if prefix != module: + removed.append(name) + for name in removed: + del names[name] + + +def infer_reachability_of_if_statement(s: IfStmt, + pyversion: Tuple[int, int]) -> None: + for i in range(len(s.expr)): + result = infer_if_condition_value(s.expr[i], pyversion) + if result == ALWAYS_FALSE: + # The condition is always false, so we skip the if/elif body. + mark_block_unreachable(s.body[i]) + elif result == ALWAYS_TRUE: + # This condition is always true, so all of the remaining + # elif/else bodies will never be executed. + for body in s.body[i + 1:]: + mark_block_unreachable(body) + if s.else_body: + mark_block_unreachable(s.else_body) + break + + +def infer_if_condition_value(expr: Node, pyversion: Tuple[int, int]) -> int: + """Infer whether if condition is always true/false. + + Return ALWAYS_TRUE if always true, ALWAYS_FALSE if always false, + and TRUTH_VALUE_UNKNOWN otherwise. + """ + name = '' + negated = False + alias = expr + if isinstance(alias, UnaryExpr): + if alias.op == 'not': + expr = alias.expr + negated = True + if isinstance(expr, NameExpr): + name = expr.name + elif isinstance(expr, MemberExpr): + name = expr.name + result = TRUTH_VALUE_UNKNOWN + if name == 'PY2': + result = ALWAYS_TRUE if pyversion[0] == 2 else ALWAYS_FALSE + elif name == 'PY3': + result = ALWAYS_TRUE if pyversion[0] == 3 else ALWAYS_FALSE + elif name == 'MYPY': + result = ALWAYS_TRUE + if negated: + if result == ALWAYS_TRUE: + result = ALWAYS_FALSE + elif result == ALWAYS_FALSE: + result = ALWAYS_TRUE + return result + + +def mark_block_unreachable(block: Block) -> None: + block.is_unreachable = True + block.accept(MarkImportsUnreachableVisitor()) + + +class MarkImportsUnreachableVisitor(TraverserVisitor): + """Visitor that flags all imports nested within a node as unreachable.""" + + def visit_import(self, node: Import) -> None: + node.is_unreachable = True + + def visit_import_from(self, node: ImportFrom) -> None: + node.is_unreachable = True + + def visit_import_all(self, node: ImportAll) -> None: + node.is_unreachable = True + + +def is_identity_signature(sig: Type) -> bool: + """Is type a callable of form T -> T (where T is a type variable)?""" + if isinstance(sig, CallableType) and sig.arg_kinds == [ARG_POS]: + if isinstance(sig.arg_types[0], TypeVarType) and isinstance(sig.ret_type, TypeVarType): + return sig.arg_types[0].id == sig.ret_type.id + return False + + +def returns_any_if_called(expr: Node) -> bool: + """Return True if we can predict that expr will return Any if called. + + This only uses information available during semantic analysis so this + will sometimes return False because of insufficient information (as + type inference hasn't run yet). + """ + if isinstance(expr, RefExpr): + if isinstance(expr.node, FuncDef): + typ = expr.node.type + if typ is None: + # No signature -> default to Any. + return True + # Explicit Any return? + return isinstance(typ, CallableType) and isinstance(typ.ret_type, AnyType) + elif isinstance(expr.node, Var): + typ = expr.node.type + return typ is None or isinstance(typ, AnyType) + elif isinstance(expr, CallExpr): + return returns_any_if_called(expr.callee) + return False + + +def find_fixed_callable_return(expr: Node) -> Optional[CallableType]: + if isinstance(expr, RefExpr): + if isinstance(expr.node, FuncDef): + typ = expr.node.type + if typ: + if isinstance(typ, CallableType) and has_no_typevars(typ.ret_type): + if isinstance(typ.ret_type, CallableType): + return typ.ret_type + elif isinstance(expr, CallExpr): + t = find_fixed_callable_return(expr.callee) + if t: + if isinstance(t.ret_type, CallableType): + return t.ret_type + return None + + +def has_no_typevars(typ: Type) -> bool: + return is_same_type(typ, erase_typevars(typ)) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 6872e297c3a64..42405695d5aed 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -1,409 +1,409 @@ -from typing import cast, List, Dict, Callable - -from mypy.types import ( - Type, AnyType, UnboundType, TypeVisitor, ErrorType, Void, NoneTyp, - Instance, TypeVarType, CallableType, TupleType, NamedTupleType, UnionType, Overloaded, ErasedType, TypeList, - PartialType, DeletedType, UninhabitedType, TypeType, is_named_instance -) -import mypy.applytype -import mypy.constraints -# Circular import; done in the function instead. -# import mypy.solve -from mypy import messages, sametypes -from mypy.nodes import CONTRAVARIANT, COVARIANT -from mypy.maptype import map_instance_to_supertype - -from mypy import experiments - - -TypeParameterChecker = Callable[[Type, Type, int], bool] - - -def check_type_parameter(lefta: Type, righta: Type, variance: int) -> bool: - if variance == COVARIANT: - return is_subtype(lefta, righta, check_type_parameter) - elif variance == CONTRAVARIANT: - return is_subtype(righta, lefta, check_type_parameter) - else: - return is_equivalent(lefta, righta, check_type_parameter) - - -def is_subtype(left: Type, right: Type, - type_parameter_checker: TypeParameterChecker = check_type_parameter) -> bool: - """Is 'left' subtype of 'right'? - - Also consider Any to be a subtype of any type, and vice versa. This - recursively applies to components of composite types (List[int] is subtype - of List[Any], for example). - - type_parameter_checker is used to check the type parameters (for example, - A with B in is_subtype(C[A], C[B]). The default checks for subtype relation - between the type arguments (e.g., A and B), taking the variance of the - type var into account. - """ - if (isinstance(right, AnyType) or isinstance(right, UnboundType) - or isinstance(right, ErasedType)): - return True - elif isinstance(right, UnionType) and not isinstance(left, UnionType): - return any(is_subtype(left, item, type_parameter_checker) - for item in right.items) - else: - return left.accept(SubtypeVisitor(right, type_parameter_checker)) - - -def is_subtype_ignoring_tvars(left: Type, right: Type) -> bool: - def ignore_tvars(s: Type, t: Type, v: int) -> bool: - return True - return is_subtype(left, right, ignore_tvars) - - -def is_equivalent(a: Type, b: Type, - type_parameter_checker=check_type_parameter) -> bool: - return is_subtype(a, b, type_parameter_checker) and is_subtype(b, a, type_parameter_checker) - - -def satisfies_upper_bound(a: Type, upper_bound: Type) -> bool: - """Is 'a' valid value for a type variable with the given 'upper_bound'? - - Same as is_subtype except that Void is considered to be a subtype of - any upper_bound. This is needed in a case like - - def f(g: Callable[[], T]) -> T: ... - def h() -> None: ... - f(h) - """ - return isinstance(a, Void) or is_subtype(a, upper_bound) - - -class SubtypeVisitor(TypeVisitor[bool]): - - def __init__(self, right: Type, - type_parameter_checker: TypeParameterChecker) -> None: - self.right = right - self.check_type_parameter = type_parameter_checker - - # visit_x(left) means: is left (which is an instance of X) a subtype of - # right? - - def visit_unbound_type(self, left: UnboundType) -> bool: - return True - - def visit_error_type(self, left: ErrorType) -> bool: - return False - - def visit_type_list(self, t: TypeList) -> bool: - assert False, 'Not supported' - - def visit_any(self, left: AnyType) -> bool: - return True - - def visit_void(self, left: Void) -> bool: - return isinstance(self.right, Void) - - def visit_none_type(self, left: NoneTyp) -> bool: - if experiments.STRICT_OPTIONAL: - return (isinstance(self.right, NoneTyp) or - is_named_instance(self.right, 'builtins.object')) - else: - return not isinstance(self.right, Void) - - def visit_uninhabited_type(self, left: UninhabitedType) -> bool: - return not isinstance(self.right, Void) - - def visit_erased_type(self, left: ErasedType) -> bool: - return True - - def visit_deleted_type(self, left: DeletedType) -> bool: - return True - - def visit_instance(self, left: Instance) -> bool: - if left.type.fallback_to_any: - return True - right = self.right - if isinstance(right, Instance): - if left.type._promote and is_subtype(left.type._promote, - self.right, - self.check_type_parameter): - return True - rname = right.type.fullname() - if not left.type.has_base(rname) and rname != 'builtins.object': - return False - - # Map left type to corresponding right instances. - t = map_instance_to_supertype(left, right.type) - - return all(self.check_type_parameter(lefta, righta, tvar.variance) - for lefta, righta, tvar in - zip(t.args, right.args, right.type.defn.type_vars)) - else: - return False - - def visit_type_var(self, left: TypeVarType) -> bool: - right = self.right - if isinstance(right, TypeVarType): - return left.id == right.id - else: - return is_subtype(left.upper_bound, self.right) - - def visit_callable_type(self, left: CallableType) -> bool: - right = self.right - if isinstance(right, CallableType): - return is_callable_subtype(left, right) - elif isinstance(right, Overloaded): - return all(is_subtype(left, item, self.check_type_parameter) - for item in right.items()) - elif isinstance(right, Instance): - return is_subtype(left.fallback, right) - elif isinstance(right, TypeType): - # This is unsound, we don't check the __init__ signature. - return left.is_type_obj() and is_subtype(left.ret_type, right.item) - else: - return False - - def visit_tuple_type(self, left: TupleType) -> bool: - right = self.right - if isinstance(right, Instance): - if is_named_instance(right, 'builtins.object'): - return True - if is_named_instance(right, 'builtins.tuple'): - target_item_type = right.args[0] - return all(is_subtype(item, target_item_type) - for item in left.items) - elif is_named_instance(right, 'typing.Sized'): - return True - elif (is_named_instance(right, 'typing.Iterable') or - is_named_instance(right, 'typing.Container') or - is_named_instance(right, 'typing.Sequence') or - is_named_instance(right, 'typing.Reversible')): - iter_type = right.args[0] - return all(is_subtype(li, iter_type) for li in left.items) - return False - elif isinstance(right, TupleType): - if len(left.items) != len(right.items): - return False - for l, r in zip(left.items, right.items): - if not is_subtype(l, r, self.check_type_parameter): - return False - if not is_subtype(left.fallback, right.fallback, self.check_type_parameter): - return False - return True - else: - return False - - def visit_namedtuple_type(self, left: NamedTupleType) -> bool: - if isinstance(self.right, NamedTupleType): - # left.attrs.issubset(right.attrs) is insufficient - # (a: int, b: str) is not subtype of (b: int, a: str) - if any(l != r for l, r in zip(left.attrs, self.right.attrs)): - return False - return self.visit_tuple_type(left) - - def visit_overloaded(self, left: Overloaded) -> bool: - right = self.right - if isinstance(right, Instance): - return is_subtype(left.fallback, right) - elif isinstance(right, CallableType) or is_named_instance( - right, 'builtins.type'): - for item in left.items(): - if is_subtype(item, right, self.check_type_parameter): - return True - return False - elif isinstance(right, Overloaded): - # TODO: this may be too restrictive - if len(left.items()) != len(right.items()): - return False - for i in range(len(left.items())): - if not is_subtype(left.items()[i], right.items()[i], self.check_type_parameter): - return False - return True - elif isinstance(right, UnboundType): - return True - elif isinstance(right, TypeType): - # All the items must have the same type object status, so - # it's sufficient to query only (any) one of them. - # This is unsound, we don't check the __init__ signature. - return left.is_type_obj() and is_subtype(left.items()[0].ret_type, right.item) - else: - return False - - def visit_union_type(self, left: UnionType) -> bool: - return all(is_subtype(item, self.right, self.check_type_parameter) - for item in left.items) - - def visit_partial_type(self, left: PartialType) -> bool: - # This is indeterminate as we don't really know the complete type yet. - raise RuntimeError - - def visit_type_type(self, left: TypeType) -> bool: - right = self.right - if isinstance(right, TypeType): - return is_subtype(left.item, right.item) - if isinstance(right, CallableType): - # This is unsound, we don't check the __init__ signature. - return right.is_type_obj() and is_subtype(left.item, right.ret_type) - if (isinstance(right, Instance) and - right.type.fullname() in ('builtins.type', 'builtins.object')): - # Treat builtins.type the same as Type[Any]; - # treat builtins.object the same as Any. - return True - return False - - -def is_callable_subtype(left: CallableType, right: CallableType, - ignore_return: bool = False) -> bool: - """Is left a subtype of right?""" - # TODO: Support named arguments, **args, etc. - # Non-type cannot be a subtype of type. - if right.is_type_obj() and not left.is_type_obj(): - return False - - # A callable L is a subtype of a generic callable R if L is a - # subtype of every type obtained from R by substituting types for - # the variables of R. We can check this by simply leaving the - # generic variables of R as type variables, effectively varying - # over all possible values. - - # It's okay even if these variables share ids with generic - # type variables of L, because generating and solving - # constraints for the variables of L to make L a subtype of R - # (below) treats type variables on the two sides as independent. - - if left.variables: - # Apply generic type variables away in left via type inference. - left = unify_generic_callable(left, right, ignore_return=ignore_return) - if left is None: - return False - - # Check return types. - if not ignore_return and not is_subtype(left.ret_type, right.ret_type): - return False - - if right.is_ellipsis_args: - return True - - # Check argument types. - if left.min_args > right.min_args: - return False - if left.is_var_arg: - return is_var_arg_callable_subtype_helper(left, right) - if right.is_var_arg: - return False - if len(left.arg_types) < len(right.arg_types): - return False - for i in range(len(right.arg_types)): - if not is_subtype(right.arg_types[i], left.arg_types[i]): - return False - return True - - -def is_var_arg_callable_subtype_helper(left: CallableType, right: CallableType) -> bool: - """Is left a subtype of right, assuming left has *args? - - See also is_callable_subtype for additional assumptions we can make. - """ - left_fixed = left.max_fixed_args() - right_fixed = right.max_fixed_args() - num_fixed_matching = min(left_fixed, right_fixed) - for i in range(num_fixed_matching): - if not is_subtype(right.arg_types[i], left.arg_types[i]): - return False - if not right.is_var_arg: - for i in range(num_fixed_matching, len(right.arg_types)): - if not is_subtype(right.arg_types[i], left.arg_types[-1]): - return False - return True - else: - for i in range(left_fixed, right_fixed): - if not is_subtype(right.arg_types[i], left.arg_types[-1]): - return False - for i in range(right_fixed, left_fixed): - if not is_subtype(right.arg_types[-1], left.arg_types[i]): - return False - return is_subtype(right.arg_types[-1], left.arg_types[-1]) - - -def unify_generic_callable(type: CallableType, target: CallableType, - ignore_return: bool) -> CallableType: - """Try to unify a generic callable type with another callable type. - - Return unified CallableType if successful; otherwise, return None. - """ - import mypy.solve - constraints = [] # type: List[mypy.constraints.Constraint] - for arg_type, target_arg_type in zip(type.arg_types, target.arg_types): - c = mypy.constraints.infer_constraints( - arg_type, target_arg_type, mypy.constraints.SUPERTYPE_OF) - constraints.extend(c) - if not ignore_return: - c = mypy.constraints.infer_constraints( - type.ret_type, target.ret_type, mypy.constraints.SUBTYPE_OF) - constraints.extend(c) - type_var_ids = [tvar.id for tvar in type.variables] - inferred_vars = mypy.solve.solve_constraints(type_var_ids, constraints) - if None in inferred_vars: - return None - msg = messages.temp_message_builder() - applied = mypy.applytype.apply_generic_arguments(type, inferred_vars, msg, context=target) - if msg.is_errors() or not isinstance(applied, CallableType): - return None - return cast(CallableType, applied) - - -def restrict_subtype_away(t: Type, s: Type) -> Type: - """Return a supertype of (t intersect not s) - - Currently just remove elements of a union type. - """ - if isinstance(t, UnionType): - new_items = [item for item in t.items if not is_subtype(item, s)] - return UnionType.make_union(new_items) - else: - return t - - -def is_proper_subtype(t: Type, s: Type) -> bool: - """Check if t is a proper subtype of s? - - For proper subtypes, there's no need to rely on compatibility due to - Any types. Any instance type t is also a proper subtype of t. - """ - # FIX tuple types - if isinstance(t, Instance): - if isinstance(s, Instance): - if not t.type.has_base(s.type.fullname()): - return False - - def check_argument(left: Type, right: Type, variance: int) -> bool: - if variance == COVARIANT: - return is_proper_subtype(left, right) - elif variance == CONTRAVARIANT: - return is_proper_subtype(right, left) - else: - return sametypes.is_same_type(left, right) - - # Map left type to corresponding right instances. - t = map_instance_to_supertype(t, s.type) - - return all(check_argument(ta, ra, tvar.variance) for ta, ra, tvar in - zip(t.args, s.args, s.type.defn.type_vars)) - return False - else: - return sametypes.is_same_type(t, s) - - -def is_more_precise(t: Type, s: Type) -> bool: - """Check if t is a more precise type than s. - - A t is a proper subtype of s, t is also more precise than s. Also, if - s is Any, t is more precise than s for any t. Finally, if t is the same - type as s, t is more precise than s. - """ - # TODO Should List[int] be more precise than List[Any]? - if isinstance(s, AnyType): - return True - if isinstance(s, Instance): - if isinstance(t, CallableType): - # Fall back to subclass check and ignore other properties of the callable. - return is_proper_subtype(t.fallback, s) - return is_proper_subtype(t, s) - return sametypes.is_same_type(t, s) +from typing import cast, List, Dict, Callable + +from mypy.types import ( + Type, AnyType, UnboundType, TypeVisitor, ErrorType, Void, NoneTyp, + Instance, TypeVarType, CallableType, TupleType, NamedTupleType, UnionType, Overloaded, ErasedType, TypeList, + PartialType, DeletedType, UninhabitedType, TypeType, is_named_instance +) +import mypy.applytype +import mypy.constraints +# Circular import; done in the function instead. +# import mypy.solve +from mypy import messages, sametypes +from mypy.nodes import CONTRAVARIANT, COVARIANT +from mypy.maptype import map_instance_to_supertype + +from mypy import experiments + + +TypeParameterChecker = Callable[[Type, Type, int], bool] + + +def check_type_parameter(lefta: Type, righta: Type, variance: int) -> bool: + if variance == COVARIANT: + return is_subtype(lefta, righta, check_type_parameter) + elif variance == CONTRAVARIANT: + return is_subtype(righta, lefta, check_type_parameter) + else: + return is_equivalent(lefta, righta, check_type_parameter) + + +def is_subtype(left: Type, right: Type, + type_parameter_checker: TypeParameterChecker = check_type_parameter) -> bool: + """Is 'left' subtype of 'right'? + + Also consider Any to be a subtype of any type, and vice versa. This + recursively applies to components of composite types (List[int] is subtype + of List[Any], for example). + + type_parameter_checker is used to check the type parameters (for example, + A with B in is_subtype(C[A], C[B]). The default checks for subtype relation + between the type arguments (e.g., A and B), taking the variance of the + type var into account. + """ + if (isinstance(right, AnyType) or isinstance(right, UnboundType) + or isinstance(right, ErasedType)): + return True + elif isinstance(right, UnionType) and not isinstance(left, UnionType): + return any(is_subtype(left, item, type_parameter_checker) + for item in right.items) + else: + return left.accept(SubtypeVisitor(right, type_parameter_checker)) + + +def is_subtype_ignoring_tvars(left: Type, right: Type) -> bool: + def ignore_tvars(s: Type, t: Type, v: int) -> bool: + return True + return is_subtype(left, right, ignore_tvars) + + +def is_equivalent(a: Type, b: Type, + type_parameter_checker=check_type_parameter) -> bool: + return is_subtype(a, b, type_parameter_checker) and is_subtype(b, a, type_parameter_checker) + + +def satisfies_upper_bound(a: Type, upper_bound: Type) -> bool: + """Is 'a' valid value for a type variable with the given 'upper_bound'? + + Same as is_subtype except that Void is considered to be a subtype of + any upper_bound. This is needed in a case like + + def f(g: Callable[[], T]) -> T: ... + def h() -> None: ... + f(h) + """ + return isinstance(a, Void) or is_subtype(a, upper_bound) + + +class SubtypeVisitor(TypeVisitor[bool]): + + def __init__(self, right: Type, + type_parameter_checker: TypeParameterChecker) -> None: + self.right = right + self.check_type_parameter = type_parameter_checker + + # visit_x(left) means: is left (which is an instance of X) a subtype of + # right? + + def visit_unbound_type(self, left: UnboundType) -> bool: + return True + + def visit_error_type(self, left: ErrorType) -> bool: + return False + + def visit_type_list(self, t: TypeList) -> bool: + assert False, 'Not supported' + + def visit_any(self, left: AnyType) -> bool: + return True + + def visit_void(self, left: Void) -> bool: + return isinstance(self.right, Void) + + def visit_none_type(self, left: NoneTyp) -> bool: + if experiments.STRICT_OPTIONAL: + return (isinstance(self.right, NoneTyp) or + is_named_instance(self.right, 'builtins.object')) + else: + return not isinstance(self.right, Void) + + def visit_uninhabited_type(self, left: UninhabitedType) -> bool: + return not isinstance(self.right, Void) + + def visit_erased_type(self, left: ErasedType) -> bool: + return True + + def visit_deleted_type(self, left: DeletedType) -> bool: + return True + + def visit_instance(self, left: Instance) -> bool: + if left.type.fallback_to_any: + return True + right = self.right + if isinstance(right, Instance): + if left.type._promote and is_subtype(left.type._promote, + self.right, + self.check_type_parameter): + return True + rname = right.type.fullname() + if not left.type.has_base(rname) and rname != 'builtins.object': + return False + + # Map left type to corresponding right instances. + t = map_instance_to_supertype(left, right.type) + + return all(self.check_type_parameter(lefta, righta, tvar.variance) + for lefta, righta, tvar in + zip(t.args, right.args, right.type.defn.type_vars)) + else: + return False + + def visit_type_var(self, left: TypeVarType) -> bool: + right = self.right + if isinstance(right, TypeVarType): + return left.id == right.id + else: + return is_subtype(left.upper_bound, self.right) + + def visit_callable_type(self, left: CallableType) -> bool: + right = self.right + if isinstance(right, CallableType): + return is_callable_subtype(left, right) + elif isinstance(right, Overloaded): + return all(is_subtype(left, item, self.check_type_parameter) + for item in right.items()) + elif isinstance(right, Instance): + return is_subtype(left.fallback, right) + elif isinstance(right, TypeType): + # This is unsound, we don't check the __init__ signature. + return left.is_type_obj() and is_subtype(left.ret_type, right.item) + else: + return False + + def visit_tuple_type(self, left: TupleType) -> bool: + right = self.right + if isinstance(right, Instance): + if is_named_instance(right, 'builtins.object'): + return True + if is_named_instance(right, 'builtins.tuple'): + target_item_type = right.args[0] + return all(is_subtype(item, target_item_type) + for item in left.items) + elif is_named_instance(right, 'typing.Sized'): + return True + elif (is_named_instance(right, 'typing.Iterable') or + is_named_instance(right, 'typing.Container') or + is_named_instance(right, 'typing.Sequence') or + is_named_instance(right, 'typing.Reversible')): + iter_type = right.args[0] + return all(is_subtype(li, iter_type) for li in left.items) + return False + elif isinstance(right, TupleType): + if len(left.items) != len(right.items): + return False + for l, r in zip(left.items, right.items): + if not is_subtype(l, r, self.check_type_parameter): + return False + if not is_subtype(left.fallback, right.fallback, self.check_type_parameter): + return False + return True + else: + return False + + def visit_namedtuple_type(self, left: NamedTupleType) -> bool: + if isinstance(self.right, NamedTupleType): + # left.attrs.issubset(right.attrs) is insufficient + # (a: int, b: str) is not subtype of (b: int, a: str) + if any(l != r for l, r in zip(left.attrs, self.right.attrs)): + return False + return self.visit_tuple_type(left) + + def visit_overloaded(self, left: Overloaded) -> bool: + right = self.right + if isinstance(right, Instance): + return is_subtype(left.fallback, right) + elif isinstance(right, CallableType) or is_named_instance( + right, 'builtins.type'): + for item in left.items(): + if is_subtype(item, right, self.check_type_parameter): + return True + return False + elif isinstance(right, Overloaded): + # TODO: this may be too restrictive + if len(left.items()) != len(right.items()): + return False + for i in range(len(left.items())): + if not is_subtype(left.items()[i], right.items()[i], self.check_type_parameter): + return False + return True + elif isinstance(right, UnboundType): + return True + elif isinstance(right, TypeType): + # All the items must have the same type object status, so + # it's sufficient to query only (any) one of them. + # This is unsound, we don't check the __init__ signature. + return left.is_type_obj() and is_subtype(left.items()[0].ret_type, right.item) + else: + return False + + def visit_union_type(self, left: UnionType) -> bool: + return all(is_subtype(item, self.right, self.check_type_parameter) + for item in left.items) + + def visit_partial_type(self, left: PartialType) -> bool: + # This is indeterminate as we don't really know the complete type yet. + raise RuntimeError + + def visit_type_type(self, left: TypeType) -> bool: + right = self.right + if isinstance(right, TypeType): + return is_subtype(left.item, right.item) + if isinstance(right, CallableType): + # This is unsound, we don't check the __init__ signature. + return right.is_type_obj() and is_subtype(left.item, right.ret_type) + if (isinstance(right, Instance) and + right.type.fullname() in ('builtins.type', 'builtins.object')): + # Treat builtins.type the same as Type[Any]; + # treat builtins.object the same as Any. + return True + return False + + +def is_callable_subtype(left: CallableType, right: CallableType, + ignore_return: bool = False) -> bool: + """Is left a subtype of right?""" + # TODO: Support named arguments, **args, etc. + # Non-type cannot be a subtype of type. + if right.is_type_obj() and not left.is_type_obj(): + return False + + # A callable L is a subtype of a generic callable R if L is a + # subtype of every type obtained from R by substituting types for + # the variables of R. We can check this by simply leaving the + # generic variables of R as type variables, effectively varying + # over all possible values. + + # It's okay even if these variables share ids with generic + # type variables of L, because generating and solving + # constraints for the variables of L to make L a subtype of R + # (below) treats type variables on the two sides as independent. + + if left.variables: + # Apply generic type variables away in left via type inference. + left = unify_generic_callable(left, right, ignore_return=ignore_return) + if left is None: + return False + + # Check return types. + if not ignore_return and not is_subtype(left.ret_type, right.ret_type): + return False + + if right.is_ellipsis_args: + return True + + # Check argument types. + if left.min_args > right.min_args: + return False + if left.is_var_arg: + return is_var_arg_callable_subtype_helper(left, right) + if right.is_var_arg: + return False + if len(left.arg_types) < len(right.arg_types): + return False + for i in range(len(right.arg_types)): + if not is_subtype(right.arg_types[i], left.arg_types[i]): + return False + return True + + +def is_var_arg_callable_subtype_helper(left: CallableType, right: CallableType) -> bool: + """Is left a subtype of right, assuming left has *args? + + See also is_callable_subtype for additional assumptions we can make. + """ + left_fixed = left.max_fixed_args() + right_fixed = right.max_fixed_args() + num_fixed_matching = min(left_fixed, right_fixed) + for i in range(num_fixed_matching): + if not is_subtype(right.arg_types[i], left.arg_types[i]): + return False + if not right.is_var_arg: + for i in range(num_fixed_matching, len(right.arg_types)): + if not is_subtype(right.arg_types[i], left.arg_types[-1]): + return False + return True + else: + for i in range(left_fixed, right_fixed): + if not is_subtype(right.arg_types[i], left.arg_types[-1]): + return False + for i in range(right_fixed, left_fixed): + if not is_subtype(right.arg_types[-1], left.arg_types[i]): + return False + return is_subtype(right.arg_types[-1], left.arg_types[-1]) + + +def unify_generic_callable(type: CallableType, target: CallableType, + ignore_return: bool) -> CallableType: + """Try to unify a generic callable type with another callable type. + + Return unified CallableType if successful; otherwise, return None. + """ + import mypy.solve + constraints = [] # type: List[mypy.constraints.Constraint] + for arg_type, target_arg_type in zip(type.arg_types, target.arg_types): + c = mypy.constraints.infer_constraints( + arg_type, target_arg_type, mypy.constraints.SUPERTYPE_OF) + constraints.extend(c) + if not ignore_return: + c = mypy.constraints.infer_constraints( + type.ret_type, target.ret_type, mypy.constraints.SUBTYPE_OF) + constraints.extend(c) + type_var_ids = [tvar.id for tvar in type.variables] + inferred_vars = mypy.solve.solve_constraints(type_var_ids, constraints) + if None in inferred_vars: + return None + msg = messages.temp_message_builder() + applied = mypy.applytype.apply_generic_arguments(type, inferred_vars, msg, context=target) + if msg.is_errors() or not isinstance(applied, CallableType): + return None + return cast(CallableType, applied) + + +def restrict_subtype_away(t: Type, s: Type) -> Type: + """Return a supertype of (t intersect not s) + + Currently just remove elements of a union type. + """ + if isinstance(t, UnionType): + new_items = [item for item in t.items if not is_subtype(item, s)] + return UnionType.make_union(new_items) + else: + return t + + +def is_proper_subtype(t: Type, s: Type) -> bool: + """Check if t is a proper subtype of s? + + For proper subtypes, there's no need to rely on compatibility due to + Any types. Any instance type t is also a proper subtype of t. + """ + # FIX tuple types + if isinstance(t, Instance): + if isinstance(s, Instance): + if not t.type.has_base(s.type.fullname()): + return False + + def check_argument(left: Type, right: Type, variance: int) -> bool: + if variance == COVARIANT: + return is_proper_subtype(left, right) + elif variance == CONTRAVARIANT: + return is_proper_subtype(right, left) + else: + return sametypes.is_same_type(left, right) + + # Map left type to corresponding right instances. + t = map_instance_to_supertype(t, s.type) + + return all(check_argument(ta, ra, tvar.variance) for ta, ra, tvar in + zip(t.args, s.args, s.type.defn.type_vars)) + return False + else: + return sametypes.is_same_type(t, s) + + +def is_more_precise(t: Type, s: Type) -> bool: + """Check if t is a more precise type than s. + + A t is a proper subtype of s, t is also more precise than s. Also, if + s is Any, t is more precise than s for any t. Finally, if t is the same + type as s, t is more precise than s. + """ + # TODO Should List[int] be more precise than List[Any]? + if isinstance(s, AnyType): + return True + if isinstance(s, Instance): + if isinstance(t, CallableType): + # Fall back to subclass check and ignore other properties of the callable. + return is_proper_subtype(t.fallback, s) + return is_proper_subtype(t, s) + return sametypes.is_same_type(t, s) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 1841f283a19e1..f10048c4f0fe4 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -1,430 +1,430 @@ -"""Semantic analysis of types""" - -from typing import Callable, cast, List, Tuple - -from mypy.types import ( - Type, UnboundType, TypeVarType, TupleType, NamedTupleType, UnionType, Instance, AnyType, CallableType, - Void, NoneTyp, DeletedType, TypeList, TypeVarDef, TypeVisitor, StarType, PartialType, - EllipsisType, UninhabitedType, TypeType -) -from mypy.nodes import ( - BOUND_TVAR, TYPE_ALIAS, UNBOUND_IMPORTED, - TypeInfo, Context, SymbolTableNode, TypeVarExpr, Var, Node, - IndexExpr, RefExpr -) -from mypy.sametypes import is_same_type -from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError -from mypy.subtypes import satisfies_upper_bound -from mypy import nodes -from mypy import experiments - - -type_constructors = {'typing.Tuple', 'typing.Union', 'typing.Callable', 'typing.Type'} - - -def analyze_type_alias(node: Node, - lookup_func: Callable[[str, Context], SymbolTableNode], - lookup_fqn_func: Callable[[str], SymbolTableNode], - fail_func: Callable[[str, Context], None]) -> Type: - """Return type if node is valid as a type alias rvalue. - - Return None otherwise. 'node' must have been semantically analyzed. - """ - # Quickly return None if the expression doesn't look like a type. Note - # that we don't support straight string literals as type aliases - # (only string literals within index expressions). - if isinstance(node, RefExpr): - if not (isinstance(node.node, TypeInfo) or - node.fullname == 'typing.Any' or - node.kind == TYPE_ALIAS): - return None - elif isinstance(node, IndexExpr): - base = node.base - if isinstance(base, RefExpr): - if not (isinstance(base.node, TypeInfo) or - base.fullname in type_constructors): - return None - else: - return None - else: - return None - - # It's a type alias (though it may be an invalid one). - try: - type = expr_to_unanalyzed_type(node) - except TypeTranslationError: - fail_func('Invalid type alias', node) - return None - analyzer = TypeAnalyser(lookup_func, lookup_fqn_func, fail_func) - return type.accept(analyzer) - - -class TypeAnalyser(TypeVisitor[Type]): - """Semantic analyzer for types (semantic analysis pass 2). - - Converts unbound types into bound types. - """ - - def __init__(self, - lookup_func: Callable[[str, Context], SymbolTableNode], - lookup_fqn_func: Callable[[str], SymbolTableNode], - fail_func: Callable[[str, Context], None]) -> None: - self.lookup = lookup_func - self.lookup_fqn_func = lookup_fqn_func - self.fail = fail_func - - def visit_unbound_type(self, t: UnboundType) -> Type: - if t.optional: - t.optional = False - # We don't need to worry about double-wrapping Optionals or - # wrapping Anys: Union simplification will take care of that. - return UnionType.make_simplified_union([self.visit_unbound_type(t), NoneTyp()]) - sym = self.lookup(t.name, t) - if sym is not None: - if sym.node is None: - # UNBOUND_IMPORTED can happen if an unknown name was imported. - if sym.kind != UNBOUND_IMPORTED: - self.fail('Internal error (node is None, kind={})'.format(sym.kind), t) - return AnyType() - fullname = sym.node.fullname() - if sym.kind == BOUND_TVAR: - if len(t.args) > 0: - self.fail('Type variable "{}" used with arguments'.format( - t.name), t) - assert sym.tvar_def is not None - return TypeVarType(sym.tvar_def, t.line) - elif fullname == 'builtins.None': - if experiments.STRICT_OPTIONAL: - if t.is_ret_type: - return Void() - else: - return NoneTyp() - else: - return Void() - elif fullname == 'typing.Any': - return AnyType() - elif fullname == 'typing.Tuple': - if len(t.args) == 2 and isinstance(t.args[1], EllipsisType): - # Tuple[T, ...] (uniform, variable-length tuple) - node = self.lookup_fqn_func('builtins.tuple') - tuple_info = cast(TypeInfo, node.node) - return Instance(tuple_info, [t.args[0].accept(self)], t.line) - return self.tuple_type(self.anal_array(t.args)) - elif fullname == 'typing.Union': - items = self.anal_array(t.args) - items = [item for item in items if not isinstance(item, Void)] - return UnionType.make_union(items) - elif fullname == 'typing.Optional': - if len(t.args) != 1: - self.fail('Optional[...] must have exactly one type argument', t) - return AnyType() - items = self.anal_array(t.args) - if experiments.STRICT_OPTIONAL: - return UnionType.make_simplified_union([items[0], NoneTyp()]) - else: - # Without strict Optional checking Optional[t] is just an alias for t. - return items[0] - elif fullname == 'typing.Callable': - return self.analyze_callable_type(t) - elif fullname == 'typing.Type': - if len(t.args) == 0: - return TypeType(AnyType(), line=t.line) - if len(t.args) != 1: - self.fail('Type[...] must have exactly one type argument', t) - items = self.anal_array(t.args) - item = items[0] - return TypeType(item, line=t.line) - elif sym.kind == TYPE_ALIAS: - # TODO: Generic type aliases. - return sym.type_override - elif not isinstance(sym.node, TypeInfo): - name = sym.fullname - if name is None: - name = sym.node.name() - if isinstance(sym.node, Var) and isinstance(sym.node.type, AnyType): - # Something with an Any type -- make it an alias for Any in a type - # context. This is slightly problematic as it allows using the type 'Any' - # as a base class -- however, this will fail soon at runtime so the problem - # is pretty minor. - return AnyType() - self.fail('Invalid type "{}"'.format(name), t) - return t - info = sym.node # type: TypeInfo - if len(t.args) > 0 and info.fullname() == 'builtins.tuple': - return TupleType(self.anal_array(t.args), - Instance(info, [AnyType()], t.line), - t.line) - else: - # Analyze arguments and construct Instance type. The - # number of type arguments and their values are - # checked only later, since we do not always know the - # valid count at this point. Thus we may construct an - # Instance with an invalid number of type arguments. - instance = Instance(info, self.anal_array(t.args), t.line) - if info.tuple_type is None: - return instance - else: - # The class has a Tuple[...] base class so it will be - # represented as a tuple type. - if t.args: - self.fail('Generic tuple types not supported', t) - return AnyType() - # import pdb; pdb.set_trace() - if info.is_named_tuple: - return NamedTupleType(info.name(), info.attrs, - self.anal_array(info.tuple_type.items), - fallback=instance, - line=t.line) - return TupleType(self.anal_array(info.tuple_type.items), - fallback=instance, - line=t.line) - else: - return AnyType() - - def visit_any(self, t: AnyType) -> Type: - return t - - def visit_void(self, t: Void) -> Type: - return t - - def visit_none_type(self, t: NoneTyp) -> Type: - return t - - def visit_uninhabited_type(self, t: UninhabitedType) -> Type: - return t - - def visit_deleted_type(self, t: DeletedType) -> Type: - return t - - def visit_type_list(self, t: TypeList) -> Type: - self.fail('Invalid type', t) - return AnyType() - - def visit_instance(self, t: Instance) -> Type: - return t - - def visit_type_var(self, t: TypeVarType) -> Type: - raise RuntimeError('TypeVarType is already analyzed') - - def visit_callable_type(self, t: CallableType) -> Type: - return t.copy_modified(arg_types=self.anal_array(t.arg_types), - ret_type=t.ret_type.accept(self), - fallback=t.fallback or self.builtin_type('builtins.function'), - variables=self.anal_var_defs(t.variables)) - - def visit_tuple_type(self, t: TupleType) -> Type: - if t.implicit: - self.fail('Invalid tuple literal type', t) - return AnyType() - star_count = sum(1 for item in t.items if isinstance(item, StarType)) - if star_count > 1: - self.fail('At most one star type allowed in a tuple', t) - return AnyType() - fallback = t.fallback if t.fallback else self.builtin_type('builtins.tuple', [AnyType()]) - return TupleType(self.anal_array(t.items), fallback, t.line) - - def visit_namedtuple_type(self, t: NamedTupleType) -> Type: - return NamedTupleType(t.name, t.attrs, - self.anal_array(t.items), - fallback=t.fallback, - line=t.line) - - def visit_star_type(self, t: StarType) -> Type: - return StarType(t.type.accept(self), t.line) - - def visit_union_type(self, t: UnionType) -> Type: - return UnionType(self.anal_array(t.items), t.line) - - def visit_partial_type(self, t: PartialType) -> Type: - assert False, "Internal error: Unexpected partial type" - - def visit_ellipsis_type(self, t: EllipsisType) -> Type: - self.fail("Unexpected '...'", t) - return AnyType() - - def visit_type_type(self, t: TypeType) -> Type: - return TypeType(t.item.accept(self), line=t.line) - - def analyze_callable_type(self, t: UnboundType) -> Type: - fallback = self.builtin_type('builtins.function') - if len(t.args) == 0: - # Callable (bare). Treat as Callable[..., Any]. - return CallableType([AnyType(), AnyType()], - [nodes.ARG_STAR, nodes.ARG_STAR2], - [None, None], - ret_type=AnyType(), - fallback=fallback, - is_ellipsis_args=True) - elif len(t.args) == 2: - ret_type = t.args[1].accept(self) - if isinstance(t.args[0], TypeList): - # Callable[[ARG, ...], RET] (ordinary callable type) - args = t.args[0].items - return CallableType(self.anal_array(args), - [nodes.ARG_POS] * len(args), - [None] * len(args), - ret_type=ret_type, - fallback=fallback) - elif isinstance(t.args[0], EllipsisType): - # Callable[..., RET] (with literal ellipsis; accept arbitrary arguments) - return CallableType([AnyType(), AnyType()], - [nodes.ARG_STAR, nodes.ARG_STAR2], - [None, None], - ret_type=ret_type, - fallback=fallback, - is_ellipsis_args=True) - else: - self.fail('The first argument to Callable must be a list of types or "..."', t) - return AnyType() - - self.fail('Invalid function type', t) - return AnyType() - - def anal_array(self, a: List[Type]) -> List[Type]: - res = [] # type: List[Type] - for t in a: - res.append(t.accept(self)) - return res - - def anal_var_defs(self, var_defs: List[TypeVarDef]) -> List[TypeVarDef]: - a = [] # type: List[TypeVarDef] - for vd in var_defs: - a.append(TypeVarDef(vd.name, vd.id.raw_id, self.anal_array(vd.values), - vd.upper_bound.accept(self), - vd.variance, - vd.line)) - return a - - def builtin_type(self, fully_qualified_name: str, args: List[Type] = None) -> Instance: - node = self.lookup_fqn_func(fully_qualified_name) - info = cast(TypeInfo, node.node) - return Instance(info, args or []) - - def tuple_type(self, items: List[Type]) -> TupleType: - return TupleType(items, fallback=self.builtin_type('builtins.tuple', [AnyType()])) - - -class TypeAnalyserPass3(TypeVisitor[None]): - """Analyze type argument counts and values of generic types. - - This is semantic analysis pass 3 for types. - - Perform these operations: - - * Report error for invalid type argument counts, such as List[x, y]. - * Make implicit Any type arguments explicit my modifying types - in-place. For example, modify Foo into Foo[Any] if Foo expects a single - type argument. - * If a type variable has a value restriction, ensure that the value is - valid. For example, reject IO[int] if the type argument must be str - or bytes. - - We can't do this earlier than the third pass, since type argument counts - are only determined in pass 2, and we have to support forward references - to types. - """ - - def __init__(self, fail_func: Callable[[str, Context], None]) -> None: - self.fail = fail_func - - def visit_instance(self, t: Instance) -> None: - info = t.type - # Check type argument count. - if len(t.args) != len(info.type_vars): - if len(t.args) == 0: - # Insert implicit 'Any' type arguments. - t.args = [AnyType()] * len(info.type_vars) - return - # Invalid number of type parameters. - n = len(info.type_vars) - s = '{} type arguments'.format(n) - if n == 0: - s = 'no type arguments' - elif n == 1: - s = '1 type argument' - act = str(len(t.args)) - if act == '0': - act = 'none' - self.fail('"{}" expects {}, but {} given'.format( - info.name(), s, act), t) - # Construct the correct number of type arguments, as - # otherwise the type checker may crash as it expects - # things to be right. - t.args = [AnyType() for _ in info.type_vars] - elif info.defn.type_vars: - # Check type argument values. - for arg, TypeVar in zip(t.args, info.defn.type_vars): - if TypeVar.values: - if isinstance(arg, TypeVarType): - arg_values = arg.values - if not arg_values: - self.fail('Type variable "{}" not valid as type ' - 'argument value for "{}"'.format( - arg.name, info.name()), t) - continue - else: - arg_values = [arg] - self.check_type_var_values(info, arg_values, - TypeVar.values, t) - if not satisfies_upper_bound(arg, TypeVar.upper_bound): - self.fail('Type argument "{}" of "{}" must be ' - 'a subtype of "{}"'.format( - arg, info.name(), TypeVar.upper_bound), t) - for arg in t.args: - arg.accept(self) - - def check_type_var_values(self, type: TypeInfo, actuals: List[Type], - valids: List[Type], context: Context) -> None: - for actual in actuals: - if (not isinstance(actual, AnyType) and - not any(is_same_type(actual, value) for value in valids)): - self.fail('Invalid type argument value for "{}"'.format( - type.name()), context) - - def visit_callable_type(self, t: CallableType) -> None: - t.ret_type.accept(self) - for arg_type in t.arg_types: - arg_type.accept(self) - - def visit_tuple_type(self, t: TupleType) -> None: - for item in t.items: - item.accept(self) - - def visit_union_type(self, t: UnionType) -> None: - for item in t.items: - item.accept(self) - - def visit_star_type(self, t: StarType) -> None: - t.type.accept(self) - - # Other kinds of type are trivial, since they are atomic (or invalid). - - def visit_unbound_type(self, t: UnboundType) -> None: - pass - - def visit_any(self, t: AnyType) -> None: - pass - - def visit_void(self, t: Void) -> None: - pass - - def visit_none_type(self, t: NoneTyp) -> None: - pass - - def visit_uninhabited_type(self, t: UninhabitedType) -> None: - pass - - def visit_deleted_type(self, t: DeletedType) -> None: - pass - - def visit_type_list(self, t: TypeList) -> None: - self.fail('Invalid type', t) - - def visit_type_var(self, t: TypeVarType) -> None: - pass - - def visit_partial_type(self, t: PartialType) -> None: - pass - - def visit_type_type(self, t: TypeType) -> None: - pass +"""Semantic analysis of types""" + +from typing import Callable, cast, List, Tuple + +from mypy.types import ( + Type, UnboundType, TypeVarType, TupleType, NamedTupleType, UnionType, Instance, AnyType, CallableType, + Void, NoneTyp, DeletedType, TypeList, TypeVarDef, TypeVisitor, StarType, PartialType, + EllipsisType, UninhabitedType, TypeType +) +from mypy.nodes import ( + BOUND_TVAR, TYPE_ALIAS, UNBOUND_IMPORTED, + TypeInfo, Context, SymbolTableNode, TypeVarExpr, Var, Node, + IndexExpr, RefExpr +) +from mypy.sametypes import is_same_type +from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError +from mypy.subtypes import satisfies_upper_bound +from mypy import nodes +from mypy import experiments + + +type_constructors = {'typing.Tuple', 'typing.Union', 'typing.Callable', 'typing.Type'} + + +def analyze_type_alias(node: Node, + lookup_func: Callable[[str, Context], SymbolTableNode], + lookup_fqn_func: Callable[[str], SymbolTableNode], + fail_func: Callable[[str, Context], None]) -> Type: + """Return type if node is valid as a type alias rvalue. + + Return None otherwise. 'node' must have been semantically analyzed. + """ + # Quickly return None if the expression doesn't look like a type. Note + # that we don't support straight string literals as type aliases + # (only string literals within index expressions). + if isinstance(node, RefExpr): + if not (isinstance(node.node, TypeInfo) or + node.fullname == 'typing.Any' or + node.kind == TYPE_ALIAS): + return None + elif isinstance(node, IndexExpr): + base = node.base + if isinstance(base, RefExpr): + if not (isinstance(base.node, TypeInfo) or + base.fullname in type_constructors): + return None + else: + return None + else: + return None + + # It's a type alias (though it may be an invalid one). + try: + type = expr_to_unanalyzed_type(node) + except TypeTranslationError: + fail_func('Invalid type alias', node) + return None + analyzer = TypeAnalyser(lookup_func, lookup_fqn_func, fail_func) + return type.accept(analyzer) + + +class TypeAnalyser(TypeVisitor[Type]): + """Semantic analyzer for types (semantic analysis pass 2). + + Converts unbound types into bound types. + """ + + def __init__(self, + lookup_func: Callable[[str, Context], SymbolTableNode], + lookup_fqn_func: Callable[[str], SymbolTableNode], + fail_func: Callable[[str, Context], None]) -> None: + self.lookup = lookup_func + self.lookup_fqn_func = lookup_fqn_func + self.fail = fail_func + + def visit_unbound_type(self, t: UnboundType) -> Type: + if t.optional: + t.optional = False + # We don't need to worry about double-wrapping Optionals or + # wrapping Anys: Union simplification will take care of that. + return UnionType.make_simplified_union([self.visit_unbound_type(t), NoneTyp()]) + sym = self.lookup(t.name, t) + if sym is not None: + if sym.node is None: + # UNBOUND_IMPORTED can happen if an unknown name was imported. + if sym.kind != UNBOUND_IMPORTED: + self.fail('Internal error (node is None, kind={})'.format(sym.kind), t) + return AnyType() + fullname = sym.node.fullname() + if sym.kind == BOUND_TVAR: + if len(t.args) > 0: + self.fail('Type variable "{}" used with arguments'.format( + t.name), t) + assert sym.tvar_def is not None + return TypeVarType(sym.tvar_def, t.line) + elif fullname == 'builtins.None': + if experiments.STRICT_OPTIONAL: + if t.is_ret_type: + return Void() + else: + return NoneTyp() + else: + return Void() + elif fullname == 'typing.Any': + return AnyType() + elif fullname == 'typing.Tuple': + if len(t.args) == 2 and isinstance(t.args[1], EllipsisType): + # Tuple[T, ...] (uniform, variable-length tuple) + node = self.lookup_fqn_func('builtins.tuple') + tuple_info = cast(TypeInfo, node.node) + return Instance(tuple_info, [t.args[0].accept(self)], t.line) + return self.tuple_type(self.anal_array(t.args)) + elif fullname == 'typing.Union': + items = self.anal_array(t.args) + items = [item for item in items if not isinstance(item, Void)] + return UnionType.make_union(items) + elif fullname == 'typing.Optional': + if len(t.args) != 1: + self.fail('Optional[...] must have exactly one type argument', t) + return AnyType() + items = self.anal_array(t.args) + if experiments.STRICT_OPTIONAL: + return UnionType.make_simplified_union([items[0], NoneTyp()]) + else: + # Without strict Optional checking Optional[t] is just an alias for t. + return items[0] + elif fullname == 'typing.Callable': + return self.analyze_callable_type(t) + elif fullname == 'typing.Type': + if len(t.args) == 0: + return TypeType(AnyType(), line=t.line) + if len(t.args) != 1: + self.fail('Type[...] must have exactly one type argument', t) + items = self.anal_array(t.args) + item = items[0] + return TypeType(item, line=t.line) + elif sym.kind == TYPE_ALIAS: + # TODO: Generic type aliases. + return sym.type_override + elif not isinstance(sym.node, TypeInfo): + name = sym.fullname + if name is None: + name = sym.node.name() + if isinstance(sym.node, Var) and isinstance(sym.node.type, AnyType): + # Something with an Any type -- make it an alias for Any in a type + # context. This is slightly problematic as it allows using the type 'Any' + # as a base class -- however, this will fail soon at runtime so the problem + # is pretty minor. + return AnyType() + self.fail('Invalid type "{}"'.format(name), t) + return t + info = sym.node # type: TypeInfo + if len(t.args) > 0 and info.fullname() == 'builtins.tuple': + return TupleType(self.anal_array(t.args), + Instance(info, [AnyType()], t.line), + t.line) + else: + # Analyze arguments and construct Instance type. The + # number of type arguments and their values are + # checked only later, since we do not always know the + # valid count at this point. Thus we may construct an + # Instance with an invalid number of type arguments. + instance = Instance(info, self.anal_array(t.args), t.line) + if info.tuple_type is None: + return instance + else: + # The class has a Tuple[...] base class so it will be + # represented as a tuple type. + if t.args: + self.fail('Generic tuple types not supported', t) + return AnyType() + # import pdb; pdb.set_trace() + if info.is_named_tuple: + return NamedTupleType(info.name(), info.attrs, + self.anal_array(info.tuple_type.items), + fallback=instance, + line=t.line) + return TupleType(self.anal_array(info.tuple_type.items), + fallback=instance, + line=t.line) + else: + return AnyType() + + def visit_any(self, t: AnyType) -> Type: + return t + + def visit_void(self, t: Void) -> Type: + return t + + def visit_none_type(self, t: NoneTyp) -> Type: + return t + + def visit_uninhabited_type(self, t: UninhabitedType) -> Type: + return t + + def visit_deleted_type(self, t: DeletedType) -> Type: + return t + + def visit_type_list(self, t: TypeList) -> Type: + self.fail('Invalid type', t) + return AnyType() + + def visit_instance(self, t: Instance) -> Type: + return t + + def visit_type_var(self, t: TypeVarType) -> Type: + raise RuntimeError('TypeVarType is already analyzed') + + def visit_callable_type(self, t: CallableType) -> Type: + return t.copy_modified(arg_types=self.anal_array(t.arg_types), + ret_type=t.ret_type.accept(self), + fallback=t.fallback or self.builtin_type('builtins.function'), + variables=self.anal_var_defs(t.variables)) + + def visit_tuple_type(self, t: TupleType) -> Type: + if t.implicit: + self.fail('Invalid tuple literal type', t) + return AnyType() + star_count = sum(1 for item in t.items if isinstance(item, StarType)) + if star_count > 1: + self.fail('At most one star type allowed in a tuple', t) + return AnyType() + fallback = t.fallback if t.fallback else self.builtin_type('builtins.tuple', [AnyType()]) + return TupleType(self.anal_array(t.items), fallback, t.line) + + def visit_namedtuple_type(self, t: NamedTupleType) -> Type: + return NamedTupleType(t.name, t.attrs, + self.anal_array(t.items), + fallback=t.fallback, + line=t.line) + + def visit_star_type(self, t: StarType) -> Type: + return StarType(t.type.accept(self), t.line) + + def visit_union_type(self, t: UnionType) -> Type: + return UnionType(self.anal_array(t.items), t.line) + + def visit_partial_type(self, t: PartialType) -> Type: + assert False, "Internal error: Unexpected partial type" + + def visit_ellipsis_type(self, t: EllipsisType) -> Type: + self.fail("Unexpected '...'", t) + return AnyType() + + def visit_type_type(self, t: TypeType) -> Type: + return TypeType(t.item.accept(self), line=t.line) + + def analyze_callable_type(self, t: UnboundType) -> Type: + fallback = self.builtin_type('builtins.function') + if len(t.args) == 0: + # Callable (bare). Treat as Callable[..., Any]. + return CallableType([AnyType(), AnyType()], + [nodes.ARG_STAR, nodes.ARG_STAR2], + [None, None], + ret_type=AnyType(), + fallback=fallback, + is_ellipsis_args=True) + elif len(t.args) == 2: + ret_type = t.args[1].accept(self) + if isinstance(t.args[0], TypeList): + # Callable[[ARG, ...], RET] (ordinary callable type) + args = t.args[0].items + return CallableType(self.anal_array(args), + [nodes.ARG_POS] * len(args), + [None] * len(args), + ret_type=ret_type, + fallback=fallback) + elif isinstance(t.args[0], EllipsisType): + # Callable[..., RET] (with literal ellipsis; accept arbitrary arguments) + return CallableType([AnyType(), AnyType()], + [nodes.ARG_STAR, nodes.ARG_STAR2], + [None, None], + ret_type=ret_type, + fallback=fallback, + is_ellipsis_args=True) + else: + self.fail('The first argument to Callable must be a list of types or "..."', t) + return AnyType() + + self.fail('Invalid function type', t) + return AnyType() + + def anal_array(self, a: List[Type]) -> List[Type]: + res = [] # type: List[Type] + for t in a: + res.append(t.accept(self)) + return res + + def anal_var_defs(self, var_defs: List[TypeVarDef]) -> List[TypeVarDef]: + a = [] # type: List[TypeVarDef] + for vd in var_defs: + a.append(TypeVarDef(vd.name, vd.id.raw_id, self.anal_array(vd.values), + vd.upper_bound.accept(self), + vd.variance, + vd.line)) + return a + + def builtin_type(self, fully_qualified_name: str, args: List[Type] = None) -> Instance: + node = self.lookup_fqn_func(fully_qualified_name) + info = cast(TypeInfo, node.node) + return Instance(info, args or []) + + def tuple_type(self, items: List[Type]) -> TupleType: + return TupleType(items, fallback=self.builtin_type('builtins.tuple', [AnyType()])) + + +class TypeAnalyserPass3(TypeVisitor[None]): + """Analyze type argument counts and values of generic types. + + This is semantic analysis pass 3 for types. + + Perform these operations: + + * Report error for invalid type argument counts, such as List[x, y]. + * Make implicit Any type arguments explicit my modifying types + in-place. For example, modify Foo into Foo[Any] if Foo expects a single + type argument. + * If a type variable has a value restriction, ensure that the value is + valid. For example, reject IO[int] if the type argument must be str + or bytes. + + We can't do this earlier than the third pass, since type argument counts + are only determined in pass 2, and we have to support forward references + to types. + """ + + def __init__(self, fail_func: Callable[[str, Context], None]) -> None: + self.fail = fail_func + + def visit_instance(self, t: Instance) -> None: + info = t.type + # Check type argument count. + if len(t.args) != len(info.type_vars): + if len(t.args) == 0: + # Insert implicit 'Any' type arguments. + t.args = [AnyType()] * len(info.type_vars) + return + # Invalid number of type parameters. + n = len(info.type_vars) + s = '{} type arguments'.format(n) + if n == 0: + s = 'no type arguments' + elif n == 1: + s = '1 type argument' + act = str(len(t.args)) + if act == '0': + act = 'none' + self.fail('"{}" expects {}, but {} given'.format( + info.name(), s, act), t) + # Construct the correct number of type arguments, as + # otherwise the type checker may crash as it expects + # things to be right. + t.args = [AnyType() for _ in info.type_vars] + elif info.defn.type_vars: + # Check type argument values. + for arg, TypeVar in zip(t.args, info.defn.type_vars): + if TypeVar.values: + if isinstance(arg, TypeVarType): + arg_values = arg.values + if not arg_values: + self.fail('Type variable "{}" not valid as type ' + 'argument value for "{}"'.format( + arg.name, info.name()), t) + continue + else: + arg_values = [arg] + self.check_type_var_values(info, arg_values, + TypeVar.values, t) + if not satisfies_upper_bound(arg, TypeVar.upper_bound): + self.fail('Type argument "{}" of "{}" must be ' + 'a subtype of "{}"'.format( + arg, info.name(), TypeVar.upper_bound), t) + for arg in t.args: + arg.accept(self) + + def check_type_var_values(self, type: TypeInfo, actuals: List[Type], + valids: List[Type], context: Context) -> None: + for actual in actuals: + if (not isinstance(actual, AnyType) and + not any(is_same_type(actual, value) for value in valids)): + self.fail('Invalid type argument value for "{}"'.format( + type.name()), context) + + def visit_callable_type(self, t: CallableType) -> None: + t.ret_type.accept(self) + for arg_type in t.arg_types: + arg_type.accept(self) + + def visit_tuple_type(self, t: TupleType) -> None: + for item in t.items: + item.accept(self) + + def visit_union_type(self, t: UnionType) -> None: + for item in t.items: + item.accept(self) + + def visit_star_type(self, t: StarType) -> None: + t.type.accept(self) + + # Other kinds of type are trivial, since they are atomic (or invalid). + + def visit_unbound_type(self, t: UnboundType) -> None: + pass + + def visit_any(self, t: AnyType) -> None: + pass + + def visit_void(self, t: Void) -> None: + pass + + def visit_none_type(self, t: NoneTyp) -> None: + pass + + def visit_uninhabited_type(self, t: UninhabitedType) -> None: + pass + + def visit_deleted_type(self, t: DeletedType) -> None: + pass + + def visit_type_list(self, t: TypeList) -> None: + self.fail('Invalid type', t) + + def visit_type_var(self, t: TypeVarType) -> None: + pass + + def visit_partial_type(self, t: PartialType) -> None: + pass + + def visit_type_type(self, t: TypeType) -> None: + pass diff --git a/mypy/types.py b/mypy/types.py index 7d8dcb1c1a603..57bc7bd6b26f7 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1,1451 +1,1451 @@ -"""Classes for representing mypy types.""" - -from abc import abstractmethod -from typing import ( - Any, TypeVar, Dict, List, Tuple, cast, Generic, Set, Sequence, Optional, Union -) - -import mypy.nodes -from mypy.nodes import INVARIANT, SymbolNode - -from mypy import experiments - - -T = TypeVar('T') - -JsonDict = Dict[str, Any] - - -class Type(mypy.nodes.Context): - """Abstract base class for all types.""" - - line = 0 - - def __init__(self, line: int = -1) -> None: - self.line = line - - def get_line(self) -> int: - return self.line - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - raise RuntimeError('Not implemented') - - def __repr__(self) -> str: - return self.accept(TypeStrVisitor()) - - def serialize(self) -> JsonDict: - raise NotImplementedError('Cannot serialize {} instance'.format(self.__class__.__name__)) - - @classmethod - def deserialize(cls, data: JsonDict) -> 'Type': - classname = data['.class'] - glo = globals() - if classname in glo: - cl = glo[classname] - if 'deserialize' in cl.__dict__: - return cl.deserialize(data) - raise NotImplementedError('unexpected .class {}'.format(classname)) - - -class TypeVarId: - # A type variable is uniquely identified by its raw id and meta level. - - # For plain variables (type parameters of generic classes and - # functions) raw ids are allocated by semantic analysis, using - # positive ids 1, 2, ... for generic class parameters and negative - # ids -1, ... for generic function type arguments. This convention - # is only used to keep type variable ids distinct when allocating - # them; the type checker makes no distinction between class and - # function type variables. - - # Metavariables are allocated unique ids starting from 1. - raw_id = 0 # type: int - - # Level of the variable in type inference. Currently either 0 for - # declared types, or 1 for type inference metavariables. - meta_level = 0 # type: int - - # Class variable used for allocating fresh ids for metavariables. - next_raw_id = 1 # type: int - - def __init__(self, raw_id: int, meta_level: int = 0) -> None: - self.raw_id = raw_id - self.meta_level = meta_level - - @staticmethod - def new(meta_level: int) -> 'TypeVarId': - raw_id = TypeVarId.next_raw_id - TypeVarId.next_raw_id += 1 - return TypeVarId(raw_id, meta_level) - - def __repr__(self) -> str: - return self.raw_id.__repr__() - - def __eq__(self, other: object) -> bool: - if isinstance(other, TypeVarId): - return (self.raw_id == other.raw_id and - self.meta_level == other.meta_level) - else: - return False - - def __ne__(self, other: object) -> bool: - return not (self == other) - - def __hash__(self) -> int: - return hash((self.raw_id, self.meta_level)) - - def is_meta_var(self) -> bool: - return self.meta_level > 0 - - -class TypeVarDef(mypy.nodes.Context): - """Definition of a single type variable.""" - - name = '' - id = None # type: TypeVarId - values = None # type: List[Type] # Value restriction, empty list if no restriction - upper_bound = None # type: Type - variance = INVARIANT # type: int - line = 0 - - def __init__(self, name: str, id: Union[TypeVarId, int], values: Optional[List[Type]], - upper_bound: Type, variance: int = INVARIANT, line: int = -1) -> None: - self.name = name - if isinstance(id, int): - id = TypeVarId(id) - self.id = id - self.values = values - self.upper_bound = upper_bound - self.variance = variance - self.line = line - - @staticmethod - def new_unification_variable(old: 'TypeVarDef') -> 'TypeVarDef': - new_id = TypeVarId.new(meta_level=1) - return TypeVarDef(old.name, new_id, old.values, - old.upper_bound, old.variance, old.line) - - def get_line(self) -> int: - return self.line - - def __repr__(self) -> str: - if self.values: - return '{} in {}'.format(self.name, tuple(self.values)) - elif not is_named_instance(self.upper_bound, 'builtins.object'): - return '{} <: {}'.format(self.name, self.upper_bound) - else: - return self.name - - def serialize(self) -> JsonDict: - assert not self.id.is_meta_var() - return {'.class': 'TypeVarDef', - 'name': self.name, - 'id': self.id.raw_id, - 'values': None if self.values is None else [v.serialize() for v in self.values], - 'upper_bound': self.upper_bound.serialize(), - 'variance': self.variance, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'TypeVarDef': - assert data['.class'] == 'TypeVarDef' - return TypeVarDef(data['name'], - data['id'], - None if data['values'] is None - else [Type.deserialize(v) for v in data['values']], - Type.deserialize(data['upper_bound']), - data['variance'], - ) - - -class UnboundType(Type): - """Instance type that has not been bound during semantic analysis.""" - - name = '' - args = None # type: List[Type] - # should this type be wrapped in an Optional? - optional = False - # is this type a return type? - is_ret_type = False - - def __init__(self, - name: str, - args: List[Type] = None, - line: int = -1, - optional: bool = False, - is_ret_type: bool = False) -> None: - if not args: - args = [] - self.name = name - self.args = args - self.optional = optional - self.is_ret_type = is_ret_type - super().__init__(line) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_unbound_type(self) - - def serialize(self) -> JsonDict: - return {'.class': 'UnboundType', - 'name': self.name, - 'args': [a.serialize() for a in self.args], - } - - @classmethod - def deserialize(self, data: JsonDict) -> 'UnboundType': - assert data['.class'] == 'UnboundType' - return UnboundType(data['name'], - [Type.deserialize(a) for a in data['args']]) - - -class ErrorType(Type): - """The error type is used as the result of failed type operations.""" - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_error_type(self) - - -class TypeList(Type): - """A list of types [...]. - - This is only used for the arguments of a Callable type, i.e. for - [arg, ...] in Callable[[arg, ...], ret]. This is not a real type - but a syntactic AST construct. - """ - - items = None # type: List[Type] - - def __init__(self, items: List[Type], line: int = -1) -> None: - super().__init__(line) - self.items = items - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_type_list(self) - - def serialize(self) -> JsonDict: - return {'.class': 'TypeList', - 'items': [t.serialize() for t in self.items], - } - - @classmethod - def deserialize(self, data: JsonDict) -> 'TypeList': - assert data['.class'] == 'TypeList' - return TypeList([Type.deserialize(t) for t in data['items']]) - - -class AnyType(Type): - """The type 'Any'.""" - - def __init__(self, implicit=False, line: int = -1) -> None: - super().__init__(line) - self.implicit = implicit - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_any(self) - - def serialize(self) -> JsonDict: - return {'.class': 'AnyType'} - - @classmethod - def deserialize(cls, data: JsonDict) -> 'AnyType': - assert data['.class'] == 'AnyType' - return AnyType() - - -class Void(Type): - """The return type 'None'. - - This can only be used as the return type in a callable type and as - the result type of calling such callable. - """ - - source = '' # May be None; function that generated this value - - def __init__(self, source: str = None, line: int = -1) -> None: - self.source = source - super().__init__(line) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_void(self) - - def with_source(self, source: str) -> 'Void': - return Void(source, self.line) - - def serialize(self) -> JsonDict: - return {'.class': 'Void'} - - @classmethod - def deserialize(cls, data: JsonDict) -> 'Void': - assert data['.class'] == 'Void' - return Void() - - -class UninhabitedType(Type): - """This type has no members. - - This type is almost the bottom type, except it is not a subtype of Void. - With strict Optional checking, it is the only common subtype between all - other types, which allows `meet` to be well defined. Without strict - Optional checking, NoneTyp fills this role. - - In general, for any type T that isn't Void: - join(UninhabitedType, T) = T - meet(UninhabitedType, T) = UninhabitedType - is_subtype(UninhabitedType, T) = True - """ - - def __init__(self, line: int = -1) -> None: - super().__init__(line) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_uninhabited_type(self) - - def serialize(self) -> JsonDict: - return {'.class': 'UninhabitedType'} - - @classmethod - def deserialize(cls, data: JsonDict) -> 'UninhabitedType': - assert data['.class'] == 'UninhabitedType' - return UninhabitedType() - - -class NoneTyp(Type): - """The type of 'None'. - - Without strict Optional checking: - This is only used internally during type inference. Programs - cannot declare a variable of this type, and the type checker - refuses to infer this type for a variable. However, subexpressions - often have this type. Note that this is not used as the result - type when calling a function with a void type, even though - semantically such a function returns a None value; the void type - is used instead so that we can report an error if the caller tries - to do anything with the return value. - - With strict Optional checking: - This type can be written by users as 'None', except as the return value - of a function, where 'None' means Void. - """ - - def __init__(self, line: int = -1) -> None: - super().__init__(line) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_none_type(self) - - def serialize(self) -> JsonDict: - return {'.class': 'NoneTyp'} - - @classmethod - def deserialize(self, data: JsonDict) -> 'NoneTyp': - assert data['.class'] == 'NoneTyp' - return NoneTyp() - - -class ErasedType(Type): - """Placeholder for an erased type. - - This is used during type inference. This has the special property that - it is ignored during type inference. - """ - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_erased_type(self) - - -class DeletedType(Type): - """Type of deleted variables. - - These can be used as lvalues but not rvalues. - """ - - source = '' # May be None; name that generated this value - - def __init__(self, source: str = None, line: int = -1) -> None: - self.source = source - super().__init__(line) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_deleted_type(self) - - def serialize(self) -> JsonDict: - return {'.class': 'DeletedType', - 'source': self.source} - - @classmethod - def deserialize(self, data: JsonDict) -> 'DeletedType': - assert data['.class'] == 'DeletedType' - return DeletedType(data['source']) - - -class Instance(Type): - """An instance type of form C[T1, ..., Tn]. - - The list of type variables may be empty. - """ - - type = None # type: mypy.nodes.TypeInfo - args = None # type: List[Type] - erased = False # True if result of type variable substitution - - def __init__(self, typ: mypy.nodes.TypeInfo, args: List[Type], - line: int = -1, erased: bool = False) -> None: - self.type = typ - self.args = args - self.erased = erased - super().__init__(line) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_instance(self) - - type_ref = None # type: str - - def serialize(self) -> JsonDict: - data = {'.class': 'Instance', - } # type: JsonDict - assert self.type is not None - data['type_ref'] = self.type.alt_fullname or self.type.fullname() - if self.args: - data['args'] = [arg.serialize() for arg in self.args] - return data - - @classmethod - def deserialize(cls, data: JsonDict) -> 'Instance': - assert data['.class'] == 'Instance' - args = [] # type: List[Type] - if 'args' in data: - args_list = data['args'] - assert isinstance(args_list, list) - args = [Type.deserialize(arg) for arg in args_list] - inst = Instance(None, args) - inst.type_ref = data['type_ref'] # Will be fixed up by fixup.py later. - return inst - - -class TypeVarType(Type): - """A type variable type. - - This refers to either a class type variable (id > 0) or a function - type variable (id < 0). - """ - - name = '' # Name of the type variable (for messages and debugging) - id = None # type: TypeVarId - values = None # type: List[Type] # Value restriction, empty list if no restriction - upper_bound = None # type: Type # Upper bound for values - # See comments in TypeVarDef for more about variance. - variance = INVARIANT # type: int - - def __init__(self, binder: TypeVarDef, line: int = -1) -> None: - self.name = binder.name - self.id = binder.id - self.values = binder.values - self.upper_bound = binder.upper_bound - self.variance = binder.variance - super().__init__(line) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_type_var(self) - - def erase_to_union_or_bound(self) -> Type: - if self.values: - return UnionType.make_simplified_union(self.values) - else: - return self.upper_bound - - def serialize(self) -> JsonDict: - assert not self.id.is_meta_var() - return {'.class': 'TypeVarType', - 'name': self.name, - 'id': self.id.raw_id, - 'values': [v.serialize() for v in self.values], - 'upper_bound': self.upper_bound.serialize(), - 'variance': self.variance, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'TypeVarType': - assert data['.class'] == 'TypeVarType' - tvdef = TypeVarDef(data['name'], - data['id'], - [Type.deserialize(v) for v in data['values']], - Type.deserialize(data['upper_bound']), - data['variance']) - return TypeVarType(tvdef) - - -class FunctionLike(Type): - """Abstract base class for function types.""" - - @abstractmethod - def is_type_obj(self) -> bool: pass - - def is_concrete_type_obj(self) -> bool: - return self.is_type_obj() - - @abstractmethod - def type_object(self) -> mypy.nodes.TypeInfo: pass - - @abstractmethod - def items(self) -> List['CallableType']: pass - - @abstractmethod - def with_name(self, name: str) -> 'FunctionLike': pass - - # Corresponding instance type (e.g. builtins.type) - fallback = None # type: Instance - - @classmethod - def deserialize(cls, data: JsonDict) -> 'FunctionLike': - return cast(FunctionLike, super().deserialize(data)) - - -_dummy = object() # type: Any - - -class CallableType(FunctionLike): - """Type of a non-overloaded callable object (function).""" - - arg_types = None # type: List[Type] # Types of function arguments - arg_kinds = None # type: List[int] # mypy.nodes.ARG_ constants - arg_names = None # type: List[str] # None if not a keyword argument - min_args = 0 # Minimum number of arguments; derived from arg_kinds - is_var_arg = False # Is it a varargs function? derived from arg_kinds - ret_type = None # type: Type # Return value type - name = '' # Name (may be None; for error messages) - definition = None # type: SymbolNode # For error messages. May be None. - # Type variables for a generic function - variables = None # type: List[TypeVarDef] - - # Is this Callable[..., t] (with literal '...')? - is_ellipsis_args = False - # Is this callable constructed for the benefit of a classmethod's 'cls' argument? - is_classmethod_class = False - # Was this type implicitly generated instead of explicitly specified by the user? - implicit = False - # Defined for signatures that require special handling (currently only value is 'dict' - # for a signature similar to 'dict') - special_sig = None # type: Optional[str] - - def __init__(self, - arg_types: List[Type], - arg_kinds: List[int], - arg_names: List[str], - ret_type: Type, - fallback: Instance, - name: str = None, - definition: SymbolNode = None, - variables: List[TypeVarDef] = None, - line: int = -1, - is_ellipsis_args: bool = False, - implicit=False, - is_classmethod_class=False, - special_sig=None, - ) -> None: - if variables is None: - variables = [] - self.arg_types = arg_types - self.arg_kinds = arg_kinds - self.arg_names = arg_names - self.min_args = arg_kinds.count(mypy.nodes.ARG_POS) - self.is_var_arg = mypy.nodes.ARG_STAR in arg_kinds - self.ret_type = ret_type - self.fallback = fallback - assert not name or ' 'CallableType': - return CallableType( - arg_types=arg_types if arg_types is not _dummy else self.arg_types, - arg_kinds=arg_kinds if arg_kinds is not _dummy else self.arg_kinds, - arg_names=arg_names if arg_names is not _dummy else self.arg_names, - ret_type=ret_type if ret_type is not _dummy else self.ret_type, - fallback=fallback if fallback is not _dummy else self.fallback, - name=name if name is not _dummy else self.name, - definition=definition if definition is not _dummy else self.definition, - variables=variables if variables is not _dummy else self.variables, - line=line if line is not _dummy else self.line, - is_ellipsis_args=( - is_ellipsis_args if is_ellipsis_args is not _dummy else self.is_ellipsis_args), - implicit=self.implicit, - is_classmethod_class=self.is_classmethod_class, - special_sig=special_sig if special_sig is not _dummy else self.special_sig, - ) - - def is_type_obj(self) -> bool: - return self.fallback.type is not None and self.fallback.type.fullname() == 'builtins.type' - - def is_concrete_type_obj(self) -> bool: - return self.is_type_obj() and self.is_classmethod_class - - def type_object(self) -> mypy.nodes.TypeInfo: - assert self.is_type_obj() - ret = self.ret_type - if isinstance(ret, TupleType): - ret = ret.fallback - return cast(Instance, ret).type - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_callable_type(self) - - def with_name(self, name: str) -> 'CallableType': - """Return a copy of this type with the specified name.""" - ret = self.ret_type - if isinstance(ret, Void): - ret = ret.with_source(name) - return self.copy_modified(ret_type=ret, name=name) - - def max_fixed_args(self) -> int: - n = len(self.arg_types) - if self.is_var_arg: - n -= 1 - return n - - def items(self) -> List['CallableType']: - return [self] - - def is_generic(self) -> bool: - return bool(self.variables) - - def type_var_ids(self) -> List[TypeVarId]: - a = [] # type: List[TypeVarId] - for tv in self.variables: - a.append(tv.id) - return a - - def serialize(self) -> JsonDict: - # TODO: As an optimization, leave out everything related to - # generic functions for non-generic functions. - return {'.class': 'CallableType', - 'arg_types': [(None if t is None else t.serialize()) - for t in self.arg_types], - 'arg_kinds': self.arg_kinds, - 'arg_names': self.arg_names, - 'ret_type': self.ret_type.serialize(), - 'fallback': self.fallback.serialize(), - 'name': self.name, - # We don't serialize the definition (only used for error messages). - 'variables': [v.serialize() for v in self.variables], - 'is_ellipsis_args': self.is_ellipsis_args, - 'implicit': self.implicit, - 'is_classmethod_class': self.is_classmethod_class, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'CallableType': - assert data['.class'] == 'CallableType' - # TODO: Set definition to the containing SymbolNode? - return CallableType([(None if t is None else Type.deserialize(t)) - for t in data['arg_types']], - data['arg_kinds'], - data['arg_names'], - Type.deserialize(data['ret_type']), - Instance.deserialize(data['fallback']), - name=data['name'], - variables=[TypeVarDef.deserialize(v) for v in data['variables']], - is_ellipsis_args=data['is_ellipsis_args'], - implicit=data['implicit'], - is_classmethod_class=data['is_classmethod_class'], - ) - - -class Overloaded(FunctionLike): - """Overloaded function type T1, ... Tn, where each Ti is CallableType. - - The variant to call is chosen based on static argument - types. Overloaded function types can only be defined in stub - files, and thus there is no explicit runtime dispatch - implementation. - """ - - _items = None # type: List[CallableType] # Must not be empty - - def __init__(self, items: List[CallableType]) -> None: - self._items = items - self.fallback = items[0].fallback - super().__init__(items[0].line) - - def items(self) -> List[CallableType]: - return self._items - - def name(self) -> str: - return self._items[0].name - - def is_type_obj(self) -> bool: - # All the items must have the same type object status, so it's - # sufficient to query only (any) one of them. - return self._items[0].is_type_obj() - - def type_object(self) -> mypy.nodes.TypeInfo: - # All the items must have the same type object, so it's sufficient to - # query only (any) one of them. - return self._items[0].type_object() - - def with_name(self, name: str) -> 'Overloaded': - ni = [] # type: List[CallableType] - for it in self._items: - ni.append(it.with_name(name)) - return Overloaded(ni) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_overloaded(self) - - def serialize(self) -> JsonDict: - return {'.class': 'Overloaded', - 'items': [t.serialize() for t in self.items()], - } - - @classmethod - def deserialize(self, data: JsonDict) -> 'Overloaded': - assert data['.class'] == 'Overloaded' - return Overloaded([CallableType.deserialize(t) for t in data['items']]) - - -class TupleType(Type): - """The tuple type Tuple[T1, ..., Tn] (at least one type argument). - - Instance variables: - items: tuple item types - fallback: the underlying instance type that is used for non-tuple methods - (this is currently always builtins.tuple, but it could be different for named - tuples, for example) - implicit: if True, derived from a tuple expression (t,....) instead of Tuple[t, ...] - """ - - items = None # type: List[Type] - fallback = None # type: Instance - implicit = False - - def __init__(self, items: List[Type], fallback: Instance, line: int = -1, - implicit: bool = False) -> None: - #import pdb; pdb.set_trace() - self.items = items - self.fallback = fallback - self.implicit = implicit - super().__init__(line) - - def length(self) -> int: - return len(self.items) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_tuple_type(self) - - def serialize(self) -> JsonDict: - return {'.class': 'TupleType', - 'items': [t.serialize() for t in self.items], - 'fallback': self.fallback.serialize(), - 'implicit': self.implicit, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'TupleType': - assert data['.class'] == 'TupleType' - return TupleType([Type.deserialize(t) for t in data['items']], - Instance.deserialize(data['fallback']), - implicit=data['implicit']) - - def self_type(self, inst: Instance) -> 'TupleType': - return TupleType(self.items, inst, self.line, self.implicit) - - def with_types(self, items: List[Type]) -> 'TupleType': - return TupleType(items, self.fallback, self.line, self.implicit) - - def slice(self, begin: int, stride: int, end: int) -> 'TupleType': - return TupleType(self.items[begin:end:stride], self.fallback, - self.line, self.implicit) - - -class NamedTupleType(TupleType): - attrs = None # type: List[str] - name = None # type: str - - def __init__(self, name: str, attrs: List[str], *args, **kwargs) -> None: - self.attrs = attrs - self.name = name - super().__init__(*args, **kwargs) - - def self_type(self, inst: Instance) -> 'NamedTupleType': - return NamedTupleType(self.name, self.attrs, self.items, inst, self.line) - - def with_types(self, items: List[Type]) -> 'NamedTupleType': - return NamedTupleType(self.name, self.attrs, items, - self.fallback, self.line, self.implicit) - - def __eq__(self, other) -> bool: - return self.serialize() == other.serialize() - - def serialize(self) -> JsonDict: - d = super().serialize() - d.update({'.class': 'NamedTupleType', - 'name': self.name, - 'attrs': self.attrs}) - return d - - @classmethod - def deserialize(cls, data: JsonDict) -> 'NamedTupleType': - assert data['.class'] == 'NamedTupleType' - data['.class'] = 'TupleType' - tup = TupleType.deserialize(data) - return NamedTupleType(data['name'], - data['attrs'], - tup.items, - tup.fallback, - implicit=tup.implicit) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - try: - return visitor.visit_namedtuple_type(self) - except: - return super().accept(visitor) - - -class StarType(Type): - """The star type *type_parameter. - - This is not a real type but a syntactic AST construct. - """ - - type = None # type: Type - - def __init__(self, type: Type, line: int = -1) -> None: - self.type = type - super().__init__(line) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_star_type(self) - - -class UnionType(Type): - """The union type Union[T1, ..., Tn] (at least one type argument).""" - - items = None # type: List[Type] - - def __init__(self, items: List[Type], line: int = -1) -> None: - self.items = items - super().__init__(line) - - @staticmethod - def make_union(items: List[Type], line: int = -1) -> Type: - if len(items) > 1: - return UnionType(items, line) - elif len(items) == 1: - return items[0] - else: - if experiments.STRICT_OPTIONAL: - return UninhabitedType() - else: - return Void() - - @staticmethod - def make_simplified_union(items: List[Type], line: int = -1) -> Type: - while any(isinstance(typ, UnionType) for typ in items): - all_items = [] # type: List[Type] - for typ in items: - if isinstance(typ, UnionType): - all_items.extend(typ.items) - else: - all_items.append(typ) - items = all_items - - if any(isinstance(typ, AnyType) for typ in items): - return AnyType() - - from mypy.subtypes import is_subtype - removed = set() # type: Set[int] - for i in range(len(items)): - if any(is_subtype(items[i], items[j]) for j in range(len(items)) - if j not in removed and j != i): - removed.add(i) - - simplified_set = [items[i] for i in range(len(items)) if i not in removed] - return UnionType.make_union(simplified_set) - - def length(self) -> int: - return len(self.items) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_union_type(self) - - def has_readable_member(self, name: str) -> bool: - """For a tree of unions of instances, check whether all instances have a given member. - - TODO: Deal with attributes of TupleType etc. - TODO: This should probably be refactored to go elsewhere. - """ - return all((isinstance(x, UnionType) and x.has_readable_member(name)) or - (isinstance(x, Instance) and x.type.has_readable_member(name)) - for x in self.items) - - def serialize(self) -> JsonDict: - return {'.class': 'UnionType', - 'items': [t.serialize() for t in self.items], - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'UnionType': - assert data['.class'] == 'UnionType' - return UnionType([Type.deserialize(t) for t in data['items']]) - - -class PartialType(Type): - """Type such as List[?] where type arguments are unknown, or partial None type. - - These are used for inferring types in multiphase initialization such as this: - - x = [] # x gets a partial type List[?], as item type is unknown - x.append(1) # partial type gets replaced with normal type List[int] - - Or with None: - - x = None # x gets a partial type None - if c: - x = 1 # Infer actual type int for x - """ - - # None for the 'None' partial type; otherwise a generic class - type = None # type: Optional[mypy.nodes.TypeInfo] - var = None # type: mypy.nodes.Var - inner_types = None # type: List[Type] - - def __init__(self, - type: Optional['mypy.nodes.TypeInfo'], - var: 'mypy.nodes.Var', - inner_types: List[Type]) -> None: - self.type = type - self.var = var - self.inner_types = inner_types - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_partial_type(self) - - -class EllipsisType(Type): - """The type ... (ellipsis). - - This is not a real type but a syntactic AST construct, used in Callable[..., T], for example. - - A semantically analyzed type will never have ellipsis types. - """ - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_ellipsis_type(self) - - def serialize(self) -> JsonDict: - return {'.class': 'EllipsisType'} - - @classmethod - def deserialize(self, data: JsonDict) -> 'EllipsisType': - assert data['.class'] == 'EllipsisType' - return EllipsisType() - - -class TypeType(Type): - """For types like Type[User]. - - This annotates variables that are class objects, constrained by - the type argument. See PEP 484 for more details. - - We may encounter expressions whose values are specific classes; - those are represented as callables (possibly overloaded) - corresponding to the class's constructor's signature and returning - an instance of that class. The difference with Type[C] is that - those callables always represent the exact class given as the - return type; Type[C] represents any class that's a subclass of C, - and C may also be a type variable or a union (or Any). - - Many questions around subtype relationships between Type[C1] and - def(...) -> C2 are answered by looking at the subtype - relationships between C1 and C2, since Type[] is considered - covariant. - - There's an unsolved problem with constructor signatures (also - unsolved in PEP 484): calling a variable whose type is Type[C] - assumes the constructor signature for C, even though a subclass of - C might completely change the constructor signature. For now we - just assume that users of Type[C] are careful not to do that (in - the future we might detect when they are violating that - assumption). - """ - - # This can't be everything, but it can be a class reference, - # a generic class instance, a union, Any, a type variable... - item = None # type: Type - - def __init__(self, item: Type, *, line: int = -1) -> None: - super().__init__(line) - self.item = item - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_type_type(self) - - def serialize(self) -> JsonDict: - return {'.class': 'TypeType', 'item': self.item.serialize()} - - def deserialize(self, data: JsonDict) -> 'TypeType': - assert data['.class'] == 'TypeType' - return TypeType(Type.deserialize(data['item'])) - - -# -# Visitor-related classes -# - - -class TypeVisitor(Generic[T]): - """Visitor class for types (Type subclasses). - - The parameter T is the return type of the visit methods. - """ - - def _notimplemented_helper(self, name: str) -> NotImplementedError: - return NotImplementedError("Method {}.visit_{}() not implemented\n" - .format(type(self).__name__, name) - + "This is a known bug, track development in " - + "'https://github.com/JukkaL/mypy/issues/730'") - - @abstractmethod - def visit_unbound_type(self, t: UnboundType) -> T: - pass - - def visit_type_list(self, t: TypeList) -> T: - raise self._notimplemented_helper('type_list') - - def visit_error_type(self, t: ErrorType) -> T: - raise self._notimplemented_helper('error_type') - - @abstractmethod - def visit_any(self, t: AnyType) -> T: - pass - - @abstractmethod - def visit_void(self, t: Void) -> T: - pass - - @abstractmethod - def visit_none_type(self, t: NoneTyp) -> T: - pass - - @abstractmethod - def visit_uninhabited_type(self, t: UninhabitedType) -> T: - pass - - def visit_erased_type(self, t: ErasedType) -> T: - raise self._notimplemented_helper('erased_type') - - @abstractmethod - def visit_deleted_type(self, t: DeletedType) -> T: - pass - - @abstractmethod - def visit_type_var(self, t: TypeVarType) -> T: - pass - - @abstractmethod - def visit_instance(self, t: Instance) -> T: - pass - - @abstractmethod - def visit_callable_type(self, t: CallableType) -> T: - pass - - def visit_overloaded(self, t: Overloaded) -> T: - raise self._notimplemented_helper('overloaded') - - @abstractmethod - def visit_tuple_type(self, t: TupleType) -> T: - pass - - def visit_star_type(self, t: StarType) -> T: - raise self._notimplemented_helper('star_type') - - @abstractmethod - def visit_union_type(self, t: UnionType) -> T: - pass - - @abstractmethod - def visit_partial_type(self, t: PartialType) -> T: - pass - - def visit_ellipsis_type(self, t: EllipsisType) -> T: - raise self._notimplemented_helper('ellipsis_type') - - @abstractmethod - def visit_type_type(self, t: TypeType) -> T: - pass - - -class TypeTranslator(TypeVisitor[Type]): - """Identity type transformation. - - Subclass this and override some methods to implement a non-trivial - transformation. - """ - - def visit_unbound_type(self, t: UnboundType) -> Type: - return t - - def visit_type_list(self, t: TypeList) -> Type: - return t - - def visit_error_type(self, t: ErrorType) -> Type: - return t - - def visit_any(self, t: AnyType) -> Type: - return t - - def visit_void(self, t: Void) -> Type: - return t - - def visit_none_type(self, t: NoneTyp) -> Type: - return t - - def visit_uninhabited_type(self, t: UninhabitedType) -> Type: - return t - - def visit_erased_type(self, t: ErasedType) -> Type: - return t - - def visit_deleted_type(self, t: DeletedType) -> Type: - return t - - def visit_instance(self, t: Instance) -> Type: - return Instance(t.type, self.translate_types(t.args), t.line) - - def visit_type_var(self, t: TypeVarType) -> Type: - return t - - def visit_partial_type(self, t: PartialType) -> Type: - return t - - def visit_callable_type(self, t: CallableType) -> Type: - return t.copy_modified(arg_types=self.translate_types(t.arg_types), - ret_type=t.ret_type.accept(self), - variables=self.translate_variables(t.variables)) - - def visit_tuple_type(self, t: TupleType) -> Type: - return TupleType(self.translate_types(t.items), - cast(Any, t.fallback.accept(self)), - t.line) - - def visit_star_type(self, t: StarType) -> Type: - return StarType(t.type.accept(self), t.line) - - def visit_union_type(self, t: UnionType) -> Type: - return UnionType(self.translate_types(t.items), t.line) - - def visit_ellipsis_type(self, t: EllipsisType) -> Type: - return t - - def translate_types(self, types: List[Type]) -> List[Type]: - return [t.accept(self) for t in types] - - def translate_variables(self, - variables: List[TypeVarDef]) -> List[TypeVarDef]: - return variables - - def visit_overloaded(self, t: Overloaded) -> Type: - items = [] # type: List[CallableType] - for item in t.items(): - new = item.accept(self) - if isinstance(new, CallableType): - items.append(new) - else: - raise RuntimeError('CallableType expectected, but got {}'.format(type(new))) - return Overloaded(items=items) - - def visit_type_type(self, t: TypeType) -> Type: - return TypeType(t.item.accept(self), line=t.line) - - -class TypeStrVisitor(TypeVisitor[str]): - """Visitor for pretty-printing types into strings. - - This is mostly for debugging/testing. - - Do not preserve original formatting. - - Notes: - - Represent unbound types as Foo? or Foo?[...]. - - Represent the NoneTyp type as None. - """ - - def visit_unbound_type(self, t): - s = t.name + '?' - if t.args != []: - s += '[{}]'.format(self.list_str(t.args)) - return s - - def visit_type_list(self, t): - return ''.format(self.list_str(t.items)) - - def visit_error_type(self, t): - return '' - - def visit_any(self, t): - return 'Any' - - def visit_void(self, t): - return 'void' - - def visit_none_type(self, t): - # Fully qualify to make this distinct from the None value. - return "builtins.None" - - def visit_uninhabited_type(self, t): - return "" - - def visit_erased_type(self, t): - return "" - - def visit_deleted_type(self, t): - if t.source is None: - return "" - else: - return "".format(t.source) - - def visit_instance(self, t): - s = t.type.fullname() if t.type is not None else '' - if t.erased: - s += '*' - if t.args != []: - s += '[{}]'.format(self.list_str(t.args)) - return s - - def visit_type_var(self, t): - if t.name is None: - # Anonymous type variable type (only numeric id). - return '`{}'.format(t.id) - else: - # Named type variable type. - return '{}`{}'.format(t.name, t.id) - - def visit_callable_type(self, t): - s = '' - bare_asterisk = False - for i in range(len(t.arg_types)): - if s != '': - s += ', ' - if t.arg_kinds[i] == mypy.nodes.ARG_NAMED and not bare_asterisk: - s += '*, ' - bare_asterisk = True - if t.arg_kinds[i] == mypy.nodes.ARG_STAR: - s += '*' - if t.arg_kinds[i] == mypy.nodes.ARG_STAR2: - s += '**' - if t.arg_names[i]: - s += t.arg_names[i] + ': ' - s += str(t.arg_types[i]) - if t.arg_kinds[i] == mypy.nodes.ARG_OPT: - s += ' =' - - s = '({})'.format(s) - - if not isinstance(t.ret_type, Void): - s += ' -> {}'.format(t.ret_type) - - if t.variables: - s = '{} {}'.format(t.variables, s) - - return 'def {}'.format(s) - - def visit_overloaded(self, t): - a = [] - for i in t.items(): - a.append(i.accept(self)) - return 'Overload({})'.format(', '.join(a)) - - def visit_tuple_type(self, t): - s = self.list_str(t.items) - if t.fallback and t.fallback.type: - fallback_name = t.fallback.type.fullname() - if fallback_name != 'builtins.tuple': - return 'Tuple[{}, fallback={}]'.format(s, t.fallback.accept(self)) - return 'Tuple[{}]'.format(s) - - def visit_namedtuple_type(self, t): - s = self.list_str(zip(t.attrs, t.items)) - return '{}[{}]'.format(t.name, s) - - def visit_star_type(self, t): - s = t.type.accept(self) - return '*{}'.format(s) - - def visit_union_type(self, t): - s = self.list_str(t.items) - return 'Union[{}]'.format(s) - - def visit_partial_type(self, t: PartialType) -> str: - if t.type is None: - return '' - else: - return ''.format(t.type.name(), - ', '.join(['?'] * len(t.type.type_vars))) - - def visit_ellipsis_type(self, t): - return '...' - - def visit_type_type(self, t): - return 'Type[{}]'.format(t.item.accept(self)) - - def list_str(self, a): - """Convert items of an array to strings (pretty-print types) - and join the results with commas. - """ - res = [] - for t in a: - if isinstance(t, Type): - res.append(t.accept(self)) - else: - res.append(str(t)) - return ', '.join(res) - - -# These constants define the method used by TypeQuery to combine multiple -# query results, e.g. for tuple types. The strategy is not used for empty -# result lists; in that case the default value takes precedence. -ANY_TYPE_STRATEGY = 0 # Return True if any of the results are True. -ALL_TYPES_STRATEGY = 1 # Return True if all of the results are True. - - -class TypeQuery(TypeVisitor[bool]): - """Visitor for performing simple boolean queries of types. - - This class allows defining the default value for leafs to simplify the - implementation of many queries. - """ - - default = False # Default result - strategy = 0 # Strategy for combining multiple values (ANY_TYPE_STRATEGY or ALL_TYPES_...). - - def __init__(self, default: bool, strategy: int) -> None: - """Construct a query visitor. - - Use the given default result and strategy for combining - multiple results. The strategy must be either - ANY_TYPE_STRATEGY or ALL_TYPES_STRATEGY. - """ - self.default = default - self.strategy = strategy - - def visit_unbound_type(self, t: UnboundType) -> bool: - return self.default - - def visit_type_list(self, t: TypeList) -> bool: - return self.default - - def visit_error_type(self, t: ErrorType) -> bool: - return self.default - - def visit_any(self, t: AnyType) -> bool: - return self.default - - def visit_void(self, t: Void) -> bool: - return self.default - - def visit_uninhabited_type(self, t: UninhabitedType) -> bool: - return self.default - - def visit_none_type(self, t: NoneTyp) -> bool: - return self.default - - def visit_erased_type(self, t: ErasedType) -> bool: - return self.default - - def visit_deleted_type(self, t: DeletedType) -> bool: - return self.default - - def visit_type_var(self, t: TypeVarType) -> bool: - return self.default - - def visit_partial_type(self, t: PartialType) -> bool: - return self.default - - def visit_instance(self, t: Instance) -> bool: - return self.query_types(t.args) - - def visit_callable_type(self, t: CallableType) -> bool: - # FIX generics - return self.query_types(t.arg_types + [t.ret_type]) - - def visit_tuple_type(self, t: TupleType) -> bool: - return self.query_types(t.items) - - def visit_star_type(self, t: StarType) -> bool: - return t.type.accept(self) - - def visit_union_type(self, t: UnionType) -> bool: - return self.query_types(t.items) - - def visit_overloaded(self, t: Overloaded) -> bool: - return self.query_types(t.items()) - - def visit_type_type(self, t: TypeType) -> bool: - return t.item.accept(self) - - def query_types(self, types: Sequence[Type]) -> bool: - """Perform a query for a list of types. - - Use the strategy constant to combine the results. - """ - if not types: - # Use default result for empty list. - return self.default - if self.strategy == ANY_TYPE_STRATEGY: - # Return True if at least one component is true. - res = False - for t in types: - res = res or t.accept(self) - if res: - break - return res - else: - # Return True if all components are true. - res = True - for t in types: - res = res and t.accept(self) - if not res: - break - return res - - -def strip_type(typ: Type) -> Type: - """Make a copy of type without 'debugging info' (function name).""" - - if isinstance(typ, CallableType): - return typ.copy_modified(name=None) - elif isinstance(typ, Overloaded): - return Overloaded([cast(CallableType, strip_type(item)) - for item in typ.items()]) - else: - return typ - - -def replace_leading_arg_type(t: CallableType, self_type: Type) -> CallableType: - """Return a copy of a callable type with a different self argument type. - - Assume that the callable is the signature of a method. - """ - return t.copy_modified(arg_types=[self_type] + t.arg_types[1:]) - - -def is_named_instance(t: Type, fullname: str) -> bool: - return (isinstance(t, Instance) and - t.type is not None and - t.type.fullname() == fullname) +"""Classes for representing mypy types.""" + +from abc import abstractmethod +from typing import ( + Any, TypeVar, Dict, List, Tuple, cast, Generic, Set, Sequence, Optional, Union +) + +import mypy.nodes +from mypy.nodes import INVARIANT, SymbolNode + +from mypy import experiments + + +T = TypeVar('T') + +JsonDict = Dict[str, Any] + + +class Type(mypy.nodes.Context): + """Abstract base class for all types.""" + + line = 0 + + def __init__(self, line: int = -1) -> None: + self.line = line + + def get_line(self) -> int: + return self.line + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + raise RuntimeError('Not implemented') + + def __repr__(self) -> str: + return self.accept(TypeStrVisitor()) + + def serialize(self) -> JsonDict: + raise NotImplementedError('Cannot serialize {} instance'.format(self.__class__.__name__)) + + @classmethod + def deserialize(cls, data: JsonDict) -> 'Type': + classname = data['.class'] + glo = globals() + if classname in glo: + cl = glo[classname] + if 'deserialize' in cl.__dict__: + return cl.deserialize(data) + raise NotImplementedError('unexpected .class {}'.format(classname)) + + +class TypeVarId: + # A type variable is uniquely identified by its raw id and meta level. + + # For plain variables (type parameters of generic classes and + # functions) raw ids are allocated by semantic analysis, using + # positive ids 1, 2, ... for generic class parameters and negative + # ids -1, ... for generic function type arguments. This convention + # is only used to keep type variable ids distinct when allocating + # them; the type checker makes no distinction between class and + # function type variables. + + # Metavariables are allocated unique ids starting from 1. + raw_id = 0 # type: int + + # Level of the variable in type inference. Currently either 0 for + # declared types, or 1 for type inference metavariables. + meta_level = 0 # type: int + + # Class variable used for allocating fresh ids for metavariables. + next_raw_id = 1 # type: int + + def __init__(self, raw_id: int, meta_level: int = 0) -> None: + self.raw_id = raw_id + self.meta_level = meta_level + + @staticmethod + def new(meta_level: int) -> 'TypeVarId': + raw_id = TypeVarId.next_raw_id + TypeVarId.next_raw_id += 1 + return TypeVarId(raw_id, meta_level) + + def __repr__(self) -> str: + return self.raw_id.__repr__() + + def __eq__(self, other: object) -> bool: + if isinstance(other, TypeVarId): + return (self.raw_id == other.raw_id and + self.meta_level == other.meta_level) + else: + return False + + def __ne__(self, other: object) -> bool: + return not (self == other) + + def __hash__(self) -> int: + return hash((self.raw_id, self.meta_level)) + + def is_meta_var(self) -> bool: + return self.meta_level > 0 + + +class TypeVarDef(mypy.nodes.Context): + """Definition of a single type variable.""" + + name = '' + id = None # type: TypeVarId + values = None # type: List[Type] # Value restriction, empty list if no restriction + upper_bound = None # type: Type + variance = INVARIANT # type: int + line = 0 + + def __init__(self, name: str, id: Union[TypeVarId, int], values: Optional[List[Type]], + upper_bound: Type, variance: int = INVARIANT, line: int = -1) -> None: + self.name = name + if isinstance(id, int): + id = TypeVarId(id) + self.id = id + self.values = values + self.upper_bound = upper_bound + self.variance = variance + self.line = line + + @staticmethod + def new_unification_variable(old: 'TypeVarDef') -> 'TypeVarDef': + new_id = TypeVarId.new(meta_level=1) + return TypeVarDef(old.name, new_id, old.values, + old.upper_bound, old.variance, old.line) + + def get_line(self) -> int: + return self.line + + def __repr__(self) -> str: + if self.values: + return '{} in {}'.format(self.name, tuple(self.values)) + elif not is_named_instance(self.upper_bound, 'builtins.object'): + return '{} <: {}'.format(self.name, self.upper_bound) + else: + return self.name + + def serialize(self) -> JsonDict: + assert not self.id.is_meta_var() + return {'.class': 'TypeVarDef', + 'name': self.name, + 'id': self.id.raw_id, + 'values': None if self.values is None else [v.serialize() for v in self.values], + 'upper_bound': self.upper_bound.serialize(), + 'variance': self.variance, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'TypeVarDef': + assert data['.class'] == 'TypeVarDef' + return TypeVarDef(data['name'], + data['id'], + None if data['values'] is None + else [Type.deserialize(v) for v in data['values']], + Type.deserialize(data['upper_bound']), + data['variance'], + ) + + +class UnboundType(Type): + """Instance type that has not been bound during semantic analysis.""" + + name = '' + args = None # type: List[Type] + # should this type be wrapped in an Optional? + optional = False + # is this type a return type? + is_ret_type = False + + def __init__(self, + name: str, + args: List[Type] = None, + line: int = -1, + optional: bool = False, + is_ret_type: bool = False) -> None: + if not args: + args = [] + self.name = name + self.args = args + self.optional = optional + self.is_ret_type = is_ret_type + super().__init__(line) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_unbound_type(self) + + def serialize(self) -> JsonDict: + return {'.class': 'UnboundType', + 'name': self.name, + 'args': [a.serialize() for a in self.args], + } + + @classmethod + def deserialize(self, data: JsonDict) -> 'UnboundType': + assert data['.class'] == 'UnboundType' + return UnboundType(data['name'], + [Type.deserialize(a) for a in data['args']]) + + +class ErrorType(Type): + """The error type is used as the result of failed type operations.""" + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_error_type(self) + + +class TypeList(Type): + """A list of types [...]. + + This is only used for the arguments of a Callable type, i.e. for + [arg, ...] in Callable[[arg, ...], ret]. This is not a real type + but a syntactic AST construct. + """ + + items = None # type: List[Type] + + def __init__(self, items: List[Type], line: int = -1) -> None: + super().__init__(line) + self.items = items + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_type_list(self) + + def serialize(self) -> JsonDict: + return {'.class': 'TypeList', + 'items': [t.serialize() for t in self.items], + } + + @classmethod + def deserialize(self, data: JsonDict) -> 'TypeList': + assert data['.class'] == 'TypeList' + return TypeList([Type.deserialize(t) for t in data['items']]) + + +class AnyType(Type): + """The type 'Any'.""" + + def __init__(self, implicit=False, line: int = -1) -> None: + super().__init__(line) + self.implicit = implicit + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_any(self) + + def serialize(self) -> JsonDict: + return {'.class': 'AnyType'} + + @classmethod + def deserialize(cls, data: JsonDict) -> 'AnyType': + assert data['.class'] == 'AnyType' + return AnyType() + + +class Void(Type): + """The return type 'None'. + + This can only be used as the return type in a callable type and as + the result type of calling such callable. + """ + + source = '' # May be None; function that generated this value + + def __init__(self, source: str = None, line: int = -1) -> None: + self.source = source + super().__init__(line) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_void(self) + + def with_source(self, source: str) -> 'Void': + return Void(source, self.line) + + def serialize(self) -> JsonDict: + return {'.class': 'Void'} + + @classmethod + def deserialize(cls, data: JsonDict) -> 'Void': + assert data['.class'] == 'Void' + return Void() + + +class UninhabitedType(Type): + """This type has no members. + + This type is almost the bottom type, except it is not a subtype of Void. + With strict Optional checking, it is the only common subtype between all + other types, which allows `meet` to be well defined. Without strict + Optional checking, NoneTyp fills this role. + + In general, for any type T that isn't Void: + join(UninhabitedType, T) = T + meet(UninhabitedType, T) = UninhabitedType + is_subtype(UninhabitedType, T) = True + """ + + def __init__(self, line: int = -1) -> None: + super().__init__(line) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_uninhabited_type(self) + + def serialize(self) -> JsonDict: + return {'.class': 'UninhabitedType'} + + @classmethod + def deserialize(cls, data: JsonDict) -> 'UninhabitedType': + assert data['.class'] == 'UninhabitedType' + return UninhabitedType() + + +class NoneTyp(Type): + """The type of 'None'. + + Without strict Optional checking: + This is only used internally during type inference. Programs + cannot declare a variable of this type, and the type checker + refuses to infer this type for a variable. However, subexpressions + often have this type. Note that this is not used as the result + type when calling a function with a void type, even though + semantically such a function returns a None value; the void type + is used instead so that we can report an error if the caller tries + to do anything with the return value. + + With strict Optional checking: + This type can be written by users as 'None', except as the return value + of a function, where 'None' means Void. + """ + + def __init__(self, line: int = -1) -> None: + super().__init__(line) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_none_type(self) + + def serialize(self) -> JsonDict: + return {'.class': 'NoneTyp'} + + @classmethod + def deserialize(self, data: JsonDict) -> 'NoneTyp': + assert data['.class'] == 'NoneTyp' + return NoneTyp() + + +class ErasedType(Type): + """Placeholder for an erased type. + + This is used during type inference. This has the special property that + it is ignored during type inference. + """ + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_erased_type(self) + + +class DeletedType(Type): + """Type of deleted variables. + + These can be used as lvalues but not rvalues. + """ + + source = '' # May be None; name that generated this value + + def __init__(self, source: str = None, line: int = -1) -> None: + self.source = source + super().__init__(line) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_deleted_type(self) + + def serialize(self) -> JsonDict: + return {'.class': 'DeletedType', + 'source': self.source} + + @classmethod + def deserialize(self, data: JsonDict) -> 'DeletedType': + assert data['.class'] == 'DeletedType' + return DeletedType(data['source']) + + +class Instance(Type): + """An instance type of form C[T1, ..., Tn]. + + The list of type variables may be empty. + """ + + type = None # type: mypy.nodes.TypeInfo + args = None # type: List[Type] + erased = False # True if result of type variable substitution + + def __init__(self, typ: mypy.nodes.TypeInfo, args: List[Type], + line: int = -1, erased: bool = False) -> None: + self.type = typ + self.args = args + self.erased = erased + super().__init__(line) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_instance(self) + + type_ref = None # type: str + + def serialize(self) -> JsonDict: + data = {'.class': 'Instance', + } # type: JsonDict + assert self.type is not None + data['type_ref'] = self.type.alt_fullname or self.type.fullname() + if self.args: + data['args'] = [arg.serialize() for arg in self.args] + return data + + @classmethod + def deserialize(cls, data: JsonDict) -> 'Instance': + assert data['.class'] == 'Instance' + args = [] # type: List[Type] + if 'args' in data: + args_list = data['args'] + assert isinstance(args_list, list) + args = [Type.deserialize(arg) for arg in args_list] + inst = Instance(None, args) + inst.type_ref = data['type_ref'] # Will be fixed up by fixup.py later. + return inst + + +class TypeVarType(Type): + """A type variable type. + + This refers to either a class type variable (id > 0) or a function + type variable (id < 0). + """ + + name = '' # Name of the type variable (for messages and debugging) + id = None # type: TypeVarId + values = None # type: List[Type] # Value restriction, empty list if no restriction + upper_bound = None # type: Type # Upper bound for values + # See comments in TypeVarDef for more about variance. + variance = INVARIANT # type: int + + def __init__(self, binder: TypeVarDef, line: int = -1) -> None: + self.name = binder.name + self.id = binder.id + self.values = binder.values + self.upper_bound = binder.upper_bound + self.variance = binder.variance + super().__init__(line) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_type_var(self) + + def erase_to_union_or_bound(self) -> Type: + if self.values: + return UnionType.make_simplified_union(self.values) + else: + return self.upper_bound + + def serialize(self) -> JsonDict: + assert not self.id.is_meta_var() + return {'.class': 'TypeVarType', + 'name': self.name, + 'id': self.id.raw_id, + 'values': [v.serialize() for v in self.values], + 'upper_bound': self.upper_bound.serialize(), + 'variance': self.variance, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'TypeVarType': + assert data['.class'] == 'TypeVarType' + tvdef = TypeVarDef(data['name'], + data['id'], + [Type.deserialize(v) for v in data['values']], + Type.deserialize(data['upper_bound']), + data['variance']) + return TypeVarType(tvdef) + + +class FunctionLike(Type): + """Abstract base class for function types.""" + + @abstractmethod + def is_type_obj(self) -> bool: pass + + def is_concrete_type_obj(self) -> bool: + return self.is_type_obj() + + @abstractmethod + def type_object(self) -> mypy.nodes.TypeInfo: pass + + @abstractmethod + def items(self) -> List['CallableType']: pass + + @abstractmethod + def with_name(self, name: str) -> 'FunctionLike': pass + + # Corresponding instance type (e.g. builtins.type) + fallback = None # type: Instance + + @classmethod + def deserialize(cls, data: JsonDict) -> 'FunctionLike': + return cast(FunctionLike, super().deserialize(data)) + + +_dummy = object() # type: Any + + +class CallableType(FunctionLike): + """Type of a non-overloaded callable object (function).""" + + arg_types = None # type: List[Type] # Types of function arguments + arg_kinds = None # type: List[int] # mypy.nodes.ARG_ constants + arg_names = None # type: List[str] # None if not a keyword argument + min_args = 0 # Minimum number of arguments; derived from arg_kinds + is_var_arg = False # Is it a varargs function? derived from arg_kinds + ret_type = None # type: Type # Return value type + name = '' # Name (may be None; for error messages) + definition = None # type: SymbolNode # For error messages. May be None. + # Type variables for a generic function + variables = None # type: List[TypeVarDef] + + # Is this Callable[..., t] (with literal '...')? + is_ellipsis_args = False + # Is this callable constructed for the benefit of a classmethod's 'cls' argument? + is_classmethod_class = False + # Was this type implicitly generated instead of explicitly specified by the user? + implicit = False + # Defined for signatures that require special handling (currently only value is 'dict' + # for a signature similar to 'dict') + special_sig = None # type: Optional[str] + + def __init__(self, + arg_types: List[Type], + arg_kinds: List[int], + arg_names: List[str], + ret_type: Type, + fallback: Instance, + name: str = None, + definition: SymbolNode = None, + variables: List[TypeVarDef] = None, + line: int = -1, + is_ellipsis_args: bool = False, + implicit=False, + is_classmethod_class=False, + special_sig=None, + ) -> None: + if variables is None: + variables = [] + self.arg_types = arg_types + self.arg_kinds = arg_kinds + self.arg_names = arg_names + self.min_args = arg_kinds.count(mypy.nodes.ARG_POS) + self.is_var_arg = mypy.nodes.ARG_STAR in arg_kinds + self.ret_type = ret_type + self.fallback = fallback + assert not name or ' 'CallableType': + return CallableType( + arg_types=arg_types if arg_types is not _dummy else self.arg_types, + arg_kinds=arg_kinds if arg_kinds is not _dummy else self.arg_kinds, + arg_names=arg_names if arg_names is not _dummy else self.arg_names, + ret_type=ret_type if ret_type is not _dummy else self.ret_type, + fallback=fallback if fallback is not _dummy else self.fallback, + name=name if name is not _dummy else self.name, + definition=definition if definition is not _dummy else self.definition, + variables=variables if variables is not _dummy else self.variables, + line=line if line is not _dummy else self.line, + is_ellipsis_args=( + is_ellipsis_args if is_ellipsis_args is not _dummy else self.is_ellipsis_args), + implicit=self.implicit, + is_classmethod_class=self.is_classmethod_class, + special_sig=special_sig if special_sig is not _dummy else self.special_sig, + ) + + def is_type_obj(self) -> bool: + return self.fallback.type is not None and self.fallback.type.fullname() == 'builtins.type' + + def is_concrete_type_obj(self) -> bool: + return self.is_type_obj() and self.is_classmethod_class + + def type_object(self) -> mypy.nodes.TypeInfo: + assert self.is_type_obj() + ret = self.ret_type + if isinstance(ret, TupleType): + ret = ret.fallback + return cast(Instance, ret).type + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_callable_type(self) + + def with_name(self, name: str) -> 'CallableType': + """Return a copy of this type with the specified name.""" + ret = self.ret_type + if isinstance(ret, Void): + ret = ret.with_source(name) + return self.copy_modified(ret_type=ret, name=name) + + def max_fixed_args(self) -> int: + n = len(self.arg_types) + if self.is_var_arg: + n -= 1 + return n + + def items(self) -> List['CallableType']: + return [self] + + def is_generic(self) -> bool: + return bool(self.variables) + + def type_var_ids(self) -> List[TypeVarId]: + a = [] # type: List[TypeVarId] + for tv in self.variables: + a.append(tv.id) + return a + + def serialize(self) -> JsonDict: + # TODO: As an optimization, leave out everything related to + # generic functions for non-generic functions. + return {'.class': 'CallableType', + 'arg_types': [(None if t is None else t.serialize()) + for t in self.arg_types], + 'arg_kinds': self.arg_kinds, + 'arg_names': self.arg_names, + 'ret_type': self.ret_type.serialize(), + 'fallback': self.fallback.serialize(), + 'name': self.name, + # We don't serialize the definition (only used for error messages). + 'variables': [v.serialize() for v in self.variables], + 'is_ellipsis_args': self.is_ellipsis_args, + 'implicit': self.implicit, + 'is_classmethod_class': self.is_classmethod_class, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'CallableType': + assert data['.class'] == 'CallableType' + # TODO: Set definition to the containing SymbolNode? + return CallableType([(None if t is None else Type.deserialize(t)) + for t in data['arg_types']], + data['arg_kinds'], + data['arg_names'], + Type.deserialize(data['ret_type']), + Instance.deserialize(data['fallback']), + name=data['name'], + variables=[TypeVarDef.deserialize(v) for v in data['variables']], + is_ellipsis_args=data['is_ellipsis_args'], + implicit=data['implicit'], + is_classmethod_class=data['is_classmethod_class'], + ) + + +class Overloaded(FunctionLike): + """Overloaded function type T1, ... Tn, where each Ti is CallableType. + + The variant to call is chosen based on static argument + types. Overloaded function types can only be defined in stub + files, and thus there is no explicit runtime dispatch + implementation. + """ + + _items = None # type: List[CallableType] # Must not be empty + + def __init__(self, items: List[CallableType]) -> None: + self._items = items + self.fallback = items[0].fallback + super().__init__(items[0].line) + + def items(self) -> List[CallableType]: + return self._items + + def name(self) -> str: + return self._items[0].name + + def is_type_obj(self) -> bool: + # All the items must have the same type object status, so it's + # sufficient to query only (any) one of them. + return self._items[0].is_type_obj() + + def type_object(self) -> mypy.nodes.TypeInfo: + # All the items must have the same type object, so it's sufficient to + # query only (any) one of them. + return self._items[0].type_object() + + def with_name(self, name: str) -> 'Overloaded': + ni = [] # type: List[CallableType] + for it in self._items: + ni.append(it.with_name(name)) + return Overloaded(ni) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_overloaded(self) + + def serialize(self) -> JsonDict: + return {'.class': 'Overloaded', + 'items': [t.serialize() for t in self.items()], + } + + @classmethod + def deserialize(self, data: JsonDict) -> 'Overloaded': + assert data['.class'] == 'Overloaded' + return Overloaded([CallableType.deserialize(t) for t in data['items']]) + + +class TupleType(Type): + """The tuple type Tuple[T1, ..., Tn] (at least one type argument). + + Instance variables: + items: tuple item types + fallback: the underlying instance type that is used for non-tuple methods + (this is currently always builtins.tuple, but it could be different for named + tuples, for example) + implicit: if True, derived from a tuple expression (t,....) instead of Tuple[t, ...] + """ + + items = None # type: List[Type] + fallback = None # type: Instance + implicit = False + + def __init__(self, items: List[Type], fallback: Instance, line: int = -1, + implicit: bool = False) -> None: + #import pdb; pdb.set_trace() + self.items = items + self.fallback = fallback + self.implicit = implicit + super().__init__(line) + + def length(self) -> int: + return len(self.items) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_tuple_type(self) + + def serialize(self) -> JsonDict: + return {'.class': 'TupleType', + 'items': [t.serialize() for t in self.items], + 'fallback': self.fallback.serialize(), + 'implicit': self.implicit, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'TupleType': + assert data['.class'] == 'TupleType' + return TupleType([Type.deserialize(t) for t in data['items']], + Instance.deserialize(data['fallback']), + implicit=data['implicit']) + + def self_type(self, inst: Instance) -> 'TupleType': + return TupleType(self.items, inst, self.line, self.implicit) + + def with_types(self, items: List[Type]) -> 'TupleType': + return TupleType(items, self.fallback, self.line, self.implicit) + + def slice(self, begin: int, stride: int, end: int) -> 'TupleType': + return TupleType(self.items[begin:end:stride], self.fallback, + self.line, self.implicit) + + +class NamedTupleType(TupleType): + attrs = None # type: List[str] + name = None # type: str + + def __init__(self, name: str, attrs: List[str], *args, **kwargs) -> None: + self.attrs = attrs + self.name = name + super().__init__(*args, **kwargs) + + def self_type(self, inst: Instance) -> 'NamedTupleType': + return NamedTupleType(self.name, self.attrs, self.items, inst, self.line) + + def with_types(self, items: List[Type]) -> 'NamedTupleType': + return NamedTupleType(self.name, self.attrs, items, + self.fallback, self.line, self.implicit) + + def __eq__(self, other) -> bool: + return self.serialize() == other.serialize() + + def serialize(self) -> JsonDict: + d = super().serialize() + d.update({'.class': 'NamedTupleType', + 'name': self.name, + 'attrs': self.attrs}) + return d + + @classmethod + def deserialize(cls, data: JsonDict) -> 'NamedTupleType': + assert data['.class'] == 'NamedTupleType' + data['.class'] = 'TupleType' + tup = TupleType.deserialize(data) + return NamedTupleType(data['name'], + data['attrs'], + tup.items, + tup.fallback, + implicit=tup.implicit) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + try: + return visitor.visit_namedtuple_type(self) + except: + return super().accept(visitor) + + +class StarType(Type): + """The star type *type_parameter. + + This is not a real type but a syntactic AST construct. + """ + + type = None # type: Type + + def __init__(self, type: Type, line: int = -1) -> None: + self.type = type + super().__init__(line) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_star_type(self) + + +class UnionType(Type): + """The union type Union[T1, ..., Tn] (at least one type argument).""" + + items = None # type: List[Type] + + def __init__(self, items: List[Type], line: int = -1) -> None: + self.items = items + super().__init__(line) + + @staticmethod + def make_union(items: List[Type], line: int = -1) -> Type: + if len(items) > 1: + return UnionType(items, line) + elif len(items) == 1: + return items[0] + else: + if experiments.STRICT_OPTIONAL: + return UninhabitedType() + else: + return Void() + + @staticmethod + def make_simplified_union(items: List[Type], line: int = -1) -> Type: + while any(isinstance(typ, UnionType) for typ in items): + all_items = [] # type: List[Type] + for typ in items: + if isinstance(typ, UnionType): + all_items.extend(typ.items) + else: + all_items.append(typ) + items = all_items + + if any(isinstance(typ, AnyType) for typ in items): + return AnyType() + + from mypy.subtypes import is_subtype + removed = set() # type: Set[int] + for i in range(len(items)): + if any(is_subtype(items[i], items[j]) for j in range(len(items)) + if j not in removed and j != i): + removed.add(i) + + simplified_set = [items[i] for i in range(len(items)) if i not in removed] + return UnionType.make_union(simplified_set) + + def length(self) -> int: + return len(self.items) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_union_type(self) + + def has_readable_member(self, name: str) -> bool: + """For a tree of unions of instances, check whether all instances have a given member. + + TODO: Deal with attributes of TupleType etc. + TODO: This should probably be refactored to go elsewhere. + """ + return all((isinstance(x, UnionType) and x.has_readable_member(name)) or + (isinstance(x, Instance) and x.type.has_readable_member(name)) + for x in self.items) + + def serialize(self) -> JsonDict: + return {'.class': 'UnionType', + 'items': [t.serialize() for t in self.items], + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'UnionType': + assert data['.class'] == 'UnionType' + return UnionType([Type.deserialize(t) for t in data['items']]) + + +class PartialType(Type): + """Type such as List[?] where type arguments are unknown, or partial None type. + + These are used for inferring types in multiphase initialization such as this: + + x = [] # x gets a partial type List[?], as item type is unknown + x.append(1) # partial type gets replaced with normal type List[int] + + Or with None: + + x = None # x gets a partial type None + if c: + x = 1 # Infer actual type int for x + """ + + # None for the 'None' partial type; otherwise a generic class + type = None # type: Optional[mypy.nodes.TypeInfo] + var = None # type: mypy.nodes.Var + inner_types = None # type: List[Type] + + def __init__(self, + type: Optional['mypy.nodes.TypeInfo'], + var: 'mypy.nodes.Var', + inner_types: List[Type]) -> None: + self.type = type + self.var = var + self.inner_types = inner_types + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_partial_type(self) + + +class EllipsisType(Type): + """The type ... (ellipsis). + + This is not a real type but a syntactic AST construct, used in Callable[..., T], for example. + + A semantically analyzed type will never have ellipsis types. + """ + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_ellipsis_type(self) + + def serialize(self) -> JsonDict: + return {'.class': 'EllipsisType'} + + @classmethod + def deserialize(self, data: JsonDict) -> 'EllipsisType': + assert data['.class'] == 'EllipsisType' + return EllipsisType() + + +class TypeType(Type): + """For types like Type[User]. + + This annotates variables that are class objects, constrained by + the type argument. See PEP 484 for more details. + + We may encounter expressions whose values are specific classes; + those are represented as callables (possibly overloaded) + corresponding to the class's constructor's signature and returning + an instance of that class. The difference with Type[C] is that + those callables always represent the exact class given as the + return type; Type[C] represents any class that's a subclass of C, + and C may also be a type variable or a union (or Any). + + Many questions around subtype relationships between Type[C1] and + def(...) -> C2 are answered by looking at the subtype + relationships between C1 and C2, since Type[] is considered + covariant. + + There's an unsolved problem with constructor signatures (also + unsolved in PEP 484): calling a variable whose type is Type[C] + assumes the constructor signature for C, even though a subclass of + C might completely change the constructor signature. For now we + just assume that users of Type[C] are careful not to do that (in + the future we might detect when they are violating that + assumption). + """ + + # This can't be everything, but it can be a class reference, + # a generic class instance, a union, Any, a type variable... + item = None # type: Type + + def __init__(self, item: Type, *, line: int = -1) -> None: + super().__init__(line) + self.item = item + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_type_type(self) + + def serialize(self) -> JsonDict: + return {'.class': 'TypeType', 'item': self.item.serialize()} + + def deserialize(self, data: JsonDict) -> 'TypeType': + assert data['.class'] == 'TypeType' + return TypeType(Type.deserialize(data['item'])) + + +# +# Visitor-related classes +# + + +class TypeVisitor(Generic[T]): + """Visitor class for types (Type subclasses). + + The parameter T is the return type of the visit methods. + """ + + def _notimplemented_helper(self, name: str) -> NotImplementedError: + return NotImplementedError("Method {}.visit_{}() not implemented\n" + .format(type(self).__name__, name) + + "This is a known bug, track development in " + + "'https://github.com/JukkaL/mypy/issues/730'") + + @abstractmethod + def visit_unbound_type(self, t: UnboundType) -> T: + pass + + def visit_type_list(self, t: TypeList) -> T: + raise self._notimplemented_helper('type_list') + + def visit_error_type(self, t: ErrorType) -> T: + raise self._notimplemented_helper('error_type') + + @abstractmethod + def visit_any(self, t: AnyType) -> T: + pass + + @abstractmethod + def visit_void(self, t: Void) -> T: + pass + + @abstractmethod + def visit_none_type(self, t: NoneTyp) -> T: + pass + + @abstractmethod + def visit_uninhabited_type(self, t: UninhabitedType) -> T: + pass + + def visit_erased_type(self, t: ErasedType) -> T: + raise self._notimplemented_helper('erased_type') + + @abstractmethod + def visit_deleted_type(self, t: DeletedType) -> T: + pass + + @abstractmethod + def visit_type_var(self, t: TypeVarType) -> T: + pass + + @abstractmethod + def visit_instance(self, t: Instance) -> T: + pass + + @abstractmethod + def visit_callable_type(self, t: CallableType) -> T: + pass + + def visit_overloaded(self, t: Overloaded) -> T: + raise self._notimplemented_helper('overloaded') + + @abstractmethod + def visit_tuple_type(self, t: TupleType) -> T: + pass + + def visit_star_type(self, t: StarType) -> T: + raise self._notimplemented_helper('star_type') + + @abstractmethod + def visit_union_type(self, t: UnionType) -> T: + pass + + @abstractmethod + def visit_partial_type(self, t: PartialType) -> T: + pass + + def visit_ellipsis_type(self, t: EllipsisType) -> T: + raise self._notimplemented_helper('ellipsis_type') + + @abstractmethod + def visit_type_type(self, t: TypeType) -> T: + pass + + +class TypeTranslator(TypeVisitor[Type]): + """Identity type transformation. + + Subclass this and override some methods to implement a non-trivial + transformation. + """ + + def visit_unbound_type(self, t: UnboundType) -> Type: + return t + + def visit_type_list(self, t: TypeList) -> Type: + return t + + def visit_error_type(self, t: ErrorType) -> Type: + return t + + def visit_any(self, t: AnyType) -> Type: + return t + + def visit_void(self, t: Void) -> Type: + return t + + def visit_none_type(self, t: NoneTyp) -> Type: + return t + + def visit_uninhabited_type(self, t: UninhabitedType) -> Type: + return t + + def visit_erased_type(self, t: ErasedType) -> Type: + return t + + def visit_deleted_type(self, t: DeletedType) -> Type: + return t + + def visit_instance(self, t: Instance) -> Type: + return Instance(t.type, self.translate_types(t.args), t.line) + + def visit_type_var(self, t: TypeVarType) -> Type: + return t + + def visit_partial_type(self, t: PartialType) -> Type: + return t + + def visit_callable_type(self, t: CallableType) -> Type: + return t.copy_modified(arg_types=self.translate_types(t.arg_types), + ret_type=t.ret_type.accept(self), + variables=self.translate_variables(t.variables)) + + def visit_tuple_type(self, t: TupleType) -> Type: + return TupleType(self.translate_types(t.items), + cast(Any, t.fallback.accept(self)), + t.line) + + def visit_star_type(self, t: StarType) -> Type: + return StarType(t.type.accept(self), t.line) + + def visit_union_type(self, t: UnionType) -> Type: + return UnionType(self.translate_types(t.items), t.line) + + def visit_ellipsis_type(self, t: EllipsisType) -> Type: + return t + + def translate_types(self, types: List[Type]) -> List[Type]: + return [t.accept(self) for t in types] + + def translate_variables(self, + variables: List[TypeVarDef]) -> List[TypeVarDef]: + return variables + + def visit_overloaded(self, t: Overloaded) -> Type: + items = [] # type: List[CallableType] + for item in t.items(): + new = item.accept(self) + if isinstance(new, CallableType): + items.append(new) + else: + raise RuntimeError('CallableType expectected, but got {}'.format(type(new))) + return Overloaded(items=items) + + def visit_type_type(self, t: TypeType) -> Type: + return TypeType(t.item.accept(self), line=t.line) + + +class TypeStrVisitor(TypeVisitor[str]): + """Visitor for pretty-printing types into strings. + + This is mostly for debugging/testing. + + Do not preserve original formatting. + + Notes: + - Represent unbound types as Foo? or Foo?[...]. + - Represent the NoneTyp type as None. + """ + + def visit_unbound_type(self, t): + s = t.name + '?' + if t.args != []: + s += '[{}]'.format(self.list_str(t.args)) + return s + + def visit_type_list(self, t): + return ''.format(self.list_str(t.items)) + + def visit_error_type(self, t): + return '' + + def visit_any(self, t): + return 'Any' + + def visit_void(self, t): + return 'void' + + def visit_none_type(self, t): + # Fully qualify to make this distinct from the None value. + return "builtins.None" + + def visit_uninhabited_type(self, t): + return "" + + def visit_erased_type(self, t): + return "" + + def visit_deleted_type(self, t): + if t.source is None: + return "" + else: + return "".format(t.source) + + def visit_instance(self, t): + s = t.type.fullname() if t.type is not None else '' + if t.erased: + s += '*' + if t.args != []: + s += '[{}]'.format(self.list_str(t.args)) + return s + + def visit_type_var(self, t): + if t.name is None: + # Anonymous type variable type (only numeric id). + return '`{}'.format(t.id) + else: + # Named type variable type. + return '{}`{}'.format(t.name, t.id) + + def visit_callable_type(self, t): + s = '' + bare_asterisk = False + for i in range(len(t.arg_types)): + if s != '': + s += ', ' + if t.arg_kinds[i] == mypy.nodes.ARG_NAMED and not bare_asterisk: + s += '*, ' + bare_asterisk = True + if t.arg_kinds[i] == mypy.nodes.ARG_STAR: + s += '*' + if t.arg_kinds[i] == mypy.nodes.ARG_STAR2: + s += '**' + if t.arg_names[i]: + s += t.arg_names[i] + ': ' + s += str(t.arg_types[i]) + if t.arg_kinds[i] == mypy.nodes.ARG_OPT: + s += ' =' + + s = '({})'.format(s) + + if not isinstance(t.ret_type, Void): + s += ' -> {}'.format(t.ret_type) + + if t.variables: + s = '{} {}'.format(t.variables, s) + + return 'def {}'.format(s) + + def visit_overloaded(self, t): + a = [] + for i in t.items(): + a.append(i.accept(self)) + return 'Overload({})'.format(', '.join(a)) + + def visit_tuple_type(self, t): + s = self.list_str(t.items) + if t.fallback and t.fallback.type: + fallback_name = t.fallback.type.fullname() + if fallback_name != 'builtins.tuple': + return 'Tuple[{}, fallback={}]'.format(s, t.fallback.accept(self)) + return 'Tuple[{}]'.format(s) + + def visit_namedtuple_type(self, t): + s = self.list_str(zip(t.attrs, t.items)) + return '{}[{}]'.format(t.name, s) + + def visit_star_type(self, t): + s = t.type.accept(self) + return '*{}'.format(s) + + def visit_union_type(self, t): + s = self.list_str(t.items) + return 'Union[{}]'.format(s) + + def visit_partial_type(self, t: PartialType) -> str: + if t.type is None: + return '' + else: + return ''.format(t.type.name(), + ', '.join(['?'] * len(t.type.type_vars))) + + def visit_ellipsis_type(self, t): + return '...' + + def visit_type_type(self, t): + return 'Type[{}]'.format(t.item.accept(self)) + + def list_str(self, a): + """Convert items of an array to strings (pretty-print types) + and join the results with commas. + """ + res = [] + for t in a: + if isinstance(t, Type): + res.append(t.accept(self)) + else: + res.append(str(t)) + return ', '.join(res) + + +# These constants define the method used by TypeQuery to combine multiple +# query results, e.g. for tuple types. The strategy is not used for empty +# result lists; in that case the default value takes precedence. +ANY_TYPE_STRATEGY = 0 # Return True if any of the results are True. +ALL_TYPES_STRATEGY = 1 # Return True if all of the results are True. + + +class TypeQuery(TypeVisitor[bool]): + """Visitor for performing simple boolean queries of types. + + This class allows defining the default value for leafs to simplify the + implementation of many queries. + """ + + default = False # Default result + strategy = 0 # Strategy for combining multiple values (ANY_TYPE_STRATEGY or ALL_TYPES_...). + + def __init__(self, default: bool, strategy: int) -> None: + """Construct a query visitor. + + Use the given default result and strategy for combining + multiple results. The strategy must be either + ANY_TYPE_STRATEGY or ALL_TYPES_STRATEGY. + """ + self.default = default + self.strategy = strategy + + def visit_unbound_type(self, t: UnboundType) -> bool: + return self.default + + def visit_type_list(self, t: TypeList) -> bool: + return self.default + + def visit_error_type(self, t: ErrorType) -> bool: + return self.default + + def visit_any(self, t: AnyType) -> bool: + return self.default + + def visit_void(self, t: Void) -> bool: + return self.default + + def visit_uninhabited_type(self, t: UninhabitedType) -> bool: + return self.default + + def visit_none_type(self, t: NoneTyp) -> bool: + return self.default + + def visit_erased_type(self, t: ErasedType) -> bool: + return self.default + + def visit_deleted_type(self, t: DeletedType) -> bool: + return self.default + + def visit_type_var(self, t: TypeVarType) -> bool: + return self.default + + def visit_partial_type(self, t: PartialType) -> bool: + return self.default + + def visit_instance(self, t: Instance) -> bool: + return self.query_types(t.args) + + def visit_callable_type(self, t: CallableType) -> bool: + # FIX generics + return self.query_types(t.arg_types + [t.ret_type]) + + def visit_tuple_type(self, t: TupleType) -> bool: + return self.query_types(t.items) + + def visit_star_type(self, t: StarType) -> bool: + return t.type.accept(self) + + def visit_union_type(self, t: UnionType) -> bool: + return self.query_types(t.items) + + def visit_overloaded(self, t: Overloaded) -> bool: + return self.query_types(t.items()) + + def visit_type_type(self, t: TypeType) -> bool: + return t.item.accept(self) + + def query_types(self, types: Sequence[Type]) -> bool: + """Perform a query for a list of types. + + Use the strategy constant to combine the results. + """ + if not types: + # Use default result for empty list. + return self.default + if self.strategy == ANY_TYPE_STRATEGY: + # Return True if at least one component is true. + res = False + for t in types: + res = res or t.accept(self) + if res: + break + return res + else: + # Return True if all components are true. + res = True + for t in types: + res = res and t.accept(self) + if not res: + break + return res + + +def strip_type(typ: Type) -> Type: + """Make a copy of type without 'debugging info' (function name).""" + + if isinstance(typ, CallableType): + return typ.copy_modified(name=None) + elif isinstance(typ, Overloaded): + return Overloaded([cast(CallableType, strip_type(item)) + for item in typ.items()]) + else: + return typ + + +def replace_leading_arg_type(t: CallableType, self_type: Type) -> CallableType: + """Return a copy of a callable type with a different self argument type. + + Assume that the callable is the signature of a method. + """ + return t.copy_modified(arg_types=[self_type] + t.arg_types[1:]) + + +def is_named_instance(t: Type, fullname: str) -> bool: + return (isinstance(t, Instance) and + t.type is not None and + t.type.fullname() == fullname) diff --git a/test-data/unit/check-lists.test b/test-data/unit/check-lists.test index 032e02314f5c4..a6df5ae740c61 100644 --- a/test-data/unit/check-lists.test +++ b/test-data/unit/check-lists.test @@ -1,66 +1,63 @@ --- Nested list assignment --- ----------------------------- -[case testListSimplest] -l = [1, 1] - - -[case testNestedListAssignment] -from typing import List -a1, b1, c1 = None, None, None # type: (A, B, C) -a2, b2, c2 = None, None, None # type: (A, B, C) - -a1, [b1, c1] = a2, [b2, c2] -a1, [a1, [b1, c1]] = a2, [a2, [b2, c2]] -a1, [a1, [a1, b1]] = a1, [a1, [a1, c1]] # E: Incompatible types in assignment (expression has type "C", variable has type "B") - -class A: pass -class B: pass -class C: pass -[builtins fixtures/list.py] -[out] - -[case testNestedListAssignmentToTuple] -from typing import List -a, b, c = None, None, None # type: (A, B, C) - -a, b = [a, b] -a, b = [a] # E: Need more than 1 value to unpack (2 expected) -a, b = [a, b, c] # E: Too many values to unpack (2 expected, 3 provided) - -class A: pass -class B: pass -class C: pass -[builtins fixtures/list.py] -[out] - -[case testListAssignmentFromTuple] -from typing import List -a, b, c = None, None, None # type: (A, B, C) -t = a, b - -[a, b], c = t, c -[a, c], c = t, c # E: Incompatible types in assignment (expression has type "B", variable has type "C") -[a, a, a], c = t, c # E: Need more than 2 values to unpack (3 expected) -[a], c = t, c # E: Too many values to unpack (1 expected, 2 provided) - -class A: pass -class B: pass -class C: pass -[builtins fixtures/list.py] -[out] - -[case testListAssignmentUnequalAmountToUnpack] -from typing import List -a, b, c = None, None, None # type: (A, B, C) - -def f() -> None: # needed because test parser tries to parse [a, b] as section header - [a, b] = [a, b] - [a, b] = [a] # E: Need more than 1 value to unpack (2 expected) - [a, b] = [a, b, c] # E: Too many values to unpack (2 expected, 3 provided) - -class A: pass -class B: pass -class C: pass -[builtins fixtures/list.py] -[out] -main: note: In function "f": +-- Nested list assignment +-- ----------------------------- + +[case testNestedListAssignment] +from typing import List +a1, b1, c1 = None, None, None # type: (A, B, C) +a2, b2, c2 = None, None, None # type: (A, B, C) + +a1, [b1, c1] = a2, [b2, c2] +a1, [a1, [b1, c1]] = a2, [a2, [b2, c2]] +a1, [a1, [a1, b1]] = a1, [a1, [a1, c1]] # E: Incompatible types in assignment (expression has type "C", variable has type "B") + +class A: pass +class B: pass +class C: pass +[builtins fixtures/list.py] +[out] + +[case testNestedListAssignmentToTuple] +from typing import List +a, b, c = None, None, None # type: (A, B, C) + +a, b = [a, b] +a, b = [a] # E: Need more than 1 value to unpack (2 expected) +a, b = [a, b, c] # E: Too many values to unpack (2 expected, 3 provided) + +class A: pass +class B: pass +class C: pass +[builtins fixtures/list.py] +[out] + +[case testListAssignmentFromTuple] +from typing import List +a, b, c = None, None, None # type: (A, B, C) +t = a, b + +[a, b], c = t, c +[a, c], c = t, c # E: Incompatible types in assignment (expression has type "B", variable has type "C") +[a, a, a], c = t, c # E: Need more than 2 values to unpack (3 expected) +[a], c = t, c # E: Too many values to unpack (1 expected, 2 provided) + +class A: pass +class B: pass +class C: pass +[builtins fixtures/list.py] +[out] + +[case testListAssignmentUnequalAmountToUnpack] +from typing import List +a, b, c = None, None, None # type: (A, B, C) + +def f() -> None: # needed because test parser tries to parse [a, b] as section header + [a, b] = [a, b] + [a, b] = [a] # E: Need more than 1 value to unpack (2 expected) + [a, b] = [a, b, c] # E: Too many values to unpack (2 expected, 3 provided) + +class A: pass +class B: pass +class C: pass +[builtins fixtures/list.py] +[out] +main: note: In function "f": diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index adb07ec522705..86c19956ca634 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -1,223 +1,223 @@ -[case testNamedTupleDeclaration] -from typing import NamedTuple - -A = NamedTuple('A', [('x', str), ('y', int)]) - -[case testNamedTupleUsedAsTuple] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -a, b = x -b = x[0] -a = x[1] -a, b, c = x # E: Need more than 2 values to unpack (3 expected) -x[2] # E: Tuple index out of range - -[case testAccessingNamedTupleAttributes] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -x.x -x.y -x.z # E: "X" has no attribute "z" - -[case testCreateNamedTupleWithPositionalArguments] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = X(1, 'x') -x.x -x.z # E: "X" has no attribute "z" -x = X(1) # E: Too few arguments for "X" -x = X(1, 2, 3) # E: Too many arguments for "X" - -[case testCreateNamedTupleWithKeywordArguments] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = X(x=1, y='x') -x = X(1, y='x') -x = X(x=1, z=1) # E: Unexpected keyword argument "z" for "X" -x = X(y=1) # E: Missing positional argument "x" in call to "X" - -[case testCreateNamedTupleAndUseAsTuple] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = X(1, 'x') -a, b = x -a, b, c = x # E: Need more than 2 values to unpack (3 expected) - -[case testNamedTupleWithItemTypes] -from typing import NamedTuple -N = NamedTuple('N', [('a', int), - ('b', str)]) -n = N(1, 'x') -s = n.a # type: str # E: Incompatible types in assignment (expression has type "int", \ - variable has type "str") -i = n.b # type: int # E: Incompatible types in assignment (expression has type "str", \ - variable has type "int") -x, y = n -x = y # E: Incompatible types in assignment (expression has type "str", variable has type "int") - -[case testNamedTupleConstructorArgumentTypes] -from typing import NamedTuple -N = NamedTuple('N', [('a', int), - ('b', str)]) -n = N('x', 'x') # E: Argument 1 to "N" has incompatible type "str"; expected "int" -n = N(1, b=2) # E: Argument 2 to "N" has incompatible type "int"; expected "str" -N(1, 'x') -N(b='x', a=1) - -[case testNamedTupleAsBaseClass] -from typing import NamedTuple -N = NamedTuple('N', [('a', int), - ('b', str)]) -class X(N): - pass -x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" -s = '' -i = 0 -s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") -i, s = x -s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") - -[case testNamedTupleAsBaseClass2] -from typing import NamedTuple -class X(NamedTuple('N', [('a', int), - ('b', str)])): - pass -x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" -s = '' -i = 0 -s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") -i, s = x -s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") - -[case testTwoNamedTuplesAsBaseClasses] -from typing import NamedTuple -A = NamedTuple('A', [('a', int)]) -B = NamedTuple('B', [('a', int)]) -class X(A, B): # E: Class has two incompatible bases derived from tuple - pass - -[case testTwoNamedTuplesAsBaseClasses2] -from typing import NamedTuple -A = NamedTuple('A', [('a', int)]) -class X(A, NamedTuple('B', [('a', int)])): # E: Class has two incompatible bases derived from tuple - pass - -[case testSelfTypeWithNamedTupleAsBase] -from typing import NamedTuple -A = NamedTuple('A', [('a', int), ('b', str)]) -class B(A): - def f(self, x: int) -> None: - self.f(self.a) - self.f(self.b) # E: Argument 1 to "f" of "B" has incompatible type "str"; expected "int" - i = 0 - s = '' - i, s = self - i, i = self # E: Incompatible types in assignment (expression has type "str", \ - variable has type "int") -[out] -main: note: In member "f" of class "B": - -[case testTypeReferenceToClassDerivedFromNamedTuple] -from typing import NamedTuple -A = NamedTuple('A', [('a', int), ('b', str)]) -class B(A): - def f(self, x: 'B') -> None: - i = 0 - s = '' - self = x - i, s = x - i, s = x.a, x.b - i, s = x.a, x.a # E: Incompatible types in assignment (expression has type "int", \ - variable has type "str") - i, i = self # E: Incompatible types in assignment (expression has type "str", \ - variable has type "int") -[out] -main: note: In member "f" of class "B": - -[case testNamedTupleSubtyping] -from typing import NamedTuple, Tuple -A = NamedTuple('A', [('a', int), ('b', str)]) -class B(A): pass -a = A(1, '') -b = B(1, '') -t = None # type: Tuple[int, str] -b = a # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "A") -b = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "B") -t = a -t = (1, '') -t = b -a = b - -[case testNamedTupleSimpleTypeInference] -from typing import NamedTuple, Tuple -A = NamedTuple('A', [('a', int)]) -l = [A(1), A(2)] -a = A(1) -a = l[0] -(i,) = l[0] -i, i = l[0] # E: Need more than 1 value to unpack (2 expected) -a = (1,) # E: Incompatible types in assignment (expression has type "Tuple[int]", \ - variable has type "A") -[builtins fixtures/list.py] - -[case testNamedTupleMissingClassAttribute] -import collections -MyNamedTuple = collections.namedtuple('MyNamedTuple', ['spam', 'eggs']) -MyNamedTuple.x # E: "MyNamedTuple" has no attribute "x" - -[case testNamedTupleEmptyItems] -from typing import NamedTuple -A = NamedTuple('A', []) -[builtins fixtures/list.py] - -[case testNamedTupleProperty] -from typing import NamedTuple -A = NamedTuple('A', [('a', int)]) -class B(A): - @property - def b(self) -> int: - return self.a -class C(B): pass -B(1).b -C(2).b -[builtins fixtures/property.py] - -[case testNamedTupleAsDict] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -a = x._asdict() - -[case testNamedTupleReplace] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -x = x._replace(y=5) -x = x._replace(z=5) # E: Unexpected keyword argument "z" for X._replace -x = x._replace(5) # E: Too many positional arguments for X._replace - -[case testNamedTupleReplaceAsClass] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -X._replace(x, x=1, y=2) -X._replace(x=1, y=2) # E: Missing positional argument "self" in call to X._replace - -[case testNamedTupleReplaceTyped] -from typing import NamedTuple - -X = NamedTuple('X', [('x', int), ('y', str)]) -x = None # type: X -x = x._replace(x=5) -x = x._replace(y=5) # E: Argument 1 to X._replace has incompatible type "int"; expected "str" +[case testNamedTupleDeclaration] +from typing import NamedTuple + +A = NamedTuple('A', [('x', str), ('y', int)]) + +[case testNamedTupleUsedAsTuple] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +a, b = x +b = x[0] +a = x[1] +a, b, c = x # E: Need more than 2 values to unpack (3 expected) +x[2] # E: Tuple index out of range + +[case testAccessingNamedTupleAttributes] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +x.x +x.y +x.z # E: "X" has no attribute "z" + +[case testCreateNamedTupleWithPositionalArguments] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = X(1, 'x') +x.x +x.z # E: "X" has no attribute "z" +x = X(1) # E: Too few arguments for "X" +x = X(1, 2, 3) # E: Too many arguments for "X" + +[case testCreateNamedTupleWithKeywordArguments] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = X(x=1, y='x') +x = X(1, y='x') +x = X(x=1, z=1) # E: Unexpected keyword argument "z" for "X" +x = X(y=1) # E: Missing positional argument "x" in call to "X" + +[case testCreateNamedTupleAndUseAsTuple] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = X(1, 'x') +a, b = x +a, b, c = x # E: Need more than 2 values to unpack (3 expected) + +[case testNamedTupleWithItemTypes] +from typing import NamedTuple +N = NamedTuple('N', [('a', int), + ('b', str)]) +n = N(1, 'x') +s = n.a # type: str # E: Incompatible types in assignment (expression has type "int", \ + variable has type "str") +i = n.b # type: int # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +x, y = n +x = y # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testNamedTupleConstructorArgumentTypes] +from typing import NamedTuple +N = NamedTuple('N', [('a', int), + ('b', str)]) +n = N('x', 'x') # E: Argument 1 to "N" has incompatible type "str"; expected "int" +n = N(1, b=2) # E: Argument 2 to "N" has incompatible type "int"; expected "str" +N(1, 'x') +N(b='x', a=1) + +[case testNamedTupleAsBaseClass] +from typing import NamedTuple +N = NamedTuple('N', [('a', int), + ('b', str)]) +class X(N): + pass +x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" +s = '' +i = 0 +s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") +i, s = x +s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testNamedTupleAsBaseClass2] +from typing import NamedTuple +class X(NamedTuple('N', [('a', int), + ('b', str)])): + pass +x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" +s = '' +i = 0 +s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") +i, s = x +s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testTwoNamedTuplesAsBaseClasses] +from typing import NamedTuple +A = NamedTuple('A', [('a', int)]) +B = NamedTuple('B', [('a', int)]) +class X(A, B): # E: Class has two incompatible bases derived from tuple + pass + +[case testTwoNamedTuplesAsBaseClasses2] +from typing import NamedTuple +A = NamedTuple('A', [('a', int)]) +class X(A, NamedTuple('B', [('a', int)])): # E: Class has two incompatible bases derived from tuple + pass + +[case testSelfTypeWithNamedTupleAsBase] +from typing import NamedTuple +A = NamedTuple('A', [('a', int), ('b', str)]) +class B(A): + def f(self, x: int) -> None: + self.f(self.a) + self.f(self.b) # E: Argument 1 to "f" of "B" has incompatible type "str"; expected "int" + i = 0 + s = '' + i, s = self + i, i = self # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +[out] +main: note: In member "f" of class "B": + +[case testTypeReferenceToClassDerivedFromNamedTuple] +from typing import NamedTuple +A = NamedTuple('A', [('a', int), ('b', str)]) +class B(A): + def f(self, x: 'B') -> None: + i = 0 + s = '' + self = x + i, s = x + i, s = x.a, x.b + i, s = x.a, x.a # E: Incompatible types in assignment (expression has type "int", \ + variable has type "str") + i, i = self # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +[out] +main: note: In member "f" of class "B": + +[case testNamedTupleSubtyping] +from typing import NamedTuple, Tuple +A = NamedTuple('A', [('a', int), ('b', str)]) +class B(A): pass +a = A(1, '') +b = B(1, '') +t = None # type: Tuple[int, str] +b = a # E: Incompatible types in assignment (expression has type "A", variable has type "B") +a = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "A") +b = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "B") +t = a +t = (1, '') +t = b +a = b + +[case testNamedTupleSimpleTypeInference] +from typing import NamedTuple, Tuple +A = NamedTuple('A', [('a', int)]) +l = [A(1), A(2)] +a = A(1) +a = l[0] +(i,) = l[0] +i, i = l[0] # E: Need more than 1 value to unpack (2 expected) +a = (1,) # E: Incompatible types in assignment (expression has type "Tuple[int]", \ + variable has type "A") +[builtins fixtures/list.py] + +[case testNamedTupleMissingClassAttribute] +import collections +MyNamedTuple = collections.namedtuple('MyNamedTuple', ['spam', 'eggs']) +MyNamedTuple.x # E: "MyNamedTuple" has no attribute "x" + +[case testNamedTupleEmptyItems] +from typing import NamedTuple +A = NamedTuple('A', []) +[builtins fixtures/list.py] + +[case testNamedTupleProperty] +from typing import NamedTuple +A = NamedTuple('A', [('a', int)]) +class B(A): + @property + def b(self) -> int: + return self.a +class C(B): pass +B(1).b +C(2).b +[builtins fixtures/property.py] + +[case testNamedTupleAsDict] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +a = x._asdict() + +[case testNamedTupleReplace] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +x = x._replace(y=5) +x = x._replace(z=5) # E: Unexpected keyword argument "z" for X._replace +x = x._replace(5) # E: Too many positional arguments for X._replace + +[case testNamedTupleReplaceAsClass] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +X._replace(x, x=1, y=2) +X._replace(x=1, y=2) # E: Missing positional argument "self" in call to X._replace + +[case testNamedTupleReplaceTyped] +from typing import NamedTuple + +X = NamedTuple('X', [('x', int), ('y', str)]) +x = None # type: X +x = x._replace(x=5) +x = x._replace(y=5) # E: Argument 1 to X._replace has incompatible type "int"; expected "str" diff --git a/test-data/unit/check-tuples.test b/test-data/unit/check-tuples.test index 50d1b3e64598d..f946c451123fd 100644 --- a/test-data/unit/check-tuples.test +++ b/test-data/unit/check-tuples.test @@ -1,770 +1,770 @@ --- Normal assignment and subtyping --- ------------------------------- - - -[case testTupleAssignmentWithTupleTypes] -from typing import Tuple -t1 = None # type: Tuple[A] -t2 = None # type: Tuple[B] -t3 = None # type: Tuple[A, A] -t4 = None # type: Tuple[A, B] -t5 = None # type: Tuple[B, A] - -t1 = t2 # E: Incompatible types in assignment (expression has type "Tuple[B]", variable has type "Tuple[A]") -t1 = t3 # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A]") -t3 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A]", variable has type "Tuple[A, A]") -t3 = t4 # E: Incompatible types in assignment (expression has type "Tuple[A, B]", variable has type "Tuple[A, A]") -t3 = t5 # E: Incompatible types in assignment (expression has type "Tuple[B, A]", variable has type "Tuple[A, A]") - -# Ok -t1 = t1 -t2 = t2 -t3 = t3 -t4 = t4 -t5 = t5 - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - -[case testTupleSubtyping] -from typing import Tuple -t1 = None # type: Tuple[A, A] -t2 = None # type: Tuple[A, B] -t3 = None # type: Tuple[B, A] - -t2 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A, B]") -t2 = t3 # E: Incompatible types in assignment (expression has type "Tuple[B, A]", variable has type "Tuple[A, B]") -t3 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[B, A]") -t3 = t2 # E: Incompatible types in assignment (expression has type "Tuple[A, B]", variable has type "Tuple[B, A]") - -t1 = t2 -t1 = t3 - -class A: pass -class B(A): pass -[builtins fixtures/tuple.py] - -[case testTupleCompatibilityWithOtherTypes] -from typing import Tuple -a, o = None, None # type: (A, object) -t = None # type: Tuple[A, A] - -a = t # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "A") -t = o # E: Incompatible types in assignment (expression has type "object", variable has type "Tuple[A, A]") -t = a # E: Incompatible types in assignment (expression has type "A", variable has type "Tuple[A, A]") -# TODO: callable types + tuples - -# Ok -o = t -t = None - -class A: pass -[builtins fixtures/tuple.py] - -[case testNestedTupleTypes] -from typing import Tuple -t1 = None # type: Tuple[A, Tuple[A, A]] -t2 = None # type: Tuple[B, Tuple[B, B]] - -t2 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, Tuple[A, A]]", variable has type "Tuple[B, Tuple[B, B]]") -t1 = t2 - -class A: pass -class B(A): pass -[builtins fixtures/tuple.py] - -[case testNestedTupleTypes2] -from typing import Tuple -t1 = None # type: Tuple[A, Tuple[A, A]] -t2 = None # type: Tuple[B, Tuple[B, B]] - -t2 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, Tuple[A, A]]", variable has type "Tuple[B, Tuple[B, B]]") -t1 = t2 - -class A: pass -class B(A): pass -[builtins fixtures/tuple.py] - -[case testSubtypingWithNamedTupleType] -from typing import Tuple -t1 = None # type: Tuple[A, A] -t2 = None # type: tuple - -t1 = t2 # E: Incompatible types in assignment (expression has type Tuple[Any, ...], variable has type "Tuple[A, A]") -t2 = t1 - -class A: pass -[builtins fixtures/tuple.py] - -[case testTupleInitializationWithNone] -from typing import Tuple -t = None # type: Tuple[A, A] -t = None -class A: pass -[builtins fixtures/tuple.py] - - --- Tuple expressions --- ----------------- - - -[case testTupleExpressions] -from typing import Tuple -t1 = None # type: tuple -t2 = None # type: Tuple[A] -t3 = None # type: Tuple[A, B] - -a, b, c = None, None, None # type: (A, B, C) - -t2 = () # E: Incompatible types in assignment (expression has type "Tuple[]", variable has type "Tuple[A]") -t2 = (a, a) # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A]") -t3 = (a, a) # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A, B]") -t3 = (b, b) # E: Incompatible types in assignment (expression has type "Tuple[B, B]", variable has type "Tuple[A, B]") -t3 = (a, b, a) # E: Incompatible types in assignment (expression has type "Tuple[A, B, A]", variable has type "Tuple[A, B]") - -t1 = () -t1 = (a,) -t2 = (a,) -t3 = (a, b) -t3 = (a, c) -t3 = (None, None) - -class A: pass -class B: pass -class C(B): pass -[builtins fixtures/tuple.py] - -[case testVoidValueInTuple] -import typing -(None, f()) # E: "f" does not return a value -(f(), None) # E: "f" does not return a value - -def f() -> None: pass -[builtins fixtures/tuple.py] - - --- Indexing --- -------- - - -[case testIndexingTuples] -from typing import Tuple -t1 = None # type: Tuple[A, B] -t2 = None # type: Tuple[A] -t3 = None # type: Tuple[A, B, C, D, E] -a, b = None, None # type: (A, B) -x = None # type: Tuple[A, B, C] -y = None # type: Tuple[A, C, E] -n = 0 - -a = t1[1] # E: Incompatible types in assignment (expression has type "B", variable has type "A") -b = t1[0] # E: Incompatible types in assignment (expression has type "A", variable has type "B") -t1[2] # E: Tuple index out of range -t1[3] # E: Tuple index out of range -t2[1] # E: Tuple index out of range -t1[n] # E: Tuple index must be an integer literal -t3[n:] # E: Tuple slice must be an integer literal -b = t1[(0)] # E: Incompatible types in assignment (expression has type "A", variable has type "B") - -a = t1[0] -b = t1[1] -b = t1[-1] -a = t1[(0)] -x = t3[0:3] # type (A, B, C) -y = t3[0:5:2] # type (A, C, E) -x = t3[:-2] # type (A, B, C) - -class A: pass -class B: pass -class C: pass -class D: pass -class E: pass -[builtins fixtures/tuple.py] - -[case testIndexingTuplesWithNegativeIntegers] -from typing import Tuple -t1 = None # type: Tuple[A, B] -t2 = None # type: Tuple[A] -a, b = None, None # type: A, B - -a = t1[-1] # E: Incompatible types in assignment (expression has type "B", variable has type "A") -b = t1[-2] # E: Incompatible types in assignment (expression has type "A", variable has type "B") -t1[-3] # E: Tuple index out of range -t1[-4] # E: Tuple index out of range -b = t2[(-1)] # E: Incompatible types in assignment (expression has type "A", variable has type "B") - -a = t1[-2] -b = t1[-1] -a = t2[(-1)] - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - -[case testAssigningToTupleItems] -from typing import Tuple -t = None # type: Tuple[A, B] -n = 0 - -t[0] = A() # E: Unsupported target for indexed assignment -t[2] = A() # E: Unsupported target for indexed assignment -t[n] = A() # E: Unsupported target for indexed assignment - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - - --- Multiple assignment --- ------------------- - - -[case testMultipleAssignmentWithTuples] -from typing import Tuple -t1 = None # type: Tuple[A, B] -t2 = None # type: Tuple[A, B, A] -a, b = None, None # type: (A, B) - -a, a = t1 # E: Incompatible types in assignment (expression has type "B", variable has type "A") -b, b = t1 # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a, b, b = t2 # E: Incompatible types in assignment (expression has type "A", variable has type "B") - -a, b = t1 -a, b, a = t2 - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - -[case testMultipleAssignmentWithInvalidNumberOfValues] -from typing import Tuple -t1 = None # type: Tuple[A, A, A] -a = None # type: A - -a, a = t1 # E: Too many values to unpack (2 expected, 3 provided) -a, a, a, a = t1 # E: Need more than 3 values to unpack (4 expected) - -a, a, a = t1 - -class A: pass -[builtins fixtures/tuple.py] - -[case testMultipleAssignmentWithTupleExpressionRvalue] - -a, b = None, None # type: (A, B) - -a, b = a, a # Fail -a, b = b, a # Fail - -a, b = a, b -a, a = a, a - -class A: pass -class B: pass -[builtins fixtures/tuple.py] -[out] -main:4: error: Incompatible types in assignment (expression has type "A", variable has type "B") -main:5: error: Incompatible types in assignment (expression has type "B", variable has type "A") -main:5: error: Incompatible types in assignment (expression has type "A", variable has type "B") - -[case testSubtypingInMultipleAssignment] - -a, b = None, None # type: (A, B) - -b, b = a, b # E: Incompatible types in assignment (expression has type "A", variable has type "B") -b, b = b, a # E: Incompatible types in assignment (expression has type "A", variable has type "B") - -a, b = b, b -b, a = b, b - -class A: pass -class B(A): pass -[builtins fixtures/tuple.py] - -[case testInitializationWithMultipleValues] - -a, b = None, None # type: (A, B) - -a1, b1 = a, a # type: (A, B) # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a2, b2 = b, b # type: (A, B) # E: Incompatible types in assignment (expression has type "B", variable has type "A") -a3, b3 = a # type: (A, B) # E: '__main__.A' object is not iterable -a4, b4 = None # type: (A, B) # E: 'builtins.None' object is not iterable -a5, b5 = a, b, a # type: (A, B) # E: Too many values to unpack (2 expected, 3 provided) - -ax, bx = a, b # type: (A, B) - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - -[case testMultipleAssignmentWithNonTupleRvalue] - -a, b = None, None # type: (A, B) -def f(): pass - -a, b = None # E: 'builtins.None' object is not iterable -a, b = a # E: '__main__.A' object is not iterable -a, b = f # E: 'def () -> Any' object is not iterable - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - -[case testMultipleAssignmentWithIndexedLvalues] - -a, b = None, None # type: (A, B) -aa, bb = None, None # type: (AA, BB) - -a[a], b[b] = a, bb # E: Incompatible types in assignment (expression has type "A", target has type "AA") -a[a], b[b] = aa, b # E: Incompatible types in assignment (expression has type "B", target has type "BB") -a[aa], b[b] = aa, bb # E: Invalid index type "AA" for "A" -a[a], b[bb] = aa, bb # E: Invalid index type "BB" for "B" -a[a], b[b] = aa, bb - -class A: - def __setitem__(self, x: 'A', y: 'AA') -> None: pass -class B: - def __setitem__(self, x: 'B', y: 'BB') -> None: pass - -class AA: pass -class BB: pass -[builtins fixtures/tuple.py] - -[case testMultipleDeclarationWithParentheses] - -(a, b) = (None, None) # type: int, str -a = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -b = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") -a = 1 -b = '' - -[case testMultipleAssignmentWithExtraParentheses] - -a, b = None, None # type: (A, B) - -(a, b) = (a, a) # E: Incompatible types in assignment (expression has type "A", variable has type "B") -(a, b) = (b, b) # E: Incompatible types in assignment (expression has type "B", variable has type "A") -((a), (b)) = ((a), (a)) # E: Incompatible types in assignment (expression has type "A", variable has type "B") -((a), (b)) = ((b), (b)) # E: Incompatible types in assignment (expression has type "B", variable has type "A") -[a, b] = a, a # E: Incompatible types in assignment (expression has type "A", variable has type "B") -[a, b] = b, b # E: Incompatible types in assignment (expression has type "B", variable has type "A") - -(a, b) = (a, b) -((a), (b)) = ((a), (b)) -[a, b] = a, b - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - -[case testMultipleAssignmentUsingSingleTupleType] -from typing import Tuple -a, b = None, None # type: Tuple[int, str] -a = 1 -b = '' -a = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -b = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") - -[case testMultipleAssignmentWithMixedVariables] -a = b, c = 1, 1 -x, y = p, q = 1, 1 -u, v, w = r, s = 1, 1 # E: Need more than 2 values to unpack (3 expected) -d, e = f, g, h = 1, 1 # E: Need more than 2 values to unpack (3 expected) - - --- Assignment to starred expressions --- --------------------------------- - - -[case testAssignmentToStarMissingAnnotation] -from typing import List -t = 1, 2 -a, b, *c = 1, 2 # E: Need type annotation for variable -aa, bb, *cc = t # E: Need type annotation for variable -[builtins fixtures/list.py] - -[case testAssignmentToStarAnnotation] -from typing import List -li, lo = None, None # type: List[int], List[object] -a, b, *c = 1, 2 # type: int, int, *List[int] -c = lo # E: Incompatible types in assignment (expression has type List[object], variable has type List[int]) -c = li -[builtins fixtures/list.py] - -[case testAssignmentToStarCount1] -from typing import List -ca = None # type: List[int] -c = [1] -a, b, *c = 1, # E: Need more than 1 value to unpack (2 expected) -a, b, *c = 1, 2 -a, b, *c = 1, 2, 3 -a, b, *c = 1, 2, 3, 4 -[builtins fixtures/list.py] - -[case testAssignmentToStarCount2] -from typing import List -ca = None # type: List[int] -t1 = 1, -t2 = 1, 2 -t3 = 1, 2, 3 -t4 = 1, 2, 3, 4 -c = [1] -a, b, *c = t1 # E: Need more than 1 value to unpack (2 expected) -a, b, *c = t2 -a, b, *c = t3 -a, b, *c = t4 -[builtins fixtures/list.py] - -[case testAssignmentToStarFromAny] -from typing import Any -a, c = Any(1), C() -p, *q = a -c = a -c = q - -class C: pass - -[case testAssignmentToComplexStar] -from typing import List -li = None # type: List[int] -a, *(li) = 1, -a, *(b, c) = 1, 2 # E: Need more than 1 value to unpack (2 expected) -a, *(b, c) = 1, 2, 3 -a, *(b, c) = 1, 2, 3, 4 # E: Too many values to unpack (2 expected, 3 provided) -[builtins fixtures/list.py] - -[case testAssignmentToStarFromTupleType] -from typing import List, Tuple -li = None # type: List[int] -la = None # type: List[A] -ta = None # type: Tuple[A, A, A] -a, *la = ta -a, *li = ta # E -a, *na = ta -na = la -na = a # E - -class A: pass -[builtins fixtures/list.py] -[out] -main:6: error: List item 0 has incompatible type "A" -main:6: error: List item 1 has incompatible type "A" -main:9: error: Incompatible types in assignment (expression has type "A", variable has type List[A]) - -[case testAssignmentToStarFromTupleInference] -from typing import List -li = None # type: List[int] -la = None # type: List[A] -a, *l = A(), A() -l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) -l = la - -class A: pass -[builtins fixtures/list.py] -[out] - -[case testAssignmentToStarFromListInference] -from typing import List -li = None # type: List[int] -la = None # type: List[A] -a, *l = [A(), A()] -l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) -l = la - -class A: pass -[builtins fixtures/list.py] -[out] - -[case testAssignmentToStarFromTupleTypeInference] -from typing import List, Tuple -li = None # type: List[int] -la = None # type: List[A] -ta = None # type: Tuple[A, A, A] -a, *l = ta -l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) -l = la - -class A: pass -[builtins fixtures/list.py] -[out] - -[case testAssignmentToStarFromListTypeInference] -from typing import List -li = None # type: List[int] -la = None # type: List[A] -a, *l = la -l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) -l = la - -class A: pass -[builtins fixtures/list.py] -[out] - - --- Nested tuple assignment --- ---------------------------- - - -[case testNestedTupleAssignment1] - -a1, b1, c1 = None, None, None # type: (A, B, C) -a2, b2, c2 = None, None, None # type: (A, B, C) - -a1, (b1, c1) = a2, (b2, c2) -a1, (a1, (b1, c1)) = a2, (a2, (b2, c2)) -a1, (a1, (a1, b1)) = a1, (a1, (a1, c1)) # Fail - -class A: pass -class B: pass -class C: pass -[out] -main:7: error: Incompatible types in assignment (expression has type "C", variable has type "B") - -[case testNestedTupleAssignment2] - -a1, b1, c1 = None, None, None # type: (A, B, C) -a2, b2, c2 = None, None, None # type: (A, B, C) -t = a1, b1 - -a2, b2 = t -(a2, b2), c2 = t, c1 -(a2, c2), c2 = t, c1 # Fail -t, c2 = (a2, b2), c2 -t, c2 = (a2, a2), c2 # Fail -t = a1, a1, a1 # Fail -t = a1 # Fail -a2, a2, a2 = t # Fail -a2, = t # Fail -a2 = t # Fail - -class A: pass -class B: pass -class C: pass -[out] -main:8: error: Incompatible types in assignment (expression has type "B", variable has type "C") -main:10: error: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A, B]") -main:11: error: Incompatible types in assignment (expression has type "Tuple[A, A, A]", variable has type "Tuple[A, B]") -main:12: error: Incompatible types in assignment (expression has type "A", variable has type "Tuple[A, B]") -main:13: error: Need more than 2 values to unpack (3 expected) -main:14: error: Too many values to unpack (1 expected, 2 provided) -main:15: error: Incompatible types in assignment (expression has type "Tuple[A, B]", variable has type "A") - - --- Error messages --- -------------- - - -[case testTupleErrorMessages] - -a = None # type: A - -(a, a) + a # E: Unsupported left operand type for + (Tuple[A, ...]) -a + (a, a) # E: Unsupported operand types for + ("A" and "Tuple[A, A]") -f((a, a)) # E: Argument 1 to "f" has incompatible type "Tuple[A, A]"; expected "A" -(a, a).foo # E: Tuple[A, ...] has no attribute "foo" - -def f(x: 'A') -> None: pass - -class A: - def __add__(self, x: 'A') -> 'A': pass -[builtins fixtures/tuple.py] - -[case testLargeTuplesInErrorMessages] - -a = None # type: LongTypeName -a + (a, a, a, a, a, a, a) # Fail - -class LongTypeName: - def __add__(self, x: 'LongTypeName') -> 'LongTypeName': pass -[builtins fixtures/tuple.py] -[out] -main:3: error: Unsupported operand types for + ("LongTypeName" and tuple(length 7)) - - --- Tuple methods --- ------------- - - -[case testTupleMethods] -from typing import Tuple -t = None # type: Tuple[int, str] -i = 0 -s = '' -b = bool() - -s = t.__len__() # E: Incompatible types in assignment (expression has type "int", variable has type "str") -i = t.__str__() # E: Incompatible types in assignment (expression has type "str", variable has type "int") -i = s in t # E: Incompatible types in assignment (expression has type "bool", variable has type "int") -t.foo # E: Tuple[Any, ...] has no attribute "foo" - -i = t.__len__() -s = t.__str__() -b = s in t - -[file builtins.py] -from typing import TypeVar, Generic -_T = TypeVar('_T') -class object: - def __init__(self) -> None: pass -class tuple(Generic[_T]): - def __len__(self) -> int: pass - def __str__(self) -> str: pass - def __contains__(self, o: object) -> bool: pass -class int: pass -class str: pass -class bool: pass -class type: pass -class function: pass - - --- For loop over tuple --- ------------------- - - -[case testForLoopOverTuple] -import typing -t = 1, 2 -for x in t: - x = 1 - x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -[builtins fixtures/for.py] - -[case testForLoopOverEmptyTuple] -import typing -t = () -for x in t: pass # E: Need type annotation for variable -[builtins fixtures/for.py] - -[case testForLoopOverNoneValuedTuple] -import typing -t = () -for x in None, None: pass # E: Need type annotation for variable -[builtins fixtures/for.py] - -[case testForLoopOverTupleAndSubtyping] -import typing -class A: pass -class B(A): pass -for x in B(), A(): - x = A() - x = B() - x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "A") -[builtins fixtures/for.py] - -[case testTupleIterable] -y = 'a' -x = sum((1,2)) -y = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") -[builtins fixtures/tuple.py] - - --- Tuple as a base type --- -------------------- - - -[case testTupleBaseClass] -import m -[file m.pyi] -from typing import Tuple -class A(Tuple[int, str]): - def f(self, x: int) -> None: - a, b = 1, '' - a, b = self - b, a = self # Error - self.f('') # Error -[builtins fixtures/tuple.py] -[out] -main:1: note: In module imported here: -tmp/m.pyi: note: In member "f" of class "A": -tmp/m.pyi:6: error: Incompatible types in assignment (expression has type "int", variable has type "str") -tmp/m.pyi:6: error: Incompatible types in assignment (expression has type "str", variable has type "int") -tmp/m.pyi:7: error: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" - -[case testInvalidTupleBaseClass] -from typing import Tuple -class A(Tuple[int, str]): pass # E: Tuple[...] not supported as a base class outside a stub file -[builtins fixtures/tuple.py] -[out] - -[case testValidTupleBaseClass] -from typing import Tuple -class A(tuple): pass -[out] - -[case testTupleBaseClass2-skip] -import m -[file m.pyi] -# This doesn't work correctly -- no errors are reported (#867) -from typing import Tuple -a = None # type: A -class A(Tuple[int, str]): pass -x, y = a -x() # Expected: "int" not callable -y() # Expected: "str" not callable -[out] -(should fail) - -[case testGenericClassWithTupleBaseClass] -from typing import TypeVar, Generic, Tuple -T = TypeVar('T') -class Test(Generic[T], Tuple[T]): pass -x = Test() # type: Test[int] -[builtins fixtures/tuple.py] -[out] -main:3: error: Tuple[...] not supported as a base class outside a stub file -main:4: error: Generic tuple types not supported - - --- Variable-length tuples (Tuple[t, ...] with literal '...') --- --------------------------------------------------------- - - -[case testIndexingVariableLengthTuple] -from typing import Tuple -x = () # type: Tuple[str, ...] -n = 5 -x[n]() # E: "str" not callable -x[3]() # E: "str" not callable -[builtins fixtures/tuple.py] - -[case testSubtypingVariableLengthTuple] -from typing import Tuple -class A: pass -class B(A): pass -def fa(t: Tuple[A, ...]) -> None: pass -def fb(t: Tuple[B, ...]) -> None: pass -ta = () # type: Tuple[A, ...] -tb = () # type: Tuple[B, ...] -fa(ta) -fa(tb) -fb(tb) -fb(ta) # E: Argument 1 to "fb" has incompatible type Tuple[A, ...]; expected Tuple[B, ...] -[builtins fixtures/tuple.py] - -[case testSubtypingFixedAndVariableLengthTuples] -from typing import Tuple -class A: pass -class B(A): pass -def fa(t: Tuple[A, ...]) -> None: pass -def fb(t: Tuple[B, ...]) -> None: pass -aa = (A(), A()) -ab = (A(), B()) -bb = (B(), B()) -fa(aa) -fa(ab) -fa(bb) -fb(bb) -fb(ab) # E: Argument 1 to "fb" has incompatible type "Tuple[A, B]"; expected Tuple[B, ...] -fb(aa) # E: Argument 1 to "fb" has incompatible type "Tuple[A, A]"; expected Tuple[B, ...] -[builtins fixtures/tuple.py] - -[case testSubtypingTupleIsContainer] -from typing import Container -a = None # type: Container[str] -a = () - -[case testSubtypingTupleIsSized] -from typing import Sized -a = None # type: Sized -a = () +-- Normal assignment and subtyping +-- ------------------------------- + + +[case testTupleAssignmentWithTupleTypes] +from typing import Tuple +t1 = None # type: Tuple[A] +t2 = None # type: Tuple[B] +t3 = None # type: Tuple[A, A] +t4 = None # type: Tuple[A, B] +t5 = None # type: Tuple[B, A] + +t1 = t2 # E: Incompatible types in assignment (expression has type "Tuple[B]", variable has type "Tuple[A]") +t1 = t3 # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A]") +t3 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A]", variable has type "Tuple[A, A]") +t3 = t4 # E: Incompatible types in assignment (expression has type "Tuple[A, B]", variable has type "Tuple[A, A]") +t3 = t5 # E: Incompatible types in assignment (expression has type "Tuple[B, A]", variable has type "Tuple[A, A]") + +# Ok +t1 = t1 +t2 = t2 +t3 = t3 +t4 = t4 +t5 = t5 + +class A: pass +class B: pass +[builtins fixtures/tuple.py] + +[case testTupleSubtyping] +from typing import Tuple +t1 = None # type: Tuple[A, A] +t2 = None # type: Tuple[A, B] +t3 = None # type: Tuple[B, A] + +t2 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A, B]") +t2 = t3 # E: Incompatible types in assignment (expression has type "Tuple[B, A]", variable has type "Tuple[A, B]") +t3 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[B, A]") +t3 = t2 # E: Incompatible types in assignment (expression has type "Tuple[A, B]", variable has type "Tuple[B, A]") + +t1 = t2 +t1 = t3 + +class A: pass +class B(A): pass +[builtins fixtures/tuple.py] + +[case testTupleCompatibilityWithOtherTypes] +from typing import Tuple +a, o = None, None # type: (A, object) +t = None # type: Tuple[A, A] + +a = t # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "A") +t = o # E: Incompatible types in assignment (expression has type "object", variable has type "Tuple[A, A]") +t = a # E: Incompatible types in assignment (expression has type "A", variable has type "Tuple[A, A]") +# TODO: callable types + tuples + +# Ok +o = t +t = None + +class A: pass +[builtins fixtures/tuple.py] + +[case testNestedTupleTypes] +from typing import Tuple +t1 = None # type: Tuple[A, Tuple[A, A]] +t2 = None # type: Tuple[B, Tuple[B, B]] + +t2 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, Tuple[A, A]]", variable has type "Tuple[B, Tuple[B, B]]") +t1 = t2 + +class A: pass +class B(A): pass +[builtins fixtures/tuple.py] + +[case testNestedTupleTypes2] +from typing import Tuple +t1 = None # type: Tuple[A, Tuple[A, A]] +t2 = None # type: Tuple[B, Tuple[B, B]] + +t2 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, Tuple[A, A]]", variable has type "Tuple[B, Tuple[B, B]]") +t1 = t2 + +class A: pass +class B(A): pass +[builtins fixtures/tuple.py] + +[case testSubtypingWithNamedTupleType] +from typing import Tuple +t1 = None # type: Tuple[A, A] +t2 = None # type: tuple + +t1 = t2 # E: Incompatible types in assignment (expression has type Tuple[Any, ...], variable has type "Tuple[A, A]") +t2 = t1 + +class A: pass +[builtins fixtures/tuple.py] + +[case testTupleInitializationWithNone] +from typing import Tuple +t = None # type: Tuple[A, A] +t = None +class A: pass +[builtins fixtures/tuple.py] + + +-- Tuple expressions +-- ----------------- + + +[case testTupleExpressions] +from typing import Tuple +t1 = None # type: tuple +t2 = None # type: Tuple[A] +t3 = None # type: Tuple[A, B] + +a, b, c = None, None, None # type: (A, B, C) + +t2 = () # E: Incompatible types in assignment (expression has type "Tuple[]", variable has type "Tuple[A]") +t2 = (a, a) # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A]") +t3 = (a, a) # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A, B]") +t3 = (b, b) # E: Incompatible types in assignment (expression has type "Tuple[B, B]", variable has type "Tuple[A, B]") +t3 = (a, b, a) # E: Incompatible types in assignment (expression has type "Tuple[A, B, A]", variable has type "Tuple[A, B]") + +t1 = () +t1 = (a,) +t2 = (a,) +t3 = (a, b) +t3 = (a, c) +t3 = (None, None) + +class A: pass +class B: pass +class C(B): pass +[builtins fixtures/tuple.py] + +[case testVoidValueInTuple] +import typing +(None, f()) # E: "f" does not return a value +(f(), None) # E: "f" does not return a value + +def f() -> None: pass +[builtins fixtures/tuple.py] + + +-- Indexing +-- -------- + + +[case testIndexingTuples] +from typing import Tuple +t1 = None # type: Tuple[A, B] +t2 = None # type: Tuple[A] +t3 = None # type: Tuple[A, B, C, D, E] +a, b = None, None # type: (A, B) +x = None # type: Tuple[A, B, C] +y = None # type: Tuple[A, C, E] +n = 0 + +a = t1[1] # E: Incompatible types in assignment (expression has type "B", variable has type "A") +b = t1[0] # E: Incompatible types in assignment (expression has type "A", variable has type "B") +t1[2] # E: Tuple index out of range +t1[3] # E: Tuple index out of range +t2[1] # E: Tuple index out of range +t1[n] # E: Tuple index must be an integer literal +t3[n:] # E: Tuple slice must be an integer literal +b = t1[(0)] # E: Incompatible types in assignment (expression has type "A", variable has type "B") + +a = t1[0] +b = t1[1] +b = t1[-1] +a = t1[(0)] +x = t3[0:3] # type (A, B, C) +y = t3[0:5:2] # type (A, C, E) +x = t3[:-2] # type (A, B, C) + +class A: pass +class B: pass +class C: pass +class D: pass +class E: pass +[builtins fixtures/tuple.py] + +[case testIndexingTuplesWithNegativeIntegers] +from typing import Tuple +t1 = None # type: Tuple[A, B] +t2 = None # type: Tuple[A] +a, b = None, None # type: A, B + +a = t1[-1] # E: Incompatible types in assignment (expression has type "B", variable has type "A") +b = t1[-2] # E: Incompatible types in assignment (expression has type "A", variable has type "B") +t1[-3] # E: Tuple index out of range +t1[-4] # E: Tuple index out of range +b = t2[(-1)] # E: Incompatible types in assignment (expression has type "A", variable has type "B") + +a = t1[-2] +b = t1[-1] +a = t2[(-1)] + +class A: pass +class B: pass +[builtins fixtures/tuple.py] + +[case testAssigningToTupleItems] +from typing import Tuple +t = None # type: Tuple[A, B] +n = 0 + +t[0] = A() # E: Unsupported target for indexed assignment +t[2] = A() # E: Unsupported target for indexed assignment +t[n] = A() # E: Unsupported target for indexed assignment + +class A: pass +class B: pass +[builtins fixtures/tuple.py] + + +-- Multiple assignment +-- ------------------- + + +[case testMultipleAssignmentWithTuples] +from typing import Tuple +t1 = None # type: Tuple[A, B] +t2 = None # type: Tuple[A, B, A] +a, b = None, None # type: (A, B) + +a, a = t1 # E: Incompatible types in assignment (expression has type "B", variable has type "A") +b, b = t1 # E: Incompatible types in assignment (expression has type "A", variable has type "B") +a, b, b = t2 # E: Incompatible types in assignment (expression has type "A", variable has type "B") + +a, b = t1 +a, b, a = t2 + +class A: pass +class B: pass +[builtins fixtures/tuple.py] + +[case testMultipleAssignmentWithInvalidNumberOfValues] +from typing import Tuple +t1 = None # type: Tuple[A, A, A] +a = None # type: A + +a, a = t1 # E: Too many values to unpack (2 expected, 3 provided) +a, a, a, a = t1 # E: Need more than 3 values to unpack (4 expected) + +a, a, a = t1 + +class A: pass +[builtins fixtures/tuple.py] + +[case testMultipleAssignmentWithTupleExpressionRvalue] + +a, b = None, None # type: (A, B) + +a, b = a, a # Fail +a, b = b, a # Fail + +a, b = a, b +a, a = a, a + +class A: pass +class B: pass +[builtins fixtures/tuple.py] +[out] +main:4: error: Incompatible types in assignment (expression has type "A", variable has type "B") +main:5: error: Incompatible types in assignment (expression has type "B", variable has type "A") +main:5: error: Incompatible types in assignment (expression has type "A", variable has type "B") + +[case testSubtypingInMultipleAssignment] + +a, b = None, None # type: (A, B) + +b, b = a, b # E: Incompatible types in assignment (expression has type "A", variable has type "B") +b, b = b, a # E: Incompatible types in assignment (expression has type "A", variable has type "B") + +a, b = b, b +b, a = b, b + +class A: pass +class B(A): pass +[builtins fixtures/tuple.py] + +[case testInitializationWithMultipleValues] + +a, b = None, None # type: (A, B) + +a1, b1 = a, a # type: (A, B) # E: Incompatible types in assignment (expression has type "A", variable has type "B") +a2, b2 = b, b # type: (A, B) # E: Incompatible types in assignment (expression has type "B", variable has type "A") +a3, b3 = a # type: (A, B) # E: '__main__.A' object is not iterable +a4, b4 = None # type: (A, B) # E: 'builtins.None' object is not iterable +a5, b5 = a, b, a # type: (A, B) # E: Too many values to unpack (2 expected, 3 provided) + +ax, bx = a, b # type: (A, B) + +class A: pass +class B: pass +[builtins fixtures/tuple.py] + +[case testMultipleAssignmentWithNonTupleRvalue] + +a, b = None, None # type: (A, B) +def f(): pass + +a, b = None # E: 'builtins.None' object is not iterable +a, b = a # E: '__main__.A' object is not iterable +a, b = f # E: 'def () -> Any' object is not iterable + +class A: pass +class B: pass +[builtins fixtures/tuple.py] + +[case testMultipleAssignmentWithIndexedLvalues] + +a, b = None, None # type: (A, B) +aa, bb = None, None # type: (AA, BB) + +a[a], b[b] = a, bb # E: Incompatible types in assignment (expression has type "A", target has type "AA") +a[a], b[b] = aa, b # E: Incompatible types in assignment (expression has type "B", target has type "BB") +a[aa], b[b] = aa, bb # E: Invalid index type "AA" for "A" +a[a], b[bb] = aa, bb # E: Invalid index type "BB" for "B" +a[a], b[b] = aa, bb + +class A: + def __setitem__(self, x: 'A', y: 'AA') -> None: pass +class B: + def __setitem__(self, x: 'B', y: 'BB') -> None: pass + +class AA: pass +class BB: pass +[builtins fixtures/tuple.py] + +[case testMultipleDeclarationWithParentheses] + +(a, b) = (None, None) # type: int, str +a = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +b = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") +a = 1 +b = '' + +[case testMultipleAssignmentWithExtraParentheses] + +a, b = None, None # type: (A, B) + +(a, b) = (a, a) # E: Incompatible types in assignment (expression has type "A", variable has type "B") +(a, b) = (b, b) # E: Incompatible types in assignment (expression has type "B", variable has type "A") +((a), (b)) = ((a), (a)) # E: Incompatible types in assignment (expression has type "A", variable has type "B") +((a), (b)) = ((b), (b)) # E: Incompatible types in assignment (expression has type "B", variable has type "A") +[a, b] = a, a # E: Incompatible types in assignment (expression has type "A", variable has type "B") +[a, b] = b, b # E: Incompatible types in assignment (expression has type "B", variable has type "A") + +(a, b) = (a, b) +((a), (b)) = ((a), (b)) +[a, b] = a, b + +class A: pass +class B: pass +[builtins fixtures/tuple.py] + +[case testMultipleAssignmentUsingSingleTupleType] +from typing import Tuple +a, b = None, None # type: Tuple[int, str] +a = 1 +b = '' +a = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +b = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testMultipleAssignmentWithMixedVariables] +a = b, c = 1, 1 +x, y = p, q = 1, 1 +u, v, w = r, s = 1, 1 # E: Need more than 2 values to unpack (3 expected) +d, e = f, g, h = 1, 1 # E: Need more than 2 values to unpack (3 expected) + + +-- Assignment to starred expressions +-- --------------------------------- + + +[case testAssignmentToStarMissingAnnotation] +from typing import List +t = 1, 2 +a, b, *c = 1, 2 # E: Need type annotation for variable +aa, bb, *cc = t # E: Need type annotation for variable +[builtins fixtures/list.py] + +[case testAssignmentToStarAnnotation] +from typing import List +li, lo = None, None # type: List[int], List[object] +a, b, *c = 1, 2 # type: int, int, *List[int] +c = lo # E: Incompatible types in assignment (expression has type List[object], variable has type List[int]) +c = li +[builtins fixtures/list.py] + +[case testAssignmentToStarCount1] +from typing import List +ca = None # type: List[int] +c = [1] +a, b, *c = 1, # E: Need more than 1 value to unpack (2 expected) +a, b, *c = 1, 2 +a, b, *c = 1, 2, 3 +a, b, *c = 1, 2, 3, 4 +[builtins fixtures/list.py] + +[case testAssignmentToStarCount2] +from typing import List +ca = None # type: List[int] +t1 = 1, +t2 = 1, 2 +t3 = 1, 2, 3 +t4 = 1, 2, 3, 4 +c = [1] +a, b, *c = t1 # E: Need more than 1 value to unpack (2 expected) +a, b, *c = t2 +a, b, *c = t3 +a, b, *c = t4 +[builtins fixtures/list.py] + +[case testAssignmentToStarFromAny] +from typing import Any +a, c = Any(1), C() +p, *q = a +c = a +c = q + +class C: pass + +[case testAssignmentToComplexStar] +from typing import List +li = None # type: List[int] +a, *(li) = 1, +a, *(b, c) = 1, 2 # E: Need more than 1 value to unpack (2 expected) +a, *(b, c) = 1, 2, 3 +a, *(b, c) = 1, 2, 3, 4 # E: Too many values to unpack (2 expected, 3 provided) +[builtins fixtures/list.py] + +[case testAssignmentToStarFromTupleType] +from typing import List, Tuple +li = None # type: List[int] +la = None # type: List[A] +ta = None # type: Tuple[A, A, A] +a, *la = ta +a, *li = ta # E +a, *na = ta +na = la +na = a # E + +class A: pass +[builtins fixtures/list.py] +[out] +main:6: error: List item 0 has incompatible type "A" +main:6: error: List item 1 has incompatible type "A" +main:9: error: Incompatible types in assignment (expression has type "A", variable has type List[A]) + +[case testAssignmentToStarFromTupleInference] +from typing import List +li = None # type: List[int] +la = None # type: List[A] +a, *l = A(), A() +l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) +l = la + +class A: pass +[builtins fixtures/list.py] +[out] + +[case testAssignmentToStarFromListInference] +from typing import List +li = None # type: List[int] +la = None # type: List[A] +a, *l = [A(), A()] +l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) +l = la + +class A: pass +[builtins fixtures/list.py] +[out] + +[case testAssignmentToStarFromTupleTypeInference] +from typing import List, Tuple +li = None # type: List[int] +la = None # type: List[A] +ta = None # type: Tuple[A, A, A] +a, *l = ta +l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) +l = la + +class A: pass +[builtins fixtures/list.py] +[out] + +[case testAssignmentToStarFromListTypeInference] +from typing import List +li = None # type: List[int] +la = None # type: List[A] +a, *l = la +l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) +l = la + +class A: pass +[builtins fixtures/list.py] +[out] + + +-- Nested tuple assignment +-- ---------------------------- + + +[case testNestedTupleAssignment1] + +a1, b1, c1 = None, None, None # type: (A, B, C) +a2, b2, c2 = None, None, None # type: (A, B, C) + +a1, (b1, c1) = a2, (b2, c2) +a1, (a1, (b1, c1)) = a2, (a2, (b2, c2)) +a1, (a1, (a1, b1)) = a1, (a1, (a1, c1)) # Fail + +class A: pass +class B: pass +class C: pass +[out] +main:7: error: Incompatible types in assignment (expression has type "C", variable has type "B") + +[case testNestedTupleAssignment2] + +a1, b1, c1 = None, None, None # type: (A, B, C) +a2, b2, c2 = None, None, None # type: (A, B, C) +t = a1, b1 + +a2, b2 = t +(a2, b2), c2 = t, c1 +(a2, c2), c2 = t, c1 # Fail +t, c2 = (a2, b2), c2 +t, c2 = (a2, a2), c2 # Fail +t = a1, a1, a1 # Fail +t = a1 # Fail +a2, a2, a2 = t # Fail +a2, = t # Fail +a2 = t # Fail + +class A: pass +class B: pass +class C: pass +[out] +main:8: error: Incompatible types in assignment (expression has type "B", variable has type "C") +main:10: error: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A, B]") +main:11: error: Incompatible types in assignment (expression has type "Tuple[A, A, A]", variable has type "Tuple[A, B]") +main:12: error: Incompatible types in assignment (expression has type "A", variable has type "Tuple[A, B]") +main:13: error: Need more than 2 values to unpack (3 expected) +main:14: error: Too many values to unpack (1 expected, 2 provided) +main:15: error: Incompatible types in assignment (expression has type "Tuple[A, B]", variable has type "A") + + +-- Error messages +-- -------------- + + +[case testTupleErrorMessages] + +a = None # type: A + +(a, a) + a # E: Unsupported left operand type for + (Tuple[A, ...]) +a + (a, a) # E: Unsupported operand types for + ("A" and "Tuple[A, A]") +f((a, a)) # E: Argument 1 to "f" has incompatible type "Tuple[A, A]"; expected "A" +(a, a).foo # E: Tuple[A, ...] has no attribute "foo" + +def f(x: 'A') -> None: pass + +class A: + def __add__(self, x: 'A') -> 'A': pass +[builtins fixtures/tuple.py] + +[case testLargeTuplesInErrorMessages] + +a = None # type: LongTypeName +a + (a, a, a, a, a, a, a) # Fail + +class LongTypeName: + def __add__(self, x: 'LongTypeName') -> 'LongTypeName': pass +[builtins fixtures/tuple.py] +[out] +main:3: error: Unsupported operand types for + ("LongTypeName" and tuple(length 7)) + + +-- Tuple methods +-- ------------- + + +[case testTupleMethods] +from typing import Tuple +t = None # type: Tuple[int, str] +i = 0 +s = '' +b = bool() + +s = t.__len__() # E: Incompatible types in assignment (expression has type "int", variable has type "str") +i = t.__str__() # E: Incompatible types in assignment (expression has type "str", variable has type "int") +i = s in t # E: Incompatible types in assignment (expression has type "bool", variable has type "int") +t.foo # E: Tuple[Any, ...] has no attribute "foo" + +i = t.__len__() +s = t.__str__() +b = s in t + +[file builtins.py] +from typing import TypeVar, Generic +_T = TypeVar('_T') +class object: + def __init__(self) -> None: pass +class tuple(Generic[_T]): + def __len__(self) -> int: pass + def __str__(self) -> str: pass + def __contains__(self, o: object) -> bool: pass +class int: pass +class str: pass +class bool: pass +class type: pass +class function: pass + + +-- For loop over tuple +-- ------------------- + + +[case testForLoopOverTuple] +import typing +t = 1, 2 +for x in t: + x = 1 + x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +[builtins fixtures/for.py] + +[case testForLoopOverEmptyTuple] +import typing +t = () +for x in t: pass # E: Need type annotation for variable +[builtins fixtures/for.py] + +[case testForLoopOverNoneValuedTuple] +import typing +t = () +for x in None, None: pass # E: Need type annotation for variable +[builtins fixtures/for.py] + +[case testForLoopOverTupleAndSubtyping] +import typing +class A: pass +class B(A): pass +for x in B(), A(): + x = A() + x = B() + x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "A") +[builtins fixtures/for.py] + +[case testTupleIterable] +y = 'a' +x = sum((1,2)) +y = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") +[builtins fixtures/tuple.py] + + +-- Tuple as a base type +-- -------------------- + + +[case testTupleBaseClass] +import m +[file m.pyi] +from typing import Tuple +class A(Tuple[int, str]): + def f(self, x: int) -> None: + a, b = 1, '' + a, b = self + b, a = self # Error + self.f('') # Error +[builtins fixtures/tuple.py] +[out] +main:1: note: In module imported here: +tmp/m.pyi: note: In member "f" of class "A": +tmp/m.pyi:6: error: Incompatible types in assignment (expression has type "int", variable has type "str") +tmp/m.pyi:6: error: Incompatible types in assignment (expression has type "str", variable has type "int") +tmp/m.pyi:7: error: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" + +[case testInvalidTupleBaseClass] +from typing import Tuple +class A(Tuple[int, str]): pass # E: Tuple[...] not supported as a base class outside a stub file +[builtins fixtures/tuple.py] +[out] + +[case testValidTupleBaseClass] +from typing import Tuple +class A(tuple): pass +[out] + +[case testTupleBaseClass2-skip] +import m +[file m.pyi] +# This doesn't work correctly -- no errors are reported (#867) +from typing import Tuple +a = None # type: A +class A(Tuple[int, str]): pass +x, y = a +x() # Expected: "int" not callable +y() # Expected: "str" not callable +[out] +(should fail) + +[case testGenericClassWithTupleBaseClass] +from typing import TypeVar, Generic, Tuple +T = TypeVar('T') +class Test(Generic[T], Tuple[T]): pass +x = Test() # type: Test[int] +[builtins fixtures/tuple.py] +[out] +main:3: error: Tuple[...] not supported as a base class outside a stub file +main:4: error: Generic tuple types not supported + + +-- Variable-length tuples (Tuple[t, ...] with literal '...') +-- --------------------------------------------------------- + + +[case testIndexingVariableLengthTuple] +from typing import Tuple +x = () # type: Tuple[str, ...] +n = 5 +x[n]() # E: "str" not callable +x[3]() # E: "str" not callable +[builtins fixtures/tuple.py] + +[case testSubtypingVariableLengthTuple] +from typing import Tuple +class A: pass +class B(A): pass +def fa(t: Tuple[A, ...]) -> None: pass +def fb(t: Tuple[B, ...]) -> None: pass +ta = () # type: Tuple[A, ...] +tb = () # type: Tuple[B, ...] +fa(ta) +fa(tb) +fb(tb) +fb(ta) # E: Argument 1 to "fb" has incompatible type Tuple[A, ...]; expected Tuple[B, ...] +[builtins fixtures/tuple.py] + +[case testSubtypingFixedAndVariableLengthTuples] +from typing import Tuple +class A: pass +class B(A): pass +def fa(t: Tuple[A, ...]) -> None: pass +def fb(t: Tuple[B, ...]) -> None: pass +aa = (A(), A()) +ab = (A(), B()) +bb = (B(), B()) +fa(aa) +fa(ab) +fa(bb) +fb(bb) +fb(ab) # E: Argument 1 to "fb" has incompatible type "Tuple[A, B]"; expected Tuple[B, ...] +fb(aa) # E: Argument 1 to "fb" has incompatible type "Tuple[A, A]"; expected Tuple[B, ...] +[builtins fixtures/tuple.py] + +[case testSubtypingTupleIsContainer] +from typing import Container +a = None # type: Container[str] +a = () + +[case testSubtypingTupleIsSized] +from typing import Sized +a = None # type: Sized +a = () From 99635c57e9f7f89e8aa2269c6e74eab95b33c43f Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Sun, 14 Aug 2016 19:12:09 +0300 Subject: [PATCH 50/74] fix line endings --- mypy/build.py | 3356 +++++++-------- mypy/checker.py | 4918 +++++++++++----------- mypy/checkexpr.py | 3644 ++++++++--------- mypy/expandtype.py | 224 +- mypy/join.py | 734 ++-- mypy/meet.py | 570 +-- mypy/messages.py | 1840 ++++----- mypy/nodes.py | 4484 ++++++++++---------- mypy/parse.py | 4068 +++++++++--------- mypy/semanal.py | 5678 +++++++++++++------------- mypy/subtypes.py | 818 ++-- mypy/typeanal.py | 860 ++-- mypy/types.py | 2902 ++++++------- test-data/unit/check-lists.test | 126 +- test-data/unit/check-namedtuple.test | 446 +- test-data/unit/check-tuples.test | 1540 +++---- 16 files changed, 18104 insertions(+), 18104 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index a930614989f90..e8a84c97c7c76 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -1,1678 +1,1678 @@ -"""Facilities to analyze entire programs, including imported modules. - -Parse and analyze the source files of a program in the correct order -(based on file dependencies), and collect the results. - -This module only directs a build, which is performed in multiple passes per -file. The individual passes are implemented in separate modules. - -The function build() is the main interface to this module. -""" -# TODO: More consistent terminology, e.g. path/fnam, module/id, state/file - -import binascii -import collections -import contextlib -import json -import os -import os.path -import sys -import time -from os.path import dirname, basename - -from typing import (AbstractSet, Dict, Iterable, Iterator, List, - NamedTuple, Optional, Set, Tuple, Union, Mapping) - -from mypy.types import Type -from mypy.nodes import (MypyFile, Node, Import, ImportFrom, ImportAll, - SymbolTableNode, MODULE_REF) -from mypy.semanal import FirstPass, SemanticAnalyzer, ThirdPass -from mypy.checker import TypeChecker -from mypy.errors import Errors, CompileError, DecodeError, report_internal_error -from mypy import fixup -from mypy.report import Reports -from mypy import defaults -from mypy import moduleinfo -from mypy import util -from mypy.fixup import fixup_module_pass_one, fixup_module_pass_two -from mypy.options import Options -from mypy.parse import parse -from mypy.stats import dump_type_stats - - -# We need to know the location of this file to load data, but -# until Python 3.4, __file__ is relative. -__file__ = os.path.realpath(__file__) - -PYTHON_EXTENSIONS = ['.pyi', '.py'] - - -class BuildResult: - """The result of a successful build. - - Attributes: - manager: The build manager. - files: Dictionary from module name to related AST node. - types: Dictionary from parse tree node to its inferred type. - errors: List of error messages. - """ - - def __init__(self, manager: 'BuildManager') -> None: - self.manager = manager - self.files = manager.modules - self.types = manager.type_checker.type_map - self.errors = manager.errors.messages() - - -class BuildSource: - def __init__(self, path: Optional[str], module: Optional[str], - text: Optional[str]) -> None: - self.path = path - self.module = module or '__main__' - self.text = text - - @property - def effective_path(self) -> str: - """Return the effective path (ie, if its from in memory)""" - return self.path or '' - - -class BuildSourceSet: - """Efficiently test a file's membership in the set of build sources.""" - - def __init__(self, sources: List[BuildSource]) -> None: - self.source_text_present = False - self.source_modules = set() # type: Set[str] - self.source_paths = set() # type: Set[str] - - for source in sources: - if source.text is not None: - self.source_text_present = True - elif source.path: - self.source_paths.add(source.path) - else: - self.source_modules.add(source.module) - - def is_source(self, file: MypyFile) -> bool: - if file.path and file.path in self.source_paths: - return True - elif file._fullname in self.source_modules: - return True - elif file.path is None and self.source_text_present: - return True - else: - return False - - -def build(sources: List[BuildSource], - options: Options, - alt_lib_path: str = None, - bin_dir: str = None) -> BuildResult: - """Analyze a program. - - A single call to build performs parsing, semantic analysis and optionally - type checking for the program *and* all imported modules, recursively. - - Return BuildResult if successful or only non-blocking errors were found; - otherwise raise CompileError. - - Args: - sources: list of sources to build - options: build options - alt_lib_path: an additional directory for looking up library modules - (takes precedence over other directories) - bin_dir: directory containing the mypy script, used for finding data - directories; if omitted, use '.' as the data directory - """ - - data_dir = default_data_dir(bin_dir) - - find_module_clear_caches() - - # Determine the default module search path. - lib_path = default_lib_path(data_dir, options.python_version) - - if options.use_builtins_fixtures: - # Use stub builtins (to speed up test cases and to make them easier to - # debug). This is a test-only feature, so assume our files are laid out - # as in the source tree. - root_dir = os.path.dirname(os.path.dirname(__file__)) - lib_path.insert(0, os.path.join(root_dir, 'test-data', 'unit', 'lib-stub')) - else: - for source in sources: - if source.path: - # Include directory of the program file in the module search path. - dir = remove_cwd_prefix_from_path(dirname(source.path)) - if dir not in lib_path: - lib_path.insert(0, dir) - - # Do this even if running as a file, for sanity (mainly because with - # multiple builds, there could be a mix of files/modules, so its easier - # to just define the semantics that we always add the current director - # to the lib_path - lib_path.insert(0, os.getcwd()) - - # Add MYPYPATH environment variable to front of library path, if defined. - lib_path[:0] = mypy_path() - - # If provided, insert the caller-supplied extra module path to the - # beginning (highest priority) of the search path. - if alt_lib_path: - lib_path.insert(0, alt_lib_path) - - reports = Reports(data_dir, options.report_dirs) - - source_set = BuildSourceSet(sources) - - # Construct a build manager object to hold state during the build. - # - # Ignore current directory prefix in error messages. - manager = BuildManager(data_dir, lib_path, - ignore_prefix=os.getcwd(), - source_set=source_set, - reports=reports, - options=options) - - try: - dispatch(sources, manager) - return BuildResult(manager) - finally: - manager.log("Build finished with %d modules, %d types, and %d errors" % - (len(manager.modules), - len(manager.type_checker.type_map), - manager.errors.num_messages())) - # Finish the HTML or XML reports even if CompileError was raised. - reports.finish() - - -def default_data_dir(bin_dir: Optional[str]) -> str: - """Returns directory containing typeshed directory - - Args: - bin_dir: directory containing the mypy script - """ - if not bin_dir: - mypy_package = os.path.dirname(__file__) - parent = os.path.dirname(mypy_package) - if (os.path.basename(parent) == 'site-packages' or - os.path.basename(parent) == 'dist-packages'): - # Installed in site-packages or dist-packages, but invoked with python3 -m mypy; - # __file__ is .../blah/lib/python3.N/site-packages/mypy/build.py - # or .../blah/lib/python3.N/dist-packages/mypy/build.py (Debian) - # or .../blah/lib/site-packages/mypy/build.py (Windows) - # blah may be a virtualenv or /usr/local. We want .../blah/lib/mypy. - lib = parent - for i in range(2): - lib = os.path.dirname(lib) - if os.path.basename(lib) == 'lib': - return os.path.join(lib, 'mypy') - subdir = os.path.join(parent, 'lib', 'mypy') - if os.path.isdir(subdir): - # If installed via buildout, the __file__ is - # somewhere/mypy/__init__.py and what we want is - # somewhere/lib/mypy. - return subdir - # Default to directory containing this file's parent. - return parent - base = os.path.basename(bin_dir) - dir = os.path.dirname(bin_dir) - if (sys.platform == 'win32' and base.lower() == 'scripts' - and not os.path.isdir(os.path.join(dir, 'typeshed'))): - # Installed, on Windows. - return os.path.join(dir, 'Lib', 'mypy') - elif base == 'scripts': - # Assume that we have a repo check out or unpacked source tarball. - return dir - elif base == 'bin': - # Installed to somewhere (can be under /usr/local or anywhere). - return os.path.join(dir, 'lib', 'mypy') - elif base == 'python3': - # Assume we installed python3 with brew on os x - return os.path.join(os.path.dirname(dir), 'lib', 'mypy') - elif dir.endswith('python-exec'): - # Gentoo uses a python wrapper in /usr/lib to which mypy is a symlink. - return os.path.join(os.path.dirname(dir), 'mypy') - else: - # Don't know where to find the data files! - raise RuntimeError("Broken installation: can't determine base dir") - - -def mypy_path() -> List[str]: - path_env = os.getenv('MYPYPATH') - if not path_env: - return [] - return path_env.split(os.pathsep) - - -def default_lib_path(data_dir: str, pyversion: Tuple[int, int]) -> List[str]: - """Return default standard library search paths.""" - # IDEA: Make this more portable. - path = [] # type: List[str] - - auto = os.path.join(data_dir, 'stubs-auto') - if os.path.isdir(auto): - data_dir = auto - - # We allow a module for e.g. version 3.5 to be in 3.4/. The assumption - # is that a module added with 3.4 will still be present in Python 3.5. - versions = ["%d.%d" % (pyversion[0], minor) - for minor in reversed(range(pyversion[1] + 1))] - # E.g. for Python 3.2, try 3.2/, 3.1/, 3.0/, 3/, 2and3/. - # (Note that 3.1 and 3.0 aren't really supported, but we don't care.) - for v in versions + [str(pyversion[0]), '2and3']: - for lib_type in ['stdlib', 'third_party']: - stubdir = os.path.join(data_dir, 'typeshed', lib_type, v) - if os.path.isdir(stubdir): - path.append(stubdir) - - # Add fallback path that can be used if we have a broken installation. - if sys.platform != 'win32': - path.append('/usr/local/lib/mypy') - - return path - - -CacheMeta = NamedTuple('CacheMeta', - [('id', str), - ('path', str), - ('mtime', float), - ('size', int), - ('dependencies', List[str]), # names of imported modules - ('data_mtime', float), # mtime of data_json - ('data_json', str), # path of .data.json - ('suppressed', List[str]), # dependencies that weren't imported - ('options', Optional[Dict[str, bool]]), # build options - ('dep_prios', List[int]), - ]) -# NOTE: dependencies + suppressed == all reachable imports; -# suppressed contains those reachable imports that were prevented by -# --silent-imports or simply not found. - - -# Priorities used for imports. (Here, top-level includes inside a class.) -# These are used to determine a more predictable order in which the -# nodes in an import cycle are processed. -PRI_HIGH = 5 # top-level "from X import blah" -PRI_MED = 10 # top-level "import X" -PRI_LOW = 20 # either form inside a function -PRI_ALL = 99 # include all priorities - - -class BuildManager: - """This class holds shared state for building a mypy program. - - It is used to coordinate parsing, import processing, semantic - analysis and type checking. The actual build steps are carried - out by dispatch(). - - Attributes: - data_dir: Mypy data directory (contains stubs) - lib_path: Library path for looking up modules - modules: Mapping of module ID to MypyFile (shared by the passes) - semantic_analyzer: - Semantic analyzer, pass 2 - semantic_analyzer_pass3: - Semantic analyzer, pass 3 - type_checker: Type checker - errors: Used for reporting all errors - options: Build options - missing_modules: Set of modules that could not be imported encountered so far - """ - - def __init__(self, data_dir: str, - lib_path: List[str], - ignore_prefix: str, - source_set: BuildSourceSet, - reports: Reports, - options: Options) -> None: - self.start_time = time.time() - self.data_dir = data_dir - self.errors = Errors() - self.errors.set_ignore_prefix(ignore_prefix) - self.lib_path = tuple(lib_path) - self.source_set = source_set - self.reports = reports - self.options = options - self.semantic_analyzer = SemanticAnalyzer(lib_path, self.errors, options=options) - self.modules = self.semantic_analyzer.modules - self.semantic_analyzer_pass3 = ThirdPass(self.modules, self.errors) - self.type_checker = TypeChecker(self.errors, self.modules, options=options) - self.missing_modules = set() # type: Set[str] - - def all_imported_modules_in_file(self, - file: MypyFile) -> List[Tuple[int, str, int]]: - """Find all reachable import statements in a file. - - Return list of tuples (priority, module id, import line number) - for all modules imported in file; lower numbers == higher priority. - """ - - def correct_rel_imp(imp: Union[ImportFrom, ImportAll]) -> str: - """Function to correct for relative imports.""" - file_id = file.fullname() - rel = imp.relative - if rel == 0: - return imp.id - if os.path.basename(file.path).startswith('__init__.'): - rel -= 1 - if rel != 0: - file_id = ".".join(file_id.split(".")[:-rel]) - new_id = file_id + "." + imp.id if imp.id else file_id - - return new_id - - res = [] # type: List[Tuple[int, str, int]] - for imp in file.imports: - if not imp.is_unreachable: - if isinstance(imp, Import): - pri = PRI_MED if imp.is_top_level else PRI_LOW - for id, _ in imp.ids: - res.append((pri, id, imp.line)) - elif isinstance(imp, ImportFrom): - cur_id = correct_rel_imp(imp) - pos = len(res) - all_are_submodules = True - # Also add any imported names that are submodules. - pri = PRI_MED if imp.is_top_level else PRI_LOW - for name, __ in imp.names: - sub_id = cur_id + '.' + name - if self.is_module(sub_id): - res.append((pri, sub_id, imp.line)) - else: - all_are_submodules = False - # If all imported names are submodules, don't add - # cur_id as a dependency. Otherwise (i.e., if at - # least one imported name isn't a submodule) - # cur_id is also a dependency, and we should - # insert it *before* any submodules. - if not all_are_submodules: - pri = PRI_HIGH if imp.is_top_level else PRI_LOW - res.insert(pos, ((pri, cur_id, imp.line))) - elif isinstance(imp, ImportAll): - pri = PRI_HIGH if imp.is_top_level else PRI_LOW - res.append((pri, correct_rel_imp(imp), imp.line)) - - return res - - def is_module(self, id: str) -> bool: - """Is there a file in the file system corresponding to module id?""" - return find_module(id, self.lib_path) is not None - - def parse_file(self, id: str, path: str, source: str) -> MypyFile: - """Parse the source of a file with the given name. - - Raise CompileError if there is a parse error. - """ - num_errs = self.errors.num_messages() - tree = parse(source, path, self.errors, options=self.options) - tree._fullname = id - - # We don't want to warn about 'type: ignore' comments on - # imports, but we're about to modify tree.imports, so grab - # these first. - import_lines = set(node.line for node in tree.imports) - - # Skip imports that have been ignored (so that we can ignore a C extension module without - # stub, for example), except for 'from x import *', because we wouldn't be able to - # determine which names should be defined unless we process the module. We can still - # ignore errors such as redefinitions when using the latter form. - imports = [node for node in tree.imports - if node.line not in tree.ignored_lines or isinstance(node, ImportAll)] - tree.imports = imports - - if self.errors.num_messages() != num_errs: - self.log("Bailing due to parse errors") - self.errors.raise_error() - - self.errors.set_file_ignored_lines(path, tree.ignored_lines) - self.errors.mark_file_ignored_lines_used(path, import_lines) - return tree - - def module_not_found(self, path: str, line: int, id: str) -> None: - self.errors.set_file(path) - stub_msg = "(Stub files are from https://github.com/python/typeshed)" - if ((self.options.python_version[0] == 2 and moduleinfo.is_py2_std_lib_module(id)) or - (self.options.python_version[0] >= 3 and moduleinfo.is_py3_std_lib_module(id))): - self.errors.report( - line, "No library stub file for standard library module '{}'".format(id)) - self.errors.report(line, stub_msg, severity='note', only_once=True) - elif moduleinfo.is_third_party_module(id): - self.errors.report(line, "No library stub file for module '{}'".format(id)) - self.errors.report(line, stub_msg, severity='note', only_once=True) - else: - self.errors.report(line, "Cannot find module named '{}'".format(id)) - self.errors.report(line, '(Perhaps setting MYPYPATH ' - 'or using the "--silent-imports" flag would help)', - severity='note', only_once=True) - - def report_file(self, file: MypyFile) -> None: - if self.source_set.is_source(file): - self.reports.file(file, type_map=self.type_checker.type_map) - - def log(self, *message: str) -> None: - if self.options.verbosity >= 1: - print('%.3f:LOG: ' % (time.time() - self.start_time), *message, file=sys.stderr) - sys.stderr.flush() - - def trace(self, *message: str) -> None: - if self.options.verbosity >= 2: - print('%.3f:TRACE:' % (time.time() - self.start_time), *message, file=sys.stderr) - sys.stderr.flush() - - -def remove_cwd_prefix_from_path(p: str) -> str: - """Remove current working directory prefix from p, if present. - - Also crawl up until a directory without __init__.py is found. - - If the result would be empty, return '.' instead. - """ - cur = os.getcwd() - # Add separator to the end of the path, unless one is already present. - if basename(cur) != '': - cur += os.sep - # Compute root path. - while (p and - (os.path.isfile(os.path.join(p, '__init__.py')) or - os.path.isfile(os.path.join(p, '__init__.pyi')))): - dir, base = os.path.split(p) - if not base: - break - p = dir - # Remove current directory prefix from the path, if present. - if p.startswith(cur): - p = p[len(cur):] - # Avoid returning an empty path; replace that with '.'. - if p == '': - p = '.' - return p - - -# Cache find_module: (id, lib_path) -> result. -find_module_cache = {} # type: Dict[Tuple[str, Tuple[str, ...]], str] - -# Cache some repeated work within distinct find_module calls: finding which -# elements of lib_path have even the subdirectory they'd need for the module -# to exist. This is shared among different module ids when they differ only -# in the last component. -find_module_dir_cache = {} # type: Dict[Tuple[str, Tuple[str, ...]], List[str]] - -# Cache directory listings. We assume that while one os.listdir() -# call may be more expensive than one os.stat() call, a small number -# of os.stat() calls is quickly more expensive than caching the -# os.listdir() outcome, and the advantage of the latter is that it -# gives us the case-correct filename on Windows and Mac. -find_module_listdir_cache = {} # type: Dict[str, Optional[List[str]]] - - -def find_module_clear_caches(): - find_module_cache.clear() - find_module_dir_cache.clear() - find_module_listdir_cache.clear() - - -def list_dir(path: str) -> Optional[List[str]]: - """Return a cached directory listing. - - Returns None if the path doesn't exist or isn't a directory. - """ - if path in find_module_listdir_cache: - return find_module_listdir_cache[path] - try: - res = os.listdir(path) - except OSError: - res = None - find_module_listdir_cache[path] = res - return res - - -def is_file(path: str) -> bool: - """Return whether path exists and is a file. - - On case-insensitive filesystems (like Mac or Windows) this returns - False if the case of the path's last component does not exactly - match the case found in the filesystem. - """ - head, tail = os.path.split(path) - if not tail: - return False - names = list_dir(head) - if not names: - return False - if tail not in names: - return False - return os.path.isfile(path) - - -def find_module(id: str, lib_path: Iterable[str]) -> str: - """Return the path of the module source file, or None if not found.""" - if not isinstance(lib_path, tuple): - lib_path = tuple(lib_path) - - def find(): - # If we're looking for a module like 'foo.bar.baz', it's likely that most of the - # many elements of lib_path don't even have a subdirectory 'foo/bar'. Discover - # that only once and cache it for when we look for modules like 'foo.bar.blah' - # that will require the same subdirectory. - components = id.split('.') - dir_chain = os.sep.join(components[:-1]) # e.g., 'foo/bar' - if (dir_chain, lib_path) not in find_module_dir_cache: - dirs = [] - for pathitem in lib_path: - # e.g., '/usr/lib/python3.4/foo/bar' - dir = os.path.normpath(os.path.join(pathitem, dir_chain)) - if os.path.isdir(dir): - dirs.append(dir) - find_module_dir_cache[dir_chain, lib_path] = dirs - candidate_base_dirs = find_module_dir_cache[dir_chain, lib_path] - - # If we're looking for a module like 'foo.bar.baz', then candidate_base_dirs now - # contains just the subdirectories 'foo/bar' that actually exist under the - # elements of lib_path. This is probably much shorter than lib_path itself. - # Now just look for 'baz.pyi', 'baz/__init__.py', etc., inside those directories. - seplast = os.sep + components[-1] # so e.g. '/baz' - sepinit = os.sep + '__init__' - for base_dir in candidate_base_dirs: - base_path = base_dir + seplast # so e.g. '/usr/lib/python3.4/foo/bar/baz' - # Prefer package over module, i.e. baz/__init__.py* over baz.py*. - for extension in PYTHON_EXTENSIONS: - path = base_path + sepinit + extension - if is_file(path) and verify_module(id, path): - return path - # No package, look for module. - for extension in PYTHON_EXTENSIONS: - path = base_path + extension - if is_file(path) and verify_module(id, path): - return path - return None - - key = (id, lib_path) - if key not in find_module_cache: - find_module_cache[key] = find() - return find_module_cache[key] - - -def find_modules_recursive(module: str, lib_path: List[str]) -> List[BuildSource]: - module_path = find_module(module, lib_path) - if not module_path: - return [] - result = [BuildSource(module_path, module, None)] - if module_path.endswith(('__init__.py', '__init__.pyi')): - # Subtle: this code prefers the .pyi over the .py if both - # exists, and also prefers packages over modules if both x/ - # and x.py* exist. How? We sort the directory items, so x - # comes before x.py and x.pyi. But the preference for .pyi - # over .py is encoded in find_module(); even though we see - # x.py before x.pyi, find_module() will find x.pyi first. We - # use hits to avoid adding it a second time when we see x.pyi. - # This also avoids both x.py and x.pyi when x/ was seen first. - hits = set() # type: Set[str] - for item in sorted(os.listdir(os.path.dirname(module_path))): - abs_path = os.path.join(os.path.dirname(module_path), item) - if os.path.isdir(abs_path) and \ - (os.path.isfile(os.path.join(abs_path, '__init__.py')) or - os.path.isfile(os.path.join(abs_path, '__init__.pyi'))): - hits.add(item) - result += find_modules_recursive(module + '.' + item, lib_path) - elif item != '__init__.py' and item != '__init__.pyi' and \ - item.endswith(('.py', '.pyi')): - mod = item.split('.')[0] - if mod not in hits: - hits.add(mod) - result += find_modules_recursive( - module + '.' + mod, lib_path) - return result - - -def verify_module(id: str, path: str) -> bool: - """Check that all packages containing id have a __init__ file.""" - if path.endswith(('__init__.py', '__init__.pyi')): - path = dirname(path) - for i in range(id.count('.')): - path = dirname(path) - if not any(os.path.isfile(os.path.join(path, '__init__{}'.format(extension))) - for extension in PYTHON_EXTENSIONS): - return False - return True - - -def read_with_python_encoding(path: str, pyversion: Tuple[int, int]) -> str: - """Read the Python file with while obeying PEP-263 encoding detection""" - source_bytearray = bytearray() - encoding = 'utf8' if pyversion[0] >= 3 else 'ascii' - - with open(path, 'rb') as f: - # read first two lines and check if PEP-263 coding is present - source_bytearray.extend(f.readline()) - source_bytearray.extend(f.readline()) - - # check for BOM UTF-8 encoding and strip it out if present - if source_bytearray.startswith(b'\xef\xbb\xbf'): - encoding = 'utf8' - source_bytearray = source_bytearray[3:] - else: - _encoding, _ = util.find_python_encoding(source_bytearray, pyversion) - # check that the coding isn't mypy. We skip it since - # registering may not have happened yet - if _encoding != 'mypy': - encoding = _encoding - - source_bytearray.extend(f.read()) - try: - source_bytearray.decode(encoding) - except LookupError as lookuperr: - raise DecodeError(str(lookuperr)) - return source_bytearray.decode(encoding) - - -MYPY_CACHE = '.mypy_cache' - - -def get_cache_names(id: str, path: str, pyversion: Tuple[int, int]) -> Tuple[str, str]: - """Return the file names for the cache files. - - Args: - id: module ID - path: module path (used to recognize packages) - pyversion: Python version (major, minor) - - Returns: - A tuple with the file names to be used for the meta JSON and the - data JSON, respectively. - """ - prefix = os.path.join(MYPY_CACHE, '%d.%d' % pyversion, *id.split('.')) - is_package = os.path.basename(path).startswith('__init__.py') - if is_package: - prefix = os.path.join(prefix, '__init__') - return (prefix + '.meta.json', prefix + '.data.json') - - -def find_cache_meta(id: str, path: str, manager: BuildManager) -> Optional[CacheMeta]: - """Find cache data for a module. - - Args: - id: module ID - path: module path - manager: the build manager (for pyversion, log/trace, and build options) - - Returns: - A CacheMeta instance if the cache data was found and appears - valid; otherwise None. - """ - # TODO: May need to take more build options into account - meta_json, data_json = get_cache_names(id, path, manager.options.python_version) - manager.trace('Looking for {} {}'.format(id, data_json)) - if not os.path.exists(meta_json): - return None - with open(meta_json, 'r') as f: - meta_str = f.read() - manager.trace('Meta {} {}'.format(id, meta_str.rstrip())) - meta = json.loads(meta_str) # TODO: Errors - if not isinstance(meta, dict): - return None - path = os.path.abspath(path) - m = CacheMeta( - meta.get('id'), - meta.get('path'), - meta.get('mtime'), - meta.get('size'), - meta.get('dependencies', []), - meta.get('data_mtime'), - data_json, - meta.get('suppressed', []), - meta.get('options'), - meta.get('dep_prios', []), - ) - if (m.id != id or m.path != path or - m.mtime is None or m.size is None or - m.dependencies is None or m.data_mtime is None): - return None - - # Ignore cache if generated by an older mypy version. - if m.options is None or len(m.dependencies) != len(m.dep_prios): - return None - - # Ignore cache if (relevant) options aren't the same. - cached_options = m.options - current_options = select_options_affecting_cache(manager.options) - if cached_options != current_options: - return None - - # TODO: Share stat() outcome with find_module() - st = os.stat(path) # TODO: Errors - if st.st_mtime != m.mtime or st.st_size != m.size: - manager.log('Metadata abandoned because of modified file {}'.format(path)) - return None - # It's a match on (id, path, mtime, size). - # Check data_json; assume if its mtime matches it's good. - # TODO: stat() errors - if os.path.getmtime(data_json) != m.data_mtime: - return None - manager.log('Found {} {}'.format(id, meta_json)) - return m - - -def select_options_affecting_cache(options: Options) -> Mapping[str, bool]: - return {opt: getattr(options, opt) for opt in OPTIONS_AFFECTING_CACHE} - - -OPTIONS_AFFECTING_CACHE = [ - "silent_imports", - "almost_silent", - "disallow_untyped_calls", - "disallow_untyped_defs", - "check_untyped_defs", -] - - -def random_string(): - return binascii.hexlify(os.urandom(8)).decode('ascii') - - -def write_cache(id: str, path: str, tree: MypyFile, - dependencies: List[str], suppressed: List[str], - dep_prios: List[int], - manager: BuildManager) -> None: - """Write cache files for a module. - - Args: - id: module ID - path: module path - tree: the fully checked module data - dependencies: module IDs on which this module depends - suppressed: module IDs which were suppressed as dependencies - dep_prios: priorities (parallel array to dependencies) - manager: the build manager (for pyversion, log/trace) - """ - path = os.path.abspath(path) - manager.trace('Dumping {} {}'.format(id, path)) - st = os.stat(path) # TODO: Errors - mtime = st.st_mtime - size = st.st_size - meta_json, data_json = get_cache_names(id, path, manager.options.python_version) - manager.log('Writing {} {} {}'.format(id, meta_json, data_json)) - data = tree.serialize() - parent = os.path.dirname(data_json) - if not os.path.isdir(parent): - os.makedirs(parent) - assert os.path.dirname(meta_json) == parent - nonce = '.' + random_string() - data_json_tmp = data_json + nonce - meta_json_tmp = meta_json + nonce - with open(data_json_tmp, 'w') as f: - json.dump(data, f, indent=2, sort_keys=True) - f.write('\n') - data_mtime = os.path.getmtime(data_json_tmp) - meta = {'id': id, - 'path': path, - 'mtime': mtime, - 'size': size, - 'data_mtime': data_mtime, - 'dependencies': dependencies, - 'suppressed': suppressed, - 'options': select_options_affecting_cache(manager.options), - 'dep_prios': dep_prios, - } - with open(meta_json_tmp, 'w') as f: - json.dump(meta, f, sort_keys=True) - f.write('\n') - os.replace(data_json_tmp, data_json) - os.replace(meta_json_tmp, meta_json) - - -"""Dependency manager. - -Design -====== - -Ideally -------- - -A. Collapse cycles (each SCC -- strongly connected component -- - becomes one "supernode"). - -B. Topologically sort nodes based on dependencies. - -C. Process from leaves towards roots. - -Wrinkles --------- - -a. Need to parse source modules to determine dependencies. - -b. Processing order for modules within an SCC. - -c. Must order mtimes of files to decide whether to re-process; depends - on clock never resetting. - -d. from P import M; checks filesystem whether module P.M exists in - filesystem. - -e. Race conditions, where somebody modifies a file while we're - processing. I propose not to modify the algorithm to handle this, - but to detect when this could lead to inconsistencies. (For - example, when we decide on the dependencies based on cache - metadata, and then we decide to re-parse a file because of a stale - dependency, if the re-parsing leads to a different list of - dependencies we should warn the user or start over.) - -Steps ------ - -1. For each explicitly given module find the source file location. - -2. For each such module load and check the cache metadata, and decide - whether it's valid. - -3. Now recursively (or iteratively) find dependencies and add those to - the graph: - - - for cached nodes use the list of dependencies from the cache - metadata (this will be valid even if we later end up re-parsing - the same source); - - - for uncached nodes parse the file and process all imports found, - taking care of (a) above. - -Step 3 should also address (d) above. - -Once step 3 terminates we have the entire dependency graph, and for -each module we've either loaded the cache metadata or parsed the -source code. (However, we may still need to parse those modules for -which we have cache metadata but that depend, directly or indirectly, -on at least one module for which the cache metadata is stale.) - -Now we can execute steps A-C from the first section. Finding SCCs for -step A shouldn't be hard; there's a recipe here: -http://code.activestate.com/recipes/578507/. There's also a plethora -of topsort recipes, e.g. http://code.activestate.com/recipes/577413/. - -For single nodes, processing is simple. If the node was cached, we -deserialize the cache data and fix up cross-references. Otherwise, we -do semantic analysis followed by type checking. We also handle (c) -above; if a module has valid cache data *but* any of its -dependencies was processed from source, then the module should be -processed from source. - -A relatively simple optimization (outside SCCs) we might do in the -future is as follows: if a node's cache data is valid, but one or more -of its dependencies are out of date so we have to re-parse the node -from source, once we have fully type-checked the node, we can decide -whether its symbol table actually changed compared to the cache data -(by reading the cache data and comparing it to the data we would be -writing). If there is no change we can declare the node up to date, -and any node that depends (and for which we have cached data, and -whose other dependencies are up to date) on it won't need to be -re-parsed from source. - -Import cycles -------------- - -Finally we have to decide how to handle (c), import cycles. Here -we'll need a modified version of the original state machine -(build.py), but we only need to do this per SCC, and we won't have to -deal with changes to the list of nodes while we're processing it. - -If all nodes in the SCC have valid cache metadata and all dependencies -outside the SCC are still valid, we can proceed as follows: - - 1. Load cache data for all nodes in the SCC. - - 2. Fix up cross-references for all nodes in the SCC. - -Otherwise, the simplest (but potentially slow) way to proceed is to -invalidate all cache data in the SCC and re-parse all nodes in the SCC -from source. We can do this as follows: - - 1. Parse source for all nodes in the SCC. - - 2. Semantic analysis for all nodes in the SCC. - - 3. Type check all nodes in the SCC. - -(If there are more passes the process is the same -- each pass should -be done for all nodes before starting the next pass for any nodes in -the SCC.) - -We could process the nodes in the SCC in any order. For sentimental -reasons, I've decided to process them in the reverse order in which we -encountered them when originally constructing the graph. That's how -the old build.py deals with cycles, and at least this reproduces the -previous implementation more accurately. - -Can we do better than re-parsing all nodes in the SCC when any of its -dependencies are out of date? It's doubtful. The optimization -mentioned at the end of the previous section would require re-parsing -and type-checking a node and then comparing its symbol table to the -cached data; but because the node is part of a cycle we can't -technically type-check it until the semantic analysis of all other -nodes in the cycle has completed. (This is an important issue because -Dropbox has a very large cycle in production code. But I'd like to -deal with it later.) - -Additional wrinkles -------------------- - -During implementation more wrinkles were found. - -- When a submodule of a package (e.g. x.y) is encountered, the parent - package (e.g. x) must also be loaded, but it is not strictly a - dependency. See State.add_ancestors() below. -""" - - -class ModuleNotFound(Exception): - """Control flow exception to signal that a module was not found.""" - - -class State: - """The state for a module. - - The source is only used for the -c command line option; in that - case path is None. Otherwise source is None and path isn't. - """ - - manager = None # type: BuildManager - order_counter = 0 # Class variable - order = None # type: int # Order in which modules were encountered - id = None # type: str # Fully qualified module name - path = None # type: Optional[str] # Path to module source - xpath = None # type: str # Path or '' - source = None # type: Optional[str] # Module source code - meta = None # type: Optional[CacheMeta] - data = None # type: Optional[str] - tree = None # type: Optional[MypyFile] - dependencies = None # type: List[str] - suppressed = None # type: List[str] # Suppressed/missing dependencies - priorities = None # type: Dict[str, int] - - # Map each dependency to the line number where it is first imported - dep_line_map = None # type: Dict[str, int] - - # Parent package, its parent, etc. - ancestors = None # type: Optional[List[str]] - - # List of (path, line number) tuples giving context for import - import_context = None # type: List[Tuple[str, int]] - - # The State from which this module was imported, if any - caller_state = None # type: Optional[State] - - # If caller_state is set, the line number in the caller where the import occurred - caller_line = 0 - - def __init__(self, - id: Optional[str], - path: Optional[str], - source: Optional[str], - manager: BuildManager, - caller_state: 'State' = None, - caller_line: int = 0, - ancestor_for: 'State' = None, - ) -> None: - assert id or path or source is not None, "Neither id, path nor source given" - self.manager = manager - State.order_counter += 1 - self.order = State.order_counter - self.caller_state = caller_state - self.caller_line = caller_line - if caller_state: - self.import_context = caller_state.import_context[:] - self.import_context.append((caller_state.xpath, caller_line)) - else: - self.import_context = [] - self.id = id or '__main__' - if not path and source is None: - file_id = id - if id == 'builtins' and manager.options.python_version[0] == 2: - # The __builtin__ module is called internally by mypy - # 'builtins' in Python 2 mode (similar to Python 3), - # but the stub file is __builtin__.pyi. The reason is - # that a lot of code hard-codes 'builtins.x' and it's - # easier to work it around like this. It also means - # that the implementation can mostly ignore the - # difference and just assume 'builtins' everywhere, - # which simplifies code. - file_id = '__builtin__' - path = find_module(file_id, manager.lib_path) - if path: - # In silent mode, don't import .py files, except from stubs. - if (manager.options.silent_imports and - path.endswith('.py') and (caller_state or ancestor_for)): - # (Never silence builtins, even if it's a .py file; - # this can happen in tests!) - if (id != 'builtins' and - not ((caller_state and - caller_state.tree and - caller_state.tree.is_stub))): - if manager.options.almost_silent: - if ancestor_for: - self.skipping_ancestor(id, path, ancestor_for) - else: - self.skipping_module(id, path) - path = None - manager.missing_modules.add(id) - raise ModuleNotFound - else: - # Could not find a module. Typically the reason is a - # misspelled module name, missing stub, module not in - # search path or the module has not been installed. - if caller_state: - suppress_message = ((manager.options.silent_imports and - not manager.options.almost_silent) or - (caller_state.tree is not None and - 'import' in caller_state.tree.weak_opts)) - if not suppress_message: - save_import_context = manager.errors.import_context() - manager.errors.set_import_context(caller_state.import_context) - manager.module_not_found(caller_state.xpath, caller_line, id) - manager.errors.set_import_context(save_import_context) - manager.missing_modules.add(id) - raise ModuleNotFound - else: - # If we can't find a root source it's always fatal. - # TODO: This might hide non-fatal errors from - # root sources processed earlier. - raise CompileError(["mypy: can't find module '%s'" % id]) - self.path = path - self.xpath = path or '' - self.source = source - if path and source is None and manager.options.incremental: - self.meta = find_cache_meta(self.id, self.path, manager) - # TODO: Get mtime if not cached. - self.add_ancestors() - if self.meta: - # Make copies, since we may modify these and want to - # compare them to the originals later. - self.dependencies = list(self.meta.dependencies) - self.suppressed = list(self.meta.suppressed) - assert len(self.meta.dependencies) == len(self.meta.dep_prios) - self.priorities = {id: pri - for id, pri in zip(self.meta.dependencies, self.meta.dep_prios)} - self.dep_line_map = {} - else: - # Parse the file (and then some) to get the dependencies. - self.parse_file() - self.suppressed = [] - - def skipping_ancestor(self, id: str, path: str, ancestor_for: 'State') -> None: - # TODO: Read the path (the __init__.py file) and return - # immediately if it's empty or only contains comments. - # But beware, some package may be the ancestor of many modules, - # so we'd need to cache the decision. - manager = self.manager - manager.errors.set_import_context([]) - manager.errors.set_file(ancestor_for.xpath) - manager.errors.report(-1, "Ancestor package '%s' silently ignored" % (id,), - severity='note', only_once=True) - manager.errors.report(-1, "(Using --silent-imports, submodule passed on command line)", - severity='note', only_once=True) - manager.errors.report(-1, "(This note brought to you by --almost-silent)", - severity='note', only_once=True) - - def skipping_module(self, id: str, path: str) -> None: - assert self.caller_state, (id, path) - manager = self.manager - save_import_context = manager.errors.import_context() - manager.errors.set_import_context(self.caller_state.import_context) - manager.errors.set_file(self.caller_state.xpath) - line = self.caller_line - manager.errors.report(line, "Import of '%s' silently ignored" % (id,), - severity='note') - manager.errors.report(line, "(Using --silent-imports, module not passed on command line)", - severity='note', only_once=True) - manager.errors.report(line, "(This note courtesy of --almost-silent)", - severity='note', only_once=True) - manager.errors.set_import_context(save_import_context) - - def add_ancestors(self) -> None: - # All parent packages are new ancestors. - ancestors = [] - parent = self.id - while '.' in parent: - parent, _ = parent.rsplit('.', 1) - ancestors.append(parent) - self.ancestors = ancestors - - def is_fresh(self) -> bool: - """Return whether the cache data for this file is fresh.""" - # NOTE: self.dependencies may differ from - # self.meta.dependencies when a dependency is dropped due to - # suppression by --silent-imports. However when a suppressed - # dependency is added back we find out later in the process. - return self.meta is not None and self.dependencies == self.meta.dependencies - - def mark_stale(self) -> None: - """Throw away the cache data for this file, marking it as stale.""" - self.meta = None - - def check_blockers(self) -> None: - """Raise CompileError if a blocking error is detected.""" - if self.manager.errors.is_blockers(): - self.manager.log("Bailing due to blocking errors") - self.manager.errors.raise_error() - - @contextlib.contextmanager - def wrap_context(self) -> Iterator[None]: - save_import_context = self.manager.errors.import_context() - self.manager.errors.set_import_context(self.import_context) - try: - yield - except CompileError: - raise - except Exception as err: - report_internal_error(err, self.path, 0) - self.manager.errors.set_import_context(save_import_context) - self.check_blockers() - - # Methods for processing cached modules. - - def load_tree(self) -> None: - with open(self.meta.data_json) as f: - data = json.load(f) - # TODO: Assert data file wasn't changed. - self.tree = MypyFile.deserialize(data) - self.manager.modules[self.id] = self.tree - - def fix_cross_refs(self) -> None: - fixup_module_pass_one(self.tree, self.manager.modules) - - def calculate_mros(self) -> None: - fixup_module_pass_two(self.tree, self.manager.modules) - - # Methods for processing modules from source code. - - def parse_file(self) -> None: - if self.tree is not None: - # The file was already parsed (in __init__()). - return - - manager = self.manager - modules = manager.modules - manager.log("Parsing %s (%s)" % (self.xpath, self.id)) - - with self.wrap_context(): - source = self.source - self.source = None # We won't need it again. - if self.path and source is None: - try: - source = read_with_python_encoding(self.path, manager.options.python_version) - except IOError as ioerr: - raise CompileError([ - "mypy: can't read file '{}': {}".format(self.path, ioerr.strerror)]) - except (UnicodeDecodeError, DecodeError) as decodeerr: - raise CompileError([ - "mypy: can't decode file '{}': {}".format(self.path, str(decodeerr))]) - self.tree = manager.parse_file(self.id, self.xpath, source) - - modules[self.id] = self.tree - - # Do the first pass of semantic analysis: add top-level - # definitions in the file to the symbol table. We must do - # this before processing imports, since this may mark some - # import statements as unreachable. - first = FirstPass(manager.semantic_analyzer) - first.analyze(self.tree, self.xpath, self.id) - - # Initialize module symbol table, which was populated by the - # semantic analyzer. - # TODO: Why can't FirstPass .analyze() do this? - self.tree.names = manager.semantic_analyzer.globals - - # Compute (direct) dependencies. - # Add all direct imports (this is why we needed the first pass). - # Also keep track of each dependency's source line. - dependencies = [] - suppressed = [] - priorities = {} # type: Dict[str, int] # id -> priority - dep_line_map = {} # type: Dict[str, int] # id -> line - for pri, id, line in manager.all_imported_modules_in_file(self.tree): - priorities[id] = min(pri, priorities.get(id, PRI_ALL)) - if id == self.id: - continue - # Omit missing modules, as otherwise we could not type-check - # programs with missing modules. - if id in manager.missing_modules: - if id not in dep_line_map: - suppressed.append(id) - dep_line_map[id] = line - continue - if id == '': - # Must be from a relative import. - manager.errors.set_file(self.xpath) - manager.errors.report(line, "No parent module -- cannot perform relative import", - blocker=True) - continue - if id not in dep_line_map: - dependencies.append(id) - dep_line_map[id] = line - # Every module implicitly depends on builtins. - if self.id != 'builtins' and 'builtins' not in dep_line_map: - dependencies.append('builtins') - - # If self.dependencies is already set, it was read from the - # cache, but for some reason we're re-parsing the file. - # NOTE: What to do about race conditions (like editing the - # file while mypy runs)? A previous version of this code - # explicitly checked for this, but ran afoul of other reasons - # for differences (e.g. --silent-imports). - self.dependencies = dependencies - self.suppressed = suppressed - self.priorities = priorities - self.dep_line_map = dep_line_map - self.check_blockers() - - def patch_parent(self) -> None: - # Include module in the symbol table of the enclosing package. - if '.' not in self.id: - return - manager = self.manager - modules = manager.modules - parent, child = self.id.rsplit('.', 1) - if parent in modules: - manager.trace("Added %s.%s" % (parent, child)) - modules[parent].names[child] = SymbolTableNode(MODULE_REF, self.tree, parent) - else: - manager.log("Hm... couldn't add %s.%s" % (parent, child)) - - def semantic_analysis(self) -> None: - with self.wrap_context(): - self.manager.semantic_analyzer.visit_file(self.tree, self.xpath) - - def semantic_analysis_pass_three(self) -> None: - with self.wrap_context(): - self.manager.semantic_analyzer_pass3.visit_file(self.tree, self.xpath) - if self.manager.options.dump_type_stats: - dump_type_stats(self.tree, self.xpath) - - def type_check(self) -> None: - manager = self.manager - if manager.options.semantic_analysis_only: - return - with self.wrap_context(): - manager.type_checker.visit_file(self.tree, self.xpath) - if manager.options.dump_inference_stats: - dump_type_stats(self.tree, self.xpath, inferred=True, - typemap=manager.type_checker.type_map) - manager.report_file(self.tree) - - def write_cache(self) -> None: - if self.path and self.manager.options.incremental and not self.manager.errors.is_errors(): - dep_prios = [self.priorities.get(dep, PRI_HIGH) for dep in self.dependencies] - write_cache(self.id, self.path, self.tree, - list(self.dependencies), list(self.suppressed), - dep_prios, - self.manager) - - -Graph = Dict[str, State] - - -def dispatch(sources: List[BuildSource], manager: BuildManager) -> None: - manager.log("Using new dependency manager") - graph = load_graph(sources, manager) - manager.log("Loaded graph with %d nodes" % len(graph)) - process_graph(graph, manager) - if manager.options.warn_unused_ignores: - manager.errors.generate_unused_ignore_notes() - - -def load_graph(sources: List[BuildSource], manager: BuildManager) -> Graph: - """Given some source files, load the full dependency graph.""" - graph = {} # type: Graph - # The deque is used to implement breadth-first traversal. - # TODO: Consider whether to go depth-first instead. This may - # affect the order in which we process files within import cycles. - new = collections.deque() # type: collections.deque[State] - # Seed the graph with the initial root sources. - for bs in sources: - try: - st = State(id=bs.module, path=bs.path, source=bs.text, manager=manager) - except ModuleNotFound: - continue - if st.id in graph: - manager.errors.set_file(st.xpath) - manager.errors.report(1, "Duplicate module named '%s'" % st.id) - manager.errors.raise_error() - graph[st.id] = st - new.append(st) - # Collect dependencies. We go breadth-first. - while new: - st = new.popleft() - for dep in st.ancestors + st.dependencies: - if dep not in graph: - try: - if dep in st.ancestors: - # TODO: Why not 'if dep not in st.dependencies' ? - # Ancestors don't have import context. - newst = State(id=dep, path=None, source=None, manager=manager, - ancestor_for=st) - else: - newst = State(id=dep, path=None, source=None, manager=manager, - caller_state=st, caller_line=st.dep_line_map.get(dep, 1)) - except ModuleNotFound: - if dep in st.dependencies: - st.dependencies.remove(dep) - st.suppressed.append(dep) - else: - assert newst.id not in graph, newst.id - graph[newst.id] = newst - new.append(newst) - return graph - - -def process_graph(graph: Graph, manager: BuildManager) -> None: - """Process everything in dependency order.""" - sccs = sorted_components(graph) - manager.log("Found %d SCCs; largest has %d nodes" % - (len(sccs), max(len(scc) for scc in sccs))) - # We're processing SCCs from leaves (those without further - # dependencies) to roots (those from which everything else can be - # reached). - for ascc in sccs: - # Order the SCC's nodes using a heuristic. - # Note that ascc is a set, and scc is a list. - scc = order_ascc(graph, ascc) - # If builtins is in the list, move it last. (This is a bit of - # a hack, but it's necessary because the builtins module is - # part of a small cycle involving at least {builtins, abc, - # typing}. Of these, builtins must be processed last or else - # some builtin objects will be incompletely processed.) - if 'builtins' in ascc: - scc.remove('builtins') - scc.append('builtins') - if manager.options.verbosity >= 2: - for id in scc: - manager.trace("Priorities for %s:" % id, - " ".join("%s:%d" % (x, graph[id].priorities[x]) - for x in graph[id].dependencies - if x in ascc and x in graph[id].priorities)) - # Because the SCCs are presented in topological sort order, we - # don't need to look at dependencies recursively for staleness - # -- the immediate dependencies are sufficient. - stale_scc = {id for id in scc if not graph[id].is_fresh()} - fresh = not stale_scc - deps = set() - for id in scc: - deps.update(graph[id].dependencies) - deps -= ascc - stale_deps = {id for id in deps if not graph[id].is_fresh()} - fresh = fresh and not stale_deps - undeps = set() - if fresh: - # Check if any dependencies that were suppressed according - # to the cache have heen added back in this run. - # NOTE: Newly suppressed dependencies are handled by is_fresh(). - for id in scc: - undeps.update(graph[id].suppressed) - undeps &= graph.keys() - if undeps: - fresh = False - if fresh: - # All cache files are fresh. Check that no dependency's - # cache file is newer than any scc node's cache file. - oldest_in_scc = min(graph[id].meta.data_mtime for id in scc) - newest_in_deps = 0 if not deps else max(graph[dep].meta.data_mtime for dep in deps) - if manager.options.verbosity >= 3: # Dump all mtimes for extreme debugging. - all_ids = sorted(ascc | deps, key=lambda id: graph[id].meta.data_mtime) - for id in all_ids: - if id in scc: - if graph[id].meta.data_mtime < newest_in_deps: - key = "*id:" - else: - key = "id:" - else: - if graph[id].meta.data_mtime > oldest_in_scc: - key = "+dep:" - else: - key = "dep:" - manager.trace(" %5s %.0f %s" % (key, graph[id].meta.data_mtime, id)) - # If equal, give the benefit of the doubt, due to 1-sec time granularity - # (on some platforms). - if oldest_in_scc < newest_in_deps: - fresh = False - fresh_msg = "out of date by %.0f seconds" % (newest_in_deps - oldest_in_scc) - else: - fresh_msg = "fresh" - elif undeps: - fresh_msg = "stale due to changed suppression (%s)" % " ".join(sorted(undeps)) - elif stale_scc: - fresh_msg = "inherently stale (%s)" % " ".join(sorted(stale_scc)) - if stale_deps: - fresh_msg += " with stale deps (%s)" % " ".join(sorted(stale_deps)) - else: - fresh_msg = "stale due to deps (%s)" % " ".join(sorted(stale_deps)) - if len(scc) == 1: - manager.log("Processing SCC singleton (%s) as %s" % (" ".join(scc), fresh_msg)) - else: - manager.log("Processing SCC of size %d (%s) as %s" % - (len(scc), " ".join(scc), fresh_msg)) - if fresh: - process_fresh_scc(graph, scc) - else: - process_stale_scc(graph, scc) - - -def order_ascc(graph: Graph, ascc: AbstractSet[str], pri_max: int = PRI_ALL) -> List[str]: - """Come up with the ideal processing order within an SCC. - - Using the priorities assigned by all_imported_modules_in_file(), - try to reduce the cycle to a DAG, by omitting arcs representing - dependencies of lower priority. - - In the simplest case, if we have A <--> B where A has a top-level - "import B" (medium priority) but B only has the reverse "import A" - inside a function (low priority), we turn the cycle into a DAG by - dropping the B --> A arc, which leaves only A --> B. - - If all arcs have the same priority, we fall back to sorting by - reverse global order (the order in which modules were first - encountered). - - The algorithm is recursive, as follows: when as arcs of different - priorities are present, drop all arcs of the lowest priority, - identify SCCs in the resulting graph, and apply the algorithm to - each SCC thus found. The recursion is bounded because at each - recursion the spread in priorities is (at least) one less. - - In practice there are only a few priority levels (currently - N=3) and in the worst case we just carry out the same algorithm - for finding SCCs N times. Thus the complexity is no worse than - the complexity of the original SCC-finding algorithm -- see - strongly_connected_components() below for a reference. - """ - if len(ascc) == 1: - return [s for s in ascc] - pri_spread = set() - for id in ascc: - state = graph[id] - for dep in state.dependencies: - if dep in ascc: - pri = state.priorities.get(dep, PRI_HIGH) - if pri < pri_max: - pri_spread.add(pri) - if len(pri_spread) == 1: - # Filtered dependencies are uniform -- order by global order. - return sorted(ascc, key=lambda id: -graph[id].order) - pri_max = max(pri_spread) - sccs = sorted_components(graph, ascc, pri_max) - # The recursion is bounded by the len(pri_spread) check above. - return [s for ss in sccs for s in order_ascc(graph, ss, pri_max)] - - -def process_fresh_scc(graph: Graph, scc: List[str]) -> None: - """Process the modules in one SCC from their cached data.""" - for id in scc: - graph[id].load_tree() - for id in scc: - graph[id].patch_parent() - for id in scc: - graph[id].fix_cross_refs() - for id in scc: - graph[id].calculate_mros() - - -def process_stale_scc(graph: Graph, scc: List[str]) -> None: - """Process the modules in one SCC from source code.""" - for id in scc: - graph[id].mark_stale() - for id in scc: - # We may already have parsed the module, or not. - # If the former, parse_file() is a no-op. - graph[id].parse_file() - for id in scc: - graph[id].patch_parent() - for id in scc: - graph[id].semantic_analysis() - for id in scc: - graph[id].semantic_analysis_pass_three() - for id in scc: - graph[id].type_check() - graph[id].write_cache() - - -def sorted_components(graph: Graph, - vertices: Optional[AbstractSet[str]] = None, - pri_max: int = PRI_ALL) -> List[AbstractSet[str]]: - """Return the graph's SCCs, topologically sorted by dependencies. - - The sort order is from leaves (nodes without dependencies) to - roots (nodes on which no other nodes depend). - - This works for a subset of the full dependency graph too; - dependencies that aren't present in graph.keys() are ignored. - """ - # Compute SCCs. - if vertices is None: - vertices = set(graph) - edges = {id: deps_filtered(graph, vertices, id, pri_max) for id in vertices} - sccs = list(strongly_connected_components(vertices, edges)) - # Topsort. - sccsmap = {id: frozenset(scc) for scc in sccs for id in scc} - data = {} # type: Dict[AbstractSet[str], Set[AbstractSet[str]]] - for scc in sccs: - deps = set() # type: Set[AbstractSet[str]] - for id in scc: - deps.update(sccsmap[x] for x in deps_filtered(graph, vertices, id, pri_max)) - data[frozenset(scc)] = deps - res = [] - for ready in topsort(data): - # Sort the sets in ready by reversed smallest State.order. Examples: - # - # - If ready is [{x}, {y}], x.order == 1, y.order == 2, we get - # [{y}, {x}]. - # - # - If ready is [{a, b}, {c, d}], a.order == 1, b.order == 3, - # c.order == 2, d.order == 4, the sort keys become [1, 2] - # and the result is [{c, d}, {a, b}]. - res.extend(sorted(ready, - key=lambda scc: -min(graph[id].order for id in scc))) - return res - - -def deps_filtered(graph: Graph, vertices: AbstractSet[str], id: str, pri_max: int) -> List[str]: - """Filter dependencies for id with pri < pri_max.""" - if id not in vertices: - return [] - state = graph[id] - return [dep - for dep in state.dependencies - if dep in vertices and state.priorities.get(dep, PRI_HIGH) < pri_max] - - -def strongly_connected_components(vertices: AbstractSet[str], - edges: Dict[str, List[str]]) -> Iterator[Set[str]]: - """Compute Strongly Connected Components of a directed graph. - - Args: - vertices: the labels for the vertices - edges: for each vertex, gives the target vertices of its outgoing edges - - Returns: - An iterator yielding strongly connected components, each - represented as a set of vertices. Each input vertex will occur - exactly once; vertices not part of a SCC are returned as - singleton sets. - - From http://code.activestate.com/recipes/578507/. - """ - identified = set() # type: Set[str] - stack = [] # type: List[str] - index = {} # type: Dict[str, int] - boundaries = [] # type: List[int] - - def dfs(v: str) -> Iterator[Set[str]]: - index[v] = len(stack) - stack.append(v) - boundaries.append(index[v]) - - for w in edges[v]: - if w not in index: - # For Python >= 3.3, replace with "yield from dfs(w)" - for scc in dfs(w): - yield scc - elif w not in identified: - while index[w] < boundaries[-1]: - boundaries.pop() - - if boundaries[-1] == index[v]: - boundaries.pop() - scc = set(stack[index[v]:]) - del stack[index[v]:] - identified.update(scc) - yield scc - - for v in vertices: - if v not in index: - # For Python >= 3.3, replace with "yield from dfs(v)" - for scc in dfs(v): - yield scc - - -def topsort(data: Dict[AbstractSet[str], - Set[AbstractSet[str]]]) -> Iterable[Set[AbstractSet[str]]]: - """Topological sort. - - Args: - data: A map from SCCs (represented as frozen sets of strings) to - sets of SCCs, its dependencies. NOTE: This data structure - is modified in place -- for normalization purposes, - self-dependencies are removed and entries representing - orphans are added. - - Returns: - An iterator yielding sets of SCCs that have an equivalent - ordering. NOTE: The algorithm doesn't care about the internal - structure of SCCs. - - Example: - Suppose the input has the following structure: - - {A: {B, C}, B: {D}, C: {D}} - - This is normalized to: - - {A: {B, C}, B: {D}, C: {D}, D: {}} - - The algorithm will yield the following values: - - {D} - {B, C} - {A} - - From http://code.activestate.com/recipes/577413/. - """ - # TODO: Use a faster algorithm? - for k, v in data.items(): - v.discard(k) # Ignore self dependencies. - for item in set.union(*data.values()) - set(data.keys()): - data[item] = set() - while True: - ready = {item for item, dep in data.items() if not dep} - if not ready: - break - yield ready - data = {item: (dep - ready) - for item, dep in data.items() - if item not in ready} - assert not data, "A cyclic dependency exists amongst %r" % data +"""Facilities to analyze entire programs, including imported modules. + +Parse and analyze the source files of a program in the correct order +(based on file dependencies), and collect the results. + +This module only directs a build, which is performed in multiple passes per +file. The individual passes are implemented in separate modules. + +The function build() is the main interface to this module. +""" +# TODO: More consistent terminology, e.g. path/fnam, module/id, state/file + +import binascii +import collections +import contextlib +import json +import os +import os.path +import sys +import time +from os.path import dirname, basename + +from typing import (AbstractSet, Dict, Iterable, Iterator, List, + NamedTuple, Optional, Set, Tuple, Union, Mapping) + +from mypy.types import Type +from mypy.nodes import (MypyFile, Node, Import, ImportFrom, ImportAll, + SymbolTableNode, MODULE_REF) +from mypy.semanal import FirstPass, SemanticAnalyzer, ThirdPass +from mypy.checker import TypeChecker +from mypy.errors import Errors, CompileError, DecodeError, report_internal_error +from mypy import fixup +from mypy.report import Reports +from mypy import defaults +from mypy import moduleinfo +from mypy import util +from mypy.fixup import fixup_module_pass_one, fixup_module_pass_two +from mypy.options import Options +from mypy.parse import parse +from mypy.stats import dump_type_stats + + +# We need to know the location of this file to load data, but +# until Python 3.4, __file__ is relative. +__file__ = os.path.realpath(__file__) + +PYTHON_EXTENSIONS = ['.pyi', '.py'] + + +class BuildResult: + """The result of a successful build. + + Attributes: + manager: The build manager. + files: Dictionary from module name to related AST node. + types: Dictionary from parse tree node to its inferred type. + errors: List of error messages. + """ + + def __init__(self, manager: 'BuildManager') -> None: + self.manager = manager + self.files = manager.modules + self.types = manager.type_checker.type_map + self.errors = manager.errors.messages() + + +class BuildSource: + def __init__(self, path: Optional[str], module: Optional[str], + text: Optional[str]) -> None: + self.path = path + self.module = module or '__main__' + self.text = text + + @property + def effective_path(self) -> str: + """Return the effective path (ie, if its from in memory)""" + return self.path or '' + + +class BuildSourceSet: + """Efficiently test a file's membership in the set of build sources.""" + + def __init__(self, sources: List[BuildSource]) -> None: + self.source_text_present = False + self.source_modules = set() # type: Set[str] + self.source_paths = set() # type: Set[str] + + for source in sources: + if source.text is not None: + self.source_text_present = True + elif source.path: + self.source_paths.add(source.path) + else: + self.source_modules.add(source.module) + + def is_source(self, file: MypyFile) -> bool: + if file.path and file.path in self.source_paths: + return True + elif file._fullname in self.source_modules: + return True + elif file.path is None and self.source_text_present: + return True + else: + return False + + +def build(sources: List[BuildSource], + options: Options, + alt_lib_path: str = None, + bin_dir: str = None) -> BuildResult: + """Analyze a program. + + A single call to build performs parsing, semantic analysis and optionally + type checking for the program *and* all imported modules, recursively. + + Return BuildResult if successful or only non-blocking errors were found; + otherwise raise CompileError. + + Args: + sources: list of sources to build + options: build options + alt_lib_path: an additional directory for looking up library modules + (takes precedence over other directories) + bin_dir: directory containing the mypy script, used for finding data + directories; if omitted, use '.' as the data directory + """ + + data_dir = default_data_dir(bin_dir) + + find_module_clear_caches() + + # Determine the default module search path. + lib_path = default_lib_path(data_dir, options.python_version) + + if options.use_builtins_fixtures: + # Use stub builtins (to speed up test cases and to make them easier to + # debug). This is a test-only feature, so assume our files are laid out + # as in the source tree. + root_dir = os.path.dirname(os.path.dirname(__file__)) + lib_path.insert(0, os.path.join(root_dir, 'test-data', 'unit', 'lib-stub')) + else: + for source in sources: + if source.path: + # Include directory of the program file in the module search path. + dir = remove_cwd_prefix_from_path(dirname(source.path)) + if dir not in lib_path: + lib_path.insert(0, dir) + + # Do this even if running as a file, for sanity (mainly because with + # multiple builds, there could be a mix of files/modules, so its easier + # to just define the semantics that we always add the current director + # to the lib_path + lib_path.insert(0, os.getcwd()) + + # Add MYPYPATH environment variable to front of library path, if defined. + lib_path[:0] = mypy_path() + + # If provided, insert the caller-supplied extra module path to the + # beginning (highest priority) of the search path. + if alt_lib_path: + lib_path.insert(0, alt_lib_path) + + reports = Reports(data_dir, options.report_dirs) + + source_set = BuildSourceSet(sources) + + # Construct a build manager object to hold state during the build. + # + # Ignore current directory prefix in error messages. + manager = BuildManager(data_dir, lib_path, + ignore_prefix=os.getcwd(), + source_set=source_set, + reports=reports, + options=options) + + try: + dispatch(sources, manager) + return BuildResult(manager) + finally: + manager.log("Build finished with %d modules, %d types, and %d errors" % + (len(manager.modules), + len(manager.type_checker.type_map), + manager.errors.num_messages())) + # Finish the HTML or XML reports even if CompileError was raised. + reports.finish() + + +def default_data_dir(bin_dir: Optional[str]) -> str: + """Returns directory containing typeshed directory + + Args: + bin_dir: directory containing the mypy script + """ + if not bin_dir: + mypy_package = os.path.dirname(__file__) + parent = os.path.dirname(mypy_package) + if (os.path.basename(parent) == 'site-packages' or + os.path.basename(parent) == 'dist-packages'): + # Installed in site-packages or dist-packages, but invoked with python3 -m mypy; + # __file__ is .../blah/lib/python3.N/site-packages/mypy/build.py + # or .../blah/lib/python3.N/dist-packages/mypy/build.py (Debian) + # or .../blah/lib/site-packages/mypy/build.py (Windows) + # blah may be a virtualenv or /usr/local. We want .../blah/lib/mypy. + lib = parent + for i in range(2): + lib = os.path.dirname(lib) + if os.path.basename(lib) == 'lib': + return os.path.join(lib, 'mypy') + subdir = os.path.join(parent, 'lib', 'mypy') + if os.path.isdir(subdir): + # If installed via buildout, the __file__ is + # somewhere/mypy/__init__.py and what we want is + # somewhere/lib/mypy. + return subdir + # Default to directory containing this file's parent. + return parent + base = os.path.basename(bin_dir) + dir = os.path.dirname(bin_dir) + if (sys.platform == 'win32' and base.lower() == 'scripts' + and not os.path.isdir(os.path.join(dir, 'typeshed'))): + # Installed, on Windows. + return os.path.join(dir, 'Lib', 'mypy') + elif base == 'scripts': + # Assume that we have a repo check out or unpacked source tarball. + return dir + elif base == 'bin': + # Installed to somewhere (can be under /usr/local or anywhere). + return os.path.join(dir, 'lib', 'mypy') + elif base == 'python3': + # Assume we installed python3 with brew on os x + return os.path.join(os.path.dirname(dir), 'lib', 'mypy') + elif dir.endswith('python-exec'): + # Gentoo uses a python wrapper in /usr/lib to which mypy is a symlink. + return os.path.join(os.path.dirname(dir), 'mypy') + else: + # Don't know where to find the data files! + raise RuntimeError("Broken installation: can't determine base dir") + + +def mypy_path() -> List[str]: + path_env = os.getenv('MYPYPATH') + if not path_env: + return [] + return path_env.split(os.pathsep) + + +def default_lib_path(data_dir: str, pyversion: Tuple[int, int]) -> List[str]: + """Return default standard library search paths.""" + # IDEA: Make this more portable. + path = [] # type: List[str] + + auto = os.path.join(data_dir, 'stubs-auto') + if os.path.isdir(auto): + data_dir = auto + + # We allow a module for e.g. version 3.5 to be in 3.4/. The assumption + # is that a module added with 3.4 will still be present in Python 3.5. + versions = ["%d.%d" % (pyversion[0], minor) + for minor in reversed(range(pyversion[1] + 1))] + # E.g. for Python 3.2, try 3.2/, 3.1/, 3.0/, 3/, 2and3/. + # (Note that 3.1 and 3.0 aren't really supported, but we don't care.) + for v in versions + [str(pyversion[0]), '2and3']: + for lib_type in ['stdlib', 'third_party']: + stubdir = os.path.join(data_dir, 'typeshed', lib_type, v) + if os.path.isdir(stubdir): + path.append(stubdir) + + # Add fallback path that can be used if we have a broken installation. + if sys.platform != 'win32': + path.append('/usr/local/lib/mypy') + + return path + + +CacheMeta = NamedTuple('CacheMeta', + [('id', str), + ('path', str), + ('mtime', float), + ('size', int), + ('dependencies', List[str]), # names of imported modules + ('data_mtime', float), # mtime of data_json + ('data_json', str), # path of .data.json + ('suppressed', List[str]), # dependencies that weren't imported + ('options', Optional[Dict[str, bool]]), # build options + ('dep_prios', List[int]), + ]) +# NOTE: dependencies + suppressed == all reachable imports; +# suppressed contains those reachable imports that were prevented by +# --silent-imports or simply not found. + + +# Priorities used for imports. (Here, top-level includes inside a class.) +# These are used to determine a more predictable order in which the +# nodes in an import cycle are processed. +PRI_HIGH = 5 # top-level "from X import blah" +PRI_MED = 10 # top-level "import X" +PRI_LOW = 20 # either form inside a function +PRI_ALL = 99 # include all priorities + + +class BuildManager: + """This class holds shared state for building a mypy program. + + It is used to coordinate parsing, import processing, semantic + analysis and type checking. The actual build steps are carried + out by dispatch(). + + Attributes: + data_dir: Mypy data directory (contains stubs) + lib_path: Library path for looking up modules + modules: Mapping of module ID to MypyFile (shared by the passes) + semantic_analyzer: + Semantic analyzer, pass 2 + semantic_analyzer_pass3: + Semantic analyzer, pass 3 + type_checker: Type checker + errors: Used for reporting all errors + options: Build options + missing_modules: Set of modules that could not be imported encountered so far + """ + + def __init__(self, data_dir: str, + lib_path: List[str], + ignore_prefix: str, + source_set: BuildSourceSet, + reports: Reports, + options: Options) -> None: + self.start_time = time.time() + self.data_dir = data_dir + self.errors = Errors() + self.errors.set_ignore_prefix(ignore_prefix) + self.lib_path = tuple(lib_path) + self.source_set = source_set + self.reports = reports + self.options = options + self.semantic_analyzer = SemanticAnalyzer(lib_path, self.errors, options=options) + self.modules = self.semantic_analyzer.modules + self.semantic_analyzer_pass3 = ThirdPass(self.modules, self.errors) + self.type_checker = TypeChecker(self.errors, self.modules, options=options) + self.missing_modules = set() # type: Set[str] + + def all_imported_modules_in_file(self, + file: MypyFile) -> List[Tuple[int, str, int]]: + """Find all reachable import statements in a file. + + Return list of tuples (priority, module id, import line number) + for all modules imported in file; lower numbers == higher priority. + """ + + def correct_rel_imp(imp: Union[ImportFrom, ImportAll]) -> str: + """Function to correct for relative imports.""" + file_id = file.fullname() + rel = imp.relative + if rel == 0: + return imp.id + if os.path.basename(file.path).startswith('__init__.'): + rel -= 1 + if rel != 0: + file_id = ".".join(file_id.split(".")[:-rel]) + new_id = file_id + "." + imp.id if imp.id else file_id + + return new_id + + res = [] # type: List[Tuple[int, str, int]] + for imp in file.imports: + if not imp.is_unreachable: + if isinstance(imp, Import): + pri = PRI_MED if imp.is_top_level else PRI_LOW + for id, _ in imp.ids: + res.append((pri, id, imp.line)) + elif isinstance(imp, ImportFrom): + cur_id = correct_rel_imp(imp) + pos = len(res) + all_are_submodules = True + # Also add any imported names that are submodules. + pri = PRI_MED if imp.is_top_level else PRI_LOW + for name, __ in imp.names: + sub_id = cur_id + '.' + name + if self.is_module(sub_id): + res.append((pri, sub_id, imp.line)) + else: + all_are_submodules = False + # If all imported names are submodules, don't add + # cur_id as a dependency. Otherwise (i.e., if at + # least one imported name isn't a submodule) + # cur_id is also a dependency, and we should + # insert it *before* any submodules. + if not all_are_submodules: + pri = PRI_HIGH if imp.is_top_level else PRI_LOW + res.insert(pos, ((pri, cur_id, imp.line))) + elif isinstance(imp, ImportAll): + pri = PRI_HIGH if imp.is_top_level else PRI_LOW + res.append((pri, correct_rel_imp(imp), imp.line)) + + return res + + def is_module(self, id: str) -> bool: + """Is there a file in the file system corresponding to module id?""" + return find_module(id, self.lib_path) is not None + + def parse_file(self, id: str, path: str, source: str) -> MypyFile: + """Parse the source of a file with the given name. + + Raise CompileError if there is a parse error. + """ + num_errs = self.errors.num_messages() + tree = parse(source, path, self.errors, options=self.options) + tree._fullname = id + + # We don't want to warn about 'type: ignore' comments on + # imports, but we're about to modify tree.imports, so grab + # these first. + import_lines = set(node.line for node in tree.imports) + + # Skip imports that have been ignored (so that we can ignore a C extension module without + # stub, for example), except for 'from x import *', because we wouldn't be able to + # determine which names should be defined unless we process the module. We can still + # ignore errors such as redefinitions when using the latter form. + imports = [node for node in tree.imports + if node.line not in tree.ignored_lines or isinstance(node, ImportAll)] + tree.imports = imports + + if self.errors.num_messages() != num_errs: + self.log("Bailing due to parse errors") + self.errors.raise_error() + + self.errors.set_file_ignored_lines(path, tree.ignored_lines) + self.errors.mark_file_ignored_lines_used(path, import_lines) + return tree + + def module_not_found(self, path: str, line: int, id: str) -> None: + self.errors.set_file(path) + stub_msg = "(Stub files are from https://github.com/python/typeshed)" + if ((self.options.python_version[0] == 2 and moduleinfo.is_py2_std_lib_module(id)) or + (self.options.python_version[0] >= 3 and moduleinfo.is_py3_std_lib_module(id))): + self.errors.report( + line, "No library stub file for standard library module '{}'".format(id)) + self.errors.report(line, stub_msg, severity='note', only_once=True) + elif moduleinfo.is_third_party_module(id): + self.errors.report(line, "No library stub file for module '{}'".format(id)) + self.errors.report(line, stub_msg, severity='note', only_once=True) + else: + self.errors.report(line, "Cannot find module named '{}'".format(id)) + self.errors.report(line, '(Perhaps setting MYPYPATH ' + 'or using the "--silent-imports" flag would help)', + severity='note', only_once=True) + + def report_file(self, file: MypyFile) -> None: + if self.source_set.is_source(file): + self.reports.file(file, type_map=self.type_checker.type_map) + + def log(self, *message: str) -> None: + if self.options.verbosity >= 1: + print('%.3f:LOG: ' % (time.time() - self.start_time), *message, file=sys.stderr) + sys.stderr.flush() + + def trace(self, *message: str) -> None: + if self.options.verbosity >= 2: + print('%.3f:TRACE:' % (time.time() - self.start_time), *message, file=sys.stderr) + sys.stderr.flush() + + +def remove_cwd_prefix_from_path(p: str) -> str: + """Remove current working directory prefix from p, if present. + + Also crawl up until a directory without __init__.py is found. + + If the result would be empty, return '.' instead. + """ + cur = os.getcwd() + # Add separator to the end of the path, unless one is already present. + if basename(cur) != '': + cur += os.sep + # Compute root path. + while (p and + (os.path.isfile(os.path.join(p, '__init__.py')) or + os.path.isfile(os.path.join(p, '__init__.pyi')))): + dir, base = os.path.split(p) + if not base: + break + p = dir + # Remove current directory prefix from the path, if present. + if p.startswith(cur): + p = p[len(cur):] + # Avoid returning an empty path; replace that with '.'. + if p == '': + p = '.' + return p + + +# Cache find_module: (id, lib_path) -> result. +find_module_cache = {} # type: Dict[Tuple[str, Tuple[str, ...]], str] + +# Cache some repeated work within distinct find_module calls: finding which +# elements of lib_path have even the subdirectory they'd need for the module +# to exist. This is shared among different module ids when they differ only +# in the last component. +find_module_dir_cache = {} # type: Dict[Tuple[str, Tuple[str, ...]], List[str]] + +# Cache directory listings. We assume that while one os.listdir() +# call may be more expensive than one os.stat() call, a small number +# of os.stat() calls is quickly more expensive than caching the +# os.listdir() outcome, and the advantage of the latter is that it +# gives us the case-correct filename on Windows and Mac. +find_module_listdir_cache = {} # type: Dict[str, Optional[List[str]]] + + +def find_module_clear_caches(): + find_module_cache.clear() + find_module_dir_cache.clear() + find_module_listdir_cache.clear() + + +def list_dir(path: str) -> Optional[List[str]]: + """Return a cached directory listing. + + Returns None if the path doesn't exist or isn't a directory. + """ + if path in find_module_listdir_cache: + return find_module_listdir_cache[path] + try: + res = os.listdir(path) + except OSError: + res = None + find_module_listdir_cache[path] = res + return res + + +def is_file(path: str) -> bool: + """Return whether path exists and is a file. + + On case-insensitive filesystems (like Mac or Windows) this returns + False if the case of the path's last component does not exactly + match the case found in the filesystem. + """ + head, tail = os.path.split(path) + if not tail: + return False + names = list_dir(head) + if not names: + return False + if tail not in names: + return False + return os.path.isfile(path) + + +def find_module(id: str, lib_path: Iterable[str]) -> str: + """Return the path of the module source file, or None if not found.""" + if not isinstance(lib_path, tuple): + lib_path = tuple(lib_path) + + def find(): + # If we're looking for a module like 'foo.bar.baz', it's likely that most of the + # many elements of lib_path don't even have a subdirectory 'foo/bar'. Discover + # that only once and cache it for when we look for modules like 'foo.bar.blah' + # that will require the same subdirectory. + components = id.split('.') + dir_chain = os.sep.join(components[:-1]) # e.g., 'foo/bar' + if (dir_chain, lib_path) not in find_module_dir_cache: + dirs = [] + for pathitem in lib_path: + # e.g., '/usr/lib/python3.4/foo/bar' + dir = os.path.normpath(os.path.join(pathitem, dir_chain)) + if os.path.isdir(dir): + dirs.append(dir) + find_module_dir_cache[dir_chain, lib_path] = dirs + candidate_base_dirs = find_module_dir_cache[dir_chain, lib_path] + + # If we're looking for a module like 'foo.bar.baz', then candidate_base_dirs now + # contains just the subdirectories 'foo/bar' that actually exist under the + # elements of lib_path. This is probably much shorter than lib_path itself. + # Now just look for 'baz.pyi', 'baz/__init__.py', etc., inside those directories. + seplast = os.sep + components[-1] # so e.g. '/baz' + sepinit = os.sep + '__init__' + for base_dir in candidate_base_dirs: + base_path = base_dir + seplast # so e.g. '/usr/lib/python3.4/foo/bar/baz' + # Prefer package over module, i.e. baz/__init__.py* over baz.py*. + for extension in PYTHON_EXTENSIONS: + path = base_path + sepinit + extension + if is_file(path) and verify_module(id, path): + return path + # No package, look for module. + for extension in PYTHON_EXTENSIONS: + path = base_path + extension + if is_file(path) and verify_module(id, path): + return path + return None + + key = (id, lib_path) + if key not in find_module_cache: + find_module_cache[key] = find() + return find_module_cache[key] + + +def find_modules_recursive(module: str, lib_path: List[str]) -> List[BuildSource]: + module_path = find_module(module, lib_path) + if not module_path: + return [] + result = [BuildSource(module_path, module, None)] + if module_path.endswith(('__init__.py', '__init__.pyi')): + # Subtle: this code prefers the .pyi over the .py if both + # exists, and also prefers packages over modules if both x/ + # and x.py* exist. How? We sort the directory items, so x + # comes before x.py and x.pyi. But the preference for .pyi + # over .py is encoded in find_module(); even though we see + # x.py before x.pyi, find_module() will find x.pyi first. We + # use hits to avoid adding it a second time when we see x.pyi. + # This also avoids both x.py and x.pyi when x/ was seen first. + hits = set() # type: Set[str] + for item in sorted(os.listdir(os.path.dirname(module_path))): + abs_path = os.path.join(os.path.dirname(module_path), item) + if os.path.isdir(abs_path) and \ + (os.path.isfile(os.path.join(abs_path, '__init__.py')) or + os.path.isfile(os.path.join(abs_path, '__init__.pyi'))): + hits.add(item) + result += find_modules_recursive(module + '.' + item, lib_path) + elif item != '__init__.py' and item != '__init__.pyi' and \ + item.endswith(('.py', '.pyi')): + mod = item.split('.')[0] + if mod not in hits: + hits.add(mod) + result += find_modules_recursive( + module + '.' + mod, lib_path) + return result + + +def verify_module(id: str, path: str) -> bool: + """Check that all packages containing id have a __init__ file.""" + if path.endswith(('__init__.py', '__init__.pyi')): + path = dirname(path) + for i in range(id.count('.')): + path = dirname(path) + if not any(os.path.isfile(os.path.join(path, '__init__{}'.format(extension))) + for extension in PYTHON_EXTENSIONS): + return False + return True + + +def read_with_python_encoding(path: str, pyversion: Tuple[int, int]) -> str: + """Read the Python file with while obeying PEP-263 encoding detection""" + source_bytearray = bytearray() + encoding = 'utf8' if pyversion[0] >= 3 else 'ascii' + + with open(path, 'rb') as f: + # read first two lines and check if PEP-263 coding is present + source_bytearray.extend(f.readline()) + source_bytearray.extend(f.readline()) + + # check for BOM UTF-8 encoding and strip it out if present + if source_bytearray.startswith(b'\xef\xbb\xbf'): + encoding = 'utf8' + source_bytearray = source_bytearray[3:] + else: + _encoding, _ = util.find_python_encoding(source_bytearray, pyversion) + # check that the coding isn't mypy. We skip it since + # registering may not have happened yet + if _encoding != 'mypy': + encoding = _encoding + + source_bytearray.extend(f.read()) + try: + source_bytearray.decode(encoding) + except LookupError as lookuperr: + raise DecodeError(str(lookuperr)) + return source_bytearray.decode(encoding) + + +MYPY_CACHE = '.mypy_cache' + + +def get_cache_names(id: str, path: str, pyversion: Tuple[int, int]) -> Tuple[str, str]: + """Return the file names for the cache files. + + Args: + id: module ID + path: module path (used to recognize packages) + pyversion: Python version (major, minor) + + Returns: + A tuple with the file names to be used for the meta JSON and the + data JSON, respectively. + """ + prefix = os.path.join(MYPY_CACHE, '%d.%d' % pyversion, *id.split('.')) + is_package = os.path.basename(path).startswith('__init__.py') + if is_package: + prefix = os.path.join(prefix, '__init__') + return (prefix + '.meta.json', prefix + '.data.json') + + +def find_cache_meta(id: str, path: str, manager: BuildManager) -> Optional[CacheMeta]: + """Find cache data for a module. + + Args: + id: module ID + path: module path + manager: the build manager (for pyversion, log/trace, and build options) + + Returns: + A CacheMeta instance if the cache data was found and appears + valid; otherwise None. + """ + # TODO: May need to take more build options into account + meta_json, data_json = get_cache_names(id, path, manager.options.python_version) + manager.trace('Looking for {} {}'.format(id, data_json)) + if not os.path.exists(meta_json): + return None + with open(meta_json, 'r') as f: + meta_str = f.read() + manager.trace('Meta {} {}'.format(id, meta_str.rstrip())) + meta = json.loads(meta_str) # TODO: Errors + if not isinstance(meta, dict): + return None + path = os.path.abspath(path) + m = CacheMeta( + meta.get('id'), + meta.get('path'), + meta.get('mtime'), + meta.get('size'), + meta.get('dependencies', []), + meta.get('data_mtime'), + data_json, + meta.get('suppressed', []), + meta.get('options'), + meta.get('dep_prios', []), + ) + if (m.id != id or m.path != path or + m.mtime is None or m.size is None or + m.dependencies is None or m.data_mtime is None): + return None + + # Ignore cache if generated by an older mypy version. + if m.options is None or len(m.dependencies) != len(m.dep_prios): + return None + + # Ignore cache if (relevant) options aren't the same. + cached_options = m.options + current_options = select_options_affecting_cache(manager.options) + if cached_options != current_options: + return None + + # TODO: Share stat() outcome with find_module() + st = os.stat(path) # TODO: Errors + if st.st_mtime != m.mtime or st.st_size != m.size: + manager.log('Metadata abandoned because of modified file {}'.format(path)) + return None + # It's a match on (id, path, mtime, size). + # Check data_json; assume if its mtime matches it's good. + # TODO: stat() errors + if os.path.getmtime(data_json) != m.data_mtime: + return None + manager.log('Found {} {}'.format(id, meta_json)) + return m + + +def select_options_affecting_cache(options: Options) -> Mapping[str, bool]: + return {opt: getattr(options, opt) for opt in OPTIONS_AFFECTING_CACHE} + + +OPTIONS_AFFECTING_CACHE = [ + "silent_imports", + "almost_silent", + "disallow_untyped_calls", + "disallow_untyped_defs", + "check_untyped_defs", +] + + +def random_string(): + return binascii.hexlify(os.urandom(8)).decode('ascii') + + +def write_cache(id: str, path: str, tree: MypyFile, + dependencies: List[str], suppressed: List[str], + dep_prios: List[int], + manager: BuildManager) -> None: + """Write cache files for a module. + + Args: + id: module ID + path: module path + tree: the fully checked module data + dependencies: module IDs on which this module depends + suppressed: module IDs which were suppressed as dependencies + dep_prios: priorities (parallel array to dependencies) + manager: the build manager (for pyversion, log/trace) + """ + path = os.path.abspath(path) + manager.trace('Dumping {} {}'.format(id, path)) + st = os.stat(path) # TODO: Errors + mtime = st.st_mtime + size = st.st_size + meta_json, data_json = get_cache_names(id, path, manager.options.python_version) + manager.log('Writing {} {} {}'.format(id, meta_json, data_json)) + data = tree.serialize() + parent = os.path.dirname(data_json) + if not os.path.isdir(parent): + os.makedirs(parent) + assert os.path.dirname(meta_json) == parent + nonce = '.' + random_string() + data_json_tmp = data_json + nonce + meta_json_tmp = meta_json + nonce + with open(data_json_tmp, 'w') as f: + json.dump(data, f, indent=2, sort_keys=True) + f.write('\n') + data_mtime = os.path.getmtime(data_json_tmp) + meta = {'id': id, + 'path': path, + 'mtime': mtime, + 'size': size, + 'data_mtime': data_mtime, + 'dependencies': dependencies, + 'suppressed': suppressed, + 'options': select_options_affecting_cache(manager.options), + 'dep_prios': dep_prios, + } + with open(meta_json_tmp, 'w') as f: + json.dump(meta, f, sort_keys=True) + f.write('\n') + os.replace(data_json_tmp, data_json) + os.replace(meta_json_tmp, meta_json) + + +"""Dependency manager. + +Design +====== + +Ideally +------- + +A. Collapse cycles (each SCC -- strongly connected component -- + becomes one "supernode"). + +B. Topologically sort nodes based on dependencies. + +C. Process from leaves towards roots. + +Wrinkles +-------- + +a. Need to parse source modules to determine dependencies. + +b. Processing order for modules within an SCC. + +c. Must order mtimes of files to decide whether to re-process; depends + on clock never resetting. + +d. from P import M; checks filesystem whether module P.M exists in + filesystem. + +e. Race conditions, where somebody modifies a file while we're + processing. I propose not to modify the algorithm to handle this, + but to detect when this could lead to inconsistencies. (For + example, when we decide on the dependencies based on cache + metadata, and then we decide to re-parse a file because of a stale + dependency, if the re-parsing leads to a different list of + dependencies we should warn the user or start over.) + +Steps +----- + +1. For each explicitly given module find the source file location. + +2. For each such module load and check the cache metadata, and decide + whether it's valid. + +3. Now recursively (or iteratively) find dependencies and add those to + the graph: + + - for cached nodes use the list of dependencies from the cache + metadata (this will be valid even if we later end up re-parsing + the same source); + + - for uncached nodes parse the file and process all imports found, + taking care of (a) above. + +Step 3 should also address (d) above. + +Once step 3 terminates we have the entire dependency graph, and for +each module we've either loaded the cache metadata or parsed the +source code. (However, we may still need to parse those modules for +which we have cache metadata but that depend, directly or indirectly, +on at least one module for which the cache metadata is stale.) + +Now we can execute steps A-C from the first section. Finding SCCs for +step A shouldn't be hard; there's a recipe here: +http://code.activestate.com/recipes/578507/. There's also a plethora +of topsort recipes, e.g. http://code.activestate.com/recipes/577413/. + +For single nodes, processing is simple. If the node was cached, we +deserialize the cache data and fix up cross-references. Otherwise, we +do semantic analysis followed by type checking. We also handle (c) +above; if a module has valid cache data *but* any of its +dependencies was processed from source, then the module should be +processed from source. + +A relatively simple optimization (outside SCCs) we might do in the +future is as follows: if a node's cache data is valid, but one or more +of its dependencies are out of date so we have to re-parse the node +from source, once we have fully type-checked the node, we can decide +whether its symbol table actually changed compared to the cache data +(by reading the cache data and comparing it to the data we would be +writing). If there is no change we can declare the node up to date, +and any node that depends (and for which we have cached data, and +whose other dependencies are up to date) on it won't need to be +re-parsed from source. + +Import cycles +------------- + +Finally we have to decide how to handle (c), import cycles. Here +we'll need a modified version of the original state machine +(build.py), but we only need to do this per SCC, and we won't have to +deal with changes to the list of nodes while we're processing it. + +If all nodes in the SCC have valid cache metadata and all dependencies +outside the SCC are still valid, we can proceed as follows: + + 1. Load cache data for all nodes in the SCC. + + 2. Fix up cross-references for all nodes in the SCC. + +Otherwise, the simplest (but potentially slow) way to proceed is to +invalidate all cache data in the SCC and re-parse all nodes in the SCC +from source. We can do this as follows: + + 1. Parse source for all nodes in the SCC. + + 2. Semantic analysis for all nodes in the SCC. + + 3. Type check all nodes in the SCC. + +(If there are more passes the process is the same -- each pass should +be done for all nodes before starting the next pass for any nodes in +the SCC.) + +We could process the nodes in the SCC in any order. For sentimental +reasons, I've decided to process them in the reverse order in which we +encountered them when originally constructing the graph. That's how +the old build.py deals with cycles, and at least this reproduces the +previous implementation more accurately. + +Can we do better than re-parsing all nodes in the SCC when any of its +dependencies are out of date? It's doubtful. The optimization +mentioned at the end of the previous section would require re-parsing +and type-checking a node and then comparing its symbol table to the +cached data; but because the node is part of a cycle we can't +technically type-check it until the semantic analysis of all other +nodes in the cycle has completed. (This is an important issue because +Dropbox has a very large cycle in production code. But I'd like to +deal with it later.) + +Additional wrinkles +------------------- + +During implementation more wrinkles were found. + +- When a submodule of a package (e.g. x.y) is encountered, the parent + package (e.g. x) must also be loaded, but it is not strictly a + dependency. See State.add_ancestors() below. +""" + + +class ModuleNotFound(Exception): + """Control flow exception to signal that a module was not found.""" + + +class State: + """The state for a module. + + The source is only used for the -c command line option; in that + case path is None. Otherwise source is None and path isn't. + """ + + manager = None # type: BuildManager + order_counter = 0 # Class variable + order = None # type: int # Order in which modules were encountered + id = None # type: str # Fully qualified module name + path = None # type: Optional[str] # Path to module source + xpath = None # type: str # Path or '' + source = None # type: Optional[str] # Module source code + meta = None # type: Optional[CacheMeta] + data = None # type: Optional[str] + tree = None # type: Optional[MypyFile] + dependencies = None # type: List[str] + suppressed = None # type: List[str] # Suppressed/missing dependencies + priorities = None # type: Dict[str, int] + + # Map each dependency to the line number where it is first imported + dep_line_map = None # type: Dict[str, int] + + # Parent package, its parent, etc. + ancestors = None # type: Optional[List[str]] + + # List of (path, line number) tuples giving context for import + import_context = None # type: List[Tuple[str, int]] + + # The State from which this module was imported, if any + caller_state = None # type: Optional[State] + + # If caller_state is set, the line number in the caller where the import occurred + caller_line = 0 + + def __init__(self, + id: Optional[str], + path: Optional[str], + source: Optional[str], + manager: BuildManager, + caller_state: 'State' = None, + caller_line: int = 0, + ancestor_for: 'State' = None, + ) -> None: + assert id or path or source is not None, "Neither id, path nor source given" + self.manager = manager + State.order_counter += 1 + self.order = State.order_counter + self.caller_state = caller_state + self.caller_line = caller_line + if caller_state: + self.import_context = caller_state.import_context[:] + self.import_context.append((caller_state.xpath, caller_line)) + else: + self.import_context = [] + self.id = id or '__main__' + if not path and source is None: + file_id = id + if id == 'builtins' and manager.options.python_version[0] == 2: + # The __builtin__ module is called internally by mypy + # 'builtins' in Python 2 mode (similar to Python 3), + # but the stub file is __builtin__.pyi. The reason is + # that a lot of code hard-codes 'builtins.x' and it's + # easier to work it around like this. It also means + # that the implementation can mostly ignore the + # difference and just assume 'builtins' everywhere, + # which simplifies code. + file_id = '__builtin__' + path = find_module(file_id, manager.lib_path) + if path: + # In silent mode, don't import .py files, except from stubs. + if (manager.options.silent_imports and + path.endswith('.py') and (caller_state or ancestor_for)): + # (Never silence builtins, even if it's a .py file; + # this can happen in tests!) + if (id != 'builtins' and + not ((caller_state and + caller_state.tree and + caller_state.tree.is_stub))): + if manager.options.almost_silent: + if ancestor_for: + self.skipping_ancestor(id, path, ancestor_for) + else: + self.skipping_module(id, path) + path = None + manager.missing_modules.add(id) + raise ModuleNotFound + else: + # Could not find a module. Typically the reason is a + # misspelled module name, missing stub, module not in + # search path or the module has not been installed. + if caller_state: + suppress_message = ((manager.options.silent_imports and + not manager.options.almost_silent) or + (caller_state.tree is not None and + 'import' in caller_state.tree.weak_opts)) + if not suppress_message: + save_import_context = manager.errors.import_context() + manager.errors.set_import_context(caller_state.import_context) + manager.module_not_found(caller_state.xpath, caller_line, id) + manager.errors.set_import_context(save_import_context) + manager.missing_modules.add(id) + raise ModuleNotFound + else: + # If we can't find a root source it's always fatal. + # TODO: This might hide non-fatal errors from + # root sources processed earlier. + raise CompileError(["mypy: can't find module '%s'" % id]) + self.path = path + self.xpath = path or '' + self.source = source + if path and source is None and manager.options.incremental: + self.meta = find_cache_meta(self.id, self.path, manager) + # TODO: Get mtime if not cached. + self.add_ancestors() + if self.meta: + # Make copies, since we may modify these and want to + # compare them to the originals later. + self.dependencies = list(self.meta.dependencies) + self.suppressed = list(self.meta.suppressed) + assert len(self.meta.dependencies) == len(self.meta.dep_prios) + self.priorities = {id: pri + for id, pri in zip(self.meta.dependencies, self.meta.dep_prios)} + self.dep_line_map = {} + else: + # Parse the file (and then some) to get the dependencies. + self.parse_file() + self.suppressed = [] + + def skipping_ancestor(self, id: str, path: str, ancestor_for: 'State') -> None: + # TODO: Read the path (the __init__.py file) and return + # immediately if it's empty or only contains comments. + # But beware, some package may be the ancestor of many modules, + # so we'd need to cache the decision. + manager = self.manager + manager.errors.set_import_context([]) + manager.errors.set_file(ancestor_for.xpath) + manager.errors.report(-1, "Ancestor package '%s' silently ignored" % (id,), + severity='note', only_once=True) + manager.errors.report(-1, "(Using --silent-imports, submodule passed on command line)", + severity='note', only_once=True) + manager.errors.report(-1, "(This note brought to you by --almost-silent)", + severity='note', only_once=True) + + def skipping_module(self, id: str, path: str) -> None: + assert self.caller_state, (id, path) + manager = self.manager + save_import_context = manager.errors.import_context() + manager.errors.set_import_context(self.caller_state.import_context) + manager.errors.set_file(self.caller_state.xpath) + line = self.caller_line + manager.errors.report(line, "Import of '%s' silently ignored" % (id,), + severity='note') + manager.errors.report(line, "(Using --silent-imports, module not passed on command line)", + severity='note', only_once=True) + manager.errors.report(line, "(This note courtesy of --almost-silent)", + severity='note', only_once=True) + manager.errors.set_import_context(save_import_context) + + def add_ancestors(self) -> None: + # All parent packages are new ancestors. + ancestors = [] + parent = self.id + while '.' in parent: + parent, _ = parent.rsplit('.', 1) + ancestors.append(parent) + self.ancestors = ancestors + + def is_fresh(self) -> bool: + """Return whether the cache data for this file is fresh.""" + # NOTE: self.dependencies may differ from + # self.meta.dependencies when a dependency is dropped due to + # suppression by --silent-imports. However when a suppressed + # dependency is added back we find out later in the process. + return self.meta is not None and self.dependencies == self.meta.dependencies + + def mark_stale(self) -> None: + """Throw away the cache data for this file, marking it as stale.""" + self.meta = None + + def check_blockers(self) -> None: + """Raise CompileError if a blocking error is detected.""" + if self.manager.errors.is_blockers(): + self.manager.log("Bailing due to blocking errors") + self.manager.errors.raise_error() + + @contextlib.contextmanager + def wrap_context(self) -> Iterator[None]: + save_import_context = self.manager.errors.import_context() + self.manager.errors.set_import_context(self.import_context) + try: + yield + except CompileError: + raise + except Exception as err: + report_internal_error(err, self.path, 0) + self.manager.errors.set_import_context(save_import_context) + self.check_blockers() + + # Methods for processing cached modules. + + def load_tree(self) -> None: + with open(self.meta.data_json) as f: + data = json.load(f) + # TODO: Assert data file wasn't changed. + self.tree = MypyFile.deserialize(data) + self.manager.modules[self.id] = self.tree + + def fix_cross_refs(self) -> None: + fixup_module_pass_one(self.tree, self.manager.modules) + + def calculate_mros(self) -> None: + fixup_module_pass_two(self.tree, self.manager.modules) + + # Methods for processing modules from source code. + + def parse_file(self) -> None: + if self.tree is not None: + # The file was already parsed (in __init__()). + return + + manager = self.manager + modules = manager.modules + manager.log("Parsing %s (%s)" % (self.xpath, self.id)) + + with self.wrap_context(): + source = self.source + self.source = None # We won't need it again. + if self.path and source is None: + try: + source = read_with_python_encoding(self.path, manager.options.python_version) + except IOError as ioerr: + raise CompileError([ + "mypy: can't read file '{}': {}".format(self.path, ioerr.strerror)]) + except (UnicodeDecodeError, DecodeError) as decodeerr: + raise CompileError([ + "mypy: can't decode file '{}': {}".format(self.path, str(decodeerr))]) + self.tree = manager.parse_file(self.id, self.xpath, source) + + modules[self.id] = self.tree + + # Do the first pass of semantic analysis: add top-level + # definitions in the file to the symbol table. We must do + # this before processing imports, since this may mark some + # import statements as unreachable. + first = FirstPass(manager.semantic_analyzer) + first.analyze(self.tree, self.xpath, self.id) + + # Initialize module symbol table, which was populated by the + # semantic analyzer. + # TODO: Why can't FirstPass .analyze() do this? + self.tree.names = manager.semantic_analyzer.globals + + # Compute (direct) dependencies. + # Add all direct imports (this is why we needed the first pass). + # Also keep track of each dependency's source line. + dependencies = [] + suppressed = [] + priorities = {} # type: Dict[str, int] # id -> priority + dep_line_map = {} # type: Dict[str, int] # id -> line + for pri, id, line in manager.all_imported_modules_in_file(self.tree): + priorities[id] = min(pri, priorities.get(id, PRI_ALL)) + if id == self.id: + continue + # Omit missing modules, as otherwise we could not type-check + # programs with missing modules. + if id in manager.missing_modules: + if id not in dep_line_map: + suppressed.append(id) + dep_line_map[id] = line + continue + if id == '': + # Must be from a relative import. + manager.errors.set_file(self.xpath) + manager.errors.report(line, "No parent module -- cannot perform relative import", + blocker=True) + continue + if id not in dep_line_map: + dependencies.append(id) + dep_line_map[id] = line + # Every module implicitly depends on builtins. + if self.id != 'builtins' and 'builtins' not in dep_line_map: + dependencies.append('builtins') + + # If self.dependencies is already set, it was read from the + # cache, but for some reason we're re-parsing the file. + # NOTE: What to do about race conditions (like editing the + # file while mypy runs)? A previous version of this code + # explicitly checked for this, but ran afoul of other reasons + # for differences (e.g. --silent-imports). + self.dependencies = dependencies + self.suppressed = suppressed + self.priorities = priorities + self.dep_line_map = dep_line_map + self.check_blockers() + + def patch_parent(self) -> None: + # Include module in the symbol table of the enclosing package. + if '.' not in self.id: + return + manager = self.manager + modules = manager.modules + parent, child = self.id.rsplit('.', 1) + if parent in modules: + manager.trace("Added %s.%s" % (parent, child)) + modules[parent].names[child] = SymbolTableNode(MODULE_REF, self.tree, parent) + else: + manager.log("Hm... couldn't add %s.%s" % (parent, child)) + + def semantic_analysis(self) -> None: + with self.wrap_context(): + self.manager.semantic_analyzer.visit_file(self.tree, self.xpath) + + def semantic_analysis_pass_three(self) -> None: + with self.wrap_context(): + self.manager.semantic_analyzer_pass3.visit_file(self.tree, self.xpath) + if self.manager.options.dump_type_stats: + dump_type_stats(self.tree, self.xpath) + + def type_check(self) -> None: + manager = self.manager + if manager.options.semantic_analysis_only: + return + with self.wrap_context(): + manager.type_checker.visit_file(self.tree, self.xpath) + if manager.options.dump_inference_stats: + dump_type_stats(self.tree, self.xpath, inferred=True, + typemap=manager.type_checker.type_map) + manager.report_file(self.tree) + + def write_cache(self) -> None: + if self.path and self.manager.options.incremental and not self.manager.errors.is_errors(): + dep_prios = [self.priorities.get(dep, PRI_HIGH) for dep in self.dependencies] + write_cache(self.id, self.path, self.tree, + list(self.dependencies), list(self.suppressed), + dep_prios, + self.manager) + + +Graph = Dict[str, State] + + +def dispatch(sources: List[BuildSource], manager: BuildManager) -> None: + manager.log("Using new dependency manager") + graph = load_graph(sources, manager) + manager.log("Loaded graph with %d nodes" % len(graph)) + process_graph(graph, manager) + if manager.options.warn_unused_ignores: + manager.errors.generate_unused_ignore_notes() + + +def load_graph(sources: List[BuildSource], manager: BuildManager) -> Graph: + """Given some source files, load the full dependency graph.""" + graph = {} # type: Graph + # The deque is used to implement breadth-first traversal. + # TODO: Consider whether to go depth-first instead. This may + # affect the order in which we process files within import cycles. + new = collections.deque() # type: collections.deque[State] + # Seed the graph with the initial root sources. + for bs in sources: + try: + st = State(id=bs.module, path=bs.path, source=bs.text, manager=manager) + except ModuleNotFound: + continue + if st.id in graph: + manager.errors.set_file(st.xpath) + manager.errors.report(1, "Duplicate module named '%s'" % st.id) + manager.errors.raise_error() + graph[st.id] = st + new.append(st) + # Collect dependencies. We go breadth-first. + while new: + st = new.popleft() + for dep in st.ancestors + st.dependencies: + if dep not in graph: + try: + if dep in st.ancestors: + # TODO: Why not 'if dep not in st.dependencies' ? + # Ancestors don't have import context. + newst = State(id=dep, path=None, source=None, manager=manager, + ancestor_for=st) + else: + newst = State(id=dep, path=None, source=None, manager=manager, + caller_state=st, caller_line=st.dep_line_map.get(dep, 1)) + except ModuleNotFound: + if dep in st.dependencies: + st.dependencies.remove(dep) + st.suppressed.append(dep) + else: + assert newst.id not in graph, newst.id + graph[newst.id] = newst + new.append(newst) + return graph + + +def process_graph(graph: Graph, manager: BuildManager) -> None: + """Process everything in dependency order.""" + sccs = sorted_components(graph) + manager.log("Found %d SCCs; largest has %d nodes" % + (len(sccs), max(len(scc) for scc in sccs))) + # We're processing SCCs from leaves (those without further + # dependencies) to roots (those from which everything else can be + # reached). + for ascc in sccs: + # Order the SCC's nodes using a heuristic. + # Note that ascc is a set, and scc is a list. + scc = order_ascc(graph, ascc) + # If builtins is in the list, move it last. (This is a bit of + # a hack, but it's necessary because the builtins module is + # part of a small cycle involving at least {builtins, abc, + # typing}. Of these, builtins must be processed last or else + # some builtin objects will be incompletely processed.) + if 'builtins' in ascc: + scc.remove('builtins') + scc.append('builtins') + if manager.options.verbosity >= 2: + for id in scc: + manager.trace("Priorities for %s:" % id, + " ".join("%s:%d" % (x, graph[id].priorities[x]) + for x in graph[id].dependencies + if x in ascc and x in graph[id].priorities)) + # Because the SCCs are presented in topological sort order, we + # don't need to look at dependencies recursively for staleness + # -- the immediate dependencies are sufficient. + stale_scc = {id for id in scc if not graph[id].is_fresh()} + fresh = not stale_scc + deps = set() + for id in scc: + deps.update(graph[id].dependencies) + deps -= ascc + stale_deps = {id for id in deps if not graph[id].is_fresh()} + fresh = fresh and not stale_deps + undeps = set() + if fresh: + # Check if any dependencies that were suppressed according + # to the cache have heen added back in this run. + # NOTE: Newly suppressed dependencies are handled by is_fresh(). + for id in scc: + undeps.update(graph[id].suppressed) + undeps &= graph.keys() + if undeps: + fresh = False + if fresh: + # All cache files are fresh. Check that no dependency's + # cache file is newer than any scc node's cache file. + oldest_in_scc = min(graph[id].meta.data_mtime for id in scc) + newest_in_deps = 0 if not deps else max(graph[dep].meta.data_mtime for dep in deps) + if manager.options.verbosity >= 3: # Dump all mtimes for extreme debugging. + all_ids = sorted(ascc | deps, key=lambda id: graph[id].meta.data_mtime) + for id in all_ids: + if id in scc: + if graph[id].meta.data_mtime < newest_in_deps: + key = "*id:" + else: + key = "id:" + else: + if graph[id].meta.data_mtime > oldest_in_scc: + key = "+dep:" + else: + key = "dep:" + manager.trace(" %5s %.0f %s" % (key, graph[id].meta.data_mtime, id)) + # If equal, give the benefit of the doubt, due to 1-sec time granularity + # (on some platforms). + if oldest_in_scc < newest_in_deps: + fresh = False + fresh_msg = "out of date by %.0f seconds" % (newest_in_deps - oldest_in_scc) + else: + fresh_msg = "fresh" + elif undeps: + fresh_msg = "stale due to changed suppression (%s)" % " ".join(sorted(undeps)) + elif stale_scc: + fresh_msg = "inherently stale (%s)" % " ".join(sorted(stale_scc)) + if stale_deps: + fresh_msg += " with stale deps (%s)" % " ".join(sorted(stale_deps)) + else: + fresh_msg = "stale due to deps (%s)" % " ".join(sorted(stale_deps)) + if len(scc) == 1: + manager.log("Processing SCC singleton (%s) as %s" % (" ".join(scc), fresh_msg)) + else: + manager.log("Processing SCC of size %d (%s) as %s" % + (len(scc), " ".join(scc), fresh_msg)) + if fresh: + process_fresh_scc(graph, scc) + else: + process_stale_scc(graph, scc) + + +def order_ascc(graph: Graph, ascc: AbstractSet[str], pri_max: int = PRI_ALL) -> List[str]: + """Come up with the ideal processing order within an SCC. + + Using the priorities assigned by all_imported_modules_in_file(), + try to reduce the cycle to a DAG, by omitting arcs representing + dependencies of lower priority. + + In the simplest case, if we have A <--> B where A has a top-level + "import B" (medium priority) but B only has the reverse "import A" + inside a function (low priority), we turn the cycle into a DAG by + dropping the B --> A arc, which leaves only A --> B. + + If all arcs have the same priority, we fall back to sorting by + reverse global order (the order in which modules were first + encountered). + + The algorithm is recursive, as follows: when as arcs of different + priorities are present, drop all arcs of the lowest priority, + identify SCCs in the resulting graph, and apply the algorithm to + each SCC thus found. The recursion is bounded because at each + recursion the spread in priorities is (at least) one less. + + In practice there are only a few priority levels (currently + N=3) and in the worst case we just carry out the same algorithm + for finding SCCs N times. Thus the complexity is no worse than + the complexity of the original SCC-finding algorithm -- see + strongly_connected_components() below for a reference. + """ + if len(ascc) == 1: + return [s for s in ascc] + pri_spread = set() + for id in ascc: + state = graph[id] + for dep in state.dependencies: + if dep in ascc: + pri = state.priorities.get(dep, PRI_HIGH) + if pri < pri_max: + pri_spread.add(pri) + if len(pri_spread) == 1: + # Filtered dependencies are uniform -- order by global order. + return sorted(ascc, key=lambda id: -graph[id].order) + pri_max = max(pri_spread) + sccs = sorted_components(graph, ascc, pri_max) + # The recursion is bounded by the len(pri_spread) check above. + return [s for ss in sccs for s in order_ascc(graph, ss, pri_max)] + + +def process_fresh_scc(graph: Graph, scc: List[str]) -> None: + """Process the modules in one SCC from their cached data.""" + for id in scc: + graph[id].load_tree() + for id in scc: + graph[id].patch_parent() + for id in scc: + graph[id].fix_cross_refs() + for id in scc: + graph[id].calculate_mros() + + +def process_stale_scc(graph: Graph, scc: List[str]) -> None: + """Process the modules in one SCC from source code.""" + for id in scc: + graph[id].mark_stale() + for id in scc: + # We may already have parsed the module, or not. + # If the former, parse_file() is a no-op. + graph[id].parse_file() + for id in scc: + graph[id].patch_parent() + for id in scc: + graph[id].semantic_analysis() + for id in scc: + graph[id].semantic_analysis_pass_three() + for id in scc: + graph[id].type_check() + graph[id].write_cache() + + +def sorted_components(graph: Graph, + vertices: Optional[AbstractSet[str]] = None, + pri_max: int = PRI_ALL) -> List[AbstractSet[str]]: + """Return the graph's SCCs, topologically sorted by dependencies. + + The sort order is from leaves (nodes without dependencies) to + roots (nodes on which no other nodes depend). + + This works for a subset of the full dependency graph too; + dependencies that aren't present in graph.keys() are ignored. + """ + # Compute SCCs. + if vertices is None: + vertices = set(graph) + edges = {id: deps_filtered(graph, vertices, id, pri_max) for id in vertices} + sccs = list(strongly_connected_components(vertices, edges)) + # Topsort. + sccsmap = {id: frozenset(scc) for scc in sccs for id in scc} + data = {} # type: Dict[AbstractSet[str], Set[AbstractSet[str]]] + for scc in sccs: + deps = set() # type: Set[AbstractSet[str]] + for id in scc: + deps.update(sccsmap[x] for x in deps_filtered(graph, vertices, id, pri_max)) + data[frozenset(scc)] = deps + res = [] + for ready in topsort(data): + # Sort the sets in ready by reversed smallest State.order. Examples: + # + # - If ready is [{x}, {y}], x.order == 1, y.order == 2, we get + # [{y}, {x}]. + # + # - If ready is [{a, b}, {c, d}], a.order == 1, b.order == 3, + # c.order == 2, d.order == 4, the sort keys become [1, 2] + # and the result is [{c, d}, {a, b}]. + res.extend(sorted(ready, + key=lambda scc: -min(graph[id].order for id in scc))) + return res + + +def deps_filtered(graph: Graph, vertices: AbstractSet[str], id: str, pri_max: int) -> List[str]: + """Filter dependencies for id with pri < pri_max.""" + if id not in vertices: + return [] + state = graph[id] + return [dep + for dep in state.dependencies + if dep in vertices and state.priorities.get(dep, PRI_HIGH) < pri_max] + + +def strongly_connected_components(vertices: AbstractSet[str], + edges: Dict[str, List[str]]) -> Iterator[Set[str]]: + """Compute Strongly Connected Components of a directed graph. + + Args: + vertices: the labels for the vertices + edges: for each vertex, gives the target vertices of its outgoing edges + + Returns: + An iterator yielding strongly connected components, each + represented as a set of vertices. Each input vertex will occur + exactly once; vertices not part of a SCC are returned as + singleton sets. + + From http://code.activestate.com/recipes/578507/. + """ + identified = set() # type: Set[str] + stack = [] # type: List[str] + index = {} # type: Dict[str, int] + boundaries = [] # type: List[int] + + def dfs(v: str) -> Iterator[Set[str]]: + index[v] = len(stack) + stack.append(v) + boundaries.append(index[v]) + + for w in edges[v]: + if w not in index: + # For Python >= 3.3, replace with "yield from dfs(w)" + for scc in dfs(w): + yield scc + elif w not in identified: + while index[w] < boundaries[-1]: + boundaries.pop() + + if boundaries[-1] == index[v]: + boundaries.pop() + scc = set(stack[index[v]:]) + del stack[index[v]:] + identified.update(scc) + yield scc + + for v in vertices: + if v not in index: + # For Python >= 3.3, replace with "yield from dfs(v)" + for scc in dfs(v): + yield scc + + +def topsort(data: Dict[AbstractSet[str], + Set[AbstractSet[str]]]) -> Iterable[Set[AbstractSet[str]]]: + """Topological sort. + + Args: + data: A map from SCCs (represented as frozen sets of strings) to + sets of SCCs, its dependencies. NOTE: This data structure + is modified in place -- for normalization purposes, + self-dependencies are removed and entries representing + orphans are added. + + Returns: + An iterator yielding sets of SCCs that have an equivalent + ordering. NOTE: The algorithm doesn't care about the internal + structure of SCCs. + + Example: + Suppose the input has the following structure: + + {A: {B, C}, B: {D}, C: {D}} + + This is normalized to: + + {A: {B, C}, B: {D}, C: {D}, D: {}} + + The algorithm will yield the following values: + + {D} + {B, C} + {A} + + From http://code.activestate.com/recipes/577413/. + """ + # TODO: Use a faster algorithm? + for k, v in data.items(): + v.discard(k) # Ignore self dependencies. + for item in set.union(*data.values()) - set(data.keys()): + data[item] = set() + while True: + ready = {item for item, dep in data.items() if not dep} + if not ready: + break + yield ready + data = {item: (dep - ready) + for item, dep in data.items() + if item not in ready} + assert not data, "A cyclic dependency exists amongst %r" % data diff --git a/mypy/checker.py b/mypy/checker.py index 62f4c250ea272..7063758096dfa 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -1,2459 +1,2459 @@ -"""Mypy type checker.""" - -import itertools -import contextlib -import os -import os.path - -from typing import ( - Any, Dict, Set, List, cast, Tuple, TypeVar, Union, Optional, NamedTuple -) - -from mypy.errors import Errors, report_internal_error -from mypy.nodes import ( - SymbolTable, Node, MypyFile, Var, - OverloadedFuncDef, FuncDef, FuncItem, FuncBase, TypeInfo, - ClassDef, GDEF, Block, AssignmentStmt, NameExpr, MemberExpr, IndexExpr, - TupleExpr, ListExpr, ExpressionStmt, ReturnStmt, IfStmt, - WhileStmt, OperatorAssignmentStmt, WithStmt, AssertStmt, - RaiseStmt, TryStmt, ForStmt, DelStmt, CallExpr, IntExpr, StrExpr, - BytesExpr, UnicodeExpr, FloatExpr, OpExpr, UnaryExpr, CastExpr, RevealTypeExpr, SuperExpr, - TypeApplication, DictExpr, SliceExpr, FuncExpr, TempNode, SymbolTableNode, - Context, ListComprehension, ConditionalExpr, GeneratorExpr, - Decorator, SetExpr, TypeVarExpr, PrintStmt, - LITERAL_TYPE, BreakStmt, ContinueStmt, ComparisonExpr, StarExpr, - YieldFromExpr, NamedTupleExpr, SetComprehension, - DictionaryComprehension, ComplexExpr, EllipsisExpr, TypeAliasExpr, - RefExpr, YieldExpr, BackquoteExpr, ImportFrom, ImportAll, ImportBase, - CONTRAVARIANT, COVARIANT -) -from mypy.nodes import function_type, method_type, method_type_with_fallback -from mypy import nodes -from mypy.types import ( - Type, AnyType, CallableType, Void, FunctionLike, Overloaded, TupleType, - Instance, NoneTyp, ErrorType, strip_type, - UnionType, TypeVarId, TypeVarType, PartialType, DeletedType, UninhabitedType -) -from mypy.sametypes import is_same_type -from mypy.messages import MessageBuilder -import mypy.checkexpr -from mypy.checkmember import map_type_from_supertype -from mypy import defaults -from mypy import messages -from mypy.subtypes import ( - is_subtype, is_equivalent, is_proper_subtype, - is_more_precise, restrict_subtype_away -) -from mypy.maptype import map_instance_to_supertype -from mypy.semanal import self_type, set_callable_name, refers_to_fullname -from mypy.erasetype import erase_typevars -from mypy.expandtype import expand_type_by_instance, expand_type -from mypy.visitor import NodeVisitor -from mypy.join import join_types -from mypy.treetransform import TransformVisitor -from mypy.meet import meet_simple, nearest_builtin_ancestor, is_overlapping_types -from mypy.binder import ConditionalTypeBinder -from mypy.options import Options - -from mypy import experiments - - -T = TypeVar('T') - - -# A node which is postponed to be type checked during the next pass. -DeferredNode = NamedTuple( - 'DeferredNode', - [ - ('node', Node), - ('context_type_name', Optional[str]), # Name of the surrounding class (for error messages) - ]) - - -class TypeChecker(NodeVisitor[Type]): - """Mypy type checker. - - Type check mypy source files that have been semantically analyzed. - """ - - # Are we type checking a stub? - is_stub = False - # Error message reporter - errors = None # type: Errors - # Utility for generating messages - msg = None # type: MessageBuilder - # Types of type checked nodes - type_map = None # type: Dict[Node, Type] - - # Helper for managing conditional types - binder = None # type: ConditionalTypeBinder - # Helper for type checking expressions - expr_checker = None # type: mypy.checkexpr.ExpressionChecker - - # Stack of function return types - return_types = None # type: List[Type] - # Type context for type inference - type_context = None # type: List[Type] - # Flags; true for dynamically typed functions - dynamic_funcs = None # type: List[bool] - # Stack of functions being type checked - function_stack = None # type: List[FuncItem] - # Do weak type checking in this file - weak_opts = set() # type: Set[str] - # Stack of collections of variables with partial types - partial_types = None # type: List[Dict[Var, Context]] - globals = None # type: SymbolTable - modules = None # type: Dict[str, MypyFile] - # Nodes that couldn't be checked because some types weren't available. We'll run - # another pass and try these again. - deferred_nodes = None # type: List[DeferredNode] - # Type checking pass number (0 = first pass) - pass_num = 0 - # Have we deferred the current function? If yes, don't infer additional - # types during this pass within the function. - current_node_deferred = False - is_typeshed_stub = False - options = None # type: Options - - def __init__(self, errors: Errors, modules: Dict[str, MypyFile], options: Options) -> None: - """Construct a type checker. - - Use errors to report type check errors. - """ - self.errors = errors - self.modules = modules - self.options = options - self.msg = MessageBuilder(errors, modules) - self.type_map = {} - self.binder = ConditionalTypeBinder() - self.expr_checker = mypy.checkexpr.ExpressionChecker(self, self.msg) - self.return_types = [] - self.type_context = [] - self.dynamic_funcs = [] - self.function_stack = [] - self.weak_opts = set() # type: Set[str] - self.partial_types = [] - self.deferred_nodes = [] - self.pass_num = 0 - self.current_node_deferred = False - - def visit_file(self, file_node: MypyFile, path: str) -> None: - """Type check a mypy file with the given path.""" - self.pass_num = 0 - self.is_stub = file_node.is_stub - self.errors.set_file(path) - self.globals = file_node.names - self.weak_opts = file_node.weak_opts - self.enter_partial_types() - # gross, but no other clear way to tell - self.is_typeshed_stub = self.is_stub and 'typeshed' in os.path.normpath(path).split(os.sep) - - for d in file_node.defs: - self.accept(d) - - self.leave_partial_types() - - if self.deferred_nodes: - self.check_second_pass() - - self.current_node_deferred = False - - all_ = self.globals.get('__all__') - if all_ is not None and all_.type is not None: - seq_str = self.named_generic_type('typing.Sequence', - [self.named_type('builtins.str')]) - if not is_subtype(all_.type, seq_str): - str_seq_s, all_s = self.msg.format_distinctly(seq_str, all_.type) - self.fail(messages.ALL_MUST_BE_SEQ_STR.format(str_seq_s, all_s), - all_.node) - - def check_second_pass(self): - """Run second pass of type checking which goes through deferred nodes.""" - self.pass_num = 1 - for node, type_name in self.deferred_nodes: - if type_name: - self.errors.push_type(type_name) - self.accept(node) - if type_name: - self.errors.pop_type() - self.deferred_nodes = [] - - def handle_cannot_determine_type(self, name: str, context: Context) -> None: - if self.pass_num == 0 and self.function_stack: - # Don't report an error yet. Just defer. - node = self.function_stack[-1] - if self.errors.type_name: - type_name = self.errors.type_name[-1] - else: - type_name = None - self.deferred_nodes.append(DeferredNode(node, type_name)) - # Set a marker so that we won't infer additional types in this - # function. Any inferred types could be bogus, because there's at - # least one type that we don't know. - self.current_node_deferred = True - else: - self.msg.cannot_determine_type(name, context) - - def accept(self, node: Node, type_context: Type = None) -> Type: - """Type check a node in the given type context.""" - self.type_context.append(type_context) - try: - typ = node.accept(self) - except Exception as err: - report_internal_error(err, self.errors.file, node.line) - self.type_context.pop() - self.store_type(node, typ) - if self.typing_mode_none(): - return AnyType() - else: - return typ - - def accept_loop(self, body: Node, else_body: Node = None) -> Type: - """Repeatedly type check a loop body until the frame doesn't change. - - Then check the else_body. - """ - # The outer frame accumulates the results of all iterations - with self.binder.frame_context(1) as outer_frame: - self.binder.push_loop_frame() - while True: - with self.binder.frame_context(1): - # We may skip each iteration - self.binder.options_on_return[-1].append(outer_frame) - self.accept(body) - if not self.binder.last_pop_changed: - break - self.binder.pop_loop_frame() - if else_body: - self.accept(else_body) - - # - # Definitions - # - - def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> Type: - num_abstract = 0 - if defn.is_property: - # HACK: Infer the type of the property. - self.visit_decorator(defn.items[0]) - for fdef in defn.items: - self.check_func_item(fdef.func, name=fdef.func.name()) - if fdef.func.is_abstract: - num_abstract += 1 - if num_abstract not in (0, len(defn.items)): - self.fail(messages.INCONSISTENT_ABSTRACT_OVERLOAD, defn) - if defn.info: - self.check_method_override(defn) - self.check_inplace_operator_method(defn) - self.check_overlapping_overloads(defn) - - def check_overlapping_overloads(self, defn: OverloadedFuncDef) -> None: - for i, item in enumerate(defn.items): - for j, item2 in enumerate(defn.items[i + 1:]): - # TODO overloads involving decorators - sig1 = self.function_type(item.func) - sig2 = self.function_type(item2.func) - if is_unsafe_overlapping_signatures(sig1, sig2): - self.msg.overloaded_signatures_overlap(i + 1, i + j + 2, - item.func) - - def is_generator_return_type(self, typ: Type) -> bool: - return is_subtype(self.named_generic_type('typing.Generator', - [AnyType(), AnyType(), AnyType()]), - typ) - - def get_generator_yield_type(self, return_type: Type) -> Type: - if isinstance(return_type, AnyType): - return AnyType() - elif not self.is_generator_return_type(return_type): - # If the function doesn't have a proper Generator (or superclass) return type, anything - # is permissible. - return AnyType() - elif not isinstance(return_type, Instance): - # Same as above, but written as a separate branch so the typechecker can understand. - return AnyType() - elif return_type.args: - return return_type.args[0] - else: - # If the function's declared supertype of Generator has no type - # parameters (i.e. is `object`), then the yielded values can't - # be accessed so any type is acceptable. - return AnyType() - - def get_generator_receive_type(self, return_type: Type) -> Type: - if isinstance(return_type, AnyType): - return AnyType() - elif not self.is_generator_return_type(return_type): - # If the function doesn't have a proper Generator (or superclass) return type, anything - # is permissible. - return AnyType() - elif not isinstance(return_type, Instance): - # Same as above, but written as a separate branch so the typechecker can understand. - return AnyType() - elif return_type.type.fullname() == 'typing.Generator': - # Generator is the only type which specifies the type of values it can receive. - return return_type.args[1] - else: - # `return_type` is a supertype of Generator, so callers won't be able to send it - # values. - return Void() - - def get_generator_return_type(self, return_type: Type) -> Type: - if isinstance(return_type, AnyType): - return AnyType() - elif not self.is_generator_return_type(return_type): - # If the function doesn't have a proper Generator (or superclass) return type, anything - # is permissible. - return AnyType() - elif not isinstance(return_type, Instance): - # Same as above, but written as a separate branch so the typechecker can understand. - return AnyType() - elif return_type.type.fullname() == 'typing.Generator': - # Generator is the only type which specifies the type of values it returns into - # `yield from` expressions. - return return_type.args[2] - else: - # `return_type` is supertype of Generator, so callers won't be able to see the return - # type when used in a `yield from` expression. - return AnyType() - - def visit_func_def(self, defn: FuncDef) -> Type: - """Type check a function definition.""" - self.check_func_item(defn, name=defn.name()) - if defn.info: - if not defn.is_dynamic(): - self.check_method_override(defn) - self.check_inplace_operator_method(defn) - if defn.original_def: - # Override previous definition. - new_type = self.function_type(defn) - if isinstance(defn.original_def, FuncDef): - # Function definition overrides function definition. - if not is_same_type(new_type, self.function_type(defn.original_def)): - self.msg.incompatible_conditional_function_def(defn) - else: - # Function definition overrides a variable initialized via assignment. - orig_type = defn.original_def.type - if orig_type is None: - # XXX This can be None, as happens in - # test_testcheck_TypeCheckSuite.testRedefinedFunctionInTryWithElse - self.msg.note("Internal mypy error checking function redefinition.", defn) - return None - if isinstance(orig_type, PartialType): - if orig_type.type is None: - # Ah this is a partial type. Give it the type of the function. - var = defn.original_def - partial_types = self.find_partial_types(var) - if partial_types is not None: - var.type = new_type - del partial_types[var] - else: - # Trying to redefine something like partial empty list as function. - self.fail(messages.INCOMPATIBLE_REDEFINITION, defn) - else: - # TODO: Update conditional type binder. - self.check_subtype(new_type, orig_type, defn, - messages.INCOMPATIBLE_REDEFINITION, - 'redefinition with type', - 'original type') - - def check_func_item(self, defn: FuncItem, - type_override: CallableType = None, - name: str = None) -> Type: - """Type check a function. - - If type_override is provided, use it as the function type. - """ - # We may be checking a function definition or an anonymous function. In - # the first case, set up another reference with the precise type. - fdef = None # type: FuncDef - if isinstance(defn, FuncDef): - fdef = defn - - self.function_stack.append(defn) - self.dynamic_funcs.append(defn.is_dynamic() and not type_override) - - if fdef: - self.errors.push_function(fdef.name()) - - self.enter_partial_types() - - typ = self.function_type(defn) - if type_override: - typ = type_override - if isinstance(typ, CallableType): - self.check_func_def(defn, typ, name) - else: - raise RuntimeError('Not supported') - - self.leave_partial_types() - - if fdef: - self.errors.pop_function() - - self.dynamic_funcs.pop() - self.function_stack.pop() - self.current_node_deferred = False - - def check_func_def(self, defn: FuncItem, typ: CallableType, name: str) -> None: - """Type check a function definition.""" - # Expand type variables with value restrictions to ordinary types. - for item, typ in self.expand_typevars(defn, typ): - old_binder = self.binder - self.binder = ConditionalTypeBinder() - with self.binder.frame_context(): - defn.expanded.append(item) - - # We may be checking a function definition or an anonymous - # function. In the first case, set up another reference with the - # precise type. - if isinstance(item, FuncDef): - fdef = item - else: - fdef = None - - if fdef: - # Check if __init__ has an invalid, non-None return type. - if (fdef.info and fdef.name() == '__init__' and - not isinstance(typ.ret_type, Void) and - not self.dynamic_funcs[-1]): - self.fail(messages.INIT_MUST_HAVE_NONE_RETURN_TYPE, - item.type) - - show_untyped = not self.is_typeshed_stub or self.options.warn_incomplete_stub - if self.options.disallow_untyped_defs and show_untyped: - # Check for functions with unspecified/not fully specified types. - def is_implicit_any(t: Type) -> bool: - return isinstance(t, AnyType) and t.implicit - - if fdef.type is None: - self.fail(messages.FUNCTION_TYPE_EXPECTED, fdef) - elif isinstance(fdef.type, CallableType): - if is_implicit_any(fdef.type.ret_type): - self.fail(messages.RETURN_TYPE_EXPECTED, fdef) - if any(is_implicit_any(t) for t in fdef.type.arg_types): - self.fail(messages.ARGUMENT_TYPE_EXPECTED, fdef) - - if name in nodes.reverse_op_method_set: - self.check_reverse_op_method(item, typ, name) - elif name == '__getattr__': - self.check_getattr_method(typ, defn) - - # Refuse contravariant return type variable - if isinstance(typ.ret_type, TypeVarType): - if typ.ret_type.variance == CONTRAVARIANT: - self.fail(messages.RETURN_TYPE_CANNOT_BE_CONTRAVARIANT, - typ.ret_type) - - # Check that Generator functions have the appropriate return type. - if defn.is_generator: - if not self.is_generator_return_type(typ.ret_type): - self.fail(messages.INVALID_RETURN_TYPE_FOR_GENERATOR, typ) - - # Python 2 generators aren't allowed to return values. - if (self.options.python_version[0] == 2 and - isinstance(typ.ret_type, Instance) and - typ.ret_type.type.fullname() == 'typing.Generator'): - if not (isinstance(typ.ret_type.args[2], Void) - or isinstance(typ.ret_type.args[2], AnyType)): - self.fail(messages.INVALID_GENERATOR_RETURN_ITEM_TYPE, typ) - - # Push return type. - self.return_types.append(typ.ret_type) - - # Store argument types. - for i in range(len(typ.arg_types)): - arg_type = typ.arg_types[i] - - # Refuse covariant parameter type variables - if isinstance(arg_type, TypeVarType): - if arg_type.variance == COVARIANT: - self.fail(messages.FUNCTION_PARAMETER_CANNOT_BE_COVARIANT, - arg_type) - - if typ.arg_kinds[i] == nodes.ARG_STAR: - # builtins.tuple[T] is typing.Tuple[T, ...] - arg_type = self.named_generic_type('builtins.tuple', - [arg_type]) - elif typ.arg_kinds[i] == nodes.ARG_STAR2: - arg_type = self.named_generic_type('builtins.dict', - [self.str_type(), - arg_type]) - item.arguments[i].variable.type = arg_type - - # Type check initialization expressions. - for arg in item.arguments: - init = arg.initialization_statement - if init: - self.accept(init) - - # Type check body in a new scope. - with self.binder.frame_context(): - self.accept(item.body) - - self.return_types.pop() - - self.binder = old_binder - - def check_reverse_op_method(self, defn: FuncItem, typ: CallableType, - method: str) -> None: - """Check a reverse operator method such as __radd__.""" - - # This used to check for some very obscure scenario. It now - # just decides whether it's worth calling - # check_overlapping_op_methods(). - - if method in ('__eq__', '__ne__'): - # These are defined for all objects => can't cause trouble. - return - - # With 'Any' or 'object' return type we are happy, since any possible - # return value is valid. - ret_type = typ.ret_type - if isinstance(ret_type, AnyType): - return - if isinstance(ret_type, Instance): - if ret_type.type.fullname() == 'builtins.object': - return - # Plausibly the method could have too few arguments, which would result - # in an error elsewhere. - if len(typ.arg_types) <= 2: - # TODO check self argument kind - - # Check for the issue described above. - arg_type = typ.arg_types[1] - other_method = nodes.normal_from_reverse_op[method] - if isinstance(arg_type, Instance): - if not arg_type.type.has_readable_member(other_method): - return - elif isinstance(arg_type, AnyType): - return - elif isinstance(arg_type, UnionType): - if not arg_type.has_readable_member(other_method): - return - else: - return - - typ2 = self.expr_checker.analyze_external_member_access( - other_method, arg_type, defn) - self.check_overlapping_op_methods( - typ, method, defn.info, - typ2, other_method, cast(Instance, arg_type), - defn) - - def check_overlapping_op_methods(self, - reverse_type: CallableType, - reverse_name: str, - reverse_class: TypeInfo, - forward_type: Type, - forward_name: str, - forward_base: Instance, - context: Context) -> None: - """Check for overlapping method and reverse method signatures. - - Assume reverse method has valid argument count and kinds. - """ - - # Reverse operator method that overlaps unsafely with the - # forward operator method can result in type unsafety. This is - # similar to overlapping overload variants. - # - # This example illustrates the issue: - # - # class X: pass - # class A: - # def __add__(self, x: X) -> int: - # if isinstance(x, X): - # return 1 - # return NotImplemented - # class B: - # def __radd__(self, x: A) -> str: return 'x' - # class C(X, B): pass - # def f(b: B) -> None: - # A() + b # Result is 1, even though static type seems to be str! - # f(C()) - # - # The reason for the problem is that B and X are overlapping - # types, and the return types are different. Also, if the type - # of x in __radd__ would not be A, the methods could be - # non-overlapping. - - if isinstance(forward_type, CallableType): - # TODO check argument kinds - if len(forward_type.arg_types) < 1: - # Not a valid operator method -- can't succeed anyway. - return - - # Construct normalized function signatures corresponding to the - # operator methods. The first argument is the left operand and the - # second operand is the right argument -- we switch the order of - # the arguments of the reverse method. - forward_tweaked = CallableType( - [forward_base, forward_type.arg_types[0]], - [nodes.ARG_POS] * 2, - [None] * 2, - forward_type.ret_type, - forward_type.fallback, - name=forward_type.name) - reverse_args = reverse_type.arg_types - reverse_tweaked = CallableType( - [reverse_args[1], reverse_args[0]], - [nodes.ARG_POS] * 2, - [None] * 2, - reverse_type.ret_type, - fallback=self.named_type('builtins.function'), - name=reverse_type.name) - - if is_unsafe_overlapping_signatures(forward_tweaked, - reverse_tweaked): - self.msg.operator_method_signatures_overlap( - reverse_class.name(), reverse_name, - forward_base.type.name(), forward_name, context) - elif isinstance(forward_type, Overloaded): - for item in forward_type.items(): - self.check_overlapping_op_methods( - reverse_type, reverse_name, reverse_class, - item, forward_name, forward_base, context) - elif not isinstance(forward_type, AnyType): - self.msg.forward_operator_not_callable(forward_name, context) - - def check_inplace_operator_method(self, defn: FuncBase) -> None: - """Check an inplace operator method such as __iadd__. - - They cannot arbitrarily overlap with __add__. - """ - method = defn.name() - if method not in nodes.inplace_operator_methods: - return - typ = self.method_type(defn) - cls = defn.info - other_method = '__' + method[3:] - if cls.has_readable_member(other_method): - instance = self_type(cls) - typ2 = self.expr_checker.analyze_external_member_access( - other_method, instance, defn) - fail = False - if isinstance(typ2, FunctionLike): - if not is_more_general_arg_prefix(typ, typ2): - fail = True - else: - # TODO overloads - fail = True - if fail: - self.msg.signatures_incompatible(method, other_method, defn) - - def check_getattr_method(self, typ: CallableType, context: Context) -> None: - method_type = CallableType([AnyType(), self.named_type('builtins.str')], - [nodes.ARG_POS, nodes.ARG_POS], - [None], - AnyType(), - self.named_type('builtins.function')) - if not is_subtype(typ, method_type): - self.msg.invalid_signature(typ, context) - - def expand_typevars(self, defn: FuncItem, - typ: CallableType) -> List[Tuple[FuncItem, CallableType]]: - # TODO use generator - subst = [] # type: List[List[Tuple[TypeVarId, Type]]] - tvars = typ.variables or [] - tvars = tvars[:] - if defn.info: - # Class type variables - tvars += defn.info.defn.type_vars or [] - for tvar in tvars: - if tvar.values: - subst.append([(tvar.id, value) - for value in tvar.values]) - if subst: - result = [] # type: List[Tuple[FuncItem, CallableType]] - for substitutions in itertools.product(*subst): - mapping = dict(substitutions) - expanded = cast(CallableType, expand_type(typ, mapping)) - result.append((expand_func(defn, mapping), expanded)) - return result - else: - return [(defn, typ)] - - def check_method_override(self, defn: FuncBase) -> None: - """Check if function definition is compatible with base classes.""" - # Check against definitions in base classes. - for base in defn.info.mro[1:]: - self.check_method_or_accessor_override_for_base(defn, base) - - def check_method_or_accessor_override_for_base(self, defn: FuncBase, - base: TypeInfo) -> None: - """Check if method definition is compatible with a base class.""" - if base: - name = defn.name() - if name not in ('__init__', '__new__'): - # Check method override (__init__ and __new__ are special). - self.check_method_override_for_base_with_name(defn, name, base) - if name in nodes.inplace_operator_methods: - # Figure out the name of the corresponding operator method. - method = '__' + name[3:] - # An inplace operator method such as __iadd__ might not be - # always introduced safely if a base class defined __add__. - # TODO can't come up with an example where this is - # necessary; now it's "just in case" - self.check_method_override_for_base_with_name(defn, method, - base) - - def check_method_override_for_base_with_name( - self, defn: FuncBase, name: str, base: TypeInfo) -> None: - base_attr = base.names.get(name) - if base_attr: - # The name of the method is defined in the base class. - - # Construct the type of the overriding method. - typ = self.method_type(defn) - # Map the overridden method type to subtype context so that - # it can be checked for compatibility. - original_type = base_attr.type - if original_type is None and isinstance(base_attr.node, - FuncDef): - original_type = self.function_type(base_attr.node) - if isinstance(original_type, FunctionLike): - original = map_type_from_supertype( - method_type(original_type), - defn.info, base) - # Check that the types are compatible. - # TODO overloaded signatures - self.check_override(typ, - cast(FunctionLike, original), - defn.name(), - name, - base.name(), - defn) - else: - assert original_type is not None - self.msg.signature_incompatible_with_supertype( - defn.name(), name, base.name(), defn) - - def check_override(self, override: FunctionLike, original: FunctionLike, - name: str, name_in_super: str, supertype: str, - node: Context) -> None: - """Check a method override with given signatures. - - Arguments: - override: The signature of the overriding method. - original: The signature of the original supertype method. - name: The name of the subtype. This and the next argument are - only used for generating error messages. - supertype: The name of the supertype. - """ - # Use boolean variable to clarify code. - fail = False - if not is_subtype(override, original): - fail = True - elif (not isinstance(original, Overloaded) and - isinstance(override, Overloaded) and - name in nodes.reverse_op_methods.keys()): - # Operator method overrides cannot introduce overloading, as - # this could be unsafe with reverse operator methods. - fail = True - - if fail: - emitted_msg = False - if (isinstance(override, CallableType) and - isinstance(original, CallableType) and - len(override.arg_types) == len(original.arg_types) and - override.min_args == original.min_args): - # Give more detailed messages for the common case of both - # signatures having the same number of arguments and no - # overloads. - - # override might have its own generic function type - # variables. If an argument or return type of override - # does not have the correct subtyping relationship - # with the original type even after these variables - # are erased, then it is definitely an incompatiblity. - - override_ids = override.type_var_ids() - - def erase_override(t: Type) -> Type: - return erase_typevars(t, ids_to_erase=override_ids) - - for i in range(len(override.arg_types)): - if not is_subtype(original.arg_types[i], - erase_override(override.arg_types[i])): - self.msg.argument_incompatible_with_supertype( - i + 1, name, name_in_super, supertype, node) - emitted_msg = True - - if not is_subtype(erase_override(override.ret_type), - original.ret_type): - self.msg.return_type_incompatible_with_supertype( - name, name_in_super, supertype, node) - emitted_msg = True - - if not emitted_msg: - # Fall back to generic incompatibility message. - self.msg.signature_incompatible_with_supertype( - name, name_in_super, supertype, node) - - def visit_class_def(self, defn: ClassDef) -> Type: - """Type check a class definition.""" - typ = defn.info - self.errors.push_type(defn.name) - self.enter_partial_types() - old_binder = self.binder - self.binder = ConditionalTypeBinder() - with self.binder.frame_context(): - self.accept(defn.defs) - self.binder = old_binder - if not hasattr(defn, 'incompatible_baseclass'): - # Otherwise we've already found errors; more errors are not useful - self.check_multiple_inheritance(typ) - self.leave_partial_types() - self.errors.pop_type() - - def check_multiple_inheritance(self, typ: TypeInfo) -> None: - """Check for multiple inheritance related errors.""" - - if len(typ.bases) <= 1: - # No multiple inheritance. - return - # Verify that inherited attributes are compatible. - mro = typ.mro[1:] - for i, base in enumerate(mro): - for name in base.names: - for base2 in mro[i + 1:]: - # We only need to check compatibility of attributes from classes not - # in a subclass relationship. For subclasses, normal (single inheritance) - # checks suffice (these are implemented elsewhere). - if name in base2.names and base2 not in base.mro: - self.check_compatibility(name, base, base2, typ) - # Verify that base class layouts are compatible. - builtin_bases = [nearest_builtin_ancestor(base.type) - for base in typ.bases] - for base1 in builtin_bases: - for base2 in builtin_bases: - if not (base1 in base2.mro or base2 in base1.mro): - self.fail(messages.INSTANCE_LAYOUT_CONFLICT, typ) - - def check_compatibility(self, name: str, base1: TypeInfo, - base2: TypeInfo, ctx: Context) -> None: - """Check if attribute name in base1 is compatible with base2 in multiple inheritance. - - Assume base1 comes before base2 in the MRO, and that base1 and base2 don't have - a direct subclass relationship (i.e., the compatibility requirement only derives from - multiple inheritance). - """ - if name == '__init__': - # __init__ can be incompatible -- it's a special case. - return - first = base1[name] - second = base2[name] - first_type = first.type - if first_type is None and isinstance(first.node, FuncDef): - first_type = self.function_type(first.node) - second_type = second.type - if second_type is None and isinstance(second.node, FuncDef): - second_type = self.function_type(second.node) - # TODO: What if some classes are generic? - if (isinstance(first_type, FunctionLike) and - isinstance(second_type, FunctionLike)): - # Method override - first_sig = method_type(first_type) - second_sig = method_type(second_type) - ok = is_subtype(first_sig, second_sig) - elif first_type and second_type: - ok = is_equivalent(first_type, second_type) - else: - if first_type is None: - self.msg.cannot_determine_type_in_base(name, base1.name(), ctx) - if second_type is None: - self.msg.cannot_determine_type_in_base(name, base2.name(), ctx) - ok = True - if not ok: - self.msg.base_class_definitions_incompatible(name, base1, base2, - ctx) - - def visit_import_from(self, node: ImportFrom) -> Type: - self.check_import(node) - - def visit_import_all(self, node: ImportAll) -> Type: - self.check_import(node) - - def check_import(self, node: ImportBase) -> Type: - for assign in node.assignments: - lvalue = assign.lvalues[0] - lvalue_type, _, __ = self.check_lvalue(lvalue) - if lvalue_type is None: - # TODO: This is broken. - lvalue_type = AnyType() - message = '{} "{}"'.format(messages.INCOMPATIBLE_IMPORT_OF, - cast(NameExpr, assign.rvalue).name) - self.check_simple_assignment(lvalue_type, assign.rvalue, node, - msg=message, lvalue_name='local name', - rvalue_name='imported name') - - # - # Statements - # - - def visit_block(self, b: Block) -> Type: - if b.is_unreachable: - return None - for s in b.body: - self.accept(s) - if self.binder.breaking_out: - break - - def visit_assignment_stmt(self, s: AssignmentStmt) -> Type: - """Type check an assignment statement. - - Handle all kinds of assignment statements (simple, indexed, multiple). - """ - self.check_assignment(s.lvalues[-1], s.rvalue, s.type is None) - - if len(s.lvalues) > 1: - # Chained assignment (e.g. x = y = ...). - # Make sure that rvalue type will not be reinferred. - if s.rvalue not in self.type_map: - self.accept(s.rvalue) - rvalue = self.temp_node(self.type_map[s.rvalue], s) - for lv in s.lvalues[:-1]: - self.check_assignment(lv, rvalue, s.type is None) - - def check_assignment(self, lvalue: Node, rvalue: Node, infer_lvalue_type: bool = True) -> None: - """Type check a single assignment: lvalue = rvalue.""" - if isinstance(lvalue, TupleExpr) or isinstance(lvalue, ListExpr): - ltuple = cast(Union[TupleExpr, ListExpr], lvalue) - - self.check_assignment_to_multiple_lvalues(ltuple.items, rvalue, lvalue, - infer_lvalue_type) - else: - lvalue_type, index_lvalue, inferred = self.check_lvalue(lvalue) - if lvalue_type: - if isinstance(lvalue_type, PartialType) and lvalue_type.type is None: - # Try to infer a proper type for a variable with a partial None type. - rvalue_type = self.accept(rvalue) - if isinstance(rvalue_type, NoneTyp): - # This doesn't actually provide any additional information -- multiple - # None initializers preserve the partial None type. - return - if is_valid_inferred_type(rvalue_type): - var = lvalue_type.var - partial_types = self.find_partial_types(var) - if partial_types is not None: - if not self.current_node_deferred: - if experiments.STRICT_OPTIONAL: - var.type = UnionType.make_simplified_union( - [rvalue_type, NoneTyp()]) - else: - var.type = rvalue_type - else: - var.type = None - del partial_types[var] - # Try to infer a partial type. No need to check the return value, as - # an error will be reported elsewhere. - self.infer_partial_type(lvalue_type.var, lvalue, rvalue_type) - return - if (is_literal_none(rvalue) and - isinstance(lvalue, NameExpr) and - isinstance(lvalue.node, Var) and - lvalue.node.is_initialized_in_class): - # Allow None's to be assigned to class variables with non-Optional types. - rvalue_type = lvalue_type - else: - rvalue_type = self.check_simple_assignment(lvalue_type, rvalue, lvalue) - - if rvalue_type and infer_lvalue_type: - self.binder.assign_type(lvalue, - rvalue_type, - lvalue_type, - self.typing_mode_weak()) - elif index_lvalue: - self.check_indexed_assignment(index_lvalue, rvalue, rvalue) - - if inferred: - self.infer_variable_type(inferred, lvalue, self.accept(rvalue), - rvalue) - - def check_assignment_to_multiple_lvalues(self, lvalues: List[Node], rvalue: Node, - context: Context, - infer_lvalue_type: bool = True) -> None: - if isinstance(rvalue, TupleExpr) or isinstance(rvalue, ListExpr): - # Recursively go into Tuple or List expression rhs instead of - # using the type of rhs, because this allowed more fine grained - # control in cases like: a, b = [int, str] where rhs would get - # type List[object] - - rvalues = cast(Union[TupleExpr, ListExpr], rvalue).items - - if self.check_rvalue_count_in_assignment(lvalues, len(rvalues), context): - star_index = next((i for i, lv in enumerate(lvalues) if - isinstance(lv, StarExpr)), len(lvalues)) - - left_lvs = lvalues[:star_index] - star_lv = cast(StarExpr, - lvalues[star_index]) if star_index != len(lvalues) else None - right_lvs = lvalues[star_index + 1:] - - left_rvs, star_rvs, right_rvs = self.split_around_star( - rvalues, star_index, len(lvalues)) - - lr_pairs = list(zip(left_lvs, left_rvs)) - if star_lv: - rv_list = ListExpr(star_rvs) - rv_list.set_line(rvalue.get_line()) - lr_pairs.append((star_lv.expr, rv_list)) - lr_pairs.extend(zip(right_lvs, right_rvs)) - - for lv, rv in lr_pairs: - self.check_assignment(lv, rv, infer_lvalue_type) - else: - self.check_multi_assignment(lvalues, rvalue, context, infer_lvalue_type) - - def check_rvalue_count_in_assignment(self, lvalues: List[Node], rvalue_count: int, - context: Context) -> bool: - if any(isinstance(lvalue, StarExpr) for lvalue in lvalues): - if len(lvalues) - 1 > rvalue_count: - self.msg.wrong_number_values_to_unpack(rvalue_count, - len(lvalues) - 1, context) - return False - elif rvalue_count != len(lvalues): - self.msg.wrong_number_values_to_unpack(rvalue_count, - len(lvalues), context) - return False - return True - - def check_multi_assignment(self, lvalues: List[Node], - rvalue: Node, - context: Context, - infer_lvalue_type: bool = True, - msg: str = None) -> None: - """Check the assignment of one rvalue to a number of lvalues.""" - - # Infer the type of an ordinary rvalue expression. - rvalue_type = self.accept(rvalue) # TODO maybe elsewhere; redundant - undefined_rvalue = False - - if isinstance(rvalue_type, AnyType): - for lv in lvalues: - if isinstance(lv, StarExpr): - lv = lv.expr - self.check_assignment(lv, self.temp_node(AnyType(), context), infer_lvalue_type) - elif isinstance(rvalue_type, TupleType): - self.check_multi_assignment_from_tuple(lvalues, rvalue, rvalue_type, - context, undefined_rvalue, infer_lvalue_type) - else: - self.check_multi_assignment_from_iterable(lvalues, rvalue_type, - context, infer_lvalue_type) - - def check_multi_assignment_from_tuple(self, lvalues: List[Node], rvalue: Node, - rvalue_type: TupleType, context: Context, - undefined_rvalue: bool, - infer_lvalue_type: bool = True) -> None: - if self.check_rvalue_count_in_assignment(lvalues, len(rvalue_type.items), context): - star_index = next((i for i, lv in enumerate(lvalues) - if isinstance(lv, StarExpr)), len(lvalues)) - - left_lvs = lvalues[:star_index] - star_lv = cast(StarExpr, lvalues[star_index]) if star_index != len(lvalues) else None - right_lvs = lvalues[star_index + 1:] - - if not undefined_rvalue: - # Infer rvalue again, now in the correct type context. - lvalue_type = self.lvalue_type_for_inference(lvalues, rvalue_type) - rvalue_type = cast(TupleType, self.accept(rvalue, lvalue_type)) - - left_rv_types, star_rv_types, right_rv_types = self.split_around_star( - rvalue_type.items, star_index, len(lvalues)) - - for lv, rv_type in zip(left_lvs, left_rv_types): - self.check_assignment(lv, self.temp_node(rv_type, context), infer_lvalue_type) - if star_lv: - nodes = [self.temp_node(rv_type, context) for rv_type in star_rv_types] - list_expr = ListExpr(nodes) - list_expr.set_line(context.get_line()) - self.check_assignment(star_lv.expr, list_expr, infer_lvalue_type) - for lv, rv_type in zip(right_lvs, right_rv_types): - self.check_assignment(lv, self.temp_node(rv_type, context), infer_lvalue_type) - - def lvalue_type_for_inference(self, lvalues: List[Node], rvalue_type: TupleType) -> Type: - star_index = next((i for i, lv in enumerate(lvalues) - if isinstance(lv, StarExpr)), len(lvalues)) - left_lvs = lvalues[:star_index] - star_lv = cast(StarExpr, lvalues[star_index]) if star_index != len(lvalues) else None - right_lvs = lvalues[star_index + 1:] - left_rv_types, star_rv_types, right_rv_types = self.split_around_star( - rvalue_type.items, star_index, len(lvalues)) - - type_parameters = [] # type: List[Type] - - def append_types_for_inference(lvs: List[Node], rv_types: List[Type]) -> None: - for lv, rv_type in zip(lvs, rv_types): - sub_lvalue_type, index_expr, inferred = self.check_lvalue(lv) - if sub_lvalue_type: - type_parameters.append(sub_lvalue_type) - else: # index lvalue - # TODO Figure out more precise type context, probably - # based on the type signature of the _set method. - type_parameters.append(rv_type) - - append_types_for_inference(left_lvs, left_rv_types) - - if star_lv: - sub_lvalue_type, index_expr, inferred = self.check_lvalue(star_lv.expr) - if sub_lvalue_type: - type_parameters.extend([sub_lvalue_type] * len(star_rv_types)) - else: # index lvalue - # TODO Figure out more precise type context, probably - # based on the type signature of the _set method. - type_parameters.extend(star_rv_types) - - append_types_for_inference(right_lvs, right_rv_types) - - return TupleType(type_parameters, self.named_type('builtins.tuple')) - - def split_around_star(self, items: List[T], star_index: int, - length: int) -> Tuple[List[T], List[T], List[T]]: - """Splits a list of items in three to match another list of length 'length' - that contains a starred expression at 'star_index' in the following way: - - star_index = 2, length = 5 (i.e., [a,b,*,c,d]), items = [1,2,3,4,5,6,7] - returns in: ([1,2], [3,4,5], [6,7]) - """ - nr_right_of_star = length - star_index - 1 - right_index = nr_right_of_star if -nr_right_of_star != 0 else len(items) - left = items[:star_index] - star = items[star_index:right_index] - right = items[right_index:] - return (left, star, right) - - def type_is_iterable(self, type: Type) -> bool: - return (is_subtype(type, self.named_generic_type('typing.Iterable', - [AnyType()])) and - isinstance(type, Instance)) - - def check_multi_assignment_from_iterable(self, lvalues: List[Node], rvalue_type: Type, - context: Context, - infer_lvalue_type: bool = True) -> None: - if self.type_is_iterable(rvalue_type): - item_type = self.iterable_item_type(cast(Instance, rvalue_type)) - for lv in lvalues: - if isinstance(lv, StarExpr): - self.check_assignment(lv.expr, self.temp_node(rvalue_type, context), - infer_lvalue_type) - else: - self.check_assignment(lv, self.temp_node(item_type, context), - infer_lvalue_type) - else: - self.msg.type_not_iterable(rvalue_type, context) - - def check_lvalue(self, lvalue: Node) -> Tuple[Type, IndexExpr, Var]: - lvalue_type = None # type: Type - index_lvalue = None # type: IndexExpr - inferred = None # type: Var - - if self.is_definition(lvalue): - if isinstance(lvalue, NameExpr): - inferred = cast(Var, lvalue.node) - assert isinstance(inferred, Var) - else: - m = cast(MemberExpr, lvalue) - self.accept(m.expr) - inferred = m.def_var - elif isinstance(lvalue, IndexExpr): - index_lvalue = lvalue - elif isinstance(lvalue, MemberExpr): - lvalue_type = self.expr_checker.analyze_ordinary_member_access(lvalue, - True) - self.store_type(lvalue, lvalue_type) - elif isinstance(lvalue, NameExpr): - lvalue_type = self.expr_checker.analyze_ref_expr(lvalue, lvalue=True) - self.store_type(lvalue, lvalue_type) - elif isinstance(lvalue, TupleExpr) or isinstance(lvalue, ListExpr): - lv = cast(Union[TupleExpr, ListExpr], lvalue) - types = [self.check_lvalue(sub_expr)[0] for sub_expr in lv.items] - lvalue_type = TupleType(types, self.named_type('builtins.tuple')) - else: - lvalue_type = self.accept(lvalue) - - return lvalue_type, index_lvalue, inferred - - def is_definition(self, s: Node) -> bool: - if isinstance(s, NameExpr): - if s.is_def: - return True - # If the node type is not defined, this must the first assignment - # that we process => this is a definition, even though the semantic - # analyzer did not recognize this as such. This can arise in code - # that uses isinstance checks, if type checking of the primary - # definition is skipped due to an always False type check. - node = s.node - if isinstance(node, Var): - return node.type is None - elif isinstance(s, MemberExpr): - return s.is_def - return False - - def infer_variable_type(self, name: Var, lvalue: Node, - init_type: Type, context: Context) -> None: - """Infer the type of initialized variables from initializer type.""" - if self.typing_mode_weak(): - self.set_inferred_type(name, lvalue, AnyType()) - self.binder.assign_type(lvalue, init_type, self.binder.get_declaration(lvalue), True) - elif isinstance(init_type, Void): - self.check_not_void(init_type, context) - self.set_inference_error_fallback_type(name, lvalue, init_type, context) - elif isinstance(init_type, DeletedType): - self.msg.deleted_as_rvalue(init_type, context) - elif not is_valid_inferred_type(init_type): - # We cannot use the type of the initialization expression for full type - # inference (it's not specific enough), but we might be able to give - # partial type which will be made more specific later. A partial type - # gets generated in assignment like 'x = []' where item type is not known. - if not self.infer_partial_type(name, lvalue, init_type): - self.fail(messages.NEED_ANNOTATION_FOR_VAR, context) - self.set_inference_error_fallback_type(name, lvalue, init_type, context) - else: - # Infer type of the target. - - # Make the type more general (strip away function names etc.). - init_type = strip_type(init_type) - - self.set_inferred_type(name, lvalue, init_type) - - def infer_partial_type(self, name: Var, lvalue: Node, init_type: Type) -> bool: - if isinstance(init_type, (NoneTyp, UninhabitedType)): - partial_type = PartialType(None, name, [init_type]) - elif isinstance(init_type, Instance): - fullname = init_type.type.fullname() - if (isinstance(lvalue, NameExpr) and - (fullname == 'builtins.list' or - fullname == 'builtins.set' or - fullname == 'builtins.dict') and - all(isinstance(t, (NoneTyp, UninhabitedType)) for t in init_type.args)): - partial_type = PartialType(init_type.type, name, init_type.args) - else: - return False - else: - return False - self.set_inferred_type(name, lvalue, partial_type) - self.partial_types[-1][name] = lvalue - return True - - def set_inferred_type(self, var: Var, lvalue: Node, type: Type) -> None: - """Store inferred variable type. - - Store the type to both the variable node and the expression node that - refers to the variable (lvalue). If var is None, do nothing. - """ - if var and not self.current_node_deferred: - var.type = type - self.store_type(lvalue, type) - - def set_inference_error_fallback_type(self, var: Var, lvalue: Node, type: Type, - context: Context) -> None: - """If errors on context line are ignored, store dummy type for variable. - - If a program ignores error on type inference error, the variable should get some - inferred type so that if can used later on in the program. Example: - - x = [] # type: ignore - x.append(1) # Should be ok! - - We implement this here by giving x a valid type (Any). - """ - if context.get_line() in self.errors.ignored_lines[self.errors.file]: - self.set_inferred_type(var, lvalue, AnyType()) - - def narrow_type_from_binder(self, expr: Node, known_type: Type) -> Type: - if expr.literal >= LITERAL_TYPE: - restriction = self.binder.get(expr) - if restriction: - ans = meet_simple(known_type, restriction) - return ans - return known_type - - def check_simple_assignment(self, lvalue_type: Type, rvalue: Node, - context: Node, - msg: str = messages.INCOMPATIBLE_TYPES_IN_ASSIGNMENT, - lvalue_name: str = 'variable', - rvalue_name: str = 'expression') -> Type: - if self.is_stub and isinstance(rvalue, EllipsisExpr): - # '...' is always a valid initializer in a stub. - return AnyType() - else: - rvalue_type = self.accept(rvalue, lvalue_type) - if isinstance(rvalue_type, DeletedType): - self.msg.deleted_as_rvalue(rvalue_type, context) - if self.typing_mode_weak(): - return rvalue_type - if isinstance(lvalue_type, DeletedType): - self.msg.deleted_as_lvalue(lvalue_type, context) - else: - self.check_subtype(rvalue_type, lvalue_type, context, msg, - '{} has type'.format(rvalue_name), - '{} has type'.format(lvalue_name)) - return rvalue_type - - def check_indexed_assignment(self, lvalue: IndexExpr, - rvalue: Node, context: Context) -> None: - """Type check indexed assignment base[index] = rvalue. - - The lvalue argument is the base[index] expression. - """ - self.try_infer_partial_type_from_indexed_assignment(lvalue, rvalue) - basetype = self.accept(lvalue.base) - method_type = self.expr_checker.analyze_external_member_access( - '__setitem__', basetype, context) - lvalue.method_type = method_type - self.expr_checker.check_call(method_type, [lvalue.index, rvalue], - [nodes.ARG_POS, nodes.ARG_POS], - context) - - def try_infer_partial_type_from_indexed_assignment( - self, lvalue: IndexExpr, rvalue: Node) -> None: - # TODO: Should we share some of this with try_infer_partial_type? - if isinstance(lvalue.base, RefExpr) and isinstance(lvalue.base.node, Var): - var = lvalue.base.node - if isinstance(var.type, PartialType): - type_type = var.type.type - if type_type is None: - return # The partial type is None. - partial_types = self.find_partial_types(var) - if partial_types is None: - return - typename = type_type.fullname() - if typename == 'builtins.dict': - # TODO: Don't infer things twice. - key_type = self.accept(lvalue.index) - value_type = self.accept(rvalue) - full_key_type = UnionType.make_simplified_union( - [key_type, var.type.inner_types[0]]) - full_value_type = UnionType.make_simplified_union( - [value_type, var.type.inner_types[1]]) - if (is_valid_inferred_type(full_key_type) and - is_valid_inferred_type(full_value_type)): - if not self.current_node_deferred: - var.type = self.named_generic_type('builtins.dict', - [full_key_type, full_value_type]) - del partial_types[var] - - def visit_expression_stmt(self, s: ExpressionStmt) -> Type: - self.accept(s.expr) - - def visit_return_stmt(self, s: ReturnStmt) -> Type: - """Type check a return statement.""" - self.binder.breaking_out = True - if self.is_within_function(): - if self.function_stack[-1].is_generator: - return_type = self.get_generator_return_type(self.return_types[-1]) - else: - return_type = self.return_types[-1] - - if s.expr: - # Return with a value. - typ = self.accept(s.expr, return_type) - # Returning a value of type Any is always fine. - if isinstance(typ, AnyType): - return None - - if isinstance(return_type, Void): - # Lambdas are allowed to have a Void return. - # Functions returning a value of type None are allowed to have a Void return. - if isinstance(self.function_stack[-1], FuncExpr) or isinstance(typ, NoneTyp): - return None - self.fail(messages.NO_RETURN_VALUE_EXPECTED, s) - else: - self.check_subtype( - subtype_label='got', - subtype=typ, - supertype_label='expected', - supertype=return_type, - context=s, - msg=messages.INCOMPATIBLE_RETURN_VALUE_TYPE) - else: - # Empty returns are valid in Generators with Any typed returns. - if (self.function_stack[-1].is_generator and isinstance(return_type, AnyType)): - return None - - if isinstance(return_type, Void): - return None - - if isinstance(return_type, AnyType): - return None - - if self.typing_mode_full(): - self.fail(messages.RETURN_VALUE_EXPECTED, s) - - def wrap_generic_type(self, typ: Instance, rtyp: Instance, check_type: - str, context: Context) -> Type: - n_diff = self.count_nested_types(rtyp, check_type) - self.count_nested_types(typ, - check_type) - if n_diff == 1: - return self.named_generic_type(check_type, [typ]) - elif n_diff == 0 or n_diff > 1: - self.fail(messages.INCOMPATIBLE_RETURN_VALUE_TYPE - + ": expected {}, got {}".format(rtyp, typ), context) - return typ - return typ - - def count_nested_types(self, typ: Instance, check_type: str) -> int: - c = 0 - while is_subtype(typ, self.named_type(check_type)): - c += 1 - typ = map_instance_to_supertype(self.named_generic_type(check_type, typ.args), - self.lookup_typeinfo(check_type)) - if typ.args: - typ = cast(Instance, typ.args[0]) - else: - return c - return c - - def visit_if_stmt(self, s: IfStmt) -> Type: - """Type check an if statement.""" - breaking_out = True - # This frame records the knowledge from previous if/elif clauses not being taken. - with self.binder.frame_context(): - for e, b in zip(s.expr, s.body): - t = self.accept(e) - self.check_not_void(t, e) - if_map, else_map = find_isinstance_check( - e, self.type_map, - self.typing_mode_weak() - ) - if if_map is None: - # The condition is always false - # XXX should issue a warning? - pass - else: - # Only type check body if the if condition can be true. - with self.binder.frame_context(2): - if if_map: - for var, type in if_map.items(): - self.binder.push(var, type) - - self.accept(b) - breaking_out = breaking_out and self.binder.last_pop_breaking_out - - if else_map: - for var, type in else_map.items(): - self.binder.push(var, type) - if else_map is None: - # The condition is always true => remaining elif/else blocks - # can never be reached. - - # Might also want to issue a warning - # print("Warning: isinstance always true") - break - else: # Didn't break => can't prove one of the conditions is always true - with self.binder.frame_context(2): - if s.else_body: - self.accept(s.else_body) - breaking_out = breaking_out and self.binder.last_pop_breaking_out - if breaking_out: - self.binder.breaking_out = True - return None - - def visit_while_stmt(self, s: WhileStmt) -> Type: - """Type check a while statement.""" - self.accept_loop(IfStmt([s.expr], [s.body], None), s.else_body) - - def visit_operator_assignment_stmt(self, - s: OperatorAssignmentStmt) -> Type: - """Type check an operator assignment statement, e.g. x += 1.""" - lvalue_type = self.accept(s.lvalue) - method = infer_operator_assignment_method(lvalue_type, s.op) - rvalue_type, method_type = self.expr_checker.check_op( - method, lvalue_type, s.rvalue, s) - - if isinstance(s.lvalue, IndexExpr): - self.check_indexed_assignment(s.lvalue, s.rvalue, s.rvalue) - else: - if not is_subtype(rvalue_type, lvalue_type): - self.msg.incompatible_operator_assignment(s.op, s) - - def visit_assert_stmt(self, s: AssertStmt) -> Type: - self.accept(s.expr) - - # If this is asserting some isinstance check, bind that type in the following code - true_map, _ = find_isinstance_check( - s.expr, self.type_map, - self.typing_mode_weak() - ) - - if true_map: - for var, type in true_map.items(): - self.binder.push(var, type) - - def visit_raise_stmt(self, s: RaiseStmt) -> Type: - """Type check a raise statement.""" - self.binder.breaking_out = True - if s.expr: - self.type_check_raise(s.expr, s) - if s.from_expr: - self.type_check_raise(s.from_expr, s) - - def type_check_raise(self, e: Node, s: RaiseStmt) -> None: - typ = self.accept(e) - if isinstance(typ, FunctionLike): - if typ.is_type_obj(): - # Cases like "raise/from ExceptionClass". - typeinfo = typ.type_object() - base = self.lookup_typeinfo('builtins.BaseException') - if base in typeinfo.mro or typeinfo.fallback_to_any: - # Good! - return - # Else fall back to the checks below (which will fail). - if isinstance(typ, TupleType) and self.options.python_version[0] == 2: - # allow `raise type, value, traceback` - # https://docs.python.org/2/reference/simple_stmts.html#the-raise-statement - # TODO: Also check tuple item types. - if len(typ.items) in (2, 3): - return - if isinstance(typ, Instance) and typ.type.fallback_to_any: - # OK! - return - self.check_subtype(typ, - self.named_type('builtins.BaseException'), s, - messages.INVALID_EXCEPTION) - - def visit_try_stmt(self, s: TryStmt) -> Type: - """Type check a try statement.""" - # Our enclosing frame will get the result if the try/except falls through. - # This one gets all possible intermediate states - with self.binder.frame_context(): - if s.finally_body: - self.binder.try_frames.add(len(self.binder.frames) - 1) - breaking_out = self.visit_try_without_finally(s) - self.binder.try_frames.remove(len(self.binder.frames) - 1) - # First we check finally_body is type safe for all intermediate frames - self.accept(s.finally_body) - breaking_out = breaking_out or self.binder.breaking_out - else: - breaking_out = self.visit_try_without_finally(s) - - if not breaking_out and s.finally_body: - # Then we try again for the more restricted set of options that can fall through - self.accept(s.finally_body) - self.binder.breaking_out = breaking_out - return None - - def visit_try_without_finally(self, s: TryStmt) -> bool: - """Type check a try statement, ignoring the finally block. - - Return whether we are guaranteed to be breaking out. - Otherwise, it will place the results possible frames of - that don't break out into self.binder.frames[-2]. - """ - breaking_out = True - # This frame records the possible states that exceptions can leave variables in - # during the try: block - with self.binder.frame_context(): - with self.binder.frame_context(3): - self.binder.try_frames.add(len(self.binder.frames) - 2) - self.accept(s.body) - self.binder.try_frames.remove(len(self.binder.frames) - 2) - if s.else_body: - self.accept(s.else_body) - breaking_out = breaking_out and self.binder.last_pop_breaking_out - for i in range(len(s.handlers)): - with self.binder.frame_context(3): - if s.types[i]: - t = self.visit_except_handler_test(s.types[i]) - if s.vars[i]: - # To support local variables, we make this a definition line, - # causing assignment to set the variable's type. - s.vars[i].is_def = True - self.check_assignment(s.vars[i], self.temp_node(t, s.vars[i])) - self.accept(s.handlers[i]) - if s.vars[i]: - # Exception variables are deleted in python 3 but not python 2. - # But, since it's bad form in python 2 and the type checking - # wouldn't work very well, we delete it anyway. - - # Unfortunately, this doesn't let us detect usage before the - # try/except block. - if self.options.python_version[0] >= 3: - source = s.vars[i].name - else: - source = ('(exception variable "{}", which we do not accept outside' - 'except: blocks even in python 2)'.format(s.vars[i].name)) - var = cast(Var, s.vars[i].node) - var.type = DeletedType(source=source) - self.binder.cleanse(s.vars[i]) - breaking_out = breaking_out and self.binder.last_pop_breaking_out - return breaking_out - - def visit_except_handler_test(self, n: Node) -> Type: - """Type check an exception handler test clause.""" - type = self.accept(n) - - all_types = [] # type: List[Type] - test_types = type.items if isinstance(type, TupleType) else [type] - - for ttype in test_types: - if isinstance(ttype, AnyType): - all_types.append(ttype) - continue - - if not isinstance(ttype, FunctionLike): - self.fail(messages.INVALID_EXCEPTION_TYPE, n) - return AnyType() - - item = ttype.items()[0] - ret_type = item.ret_type - if not (is_subtype(ret_type, self.named_type('builtins.BaseException')) - and item.is_type_obj()): - self.fail(messages.INVALID_EXCEPTION_TYPE, n) - return AnyType() - - all_types.append(ret_type) - - return UnionType.make_simplified_union(all_types) - - def visit_for_stmt(self, s: ForStmt) -> Type: - """Type check a for statement.""" - item_type = self.analyze_iterable_item_type(s.expr) - self.analyze_index_variables(s.index, item_type, s) - self.accept_loop(s.body, s.else_body) - - def analyze_iterable_item_type(self, expr: Node) -> Type: - """Analyse iterable expression and return iterator item type.""" - iterable = self.accept(expr) - - self.check_not_void(iterable, expr) - if isinstance(iterable, TupleType): - if experiments.STRICT_OPTIONAL: - joined = UninhabitedType() # type: Type - else: - joined = NoneTyp() - for item in iterable.items: - joined = join_types(joined, item) - if isinstance(joined, ErrorType): - self.fail(messages.CANNOT_INFER_ITEM_TYPE, expr) - return AnyType() - return joined - else: - # Non-tuple iterable. - self.check_subtype(iterable, - self.named_generic_type('typing.Iterable', - [AnyType()]), - expr, messages.ITERABLE_EXPECTED) - - echk = self.expr_checker - method = echk.analyze_external_member_access('__iter__', iterable, - expr) - iterator = echk.check_call(method, [], [], expr)[0] - if self.options.python_version[0] >= 3: - nextmethod = '__next__' - else: - nextmethod = 'next' - method = echk.analyze_external_member_access(nextmethod, iterator, - expr) - return echk.check_call(method, [], [], expr)[0] - - def analyze_index_variables(self, index: Node, item_type: Type, - context: Context) -> None: - """Type check or infer for loop or list comprehension index vars.""" - self.check_assignment(index, self.temp_node(item_type, context)) - - def visit_del_stmt(self, s: DelStmt) -> Type: - if isinstance(s.expr, IndexExpr): - e = s.expr - m = MemberExpr(e.base, '__delitem__') - m.line = s.line - c = CallExpr(m, [e.index], [nodes.ARG_POS], [None]) - c.line = s.line - return c.accept(self) - else: - def flatten(t: Node) -> List[Node]: - """Flatten a nested sequence of tuples/lists into one list of nodes.""" - if isinstance(t, TupleExpr) or isinstance(t, ListExpr): - t = cast(Union[TupleExpr, ListExpr], t) - return [b for a in t.items for b in flatten(a)] - else: - return [t] - - s.expr.accept(self) - for elt in flatten(s.expr): - if isinstance(elt, NameExpr): - self.binder.assign_type(elt, - DeletedType(source=elt.name), - self.binder.get_declaration(elt), - self.typing_mode_weak()) - return None - - def visit_decorator(self, e: Decorator) -> Type: - for d in e.decorators: - if isinstance(d, RefExpr): - if d.fullname == 'typing.no_type_check': - e.var.type = AnyType() - e.var.is_ready = True - return NoneTyp() - - e.func.accept(self) - sig = self.function_type(e.func) # type: Type - # Process decorators from the inside out. - for i in range(len(e.decorators)): - n = len(e.decorators) - 1 - i - d = e.decorators[n] - if isinstance(d, NameExpr) and d.fullname == 'typing.overload': - self.fail('Single overload definition, multiple required', e) - continue - dec = self.accept(d) - temp = self.temp_node(sig) - sig, t2 = self.expr_checker.check_call(dec, [temp], - [nodes.ARG_POS], e) - sig = cast(FunctionLike, sig) - sig = set_callable_name(sig, e.func) - e.var.type = sig - e.var.is_ready = True - if e.func.is_property: - self.check_incompatible_property_override(e) - - def check_incompatible_property_override(self, e: Decorator) -> None: - if not e.var.is_settable_property and e.func.info is not None: - name = e.func.name() - for base in e.func.info.mro[1:]: - base_attr = base.names.get(name) - if not base_attr: - continue - if (isinstance(base_attr.node, OverloadedFuncDef) and - base_attr.node.is_property and - base_attr.node.items[0].var.is_settable_property): - self.fail(messages.READ_ONLY_PROPERTY_OVERRIDES_READ_WRITE, e) - - def visit_with_stmt(self, s: WithStmt) -> Type: - echk = self.expr_checker - for expr, target in zip(s.expr, s.target): - ctx = self.accept(expr) - enter = echk.analyze_external_member_access('__enter__', ctx, expr) - obj = echk.check_call(enter, [], [], expr)[0] - if target: - self.check_assignment(target, self.temp_node(obj, expr)) - exit = echk.analyze_external_member_access('__exit__', ctx, expr) - arg = self.temp_node(AnyType(), expr) - echk.check_call(exit, [arg] * 3, [nodes.ARG_POS] * 3, expr) - self.accept(s.body) - - def visit_print_stmt(self, s: PrintStmt) -> Type: - for arg in s.args: - self.accept(arg) - if s.target: - target_type = self.accept(s.target) - if not isinstance(target_type, NoneTyp): - # TODO: Also verify the type of 'write'. - self.expr_checker.analyze_external_member_access('write', target_type, s.target) - - # - # Expressions - # - - def visit_name_expr(self, e: NameExpr) -> Type: - return self.expr_checker.visit_name_expr(e) - - def visit_call_expr(self, e: CallExpr) -> Type: - return self.expr_checker.visit_call_expr(e) - - def visit_yield_from_expr(self, e: YieldFromExpr) -> Type: - return_type = self.return_types[-1] - subexpr_type = self.accept(e.expr, return_type) - iter_type = None # type: Type - - # Check that the expr is an instance of Iterable and get the type of the iterator produced - # by __iter__. - if isinstance(subexpr_type, AnyType): - iter_type = AnyType() - elif (isinstance(subexpr_type, Instance) and - is_subtype(subexpr_type, self.named_type('typing.Iterable'))): - iter_method_type = self.expr_checker.analyze_external_member_access( - '__iter__', - subexpr_type, - AnyType()) - - generic_generator_type = self.named_generic_type('typing.Generator', - [AnyType(), AnyType(), AnyType()]) - iter_type, _ = self.expr_checker.check_call(iter_method_type, [], [], - context=generic_generator_type) - else: - self.msg.yield_from_invalid_operand_type(subexpr_type, e) - iter_type = AnyType() - - # Check that the iterator's item type matches the type yielded by the Generator function - # containing this `yield from` expression. - expected_item_type = self.get_generator_yield_type(return_type) - actual_item_type = self.get_generator_yield_type(iter_type) - - self.check_subtype(actual_item_type, expected_item_type, e, - messages.INCOMPATIBLE_TYPES_IN_YIELD_FROM, - 'actual type', 'expected type') - - # Determine the type of the entire yield from expression. - if (isinstance(iter_type, Instance) and - iter_type.type.fullname() == 'typing.Generator'): - return self.get_generator_return_type(iter_type) - else: - # Non-Generators don't return anything from `yield from` expressions. - return Void() - - def visit_member_expr(self, e: MemberExpr) -> Type: - return self.expr_checker.visit_member_expr(e) - - def visit_break_stmt(self, s: BreakStmt) -> Type: - self.binder.breaking_out = True - self.binder.allow_jump(self.binder.loop_frames[-1] - 1) - return None - - def visit_continue_stmt(self, s: ContinueStmt) -> Type: - self.binder.breaking_out = True - self.binder.allow_jump(self.binder.loop_frames[-1]) - return None - - def visit_int_expr(self, e: IntExpr) -> Type: - return self.expr_checker.visit_int_expr(e) - - def visit_str_expr(self, e: StrExpr) -> Type: - return self.expr_checker.visit_str_expr(e) - - def visit_bytes_expr(self, e: BytesExpr) -> Type: - return self.expr_checker.visit_bytes_expr(e) - - def visit_unicode_expr(self, e: UnicodeExpr) -> Type: - return self.expr_checker.visit_unicode_expr(e) - - def visit_float_expr(self, e: FloatExpr) -> Type: - return self.expr_checker.visit_float_expr(e) - - def visit_complex_expr(self, e: ComplexExpr) -> Type: - return self.expr_checker.visit_complex_expr(e) - - def visit_ellipsis(self, e: EllipsisExpr) -> Type: - return self.expr_checker.visit_ellipsis(e) - - def visit_op_expr(self, e: OpExpr) -> Type: - return self.expr_checker.visit_op_expr(e) - - def visit_comparison_expr(self, e: ComparisonExpr) -> Type: - return self.expr_checker.visit_comparison_expr(e) - - def visit_unary_expr(self, e: UnaryExpr) -> Type: - return self.expr_checker.visit_unary_expr(e) - - def visit_index_expr(self, e: IndexExpr) -> Type: - return self.expr_checker.visit_index_expr(e) - - def visit_cast_expr(self, e: CastExpr) -> Type: - return self.expr_checker.visit_cast_expr(e) - - def visit_reveal_type_expr(self, e: RevealTypeExpr) -> Type: - return self.expr_checker.visit_reveal_type_expr(e) - - def visit_super_expr(self, e: SuperExpr) -> Type: - return self.expr_checker.visit_super_expr(e) - - def visit_type_application(self, e: TypeApplication) -> Type: - return self.expr_checker.visit_type_application(e) - - def visit_type_alias_expr(self, e: TypeAliasExpr) -> Type: - return self.expr_checker.visit_type_alias_expr(e) - - def visit_type_var_expr(self, e: TypeVarExpr) -> Type: - # TODO: Perhaps return a special type used for type variables only? - return AnyType() - - def visit_namedtuple_expr(self, e: NamedTupleExpr) -> Type: - # TODO: Perhaps return a type object type? - return AnyType() - - def visit_list_expr(self, e: ListExpr) -> Type: - return self.expr_checker.visit_list_expr(e) - - def visit_set_expr(self, e: SetExpr) -> Type: - return self.expr_checker.visit_set_expr(e) - - def visit_tuple_expr(self, e: TupleExpr) -> Type: - return self.expr_checker.visit_tuple_expr(e) - - def visit_dict_expr(self, e: DictExpr) -> Type: - return self.expr_checker.visit_dict_expr(e) - - def visit_slice_expr(self, e: SliceExpr) -> Type: - return self.expr_checker.visit_slice_expr(e) - - def visit_func_expr(self, e: FuncExpr) -> Type: - return self.expr_checker.visit_func_expr(e) - - def visit_list_comprehension(self, e: ListComprehension) -> Type: - return self.expr_checker.visit_list_comprehension(e) - - def visit_set_comprehension(self, e: SetComprehension) -> Type: - return self.expr_checker.visit_set_comprehension(e) - - def visit_generator_expr(self, e: GeneratorExpr) -> Type: - return self.expr_checker.visit_generator_expr(e) - - def visit_dictionary_comprehension(self, e: DictionaryComprehension) -> Type: - return self.expr_checker.visit_dictionary_comprehension(e) - - def visit_temp_node(self, e: TempNode) -> Type: - return e.type - - def visit_conditional_expr(self, e: ConditionalExpr) -> Type: - return self.expr_checker.visit_conditional_expr(e) - - def visit_backquote_expr(self, e: BackquoteExpr) -> Type: - return self.expr_checker.visit_backquote_expr(e) - - def visit_yield_expr(self, e: YieldExpr) -> Type: - return_type = self.return_types[-1] - expected_item_type = self.get_generator_yield_type(return_type) - if e.expr is None: - if (not (isinstance(expected_item_type, Void) or - isinstance(expected_item_type, AnyType)) - and self.typing_mode_full()): - self.fail(messages.YIELD_VALUE_EXPECTED, e) - else: - actual_item_type = self.accept(e.expr, expected_item_type) - self.check_subtype(actual_item_type, expected_item_type, e, - messages.INCOMPATIBLE_TYPES_IN_YIELD, - 'actual type', 'expected type') - return self.get_generator_receive_type(return_type) - - # - # Helpers - # - - def check_subtype(self, subtype: Type, supertype: Type, context: Context, - msg: str = messages.INCOMPATIBLE_TYPES, - subtype_label: str = None, - supertype_label: str = None) -> None: - """Generate an error if the subtype is not compatible with - supertype.""" - if not is_subtype(subtype, supertype): - if isinstance(subtype, Void): - self.msg.does_not_return_value(subtype, context) - else: - extra_info = [] # type: List[str] - if subtype_label is not None or supertype_label is not None: - subtype_str, supertype_str = self.msg.format_distinctly(subtype, supertype) - if subtype_label is not None: - extra_info.append(subtype_label + ' ' + subtype_str) - if supertype_label is not None: - extra_info.append(supertype_label + ' ' + supertype_str) - if extra_info: - msg += ' (' + ', '.join(extra_info) + ')' - self.fail(msg, context) - - def named_type(self, name: str) -> Instance: - """Return an instance type with type given by the name and no - type arguments. For example, named_type('builtins.object') - produces the object type. - """ - # Assume that the name refers to a type. - sym = self.lookup_qualified(name) - return Instance(cast(TypeInfo, sym.node), []) - - def named_generic_type(self, name: str, args: List[Type]) -> Instance: - """Return an instance with the given name and type arguments. - - Assume that the number of arguments is correct. Assume that - the name refers to a compatible generic type. - """ - return Instance(self.lookup_typeinfo(name), args) - - def lookup_typeinfo(self, fullname: str) -> TypeInfo: - # Assume that the name refers to a class. - sym = self.lookup_qualified(fullname) - return cast(TypeInfo, sym.node) - - def type_type(self) -> Instance: - """Return instance type 'type'.""" - return self.named_type('builtins.type') - - def object_type(self) -> Instance: - """Return instance type 'object'.""" - return self.named_type('builtins.object') - - def bool_type(self) -> Instance: - """Return instance type 'bool'.""" - return self.named_type('builtins.bool') - - def str_type(self) -> Instance: - """Return instance type 'str'.""" - return self.named_type('builtins.str') - - def check_type_equivalency(self, t1: Type, t2: Type, node: Context, - msg: str = messages.INCOMPATIBLE_TYPES) -> None: - """Generate an error if the types are not equivalent. The - dynamic type is equivalent with all types. - """ - if not is_equivalent(t1, t2): - self.fail(msg, node) - - def store_type(self, node: Node, typ: Type) -> None: - """Store the type of a node in the type map.""" - self.type_map[node] = typ - - def typing_mode_none(self) -> bool: - if self.is_dynamic_function() and not self.options.check_untyped_defs: - return not self.weak_opts - elif self.function_stack: - return False - else: - return False - - def typing_mode_weak(self) -> bool: - if self.is_dynamic_function() and not self.options.check_untyped_defs: - return bool(self.weak_opts) - elif self.function_stack: - return False - else: - return 'global' in self.weak_opts - - def typing_mode_full(self) -> bool: - if self.is_dynamic_function() and not self.options.check_untyped_defs: - return False - elif self.function_stack: - return True - else: - return 'global' not in self.weak_opts - - def is_dynamic_function(self) -> bool: - return len(self.dynamic_funcs) > 0 and self.dynamic_funcs[-1] - - def lookup(self, name: str, kind: int) -> SymbolTableNode: - """Look up a definition from the symbol table with the given name. - TODO remove kind argument - """ - if name in self.globals: - return self.globals[name] - else: - b = self.globals.get('__builtins__', None) - if b: - table = cast(MypyFile, b.node).names - if name in table: - return table[name] - raise KeyError('Failed lookup: {}'.format(name)) - - def lookup_qualified(self, name: str) -> SymbolTableNode: - if '.' not in name: - return self.lookup(name, GDEF) # FIX kind - else: - parts = name.split('.') - n = self.modules[parts[0]] - for i in range(1, len(parts) - 1): - n = cast(MypyFile, n.names.get(parts[i], None).node) - return n.names[parts[-1]] - - def enter_partial_types(self) -> None: - """Push a new scope for collecting partial types.""" - self.partial_types.append({}) - - def leave_partial_types(self) -> None: - """Pop partial type scope. - - Also report errors for variables which still have partial - types, i.e. we couldn't infer a complete type. - """ - partial_types = self.partial_types.pop() - if not self.current_node_deferred: - for var, context in partial_types.items(): - if experiments.STRICT_OPTIONAL and cast(PartialType, var.type).type is None: - # None partial type: assume variable is intended to have type None - var.type = NoneTyp() - else: - self.msg.fail(messages.NEED_ANNOTATION_FOR_VAR, context) - var.type = AnyType() - - def find_partial_types(self, var: Var) -> Optional[Dict[Var, Context]]: - for partial_types in reversed(self.partial_types): - if var in partial_types: - return partial_types - return None - - def is_within_function(self) -> bool: - """Are we currently type checking within a function? - - I.e. not at class body or at the top level. - """ - return self.return_types != [] - - def check_not_void(self, typ: Type, context: Context) -> None: - """Generate an error if the type is Void.""" - if isinstance(typ, Void): - self.msg.does_not_return_value(typ, context) - - def temp_node(self, t: Type, context: Context = None) -> Node: - """Create a temporary node with the given, fixed type.""" - temp = TempNode(t) - if context: - temp.set_line(context.get_line()) - return temp - - def fail(self, msg: str, context: Context) -> None: - """Produce an error message.""" - self.msg.fail(msg, context) - - def iterable_item_type(self, instance: Instance) -> Type: - iterable = map_instance_to_supertype( - instance, - self.lookup_typeinfo('typing.Iterable')) - return iterable.args[0] - - def function_type(self, func: FuncBase) -> FunctionLike: - return function_type(func, self.named_type('builtins.function')) - - def method_type(self, func: FuncBase) -> FunctionLike: - return method_type_with_fallback(func, self.named_type('builtins.function')) - - -# Data structure returned by find_isinstance_check representing -# information learned from the truth or falsehood of a condition. The -# dict maps nodes representing expressions like 'a[0].x' to their -# refined types under the assumption that the condition has a -# particular truth value. A value of None means that the condition can -# never have that truth value. - -# NB: The keys of this dict are nodes in the original source program, -# which are compared by reference equality--effectively, being *the -# same* expression of the program, not just two identical expressions -# (such as two references to the same variable). TODO: it would -# probably be better to have the dict keyed by the nodes' literal_hash -# field instead. - -# NB: This should be `TypeMap = Optional[Dict[Node, Type]]`! -# But see https://github.com/python/mypy/issues/1637 -TypeMap = Dict[Node, Type] - - -def conditional_type_map(expr: Node, - current_type: Optional[Type], - proposed_type: Optional[Type], - *, - weak: bool = False - ) -> Tuple[TypeMap, TypeMap]: - """Takes in an expression, the current type of the expression, and a - proposed type of that expression. - - Returns a 2-tuple: The first element is a map from the expression to - the proposed type, if the expression can be the proposed type. The - second element is a map from the expression to the type it would hold - if it was not the proposed type, if any.""" - if proposed_type: - if current_type: - if is_proper_subtype(current_type, proposed_type): - return {expr: proposed_type}, None - elif not is_overlapping_types(current_type, proposed_type): - return None, {expr: current_type} - else: - remaining_type = restrict_subtype_away(current_type, proposed_type) - return {expr: proposed_type}, {expr: remaining_type} - else: - return {expr: proposed_type}, {} - else: - # An isinstance check, but we don't understand the type - if weak: - return {expr: AnyType()}, {expr: current_type} - else: - return {}, {} - - -def is_literal_none(n: Node) -> bool: - return isinstance(n, NameExpr) and n.fullname == 'builtins.None' - - -def and_conditional_maps(m1: TypeMap, m2: TypeMap) -> TypeMap: - """Calculate what information we can learn from the truth of (e1 and e2) - in terms of the information that we can learn from the truth of e1 and - the truth of e2. - """ - - if m1 is None or m2 is None: - # One of the conditions can never be true. - return None - # Both conditions can be true; combine the information. Anything - # we learn from either conditions's truth is valid. If the same - # expression's type is refined by both conditions, we somewhat - # arbitrarily give precedence to m2. (In the future, we could use - # an intersection type.) - result = m2.copy() - m2_keys = set(n2.literal_hash for n2 in m2) - for n1 in m1: - if n1.literal_hash not in m2_keys: - result[n1] = m1[n1] - return result - - -def or_conditional_maps(m1: TypeMap, m2: TypeMap) -> TypeMap: - """Calculate what information we can learn from the truth of (e1 or e2) - in terms of the information that we can learn from the truth of e1 and - the truth of e2. - """ - - if m1 is None: - return m2 - if m2 is None: - return m1 - # Both conditions can be true. Combine information about - # expressions whose type is refined by both conditions. (We do not - # learn anything about expressions whose type is refined by only - # one condition.) - result = {} - for n1 in m1: - for n2 in m2: - if n1.literal_hash == n2.literal_hash: - result[n1] = UnionType.make_simplified_union([m1[n1], m2[n2]]) - return result - - -def find_isinstance_check(node: Node, - type_map: Dict[Node, Type], - weak: bool=False - ) -> Tuple[TypeMap, TypeMap]: - """Find any isinstance checks (within a chain of ands). Includes - implicit and explicit checks for None. - - Return value is a map of variables to their types if the condition - is true and a map of variables to their types if the condition is false. - - If either of the values in the tuple is None, then that particular - branch can never occur. - - Guaranteed to not return None, None. (But may return {}, {}) - """ - if isinstance(node, CallExpr): - if refers_to_fullname(node.callee, 'builtins.isinstance'): - expr = node.args[0] - if expr.literal == LITERAL_TYPE: - vartype = type_map[expr] - type = get_isinstance_type(node.args[1], type_map) - return conditional_type_map(expr, vartype, type, weak=weak) - elif (isinstance(node, ComparisonExpr) and any(is_literal_none(n) for n in node.operands) and - experiments.STRICT_OPTIONAL): - # Check for `x is None` and `x is not None`. - is_not = node.operators == ['is not'] - if is_not or node.operators == ['is']: - if_vars = {} # type: Dict[Node, Type] - else_vars = {} # type: Dict[Node, Type] - for expr in node.operands: - if expr.literal == LITERAL_TYPE and not is_literal_none(expr) and expr in type_map: - # This should only be true at most once: there should be - # two elements in node.operands, and at least one of them - # should represent a None. - vartype = type_map[expr] - if_vars, else_vars = conditional_type_map(expr, vartype, NoneTyp(), weak=weak) - break - - if is_not: - if_vars, else_vars = else_vars, if_vars - return if_vars, else_vars - elif isinstance(node, RefExpr) and experiments.STRICT_OPTIONAL: - # The type could be falsy, so we can't deduce anything new about the else branch - vartype = type_map[node] - _, if_vars = conditional_type_map(node, vartype, NoneTyp(), weak=weak) - return if_vars, {} - elif isinstance(node, OpExpr) and node.op == 'and': - left_if_vars, left_else_vars = find_isinstance_check( - node.left, - type_map, - weak, - ) - - right_if_vars, right_else_vars = find_isinstance_check( - node.right, - type_map, - weak, - ) - - # (e1 and e2) is true if both e1 and e2 are true, - # and false if at least one of e1 and e2 is false. - return (and_conditional_maps(left_if_vars, right_if_vars), - or_conditional_maps(left_else_vars, right_else_vars)) - elif isinstance(node, OpExpr) and node.op == 'or': - left_if_vars, left_else_vars = find_isinstance_check( - node.left, - type_map, - weak, - ) - - right_if_vars, right_else_vars = find_isinstance_check( - node.right, - type_map, - weak, - ) - - # (e1 or e2) is true if at least one of e1 or e2 is true, - # and false if both e1 and e2 are false. - return (or_conditional_maps(left_if_vars, right_if_vars), - and_conditional_maps(left_else_vars, right_else_vars)) - elif isinstance(node, UnaryExpr) and node.op == 'not': - left, right = find_isinstance_check(node.expr, type_map, weak) - return right, left - - # Not a supported isinstance check - return {}, {} - - -def get_isinstance_type(node: Node, type_map: Dict[Node, Type]) -> Type: - type = type_map[node] - - if isinstance(type, TupleType): - all_types = type.items - else: - all_types = [type] - - types = [] # type: List[Type] - - for type in all_types: - if isinstance(type, FunctionLike): - if type.is_type_obj(): - # Type variables may be present -- erase them, which is the best - # we can do (outside disallowing them here). - type = erase_typevars(type.items()[0].ret_type) - - types.append(type) - - if len(types) == 0: - return None - elif len(types) == 1: - return types[0] - else: - return UnionType(types) - - -def expand_node(defn: Node, map: Dict[TypeVarId, Type]) -> Node: - visitor = TypeTransformVisitor(map) - return defn.accept(visitor) - - -def expand_func(defn: FuncItem, map: Dict[TypeVarId, Type]) -> FuncItem: - return cast(FuncItem, expand_node(defn, map)) - - -class TypeTransformVisitor(TransformVisitor): - def __init__(self, map: Dict[TypeVarId, Type]) -> None: - super().__init__() - self.map = map - - def type(self, type: Type) -> Type: - return expand_type(type, self.map) - - -def is_unsafe_overlapping_signatures(signature: Type, other: Type) -> bool: - """Check if two signatures may be unsafely overlapping. - - Two signatures s and t are overlapping if both can be valid for the same - statically typed values and the return types are incompatible. - - Assume calls are first checked against 'signature', then against 'other'. - Thus if 'signature' is more general than 'other', there is no unsafe - overlapping. - - TODO If argument types vary covariantly, the return type may vary - covariantly as well. - """ - if isinstance(signature, CallableType): - if isinstance(other, CallableType): - # TODO varargs - # TODO keyword args - # TODO erasure - # TODO allow to vary covariantly - # Check if the argument counts are overlapping. - min_args = max(signature.min_args, other.min_args) - max_args = min(len(signature.arg_types), len(other.arg_types)) - if min_args > max_args: - # Argument counts are not overlapping. - return False - # Signatures are overlapping iff if they are overlapping for the - # smallest common argument count. - for i in range(min_args): - t1 = signature.arg_types[i] - t2 = other.arg_types[i] - if not is_overlapping_types(t1, t2): - return False - # All arguments types for the smallest common argument count are - # overlapping => the signature is overlapping. The overlapping is - # safe if the return types are identical. - if is_same_type(signature.ret_type, other.ret_type): - return False - # If the first signature has more general argument types, the - # latter will never be called - if is_more_general_arg_prefix(signature, other): - return False - return not is_more_precise_signature(signature, other) - return True - - -def is_more_general_arg_prefix(t: FunctionLike, s: FunctionLike) -> bool: - """Does t have wider arguments than s?""" - # TODO should an overload with additional items be allowed to be more - # general than one with fewer items (or just one item)? - # TODO check argument kinds - if isinstance(t, CallableType): - if isinstance(s, CallableType): - return all(is_proper_subtype(args, argt) - for argt, args in zip(t.arg_types, s.arg_types)) - elif isinstance(t, FunctionLike): - if isinstance(s, FunctionLike): - if len(t.items()) == len(s.items()): - return all(is_same_arg_prefix(items, itemt) - for items, itemt in zip(t.items(), s.items())) - return False - - -def is_same_arg_prefix(t: CallableType, s: CallableType) -> bool: - # TODO check argument kinds - return all(is_same_type(argt, args) - for argt, args in zip(t.arg_types, s.arg_types)) - - -def is_more_precise_signature(t: CallableType, s: CallableType) -> bool: - """Is t more precise than s? - - A signature t is more precise than s if all argument types and the return - type of t are more precise than the corresponding types in s. - - Assume that the argument kinds and names are compatible, and that the - argument counts are overlapping. - """ - # TODO generic function types - # Only consider the common prefix of argument types. - for argt, args in zip(t.arg_types, s.arg_types): - if not is_more_precise(argt, args): - return False - return is_more_precise(t.ret_type, s.ret_type) - - -def infer_operator_assignment_method(type: Type, operator: str) -> str: - """Return the method used for operator assignment for given value type. - - For example, if operator is '+', return '__iadd__' or '__add__' depending - on which method is supported by the type. - """ - method = nodes.op_methods[operator] - if isinstance(type, Instance): - if operator in nodes.ops_with_inplace_method: - inplace = '__i' + method[2:] - if type.type.has_readable_member(inplace): - method = inplace - return method - - -def is_valid_inferred_type(typ: Type) -> bool: - """Is an inferred type valid? - - Examples of invalid types include the None type or a type with a None component. - """ - if is_same_type(typ, NoneTyp()): - # With strict Optional checking, we *may* eventually infer NoneTyp, but - # we only do that if we can't infer a specific Optional type. This - # resolution happens in leave_partial_types when we pop a partial types - # scope. - return False - if is_same_type(typ, UninhabitedType()): - return False - elif isinstance(typ, Instance): - for arg in typ.args: - if not is_valid_inferred_type(arg): - return False - elif isinstance(typ, TupleType): - for item in typ.items: - if not is_valid_inferred_type(item): - return False - return True +"""Mypy type checker.""" + +import itertools +import contextlib +import os +import os.path + +from typing import ( + Any, Dict, Set, List, cast, Tuple, TypeVar, Union, Optional, NamedTuple +) + +from mypy.errors import Errors, report_internal_error +from mypy.nodes import ( + SymbolTable, Node, MypyFile, Var, + OverloadedFuncDef, FuncDef, FuncItem, FuncBase, TypeInfo, + ClassDef, GDEF, Block, AssignmentStmt, NameExpr, MemberExpr, IndexExpr, + TupleExpr, ListExpr, ExpressionStmt, ReturnStmt, IfStmt, + WhileStmt, OperatorAssignmentStmt, WithStmt, AssertStmt, + RaiseStmt, TryStmt, ForStmt, DelStmt, CallExpr, IntExpr, StrExpr, + BytesExpr, UnicodeExpr, FloatExpr, OpExpr, UnaryExpr, CastExpr, RevealTypeExpr, SuperExpr, + TypeApplication, DictExpr, SliceExpr, FuncExpr, TempNode, SymbolTableNode, + Context, ListComprehension, ConditionalExpr, GeneratorExpr, + Decorator, SetExpr, TypeVarExpr, PrintStmt, + LITERAL_TYPE, BreakStmt, ContinueStmt, ComparisonExpr, StarExpr, + YieldFromExpr, NamedTupleExpr, SetComprehension, + DictionaryComprehension, ComplexExpr, EllipsisExpr, TypeAliasExpr, + RefExpr, YieldExpr, BackquoteExpr, ImportFrom, ImportAll, ImportBase, + CONTRAVARIANT, COVARIANT +) +from mypy.nodes import function_type, method_type, method_type_with_fallback +from mypy import nodes +from mypy.types import ( + Type, AnyType, CallableType, Void, FunctionLike, Overloaded, TupleType, + Instance, NoneTyp, ErrorType, strip_type, + UnionType, TypeVarId, TypeVarType, PartialType, DeletedType, UninhabitedType +) +from mypy.sametypes import is_same_type +from mypy.messages import MessageBuilder +import mypy.checkexpr +from mypy.checkmember import map_type_from_supertype +from mypy import defaults +from mypy import messages +from mypy.subtypes import ( + is_subtype, is_equivalent, is_proper_subtype, + is_more_precise, restrict_subtype_away +) +from mypy.maptype import map_instance_to_supertype +from mypy.semanal import self_type, set_callable_name, refers_to_fullname +from mypy.erasetype import erase_typevars +from mypy.expandtype import expand_type_by_instance, expand_type +from mypy.visitor import NodeVisitor +from mypy.join import join_types +from mypy.treetransform import TransformVisitor +from mypy.meet import meet_simple, nearest_builtin_ancestor, is_overlapping_types +from mypy.binder import ConditionalTypeBinder +from mypy.options import Options + +from mypy import experiments + + +T = TypeVar('T') + + +# A node which is postponed to be type checked during the next pass. +DeferredNode = NamedTuple( + 'DeferredNode', + [ + ('node', Node), + ('context_type_name', Optional[str]), # Name of the surrounding class (for error messages) + ]) + + +class TypeChecker(NodeVisitor[Type]): + """Mypy type checker. + + Type check mypy source files that have been semantically analyzed. + """ + + # Are we type checking a stub? + is_stub = False + # Error message reporter + errors = None # type: Errors + # Utility for generating messages + msg = None # type: MessageBuilder + # Types of type checked nodes + type_map = None # type: Dict[Node, Type] + + # Helper for managing conditional types + binder = None # type: ConditionalTypeBinder + # Helper for type checking expressions + expr_checker = None # type: mypy.checkexpr.ExpressionChecker + + # Stack of function return types + return_types = None # type: List[Type] + # Type context for type inference + type_context = None # type: List[Type] + # Flags; true for dynamically typed functions + dynamic_funcs = None # type: List[bool] + # Stack of functions being type checked + function_stack = None # type: List[FuncItem] + # Do weak type checking in this file + weak_opts = set() # type: Set[str] + # Stack of collections of variables with partial types + partial_types = None # type: List[Dict[Var, Context]] + globals = None # type: SymbolTable + modules = None # type: Dict[str, MypyFile] + # Nodes that couldn't be checked because some types weren't available. We'll run + # another pass and try these again. + deferred_nodes = None # type: List[DeferredNode] + # Type checking pass number (0 = first pass) + pass_num = 0 + # Have we deferred the current function? If yes, don't infer additional + # types during this pass within the function. + current_node_deferred = False + is_typeshed_stub = False + options = None # type: Options + + def __init__(self, errors: Errors, modules: Dict[str, MypyFile], options: Options) -> None: + """Construct a type checker. + + Use errors to report type check errors. + """ + self.errors = errors + self.modules = modules + self.options = options + self.msg = MessageBuilder(errors, modules) + self.type_map = {} + self.binder = ConditionalTypeBinder() + self.expr_checker = mypy.checkexpr.ExpressionChecker(self, self.msg) + self.return_types = [] + self.type_context = [] + self.dynamic_funcs = [] + self.function_stack = [] + self.weak_opts = set() # type: Set[str] + self.partial_types = [] + self.deferred_nodes = [] + self.pass_num = 0 + self.current_node_deferred = False + + def visit_file(self, file_node: MypyFile, path: str) -> None: + """Type check a mypy file with the given path.""" + self.pass_num = 0 + self.is_stub = file_node.is_stub + self.errors.set_file(path) + self.globals = file_node.names + self.weak_opts = file_node.weak_opts + self.enter_partial_types() + # gross, but no other clear way to tell + self.is_typeshed_stub = self.is_stub and 'typeshed' in os.path.normpath(path).split(os.sep) + + for d in file_node.defs: + self.accept(d) + + self.leave_partial_types() + + if self.deferred_nodes: + self.check_second_pass() + + self.current_node_deferred = False + + all_ = self.globals.get('__all__') + if all_ is not None and all_.type is not None: + seq_str = self.named_generic_type('typing.Sequence', + [self.named_type('builtins.str')]) + if not is_subtype(all_.type, seq_str): + str_seq_s, all_s = self.msg.format_distinctly(seq_str, all_.type) + self.fail(messages.ALL_MUST_BE_SEQ_STR.format(str_seq_s, all_s), + all_.node) + + def check_second_pass(self): + """Run second pass of type checking which goes through deferred nodes.""" + self.pass_num = 1 + for node, type_name in self.deferred_nodes: + if type_name: + self.errors.push_type(type_name) + self.accept(node) + if type_name: + self.errors.pop_type() + self.deferred_nodes = [] + + def handle_cannot_determine_type(self, name: str, context: Context) -> None: + if self.pass_num == 0 and self.function_stack: + # Don't report an error yet. Just defer. + node = self.function_stack[-1] + if self.errors.type_name: + type_name = self.errors.type_name[-1] + else: + type_name = None + self.deferred_nodes.append(DeferredNode(node, type_name)) + # Set a marker so that we won't infer additional types in this + # function. Any inferred types could be bogus, because there's at + # least one type that we don't know. + self.current_node_deferred = True + else: + self.msg.cannot_determine_type(name, context) + + def accept(self, node: Node, type_context: Type = None) -> Type: + """Type check a node in the given type context.""" + self.type_context.append(type_context) + try: + typ = node.accept(self) + except Exception as err: + report_internal_error(err, self.errors.file, node.line) + self.type_context.pop() + self.store_type(node, typ) + if self.typing_mode_none(): + return AnyType() + else: + return typ + + def accept_loop(self, body: Node, else_body: Node = None) -> Type: + """Repeatedly type check a loop body until the frame doesn't change. + + Then check the else_body. + """ + # The outer frame accumulates the results of all iterations + with self.binder.frame_context(1) as outer_frame: + self.binder.push_loop_frame() + while True: + with self.binder.frame_context(1): + # We may skip each iteration + self.binder.options_on_return[-1].append(outer_frame) + self.accept(body) + if not self.binder.last_pop_changed: + break + self.binder.pop_loop_frame() + if else_body: + self.accept(else_body) + + # + # Definitions + # + + def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> Type: + num_abstract = 0 + if defn.is_property: + # HACK: Infer the type of the property. + self.visit_decorator(defn.items[0]) + for fdef in defn.items: + self.check_func_item(fdef.func, name=fdef.func.name()) + if fdef.func.is_abstract: + num_abstract += 1 + if num_abstract not in (0, len(defn.items)): + self.fail(messages.INCONSISTENT_ABSTRACT_OVERLOAD, defn) + if defn.info: + self.check_method_override(defn) + self.check_inplace_operator_method(defn) + self.check_overlapping_overloads(defn) + + def check_overlapping_overloads(self, defn: OverloadedFuncDef) -> None: + for i, item in enumerate(defn.items): + for j, item2 in enumerate(defn.items[i + 1:]): + # TODO overloads involving decorators + sig1 = self.function_type(item.func) + sig2 = self.function_type(item2.func) + if is_unsafe_overlapping_signatures(sig1, sig2): + self.msg.overloaded_signatures_overlap(i + 1, i + j + 2, + item.func) + + def is_generator_return_type(self, typ: Type) -> bool: + return is_subtype(self.named_generic_type('typing.Generator', + [AnyType(), AnyType(), AnyType()]), + typ) + + def get_generator_yield_type(self, return_type: Type) -> Type: + if isinstance(return_type, AnyType): + return AnyType() + elif not self.is_generator_return_type(return_type): + # If the function doesn't have a proper Generator (or superclass) return type, anything + # is permissible. + return AnyType() + elif not isinstance(return_type, Instance): + # Same as above, but written as a separate branch so the typechecker can understand. + return AnyType() + elif return_type.args: + return return_type.args[0] + else: + # If the function's declared supertype of Generator has no type + # parameters (i.e. is `object`), then the yielded values can't + # be accessed so any type is acceptable. + return AnyType() + + def get_generator_receive_type(self, return_type: Type) -> Type: + if isinstance(return_type, AnyType): + return AnyType() + elif not self.is_generator_return_type(return_type): + # If the function doesn't have a proper Generator (or superclass) return type, anything + # is permissible. + return AnyType() + elif not isinstance(return_type, Instance): + # Same as above, but written as a separate branch so the typechecker can understand. + return AnyType() + elif return_type.type.fullname() == 'typing.Generator': + # Generator is the only type which specifies the type of values it can receive. + return return_type.args[1] + else: + # `return_type` is a supertype of Generator, so callers won't be able to send it + # values. + return Void() + + def get_generator_return_type(self, return_type: Type) -> Type: + if isinstance(return_type, AnyType): + return AnyType() + elif not self.is_generator_return_type(return_type): + # If the function doesn't have a proper Generator (or superclass) return type, anything + # is permissible. + return AnyType() + elif not isinstance(return_type, Instance): + # Same as above, but written as a separate branch so the typechecker can understand. + return AnyType() + elif return_type.type.fullname() == 'typing.Generator': + # Generator is the only type which specifies the type of values it returns into + # `yield from` expressions. + return return_type.args[2] + else: + # `return_type` is supertype of Generator, so callers won't be able to see the return + # type when used in a `yield from` expression. + return AnyType() + + def visit_func_def(self, defn: FuncDef) -> Type: + """Type check a function definition.""" + self.check_func_item(defn, name=defn.name()) + if defn.info: + if not defn.is_dynamic(): + self.check_method_override(defn) + self.check_inplace_operator_method(defn) + if defn.original_def: + # Override previous definition. + new_type = self.function_type(defn) + if isinstance(defn.original_def, FuncDef): + # Function definition overrides function definition. + if not is_same_type(new_type, self.function_type(defn.original_def)): + self.msg.incompatible_conditional_function_def(defn) + else: + # Function definition overrides a variable initialized via assignment. + orig_type = defn.original_def.type + if orig_type is None: + # XXX This can be None, as happens in + # test_testcheck_TypeCheckSuite.testRedefinedFunctionInTryWithElse + self.msg.note("Internal mypy error checking function redefinition.", defn) + return None + if isinstance(orig_type, PartialType): + if orig_type.type is None: + # Ah this is a partial type. Give it the type of the function. + var = defn.original_def + partial_types = self.find_partial_types(var) + if partial_types is not None: + var.type = new_type + del partial_types[var] + else: + # Trying to redefine something like partial empty list as function. + self.fail(messages.INCOMPATIBLE_REDEFINITION, defn) + else: + # TODO: Update conditional type binder. + self.check_subtype(new_type, orig_type, defn, + messages.INCOMPATIBLE_REDEFINITION, + 'redefinition with type', + 'original type') + + def check_func_item(self, defn: FuncItem, + type_override: CallableType = None, + name: str = None) -> Type: + """Type check a function. + + If type_override is provided, use it as the function type. + """ + # We may be checking a function definition or an anonymous function. In + # the first case, set up another reference with the precise type. + fdef = None # type: FuncDef + if isinstance(defn, FuncDef): + fdef = defn + + self.function_stack.append(defn) + self.dynamic_funcs.append(defn.is_dynamic() and not type_override) + + if fdef: + self.errors.push_function(fdef.name()) + + self.enter_partial_types() + + typ = self.function_type(defn) + if type_override: + typ = type_override + if isinstance(typ, CallableType): + self.check_func_def(defn, typ, name) + else: + raise RuntimeError('Not supported') + + self.leave_partial_types() + + if fdef: + self.errors.pop_function() + + self.dynamic_funcs.pop() + self.function_stack.pop() + self.current_node_deferred = False + + def check_func_def(self, defn: FuncItem, typ: CallableType, name: str) -> None: + """Type check a function definition.""" + # Expand type variables with value restrictions to ordinary types. + for item, typ in self.expand_typevars(defn, typ): + old_binder = self.binder + self.binder = ConditionalTypeBinder() + with self.binder.frame_context(): + defn.expanded.append(item) + + # We may be checking a function definition or an anonymous + # function. In the first case, set up another reference with the + # precise type. + if isinstance(item, FuncDef): + fdef = item + else: + fdef = None + + if fdef: + # Check if __init__ has an invalid, non-None return type. + if (fdef.info and fdef.name() == '__init__' and + not isinstance(typ.ret_type, Void) and + not self.dynamic_funcs[-1]): + self.fail(messages.INIT_MUST_HAVE_NONE_RETURN_TYPE, + item.type) + + show_untyped = not self.is_typeshed_stub or self.options.warn_incomplete_stub + if self.options.disallow_untyped_defs and show_untyped: + # Check for functions with unspecified/not fully specified types. + def is_implicit_any(t: Type) -> bool: + return isinstance(t, AnyType) and t.implicit + + if fdef.type is None: + self.fail(messages.FUNCTION_TYPE_EXPECTED, fdef) + elif isinstance(fdef.type, CallableType): + if is_implicit_any(fdef.type.ret_type): + self.fail(messages.RETURN_TYPE_EXPECTED, fdef) + if any(is_implicit_any(t) for t in fdef.type.arg_types): + self.fail(messages.ARGUMENT_TYPE_EXPECTED, fdef) + + if name in nodes.reverse_op_method_set: + self.check_reverse_op_method(item, typ, name) + elif name == '__getattr__': + self.check_getattr_method(typ, defn) + + # Refuse contravariant return type variable + if isinstance(typ.ret_type, TypeVarType): + if typ.ret_type.variance == CONTRAVARIANT: + self.fail(messages.RETURN_TYPE_CANNOT_BE_CONTRAVARIANT, + typ.ret_type) + + # Check that Generator functions have the appropriate return type. + if defn.is_generator: + if not self.is_generator_return_type(typ.ret_type): + self.fail(messages.INVALID_RETURN_TYPE_FOR_GENERATOR, typ) + + # Python 2 generators aren't allowed to return values. + if (self.options.python_version[0] == 2 and + isinstance(typ.ret_type, Instance) and + typ.ret_type.type.fullname() == 'typing.Generator'): + if not (isinstance(typ.ret_type.args[2], Void) + or isinstance(typ.ret_type.args[2], AnyType)): + self.fail(messages.INVALID_GENERATOR_RETURN_ITEM_TYPE, typ) + + # Push return type. + self.return_types.append(typ.ret_type) + + # Store argument types. + for i in range(len(typ.arg_types)): + arg_type = typ.arg_types[i] + + # Refuse covariant parameter type variables + if isinstance(arg_type, TypeVarType): + if arg_type.variance == COVARIANT: + self.fail(messages.FUNCTION_PARAMETER_CANNOT_BE_COVARIANT, + arg_type) + + if typ.arg_kinds[i] == nodes.ARG_STAR: + # builtins.tuple[T] is typing.Tuple[T, ...] + arg_type = self.named_generic_type('builtins.tuple', + [arg_type]) + elif typ.arg_kinds[i] == nodes.ARG_STAR2: + arg_type = self.named_generic_type('builtins.dict', + [self.str_type(), + arg_type]) + item.arguments[i].variable.type = arg_type + + # Type check initialization expressions. + for arg in item.arguments: + init = arg.initialization_statement + if init: + self.accept(init) + + # Type check body in a new scope. + with self.binder.frame_context(): + self.accept(item.body) + + self.return_types.pop() + + self.binder = old_binder + + def check_reverse_op_method(self, defn: FuncItem, typ: CallableType, + method: str) -> None: + """Check a reverse operator method such as __radd__.""" + + # This used to check for some very obscure scenario. It now + # just decides whether it's worth calling + # check_overlapping_op_methods(). + + if method in ('__eq__', '__ne__'): + # These are defined for all objects => can't cause trouble. + return + + # With 'Any' or 'object' return type we are happy, since any possible + # return value is valid. + ret_type = typ.ret_type + if isinstance(ret_type, AnyType): + return + if isinstance(ret_type, Instance): + if ret_type.type.fullname() == 'builtins.object': + return + # Plausibly the method could have too few arguments, which would result + # in an error elsewhere. + if len(typ.arg_types) <= 2: + # TODO check self argument kind + + # Check for the issue described above. + arg_type = typ.arg_types[1] + other_method = nodes.normal_from_reverse_op[method] + if isinstance(arg_type, Instance): + if not arg_type.type.has_readable_member(other_method): + return + elif isinstance(arg_type, AnyType): + return + elif isinstance(arg_type, UnionType): + if not arg_type.has_readable_member(other_method): + return + else: + return + + typ2 = self.expr_checker.analyze_external_member_access( + other_method, arg_type, defn) + self.check_overlapping_op_methods( + typ, method, defn.info, + typ2, other_method, cast(Instance, arg_type), + defn) + + def check_overlapping_op_methods(self, + reverse_type: CallableType, + reverse_name: str, + reverse_class: TypeInfo, + forward_type: Type, + forward_name: str, + forward_base: Instance, + context: Context) -> None: + """Check for overlapping method and reverse method signatures. + + Assume reverse method has valid argument count and kinds. + """ + + # Reverse operator method that overlaps unsafely with the + # forward operator method can result in type unsafety. This is + # similar to overlapping overload variants. + # + # This example illustrates the issue: + # + # class X: pass + # class A: + # def __add__(self, x: X) -> int: + # if isinstance(x, X): + # return 1 + # return NotImplemented + # class B: + # def __radd__(self, x: A) -> str: return 'x' + # class C(X, B): pass + # def f(b: B) -> None: + # A() + b # Result is 1, even though static type seems to be str! + # f(C()) + # + # The reason for the problem is that B and X are overlapping + # types, and the return types are different. Also, if the type + # of x in __radd__ would not be A, the methods could be + # non-overlapping. + + if isinstance(forward_type, CallableType): + # TODO check argument kinds + if len(forward_type.arg_types) < 1: + # Not a valid operator method -- can't succeed anyway. + return + + # Construct normalized function signatures corresponding to the + # operator methods. The first argument is the left operand and the + # second operand is the right argument -- we switch the order of + # the arguments of the reverse method. + forward_tweaked = CallableType( + [forward_base, forward_type.arg_types[0]], + [nodes.ARG_POS] * 2, + [None] * 2, + forward_type.ret_type, + forward_type.fallback, + name=forward_type.name) + reverse_args = reverse_type.arg_types + reverse_tweaked = CallableType( + [reverse_args[1], reverse_args[0]], + [nodes.ARG_POS] * 2, + [None] * 2, + reverse_type.ret_type, + fallback=self.named_type('builtins.function'), + name=reverse_type.name) + + if is_unsafe_overlapping_signatures(forward_tweaked, + reverse_tweaked): + self.msg.operator_method_signatures_overlap( + reverse_class.name(), reverse_name, + forward_base.type.name(), forward_name, context) + elif isinstance(forward_type, Overloaded): + for item in forward_type.items(): + self.check_overlapping_op_methods( + reverse_type, reverse_name, reverse_class, + item, forward_name, forward_base, context) + elif not isinstance(forward_type, AnyType): + self.msg.forward_operator_not_callable(forward_name, context) + + def check_inplace_operator_method(self, defn: FuncBase) -> None: + """Check an inplace operator method such as __iadd__. + + They cannot arbitrarily overlap with __add__. + """ + method = defn.name() + if method not in nodes.inplace_operator_methods: + return + typ = self.method_type(defn) + cls = defn.info + other_method = '__' + method[3:] + if cls.has_readable_member(other_method): + instance = self_type(cls) + typ2 = self.expr_checker.analyze_external_member_access( + other_method, instance, defn) + fail = False + if isinstance(typ2, FunctionLike): + if not is_more_general_arg_prefix(typ, typ2): + fail = True + else: + # TODO overloads + fail = True + if fail: + self.msg.signatures_incompatible(method, other_method, defn) + + def check_getattr_method(self, typ: CallableType, context: Context) -> None: + method_type = CallableType([AnyType(), self.named_type('builtins.str')], + [nodes.ARG_POS, nodes.ARG_POS], + [None], + AnyType(), + self.named_type('builtins.function')) + if not is_subtype(typ, method_type): + self.msg.invalid_signature(typ, context) + + def expand_typevars(self, defn: FuncItem, + typ: CallableType) -> List[Tuple[FuncItem, CallableType]]: + # TODO use generator + subst = [] # type: List[List[Tuple[TypeVarId, Type]]] + tvars = typ.variables or [] + tvars = tvars[:] + if defn.info: + # Class type variables + tvars += defn.info.defn.type_vars or [] + for tvar in tvars: + if tvar.values: + subst.append([(tvar.id, value) + for value in tvar.values]) + if subst: + result = [] # type: List[Tuple[FuncItem, CallableType]] + for substitutions in itertools.product(*subst): + mapping = dict(substitutions) + expanded = cast(CallableType, expand_type(typ, mapping)) + result.append((expand_func(defn, mapping), expanded)) + return result + else: + return [(defn, typ)] + + def check_method_override(self, defn: FuncBase) -> None: + """Check if function definition is compatible with base classes.""" + # Check against definitions in base classes. + for base in defn.info.mro[1:]: + self.check_method_or_accessor_override_for_base(defn, base) + + def check_method_or_accessor_override_for_base(self, defn: FuncBase, + base: TypeInfo) -> None: + """Check if method definition is compatible with a base class.""" + if base: + name = defn.name() + if name not in ('__init__', '__new__'): + # Check method override (__init__ and __new__ are special). + self.check_method_override_for_base_with_name(defn, name, base) + if name in nodes.inplace_operator_methods: + # Figure out the name of the corresponding operator method. + method = '__' + name[3:] + # An inplace operator method such as __iadd__ might not be + # always introduced safely if a base class defined __add__. + # TODO can't come up with an example where this is + # necessary; now it's "just in case" + self.check_method_override_for_base_with_name(defn, method, + base) + + def check_method_override_for_base_with_name( + self, defn: FuncBase, name: str, base: TypeInfo) -> None: + base_attr = base.names.get(name) + if base_attr: + # The name of the method is defined in the base class. + + # Construct the type of the overriding method. + typ = self.method_type(defn) + # Map the overridden method type to subtype context so that + # it can be checked for compatibility. + original_type = base_attr.type + if original_type is None and isinstance(base_attr.node, + FuncDef): + original_type = self.function_type(base_attr.node) + if isinstance(original_type, FunctionLike): + original = map_type_from_supertype( + method_type(original_type), + defn.info, base) + # Check that the types are compatible. + # TODO overloaded signatures + self.check_override(typ, + cast(FunctionLike, original), + defn.name(), + name, + base.name(), + defn) + else: + assert original_type is not None + self.msg.signature_incompatible_with_supertype( + defn.name(), name, base.name(), defn) + + def check_override(self, override: FunctionLike, original: FunctionLike, + name: str, name_in_super: str, supertype: str, + node: Context) -> None: + """Check a method override with given signatures. + + Arguments: + override: The signature of the overriding method. + original: The signature of the original supertype method. + name: The name of the subtype. This and the next argument are + only used for generating error messages. + supertype: The name of the supertype. + """ + # Use boolean variable to clarify code. + fail = False + if not is_subtype(override, original): + fail = True + elif (not isinstance(original, Overloaded) and + isinstance(override, Overloaded) and + name in nodes.reverse_op_methods.keys()): + # Operator method overrides cannot introduce overloading, as + # this could be unsafe with reverse operator methods. + fail = True + + if fail: + emitted_msg = False + if (isinstance(override, CallableType) and + isinstance(original, CallableType) and + len(override.arg_types) == len(original.arg_types) and + override.min_args == original.min_args): + # Give more detailed messages for the common case of both + # signatures having the same number of arguments and no + # overloads. + + # override might have its own generic function type + # variables. If an argument or return type of override + # does not have the correct subtyping relationship + # with the original type even after these variables + # are erased, then it is definitely an incompatiblity. + + override_ids = override.type_var_ids() + + def erase_override(t: Type) -> Type: + return erase_typevars(t, ids_to_erase=override_ids) + + for i in range(len(override.arg_types)): + if not is_subtype(original.arg_types[i], + erase_override(override.arg_types[i])): + self.msg.argument_incompatible_with_supertype( + i + 1, name, name_in_super, supertype, node) + emitted_msg = True + + if not is_subtype(erase_override(override.ret_type), + original.ret_type): + self.msg.return_type_incompatible_with_supertype( + name, name_in_super, supertype, node) + emitted_msg = True + + if not emitted_msg: + # Fall back to generic incompatibility message. + self.msg.signature_incompatible_with_supertype( + name, name_in_super, supertype, node) + + def visit_class_def(self, defn: ClassDef) -> Type: + """Type check a class definition.""" + typ = defn.info + self.errors.push_type(defn.name) + self.enter_partial_types() + old_binder = self.binder + self.binder = ConditionalTypeBinder() + with self.binder.frame_context(): + self.accept(defn.defs) + self.binder = old_binder + if not hasattr(defn, 'incompatible_baseclass'): + # Otherwise we've already found errors; more errors are not useful + self.check_multiple_inheritance(typ) + self.leave_partial_types() + self.errors.pop_type() + + def check_multiple_inheritance(self, typ: TypeInfo) -> None: + """Check for multiple inheritance related errors.""" + + if len(typ.bases) <= 1: + # No multiple inheritance. + return + # Verify that inherited attributes are compatible. + mro = typ.mro[1:] + for i, base in enumerate(mro): + for name in base.names: + for base2 in mro[i + 1:]: + # We only need to check compatibility of attributes from classes not + # in a subclass relationship. For subclasses, normal (single inheritance) + # checks suffice (these are implemented elsewhere). + if name in base2.names and base2 not in base.mro: + self.check_compatibility(name, base, base2, typ) + # Verify that base class layouts are compatible. + builtin_bases = [nearest_builtin_ancestor(base.type) + for base in typ.bases] + for base1 in builtin_bases: + for base2 in builtin_bases: + if not (base1 in base2.mro or base2 in base1.mro): + self.fail(messages.INSTANCE_LAYOUT_CONFLICT, typ) + + def check_compatibility(self, name: str, base1: TypeInfo, + base2: TypeInfo, ctx: Context) -> None: + """Check if attribute name in base1 is compatible with base2 in multiple inheritance. + + Assume base1 comes before base2 in the MRO, and that base1 and base2 don't have + a direct subclass relationship (i.e., the compatibility requirement only derives from + multiple inheritance). + """ + if name == '__init__': + # __init__ can be incompatible -- it's a special case. + return + first = base1[name] + second = base2[name] + first_type = first.type + if first_type is None and isinstance(first.node, FuncDef): + first_type = self.function_type(first.node) + second_type = second.type + if second_type is None and isinstance(second.node, FuncDef): + second_type = self.function_type(second.node) + # TODO: What if some classes are generic? + if (isinstance(first_type, FunctionLike) and + isinstance(second_type, FunctionLike)): + # Method override + first_sig = method_type(first_type) + second_sig = method_type(second_type) + ok = is_subtype(first_sig, second_sig) + elif first_type and second_type: + ok = is_equivalent(first_type, second_type) + else: + if first_type is None: + self.msg.cannot_determine_type_in_base(name, base1.name(), ctx) + if second_type is None: + self.msg.cannot_determine_type_in_base(name, base2.name(), ctx) + ok = True + if not ok: + self.msg.base_class_definitions_incompatible(name, base1, base2, + ctx) + + def visit_import_from(self, node: ImportFrom) -> Type: + self.check_import(node) + + def visit_import_all(self, node: ImportAll) -> Type: + self.check_import(node) + + def check_import(self, node: ImportBase) -> Type: + for assign in node.assignments: + lvalue = assign.lvalues[0] + lvalue_type, _, __ = self.check_lvalue(lvalue) + if lvalue_type is None: + # TODO: This is broken. + lvalue_type = AnyType() + message = '{} "{}"'.format(messages.INCOMPATIBLE_IMPORT_OF, + cast(NameExpr, assign.rvalue).name) + self.check_simple_assignment(lvalue_type, assign.rvalue, node, + msg=message, lvalue_name='local name', + rvalue_name='imported name') + + # + # Statements + # + + def visit_block(self, b: Block) -> Type: + if b.is_unreachable: + return None + for s in b.body: + self.accept(s) + if self.binder.breaking_out: + break + + def visit_assignment_stmt(self, s: AssignmentStmt) -> Type: + """Type check an assignment statement. + + Handle all kinds of assignment statements (simple, indexed, multiple). + """ + self.check_assignment(s.lvalues[-1], s.rvalue, s.type is None) + + if len(s.lvalues) > 1: + # Chained assignment (e.g. x = y = ...). + # Make sure that rvalue type will not be reinferred. + if s.rvalue not in self.type_map: + self.accept(s.rvalue) + rvalue = self.temp_node(self.type_map[s.rvalue], s) + for lv in s.lvalues[:-1]: + self.check_assignment(lv, rvalue, s.type is None) + + def check_assignment(self, lvalue: Node, rvalue: Node, infer_lvalue_type: bool = True) -> None: + """Type check a single assignment: lvalue = rvalue.""" + if isinstance(lvalue, TupleExpr) or isinstance(lvalue, ListExpr): + ltuple = cast(Union[TupleExpr, ListExpr], lvalue) + + self.check_assignment_to_multiple_lvalues(ltuple.items, rvalue, lvalue, + infer_lvalue_type) + else: + lvalue_type, index_lvalue, inferred = self.check_lvalue(lvalue) + if lvalue_type: + if isinstance(lvalue_type, PartialType) and lvalue_type.type is None: + # Try to infer a proper type for a variable with a partial None type. + rvalue_type = self.accept(rvalue) + if isinstance(rvalue_type, NoneTyp): + # This doesn't actually provide any additional information -- multiple + # None initializers preserve the partial None type. + return + if is_valid_inferred_type(rvalue_type): + var = lvalue_type.var + partial_types = self.find_partial_types(var) + if partial_types is not None: + if not self.current_node_deferred: + if experiments.STRICT_OPTIONAL: + var.type = UnionType.make_simplified_union( + [rvalue_type, NoneTyp()]) + else: + var.type = rvalue_type + else: + var.type = None + del partial_types[var] + # Try to infer a partial type. No need to check the return value, as + # an error will be reported elsewhere. + self.infer_partial_type(lvalue_type.var, lvalue, rvalue_type) + return + if (is_literal_none(rvalue) and + isinstance(lvalue, NameExpr) and + isinstance(lvalue.node, Var) and + lvalue.node.is_initialized_in_class): + # Allow None's to be assigned to class variables with non-Optional types. + rvalue_type = lvalue_type + else: + rvalue_type = self.check_simple_assignment(lvalue_type, rvalue, lvalue) + + if rvalue_type and infer_lvalue_type: + self.binder.assign_type(lvalue, + rvalue_type, + lvalue_type, + self.typing_mode_weak()) + elif index_lvalue: + self.check_indexed_assignment(index_lvalue, rvalue, rvalue) + + if inferred: + self.infer_variable_type(inferred, lvalue, self.accept(rvalue), + rvalue) + + def check_assignment_to_multiple_lvalues(self, lvalues: List[Node], rvalue: Node, + context: Context, + infer_lvalue_type: bool = True) -> None: + if isinstance(rvalue, TupleExpr) or isinstance(rvalue, ListExpr): + # Recursively go into Tuple or List expression rhs instead of + # using the type of rhs, because this allowed more fine grained + # control in cases like: a, b = [int, str] where rhs would get + # type List[object] + + rvalues = cast(Union[TupleExpr, ListExpr], rvalue).items + + if self.check_rvalue_count_in_assignment(lvalues, len(rvalues), context): + star_index = next((i for i, lv in enumerate(lvalues) if + isinstance(lv, StarExpr)), len(lvalues)) + + left_lvs = lvalues[:star_index] + star_lv = cast(StarExpr, + lvalues[star_index]) if star_index != len(lvalues) else None + right_lvs = lvalues[star_index + 1:] + + left_rvs, star_rvs, right_rvs = self.split_around_star( + rvalues, star_index, len(lvalues)) + + lr_pairs = list(zip(left_lvs, left_rvs)) + if star_lv: + rv_list = ListExpr(star_rvs) + rv_list.set_line(rvalue.get_line()) + lr_pairs.append((star_lv.expr, rv_list)) + lr_pairs.extend(zip(right_lvs, right_rvs)) + + for lv, rv in lr_pairs: + self.check_assignment(lv, rv, infer_lvalue_type) + else: + self.check_multi_assignment(lvalues, rvalue, context, infer_lvalue_type) + + def check_rvalue_count_in_assignment(self, lvalues: List[Node], rvalue_count: int, + context: Context) -> bool: + if any(isinstance(lvalue, StarExpr) for lvalue in lvalues): + if len(lvalues) - 1 > rvalue_count: + self.msg.wrong_number_values_to_unpack(rvalue_count, + len(lvalues) - 1, context) + return False + elif rvalue_count != len(lvalues): + self.msg.wrong_number_values_to_unpack(rvalue_count, + len(lvalues), context) + return False + return True + + def check_multi_assignment(self, lvalues: List[Node], + rvalue: Node, + context: Context, + infer_lvalue_type: bool = True, + msg: str = None) -> None: + """Check the assignment of one rvalue to a number of lvalues.""" + + # Infer the type of an ordinary rvalue expression. + rvalue_type = self.accept(rvalue) # TODO maybe elsewhere; redundant + undefined_rvalue = False + + if isinstance(rvalue_type, AnyType): + for lv in lvalues: + if isinstance(lv, StarExpr): + lv = lv.expr + self.check_assignment(lv, self.temp_node(AnyType(), context), infer_lvalue_type) + elif isinstance(rvalue_type, TupleType): + self.check_multi_assignment_from_tuple(lvalues, rvalue, rvalue_type, + context, undefined_rvalue, infer_lvalue_type) + else: + self.check_multi_assignment_from_iterable(lvalues, rvalue_type, + context, infer_lvalue_type) + + def check_multi_assignment_from_tuple(self, lvalues: List[Node], rvalue: Node, + rvalue_type: TupleType, context: Context, + undefined_rvalue: bool, + infer_lvalue_type: bool = True) -> None: + if self.check_rvalue_count_in_assignment(lvalues, len(rvalue_type.items), context): + star_index = next((i for i, lv in enumerate(lvalues) + if isinstance(lv, StarExpr)), len(lvalues)) + + left_lvs = lvalues[:star_index] + star_lv = cast(StarExpr, lvalues[star_index]) if star_index != len(lvalues) else None + right_lvs = lvalues[star_index + 1:] + + if not undefined_rvalue: + # Infer rvalue again, now in the correct type context. + lvalue_type = self.lvalue_type_for_inference(lvalues, rvalue_type) + rvalue_type = cast(TupleType, self.accept(rvalue, lvalue_type)) + + left_rv_types, star_rv_types, right_rv_types = self.split_around_star( + rvalue_type.items, star_index, len(lvalues)) + + for lv, rv_type in zip(left_lvs, left_rv_types): + self.check_assignment(lv, self.temp_node(rv_type, context), infer_lvalue_type) + if star_lv: + nodes = [self.temp_node(rv_type, context) for rv_type in star_rv_types] + list_expr = ListExpr(nodes) + list_expr.set_line(context.get_line()) + self.check_assignment(star_lv.expr, list_expr, infer_lvalue_type) + for lv, rv_type in zip(right_lvs, right_rv_types): + self.check_assignment(lv, self.temp_node(rv_type, context), infer_lvalue_type) + + def lvalue_type_for_inference(self, lvalues: List[Node], rvalue_type: TupleType) -> Type: + star_index = next((i for i, lv in enumerate(lvalues) + if isinstance(lv, StarExpr)), len(lvalues)) + left_lvs = lvalues[:star_index] + star_lv = cast(StarExpr, lvalues[star_index]) if star_index != len(lvalues) else None + right_lvs = lvalues[star_index + 1:] + left_rv_types, star_rv_types, right_rv_types = self.split_around_star( + rvalue_type.items, star_index, len(lvalues)) + + type_parameters = [] # type: List[Type] + + def append_types_for_inference(lvs: List[Node], rv_types: List[Type]) -> None: + for lv, rv_type in zip(lvs, rv_types): + sub_lvalue_type, index_expr, inferred = self.check_lvalue(lv) + if sub_lvalue_type: + type_parameters.append(sub_lvalue_type) + else: # index lvalue + # TODO Figure out more precise type context, probably + # based on the type signature of the _set method. + type_parameters.append(rv_type) + + append_types_for_inference(left_lvs, left_rv_types) + + if star_lv: + sub_lvalue_type, index_expr, inferred = self.check_lvalue(star_lv.expr) + if sub_lvalue_type: + type_parameters.extend([sub_lvalue_type] * len(star_rv_types)) + else: # index lvalue + # TODO Figure out more precise type context, probably + # based on the type signature of the _set method. + type_parameters.extend(star_rv_types) + + append_types_for_inference(right_lvs, right_rv_types) + + return TupleType(type_parameters, self.named_type('builtins.tuple')) + + def split_around_star(self, items: List[T], star_index: int, + length: int) -> Tuple[List[T], List[T], List[T]]: + """Splits a list of items in three to match another list of length 'length' + that contains a starred expression at 'star_index' in the following way: + + star_index = 2, length = 5 (i.e., [a,b,*,c,d]), items = [1,2,3,4,5,6,7] + returns in: ([1,2], [3,4,5], [6,7]) + """ + nr_right_of_star = length - star_index - 1 + right_index = nr_right_of_star if -nr_right_of_star != 0 else len(items) + left = items[:star_index] + star = items[star_index:right_index] + right = items[right_index:] + return (left, star, right) + + def type_is_iterable(self, type: Type) -> bool: + return (is_subtype(type, self.named_generic_type('typing.Iterable', + [AnyType()])) and + isinstance(type, Instance)) + + def check_multi_assignment_from_iterable(self, lvalues: List[Node], rvalue_type: Type, + context: Context, + infer_lvalue_type: bool = True) -> None: + if self.type_is_iterable(rvalue_type): + item_type = self.iterable_item_type(cast(Instance, rvalue_type)) + for lv in lvalues: + if isinstance(lv, StarExpr): + self.check_assignment(lv.expr, self.temp_node(rvalue_type, context), + infer_lvalue_type) + else: + self.check_assignment(lv, self.temp_node(item_type, context), + infer_lvalue_type) + else: + self.msg.type_not_iterable(rvalue_type, context) + + def check_lvalue(self, lvalue: Node) -> Tuple[Type, IndexExpr, Var]: + lvalue_type = None # type: Type + index_lvalue = None # type: IndexExpr + inferred = None # type: Var + + if self.is_definition(lvalue): + if isinstance(lvalue, NameExpr): + inferred = cast(Var, lvalue.node) + assert isinstance(inferred, Var) + else: + m = cast(MemberExpr, lvalue) + self.accept(m.expr) + inferred = m.def_var + elif isinstance(lvalue, IndexExpr): + index_lvalue = lvalue + elif isinstance(lvalue, MemberExpr): + lvalue_type = self.expr_checker.analyze_ordinary_member_access(lvalue, + True) + self.store_type(lvalue, lvalue_type) + elif isinstance(lvalue, NameExpr): + lvalue_type = self.expr_checker.analyze_ref_expr(lvalue, lvalue=True) + self.store_type(lvalue, lvalue_type) + elif isinstance(lvalue, TupleExpr) or isinstance(lvalue, ListExpr): + lv = cast(Union[TupleExpr, ListExpr], lvalue) + types = [self.check_lvalue(sub_expr)[0] for sub_expr in lv.items] + lvalue_type = TupleType(types, self.named_type('builtins.tuple')) + else: + lvalue_type = self.accept(lvalue) + + return lvalue_type, index_lvalue, inferred + + def is_definition(self, s: Node) -> bool: + if isinstance(s, NameExpr): + if s.is_def: + return True + # If the node type is not defined, this must the first assignment + # that we process => this is a definition, even though the semantic + # analyzer did not recognize this as such. This can arise in code + # that uses isinstance checks, if type checking of the primary + # definition is skipped due to an always False type check. + node = s.node + if isinstance(node, Var): + return node.type is None + elif isinstance(s, MemberExpr): + return s.is_def + return False + + def infer_variable_type(self, name: Var, lvalue: Node, + init_type: Type, context: Context) -> None: + """Infer the type of initialized variables from initializer type.""" + if self.typing_mode_weak(): + self.set_inferred_type(name, lvalue, AnyType()) + self.binder.assign_type(lvalue, init_type, self.binder.get_declaration(lvalue), True) + elif isinstance(init_type, Void): + self.check_not_void(init_type, context) + self.set_inference_error_fallback_type(name, lvalue, init_type, context) + elif isinstance(init_type, DeletedType): + self.msg.deleted_as_rvalue(init_type, context) + elif not is_valid_inferred_type(init_type): + # We cannot use the type of the initialization expression for full type + # inference (it's not specific enough), but we might be able to give + # partial type which will be made more specific later. A partial type + # gets generated in assignment like 'x = []' where item type is not known. + if not self.infer_partial_type(name, lvalue, init_type): + self.fail(messages.NEED_ANNOTATION_FOR_VAR, context) + self.set_inference_error_fallback_type(name, lvalue, init_type, context) + else: + # Infer type of the target. + + # Make the type more general (strip away function names etc.). + init_type = strip_type(init_type) + + self.set_inferred_type(name, lvalue, init_type) + + def infer_partial_type(self, name: Var, lvalue: Node, init_type: Type) -> bool: + if isinstance(init_type, (NoneTyp, UninhabitedType)): + partial_type = PartialType(None, name, [init_type]) + elif isinstance(init_type, Instance): + fullname = init_type.type.fullname() + if (isinstance(lvalue, NameExpr) and + (fullname == 'builtins.list' or + fullname == 'builtins.set' or + fullname == 'builtins.dict') and + all(isinstance(t, (NoneTyp, UninhabitedType)) for t in init_type.args)): + partial_type = PartialType(init_type.type, name, init_type.args) + else: + return False + else: + return False + self.set_inferred_type(name, lvalue, partial_type) + self.partial_types[-1][name] = lvalue + return True + + def set_inferred_type(self, var: Var, lvalue: Node, type: Type) -> None: + """Store inferred variable type. + + Store the type to both the variable node and the expression node that + refers to the variable (lvalue). If var is None, do nothing. + """ + if var and not self.current_node_deferred: + var.type = type + self.store_type(lvalue, type) + + def set_inference_error_fallback_type(self, var: Var, lvalue: Node, type: Type, + context: Context) -> None: + """If errors on context line are ignored, store dummy type for variable. + + If a program ignores error on type inference error, the variable should get some + inferred type so that if can used later on in the program. Example: + + x = [] # type: ignore + x.append(1) # Should be ok! + + We implement this here by giving x a valid type (Any). + """ + if context.get_line() in self.errors.ignored_lines[self.errors.file]: + self.set_inferred_type(var, lvalue, AnyType()) + + def narrow_type_from_binder(self, expr: Node, known_type: Type) -> Type: + if expr.literal >= LITERAL_TYPE: + restriction = self.binder.get(expr) + if restriction: + ans = meet_simple(known_type, restriction) + return ans + return known_type + + def check_simple_assignment(self, lvalue_type: Type, rvalue: Node, + context: Node, + msg: str = messages.INCOMPATIBLE_TYPES_IN_ASSIGNMENT, + lvalue_name: str = 'variable', + rvalue_name: str = 'expression') -> Type: + if self.is_stub and isinstance(rvalue, EllipsisExpr): + # '...' is always a valid initializer in a stub. + return AnyType() + else: + rvalue_type = self.accept(rvalue, lvalue_type) + if isinstance(rvalue_type, DeletedType): + self.msg.deleted_as_rvalue(rvalue_type, context) + if self.typing_mode_weak(): + return rvalue_type + if isinstance(lvalue_type, DeletedType): + self.msg.deleted_as_lvalue(lvalue_type, context) + else: + self.check_subtype(rvalue_type, lvalue_type, context, msg, + '{} has type'.format(rvalue_name), + '{} has type'.format(lvalue_name)) + return rvalue_type + + def check_indexed_assignment(self, lvalue: IndexExpr, + rvalue: Node, context: Context) -> None: + """Type check indexed assignment base[index] = rvalue. + + The lvalue argument is the base[index] expression. + """ + self.try_infer_partial_type_from_indexed_assignment(lvalue, rvalue) + basetype = self.accept(lvalue.base) + method_type = self.expr_checker.analyze_external_member_access( + '__setitem__', basetype, context) + lvalue.method_type = method_type + self.expr_checker.check_call(method_type, [lvalue.index, rvalue], + [nodes.ARG_POS, nodes.ARG_POS], + context) + + def try_infer_partial_type_from_indexed_assignment( + self, lvalue: IndexExpr, rvalue: Node) -> None: + # TODO: Should we share some of this with try_infer_partial_type? + if isinstance(lvalue.base, RefExpr) and isinstance(lvalue.base.node, Var): + var = lvalue.base.node + if isinstance(var.type, PartialType): + type_type = var.type.type + if type_type is None: + return # The partial type is None. + partial_types = self.find_partial_types(var) + if partial_types is None: + return + typename = type_type.fullname() + if typename == 'builtins.dict': + # TODO: Don't infer things twice. + key_type = self.accept(lvalue.index) + value_type = self.accept(rvalue) + full_key_type = UnionType.make_simplified_union( + [key_type, var.type.inner_types[0]]) + full_value_type = UnionType.make_simplified_union( + [value_type, var.type.inner_types[1]]) + if (is_valid_inferred_type(full_key_type) and + is_valid_inferred_type(full_value_type)): + if not self.current_node_deferred: + var.type = self.named_generic_type('builtins.dict', + [full_key_type, full_value_type]) + del partial_types[var] + + def visit_expression_stmt(self, s: ExpressionStmt) -> Type: + self.accept(s.expr) + + def visit_return_stmt(self, s: ReturnStmt) -> Type: + """Type check a return statement.""" + self.binder.breaking_out = True + if self.is_within_function(): + if self.function_stack[-1].is_generator: + return_type = self.get_generator_return_type(self.return_types[-1]) + else: + return_type = self.return_types[-1] + + if s.expr: + # Return with a value. + typ = self.accept(s.expr, return_type) + # Returning a value of type Any is always fine. + if isinstance(typ, AnyType): + return None + + if isinstance(return_type, Void): + # Lambdas are allowed to have a Void return. + # Functions returning a value of type None are allowed to have a Void return. + if isinstance(self.function_stack[-1], FuncExpr) or isinstance(typ, NoneTyp): + return None + self.fail(messages.NO_RETURN_VALUE_EXPECTED, s) + else: + self.check_subtype( + subtype_label='got', + subtype=typ, + supertype_label='expected', + supertype=return_type, + context=s, + msg=messages.INCOMPATIBLE_RETURN_VALUE_TYPE) + else: + # Empty returns are valid in Generators with Any typed returns. + if (self.function_stack[-1].is_generator and isinstance(return_type, AnyType)): + return None + + if isinstance(return_type, Void): + return None + + if isinstance(return_type, AnyType): + return None + + if self.typing_mode_full(): + self.fail(messages.RETURN_VALUE_EXPECTED, s) + + def wrap_generic_type(self, typ: Instance, rtyp: Instance, check_type: + str, context: Context) -> Type: + n_diff = self.count_nested_types(rtyp, check_type) - self.count_nested_types(typ, + check_type) + if n_diff == 1: + return self.named_generic_type(check_type, [typ]) + elif n_diff == 0 or n_diff > 1: + self.fail(messages.INCOMPATIBLE_RETURN_VALUE_TYPE + + ": expected {}, got {}".format(rtyp, typ), context) + return typ + return typ + + def count_nested_types(self, typ: Instance, check_type: str) -> int: + c = 0 + while is_subtype(typ, self.named_type(check_type)): + c += 1 + typ = map_instance_to_supertype(self.named_generic_type(check_type, typ.args), + self.lookup_typeinfo(check_type)) + if typ.args: + typ = cast(Instance, typ.args[0]) + else: + return c + return c + + def visit_if_stmt(self, s: IfStmt) -> Type: + """Type check an if statement.""" + breaking_out = True + # This frame records the knowledge from previous if/elif clauses not being taken. + with self.binder.frame_context(): + for e, b in zip(s.expr, s.body): + t = self.accept(e) + self.check_not_void(t, e) + if_map, else_map = find_isinstance_check( + e, self.type_map, + self.typing_mode_weak() + ) + if if_map is None: + # The condition is always false + # XXX should issue a warning? + pass + else: + # Only type check body if the if condition can be true. + with self.binder.frame_context(2): + if if_map: + for var, type in if_map.items(): + self.binder.push(var, type) + + self.accept(b) + breaking_out = breaking_out and self.binder.last_pop_breaking_out + + if else_map: + for var, type in else_map.items(): + self.binder.push(var, type) + if else_map is None: + # The condition is always true => remaining elif/else blocks + # can never be reached. + + # Might also want to issue a warning + # print("Warning: isinstance always true") + break + else: # Didn't break => can't prove one of the conditions is always true + with self.binder.frame_context(2): + if s.else_body: + self.accept(s.else_body) + breaking_out = breaking_out and self.binder.last_pop_breaking_out + if breaking_out: + self.binder.breaking_out = True + return None + + def visit_while_stmt(self, s: WhileStmt) -> Type: + """Type check a while statement.""" + self.accept_loop(IfStmt([s.expr], [s.body], None), s.else_body) + + def visit_operator_assignment_stmt(self, + s: OperatorAssignmentStmt) -> Type: + """Type check an operator assignment statement, e.g. x += 1.""" + lvalue_type = self.accept(s.lvalue) + method = infer_operator_assignment_method(lvalue_type, s.op) + rvalue_type, method_type = self.expr_checker.check_op( + method, lvalue_type, s.rvalue, s) + + if isinstance(s.lvalue, IndexExpr): + self.check_indexed_assignment(s.lvalue, s.rvalue, s.rvalue) + else: + if not is_subtype(rvalue_type, lvalue_type): + self.msg.incompatible_operator_assignment(s.op, s) + + def visit_assert_stmt(self, s: AssertStmt) -> Type: + self.accept(s.expr) + + # If this is asserting some isinstance check, bind that type in the following code + true_map, _ = find_isinstance_check( + s.expr, self.type_map, + self.typing_mode_weak() + ) + + if true_map: + for var, type in true_map.items(): + self.binder.push(var, type) + + def visit_raise_stmt(self, s: RaiseStmt) -> Type: + """Type check a raise statement.""" + self.binder.breaking_out = True + if s.expr: + self.type_check_raise(s.expr, s) + if s.from_expr: + self.type_check_raise(s.from_expr, s) + + def type_check_raise(self, e: Node, s: RaiseStmt) -> None: + typ = self.accept(e) + if isinstance(typ, FunctionLike): + if typ.is_type_obj(): + # Cases like "raise/from ExceptionClass". + typeinfo = typ.type_object() + base = self.lookup_typeinfo('builtins.BaseException') + if base in typeinfo.mro or typeinfo.fallback_to_any: + # Good! + return + # Else fall back to the checks below (which will fail). + if isinstance(typ, TupleType) and self.options.python_version[0] == 2: + # allow `raise type, value, traceback` + # https://docs.python.org/2/reference/simple_stmts.html#the-raise-statement + # TODO: Also check tuple item types. + if len(typ.items) in (2, 3): + return + if isinstance(typ, Instance) and typ.type.fallback_to_any: + # OK! + return + self.check_subtype(typ, + self.named_type('builtins.BaseException'), s, + messages.INVALID_EXCEPTION) + + def visit_try_stmt(self, s: TryStmt) -> Type: + """Type check a try statement.""" + # Our enclosing frame will get the result if the try/except falls through. + # This one gets all possible intermediate states + with self.binder.frame_context(): + if s.finally_body: + self.binder.try_frames.add(len(self.binder.frames) - 1) + breaking_out = self.visit_try_without_finally(s) + self.binder.try_frames.remove(len(self.binder.frames) - 1) + # First we check finally_body is type safe for all intermediate frames + self.accept(s.finally_body) + breaking_out = breaking_out or self.binder.breaking_out + else: + breaking_out = self.visit_try_without_finally(s) + + if not breaking_out and s.finally_body: + # Then we try again for the more restricted set of options that can fall through + self.accept(s.finally_body) + self.binder.breaking_out = breaking_out + return None + + def visit_try_without_finally(self, s: TryStmt) -> bool: + """Type check a try statement, ignoring the finally block. + + Return whether we are guaranteed to be breaking out. + Otherwise, it will place the results possible frames of + that don't break out into self.binder.frames[-2]. + """ + breaking_out = True + # This frame records the possible states that exceptions can leave variables in + # during the try: block + with self.binder.frame_context(): + with self.binder.frame_context(3): + self.binder.try_frames.add(len(self.binder.frames) - 2) + self.accept(s.body) + self.binder.try_frames.remove(len(self.binder.frames) - 2) + if s.else_body: + self.accept(s.else_body) + breaking_out = breaking_out and self.binder.last_pop_breaking_out + for i in range(len(s.handlers)): + with self.binder.frame_context(3): + if s.types[i]: + t = self.visit_except_handler_test(s.types[i]) + if s.vars[i]: + # To support local variables, we make this a definition line, + # causing assignment to set the variable's type. + s.vars[i].is_def = True + self.check_assignment(s.vars[i], self.temp_node(t, s.vars[i])) + self.accept(s.handlers[i]) + if s.vars[i]: + # Exception variables are deleted in python 3 but not python 2. + # But, since it's bad form in python 2 and the type checking + # wouldn't work very well, we delete it anyway. + + # Unfortunately, this doesn't let us detect usage before the + # try/except block. + if self.options.python_version[0] >= 3: + source = s.vars[i].name + else: + source = ('(exception variable "{}", which we do not accept outside' + 'except: blocks even in python 2)'.format(s.vars[i].name)) + var = cast(Var, s.vars[i].node) + var.type = DeletedType(source=source) + self.binder.cleanse(s.vars[i]) + breaking_out = breaking_out and self.binder.last_pop_breaking_out + return breaking_out + + def visit_except_handler_test(self, n: Node) -> Type: + """Type check an exception handler test clause.""" + type = self.accept(n) + + all_types = [] # type: List[Type] + test_types = type.items if isinstance(type, TupleType) else [type] + + for ttype in test_types: + if isinstance(ttype, AnyType): + all_types.append(ttype) + continue + + if not isinstance(ttype, FunctionLike): + self.fail(messages.INVALID_EXCEPTION_TYPE, n) + return AnyType() + + item = ttype.items()[0] + ret_type = item.ret_type + if not (is_subtype(ret_type, self.named_type('builtins.BaseException')) + and item.is_type_obj()): + self.fail(messages.INVALID_EXCEPTION_TYPE, n) + return AnyType() + + all_types.append(ret_type) + + return UnionType.make_simplified_union(all_types) + + def visit_for_stmt(self, s: ForStmt) -> Type: + """Type check a for statement.""" + item_type = self.analyze_iterable_item_type(s.expr) + self.analyze_index_variables(s.index, item_type, s) + self.accept_loop(s.body, s.else_body) + + def analyze_iterable_item_type(self, expr: Node) -> Type: + """Analyse iterable expression and return iterator item type.""" + iterable = self.accept(expr) + + self.check_not_void(iterable, expr) + if isinstance(iterable, TupleType): + if experiments.STRICT_OPTIONAL: + joined = UninhabitedType() # type: Type + else: + joined = NoneTyp() + for item in iterable.items: + joined = join_types(joined, item) + if isinstance(joined, ErrorType): + self.fail(messages.CANNOT_INFER_ITEM_TYPE, expr) + return AnyType() + return joined + else: + # Non-tuple iterable. + self.check_subtype(iterable, + self.named_generic_type('typing.Iterable', + [AnyType()]), + expr, messages.ITERABLE_EXPECTED) + + echk = self.expr_checker + method = echk.analyze_external_member_access('__iter__', iterable, + expr) + iterator = echk.check_call(method, [], [], expr)[0] + if self.options.python_version[0] >= 3: + nextmethod = '__next__' + else: + nextmethod = 'next' + method = echk.analyze_external_member_access(nextmethod, iterator, + expr) + return echk.check_call(method, [], [], expr)[0] + + def analyze_index_variables(self, index: Node, item_type: Type, + context: Context) -> None: + """Type check or infer for loop or list comprehension index vars.""" + self.check_assignment(index, self.temp_node(item_type, context)) + + def visit_del_stmt(self, s: DelStmt) -> Type: + if isinstance(s.expr, IndexExpr): + e = s.expr + m = MemberExpr(e.base, '__delitem__') + m.line = s.line + c = CallExpr(m, [e.index], [nodes.ARG_POS], [None]) + c.line = s.line + return c.accept(self) + else: + def flatten(t: Node) -> List[Node]: + """Flatten a nested sequence of tuples/lists into one list of nodes.""" + if isinstance(t, TupleExpr) or isinstance(t, ListExpr): + t = cast(Union[TupleExpr, ListExpr], t) + return [b for a in t.items for b in flatten(a)] + else: + return [t] + + s.expr.accept(self) + for elt in flatten(s.expr): + if isinstance(elt, NameExpr): + self.binder.assign_type(elt, + DeletedType(source=elt.name), + self.binder.get_declaration(elt), + self.typing_mode_weak()) + return None + + def visit_decorator(self, e: Decorator) -> Type: + for d in e.decorators: + if isinstance(d, RefExpr): + if d.fullname == 'typing.no_type_check': + e.var.type = AnyType() + e.var.is_ready = True + return NoneTyp() + + e.func.accept(self) + sig = self.function_type(e.func) # type: Type + # Process decorators from the inside out. + for i in range(len(e.decorators)): + n = len(e.decorators) - 1 - i + d = e.decorators[n] + if isinstance(d, NameExpr) and d.fullname == 'typing.overload': + self.fail('Single overload definition, multiple required', e) + continue + dec = self.accept(d) + temp = self.temp_node(sig) + sig, t2 = self.expr_checker.check_call(dec, [temp], + [nodes.ARG_POS], e) + sig = cast(FunctionLike, sig) + sig = set_callable_name(sig, e.func) + e.var.type = sig + e.var.is_ready = True + if e.func.is_property: + self.check_incompatible_property_override(e) + + def check_incompatible_property_override(self, e: Decorator) -> None: + if not e.var.is_settable_property and e.func.info is not None: + name = e.func.name() + for base in e.func.info.mro[1:]: + base_attr = base.names.get(name) + if not base_attr: + continue + if (isinstance(base_attr.node, OverloadedFuncDef) and + base_attr.node.is_property and + base_attr.node.items[0].var.is_settable_property): + self.fail(messages.READ_ONLY_PROPERTY_OVERRIDES_READ_WRITE, e) + + def visit_with_stmt(self, s: WithStmt) -> Type: + echk = self.expr_checker + for expr, target in zip(s.expr, s.target): + ctx = self.accept(expr) + enter = echk.analyze_external_member_access('__enter__', ctx, expr) + obj = echk.check_call(enter, [], [], expr)[0] + if target: + self.check_assignment(target, self.temp_node(obj, expr)) + exit = echk.analyze_external_member_access('__exit__', ctx, expr) + arg = self.temp_node(AnyType(), expr) + echk.check_call(exit, [arg] * 3, [nodes.ARG_POS] * 3, expr) + self.accept(s.body) + + def visit_print_stmt(self, s: PrintStmt) -> Type: + for arg in s.args: + self.accept(arg) + if s.target: + target_type = self.accept(s.target) + if not isinstance(target_type, NoneTyp): + # TODO: Also verify the type of 'write'. + self.expr_checker.analyze_external_member_access('write', target_type, s.target) + + # + # Expressions + # + + def visit_name_expr(self, e: NameExpr) -> Type: + return self.expr_checker.visit_name_expr(e) + + def visit_call_expr(self, e: CallExpr) -> Type: + return self.expr_checker.visit_call_expr(e) + + def visit_yield_from_expr(self, e: YieldFromExpr) -> Type: + return_type = self.return_types[-1] + subexpr_type = self.accept(e.expr, return_type) + iter_type = None # type: Type + + # Check that the expr is an instance of Iterable and get the type of the iterator produced + # by __iter__. + if isinstance(subexpr_type, AnyType): + iter_type = AnyType() + elif (isinstance(subexpr_type, Instance) and + is_subtype(subexpr_type, self.named_type('typing.Iterable'))): + iter_method_type = self.expr_checker.analyze_external_member_access( + '__iter__', + subexpr_type, + AnyType()) + + generic_generator_type = self.named_generic_type('typing.Generator', + [AnyType(), AnyType(), AnyType()]) + iter_type, _ = self.expr_checker.check_call(iter_method_type, [], [], + context=generic_generator_type) + else: + self.msg.yield_from_invalid_operand_type(subexpr_type, e) + iter_type = AnyType() + + # Check that the iterator's item type matches the type yielded by the Generator function + # containing this `yield from` expression. + expected_item_type = self.get_generator_yield_type(return_type) + actual_item_type = self.get_generator_yield_type(iter_type) + + self.check_subtype(actual_item_type, expected_item_type, e, + messages.INCOMPATIBLE_TYPES_IN_YIELD_FROM, + 'actual type', 'expected type') + + # Determine the type of the entire yield from expression. + if (isinstance(iter_type, Instance) and + iter_type.type.fullname() == 'typing.Generator'): + return self.get_generator_return_type(iter_type) + else: + # Non-Generators don't return anything from `yield from` expressions. + return Void() + + def visit_member_expr(self, e: MemberExpr) -> Type: + return self.expr_checker.visit_member_expr(e) + + def visit_break_stmt(self, s: BreakStmt) -> Type: + self.binder.breaking_out = True + self.binder.allow_jump(self.binder.loop_frames[-1] - 1) + return None + + def visit_continue_stmt(self, s: ContinueStmt) -> Type: + self.binder.breaking_out = True + self.binder.allow_jump(self.binder.loop_frames[-1]) + return None + + def visit_int_expr(self, e: IntExpr) -> Type: + return self.expr_checker.visit_int_expr(e) + + def visit_str_expr(self, e: StrExpr) -> Type: + return self.expr_checker.visit_str_expr(e) + + def visit_bytes_expr(self, e: BytesExpr) -> Type: + return self.expr_checker.visit_bytes_expr(e) + + def visit_unicode_expr(self, e: UnicodeExpr) -> Type: + return self.expr_checker.visit_unicode_expr(e) + + def visit_float_expr(self, e: FloatExpr) -> Type: + return self.expr_checker.visit_float_expr(e) + + def visit_complex_expr(self, e: ComplexExpr) -> Type: + return self.expr_checker.visit_complex_expr(e) + + def visit_ellipsis(self, e: EllipsisExpr) -> Type: + return self.expr_checker.visit_ellipsis(e) + + def visit_op_expr(self, e: OpExpr) -> Type: + return self.expr_checker.visit_op_expr(e) + + def visit_comparison_expr(self, e: ComparisonExpr) -> Type: + return self.expr_checker.visit_comparison_expr(e) + + def visit_unary_expr(self, e: UnaryExpr) -> Type: + return self.expr_checker.visit_unary_expr(e) + + def visit_index_expr(self, e: IndexExpr) -> Type: + return self.expr_checker.visit_index_expr(e) + + def visit_cast_expr(self, e: CastExpr) -> Type: + return self.expr_checker.visit_cast_expr(e) + + def visit_reveal_type_expr(self, e: RevealTypeExpr) -> Type: + return self.expr_checker.visit_reveal_type_expr(e) + + def visit_super_expr(self, e: SuperExpr) -> Type: + return self.expr_checker.visit_super_expr(e) + + def visit_type_application(self, e: TypeApplication) -> Type: + return self.expr_checker.visit_type_application(e) + + def visit_type_alias_expr(self, e: TypeAliasExpr) -> Type: + return self.expr_checker.visit_type_alias_expr(e) + + def visit_type_var_expr(self, e: TypeVarExpr) -> Type: + # TODO: Perhaps return a special type used for type variables only? + return AnyType() + + def visit_namedtuple_expr(self, e: NamedTupleExpr) -> Type: + # TODO: Perhaps return a type object type? + return AnyType() + + def visit_list_expr(self, e: ListExpr) -> Type: + return self.expr_checker.visit_list_expr(e) + + def visit_set_expr(self, e: SetExpr) -> Type: + return self.expr_checker.visit_set_expr(e) + + def visit_tuple_expr(self, e: TupleExpr) -> Type: + return self.expr_checker.visit_tuple_expr(e) + + def visit_dict_expr(self, e: DictExpr) -> Type: + return self.expr_checker.visit_dict_expr(e) + + def visit_slice_expr(self, e: SliceExpr) -> Type: + return self.expr_checker.visit_slice_expr(e) + + def visit_func_expr(self, e: FuncExpr) -> Type: + return self.expr_checker.visit_func_expr(e) + + def visit_list_comprehension(self, e: ListComprehension) -> Type: + return self.expr_checker.visit_list_comprehension(e) + + def visit_set_comprehension(self, e: SetComprehension) -> Type: + return self.expr_checker.visit_set_comprehension(e) + + def visit_generator_expr(self, e: GeneratorExpr) -> Type: + return self.expr_checker.visit_generator_expr(e) + + def visit_dictionary_comprehension(self, e: DictionaryComprehension) -> Type: + return self.expr_checker.visit_dictionary_comprehension(e) + + def visit_temp_node(self, e: TempNode) -> Type: + return e.type + + def visit_conditional_expr(self, e: ConditionalExpr) -> Type: + return self.expr_checker.visit_conditional_expr(e) + + def visit_backquote_expr(self, e: BackquoteExpr) -> Type: + return self.expr_checker.visit_backquote_expr(e) + + def visit_yield_expr(self, e: YieldExpr) -> Type: + return_type = self.return_types[-1] + expected_item_type = self.get_generator_yield_type(return_type) + if e.expr is None: + if (not (isinstance(expected_item_type, Void) or + isinstance(expected_item_type, AnyType)) + and self.typing_mode_full()): + self.fail(messages.YIELD_VALUE_EXPECTED, e) + else: + actual_item_type = self.accept(e.expr, expected_item_type) + self.check_subtype(actual_item_type, expected_item_type, e, + messages.INCOMPATIBLE_TYPES_IN_YIELD, + 'actual type', 'expected type') + return self.get_generator_receive_type(return_type) + + # + # Helpers + # + + def check_subtype(self, subtype: Type, supertype: Type, context: Context, + msg: str = messages.INCOMPATIBLE_TYPES, + subtype_label: str = None, + supertype_label: str = None) -> None: + """Generate an error if the subtype is not compatible with + supertype.""" + if not is_subtype(subtype, supertype): + if isinstance(subtype, Void): + self.msg.does_not_return_value(subtype, context) + else: + extra_info = [] # type: List[str] + if subtype_label is not None or supertype_label is not None: + subtype_str, supertype_str = self.msg.format_distinctly(subtype, supertype) + if subtype_label is not None: + extra_info.append(subtype_label + ' ' + subtype_str) + if supertype_label is not None: + extra_info.append(supertype_label + ' ' + supertype_str) + if extra_info: + msg += ' (' + ', '.join(extra_info) + ')' + self.fail(msg, context) + + def named_type(self, name: str) -> Instance: + """Return an instance type with type given by the name and no + type arguments. For example, named_type('builtins.object') + produces the object type. + """ + # Assume that the name refers to a type. + sym = self.lookup_qualified(name) + return Instance(cast(TypeInfo, sym.node), []) + + def named_generic_type(self, name: str, args: List[Type]) -> Instance: + """Return an instance with the given name and type arguments. + + Assume that the number of arguments is correct. Assume that + the name refers to a compatible generic type. + """ + return Instance(self.lookup_typeinfo(name), args) + + def lookup_typeinfo(self, fullname: str) -> TypeInfo: + # Assume that the name refers to a class. + sym = self.lookup_qualified(fullname) + return cast(TypeInfo, sym.node) + + def type_type(self) -> Instance: + """Return instance type 'type'.""" + return self.named_type('builtins.type') + + def object_type(self) -> Instance: + """Return instance type 'object'.""" + return self.named_type('builtins.object') + + def bool_type(self) -> Instance: + """Return instance type 'bool'.""" + return self.named_type('builtins.bool') + + def str_type(self) -> Instance: + """Return instance type 'str'.""" + return self.named_type('builtins.str') + + def check_type_equivalency(self, t1: Type, t2: Type, node: Context, + msg: str = messages.INCOMPATIBLE_TYPES) -> None: + """Generate an error if the types are not equivalent. The + dynamic type is equivalent with all types. + """ + if not is_equivalent(t1, t2): + self.fail(msg, node) + + def store_type(self, node: Node, typ: Type) -> None: + """Store the type of a node in the type map.""" + self.type_map[node] = typ + + def typing_mode_none(self) -> bool: + if self.is_dynamic_function() and not self.options.check_untyped_defs: + return not self.weak_opts + elif self.function_stack: + return False + else: + return False + + def typing_mode_weak(self) -> bool: + if self.is_dynamic_function() and not self.options.check_untyped_defs: + return bool(self.weak_opts) + elif self.function_stack: + return False + else: + return 'global' in self.weak_opts + + def typing_mode_full(self) -> bool: + if self.is_dynamic_function() and not self.options.check_untyped_defs: + return False + elif self.function_stack: + return True + else: + return 'global' not in self.weak_opts + + def is_dynamic_function(self) -> bool: + return len(self.dynamic_funcs) > 0 and self.dynamic_funcs[-1] + + def lookup(self, name: str, kind: int) -> SymbolTableNode: + """Look up a definition from the symbol table with the given name. + TODO remove kind argument + """ + if name in self.globals: + return self.globals[name] + else: + b = self.globals.get('__builtins__', None) + if b: + table = cast(MypyFile, b.node).names + if name in table: + return table[name] + raise KeyError('Failed lookup: {}'.format(name)) + + def lookup_qualified(self, name: str) -> SymbolTableNode: + if '.' not in name: + return self.lookup(name, GDEF) # FIX kind + else: + parts = name.split('.') + n = self.modules[parts[0]] + for i in range(1, len(parts) - 1): + n = cast(MypyFile, n.names.get(parts[i], None).node) + return n.names[parts[-1]] + + def enter_partial_types(self) -> None: + """Push a new scope for collecting partial types.""" + self.partial_types.append({}) + + def leave_partial_types(self) -> None: + """Pop partial type scope. + + Also report errors for variables which still have partial + types, i.e. we couldn't infer a complete type. + """ + partial_types = self.partial_types.pop() + if not self.current_node_deferred: + for var, context in partial_types.items(): + if experiments.STRICT_OPTIONAL and cast(PartialType, var.type).type is None: + # None partial type: assume variable is intended to have type None + var.type = NoneTyp() + else: + self.msg.fail(messages.NEED_ANNOTATION_FOR_VAR, context) + var.type = AnyType() + + def find_partial_types(self, var: Var) -> Optional[Dict[Var, Context]]: + for partial_types in reversed(self.partial_types): + if var in partial_types: + return partial_types + return None + + def is_within_function(self) -> bool: + """Are we currently type checking within a function? + + I.e. not at class body or at the top level. + """ + return self.return_types != [] + + def check_not_void(self, typ: Type, context: Context) -> None: + """Generate an error if the type is Void.""" + if isinstance(typ, Void): + self.msg.does_not_return_value(typ, context) + + def temp_node(self, t: Type, context: Context = None) -> Node: + """Create a temporary node with the given, fixed type.""" + temp = TempNode(t) + if context: + temp.set_line(context.get_line()) + return temp + + def fail(self, msg: str, context: Context) -> None: + """Produce an error message.""" + self.msg.fail(msg, context) + + def iterable_item_type(self, instance: Instance) -> Type: + iterable = map_instance_to_supertype( + instance, + self.lookup_typeinfo('typing.Iterable')) + return iterable.args[0] + + def function_type(self, func: FuncBase) -> FunctionLike: + return function_type(func, self.named_type('builtins.function')) + + def method_type(self, func: FuncBase) -> FunctionLike: + return method_type_with_fallback(func, self.named_type('builtins.function')) + + +# Data structure returned by find_isinstance_check representing +# information learned from the truth or falsehood of a condition. The +# dict maps nodes representing expressions like 'a[0].x' to their +# refined types under the assumption that the condition has a +# particular truth value. A value of None means that the condition can +# never have that truth value. + +# NB: The keys of this dict are nodes in the original source program, +# which are compared by reference equality--effectively, being *the +# same* expression of the program, not just two identical expressions +# (such as two references to the same variable). TODO: it would +# probably be better to have the dict keyed by the nodes' literal_hash +# field instead. + +# NB: This should be `TypeMap = Optional[Dict[Node, Type]]`! +# But see https://github.com/python/mypy/issues/1637 +TypeMap = Dict[Node, Type] + + +def conditional_type_map(expr: Node, + current_type: Optional[Type], + proposed_type: Optional[Type], + *, + weak: bool = False + ) -> Tuple[TypeMap, TypeMap]: + """Takes in an expression, the current type of the expression, and a + proposed type of that expression. + + Returns a 2-tuple: The first element is a map from the expression to + the proposed type, if the expression can be the proposed type. The + second element is a map from the expression to the type it would hold + if it was not the proposed type, if any.""" + if proposed_type: + if current_type: + if is_proper_subtype(current_type, proposed_type): + return {expr: proposed_type}, None + elif not is_overlapping_types(current_type, proposed_type): + return None, {expr: current_type} + else: + remaining_type = restrict_subtype_away(current_type, proposed_type) + return {expr: proposed_type}, {expr: remaining_type} + else: + return {expr: proposed_type}, {} + else: + # An isinstance check, but we don't understand the type + if weak: + return {expr: AnyType()}, {expr: current_type} + else: + return {}, {} + + +def is_literal_none(n: Node) -> bool: + return isinstance(n, NameExpr) and n.fullname == 'builtins.None' + + +def and_conditional_maps(m1: TypeMap, m2: TypeMap) -> TypeMap: + """Calculate what information we can learn from the truth of (e1 and e2) + in terms of the information that we can learn from the truth of e1 and + the truth of e2. + """ + + if m1 is None or m2 is None: + # One of the conditions can never be true. + return None + # Both conditions can be true; combine the information. Anything + # we learn from either conditions's truth is valid. If the same + # expression's type is refined by both conditions, we somewhat + # arbitrarily give precedence to m2. (In the future, we could use + # an intersection type.) + result = m2.copy() + m2_keys = set(n2.literal_hash for n2 in m2) + for n1 in m1: + if n1.literal_hash not in m2_keys: + result[n1] = m1[n1] + return result + + +def or_conditional_maps(m1: TypeMap, m2: TypeMap) -> TypeMap: + """Calculate what information we can learn from the truth of (e1 or e2) + in terms of the information that we can learn from the truth of e1 and + the truth of e2. + """ + + if m1 is None: + return m2 + if m2 is None: + return m1 + # Both conditions can be true. Combine information about + # expressions whose type is refined by both conditions. (We do not + # learn anything about expressions whose type is refined by only + # one condition.) + result = {} + for n1 in m1: + for n2 in m2: + if n1.literal_hash == n2.literal_hash: + result[n1] = UnionType.make_simplified_union([m1[n1], m2[n2]]) + return result + + +def find_isinstance_check(node: Node, + type_map: Dict[Node, Type], + weak: bool=False + ) -> Tuple[TypeMap, TypeMap]: + """Find any isinstance checks (within a chain of ands). Includes + implicit and explicit checks for None. + + Return value is a map of variables to their types if the condition + is true and a map of variables to their types if the condition is false. + + If either of the values in the tuple is None, then that particular + branch can never occur. + + Guaranteed to not return None, None. (But may return {}, {}) + """ + if isinstance(node, CallExpr): + if refers_to_fullname(node.callee, 'builtins.isinstance'): + expr = node.args[0] + if expr.literal == LITERAL_TYPE: + vartype = type_map[expr] + type = get_isinstance_type(node.args[1], type_map) + return conditional_type_map(expr, vartype, type, weak=weak) + elif (isinstance(node, ComparisonExpr) and any(is_literal_none(n) for n in node.operands) and + experiments.STRICT_OPTIONAL): + # Check for `x is None` and `x is not None`. + is_not = node.operators == ['is not'] + if is_not or node.operators == ['is']: + if_vars = {} # type: Dict[Node, Type] + else_vars = {} # type: Dict[Node, Type] + for expr in node.operands: + if expr.literal == LITERAL_TYPE and not is_literal_none(expr) and expr in type_map: + # This should only be true at most once: there should be + # two elements in node.operands, and at least one of them + # should represent a None. + vartype = type_map[expr] + if_vars, else_vars = conditional_type_map(expr, vartype, NoneTyp(), weak=weak) + break + + if is_not: + if_vars, else_vars = else_vars, if_vars + return if_vars, else_vars + elif isinstance(node, RefExpr) and experiments.STRICT_OPTIONAL: + # The type could be falsy, so we can't deduce anything new about the else branch + vartype = type_map[node] + _, if_vars = conditional_type_map(node, vartype, NoneTyp(), weak=weak) + return if_vars, {} + elif isinstance(node, OpExpr) and node.op == 'and': + left_if_vars, left_else_vars = find_isinstance_check( + node.left, + type_map, + weak, + ) + + right_if_vars, right_else_vars = find_isinstance_check( + node.right, + type_map, + weak, + ) + + # (e1 and e2) is true if both e1 and e2 are true, + # and false if at least one of e1 and e2 is false. + return (and_conditional_maps(left_if_vars, right_if_vars), + or_conditional_maps(left_else_vars, right_else_vars)) + elif isinstance(node, OpExpr) and node.op == 'or': + left_if_vars, left_else_vars = find_isinstance_check( + node.left, + type_map, + weak, + ) + + right_if_vars, right_else_vars = find_isinstance_check( + node.right, + type_map, + weak, + ) + + # (e1 or e2) is true if at least one of e1 or e2 is true, + # and false if both e1 and e2 are false. + return (or_conditional_maps(left_if_vars, right_if_vars), + and_conditional_maps(left_else_vars, right_else_vars)) + elif isinstance(node, UnaryExpr) and node.op == 'not': + left, right = find_isinstance_check(node.expr, type_map, weak) + return right, left + + # Not a supported isinstance check + return {}, {} + + +def get_isinstance_type(node: Node, type_map: Dict[Node, Type]) -> Type: + type = type_map[node] + + if isinstance(type, TupleType): + all_types = type.items + else: + all_types = [type] + + types = [] # type: List[Type] + + for type in all_types: + if isinstance(type, FunctionLike): + if type.is_type_obj(): + # Type variables may be present -- erase them, which is the best + # we can do (outside disallowing them here). + type = erase_typevars(type.items()[0].ret_type) + + types.append(type) + + if len(types) == 0: + return None + elif len(types) == 1: + return types[0] + else: + return UnionType(types) + + +def expand_node(defn: Node, map: Dict[TypeVarId, Type]) -> Node: + visitor = TypeTransformVisitor(map) + return defn.accept(visitor) + + +def expand_func(defn: FuncItem, map: Dict[TypeVarId, Type]) -> FuncItem: + return cast(FuncItem, expand_node(defn, map)) + + +class TypeTransformVisitor(TransformVisitor): + def __init__(self, map: Dict[TypeVarId, Type]) -> None: + super().__init__() + self.map = map + + def type(self, type: Type) -> Type: + return expand_type(type, self.map) + + +def is_unsafe_overlapping_signatures(signature: Type, other: Type) -> bool: + """Check if two signatures may be unsafely overlapping. + + Two signatures s and t are overlapping if both can be valid for the same + statically typed values and the return types are incompatible. + + Assume calls are first checked against 'signature', then against 'other'. + Thus if 'signature' is more general than 'other', there is no unsafe + overlapping. + + TODO If argument types vary covariantly, the return type may vary + covariantly as well. + """ + if isinstance(signature, CallableType): + if isinstance(other, CallableType): + # TODO varargs + # TODO keyword args + # TODO erasure + # TODO allow to vary covariantly + # Check if the argument counts are overlapping. + min_args = max(signature.min_args, other.min_args) + max_args = min(len(signature.arg_types), len(other.arg_types)) + if min_args > max_args: + # Argument counts are not overlapping. + return False + # Signatures are overlapping iff if they are overlapping for the + # smallest common argument count. + for i in range(min_args): + t1 = signature.arg_types[i] + t2 = other.arg_types[i] + if not is_overlapping_types(t1, t2): + return False + # All arguments types for the smallest common argument count are + # overlapping => the signature is overlapping. The overlapping is + # safe if the return types are identical. + if is_same_type(signature.ret_type, other.ret_type): + return False + # If the first signature has more general argument types, the + # latter will never be called + if is_more_general_arg_prefix(signature, other): + return False + return not is_more_precise_signature(signature, other) + return True + + +def is_more_general_arg_prefix(t: FunctionLike, s: FunctionLike) -> bool: + """Does t have wider arguments than s?""" + # TODO should an overload with additional items be allowed to be more + # general than one with fewer items (or just one item)? + # TODO check argument kinds + if isinstance(t, CallableType): + if isinstance(s, CallableType): + return all(is_proper_subtype(args, argt) + for argt, args in zip(t.arg_types, s.arg_types)) + elif isinstance(t, FunctionLike): + if isinstance(s, FunctionLike): + if len(t.items()) == len(s.items()): + return all(is_same_arg_prefix(items, itemt) + for items, itemt in zip(t.items(), s.items())) + return False + + +def is_same_arg_prefix(t: CallableType, s: CallableType) -> bool: + # TODO check argument kinds + return all(is_same_type(argt, args) + for argt, args in zip(t.arg_types, s.arg_types)) + + +def is_more_precise_signature(t: CallableType, s: CallableType) -> bool: + """Is t more precise than s? + + A signature t is more precise than s if all argument types and the return + type of t are more precise than the corresponding types in s. + + Assume that the argument kinds and names are compatible, and that the + argument counts are overlapping. + """ + # TODO generic function types + # Only consider the common prefix of argument types. + for argt, args in zip(t.arg_types, s.arg_types): + if not is_more_precise(argt, args): + return False + return is_more_precise(t.ret_type, s.ret_type) + + +def infer_operator_assignment_method(type: Type, operator: str) -> str: + """Return the method used for operator assignment for given value type. + + For example, if operator is '+', return '__iadd__' or '__add__' depending + on which method is supported by the type. + """ + method = nodes.op_methods[operator] + if isinstance(type, Instance): + if operator in nodes.ops_with_inplace_method: + inplace = '__i' + method[2:] + if type.type.has_readable_member(inplace): + method = inplace + return method + + +def is_valid_inferred_type(typ: Type) -> bool: + """Is an inferred type valid? + + Examples of invalid types include the None type or a type with a None component. + """ + if is_same_type(typ, NoneTyp()): + # With strict Optional checking, we *may* eventually infer NoneTyp, but + # we only do that if we can't infer a specific Optional type. This + # resolution happens in leave_partial_types when we pop a partial types + # scope. + return False + if is_same_type(typ, UninhabitedType()): + return False + elif isinstance(typ, Instance): + for arg in typ.args: + if not is_valid_inferred_type(arg): + return False + elif isinstance(typ, TupleType): + for item in typ.items: + if not is_valid_inferred_type(item): + return False + return True diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 2fee197d9c86d..97a50142901eb 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1,1822 +1,1822 @@ -"""Expression type checker. This file is conceptually part of TypeChecker.""" - -from typing import cast, Dict, List, Tuple, Callable, Union, Optional - -from mypy.types import ( - Type, AnyType, CallableType, Overloaded, NoneTyp, Void, TypeVarDef, - TupleType, Instance, TypeVarId, TypeVarType, ErasedType, UnionType, - PartialType, DeletedType, UnboundType, UninhabitedType, TypeType -) -from mypy.nodes import ( - NameExpr, RefExpr, Var, FuncDef, OverloadedFuncDef, TypeInfo, CallExpr, - Node, MemberExpr, IntExpr, StrExpr, BytesExpr, UnicodeExpr, FloatExpr, - OpExpr, UnaryExpr, IndexExpr, CastExpr, RevealTypeExpr, TypeApplication, ListExpr, - TupleExpr, DictExpr, FuncExpr, SuperExpr, SliceExpr, Context, - ListComprehension, GeneratorExpr, SetExpr, MypyFile, Decorator, - ConditionalExpr, ComparisonExpr, TempNode, SetComprehension, - DictionaryComprehension, ComplexExpr, EllipsisExpr, - TypeAliasExpr, BackquoteExpr, ARG_POS, ARG_NAMED, ARG_STAR2 -) -from mypy.nodes import function_type -from mypy import nodes -import mypy.checker -from mypy import types -from mypy.sametypes import is_same_type -from mypy.erasetype import replace_meta_vars -from mypy.messages import MessageBuilder -from mypy import messages -from mypy.infer import infer_type_arguments, infer_function_type_arguments -from mypy import join -from mypy.subtypes import is_subtype, is_equivalent -from mypy import applytype -from mypy import erasetype -from mypy.checkmember import analyze_member_access, type_object_type -from mypy.semanal import self_type -from mypy.constraints import get_actual_type -from mypy.checkstrformat import StringFormatterChecker -from mypy.expandtype import expand_type -import mypy.checkexpr - -from mypy import experiments - -# Type of callback user for checking individual function arguments. See -# check_args() below for details. -ArgChecker = Callable[[Type, Type, int, Type, int, int, CallableType, Context, MessageBuilder], - None] - - -class Finished(Exception): - """Raised if we can terminate overload argument check early (no match).""" - - -class ExpressionChecker: - """Expression type checker. - - This class works closely together with checker.TypeChecker. - """ - - # Some services are provided by a TypeChecker instance. - chk = None # type: mypy.checker.TypeChecker - # This is shared with TypeChecker, but stored also here for convenience. - msg = None # type: MessageBuilder - - strfrm_checker = None # type: mypy.checkstrformat.StringFormatterChecker - - def __init__(self, - chk: 'mypy.checker.TypeChecker', - msg: MessageBuilder) -> None: - """Construct an expression type checker.""" - self.chk = chk - self.msg = msg - self.strfrm_checker = mypy.checkexpr.StringFormatterChecker(self, self.chk, self.msg) - - def visit_name_expr(self, e: NameExpr) -> Type: - """Type check a name expression. - - It can be of any kind: local, member or global. - """ - result = self.analyze_ref_expr(e) - return self.chk.narrow_type_from_binder(e, result) - - def analyze_ref_expr(self, e: RefExpr, lvalue: bool = False) -> Type: - result = None # type: Type - node = e.node - if isinstance(node, Var): - # Variable reference. - result = self.analyze_var_ref(node, e) - if isinstance(result, PartialType): - if result.type is None: - # 'None' partial type. It has a well-defined type. In an lvalue context - # we want to preserve the knowledge of it being a partial type. - if not lvalue: - result = NoneTyp() - else: - partial_types = self.chk.find_partial_types(node) - if partial_types is not None and not self.chk.current_node_deferred: - context = partial_types[node] - self.msg.fail(messages.NEED_ANNOTATION_FOR_VAR, context) - result = AnyType() - elif isinstance(node, FuncDef): - # Reference to a global function. - result = function_type(node, self.named_type('builtins.function')) - elif isinstance(node, OverloadedFuncDef): - result = node.type - elif isinstance(node, TypeInfo): - # Reference to a type object. - result = type_object_type(node, self.named_type) - elif isinstance(node, MypyFile): - # Reference to a module object. - result = self.named_type('builtins.module') - elif isinstance(node, Decorator): - result = self.analyze_var_ref(node.var, e) - else: - # Unknown reference; use any type implicitly to avoid - # generating extra type errors. - result = AnyType() - return result - - def analyze_var_ref(self, var: Var, context: Context) -> Type: - if not var.type: - if not var.is_ready and self.chk.typing_mode_full(): - self.chk.handle_cannot_determine_type(var.name(), context) - # Implicit 'Any' type. - return AnyType() - else: - # Look up local type of variable with type (inferred or explicit). - val = self.chk.binder.get(var) - if val is None: - return var.type - else: - return val - - def visit_call_expr(self, e: CallExpr) -> Type: - """Type check a call expression.""" - if e.analyzed: - # It's really a special form that only looks like a call. - return self.accept(e.analyzed, self.chk.type_context[-1]) - self.try_infer_partial_type(e) - callee_type = self.accept(e.callee) - if (self.chk.options.disallow_untyped_calls and - self.chk.typing_mode_full() and - isinstance(callee_type, CallableType) - and callee_type.implicit): - return self.msg.untyped_function_call(callee_type, e) - return self.check_call_expr_with_callee_type(callee_type, e) - - # Types and methods that can be used to infer partial types. - item_args = {'builtins.list': ['append'], - 'builtins.set': ['add', 'discard'], - } - container_args = {'builtins.list': {'extend': ['builtins.list']}, - 'builtins.dict': {'update': ['builtins.dict']}, - 'builtins.set': {'update': ['builtins.set', 'builtins.list']}, - } - - def try_infer_partial_type(self, e: CallExpr) -> None: - if isinstance(e.callee, MemberExpr) and isinstance(e.callee.expr, RefExpr): - var = cast(Var, e.callee.expr.node) - partial_types = self.chk.find_partial_types(var) - if partial_types is not None and not self.chk.current_node_deferred: - partial_type = cast(PartialType, var.type) - if partial_type is None or partial_type.type is None: - # A partial None type -> can't infer anything. - return - typename = partial_type.type.fullname() - methodname = e.callee.name - # Sometimes we can infer a full type for a partial List, Dict or Set type. - # TODO: Don't infer argument expression twice. - if (typename in self.item_args and methodname in self.item_args[typename] - and e.arg_kinds == [ARG_POS]): - item_type = self.accept(e.args[0]) - full_item_type = UnionType.make_simplified_union( - [item_type, partial_type.inner_types[0]]) - if mypy.checker.is_valid_inferred_type(full_item_type): - var.type = self.chk.named_generic_type(typename, [full_item_type]) - del partial_types[var] - elif (typename in self.container_args - and methodname in self.container_args[typename] - and e.arg_kinds == [ARG_POS]): - arg_type = self.accept(e.args[0]) - if isinstance(arg_type, Instance): - arg_typename = arg_type.type.fullname() - if arg_typename in self.container_args[typename][methodname]: - full_item_types = [ - UnionType.make_simplified_union([item_type, prev_type]) - for item_type, prev_type - in zip(arg_type.args, partial_type.inner_types) - ] - if all(mypy.checker.is_valid_inferred_type(item_type) - for item_type in full_item_types): - var.type = self.chk.named_generic_type(typename, - list(full_item_types)) - del partial_types[var] - - def check_call_expr_with_callee_type(self, callee_type: Type, - e: CallExpr) -> Type: - """Type check call expression. - - The given callee type overrides the type of the callee - expression. - """ - return self.check_call(callee_type, e.args, e.arg_kinds, e, - e.arg_names, callable_node=e.callee)[0] - - def check_call(self, callee: Type, args: List[Node], - arg_kinds: List[int], context: Context, - arg_names: List[str] = None, - callable_node: Node = None, - arg_messages: MessageBuilder = None) -> Tuple[Type, Type]: - """Type check a call. - - Also infer type arguments if the callee is a generic function. - - Return (result type, inferred callee type). - - Arguments: - callee: type of the called value - args: actual argument expressions - arg_kinds: contains nodes.ARG_* constant for each argument in args - describing whether the argument is positional, *arg, etc. - arg_names: names of arguments (optional) - callable_node: associate the inferred callable type to this node, - if specified - arg_messages: TODO - """ - arg_messages = arg_messages or self.msg - if isinstance(callee, CallableType): - if callee.is_concrete_type_obj() and callee.type_object().is_abstract: - type = callee.type_object() - self.msg.cannot_instantiate_abstract_class( - callee.type_object().name(), type.abstract_attributes, - context) - - formal_to_actual = map_actuals_to_formals( - arg_kinds, arg_names, - callee.arg_kinds, callee.arg_names, - lambda i: self.accept(args[i])) - - if callee.is_generic(): - callee = freshen_generic_callable(callee) - callee = self.infer_function_type_arguments_using_context( - callee, context) - callee = self.infer_function_type_arguments( - callee, args, arg_kinds, formal_to_actual, context) - - arg_types = self.infer_arg_types_in_context2( - callee, args, arg_kinds, formal_to_actual) - - self.check_argument_count(callee, arg_types, arg_kinds, - arg_names, formal_to_actual, context, self.msg) - - self.check_argument_types(arg_types, arg_kinds, callee, - formal_to_actual, context, - messages=arg_messages) - - if (callee.is_type_obj() and (len(arg_types) == 1) - and is_equivalent(callee.ret_type, self.named_type('builtins.type'))): - callee = callee.copy_modified(ret_type=TypeType(arg_types[0])) - - if callable_node: - # Store the inferred callable type. - self.chk.store_type(callable_node, callee) - return callee.ret_type, callee - elif isinstance(callee, Overloaded): - # Type check arguments in empty context. They will be checked again - # later in a context derived from the signature; these types are - # only used to pick a signature variant. - self.msg.disable_errors() - arg_types = self.infer_arg_types_in_context(None, args) - self.msg.enable_errors() - - target = self.overload_call_target(arg_types, arg_kinds, arg_names, - callee, context, - messages=arg_messages) - return self.check_call(target, args, arg_kinds, context, arg_names, - arg_messages=arg_messages) - elif isinstance(callee, AnyType) or self.chk.typing_mode_none(): - self.infer_arg_types_in_context(None, args) - return AnyType(), AnyType() - elif isinstance(callee, UnionType): - self.msg.disable_type_names += 1 - results = [self.check_call(subtype, args, arg_kinds, context, arg_names, - arg_messages=arg_messages) - for subtype in callee.items] - self.msg.disable_type_names -= 1 - return (UnionType.make_simplified_union([res[0] for res in results]), - callee) - elif isinstance(callee, Instance): - call_function = analyze_member_access('__call__', callee, context, - False, False, self.named_type, self.not_ready_callback, - self.msg) - return self.check_call(call_function, args, arg_kinds, context, arg_names, - callable_node, arg_messages) - elif isinstance(callee, TypeVarType): - return self.check_call(callee.upper_bound, args, arg_kinds, context, arg_names, - callable_node, arg_messages) - elif isinstance(callee, TypeType): - # Pass the original Type[] as context since that's where errors should go. - item = self.analyze_type_type_callee(callee.item, callee) - return self.check_call(item, args, arg_kinds, context, arg_names, - callable_node, arg_messages) - else: - return self.msg.not_callable(callee, context), AnyType() - - def analyze_type_type_callee(self, item: Type, context: Context) -> Type: - """Analyze the callee X in X(...) where X is Type[item]. - - Return a Y that we can pass to check_call(Y, ...). - """ - if isinstance(item, AnyType): - return AnyType() - if isinstance(item, Instance): - return type_object_type(item.type, self.named_type) - if isinstance(item, UnionType): - return UnionType([self.analyze_type_type_callee(item, context) - for item in item.items], item.line) - if isinstance(item, TypeVarType): - # Pretend we're calling the typevar's upper bound, - # i.e. its constructor (a poor approximation for reality, - # but better than AnyType...), but replace the return type - # with typevar. - callee = self.analyze_type_type_callee(item.upper_bound, context) - if isinstance(callee, CallableType): - if callee.is_generic(): - callee = None - else: - callee = callee.copy_modified(ret_type=item) - elif isinstance(callee, Overloaded): - if callee.items()[0].is_generic(): - callee = None - else: - callee = Overloaded([c.copy_modified(ret_type=item) - for c in callee.items()]) - if callee: - return callee - - self.msg.unsupported_type_type(item, context) - return AnyType() - - def infer_arg_types_in_context(self, callee: Optional[CallableType], - args: List[Node]) -> List[Type]: - """Infer argument expression types using a callable type as context. - - For example, if callee argument 2 has type List[int], infer the - argument expression with List[int] type context. - """ - # TODO Always called with callee as None, i.e. empty context. - res = [] # type: List[Type] - - fixed = len(args) - if callee: - fixed = min(fixed, callee.max_fixed_args()) - - arg_type = None # type: Type - ctx = None # type: Type - for i, arg in enumerate(args): - if i < fixed: - if callee and i < len(callee.arg_types): - ctx = callee.arg_types[i] - arg_type = self.accept(arg, ctx) - else: - if callee and callee.is_var_arg: - arg_type = self.accept(arg, callee.arg_types[-1]) - else: - arg_type = self.accept(arg) - if has_erased_component(arg_type): - res.append(NoneTyp()) - else: - res.append(arg_type) - return res - - def infer_arg_types_in_context2( - self, callee: CallableType, args: List[Node], arg_kinds: List[int], - formal_to_actual: List[List[int]]) -> List[Type]: - """Infer argument expression types using a callable type as context. - - For example, if callee argument 2 has type List[int], infer the - argument expression with List[int] type context. - - Returns the inferred types of *actual arguments*. - """ - res = [None] * len(args) # type: List[Type] - - for i, actuals in enumerate(formal_to_actual): - for ai in actuals: - if arg_kinds[ai] not in (nodes.ARG_STAR, nodes.ARG_STAR2): - res[ai] = self.accept(args[ai], callee.arg_types[i]) - - # Fill in the rest of the argument types. - for i, t in enumerate(res): - if not t: - res[i] = self.accept(args[i]) - return res - - def infer_function_type_arguments_using_context( - self, callable: CallableType, error_context: Context) -> CallableType: - """Unify callable return type to type context to infer type vars. - - For example, if the return type is set[t] where 't' is a type variable - of callable, and if the context is set[int], return callable modified - by substituting 't' with 'int'. - """ - ctx = self.chk.type_context[-1] - if not ctx: - return callable - # The return type may have references to type metavariables that - # we are inferring right now. We must consider them as indeterminate - # and they are not potential results; thus we replace them with the - # special ErasedType type. On the other hand, class type variables are - # valid results. - erased_ctx = replace_meta_vars(ctx, ErasedType()) - ret_type = callable.ret_type - if isinstance(ret_type, TypeVarType): - if ret_type.values or (not isinstance(ctx, Instance) or - not cast(Instance, ctx).args): - # The return type is a type variable. If it has values, we can't easily restrict - # type inference to conform to the valid values. If it's unrestricted, we could - # infer a too general type for the type variable if we use context, and this could - # result in confusing and spurious type errors elsewhere. - # - # Give up and just use function arguments for type inference. As an exception, - # if the context is a generic instance type, actually use it as context, as - # this *seems* to usually be the reasonable thing to do. - # - # See also github issues #462 and #360. - ret_type = NoneTyp() - args = infer_type_arguments(callable.type_var_ids(), ret_type, erased_ctx) - # Only substitute non-None and non-erased types. - new_args = [] # type: List[Type] - for arg in args: - if isinstance(arg, (NoneTyp, UninhabitedType)) or has_erased_component(arg): - new_args.append(None) - else: - new_args.append(arg) - return cast(CallableType, self.apply_generic_arguments(callable, new_args, - error_context)) - - def infer_function_type_arguments(self, callee_type: CallableType, - args: List[Node], - arg_kinds: List[int], - formal_to_actual: List[List[int]], - context: Context) -> CallableType: - """Infer the type arguments for a generic callee type. - - Infer based on the types of arguments. - - Return a derived callable type that has the arguments applied. - """ - if not self.chk.typing_mode_none(): - # Disable type errors during type inference. There may be errors - # due to partial available context information at this time, but - # these errors can be safely ignored as the arguments will be - # inferred again later. - self.msg.disable_errors() - - arg_types = self.infer_arg_types_in_context2( - callee_type, args, arg_kinds, formal_to_actual) - - self.msg.enable_errors() - - arg_pass_nums = self.get_arg_infer_passes( - callee_type.arg_types, formal_to_actual, len(args)) - - pass1_args = [] # type: List[Optional[Type]] - for i, arg in enumerate(arg_types): - if arg_pass_nums[i] > 1: - pass1_args.append(None) - else: - pass1_args.append(arg) - - inferred_args = infer_function_type_arguments( - callee_type, pass1_args, arg_kinds, formal_to_actual, - strict=self.chk.typing_mode_full()) # type: List[Type] - - if 2 in arg_pass_nums: - # Second pass of type inference. - (callee_type, - inferred_args) = self.infer_function_type_arguments_pass2( - callee_type, args, arg_kinds, formal_to_actual, - inferred_args, context) - - if callee_type.special_sig == 'dict' and len(inferred_args) == 2 and ( - ARG_NAMED in arg_kinds or ARG_STAR2 in arg_kinds): - # HACK: Infer str key type for dict(...) with keyword args. The type system - # can't represent this so we special case it, as this is a pretty common - # thing. This doesn't quite work with all possible subclasses of dict - # if they shuffle type variables around, as we assume that there is a 1-1 - # correspondence with dict type variables. This is a marginal issue and - # a little tricky to fix so it's left unfixed for now. - if isinstance(inferred_args[0], (NoneTyp, UninhabitedType)): - inferred_args[0] = self.named_type('builtins.str') - elif not is_subtype(self.named_type('builtins.str'), inferred_args[0]): - self.msg.fail(messages.KEYWORD_ARGUMENT_REQUIRES_STR_KEY_TYPE, - context) - else: - # In dynamically typed functions use implicit 'Any' types for - # type variables. - inferred_args = [AnyType()] * len(callee_type.variables) - return self.apply_inferred_arguments(callee_type, inferred_args, - context) - - def infer_function_type_arguments_pass2( - self, callee_type: CallableType, - args: List[Node], - arg_kinds: List[int], - formal_to_actual: List[List[int]], - inferred_args: List[Type], - context: Context) -> Tuple[CallableType, List[Type]]: - """Perform second pass of generic function type argument inference. - - The second pass is needed for arguments with types such as Callable[[T], S], - where both T and S are type variables, when the actual argument is a - lambda with inferred types. The idea is to infer the type variable T - in the first pass (based on the types of other arguments). This lets - us infer the argument and return type of the lambda expression and - thus also the type variable S in this second pass. - - Return (the callee with type vars applied, inferred actual arg types). - """ - # None or erased types in inferred types mean that there was not enough - # information to infer the argument. Replace them with None values so - # that they are not applied yet below. - for i, arg in enumerate(inferred_args): - if isinstance(arg, (NoneTyp, UninhabitedType)) or has_erased_component(arg): - inferred_args[i] = None - - callee_type = cast(CallableType, self.apply_generic_arguments( - callee_type, inferred_args, context)) - arg_types = self.infer_arg_types_in_context2( - callee_type, args, arg_kinds, formal_to_actual) - - inferred_args = infer_function_type_arguments( - callee_type, arg_types, arg_kinds, formal_to_actual) - - return callee_type, inferred_args - - def get_arg_infer_passes(self, arg_types: List[Type], - formal_to_actual: List[List[int]], - num_actuals: int) -> List[int]: - """Return pass numbers for args for two-pass argument type inference. - - For each actual, the pass number is either 1 (first pass) or 2 (second - pass). - - Two-pass argument type inference primarily lets us infer types of - lambdas more effectively. - """ - res = [1] * num_actuals - for i, arg in enumerate(arg_types): - if arg.accept(ArgInferSecondPassQuery()): - for j in formal_to_actual[i]: - res[j] = 2 - return res - - def apply_inferred_arguments(self, callee_type: CallableType, - inferred_args: List[Type], - context: Context) -> CallableType: - """Apply inferred values of type arguments to a generic function. - - Inferred_args contains the values of function type arguments. - """ - # Report error if some of the variables could not be solved. In that - # case assume that all variables have type Any to avoid extra - # bogus error messages. - for i, inferred_type in enumerate(inferred_args): - if not inferred_type or has_erased_component(inferred_type): - # Could not infer a non-trivial type for a type variable. - self.msg.could_not_infer_type_arguments( - callee_type, i + 1, context) - inferred_args = [AnyType()] * len(inferred_args) - # Apply the inferred types to the function type. In this case the - # return type must be CallableType, since we give the right number of type - # arguments. - return cast(CallableType, self.apply_generic_arguments(callee_type, - inferred_args, context)) - - def check_argument_count(self, callee: CallableType, actual_types: List[Type], - actual_kinds: List[int], actual_names: List[str], - formal_to_actual: List[List[int]], - context: Context, - messages: Optional[MessageBuilder]) -> bool: - """Check that there is a value for all required arguments to a function. - - Also check that there are no duplicate values for arguments. Report found errors - using 'messages' if it's not None. - - Return False if there were any errors. Otherwise return True - """ - # TODO(jukka): We could return as soon as we find an error if messages is None. - formal_kinds = callee.arg_kinds - - # Collect list of all actual arguments matched to formal arguments. - all_actuals = [] # type: List[int] - for actuals in formal_to_actual: - all_actuals.extend(actuals) - - is_unexpected_arg_error = False # Keep track of errors to avoid duplicate errors. - ok = True # False if we've found any error. - for i, kind in enumerate(actual_kinds): - if i not in all_actuals and ( - kind != nodes.ARG_STAR or - not is_empty_tuple(actual_types[i])): - # Extra actual: not matched by a formal argument. - ok = False - if kind != nodes.ARG_NAMED: - if messages: - messages.too_many_arguments(callee, context) - else: - if messages: - messages.unexpected_keyword_argument( - callee, actual_names[i], context) - is_unexpected_arg_error = True - elif kind == nodes.ARG_STAR and ( - nodes.ARG_STAR not in formal_kinds): - actual_type = actual_types[i] - if isinstance(actual_type, TupleType): - if all_actuals.count(i) < len(actual_type.items): - # Too many tuple items as some did not match. - if messages: - messages.too_many_arguments(callee, context) - ok = False - # *args can be applied even if the function takes a fixed - # number of positional arguments. This may succeed at runtime. - - for i, kind in enumerate(formal_kinds): - if kind == nodes.ARG_POS and (not formal_to_actual[i] and - not is_unexpected_arg_error): - # No actual for a mandatory positional formal. - if messages: - messages.too_few_arguments(callee, context, actual_names) - ok = False - elif kind in [nodes.ARG_POS, nodes.ARG_OPT, - nodes.ARG_NAMED] and is_duplicate_mapping( - formal_to_actual[i], actual_kinds): - if (self.chk.typing_mode_full() or - isinstance(actual_types[formal_to_actual[i][0]], TupleType)): - if messages: - messages.duplicate_argument_value(callee, i, context) - ok = False - elif (kind == nodes.ARG_NAMED and formal_to_actual[i] and - actual_kinds[formal_to_actual[i][0]] != nodes.ARG_NAMED): - # Positional argument when expecting a keyword argument. - if messages: - messages.too_many_positional_arguments(callee, context) - ok = False - return ok - - def check_argument_types(self, arg_types: List[Type], arg_kinds: List[int], - callee: CallableType, - formal_to_actual: List[List[int]], - context: Context, - messages: MessageBuilder = None, - check_arg: ArgChecker = None) -> None: - """Check argument types against a callable type. - - Report errors if the argument types are not compatible. - """ - messages = messages or self.msg - check_arg = check_arg or self.check_arg - # Keep track of consumed tuple *arg items. - tuple_counter = [0] - for i, actuals in enumerate(formal_to_actual): - for actual in actuals: - arg_type = arg_types[actual] - # Check that a *arg is valid as varargs. - if (arg_kinds[actual] == nodes.ARG_STAR and - not self.is_valid_var_arg(arg_type)): - messages.invalid_var_arg(arg_type, context) - if (arg_kinds[actual] == nodes.ARG_STAR2 and - not self.is_valid_keyword_var_arg(arg_type)): - messages.invalid_keyword_var_arg(arg_type, context) - # Get the type of an individual actual argument (for *args - # and **args this is the item type, not the collection type). - actual_type = get_actual_type(arg_type, arg_kinds[actual], - tuple_counter) - check_arg(actual_type, arg_type, arg_kinds[actual], - callee.arg_types[i], - actual + 1, i + 1, callee, context, messages) - - # There may be some remaining tuple varargs items that haven't - # been checked yet. Handle them. - if (callee.arg_kinds[i] == nodes.ARG_STAR and - arg_kinds[actual] == nodes.ARG_STAR and - isinstance(arg_types[actual], TupleType)): - tuplet = cast(TupleType, arg_types[actual]) - while tuple_counter[0] < len(tuplet.items): - actual_type = get_actual_type(arg_type, - arg_kinds[actual], - tuple_counter) - check_arg(actual_type, arg_type, arg_kinds[actual], - callee.arg_types[i], - actual + 1, i + 1, callee, context, messages) - - def check_arg(self, caller_type: Type, original_caller_type: Type, - caller_kind: int, - callee_type: Type, n: int, m: int, callee: CallableType, - context: Context, messages: MessageBuilder) -> None: - """Check the type of a single argument in a call.""" - if isinstance(caller_type, Void): - messages.does_not_return_value(caller_type, context) - elif isinstance(caller_type, DeletedType): - messages.deleted_as_rvalue(caller_type, context) - elif not is_subtype(caller_type, callee_type): - messages.incompatible_argument(n, m, callee, original_caller_type, - caller_kind, context) - - def overload_call_target(self, arg_types: List[Type], arg_kinds: List[int], - arg_names: List[str], - overload: Overloaded, context: Context, - messages: MessageBuilder = None) -> Type: - """Infer the correct overload item to call with given argument types. - - The return value may be CallableType or AnyType (if an unique item - could not be determined). - """ - messages = messages or self.msg - # TODO: For overlapping signatures we should try to get a more precise - # result than 'Any'. - match = [] # type: List[CallableType] - best_match = 0 - for typ in overload.items(): - similarity = self.erased_signature_similarity(arg_types, arg_kinds, arg_names, - typ, context=context) - if similarity > 0 and similarity >= best_match: - if (match and not is_same_type(match[-1].ret_type, - typ.ret_type) and - not mypy.checker.is_more_precise_signature( - match[-1], typ)): - # Ambiguous return type. Either the function overload is - # overlapping (which we don't handle very well here) or the - # caller has provided some Any argument types; in either - # case we'll fall back to Any. It's okay to use Any types - # in calls. - # - # Overlapping overload items are generally fine if the - # overlapping is only possible when there is multiple - # inheritance, as this is rare. See docstring of - # mypy.meet.is_overlapping_types for more about this. - # - # Note that there is no ambiguity if the items are - # covariant in both argument types and return types with - # respect to type precision. We'll pick the best/closest - # match. - # - # TODO: Consider returning a union type instead if the - # overlapping is NOT due to Any types? - return AnyType() - else: - match.append(typ) - best_match = max(best_match, similarity) - if not match: - messages.no_variant_matches_arguments(overload, arg_types, context) - return AnyType() - else: - if len(match) == 1: - return match[0] - else: - # More than one signature matches. Pick the first *non-erased* - # matching signature, or default to the first one if none - # match. - for m in match: - if self.match_signature_types(arg_types, arg_kinds, arg_names, m, - context=context): - return m - return match[0] - - def erased_signature_similarity(self, arg_types: List[Type], arg_kinds: List[int], - arg_names: List[str], callee: CallableType, - context: Context) -> int: - """Determine whether arguments could match the signature at runtime. - - Return similarity level (0 = no match, 1 = can match, 2 = non-promotion match). See - overload_arg_similarity for a discussion of similarity levels. - """ - formal_to_actual = map_actuals_to_formals(arg_kinds, - arg_names, - callee.arg_kinds, - callee.arg_names, - lambda i: arg_types[i]) - - if not self.check_argument_count(callee, arg_types, arg_kinds, arg_names, - formal_to_actual, None, None): - # Too few or many arguments -> no match. - return 0 - - similarity = 2 - - def check_arg(caller_type: Type, original_caller_type: Type, caller_kind: int, - callee_type: Type, n: int, m: int, callee: CallableType, - context: Context, messages: MessageBuilder) -> None: - nonlocal similarity - similarity = min(similarity, - overload_arg_similarity(caller_type, callee_type)) - if similarity == 0: - # No match -- exit early since none of the remaining work can change - # the result. - raise Finished - - try: - self.check_argument_types(arg_types, arg_kinds, callee, formal_to_actual, - context=context, check_arg=check_arg) - except Finished: - pass - - return similarity - - def match_signature_types(self, arg_types: List[Type], arg_kinds: List[int], - arg_names: List[str], callee: CallableType, - context: Context) -> bool: - """Determine whether arguments types match the signature. - - Assume that argument counts are compatible. - - Return True if arguments match. - """ - formal_to_actual = map_actuals_to_formals(arg_kinds, - arg_names, - callee.arg_kinds, - callee.arg_names, - lambda i: arg_types[i]) - ok = True - - def check_arg(caller_type: Type, original_caller_type: Type, caller_kind: int, - callee_type: Type, n: int, m: int, callee: CallableType, - context: Context, messages: MessageBuilder) -> None: - nonlocal ok - if not is_subtype(caller_type, callee_type): - ok = False - - self.check_argument_types(arg_types, arg_kinds, callee, formal_to_actual, - context=context, check_arg=check_arg) - return ok - - def apply_generic_arguments(self, callable: CallableType, types: List[Type], - context: Context) -> Type: - """Simple wrapper around mypy.applytype.apply_generic_arguments.""" - return applytype.apply_generic_arguments(callable, types, self.msg, context) - - def apply_generic_arguments2(self, overload: Overloaded, types: List[Type], - context: Context) -> Type: - items = [] # type: List[CallableType] - for item in overload.items(): - applied = self.apply_generic_arguments(item, types, context) - if isinstance(applied, CallableType): - items.append(applied) - else: - # There was an error. - return AnyType() - return Overloaded(items) - - def visit_member_expr(self, e: MemberExpr) -> Type: - """Visit member expression (of form e.id).""" - result = self.analyze_ordinary_member_access(e, False) - return self.chk.narrow_type_from_binder(e, result) - - def analyze_ordinary_member_access(self, e: MemberExpr, - is_lvalue: bool) -> Type: - """Analyse member expression or member lvalue.""" - if e.kind is not None: - # This is a reference to a module attribute. - return self.analyze_ref_expr(e) - else: - # This is a reference to a non-module attribute. - return analyze_member_access(e.name, self.accept(e.expr), e, - is_lvalue, False, - self.named_type, self.not_ready_callback, self.msg) - - def analyze_external_member_access(self, member: str, base_type: Type, - context: Context) -> Type: - """Analyse member access that is external, i.e. it cannot - refer to private definitions. Return the result type. - """ - # TODO remove; no private definitions in mypy - return analyze_member_access(member, base_type, context, False, False, - self.named_type, self.not_ready_callback, self.msg) - - def visit_int_expr(self, e: IntExpr) -> Type: - """Type check an integer literal (trivial).""" - return self.named_type('builtins.int') - - def visit_str_expr(self, e: StrExpr) -> Type: - """Type check a string literal (trivial).""" - return self.named_type('builtins.str') - - def visit_bytes_expr(self, e: BytesExpr) -> Type: - """Type check a bytes literal (trivial).""" - return self.named_type('builtins.bytes') - - def visit_unicode_expr(self, e: UnicodeExpr) -> Type: - """Type check a unicode literal (trivial).""" - return self.named_type('builtins.unicode') - - def visit_float_expr(self, e: FloatExpr) -> Type: - """Type check a float literal (trivial).""" - return self.named_type('builtins.float') - - def visit_complex_expr(self, e: ComplexExpr) -> Type: - """Type check a complex literal.""" - return self.named_type('builtins.complex') - - def visit_ellipsis(self, e: EllipsisExpr) -> Type: - """Type check '...'.""" - if self.chk.options.python_version[0] >= 3: - return self.named_type('builtins.ellipsis') - else: - # '...' is not valid in normal Python 2 code, but it can - # be used in stubs. The parser makes sure that we only - # get this far if we are in a stub, and we can safely - # return 'object' as ellipsis is special cased elsewhere. - # The builtins.ellipsis type does not exist in Python 2. - return self.named_type('builtins.object') - - def visit_op_expr(self, e: OpExpr) -> Type: - """Type check a binary operator expression.""" - if e.op == 'and' or e.op == 'or': - return self.check_boolean_op(e, e) - if e.op == '*' and isinstance(e.left, ListExpr): - # Expressions of form [...] * e get special type inference. - return self.check_list_multiply(e) - if e.op == '%' and isinstance(e.left, (StrExpr, BytesExpr)): - return self.strfrm_checker.check_str_interpolation(cast(StrExpr, e.left), e.right) - left_type = self.accept(e.left) - - if e.op in nodes.op_methods: - method = self.get_operator_method(e.op) - result, method_type = self.check_op(method, left_type, e.right, e, - allow_reverse=True) - e.method_type = method_type - return result - else: - raise RuntimeError('Unknown operator {}'.format(e.op)) - - def visit_comparison_expr(self, e: ComparisonExpr) -> Type: - """Type check a comparison expression. - - Comparison expressions are type checked consecutive-pair-wise - That is, 'a < b > c == d' is check as 'a < b and b > c and c == d' - """ - result = None # type: mypy.types.Type - - # Check each consecutive operand pair and their operator - for left, right, operator in zip(e.operands, e.operands[1:], e.operators): - left_type = self.accept(left) - - method_type = None # type: mypy.types.Type - - if operator == 'in' or operator == 'not in': - right_type = self.accept(right) # TODO only evaluate if needed - - # Keep track of whether we get type check errors (these won't be reported, they - # are just to verify whether something is valid typing wise). - local_errors = self.msg.copy() - local_errors.disable_count = 0 - sub_result, method_type = self.check_op_local('__contains__', right_type, - left, e, local_errors) - if isinstance(right_type, PartialType): - # We don't really know if this is an error or not, so just shut up. - pass - elif (local_errors.is_errors() and - # is_valid_var_arg is True for any Iterable - self.is_valid_var_arg(right_type)): - itertype = self.chk.analyze_iterable_item_type(right) - method_type = CallableType( - [left_type], - [nodes.ARG_POS], - [None], - self.chk.bool_type(), - self.named_type('builtins.function')) - sub_result = self.chk.bool_type() - if not is_subtype(left_type, itertype): - self.msg.unsupported_operand_types('in', left_type, right_type, e) - else: - self.msg.add_errors(local_errors) - if operator == 'not in': - sub_result = self.chk.bool_type() - elif operator in nodes.op_methods: - method = self.get_operator_method(operator) - sub_result, method_type = self.check_op(method, left_type, right, e, - allow_reverse=True) - - elif operator == 'is' or operator == 'is not': - sub_result = self.chk.bool_type() - method_type = None - else: - raise RuntimeError('Unknown comparison operator {}'.format(operator)) - - e.method_types.append(method_type) - - # Determine type of boolean-and of result and sub_result - if result is None: - result = sub_result - else: - # TODO: check on void needed? - self.check_not_void(sub_result, e) - result = join.join_types(result, sub_result) - - return result - - def get_operator_method(self, op: str) -> str: - if op == '/' and self.chk.options.python_version[0] == 2: - # TODO also check for "from __future__ import division" - return '__div__' - else: - return nodes.op_methods[op] - - def check_op_local(self, method: str, base_type: Type, arg: Node, - context: Context, local_errors: MessageBuilder) -> Tuple[Type, Type]: - """Type check a binary operation which maps to a method call. - - Return tuple (result type, inferred operator method type). - """ - method_type = analyze_member_access(method, base_type, context, False, False, - self.named_type, self.not_ready_callback, local_errors) - return self.check_call(method_type, [arg], [nodes.ARG_POS], - context, arg_messages=local_errors) - - def check_op(self, method: str, base_type: Type, arg: Node, - context: Context, - allow_reverse: bool = False) -> Tuple[Type, Type]: - """Type check a binary operation which maps to a method call. - - Return tuple (result type, inferred operator method type). - """ - # Use a local error storage for errors related to invalid argument - # type (but NOT other errors). This error may need to be suppressed - # for operators which support __rX methods. - local_errors = self.msg.copy() - local_errors.disable_count = 0 - if not allow_reverse or self.has_member(base_type, method): - result = self.check_op_local(method, base_type, arg, context, - local_errors) - if allow_reverse: - arg_type = self.chk.type_map[arg] - if isinstance(arg_type, AnyType): - # If the right operand has type Any, we can't make any - # conjectures about the type of the result, since the - # operand could have a __r method that returns anything. - - # However, in weak mode, we do make conjectures. - if not self.chk.typing_mode_weak(): - result = AnyType(), result[1] - success = not local_errors.is_errors() - else: - result = AnyType(), AnyType() - success = False - if success or not allow_reverse or isinstance(base_type, AnyType): - # We were able to call the normal variant of the operator method, - # or there was some problem not related to argument type - # validity, or the operator has no __rX method. In any case, we - # don't need to consider the __rX method. - self.msg.add_errors(local_errors) - return result - else: - # Calling the operator method was unsuccessful. Try the __rX - # method of the other operand instead. - rmethod = self.get_reverse_op_method(method) - arg_type = self.accept(arg) - if self.has_member(arg_type, rmethod): - method_type = self.analyze_external_member_access( - rmethod, arg_type, context) - temp = TempNode(base_type) - return self.check_call(method_type, [temp], [nodes.ARG_POS], - context) - else: - # No __rX method either. Do deferred type checking to produce - # error message that we may have missed previously. - # TODO Fix type checking an expression more than once. - return self.check_op_local(method, base_type, arg, context, - self.msg) - - def get_reverse_op_method(self, method: str) -> str: - if method == '__div__' and self.chk.options.python_version[0] == 2: - return '__rdiv__' - else: - return nodes.reverse_op_methods[method] - - def check_boolean_op(self, e: OpExpr, context: Context) -> Type: - """Type check a boolean operation ('and' or 'or').""" - - # A boolean operation can evaluate to either of the operands. - - # We use the current type context to guide the type inference of of - # the left operand. We also use the left operand type to guide the type - # inference of the right operand so that expressions such as - # '[1] or []' are inferred correctly. - ctx = self.chk.type_context[-1] - left_type = self.accept(e.left, ctx) - - if e.op == 'and': - right_map, _ = \ - mypy.checker.find_isinstance_check(e.left, self.chk.type_map, - self.chk.typing_mode_weak()) - elif e.op == 'or': - _, right_map = \ - mypy.checker.find_isinstance_check(e.left, self.chk.type_map, - self.chk.typing_mode_weak()) - else: - right_map = None - - with self.chk.binder.frame_context(): - if right_map: - for var, type in right_map.items(): - self.chk.binder.push(var, type) - - right_type = self.accept(e.right, left_type) - - self.check_not_void(left_type, context) - self.check_not_void(right_type, context) - return UnionType.make_simplified_union([left_type, right_type]) - - def check_list_multiply(self, e: OpExpr) -> Type: - """Type check an expression of form '[...] * e'. - - Type inference is special-cased for this common construct. - """ - right_type = self.accept(e.right) - if is_subtype(right_type, self.named_type('builtins.int')): - # Special case: [...] * . Use the type context of the - # OpExpr, since the multiplication does not affect the type. - left_type = self.accept(e.left, context=self.chk.type_context[-1]) - else: - left_type = self.accept(e.left) - result, method_type = self.check_op('__mul__', left_type, e.right, e) - e.method_type = method_type - return result - - def visit_unary_expr(self, e: UnaryExpr) -> Type: - """Type check an unary operation ('not', '-', '+' or '~').""" - operand_type = self.accept(e.expr) - op = e.op - if op == 'not': - self.check_not_void(operand_type, e) - result = self.chk.bool_type() # type: Type - elif op == '-': - method_type = self.analyze_external_member_access('__neg__', - operand_type, e) - result, method_type = self.check_call(method_type, [], [], e) - e.method_type = method_type - elif op == '+': - method_type = self.analyze_external_member_access('__pos__', - operand_type, e) - result, method_type = self.check_call(method_type, [], [], e) - e.method_type = method_type - else: - assert op == '~', "unhandled unary operator" - method_type = self.analyze_external_member_access('__invert__', - operand_type, e) - result, method_type = self.check_call(method_type, [], [], e) - e.method_type = method_type - return result - - def visit_index_expr(self, e: IndexExpr) -> Type: - """Type check an index expression (base[index]). - - It may also represent type application. - """ - result = self.visit_index_expr_helper(e) - return self.chk.narrow_type_from_binder(e, result) - - def visit_index_expr_helper(self, e: IndexExpr) -> Type: - if e.analyzed: - # It's actually a type application. - return self.accept(e.analyzed) - left_type = self.accept(e.base) - if isinstance(left_type, TupleType) and self.chk.typing_mode_full(): - # Special case for tuples. They support indexing only by integer - # literals. (Except in weak type checking mode.) - index = e.index - if isinstance(index, SliceExpr): - return self.visit_tuple_slice_helper(left_type, index) - - ok = False - if isinstance(index, IntExpr): - n = index.value - ok = True - elif isinstance(index, UnaryExpr): - if index.op == '-': - operand = index.expr - if isinstance(operand, IntExpr): - n = len(left_type.items) - operand.value - ok = True - if ok: - if n >= 0 and n < len(left_type.items): - return left_type.items[n] - else: - self.chk.fail(messages.TUPLE_INDEX_OUT_OF_RANGE, e) - return AnyType() - else: - self.chk.fail(messages.TUPLE_INDEX_MUST_BE_AN_INT_LITERAL, e) - return AnyType() - else: - result, method_type = self.check_op('__getitem__', left_type, e.index, e) - e.method_type = method_type - return result - - def visit_tuple_slice_helper(self, left_type: TupleType, slic: SliceExpr): - begin = None # type: int - end = None # type: int - stride = None # type:int - - if slic.begin_index: - begin = self._get_value(slic.begin_index) - if begin is None: - self.chk.fail( - messages.TUPLE_SLICE_MUST_BE_AN_INT_LITERAL, - slic.begin_index) - return AnyType() - - if slic.end_index: - end = self._get_value(slic.end_index) - if end is None: - self.chk.fail( - messages.TUPLE_SLICE_MUST_BE_AN_INT_LITERAL, - slic.end_index) - return AnyType() - - if slic.stride: - stride = self._get_value(slic.stride) - if stride is None: - self.chk.fail( - messages.TUPLE_SLICE_MUST_BE_AN_INT_LITERAL, - slic.stride) - return AnyType() - - return left_type.slice(begin, stride, end) - - def _get_value(self, index: Node) -> Optional[int]: - if isinstance(index, IntExpr): - return index.value - elif isinstance(index, UnaryExpr): - if index.op == '-': - operand = index.expr - if isinstance(operand, IntExpr): - return -1 * operand.value - return None - - def visit_cast_expr(self, expr: CastExpr) -> Type: - """Type check a cast expression.""" - source_type = self.accept(expr.expr, context=AnyType()) - target_type = expr.type - if self.chk.options.warn_redundant_casts and is_same_type(source_type, target_type): - self.msg.redundant_cast(target_type, expr) - if not self.is_valid_cast(source_type, target_type): - self.msg.invalid_cast(target_type, source_type, expr) - return target_type - - def is_valid_cast(self, source_type: Type, target_type: Type) -> bool: - """Is a cast from source_type to target_type meaningful?""" - return (isinstance(target_type, AnyType) or - (not isinstance(source_type, Void) and - not isinstance(target_type, Void))) - - def visit_reveal_type_expr(self, expr: RevealTypeExpr) -> Type: - """Type check a reveal_type expression.""" - revealed_type = self.accept(expr.expr) - self.msg.reveal_type(revealed_type, expr) - return revealed_type - - def visit_type_application(self, tapp: TypeApplication) -> Type: - """Type check a type application (expr[type, ...]).""" - self.chk.fail(messages.GENERIC_TYPE_NOT_VALID_AS_EXPRESSION, tapp) - return AnyType() - - def visit_type_alias_expr(self, alias: TypeAliasExpr) -> Type: - return AnyType() - - def visit_list_expr(self, e: ListExpr) -> Type: - """Type check a list expression [...].""" - return self.check_list_or_set_expr(e.items, 'builtins.list', '', - e) - - def visit_set_expr(self, e: SetExpr) -> Type: - return self.check_list_or_set_expr(e.items, 'builtins.set', '', e) - - def check_list_or_set_expr(self, items: List[Node], fullname: str, - tag: str, context: Context) -> Type: - # Translate into type checking a generic function call. - tvdef = TypeVarDef('T', -1, [], self.chk.object_type()) - tv = TypeVarType(tvdef) - constructor = CallableType( - [tv], - [nodes.ARG_STAR], - [None], - self.chk.named_generic_type(fullname, [tv]), - self.named_type('builtins.function'), - name=tag, - variables=[tvdef]) - return self.check_call(constructor, - items, - [nodes.ARG_POS] * len(items), context)[0] - - def visit_tuple_expr(self, e: TupleExpr) -> Type: - """Type check a tuple expression.""" - ctx = None # type: TupleType - # Try to determine type context for type inference. - if isinstance(self.chk.type_context[-1], TupleType): - t = self.chk.type_context[-1] - if len(t.items) == len(e.items): - ctx = t - # Infer item types. - items = [] # type: List[Type] - for i in range(len(e.items)): - item = e.items[i] - tt = None # type: Type - if not ctx: - tt = self.accept(item) - else: - tt = self.accept(item, ctx.items[i]) - self.check_not_void(tt, e) - items.append(tt) - fallback_item = join.join_type_list(items) - return TupleType(items, self.chk.named_generic_type('builtins.tuple', [fallback_item])) - - def visit_dict_expr(self, e: DictExpr) -> Type: - # Translate into type checking a generic function call. - ktdef = TypeVarDef('KT', -1, [], self.chk.object_type()) - vtdef = TypeVarDef('VT', -2, [], self.chk.object_type()) - kt = TypeVarType(ktdef) - vt = TypeVarType(vtdef) - # The callable type represents a function like this: - # - # def (*v: Tuple[kt, vt]) -> Dict[kt, vt]: ... - constructor = CallableType( - [TupleType([kt, vt], self.named_type('builtins.tuple'))], - [nodes.ARG_STAR], - [None], - self.chk.named_generic_type('builtins.dict', [kt, vt]), - self.named_type('builtins.function'), - name='', - variables=[ktdef, vtdef]) - # Synthesize function arguments. - args = [] # type: List[Node] - for key, value in e.items: - args.append(TupleExpr([key, value])) - return self.check_call(constructor, - args, - [nodes.ARG_POS] * len(args), e)[0] - - def visit_func_expr(self, e: FuncExpr) -> Type: - """Type check lambda expression.""" - inferred_type = self.infer_lambda_type_using_context(e) - if not inferred_type: - # No useful type context. - ret_type = e.expr().accept(self.chk) - if not e.arguments: - # Form 'lambda: e'; just use the inferred return type. - return CallableType([], [], [], ret_type, self.named_type('builtins.function')) - else: - # TODO: Consider reporting an error. However, this is fine if - # we are just doing the first pass in contextual type - # inference. - return AnyType() - else: - # Type context available. - self.chk.check_func_item(e, type_override=inferred_type) - if e.expr() not in self.chk.type_map: - self.accept(e.expr()) - ret_type = self.chk.type_map[e.expr()] - if isinstance(ret_type, NoneTyp): - # For "lambda ...: None", just use type from the context. - # Important when the context is Callable[..., None] which - # really means Void. See #1425. - return inferred_type - return replace_callable_return_type(inferred_type, ret_type) - - def infer_lambda_type_using_context(self, e: FuncExpr) -> CallableType: - """Try to infer lambda expression type using context. - - Return None if could not infer type. - """ - # TODO also accept 'Any' context - ctx = self.chk.type_context[-1] - if not ctx or not isinstance(ctx, CallableType): - return None - - # The context may have function type variables in it. We replace them - # since these are the type variables we are ultimately trying to infer; - # they must be considered as indeterminate. We use ErasedType since it - # does not affect type inference results (it is for purposes like this - # only). - ctx = replace_meta_vars(ctx, ErasedType()) - - callable_ctx = cast(CallableType, ctx) - - arg_kinds = [arg.kind for arg in e.arguments] - - if callable_ctx.is_ellipsis_args: - # Fill in Any arguments to match the arguments of the lambda. - callable_ctx = callable_ctx.copy_modified( - is_ellipsis_args=False, - arg_types=[AnyType()] * len(arg_kinds), - arg_kinds=arg_kinds - ) - - if callable_ctx.arg_kinds != arg_kinds: - # Incompatible context; cannot use it to infer types. - self.chk.fail(messages.CANNOT_INFER_LAMBDA_TYPE, e) - return None - - return callable_ctx - - def visit_super_expr(self, e: SuperExpr) -> Type: - """Type check a super expression (non-lvalue).""" - t = self.analyze_super(e, False) - return t - - def analyze_super(self, e: SuperExpr, is_lvalue: bool) -> Type: - """Type check a super expression.""" - if e.info and e.info.bases: - # TODO fix multiple inheritance etc - if len(e.info.mro) < 2: - self.chk.fail('Internal error: unexpected mro for {}: {}'.format( - e.info.name(), e.info.mro), e) - return AnyType() - for base in e.info.mro[1:]: - if e.name in base.names or base == e.info.mro[-1]: - if e.info.fallback_to_any and base == e.info.mro[-1]: - # There's an undefined base class, and we're - # at the end of the chain. That's not an error. - return AnyType() - if not self.chk.typing_mode_full(): - return AnyType() - return analyze_member_access(e.name, self_type(e.info), e, - is_lvalue, True, - self.named_type, self.not_ready_callback, - self.msg, base) - else: - # Invalid super. This has been reported by the semantic analyzer. - return AnyType() - - def visit_slice_expr(self, e: SliceExpr) -> Type: - for index in [e.begin_index, e.end_index, e.stride]: - if index: - t = self.accept(index) - self.chk.check_subtype(t, self.named_type('builtins.int'), - index, messages.INVALID_SLICE_INDEX) - return self.named_type('builtins.slice') - - def visit_list_comprehension(self, e: ListComprehension) -> Type: - return self.check_generator_or_comprehension( - e.generator, 'builtins.list', '') - - def visit_set_comprehension(self, e: SetComprehension) -> Type: - return self.check_generator_or_comprehension( - e.generator, 'builtins.set', '') - - def visit_generator_expr(self, e: GeneratorExpr) -> Type: - return self.check_generator_or_comprehension(e, 'typing.Iterator', - '') - - def check_generator_or_comprehension(self, gen: GeneratorExpr, - type_name: str, - id_for_messages: str) -> Type: - """Type check a generator expression or a list comprehension.""" - self.check_for_comp(gen) - - # Infer the type of the list comprehension by using a synthetic generic - # callable type. - tvdef = TypeVarDef('T', -1, [], self.chk.object_type()) - tv = TypeVarType(tvdef) - constructor = CallableType( - [tv], - [nodes.ARG_POS], - [None], - self.chk.named_generic_type(type_name, [tv]), - self.chk.named_type('builtins.function'), - name=id_for_messages, - variables=[tvdef]) - return self.check_call(constructor, - [gen.left_expr], [nodes.ARG_POS], gen)[0] - - def visit_dictionary_comprehension(self, e: DictionaryComprehension): - """Type check a dictionary comprehension.""" - self.check_for_comp(e) - - # Infer the type of the list comprehension by using a synthetic generic - # callable type. - ktdef = TypeVarDef('KT', -1, [], self.chk.object_type()) - vtdef = TypeVarDef('VT', -2, [], self.chk.object_type()) - kt = TypeVarType(ktdef) - vt = TypeVarType(vtdef) - constructor = CallableType( - [kt, vt], - [nodes.ARG_POS, nodes.ARG_POS], - [None, None], - self.chk.named_generic_type('builtins.dict', [kt, vt]), - self.chk.named_type('builtins.function'), - name='', - variables=[ktdef, vtdef]) - return self.check_call(constructor, - [e.key, e.value], [nodes.ARG_POS, nodes.ARG_POS], e)[0] - - def check_for_comp(self, e: Union[GeneratorExpr, DictionaryComprehension]) -> None: - """Check the for_comp part of comprehensions. That is the part from 'for': - ... for x in y if z - """ - with self.chk.binder.frame_context(): - for index, sequence, conditions in zip(e.indices, e.sequences, - e.condlists): - sequence_type = self.chk.analyze_iterable_item_type(sequence) - self.chk.analyze_index_variables(index, sequence_type, e) - for condition in conditions: - self.accept(condition) - - def visit_conditional_expr(self, e: ConditionalExpr) -> Type: - cond_type = self.accept(e.cond) - self.check_not_void(cond_type, e) - ctx = self.chk.type_context[-1] - - # Gain type information from isinstance if it is there - # but only for the current expression - if_map, else_map = mypy.checker.find_isinstance_check( - e.cond, - self.chk.type_map, - self.chk.typing_mode_weak()) - - if_type = self.analyze_cond_branch(if_map, e.if_expr, context=ctx) - - if not mypy.checker.is_valid_inferred_type(if_type): - # Analyze the right branch disregarding the left branch. - else_type = self.analyze_cond_branch(else_map, e.else_expr, context=ctx) - - # If it would make a difference, re-analyze the left - # branch using the right branch's type as context. - if ctx is None or not is_equivalent(else_type, ctx): - # TODO: If it's possible that the previous analysis of - # the left branch produced errors that are avoided - # using this context, suppress those errors. - if_type = self.analyze_cond_branch(if_map, e.if_expr, context=else_type) - - else: - # Analyze the right branch in the context of the left - # branch's type. - else_type = self.analyze_cond_branch(else_map, e.else_expr, context=if_type) - - res = join.join_types(if_type, else_type) - - return res - - def analyze_cond_branch(self, map: Optional[Dict[Node, Type]], - node: Node, context: Optional[Type]) -> Type: - with self.chk.binder.frame_context(): - if map: - for var, type in map.items(): - self.chk.binder.push(var, type) - return self.accept(node, context=context) - - def visit_backquote_expr(self, e: BackquoteExpr) -> Type: - self.accept(e.expr) - return self.named_type('builtins.str') - - # - # Helpers - # - - def accept(self, node: Node, context: Type = None) -> Type: - """Type check a node. Alias for TypeChecker.accept.""" - return self.chk.accept(node, context) - - def check_not_void(self, typ: Type, context: Context) -> None: - """Generate an error if type is Void.""" - self.chk.check_not_void(typ, context) - - def is_boolean(self, typ: Type) -> bool: - """Is type compatible with bool?""" - return is_subtype(typ, self.chk.bool_type()) - - def named_type(self, name: str) -> Instance: - """Return an instance type with type given by the name and no type - arguments. Alias for TypeChecker.named_type. - """ - return self.chk.named_type(name) - - def is_valid_var_arg(self, typ: Type) -> bool: - """Is a type valid as a *args argument?""" - return (isinstance(typ, TupleType) or - is_subtype(typ, self.chk.named_generic_type('typing.Iterable', - [AnyType()])) or - isinstance(typ, AnyType)) - - def is_valid_keyword_var_arg(self, typ: Type) -> bool: - """Is a type valid as a **kwargs argument?""" - return is_subtype(typ, self.chk.named_generic_type( - 'builtins.dict', [self.named_type('builtins.str'), AnyType()])) - - def has_non_method(self, typ: Type, member: str) -> bool: - """Does type have a member variable / property with the given name?""" - if isinstance(typ, Instance): - return (not typ.type.has_method(member) and - typ.type.has_readable_member(member)) - else: - return False - - def has_member(self, typ: Type, member: str) -> bool: - """Does type have member with the given name?""" - # TODO TupleType => also consider tuple attributes - if isinstance(typ, Instance): - return typ.type.has_readable_member(member) - elif isinstance(typ, AnyType): - return True - elif isinstance(typ, UnionType): - result = all(self.has_member(x, member) for x in typ.items) - return result - elif isinstance(typ, TupleType): - return self.has_member(typ.fallback, member) - else: - return False - - def not_ready_callback(self, name: str, context: Context) -> None: - """Called when we can't infer the type of a variable because it's not ready yet. - - Either defer type checking of the enclosing function to the next - pass or report an error. - """ - self.chk.handle_cannot_determine_type(name, context) - - -def map_actuals_to_formals(caller_kinds: List[int], - caller_names: List[str], - callee_kinds: List[int], - callee_names: List[str], - caller_arg_type: Callable[[int], - Type]) -> List[List[int]]: - """Calculate mapping between actual (caller) args and formals. - - The result contains a list of caller argument indexes mapping to each - callee argument index, indexed by callee index. - - The caller_arg_type argument should evaluate to the type of the actual - argument type with the given index. - """ - ncallee = len(callee_kinds) - map = [None] * ncallee # type: List[List[int]] - for i in range(ncallee): - map[i] = [] - j = 0 - for i, kind in enumerate(caller_kinds): - if kind == nodes.ARG_POS: - if j < ncallee: - if callee_kinds[j] in [nodes.ARG_POS, nodes.ARG_OPT, - nodes.ARG_NAMED]: - map[j].append(i) - j += 1 - elif callee_kinds[j] == nodes.ARG_STAR: - map[j].append(i) - elif kind == nodes.ARG_STAR: - # We need to know the actual type to map varargs. - argt = caller_arg_type(i) - if isinstance(argt, TupleType): - # A tuple actual maps to a fixed number of formals. - for _ in range(len(argt.items)): - if j < ncallee: - if callee_kinds[j] != nodes.ARG_STAR2: - map[j].append(i) - else: - raise NotImplementedError() - j += 1 - else: - # Assume that it is an iterable (if it isn't, there will be - # an error later). - while j < ncallee: - if callee_kinds[j] in (nodes.ARG_NAMED, nodes.ARG_STAR2): - break - else: - map[j].append(i) - j += 1 - elif kind == nodes.ARG_NAMED: - name = caller_names[i] - if name in callee_names: - map[callee_names.index(name)].append(i) - elif nodes.ARG_STAR2 in callee_kinds: - map[callee_kinds.index(nodes.ARG_STAR2)].append(i) - else: - assert kind == nodes.ARG_STAR2 - for j in range(ncallee): - # TODO tuple varargs complicate this - no_certain_match = ( - not map[j] or caller_kinds[map[j][0]] == nodes.ARG_STAR) - if ((callee_names[j] and no_certain_match) - or callee_kinds[j] == nodes.ARG_STAR2): - map[j].append(i) - return map - - -def is_empty_tuple(t: Type) -> bool: - return isinstance(t, TupleType) and not t.items - - -def is_duplicate_mapping(mapping: List[int], actual_kinds: List[int]) -> bool: - # Multiple actuals can map to the same formal only if they both come from - # varargs (*args and **kwargs); in this case at runtime it is possible that - # there are no duplicates. We need to allow this, as the convention - # f(..., *args, **kwargs) is common enough. - return len(mapping) > 1 and not ( - len(mapping) == 2 and - actual_kinds[mapping[0]] == nodes.ARG_STAR and - actual_kinds[mapping[1]] == nodes.ARG_STAR2) - - -def replace_callable_return_type(c: CallableType, new_ret_type: Type) -> CallableType: - """Return a copy of a callable type with a different return type.""" - return c.copy_modified(ret_type=new_ret_type) - - -class ArgInferSecondPassQuery(types.TypeQuery): - """Query whether an argument type should be inferred in the second pass. - - The result is True if the type has a type variable in a callable return - type anywhere. For example, the result for Callable[[], T] is True if t is - a type variable. - """ - def __init__(self) -> None: - super().__init__(False, types.ANY_TYPE_STRATEGY) - - def visit_callable_type(self, t: CallableType) -> bool: - return self.query_types(t.arg_types) or t.accept(HasTypeVarQuery()) - - -class HasTypeVarQuery(types.TypeQuery): - """Visitor for querying whether a type has a type variable component.""" - def __init__(self) -> None: - super().__init__(False, types.ANY_TYPE_STRATEGY) - - def visit_type_var(self, t: TypeVarType) -> bool: - return True - - -def has_erased_component(t: Type) -> bool: - return t is not None and t.accept(HasErasedComponentsQuery()) - - -class HasErasedComponentsQuery(types.TypeQuery): - """Visitor for querying whether a type has an erased component.""" - def __init__(self) -> None: - super().__init__(False, types.ANY_TYPE_STRATEGY) - - def visit_erased_type(self, t: ErasedType) -> bool: - return True - - -def overload_arg_similarity(actual: Type, formal: Type) -> int: - """Return if caller argument (actual) is compatible with overloaded signature arg (formal). - - Return a similarity level: - 0: no match - 1: actual is compatible, but only using type promotions (e.g. int vs float) - 2: actual is compatible without type promotions (e.g. int vs object) - - The distinction is important in cases where multiple overload items match. We want - give priority to higher similarity matches. - """ - # Replace type variables with their upper bounds. Overloading - # resolution is based on runtime behavior which erases type - # parameters, so no need to handle type variables occurring within - # a type. - if isinstance(actual, TypeVarType): - actual = actual.erase_to_union_or_bound() - if isinstance(formal, TypeVarType): - formal = formal.erase_to_union_or_bound() - if (isinstance(actual, UninhabitedType) or isinstance(actual, AnyType) or - isinstance(formal, AnyType) or isinstance(formal, CallableType) or - (isinstance(actual, Instance) and actual.type.fallback_to_any)): - # These could match anything at runtime. - return 2 - if not experiments.STRICT_OPTIONAL and isinstance(actual, NoneTyp): - # NoneTyp matches anything if we're not doing strict Optional checking - return 2 - if isinstance(actual, UnionType): - return max(overload_arg_similarity(item, formal) - for item in actual.items) - if isinstance(formal, UnionType): - return max(overload_arg_similarity(actual, item) - for item in formal.items) - if isinstance(formal, TypeType): - if isinstance(actual, TypeType): - # Since Type[T] is covariant, check if actual = Type[A] is - # a subtype of formal = Type[F]. - return overload_arg_similarity(actual.item, formal.item) - elif isinstance(actual, CallableType) and actual.is_type_obj(): - # Check if the actual is a constructor of some sort. - # Note that this is this unsound, since we don't check the __init__ signature. - return overload_arg_similarity(actual.ret_type, formal.item) - else: - return 0 - if isinstance(formal, Instance): - if isinstance(actual, CallableType): - actual = actual.fallback - if isinstance(actual, Overloaded): - actual = actual.items()[0].fallback - if isinstance(actual, TupleType): - actual = actual.fallback - if isinstance(actual, Instance): - # First perform a quick check (as an optimization) and fall back to generic - # subtyping algorithm if type promotions are possible (e.g., int vs. float). - if formal.type in actual.type.mro: - return 2 - elif actual.type._promote and is_subtype(actual, formal): - return 1 - else: - return 0 - elif isinstance(actual, TypeType): - if formal.type.fullname() in {"builtins.object", "builtins.type"}: - return 2 - else: - return 0 - else: - return 0 - if isinstance(actual, UnboundType) or isinstance(formal, UnboundType): - # Either actual or formal is the result of an error; shut up. - return 2 - # Fall back to a conservative equality check for the remaining kinds of type. - return 2 if is_same_type(erasetype.erase_type(actual), erasetype.erase_type(formal)) else 0 - - -def freshen_generic_callable(callee: CallableType) -> CallableType: - tvdefs = [] - tvmap = {} # type: Dict[TypeVarId, Type] - for v in callee.variables: - tvdef = TypeVarDef.new_unification_variable(v) - tvdefs.append(tvdef) - tvmap[v.id] = TypeVarType(tvdef) - - return cast(CallableType, expand_type(callee, tvmap)).copy_modified(variables=tvdefs) +"""Expression type checker. This file is conceptually part of TypeChecker.""" + +from typing import cast, Dict, List, Tuple, Callable, Union, Optional + +from mypy.types import ( + Type, AnyType, CallableType, Overloaded, NoneTyp, Void, TypeVarDef, + TupleType, Instance, TypeVarId, TypeVarType, ErasedType, UnionType, + PartialType, DeletedType, UnboundType, UninhabitedType, TypeType +) +from mypy.nodes import ( + NameExpr, RefExpr, Var, FuncDef, OverloadedFuncDef, TypeInfo, CallExpr, + Node, MemberExpr, IntExpr, StrExpr, BytesExpr, UnicodeExpr, FloatExpr, + OpExpr, UnaryExpr, IndexExpr, CastExpr, RevealTypeExpr, TypeApplication, ListExpr, + TupleExpr, DictExpr, FuncExpr, SuperExpr, SliceExpr, Context, + ListComprehension, GeneratorExpr, SetExpr, MypyFile, Decorator, + ConditionalExpr, ComparisonExpr, TempNode, SetComprehension, + DictionaryComprehension, ComplexExpr, EllipsisExpr, + TypeAliasExpr, BackquoteExpr, ARG_POS, ARG_NAMED, ARG_STAR2 +) +from mypy.nodes import function_type +from mypy import nodes +import mypy.checker +from mypy import types +from mypy.sametypes import is_same_type +from mypy.erasetype import replace_meta_vars +from mypy.messages import MessageBuilder +from mypy import messages +from mypy.infer import infer_type_arguments, infer_function_type_arguments +from mypy import join +from mypy.subtypes import is_subtype, is_equivalent +from mypy import applytype +from mypy import erasetype +from mypy.checkmember import analyze_member_access, type_object_type +from mypy.semanal import self_type +from mypy.constraints import get_actual_type +from mypy.checkstrformat import StringFormatterChecker +from mypy.expandtype import expand_type +import mypy.checkexpr + +from mypy import experiments + +# Type of callback user for checking individual function arguments. See +# check_args() below for details. +ArgChecker = Callable[[Type, Type, int, Type, int, int, CallableType, Context, MessageBuilder], + None] + + +class Finished(Exception): + """Raised if we can terminate overload argument check early (no match).""" + + +class ExpressionChecker: + """Expression type checker. + + This class works closely together with checker.TypeChecker. + """ + + # Some services are provided by a TypeChecker instance. + chk = None # type: mypy.checker.TypeChecker + # This is shared with TypeChecker, but stored also here for convenience. + msg = None # type: MessageBuilder + + strfrm_checker = None # type: mypy.checkstrformat.StringFormatterChecker + + def __init__(self, + chk: 'mypy.checker.TypeChecker', + msg: MessageBuilder) -> None: + """Construct an expression type checker.""" + self.chk = chk + self.msg = msg + self.strfrm_checker = mypy.checkexpr.StringFormatterChecker(self, self.chk, self.msg) + + def visit_name_expr(self, e: NameExpr) -> Type: + """Type check a name expression. + + It can be of any kind: local, member or global. + """ + result = self.analyze_ref_expr(e) + return self.chk.narrow_type_from_binder(e, result) + + def analyze_ref_expr(self, e: RefExpr, lvalue: bool = False) -> Type: + result = None # type: Type + node = e.node + if isinstance(node, Var): + # Variable reference. + result = self.analyze_var_ref(node, e) + if isinstance(result, PartialType): + if result.type is None: + # 'None' partial type. It has a well-defined type. In an lvalue context + # we want to preserve the knowledge of it being a partial type. + if not lvalue: + result = NoneTyp() + else: + partial_types = self.chk.find_partial_types(node) + if partial_types is not None and not self.chk.current_node_deferred: + context = partial_types[node] + self.msg.fail(messages.NEED_ANNOTATION_FOR_VAR, context) + result = AnyType() + elif isinstance(node, FuncDef): + # Reference to a global function. + result = function_type(node, self.named_type('builtins.function')) + elif isinstance(node, OverloadedFuncDef): + result = node.type + elif isinstance(node, TypeInfo): + # Reference to a type object. + result = type_object_type(node, self.named_type) + elif isinstance(node, MypyFile): + # Reference to a module object. + result = self.named_type('builtins.module') + elif isinstance(node, Decorator): + result = self.analyze_var_ref(node.var, e) + else: + # Unknown reference; use any type implicitly to avoid + # generating extra type errors. + result = AnyType() + return result + + def analyze_var_ref(self, var: Var, context: Context) -> Type: + if not var.type: + if not var.is_ready and self.chk.typing_mode_full(): + self.chk.handle_cannot_determine_type(var.name(), context) + # Implicit 'Any' type. + return AnyType() + else: + # Look up local type of variable with type (inferred or explicit). + val = self.chk.binder.get(var) + if val is None: + return var.type + else: + return val + + def visit_call_expr(self, e: CallExpr) -> Type: + """Type check a call expression.""" + if e.analyzed: + # It's really a special form that only looks like a call. + return self.accept(e.analyzed, self.chk.type_context[-1]) + self.try_infer_partial_type(e) + callee_type = self.accept(e.callee) + if (self.chk.options.disallow_untyped_calls and + self.chk.typing_mode_full() and + isinstance(callee_type, CallableType) + and callee_type.implicit): + return self.msg.untyped_function_call(callee_type, e) + return self.check_call_expr_with_callee_type(callee_type, e) + + # Types and methods that can be used to infer partial types. + item_args = {'builtins.list': ['append'], + 'builtins.set': ['add', 'discard'], + } + container_args = {'builtins.list': {'extend': ['builtins.list']}, + 'builtins.dict': {'update': ['builtins.dict']}, + 'builtins.set': {'update': ['builtins.set', 'builtins.list']}, + } + + def try_infer_partial_type(self, e: CallExpr) -> None: + if isinstance(e.callee, MemberExpr) and isinstance(e.callee.expr, RefExpr): + var = cast(Var, e.callee.expr.node) + partial_types = self.chk.find_partial_types(var) + if partial_types is not None and not self.chk.current_node_deferred: + partial_type = cast(PartialType, var.type) + if partial_type is None or partial_type.type is None: + # A partial None type -> can't infer anything. + return + typename = partial_type.type.fullname() + methodname = e.callee.name + # Sometimes we can infer a full type for a partial List, Dict or Set type. + # TODO: Don't infer argument expression twice. + if (typename in self.item_args and methodname in self.item_args[typename] + and e.arg_kinds == [ARG_POS]): + item_type = self.accept(e.args[0]) + full_item_type = UnionType.make_simplified_union( + [item_type, partial_type.inner_types[0]]) + if mypy.checker.is_valid_inferred_type(full_item_type): + var.type = self.chk.named_generic_type(typename, [full_item_type]) + del partial_types[var] + elif (typename in self.container_args + and methodname in self.container_args[typename] + and e.arg_kinds == [ARG_POS]): + arg_type = self.accept(e.args[0]) + if isinstance(arg_type, Instance): + arg_typename = arg_type.type.fullname() + if arg_typename in self.container_args[typename][methodname]: + full_item_types = [ + UnionType.make_simplified_union([item_type, prev_type]) + for item_type, prev_type + in zip(arg_type.args, partial_type.inner_types) + ] + if all(mypy.checker.is_valid_inferred_type(item_type) + for item_type in full_item_types): + var.type = self.chk.named_generic_type(typename, + list(full_item_types)) + del partial_types[var] + + def check_call_expr_with_callee_type(self, callee_type: Type, + e: CallExpr) -> Type: + """Type check call expression. + + The given callee type overrides the type of the callee + expression. + """ + return self.check_call(callee_type, e.args, e.arg_kinds, e, + e.arg_names, callable_node=e.callee)[0] + + def check_call(self, callee: Type, args: List[Node], + arg_kinds: List[int], context: Context, + arg_names: List[str] = None, + callable_node: Node = None, + arg_messages: MessageBuilder = None) -> Tuple[Type, Type]: + """Type check a call. + + Also infer type arguments if the callee is a generic function. + + Return (result type, inferred callee type). + + Arguments: + callee: type of the called value + args: actual argument expressions + arg_kinds: contains nodes.ARG_* constant for each argument in args + describing whether the argument is positional, *arg, etc. + arg_names: names of arguments (optional) + callable_node: associate the inferred callable type to this node, + if specified + arg_messages: TODO + """ + arg_messages = arg_messages or self.msg + if isinstance(callee, CallableType): + if callee.is_concrete_type_obj() and callee.type_object().is_abstract: + type = callee.type_object() + self.msg.cannot_instantiate_abstract_class( + callee.type_object().name(), type.abstract_attributes, + context) + + formal_to_actual = map_actuals_to_formals( + arg_kinds, arg_names, + callee.arg_kinds, callee.arg_names, + lambda i: self.accept(args[i])) + + if callee.is_generic(): + callee = freshen_generic_callable(callee) + callee = self.infer_function_type_arguments_using_context( + callee, context) + callee = self.infer_function_type_arguments( + callee, args, arg_kinds, formal_to_actual, context) + + arg_types = self.infer_arg_types_in_context2( + callee, args, arg_kinds, formal_to_actual) + + self.check_argument_count(callee, arg_types, arg_kinds, + arg_names, formal_to_actual, context, self.msg) + + self.check_argument_types(arg_types, arg_kinds, callee, + formal_to_actual, context, + messages=arg_messages) + + if (callee.is_type_obj() and (len(arg_types) == 1) + and is_equivalent(callee.ret_type, self.named_type('builtins.type'))): + callee = callee.copy_modified(ret_type=TypeType(arg_types[0])) + + if callable_node: + # Store the inferred callable type. + self.chk.store_type(callable_node, callee) + return callee.ret_type, callee + elif isinstance(callee, Overloaded): + # Type check arguments in empty context. They will be checked again + # later in a context derived from the signature; these types are + # only used to pick a signature variant. + self.msg.disable_errors() + arg_types = self.infer_arg_types_in_context(None, args) + self.msg.enable_errors() + + target = self.overload_call_target(arg_types, arg_kinds, arg_names, + callee, context, + messages=arg_messages) + return self.check_call(target, args, arg_kinds, context, arg_names, + arg_messages=arg_messages) + elif isinstance(callee, AnyType) or self.chk.typing_mode_none(): + self.infer_arg_types_in_context(None, args) + return AnyType(), AnyType() + elif isinstance(callee, UnionType): + self.msg.disable_type_names += 1 + results = [self.check_call(subtype, args, arg_kinds, context, arg_names, + arg_messages=arg_messages) + for subtype in callee.items] + self.msg.disable_type_names -= 1 + return (UnionType.make_simplified_union([res[0] for res in results]), + callee) + elif isinstance(callee, Instance): + call_function = analyze_member_access('__call__', callee, context, + False, False, self.named_type, self.not_ready_callback, + self.msg) + return self.check_call(call_function, args, arg_kinds, context, arg_names, + callable_node, arg_messages) + elif isinstance(callee, TypeVarType): + return self.check_call(callee.upper_bound, args, arg_kinds, context, arg_names, + callable_node, arg_messages) + elif isinstance(callee, TypeType): + # Pass the original Type[] as context since that's where errors should go. + item = self.analyze_type_type_callee(callee.item, callee) + return self.check_call(item, args, arg_kinds, context, arg_names, + callable_node, arg_messages) + else: + return self.msg.not_callable(callee, context), AnyType() + + def analyze_type_type_callee(self, item: Type, context: Context) -> Type: + """Analyze the callee X in X(...) where X is Type[item]. + + Return a Y that we can pass to check_call(Y, ...). + """ + if isinstance(item, AnyType): + return AnyType() + if isinstance(item, Instance): + return type_object_type(item.type, self.named_type) + if isinstance(item, UnionType): + return UnionType([self.analyze_type_type_callee(item, context) + for item in item.items], item.line) + if isinstance(item, TypeVarType): + # Pretend we're calling the typevar's upper bound, + # i.e. its constructor (a poor approximation for reality, + # but better than AnyType...), but replace the return type + # with typevar. + callee = self.analyze_type_type_callee(item.upper_bound, context) + if isinstance(callee, CallableType): + if callee.is_generic(): + callee = None + else: + callee = callee.copy_modified(ret_type=item) + elif isinstance(callee, Overloaded): + if callee.items()[0].is_generic(): + callee = None + else: + callee = Overloaded([c.copy_modified(ret_type=item) + for c in callee.items()]) + if callee: + return callee + + self.msg.unsupported_type_type(item, context) + return AnyType() + + def infer_arg_types_in_context(self, callee: Optional[CallableType], + args: List[Node]) -> List[Type]: + """Infer argument expression types using a callable type as context. + + For example, if callee argument 2 has type List[int], infer the + argument expression with List[int] type context. + """ + # TODO Always called with callee as None, i.e. empty context. + res = [] # type: List[Type] + + fixed = len(args) + if callee: + fixed = min(fixed, callee.max_fixed_args()) + + arg_type = None # type: Type + ctx = None # type: Type + for i, arg in enumerate(args): + if i < fixed: + if callee and i < len(callee.arg_types): + ctx = callee.arg_types[i] + arg_type = self.accept(arg, ctx) + else: + if callee and callee.is_var_arg: + arg_type = self.accept(arg, callee.arg_types[-1]) + else: + arg_type = self.accept(arg) + if has_erased_component(arg_type): + res.append(NoneTyp()) + else: + res.append(arg_type) + return res + + def infer_arg_types_in_context2( + self, callee: CallableType, args: List[Node], arg_kinds: List[int], + formal_to_actual: List[List[int]]) -> List[Type]: + """Infer argument expression types using a callable type as context. + + For example, if callee argument 2 has type List[int], infer the + argument expression with List[int] type context. + + Returns the inferred types of *actual arguments*. + """ + res = [None] * len(args) # type: List[Type] + + for i, actuals in enumerate(formal_to_actual): + for ai in actuals: + if arg_kinds[ai] not in (nodes.ARG_STAR, nodes.ARG_STAR2): + res[ai] = self.accept(args[ai], callee.arg_types[i]) + + # Fill in the rest of the argument types. + for i, t in enumerate(res): + if not t: + res[i] = self.accept(args[i]) + return res + + def infer_function_type_arguments_using_context( + self, callable: CallableType, error_context: Context) -> CallableType: + """Unify callable return type to type context to infer type vars. + + For example, if the return type is set[t] where 't' is a type variable + of callable, and if the context is set[int], return callable modified + by substituting 't' with 'int'. + """ + ctx = self.chk.type_context[-1] + if not ctx: + return callable + # The return type may have references to type metavariables that + # we are inferring right now. We must consider them as indeterminate + # and they are not potential results; thus we replace them with the + # special ErasedType type. On the other hand, class type variables are + # valid results. + erased_ctx = replace_meta_vars(ctx, ErasedType()) + ret_type = callable.ret_type + if isinstance(ret_type, TypeVarType): + if ret_type.values or (not isinstance(ctx, Instance) or + not cast(Instance, ctx).args): + # The return type is a type variable. If it has values, we can't easily restrict + # type inference to conform to the valid values. If it's unrestricted, we could + # infer a too general type for the type variable if we use context, and this could + # result in confusing and spurious type errors elsewhere. + # + # Give up and just use function arguments for type inference. As an exception, + # if the context is a generic instance type, actually use it as context, as + # this *seems* to usually be the reasonable thing to do. + # + # See also github issues #462 and #360. + ret_type = NoneTyp() + args = infer_type_arguments(callable.type_var_ids(), ret_type, erased_ctx) + # Only substitute non-None and non-erased types. + new_args = [] # type: List[Type] + for arg in args: + if isinstance(arg, (NoneTyp, UninhabitedType)) or has_erased_component(arg): + new_args.append(None) + else: + new_args.append(arg) + return cast(CallableType, self.apply_generic_arguments(callable, new_args, + error_context)) + + def infer_function_type_arguments(self, callee_type: CallableType, + args: List[Node], + arg_kinds: List[int], + formal_to_actual: List[List[int]], + context: Context) -> CallableType: + """Infer the type arguments for a generic callee type. + + Infer based on the types of arguments. + + Return a derived callable type that has the arguments applied. + """ + if not self.chk.typing_mode_none(): + # Disable type errors during type inference. There may be errors + # due to partial available context information at this time, but + # these errors can be safely ignored as the arguments will be + # inferred again later. + self.msg.disable_errors() + + arg_types = self.infer_arg_types_in_context2( + callee_type, args, arg_kinds, formal_to_actual) + + self.msg.enable_errors() + + arg_pass_nums = self.get_arg_infer_passes( + callee_type.arg_types, formal_to_actual, len(args)) + + pass1_args = [] # type: List[Optional[Type]] + for i, arg in enumerate(arg_types): + if arg_pass_nums[i] > 1: + pass1_args.append(None) + else: + pass1_args.append(arg) + + inferred_args = infer_function_type_arguments( + callee_type, pass1_args, arg_kinds, formal_to_actual, + strict=self.chk.typing_mode_full()) # type: List[Type] + + if 2 in arg_pass_nums: + # Second pass of type inference. + (callee_type, + inferred_args) = self.infer_function_type_arguments_pass2( + callee_type, args, arg_kinds, formal_to_actual, + inferred_args, context) + + if callee_type.special_sig == 'dict' and len(inferred_args) == 2 and ( + ARG_NAMED in arg_kinds or ARG_STAR2 in arg_kinds): + # HACK: Infer str key type for dict(...) with keyword args. The type system + # can't represent this so we special case it, as this is a pretty common + # thing. This doesn't quite work with all possible subclasses of dict + # if they shuffle type variables around, as we assume that there is a 1-1 + # correspondence with dict type variables. This is a marginal issue and + # a little tricky to fix so it's left unfixed for now. + if isinstance(inferred_args[0], (NoneTyp, UninhabitedType)): + inferred_args[0] = self.named_type('builtins.str') + elif not is_subtype(self.named_type('builtins.str'), inferred_args[0]): + self.msg.fail(messages.KEYWORD_ARGUMENT_REQUIRES_STR_KEY_TYPE, + context) + else: + # In dynamically typed functions use implicit 'Any' types for + # type variables. + inferred_args = [AnyType()] * len(callee_type.variables) + return self.apply_inferred_arguments(callee_type, inferred_args, + context) + + def infer_function_type_arguments_pass2( + self, callee_type: CallableType, + args: List[Node], + arg_kinds: List[int], + formal_to_actual: List[List[int]], + inferred_args: List[Type], + context: Context) -> Tuple[CallableType, List[Type]]: + """Perform second pass of generic function type argument inference. + + The second pass is needed for arguments with types such as Callable[[T], S], + where both T and S are type variables, when the actual argument is a + lambda with inferred types. The idea is to infer the type variable T + in the first pass (based on the types of other arguments). This lets + us infer the argument and return type of the lambda expression and + thus also the type variable S in this second pass. + + Return (the callee with type vars applied, inferred actual arg types). + """ + # None or erased types in inferred types mean that there was not enough + # information to infer the argument. Replace them with None values so + # that they are not applied yet below. + for i, arg in enumerate(inferred_args): + if isinstance(arg, (NoneTyp, UninhabitedType)) or has_erased_component(arg): + inferred_args[i] = None + + callee_type = cast(CallableType, self.apply_generic_arguments( + callee_type, inferred_args, context)) + arg_types = self.infer_arg_types_in_context2( + callee_type, args, arg_kinds, formal_to_actual) + + inferred_args = infer_function_type_arguments( + callee_type, arg_types, arg_kinds, formal_to_actual) + + return callee_type, inferred_args + + def get_arg_infer_passes(self, arg_types: List[Type], + formal_to_actual: List[List[int]], + num_actuals: int) -> List[int]: + """Return pass numbers for args for two-pass argument type inference. + + For each actual, the pass number is either 1 (first pass) or 2 (second + pass). + + Two-pass argument type inference primarily lets us infer types of + lambdas more effectively. + """ + res = [1] * num_actuals + for i, arg in enumerate(arg_types): + if arg.accept(ArgInferSecondPassQuery()): + for j in formal_to_actual[i]: + res[j] = 2 + return res + + def apply_inferred_arguments(self, callee_type: CallableType, + inferred_args: List[Type], + context: Context) -> CallableType: + """Apply inferred values of type arguments to a generic function. + + Inferred_args contains the values of function type arguments. + """ + # Report error if some of the variables could not be solved. In that + # case assume that all variables have type Any to avoid extra + # bogus error messages. + for i, inferred_type in enumerate(inferred_args): + if not inferred_type or has_erased_component(inferred_type): + # Could not infer a non-trivial type for a type variable. + self.msg.could_not_infer_type_arguments( + callee_type, i + 1, context) + inferred_args = [AnyType()] * len(inferred_args) + # Apply the inferred types to the function type. In this case the + # return type must be CallableType, since we give the right number of type + # arguments. + return cast(CallableType, self.apply_generic_arguments(callee_type, + inferred_args, context)) + + def check_argument_count(self, callee: CallableType, actual_types: List[Type], + actual_kinds: List[int], actual_names: List[str], + formal_to_actual: List[List[int]], + context: Context, + messages: Optional[MessageBuilder]) -> bool: + """Check that there is a value for all required arguments to a function. + + Also check that there are no duplicate values for arguments. Report found errors + using 'messages' if it's not None. + + Return False if there were any errors. Otherwise return True + """ + # TODO(jukka): We could return as soon as we find an error if messages is None. + formal_kinds = callee.arg_kinds + + # Collect list of all actual arguments matched to formal arguments. + all_actuals = [] # type: List[int] + for actuals in formal_to_actual: + all_actuals.extend(actuals) + + is_unexpected_arg_error = False # Keep track of errors to avoid duplicate errors. + ok = True # False if we've found any error. + for i, kind in enumerate(actual_kinds): + if i not in all_actuals and ( + kind != nodes.ARG_STAR or + not is_empty_tuple(actual_types[i])): + # Extra actual: not matched by a formal argument. + ok = False + if kind != nodes.ARG_NAMED: + if messages: + messages.too_many_arguments(callee, context) + else: + if messages: + messages.unexpected_keyword_argument( + callee, actual_names[i], context) + is_unexpected_arg_error = True + elif kind == nodes.ARG_STAR and ( + nodes.ARG_STAR not in formal_kinds): + actual_type = actual_types[i] + if isinstance(actual_type, TupleType): + if all_actuals.count(i) < len(actual_type.items): + # Too many tuple items as some did not match. + if messages: + messages.too_many_arguments(callee, context) + ok = False + # *args can be applied even if the function takes a fixed + # number of positional arguments. This may succeed at runtime. + + for i, kind in enumerate(formal_kinds): + if kind == nodes.ARG_POS and (not formal_to_actual[i] and + not is_unexpected_arg_error): + # No actual for a mandatory positional formal. + if messages: + messages.too_few_arguments(callee, context, actual_names) + ok = False + elif kind in [nodes.ARG_POS, nodes.ARG_OPT, + nodes.ARG_NAMED] and is_duplicate_mapping( + formal_to_actual[i], actual_kinds): + if (self.chk.typing_mode_full() or + isinstance(actual_types[formal_to_actual[i][0]], TupleType)): + if messages: + messages.duplicate_argument_value(callee, i, context) + ok = False + elif (kind == nodes.ARG_NAMED and formal_to_actual[i] and + actual_kinds[formal_to_actual[i][0]] != nodes.ARG_NAMED): + # Positional argument when expecting a keyword argument. + if messages: + messages.too_many_positional_arguments(callee, context) + ok = False + return ok + + def check_argument_types(self, arg_types: List[Type], arg_kinds: List[int], + callee: CallableType, + formal_to_actual: List[List[int]], + context: Context, + messages: MessageBuilder = None, + check_arg: ArgChecker = None) -> None: + """Check argument types against a callable type. + + Report errors if the argument types are not compatible. + """ + messages = messages or self.msg + check_arg = check_arg or self.check_arg + # Keep track of consumed tuple *arg items. + tuple_counter = [0] + for i, actuals in enumerate(formal_to_actual): + for actual in actuals: + arg_type = arg_types[actual] + # Check that a *arg is valid as varargs. + if (arg_kinds[actual] == nodes.ARG_STAR and + not self.is_valid_var_arg(arg_type)): + messages.invalid_var_arg(arg_type, context) + if (arg_kinds[actual] == nodes.ARG_STAR2 and + not self.is_valid_keyword_var_arg(arg_type)): + messages.invalid_keyword_var_arg(arg_type, context) + # Get the type of an individual actual argument (for *args + # and **args this is the item type, not the collection type). + actual_type = get_actual_type(arg_type, arg_kinds[actual], + tuple_counter) + check_arg(actual_type, arg_type, arg_kinds[actual], + callee.arg_types[i], + actual + 1, i + 1, callee, context, messages) + + # There may be some remaining tuple varargs items that haven't + # been checked yet. Handle them. + if (callee.arg_kinds[i] == nodes.ARG_STAR and + arg_kinds[actual] == nodes.ARG_STAR and + isinstance(arg_types[actual], TupleType)): + tuplet = cast(TupleType, arg_types[actual]) + while tuple_counter[0] < len(tuplet.items): + actual_type = get_actual_type(arg_type, + arg_kinds[actual], + tuple_counter) + check_arg(actual_type, arg_type, arg_kinds[actual], + callee.arg_types[i], + actual + 1, i + 1, callee, context, messages) + + def check_arg(self, caller_type: Type, original_caller_type: Type, + caller_kind: int, + callee_type: Type, n: int, m: int, callee: CallableType, + context: Context, messages: MessageBuilder) -> None: + """Check the type of a single argument in a call.""" + if isinstance(caller_type, Void): + messages.does_not_return_value(caller_type, context) + elif isinstance(caller_type, DeletedType): + messages.deleted_as_rvalue(caller_type, context) + elif not is_subtype(caller_type, callee_type): + messages.incompatible_argument(n, m, callee, original_caller_type, + caller_kind, context) + + def overload_call_target(self, arg_types: List[Type], arg_kinds: List[int], + arg_names: List[str], + overload: Overloaded, context: Context, + messages: MessageBuilder = None) -> Type: + """Infer the correct overload item to call with given argument types. + + The return value may be CallableType or AnyType (if an unique item + could not be determined). + """ + messages = messages or self.msg + # TODO: For overlapping signatures we should try to get a more precise + # result than 'Any'. + match = [] # type: List[CallableType] + best_match = 0 + for typ in overload.items(): + similarity = self.erased_signature_similarity(arg_types, arg_kinds, arg_names, + typ, context=context) + if similarity > 0 and similarity >= best_match: + if (match and not is_same_type(match[-1].ret_type, + typ.ret_type) and + not mypy.checker.is_more_precise_signature( + match[-1], typ)): + # Ambiguous return type. Either the function overload is + # overlapping (which we don't handle very well here) or the + # caller has provided some Any argument types; in either + # case we'll fall back to Any. It's okay to use Any types + # in calls. + # + # Overlapping overload items are generally fine if the + # overlapping is only possible when there is multiple + # inheritance, as this is rare. See docstring of + # mypy.meet.is_overlapping_types for more about this. + # + # Note that there is no ambiguity if the items are + # covariant in both argument types and return types with + # respect to type precision. We'll pick the best/closest + # match. + # + # TODO: Consider returning a union type instead if the + # overlapping is NOT due to Any types? + return AnyType() + else: + match.append(typ) + best_match = max(best_match, similarity) + if not match: + messages.no_variant_matches_arguments(overload, arg_types, context) + return AnyType() + else: + if len(match) == 1: + return match[0] + else: + # More than one signature matches. Pick the first *non-erased* + # matching signature, or default to the first one if none + # match. + for m in match: + if self.match_signature_types(arg_types, arg_kinds, arg_names, m, + context=context): + return m + return match[0] + + def erased_signature_similarity(self, arg_types: List[Type], arg_kinds: List[int], + arg_names: List[str], callee: CallableType, + context: Context) -> int: + """Determine whether arguments could match the signature at runtime. + + Return similarity level (0 = no match, 1 = can match, 2 = non-promotion match). See + overload_arg_similarity for a discussion of similarity levels. + """ + formal_to_actual = map_actuals_to_formals(arg_kinds, + arg_names, + callee.arg_kinds, + callee.arg_names, + lambda i: arg_types[i]) + + if not self.check_argument_count(callee, arg_types, arg_kinds, arg_names, + formal_to_actual, None, None): + # Too few or many arguments -> no match. + return 0 + + similarity = 2 + + def check_arg(caller_type: Type, original_caller_type: Type, caller_kind: int, + callee_type: Type, n: int, m: int, callee: CallableType, + context: Context, messages: MessageBuilder) -> None: + nonlocal similarity + similarity = min(similarity, + overload_arg_similarity(caller_type, callee_type)) + if similarity == 0: + # No match -- exit early since none of the remaining work can change + # the result. + raise Finished + + try: + self.check_argument_types(arg_types, arg_kinds, callee, formal_to_actual, + context=context, check_arg=check_arg) + except Finished: + pass + + return similarity + + def match_signature_types(self, arg_types: List[Type], arg_kinds: List[int], + arg_names: List[str], callee: CallableType, + context: Context) -> bool: + """Determine whether arguments types match the signature. + + Assume that argument counts are compatible. + + Return True if arguments match. + """ + formal_to_actual = map_actuals_to_formals(arg_kinds, + arg_names, + callee.arg_kinds, + callee.arg_names, + lambda i: arg_types[i]) + ok = True + + def check_arg(caller_type: Type, original_caller_type: Type, caller_kind: int, + callee_type: Type, n: int, m: int, callee: CallableType, + context: Context, messages: MessageBuilder) -> None: + nonlocal ok + if not is_subtype(caller_type, callee_type): + ok = False + + self.check_argument_types(arg_types, arg_kinds, callee, formal_to_actual, + context=context, check_arg=check_arg) + return ok + + def apply_generic_arguments(self, callable: CallableType, types: List[Type], + context: Context) -> Type: + """Simple wrapper around mypy.applytype.apply_generic_arguments.""" + return applytype.apply_generic_arguments(callable, types, self.msg, context) + + def apply_generic_arguments2(self, overload: Overloaded, types: List[Type], + context: Context) -> Type: + items = [] # type: List[CallableType] + for item in overload.items(): + applied = self.apply_generic_arguments(item, types, context) + if isinstance(applied, CallableType): + items.append(applied) + else: + # There was an error. + return AnyType() + return Overloaded(items) + + def visit_member_expr(self, e: MemberExpr) -> Type: + """Visit member expression (of form e.id).""" + result = self.analyze_ordinary_member_access(e, False) + return self.chk.narrow_type_from_binder(e, result) + + def analyze_ordinary_member_access(self, e: MemberExpr, + is_lvalue: bool) -> Type: + """Analyse member expression or member lvalue.""" + if e.kind is not None: + # This is a reference to a module attribute. + return self.analyze_ref_expr(e) + else: + # This is a reference to a non-module attribute. + return analyze_member_access(e.name, self.accept(e.expr), e, + is_lvalue, False, + self.named_type, self.not_ready_callback, self.msg) + + def analyze_external_member_access(self, member: str, base_type: Type, + context: Context) -> Type: + """Analyse member access that is external, i.e. it cannot + refer to private definitions. Return the result type. + """ + # TODO remove; no private definitions in mypy + return analyze_member_access(member, base_type, context, False, False, + self.named_type, self.not_ready_callback, self.msg) + + def visit_int_expr(self, e: IntExpr) -> Type: + """Type check an integer literal (trivial).""" + return self.named_type('builtins.int') + + def visit_str_expr(self, e: StrExpr) -> Type: + """Type check a string literal (trivial).""" + return self.named_type('builtins.str') + + def visit_bytes_expr(self, e: BytesExpr) -> Type: + """Type check a bytes literal (trivial).""" + return self.named_type('builtins.bytes') + + def visit_unicode_expr(self, e: UnicodeExpr) -> Type: + """Type check a unicode literal (trivial).""" + return self.named_type('builtins.unicode') + + def visit_float_expr(self, e: FloatExpr) -> Type: + """Type check a float literal (trivial).""" + return self.named_type('builtins.float') + + def visit_complex_expr(self, e: ComplexExpr) -> Type: + """Type check a complex literal.""" + return self.named_type('builtins.complex') + + def visit_ellipsis(self, e: EllipsisExpr) -> Type: + """Type check '...'.""" + if self.chk.options.python_version[0] >= 3: + return self.named_type('builtins.ellipsis') + else: + # '...' is not valid in normal Python 2 code, but it can + # be used in stubs. The parser makes sure that we only + # get this far if we are in a stub, and we can safely + # return 'object' as ellipsis is special cased elsewhere. + # The builtins.ellipsis type does not exist in Python 2. + return self.named_type('builtins.object') + + def visit_op_expr(self, e: OpExpr) -> Type: + """Type check a binary operator expression.""" + if e.op == 'and' or e.op == 'or': + return self.check_boolean_op(e, e) + if e.op == '*' and isinstance(e.left, ListExpr): + # Expressions of form [...] * e get special type inference. + return self.check_list_multiply(e) + if e.op == '%' and isinstance(e.left, (StrExpr, BytesExpr)): + return self.strfrm_checker.check_str_interpolation(cast(StrExpr, e.left), e.right) + left_type = self.accept(e.left) + + if e.op in nodes.op_methods: + method = self.get_operator_method(e.op) + result, method_type = self.check_op(method, left_type, e.right, e, + allow_reverse=True) + e.method_type = method_type + return result + else: + raise RuntimeError('Unknown operator {}'.format(e.op)) + + def visit_comparison_expr(self, e: ComparisonExpr) -> Type: + """Type check a comparison expression. + + Comparison expressions are type checked consecutive-pair-wise + That is, 'a < b > c == d' is check as 'a < b and b > c and c == d' + """ + result = None # type: mypy.types.Type + + # Check each consecutive operand pair and their operator + for left, right, operator in zip(e.operands, e.operands[1:], e.operators): + left_type = self.accept(left) + + method_type = None # type: mypy.types.Type + + if operator == 'in' or operator == 'not in': + right_type = self.accept(right) # TODO only evaluate if needed + + # Keep track of whether we get type check errors (these won't be reported, they + # are just to verify whether something is valid typing wise). + local_errors = self.msg.copy() + local_errors.disable_count = 0 + sub_result, method_type = self.check_op_local('__contains__', right_type, + left, e, local_errors) + if isinstance(right_type, PartialType): + # We don't really know if this is an error or not, so just shut up. + pass + elif (local_errors.is_errors() and + # is_valid_var_arg is True for any Iterable + self.is_valid_var_arg(right_type)): + itertype = self.chk.analyze_iterable_item_type(right) + method_type = CallableType( + [left_type], + [nodes.ARG_POS], + [None], + self.chk.bool_type(), + self.named_type('builtins.function')) + sub_result = self.chk.bool_type() + if not is_subtype(left_type, itertype): + self.msg.unsupported_operand_types('in', left_type, right_type, e) + else: + self.msg.add_errors(local_errors) + if operator == 'not in': + sub_result = self.chk.bool_type() + elif operator in nodes.op_methods: + method = self.get_operator_method(operator) + sub_result, method_type = self.check_op(method, left_type, right, e, + allow_reverse=True) + + elif operator == 'is' or operator == 'is not': + sub_result = self.chk.bool_type() + method_type = None + else: + raise RuntimeError('Unknown comparison operator {}'.format(operator)) + + e.method_types.append(method_type) + + # Determine type of boolean-and of result and sub_result + if result is None: + result = sub_result + else: + # TODO: check on void needed? + self.check_not_void(sub_result, e) + result = join.join_types(result, sub_result) + + return result + + def get_operator_method(self, op: str) -> str: + if op == '/' and self.chk.options.python_version[0] == 2: + # TODO also check for "from __future__ import division" + return '__div__' + else: + return nodes.op_methods[op] + + def check_op_local(self, method: str, base_type: Type, arg: Node, + context: Context, local_errors: MessageBuilder) -> Tuple[Type, Type]: + """Type check a binary operation which maps to a method call. + + Return tuple (result type, inferred operator method type). + """ + method_type = analyze_member_access(method, base_type, context, False, False, + self.named_type, self.not_ready_callback, local_errors) + return self.check_call(method_type, [arg], [nodes.ARG_POS], + context, arg_messages=local_errors) + + def check_op(self, method: str, base_type: Type, arg: Node, + context: Context, + allow_reverse: bool = False) -> Tuple[Type, Type]: + """Type check a binary operation which maps to a method call. + + Return tuple (result type, inferred operator method type). + """ + # Use a local error storage for errors related to invalid argument + # type (but NOT other errors). This error may need to be suppressed + # for operators which support __rX methods. + local_errors = self.msg.copy() + local_errors.disable_count = 0 + if not allow_reverse or self.has_member(base_type, method): + result = self.check_op_local(method, base_type, arg, context, + local_errors) + if allow_reverse: + arg_type = self.chk.type_map[arg] + if isinstance(arg_type, AnyType): + # If the right operand has type Any, we can't make any + # conjectures about the type of the result, since the + # operand could have a __r method that returns anything. + + # However, in weak mode, we do make conjectures. + if not self.chk.typing_mode_weak(): + result = AnyType(), result[1] + success = not local_errors.is_errors() + else: + result = AnyType(), AnyType() + success = False + if success or not allow_reverse or isinstance(base_type, AnyType): + # We were able to call the normal variant of the operator method, + # or there was some problem not related to argument type + # validity, or the operator has no __rX method. In any case, we + # don't need to consider the __rX method. + self.msg.add_errors(local_errors) + return result + else: + # Calling the operator method was unsuccessful. Try the __rX + # method of the other operand instead. + rmethod = self.get_reverse_op_method(method) + arg_type = self.accept(arg) + if self.has_member(arg_type, rmethod): + method_type = self.analyze_external_member_access( + rmethod, arg_type, context) + temp = TempNode(base_type) + return self.check_call(method_type, [temp], [nodes.ARG_POS], + context) + else: + # No __rX method either. Do deferred type checking to produce + # error message that we may have missed previously. + # TODO Fix type checking an expression more than once. + return self.check_op_local(method, base_type, arg, context, + self.msg) + + def get_reverse_op_method(self, method: str) -> str: + if method == '__div__' and self.chk.options.python_version[0] == 2: + return '__rdiv__' + else: + return nodes.reverse_op_methods[method] + + def check_boolean_op(self, e: OpExpr, context: Context) -> Type: + """Type check a boolean operation ('and' or 'or').""" + + # A boolean operation can evaluate to either of the operands. + + # We use the current type context to guide the type inference of of + # the left operand. We also use the left operand type to guide the type + # inference of the right operand so that expressions such as + # '[1] or []' are inferred correctly. + ctx = self.chk.type_context[-1] + left_type = self.accept(e.left, ctx) + + if e.op == 'and': + right_map, _ = \ + mypy.checker.find_isinstance_check(e.left, self.chk.type_map, + self.chk.typing_mode_weak()) + elif e.op == 'or': + _, right_map = \ + mypy.checker.find_isinstance_check(e.left, self.chk.type_map, + self.chk.typing_mode_weak()) + else: + right_map = None + + with self.chk.binder.frame_context(): + if right_map: + for var, type in right_map.items(): + self.chk.binder.push(var, type) + + right_type = self.accept(e.right, left_type) + + self.check_not_void(left_type, context) + self.check_not_void(right_type, context) + return UnionType.make_simplified_union([left_type, right_type]) + + def check_list_multiply(self, e: OpExpr) -> Type: + """Type check an expression of form '[...] * e'. + + Type inference is special-cased for this common construct. + """ + right_type = self.accept(e.right) + if is_subtype(right_type, self.named_type('builtins.int')): + # Special case: [...] * . Use the type context of the + # OpExpr, since the multiplication does not affect the type. + left_type = self.accept(e.left, context=self.chk.type_context[-1]) + else: + left_type = self.accept(e.left) + result, method_type = self.check_op('__mul__', left_type, e.right, e) + e.method_type = method_type + return result + + def visit_unary_expr(self, e: UnaryExpr) -> Type: + """Type check an unary operation ('not', '-', '+' or '~').""" + operand_type = self.accept(e.expr) + op = e.op + if op == 'not': + self.check_not_void(operand_type, e) + result = self.chk.bool_type() # type: Type + elif op == '-': + method_type = self.analyze_external_member_access('__neg__', + operand_type, e) + result, method_type = self.check_call(method_type, [], [], e) + e.method_type = method_type + elif op == '+': + method_type = self.analyze_external_member_access('__pos__', + operand_type, e) + result, method_type = self.check_call(method_type, [], [], e) + e.method_type = method_type + else: + assert op == '~', "unhandled unary operator" + method_type = self.analyze_external_member_access('__invert__', + operand_type, e) + result, method_type = self.check_call(method_type, [], [], e) + e.method_type = method_type + return result + + def visit_index_expr(self, e: IndexExpr) -> Type: + """Type check an index expression (base[index]). + + It may also represent type application. + """ + result = self.visit_index_expr_helper(e) + return self.chk.narrow_type_from_binder(e, result) + + def visit_index_expr_helper(self, e: IndexExpr) -> Type: + if e.analyzed: + # It's actually a type application. + return self.accept(e.analyzed) + left_type = self.accept(e.base) + if isinstance(left_type, TupleType) and self.chk.typing_mode_full(): + # Special case for tuples. They support indexing only by integer + # literals. (Except in weak type checking mode.) + index = e.index + if isinstance(index, SliceExpr): + return self.visit_tuple_slice_helper(left_type, index) + + ok = False + if isinstance(index, IntExpr): + n = index.value + ok = True + elif isinstance(index, UnaryExpr): + if index.op == '-': + operand = index.expr + if isinstance(operand, IntExpr): + n = len(left_type.items) - operand.value + ok = True + if ok: + if n >= 0 and n < len(left_type.items): + return left_type.items[n] + else: + self.chk.fail(messages.TUPLE_INDEX_OUT_OF_RANGE, e) + return AnyType() + else: + self.chk.fail(messages.TUPLE_INDEX_MUST_BE_AN_INT_LITERAL, e) + return AnyType() + else: + result, method_type = self.check_op('__getitem__', left_type, e.index, e) + e.method_type = method_type + return result + + def visit_tuple_slice_helper(self, left_type: TupleType, slic: SliceExpr): + begin = None # type: int + end = None # type: int + stride = None # type:int + + if slic.begin_index: + begin = self._get_value(slic.begin_index) + if begin is None: + self.chk.fail( + messages.TUPLE_SLICE_MUST_BE_AN_INT_LITERAL, + slic.begin_index) + return AnyType() + + if slic.end_index: + end = self._get_value(slic.end_index) + if end is None: + self.chk.fail( + messages.TUPLE_SLICE_MUST_BE_AN_INT_LITERAL, + slic.end_index) + return AnyType() + + if slic.stride: + stride = self._get_value(slic.stride) + if stride is None: + self.chk.fail( + messages.TUPLE_SLICE_MUST_BE_AN_INT_LITERAL, + slic.stride) + return AnyType() + + return left_type.slice(begin, stride, end) + + def _get_value(self, index: Node) -> Optional[int]: + if isinstance(index, IntExpr): + return index.value + elif isinstance(index, UnaryExpr): + if index.op == '-': + operand = index.expr + if isinstance(operand, IntExpr): + return -1 * operand.value + return None + + def visit_cast_expr(self, expr: CastExpr) -> Type: + """Type check a cast expression.""" + source_type = self.accept(expr.expr, context=AnyType()) + target_type = expr.type + if self.chk.options.warn_redundant_casts and is_same_type(source_type, target_type): + self.msg.redundant_cast(target_type, expr) + if not self.is_valid_cast(source_type, target_type): + self.msg.invalid_cast(target_type, source_type, expr) + return target_type + + def is_valid_cast(self, source_type: Type, target_type: Type) -> bool: + """Is a cast from source_type to target_type meaningful?""" + return (isinstance(target_type, AnyType) or + (not isinstance(source_type, Void) and + not isinstance(target_type, Void))) + + def visit_reveal_type_expr(self, expr: RevealTypeExpr) -> Type: + """Type check a reveal_type expression.""" + revealed_type = self.accept(expr.expr) + self.msg.reveal_type(revealed_type, expr) + return revealed_type + + def visit_type_application(self, tapp: TypeApplication) -> Type: + """Type check a type application (expr[type, ...]).""" + self.chk.fail(messages.GENERIC_TYPE_NOT_VALID_AS_EXPRESSION, tapp) + return AnyType() + + def visit_type_alias_expr(self, alias: TypeAliasExpr) -> Type: + return AnyType() + + def visit_list_expr(self, e: ListExpr) -> Type: + """Type check a list expression [...].""" + return self.check_list_or_set_expr(e.items, 'builtins.list', '', + e) + + def visit_set_expr(self, e: SetExpr) -> Type: + return self.check_list_or_set_expr(e.items, 'builtins.set', '', e) + + def check_list_or_set_expr(self, items: List[Node], fullname: str, + tag: str, context: Context) -> Type: + # Translate into type checking a generic function call. + tvdef = TypeVarDef('T', -1, [], self.chk.object_type()) + tv = TypeVarType(tvdef) + constructor = CallableType( + [tv], + [nodes.ARG_STAR], + [None], + self.chk.named_generic_type(fullname, [tv]), + self.named_type('builtins.function'), + name=tag, + variables=[tvdef]) + return self.check_call(constructor, + items, + [nodes.ARG_POS] * len(items), context)[0] + + def visit_tuple_expr(self, e: TupleExpr) -> Type: + """Type check a tuple expression.""" + ctx = None # type: TupleType + # Try to determine type context for type inference. + if isinstance(self.chk.type_context[-1], TupleType): + t = self.chk.type_context[-1] + if len(t.items) == len(e.items): + ctx = t + # Infer item types. + items = [] # type: List[Type] + for i in range(len(e.items)): + item = e.items[i] + tt = None # type: Type + if not ctx: + tt = self.accept(item) + else: + tt = self.accept(item, ctx.items[i]) + self.check_not_void(tt, e) + items.append(tt) + fallback_item = join.join_type_list(items) + return TupleType(items, self.chk.named_generic_type('builtins.tuple', [fallback_item])) + + def visit_dict_expr(self, e: DictExpr) -> Type: + # Translate into type checking a generic function call. + ktdef = TypeVarDef('KT', -1, [], self.chk.object_type()) + vtdef = TypeVarDef('VT', -2, [], self.chk.object_type()) + kt = TypeVarType(ktdef) + vt = TypeVarType(vtdef) + # The callable type represents a function like this: + # + # def (*v: Tuple[kt, vt]) -> Dict[kt, vt]: ... + constructor = CallableType( + [TupleType([kt, vt], self.named_type('builtins.tuple'))], + [nodes.ARG_STAR], + [None], + self.chk.named_generic_type('builtins.dict', [kt, vt]), + self.named_type('builtins.function'), + name='', + variables=[ktdef, vtdef]) + # Synthesize function arguments. + args = [] # type: List[Node] + for key, value in e.items: + args.append(TupleExpr([key, value])) + return self.check_call(constructor, + args, + [nodes.ARG_POS] * len(args), e)[0] + + def visit_func_expr(self, e: FuncExpr) -> Type: + """Type check lambda expression.""" + inferred_type = self.infer_lambda_type_using_context(e) + if not inferred_type: + # No useful type context. + ret_type = e.expr().accept(self.chk) + if not e.arguments: + # Form 'lambda: e'; just use the inferred return type. + return CallableType([], [], [], ret_type, self.named_type('builtins.function')) + else: + # TODO: Consider reporting an error. However, this is fine if + # we are just doing the first pass in contextual type + # inference. + return AnyType() + else: + # Type context available. + self.chk.check_func_item(e, type_override=inferred_type) + if e.expr() not in self.chk.type_map: + self.accept(e.expr()) + ret_type = self.chk.type_map[e.expr()] + if isinstance(ret_type, NoneTyp): + # For "lambda ...: None", just use type from the context. + # Important when the context is Callable[..., None] which + # really means Void. See #1425. + return inferred_type + return replace_callable_return_type(inferred_type, ret_type) + + def infer_lambda_type_using_context(self, e: FuncExpr) -> CallableType: + """Try to infer lambda expression type using context. + + Return None if could not infer type. + """ + # TODO also accept 'Any' context + ctx = self.chk.type_context[-1] + if not ctx or not isinstance(ctx, CallableType): + return None + + # The context may have function type variables in it. We replace them + # since these are the type variables we are ultimately trying to infer; + # they must be considered as indeterminate. We use ErasedType since it + # does not affect type inference results (it is for purposes like this + # only). + ctx = replace_meta_vars(ctx, ErasedType()) + + callable_ctx = cast(CallableType, ctx) + + arg_kinds = [arg.kind for arg in e.arguments] + + if callable_ctx.is_ellipsis_args: + # Fill in Any arguments to match the arguments of the lambda. + callable_ctx = callable_ctx.copy_modified( + is_ellipsis_args=False, + arg_types=[AnyType()] * len(arg_kinds), + arg_kinds=arg_kinds + ) + + if callable_ctx.arg_kinds != arg_kinds: + # Incompatible context; cannot use it to infer types. + self.chk.fail(messages.CANNOT_INFER_LAMBDA_TYPE, e) + return None + + return callable_ctx + + def visit_super_expr(self, e: SuperExpr) -> Type: + """Type check a super expression (non-lvalue).""" + t = self.analyze_super(e, False) + return t + + def analyze_super(self, e: SuperExpr, is_lvalue: bool) -> Type: + """Type check a super expression.""" + if e.info and e.info.bases: + # TODO fix multiple inheritance etc + if len(e.info.mro) < 2: + self.chk.fail('Internal error: unexpected mro for {}: {}'.format( + e.info.name(), e.info.mro), e) + return AnyType() + for base in e.info.mro[1:]: + if e.name in base.names or base == e.info.mro[-1]: + if e.info.fallback_to_any and base == e.info.mro[-1]: + # There's an undefined base class, and we're + # at the end of the chain. That's not an error. + return AnyType() + if not self.chk.typing_mode_full(): + return AnyType() + return analyze_member_access(e.name, self_type(e.info), e, + is_lvalue, True, + self.named_type, self.not_ready_callback, + self.msg, base) + else: + # Invalid super. This has been reported by the semantic analyzer. + return AnyType() + + def visit_slice_expr(self, e: SliceExpr) -> Type: + for index in [e.begin_index, e.end_index, e.stride]: + if index: + t = self.accept(index) + self.chk.check_subtype(t, self.named_type('builtins.int'), + index, messages.INVALID_SLICE_INDEX) + return self.named_type('builtins.slice') + + def visit_list_comprehension(self, e: ListComprehension) -> Type: + return self.check_generator_or_comprehension( + e.generator, 'builtins.list', '') + + def visit_set_comprehension(self, e: SetComprehension) -> Type: + return self.check_generator_or_comprehension( + e.generator, 'builtins.set', '') + + def visit_generator_expr(self, e: GeneratorExpr) -> Type: + return self.check_generator_or_comprehension(e, 'typing.Iterator', + '') + + def check_generator_or_comprehension(self, gen: GeneratorExpr, + type_name: str, + id_for_messages: str) -> Type: + """Type check a generator expression or a list comprehension.""" + self.check_for_comp(gen) + + # Infer the type of the list comprehension by using a synthetic generic + # callable type. + tvdef = TypeVarDef('T', -1, [], self.chk.object_type()) + tv = TypeVarType(tvdef) + constructor = CallableType( + [tv], + [nodes.ARG_POS], + [None], + self.chk.named_generic_type(type_name, [tv]), + self.chk.named_type('builtins.function'), + name=id_for_messages, + variables=[tvdef]) + return self.check_call(constructor, + [gen.left_expr], [nodes.ARG_POS], gen)[0] + + def visit_dictionary_comprehension(self, e: DictionaryComprehension): + """Type check a dictionary comprehension.""" + self.check_for_comp(e) + + # Infer the type of the list comprehension by using a synthetic generic + # callable type. + ktdef = TypeVarDef('KT', -1, [], self.chk.object_type()) + vtdef = TypeVarDef('VT', -2, [], self.chk.object_type()) + kt = TypeVarType(ktdef) + vt = TypeVarType(vtdef) + constructor = CallableType( + [kt, vt], + [nodes.ARG_POS, nodes.ARG_POS], + [None, None], + self.chk.named_generic_type('builtins.dict', [kt, vt]), + self.chk.named_type('builtins.function'), + name='', + variables=[ktdef, vtdef]) + return self.check_call(constructor, + [e.key, e.value], [nodes.ARG_POS, nodes.ARG_POS], e)[0] + + def check_for_comp(self, e: Union[GeneratorExpr, DictionaryComprehension]) -> None: + """Check the for_comp part of comprehensions. That is the part from 'for': + ... for x in y if z + """ + with self.chk.binder.frame_context(): + for index, sequence, conditions in zip(e.indices, e.sequences, + e.condlists): + sequence_type = self.chk.analyze_iterable_item_type(sequence) + self.chk.analyze_index_variables(index, sequence_type, e) + for condition in conditions: + self.accept(condition) + + def visit_conditional_expr(self, e: ConditionalExpr) -> Type: + cond_type = self.accept(e.cond) + self.check_not_void(cond_type, e) + ctx = self.chk.type_context[-1] + + # Gain type information from isinstance if it is there + # but only for the current expression + if_map, else_map = mypy.checker.find_isinstance_check( + e.cond, + self.chk.type_map, + self.chk.typing_mode_weak()) + + if_type = self.analyze_cond_branch(if_map, e.if_expr, context=ctx) + + if not mypy.checker.is_valid_inferred_type(if_type): + # Analyze the right branch disregarding the left branch. + else_type = self.analyze_cond_branch(else_map, e.else_expr, context=ctx) + + # If it would make a difference, re-analyze the left + # branch using the right branch's type as context. + if ctx is None or not is_equivalent(else_type, ctx): + # TODO: If it's possible that the previous analysis of + # the left branch produced errors that are avoided + # using this context, suppress those errors. + if_type = self.analyze_cond_branch(if_map, e.if_expr, context=else_type) + + else: + # Analyze the right branch in the context of the left + # branch's type. + else_type = self.analyze_cond_branch(else_map, e.else_expr, context=if_type) + + res = join.join_types(if_type, else_type) + + return res + + def analyze_cond_branch(self, map: Optional[Dict[Node, Type]], + node: Node, context: Optional[Type]) -> Type: + with self.chk.binder.frame_context(): + if map: + for var, type in map.items(): + self.chk.binder.push(var, type) + return self.accept(node, context=context) + + def visit_backquote_expr(self, e: BackquoteExpr) -> Type: + self.accept(e.expr) + return self.named_type('builtins.str') + + # + # Helpers + # + + def accept(self, node: Node, context: Type = None) -> Type: + """Type check a node. Alias for TypeChecker.accept.""" + return self.chk.accept(node, context) + + def check_not_void(self, typ: Type, context: Context) -> None: + """Generate an error if type is Void.""" + self.chk.check_not_void(typ, context) + + def is_boolean(self, typ: Type) -> bool: + """Is type compatible with bool?""" + return is_subtype(typ, self.chk.bool_type()) + + def named_type(self, name: str) -> Instance: + """Return an instance type with type given by the name and no type + arguments. Alias for TypeChecker.named_type. + """ + return self.chk.named_type(name) + + def is_valid_var_arg(self, typ: Type) -> bool: + """Is a type valid as a *args argument?""" + return (isinstance(typ, TupleType) or + is_subtype(typ, self.chk.named_generic_type('typing.Iterable', + [AnyType()])) or + isinstance(typ, AnyType)) + + def is_valid_keyword_var_arg(self, typ: Type) -> bool: + """Is a type valid as a **kwargs argument?""" + return is_subtype(typ, self.chk.named_generic_type( + 'builtins.dict', [self.named_type('builtins.str'), AnyType()])) + + def has_non_method(self, typ: Type, member: str) -> bool: + """Does type have a member variable / property with the given name?""" + if isinstance(typ, Instance): + return (not typ.type.has_method(member) and + typ.type.has_readable_member(member)) + else: + return False + + def has_member(self, typ: Type, member: str) -> bool: + """Does type have member with the given name?""" + # TODO TupleType => also consider tuple attributes + if isinstance(typ, Instance): + return typ.type.has_readable_member(member) + elif isinstance(typ, AnyType): + return True + elif isinstance(typ, UnionType): + result = all(self.has_member(x, member) for x in typ.items) + return result + elif isinstance(typ, TupleType): + return self.has_member(typ.fallback, member) + else: + return False + + def not_ready_callback(self, name: str, context: Context) -> None: + """Called when we can't infer the type of a variable because it's not ready yet. + + Either defer type checking of the enclosing function to the next + pass or report an error. + """ + self.chk.handle_cannot_determine_type(name, context) + + +def map_actuals_to_formals(caller_kinds: List[int], + caller_names: List[str], + callee_kinds: List[int], + callee_names: List[str], + caller_arg_type: Callable[[int], + Type]) -> List[List[int]]: + """Calculate mapping between actual (caller) args and formals. + + The result contains a list of caller argument indexes mapping to each + callee argument index, indexed by callee index. + + The caller_arg_type argument should evaluate to the type of the actual + argument type with the given index. + """ + ncallee = len(callee_kinds) + map = [None] * ncallee # type: List[List[int]] + for i in range(ncallee): + map[i] = [] + j = 0 + for i, kind in enumerate(caller_kinds): + if kind == nodes.ARG_POS: + if j < ncallee: + if callee_kinds[j] in [nodes.ARG_POS, nodes.ARG_OPT, + nodes.ARG_NAMED]: + map[j].append(i) + j += 1 + elif callee_kinds[j] == nodes.ARG_STAR: + map[j].append(i) + elif kind == nodes.ARG_STAR: + # We need to know the actual type to map varargs. + argt = caller_arg_type(i) + if isinstance(argt, TupleType): + # A tuple actual maps to a fixed number of formals. + for _ in range(len(argt.items)): + if j < ncallee: + if callee_kinds[j] != nodes.ARG_STAR2: + map[j].append(i) + else: + raise NotImplementedError() + j += 1 + else: + # Assume that it is an iterable (if it isn't, there will be + # an error later). + while j < ncallee: + if callee_kinds[j] in (nodes.ARG_NAMED, nodes.ARG_STAR2): + break + else: + map[j].append(i) + j += 1 + elif kind == nodes.ARG_NAMED: + name = caller_names[i] + if name in callee_names: + map[callee_names.index(name)].append(i) + elif nodes.ARG_STAR2 in callee_kinds: + map[callee_kinds.index(nodes.ARG_STAR2)].append(i) + else: + assert kind == nodes.ARG_STAR2 + for j in range(ncallee): + # TODO tuple varargs complicate this + no_certain_match = ( + not map[j] or caller_kinds[map[j][0]] == nodes.ARG_STAR) + if ((callee_names[j] and no_certain_match) + or callee_kinds[j] == nodes.ARG_STAR2): + map[j].append(i) + return map + + +def is_empty_tuple(t: Type) -> bool: + return isinstance(t, TupleType) and not t.items + + +def is_duplicate_mapping(mapping: List[int], actual_kinds: List[int]) -> bool: + # Multiple actuals can map to the same formal only if they both come from + # varargs (*args and **kwargs); in this case at runtime it is possible that + # there are no duplicates. We need to allow this, as the convention + # f(..., *args, **kwargs) is common enough. + return len(mapping) > 1 and not ( + len(mapping) == 2 and + actual_kinds[mapping[0]] == nodes.ARG_STAR and + actual_kinds[mapping[1]] == nodes.ARG_STAR2) + + +def replace_callable_return_type(c: CallableType, new_ret_type: Type) -> CallableType: + """Return a copy of a callable type with a different return type.""" + return c.copy_modified(ret_type=new_ret_type) + + +class ArgInferSecondPassQuery(types.TypeQuery): + """Query whether an argument type should be inferred in the second pass. + + The result is True if the type has a type variable in a callable return + type anywhere. For example, the result for Callable[[], T] is True if t is + a type variable. + """ + def __init__(self) -> None: + super().__init__(False, types.ANY_TYPE_STRATEGY) + + def visit_callable_type(self, t: CallableType) -> bool: + return self.query_types(t.arg_types) or t.accept(HasTypeVarQuery()) + + +class HasTypeVarQuery(types.TypeQuery): + """Visitor for querying whether a type has a type variable component.""" + def __init__(self) -> None: + super().__init__(False, types.ANY_TYPE_STRATEGY) + + def visit_type_var(self, t: TypeVarType) -> bool: + return True + + +def has_erased_component(t: Type) -> bool: + return t is not None and t.accept(HasErasedComponentsQuery()) + + +class HasErasedComponentsQuery(types.TypeQuery): + """Visitor for querying whether a type has an erased component.""" + def __init__(self) -> None: + super().__init__(False, types.ANY_TYPE_STRATEGY) + + def visit_erased_type(self, t: ErasedType) -> bool: + return True + + +def overload_arg_similarity(actual: Type, formal: Type) -> int: + """Return if caller argument (actual) is compatible with overloaded signature arg (formal). + + Return a similarity level: + 0: no match + 1: actual is compatible, but only using type promotions (e.g. int vs float) + 2: actual is compatible without type promotions (e.g. int vs object) + + The distinction is important in cases where multiple overload items match. We want + give priority to higher similarity matches. + """ + # Replace type variables with their upper bounds. Overloading + # resolution is based on runtime behavior which erases type + # parameters, so no need to handle type variables occurring within + # a type. + if isinstance(actual, TypeVarType): + actual = actual.erase_to_union_or_bound() + if isinstance(formal, TypeVarType): + formal = formal.erase_to_union_or_bound() + if (isinstance(actual, UninhabitedType) or isinstance(actual, AnyType) or + isinstance(formal, AnyType) or isinstance(formal, CallableType) or + (isinstance(actual, Instance) and actual.type.fallback_to_any)): + # These could match anything at runtime. + return 2 + if not experiments.STRICT_OPTIONAL and isinstance(actual, NoneTyp): + # NoneTyp matches anything if we're not doing strict Optional checking + return 2 + if isinstance(actual, UnionType): + return max(overload_arg_similarity(item, formal) + for item in actual.items) + if isinstance(formal, UnionType): + return max(overload_arg_similarity(actual, item) + for item in formal.items) + if isinstance(formal, TypeType): + if isinstance(actual, TypeType): + # Since Type[T] is covariant, check if actual = Type[A] is + # a subtype of formal = Type[F]. + return overload_arg_similarity(actual.item, formal.item) + elif isinstance(actual, CallableType) and actual.is_type_obj(): + # Check if the actual is a constructor of some sort. + # Note that this is this unsound, since we don't check the __init__ signature. + return overload_arg_similarity(actual.ret_type, formal.item) + else: + return 0 + if isinstance(formal, Instance): + if isinstance(actual, CallableType): + actual = actual.fallback + if isinstance(actual, Overloaded): + actual = actual.items()[0].fallback + if isinstance(actual, TupleType): + actual = actual.fallback + if isinstance(actual, Instance): + # First perform a quick check (as an optimization) and fall back to generic + # subtyping algorithm if type promotions are possible (e.g., int vs. float). + if formal.type in actual.type.mro: + return 2 + elif actual.type._promote and is_subtype(actual, formal): + return 1 + else: + return 0 + elif isinstance(actual, TypeType): + if formal.type.fullname() in {"builtins.object", "builtins.type"}: + return 2 + else: + return 0 + else: + return 0 + if isinstance(actual, UnboundType) or isinstance(formal, UnboundType): + # Either actual or formal is the result of an error; shut up. + return 2 + # Fall back to a conservative equality check for the remaining kinds of type. + return 2 if is_same_type(erasetype.erase_type(actual), erasetype.erase_type(formal)) else 0 + + +def freshen_generic_callable(callee: CallableType) -> CallableType: + tvdefs = [] + tvmap = {} # type: Dict[TypeVarId, Type] + for v in callee.variables: + tvdef = TypeVarDef.new_unification_variable(v) + tvdefs.append(tvdef) + tvmap[v.id] = TypeVarType(tvdef) + + return cast(CallableType, expand_type(callee, tvmap)).copy_modified(variables=tvdefs) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index b74c29f83cd9d..ce6d5190fac6a 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -1,112 +1,112 @@ -from typing import Dict, Tuple, List, cast - -from mypy.types import ( - Type, Instance, CallableType, TypeVisitor, UnboundType, ErrorType, AnyType, - Void, NoneTyp, TypeVarType, Overloaded, TupleType, UnionType, ErasedType, TypeList, - PartialType, DeletedType, UninhabitedType, TypeType, TypeVarId -) - - -def expand_type(typ: Type, env: Dict[TypeVarId, Type]) -> Type: - """Substitute any type variable references in a type given by a type - environment. - """ - - return typ.accept(ExpandTypeVisitor(env)) - - -def expand_type_by_instance(typ: Type, instance: Instance) -> Type: - """Substitute type variables in type using values from an Instance. - Type variables are considered to be bound by the class declaration.""" - - if instance.args == []: - return typ - else: - variables = {} # type: Dict[TypeVarId, Type] - for binder, arg in zip(instance.type.defn.type_vars, instance.args): - variables[binder.id] = arg - return expand_type(typ, variables) - - -class ExpandTypeVisitor(TypeVisitor[Type]): - """Visitor that substitutes type variables with values.""" - - variables = None # type: Dict[TypeVarId, Type] # TypeVar id -> TypeVar value - - def __init__(self, variables: Dict[TypeVarId, Type]) -> None: - self.variables = variables - - def visit_unbound_type(self, t: UnboundType) -> Type: - return t - - def visit_error_type(self, t: ErrorType) -> Type: - return t - - def visit_type_list(self, t: TypeList) -> Type: - assert False, 'Not supported' - - def visit_any(self, t: AnyType) -> Type: - return t - - def visit_void(self, t: Void) -> Type: - return t - - def visit_none_type(self, t: NoneTyp) -> Type: - return t - - def visit_uninhabited_type(self, t: UninhabitedType) -> Type: - return t - - def visit_deleted_type(self, t: DeletedType) -> Type: - return t - - def visit_erased_type(self, t: ErasedType) -> Type: - # Should not get here. - raise RuntimeError() - - def visit_instance(self, t: Instance) -> Type: - args = self.expand_types(t.args) - return Instance(t.type, args, t.line) - - def visit_type_var(self, t: TypeVarType) -> Type: - repl = self.variables.get(t.id, t) - if isinstance(repl, Instance): - inst = repl - # Return copy of instance with type erasure flag on. - return Instance(inst.type, inst.args, inst.line, True) - else: - return repl - - def visit_callable_type(self, t: CallableType) -> Type: - return t.copy_modified(arg_types=self.expand_types(t.arg_types), - ret_type=t.ret_type.accept(self)) - - def visit_overloaded(self, t: Overloaded) -> Type: - items = [] # type: List[CallableType] - for item in t.items(): - items.append(cast(CallableType, item.accept(self))) - return Overloaded(items) - - def visit_tuple_type(self, t: TupleType) -> Type: - return t.with_types(self.expand_types(t.items)) - - def visit_union_type(self, t: UnionType) -> Type: - # After substituting for type variables in t.items, - # some of the resulting types might be subtypes of others. - return UnionType.make_simplified_union(self.expand_types(t.items), t.line) - - def visit_partial_type(self, t: PartialType) -> Type: - return t - - def visit_type_type(self, t: TypeType) -> Type: - # TODO: Verify that the new item type is valid (instance or - # union of instances or Any). Sadly we can't report errors - # here yet. - item = t.item.accept(self) - return TypeType(item) - - def expand_types(self, types: List[Type]) -> List[Type]: - a = [] # type: List[Type] - for t in types: - a.append(t.accept(self)) - return a +from typing import Dict, Tuple, List, cast + +from mypy.types import ( + Type, Instance, CallableType, TypeVisitor, UnboundType, ErrorType, AnyType, + Void, NoneTyp, TypeVarType, Overloaded, TupleType, UnionType, ErasedType, TypeList, + PartialType, DeletedType, UninhabitedType, TypeType, TypeVarId +) + + +def expand_type(typ: Type, env: Dict[TypeVarId, Type]) -> Type: + """Substitute any type variable references in a type given by a type + environment. + """ + + return typ.accept(ExpandTypeVisitor(env)) + + +def expand_type_by_instance(typ: Type, instance: Instance) -> Type: + """Substitute type variables in type using values from an Instance. + Type variables are considered to be bound by the class declaration.""" + + if instance.args == []: + return typ + else: + variables = {} # type: Dict[TypeVarId, Type] + for binder, arg in zip(instance.type.defn.type_vars, instance.args): + variables[binder.id] = arg + return expand_type(typ, variables) + + +class ExpandTypeVisitor(TypeVisitor[Type]): + """Visitor that substitutes type variables with values.""" + + variables = None # type: Dict[TypeVarId, Type] # TypeVar id -> TypeVar value + + def __init__(self, variables: Dict[TypeVarId, Type]) -> None: + self.variables = variables + + def visit_unbound_type(self, t: UnboundType) -> Type: + return t + + def visit_error_type(self, t: ErrorType) -> Type: + return t + + def visit_type_list(self, t: TypeList) -> Type: + assert False, 'Not supported' + + def visit_any(self, t: AnyType) -> Type: + return t + + def visit_void(self, t: Void) -> Type: + return t + + def visit_none_type(self, t: NoneTyp) -> Type: + return t + + def visit_uninhabited_type(self, t: UninhabitedType) -> Type: + return t + + def visit_deleted_type(self, t: DeletedType) -> Type: + return t + + def visit_erased_type(self, t: ErasedType) -> Type: + # Should not get here. + raise RuntimeError() + + def visit_instance(self, t: Instance) -> Type: + args = self.expand_types(t.args) + return Instance(t.type, args, t.line) + + def visit_type_var(self, t: TypeVarType) -> Type: + repl = self.variables.get(t.id, t) + if isinstance(repl, Instance): + inst = repl + # Return copy of instance with type erasure flag on. + return Instance(inst.type, inst.args, inst.line, True) + else: + return repl + + def visit_callable_type(self, t: CallableType) -> Type: + return t.copy_modified(arg_types=self.expand_types(t.arg_types), + ret_type=t.ret_type.accept(self)) + + def visit_overloaded(self, t: Overloaded) -> Type: + items = [] # type: List[CallableType] + for item in t.items(): + items.append(cast(CallableType, item.accept(self))) + return Overloaded(items) + + def visit_tuple_type(self, t: TupleType) -> Type: + return t.with_types(self.expand_types(t.items)) + + def visit_union_type(self, t: UnionType) -> Type: + # After substituting for type variables in t.items, + # some of the resulting types might be subtypes of others. + return UnionType.make_simplified_union(self.expand_types(t.items), t.line) + + def visit_partial_type(self, t: PartialType) -> Type: + return t + + def visit_type_type(self, t: TypeType) -> Type: + # TODO: Verify that the new item type is valid (instance or + # union of instances or Any). Sadly we can't report errors + # here yet. + item = t.item.accept(self) + return TypeType(item) + + def expand_types(self, types: List[Type]) -> List[Type]: + a = [] # type: List[Type] + for t in types: + a.append(t.accept(self)) + return a diff --git a/mypy/join.py b/mypy/join.py index e0709c2717911..684d62cfeaebd 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -1,367 +1,367 @@ -"""Calculation of the least upper bound types (joins).""" - -from typing import List - -from mypy.types import ( - Type, AnyType, NoneTyp, Void, TypeVisitor, Instance, UnboundType, - ErrorType, TypeVarType, CallableType, TupleType, ErasedType, TypeList, - UnionType, FunctionLike, Overloaded, PartialType, DeletedType, - UninhabitedType, TypeType, NamedTupleType -) -from mypy.maptype import map_instance_to_supertype -from mypy.subtypes import is_subtype, is_equivalent, is_subtype_ignoring_tvars - -from mypy import experiments - - -def join_simple(declaration: Type, s: Type, t: Type) -> Type: - """Return a simple least upper bound given the declared type.""" - - if isinstance(s, AnyType): - return s - - if isinstance(s, ErasedType): - return t - - if is_subtype(s, t): - return t - - if is_subtype(t, s): - return s - - if isinstance(declaration, UnionType): - return UnionType.make_simplified_union([s, t]) - - if isinstance(s, NoneTyp) and not isinstance(t, NoneTyp): - s, t = t, s - - if isinstance(s, UninhabitedType) and not isinstance(t, UninhabitedType): - s, t = t, s - - value = t.accept(TypeJoinVisitor(s)) - - if value is None: - # XXX this code path probably should be avoided. - # It seems to happen when a line (x = y) is a type error, and - # it's not clear that assuming that x is arbitrary afterward - # is a good idea. - return declaration - - if declaration is None or is_subtype(value, declaration): - return value - - return declaration - - -def join_types(s: Type, t: Type) -> Type: - """Return the least upper bound of s and t. - - For example, the join of 'int' and 'object' is 'object'. - - If the join does not exist, return an ErrorType instance. - """ - if isinstance(s, AnyType): - return s - - if isinstance(s, ErasedType): - return t - - if isinstance(s, NoneTyp) and not isinstance(t, NoneTyp): - s, t = t, s - - # Use a visitor to handle non-trivial cases. - return t.accept(TypeJoinVisitor(s)) - - -class TypeJoinVisitor(TypeVisitor[Type]): - """Implementation of the least upper bound algorithm. - - Attributes: - s: The other (left) type operand. - """ - - def __init__(self, s: Type) -> None: - self.s = s - - def visit_unbound_type(self, t: UnboundType) -> Type: - if isinstance(self.s, Void) or isinstance(self.s, ErrorType): - return ErrorType() - else: - return AnyType() - - def visit_union_type(self, t: UnionType) -> Type: - if is_subtype(self.s, t): - return t - else: - return UnionType.make_simplified_union([self.s, t]) - - def visit_error_type(self, t: ErrorType) -> Type: - return t - - def visit_type_list(self, t: TypeList) -> Type: - assert False, 'Not supported' - - def visit_any(self, t: AnyType) -> Type: - return t - - def visit_void(self, t: Void) -> Type: - if isinstance(self.s, Void): - return t - else: - return ErrorType() - - def visit_none_type(self, t: NoneTyp) -> Type: - if experiments.STRICT_OPTIONAL: - if isinstance(self.s, (NoneTyp, UninhabitedType)): - return t - else: - return self.default(self.s) - else: - if not isinstance(self.s, Void): - return self.s - else: - return self.default(self.s) - - def visit_uninhabited_type(self, t: UninhabitedType) -> Type: - if not isinstance(self.s, Void): - return self.s - else: - return self.default(self.s) - - def visit_deleted_type(self, t: DeletedType) -> Type: - if not isinstance(self.s, Void): - return self.s - else: - return self.default(self.s) - - def visit_erased_type(self, t: ErasedType) -> Type: - return self.s - - def visit_type_var(self, t: TypeVarType) -> Type: - if isinstance(self.s, TypeVarType) and self.s.id == t.id: - return self.s - else: - return self.default(self.s) - - def visit_instance(self, t: Instance) -> Type: - if isinstance(self.s, Instance): - return join_instances(t, self.s) - elif isinstance(self.s, FunctionLike): - return join_types(t, self.s.fallback) - elif isinstance(self.s, TypeType): - return join_types(t, self.s) - else: - return self.default(self.s) - - def visit_callable_type(self, t: CallableType) -> Type: - # TODO: Consider subtyping instead of just similarity. - if isinstance(self.s, CallableType) and is_similar_callables(t, self.s): - return combine_similar_callables(t, self.s) - elif isinstance(self.s, Overloaded): - # Switch the order of arguments to that we'll get to visit_overloaded. - return join_types(t, self.s) - else: - return join_types(t.fallback, self.s) - - def visit_overloaded(self, t: Overloaded) -> Type: - # This is more complex than most other cases. Here are some - # examples that illustrate how this works. - # - # First let's define a concise notation: - # - Cn are callable types (for n in 1, 2, ...) - # - Ov(C1, C2, ...) is an overloaded type with items C1, C2, ... - # - Callable[[T, ...], S] is written as [T, ...] -> S. - # - # We want some basic properties to hold (assume Cn are all - # unrelated via Any-similarity): - # - # join(Ov(C1, C2), C1) == C1 - # join(Ov(C1, C2), Ov(C1, C2)) == Ov(C1, C2) - # join(Ov(C1, C2), Ov(C1, C3)) == C1 - # join(Ov(C2, C2), C3) == join of fallback types - # - # The presence of Any types makes things more interesting. The join is the - # most general type we can get with respect to Any: - # - # join(Ov([int] -> int, [str] -> str), [Any] -> str) == Any -> str - # - # We could use a simplification step that removes redundancies, but that's not - # implemented right now. Consider this example, where we get a redundancy: - # - # join(Ov([int, Any] -> Any, [str, Any] -> Any), [Any, int] -> Any) == - # Ov([Any, int] -> Any, [Any, int] -> Any) - # - # TODO: Use callable subtyping instead of just similarity. - result = [] # type: List[CallableType] - s = self.s - if isinstance(s, FunctionLike): - # The interesting case where both types are function types. - for t_item in t.items(): - for s_item in s.items(): - if is_similar_callables(t_item, s_item): - result.append(combine_similar_callables(t_item, s_item)) - if result: - # TODO: Simplify redundancies from the result. - if len(result) == 1: - return result[0] - else: - return Overloaded(result) - return join_types(t.fallback, s.fallback) - return join_types(t.fallback, s) - - def visit_tuple_type(self, t: TupleType) -> Type: - if self.s == t: - return self.s - if isinstance(self.s, TupleType) and self.s.length() == t.length(): - items = [] # type: List[Type] - for i in range(t.length()): - items.append(self.join(t.items[i], self.s.items[i])) - if (not (isinstance(t, NamedTupleType) and isinstance(self.s, NamedTupleType)) - or self.s.attrs == t.attrs): - return self.s.with_types(items) - # TODO: What if the fallback types are different? - return TupleType(items, t.fallback) - else: - return self.default(self.s) - - def visit_partial_type(self, t: PartialType) -> Type: - # We only have partial information so we can't decide the join result. We should - # never get here. - assert False, "Internal error" - - def visit_type_type(self, t: TypeType) -> Type: - if isinstance(self.s, TypeType): - return TypeType(self.join(t.item, self.s.item), line=t.line) - elif isinstance(self.s, Instance) and self.s.type.fullname() == 'builtins.type': - return self.s - else: - return self.default(self.s) - - def join(self, s: Type, t: Type) -> Type: - return join_types(s, t) - - def default(self, typ: Type) -> Type: - if isinstance(typ, Instance): - return object_from_instance(typ) - elif isinstance(typ, UnboundType): - return AnyType() - elif isinstance(typ, Void) or isinstance(typ, ErrorType): - return ErrorType() - elif isinstance(typ, TupleType): - return self.default(typ.fallback) - elif isinstance(typ, FunctionLike): - return self.default(typ.fallback) - elif isinstance(typ, TypeVarType): - return self.default(typ.upper_bound) - else: - return AnyType() - - -def join_instances(t: Instance, s: Instance) -> Type: - """Calculate the join of two instance types. - - If allow_interfaces is True, also consider interface-type results for - non-interface types. - - Return ErrorType if the result is ambiguous. - """ - if t.type == s.type: - # Simplest case: join two types with the same base type (but - # potentially different arguments). - if is_subtype(t, s) or is_subtype(s, t): - # Compatible; combine type arguments. - args = [] # type: List[Type] - for i in range(len(t.args)): - args.append(join_types(t.args[i], s.args[i])) - return Instance(t.type, args) - else: - # Incompatible; return trivial result object. - return object_from_instance(t) - elif t.type.bases and is_subtype_ignoring_tvars(t, s): - return join_instances_via_supertype(t, s) - else: - # Now t is not a subtype of s, and t != s. Now s could be a subtype - # of t; alternatively, we need to find a common supertype. This works - # in of the both cases. - return join_instances_via_supertype(s, t) - - -def join_instances_via_supertype(t: Instance, s: Instance) -> Type: - # Give preference to joins via duck typing relationship, so that - # join(int, float) == float, for example. - if t.type._promote and is_subtype(t.type._promote, s): - return join_types(t.type._promote, s) - elif s.type._promote and is_subtype(s.type._promote, t): - return join_types(t, s.type._promote) - # Compute the "best" supertype of t when joined with s. - # The definition of "best" may evolve; for now it is the one with - # the longest MRO. Ties are broken by using the earlier base. - best = None # type: Type - for base in t.type.bases: - mapped = map_instance_to_supertype(t, base.type) - res = join_instances(mapped, s) - if best is None or is_better(res, best): - best = res - assert best is not None - return best - - -def is_better(t: Type, s: Type) -> bool: - # Given two possible results from join_instances_via_supertype(), - # indicate whether t is the better one. - if isinstance(t, Instance): - if not isinstance(s, Instance): - return True - # Use len(mro) as a proxy for the better choice. - if len(t.type.mro) > len(s.type.mro): - return True - return False - - -def is_similar_callables(t: CallableType, s: CallableType) -> bool: - """Return True if t and s are equivalent and have identical numbers of - arguments, default arguments and varargs. - """ - - return (len(t.arg_types) == len(s.arg_types) and t.min_args == s.min_args - and t.is_var_arg == s.is_var_arg and is_equivalent(t, s)) - - -def combine_similar_callables(t: CallableType, s: CallableType) -> CallableType: - arg_types = [] # type: List[Type] - for i in range(len(t.arg_types)): - arg_types.append(join_types(t.arg_types[i], s.arg_types[i])) - # TODO kinds and argument names - # The fallback type can be either 'function' or 'type'. The result should have 'type' as - # fallback only if both operands have it as 'type'. - if t.fallback.type.fullname() != 'builtins.type': - fallback = t.fallback - else: - fallback = s.fallback - return t.copy_modified(arg_types=arg_types, - ret_type=join_types(t.ret_type, s.ret_type), - fallback=fallback, - name=None) - - -def object_from_instance(instance: Instance) -> Instance: - """Construct the type 'builtins.object' from an instance type.""" - # Use the fact that 'object' is always the last class in the mro. - res = Instance(instance.type.mro[-1], []) - return res - - -def join_type_list(types: List[Type]) -> Type: - if not types: - # This is a little arbitrary but reasonable. Any empty tuple should be compatible - # with all variable length tuples, and this makes it possible. A better approach - # would be to use a special bottom type, which we do when strict Optional - # checking is enabled. - if experiments.STRICT_OPTIONAL: - return UninhabitedType() - else: - return NoneTyp() - joined = types[0] - for t in types[1:]: - joined = join_types(joined, t) - return joined +"""Calculation of the least upper bound types (joins).""" + +from typing import List + +from mypy.types import ( + Type, AnyType, NoneTyp, Void, TypeVisitor, Instance, UnboundType, + ErrorType, TypeVarType, CallableType, TupleType, ErasedType, TypeList, + UnionType, FunctionLike, Overloaded, PartialType, DeletedType, + UninhabitedType, TypeType, NamedTupleType +) +from mypy.maptype import map_instance_to_supertype +from mypy.subtypes import is_subtype, is_equivalent, is_subtype_ignoring_tvars + +from mypy import experiments + + +def join_simple(declaration: Type, s: Type, t: Type) -> Type: + """Return a simple least upper bound given the declared type.""" + + if isinstance(s, AnyType): + return s + + if isinstance(s, ErasedType): + return t + + if is_subtype(s, t): + return t + + if is_subtype(t, s): + return s + + if isinstance(declaration, UnionType): + return UnionType.make_simplified_union([s, t]) + + if isinstance(s, NoneTyp) and not isinstance(t, NoneTyp): + s, t = t, s + + if isinstance(s, UninhabitedType) and not isinstance(t, UninhabitedType): + s, t = t, s + + value = t.accept(TypeJoinVisitor(s)) + + if value is None: + # XXX this code path probably should be avoided. + # It seems to happen when a line (x = y) is a type error, and + # it's not clear that assuming that x is arbitrary afterward + # is a good idea. + return declaration + + if declaration is None or is_subtype(value, declaration): + return value + + return declaration + + +def join_types(s: Type, t: Type) -> Type: + """Return the least upper bound of s and t. + + For example, the join of 'int' and 'object' is 'object'. + + If the join does not exist, return an ErrorType instance. + """ + if isinstance(s, AnyType): + return s + + if isinstance(s, ErasedType): + return t + + if isinstance(s, NoneTyp) and not isinstance(t, NoneTyp): + s, t = t, s + + # Use a visitor to handle non-trivial cases. + return t.accept(TypeJoinVisitor(s)) + + +class TypeJoinVisitor(TypeVisitor[Type]): + """Implementation of the least upper bound algorithm. + + Attributes: + s: The other (left) type operand. + """ + + def __init__(self, s: Type) -> None: + self.s = s + + def visit_unbound_type(self, t: UnboundType) -> Type: + if isinstance(self.s, Void) or isinstance(self.s, ErrorType): + return ErrorType() + else: + return AnyType() + + def visit_union_type(self, t: UnionType) -> Type: + if is_subtype(self.s, t): + return t + else: + return UnionType.make_simplified_union([self.s, t]) + + def visit_error_type(self, t: ErrorType) -> Type: + return t + + def visit_type_list(self, t: TypeList) -> Type: + assert False, 'Not supported' + + def visit_any(self, t: AnyType) -> Type: + return t + + def visit_void(self, t: Void) -> Type: + if isinstance(self.s, Void): + return t + else: + return ErrorType() + + def visit_none_type(self, t: NoneTyp) -> Type: + if experiments.STRICT_OPTIONAL: + if isinstance(self.s, (NoneTyp, UninhabitedType)): + return t + else: + return self.default(self.s) + else: + if not isinstance(self.s, Void): + return self.s + else: + return self.default(self.s) + + def visit_uninhabited_type(self, t: UninhabitedType) -> Type: + if not isinstance(self.s, Void): + return self.s + else: + return self.default(self.s) + + def visit_deleted_type(self, t: DeletedType) -> Type: + if not isinstance(self.s, Void): + return self.s + else: + return self.default(self.s) + + def visit_erased_type(self, t: ErasedType) -> Type: + return self.s + + def visit_type_var(self, t: TypeVarType) -> Type: + if isinstance(self.s, TypeVarType) and self.s.id == t.id: + return self.s + else: + return self.default(self.s) + + def visit_instance(self, t: Instance) -> Type: + if isinstance(self.s, Instance): + return join_instances(t, self.s) + elif isinstance(self.s, FunctionLike): + return join_types(t, self.s.fallback) + elif isinstance(self.s, TypeType): + return join_types(t, self.s) + else: + return self.default(self.s) + + def visit_callable_type(self, t: CallableType) -> Type: + # TODO: Consider subtyping instead of just similarity. + if isinstance(self.s, CallableType) and is_similar_callables(t, self.s): + return combine_similar_callables(t, self.s) + elif isinstance(self.s, Overloaded): + # Switch the order of arguments to that we'll get to visit_overloaded. + return join_types(t, self.s) + else: + return join_types(t.fallback, self.s) + + def visit_overloaded(self, t: Overloaded) -> Type: + # This is more complex than most other cases. Here are some + # examples that illustrate how this works. + # + # First let's define a concise notation: + # - Cn are callable types (for n in 1, 2, ...) + # - Ov(C1, C2, ...) is an overloaded type with items C1, C2, ... + # - Callable[[T, ...], S] is written as [T, ...] -> S. + # + # We want some basic properties to hold (assume Cn are all + # unrelated via Any-similarity): + # + # join(Ov(C1, C2), C1) == C1 + # join(Ov(C1, C2), Ov(C1, C2)) == Ov(C1, C2) + # join(Ov(C1, C2), Ov(C1, C3)) == C1 + # join(Ov(C2, C2), C3) == join of fallback types + # + # The presence of Any types makes things more interesting. The join is the + # most general type we can get with respect to Any: + # + # join(Ov([int] -> int, [str] -> str), [Any] -> str) == Any -> str + # + # We could use a simplification step that removes redundancies, but that's not + # implemented right now. Consider this example, where we get a redundancy: + # + # join(Ov([int, Any] -> Any, [str, Any] -> Any), [Any, int] -> Any) == + # Ov([Any, int] -> Any, [Any, int] -> Any) + # + # TODO: Use callable subtyping instead of just similarity. + result = [] # type: List[CallableType] + s = self.s + if isinstance(s, FunctionLike): + # The interesting case where both types are function types. + for t_item in t.items(): + for s_item in s.items(): + if is_similar_callables(t_item, s_item): + result.append(combine_similar_callables(t_item, s_item)) + if result: + # TODO: Simplify redundancies from the result. + if len(result) == 1: + return result[0] + else: + return Overloaded(result) + return join_types(t.fallback, s.fallback) + return join_types(t.fallback, s) + + def visit_tuple_type(self, t: TupleType) -> Type: + if self.s == t: + return self.s + if isinstance(self.s, TupleType) and self.s.length() == t.length(): + items = [] # type: List[Type] + for i in range(t.length()): + items.append(self.join(t.items[i], self.s.items[i])) + if (not (isinstance(t, NamedTupleType) and isinstance(self.s, NamedTupleType)) + or self.s.attrs == t.attrs): + return self.s.with_types(items) + # TODO: What if the fallback types are different? + return TupleType(items, t.fallback) + else: + return self.default(self.s) + + def visit_partial_type(self, t: PartialType) -> Type: + # We only have partial information so we can't decide the join result. We should + # never get here. + assert False, "Internal error" + + def visit_type_type(self, t: TypeType) -> Type: + if isinstance(self.s, TypeType): + return TypeType(self.join(t.item, self.s.item), line=t.line) + elif isinstance(self.s, Instance) and self.s.type.fullname() == 'builtins.type': + return self.s + else: + return self.default(self.s) + + def join(self, s: Type, t: Type) -> Type: + return join_types(s, t) + + def default(self, typ: Type) -> Type: + if isinstance(typ, Instance): + return object_from_instance(typ) + elif isinstance(typ, UnboundType): + return AnyType() + elif isinstance(typ, Void) or isinstance(typ, ErrorType): + return ErrorType() + elif isinstance(typ, TupleType): + return self.default(typ.fallback) + elif isinstance(typ, FunctionLike): + return self.default(typ.fallback) + elif isinstance(typ, TypeVarType): + return self.default(typ.upper_bound) + else: + return AnyType() + + +def join_instances(t: Instance, s: Instance) -> Type: + """Calculate the join of two instance types. + + If allow_interfaces is True, also consider interface-type results for + non-interface types. + + Return ErrorType if the result is ambiguous. + """ + if t.type == s.type: + # Simplest case: join two types with the same base type (but + # potentially different arguments). + if is_subtype(t, s) or is_subtype(s, t): + # Compatible; combine type arguments. + args = [] # type: List[Type] + for i in range(len(t.args)): + args.append(join_types(t.args[i], s.args[i])) + return Instance(t.type, args) + else: + # Incompatible; return trivial result object. + return object_from_instance(t) + elif t.type.bases and is_subtype_ignoring_tvars(t, s): + return join_instances_via_supertype(t, s) + else: + # Now t is not a subtype of s, and t != s. Now s could be a subtype + # of t; alternatively, we need to find a common supertype. This works + # in of the both cases. + return join_instances_via_supertype(s, t) + + +def join_instances_via_supertype(t: Instance, s: Instance) -> Type: + # Give preference to joins via duck typing relationship, so that + # join(int, float) == float, for example. + if t.type._promote and is_subtype(t.type._promote, s): + return join_types(t.type._promote, s) + elif s.type._promote and is_subtype(s.type._promote, t): + return join_types(t, s.type._promote) + # Compute the "best" supertype of t when joined with s. + # The definition of "best" may evolve; for now it is the one with + # the longest MRO. Ties are broken by using the earlier base. + best = None # type: Type + for base in t.type.bases: + mapped = map_instance_to_supertype(t, base.type) + res = join_instances(mapped, s) + if best is None or is_better(res, best): + best = res + assert best is not None + return best + + +def is_better(t: Type, s: Type) -> bool: + # Given two possible results from join_instances_via_supertype(), + # indicate whether t is the better one. + if isinstance(t, Instance): + if not isinstance(s, Instance): + return True + # Use len(mro) as a proxy for the better choice. + if len(t.type.mro) > len(s.type.mro): + return True + return False + + +def is_similar_callables(t: CallableType, s: CallableType) -> bool: + """Return True if t and s are equivalent and have identical numbers of + arguments, default arguments and varargs. + """ + + return (len(t.arg_types) == len(s.arg_types) and t.min_args == s.min_args + and t.is_var_arg == s.is_var_arg and is_equivalent(t, s)) + + +def combine_similar_callables(t: CallableType, s: CallableType) -> CallableType: + arg_types = [] # type: List[Type] + for i in range(len(t.arg_types)): + arg_types.append(join_types(t.arg_types[i], s.arg_types[i])) + # TODO kinds and argument names + # The fallback type can be either 'function' or 'type'. The result should have 'type' as + # fallback only if both operands have it as 'type'. + if t.fallback.type.fullname() != 'builtins.type': + fallback = t.fallback + else: + fallback = s.fallback + return t.copy_modified(arg_types=arg_types, + ret_type=join_types(t.ret_type, s.ret_type), + fallback=fallback, + name=None) + + +def object_from_instance(instance: Instance) -> Instance: + """Construct the type 'builtins.object' from an instance type.""" + # Use the fact that 'object' is always the last class in the mro. + res = Instance(instance.type.mro[-1], []) + return res + + +def join_type_list(types: List[Type]) -> Type: + if not types: + # This is a little arbitrary but reasonable. Any empty tuple should be compatible + # with all variable length tuples, and this makes it possible. A better approach + # would be to use a special bottom type, which we do when strict Optional + # checking is enabled. + if experiments.STRICT_OPTIONAL: + return UninhabitedType() + else: + return NoneTyp() + joined = types[0] + for t in types[1:]: + joined = join_types(joined, t) + return joined diff --git a/mypy/meet.py b/mypy/meet.py index c4e3831e6f5f7..5c98385c7f1f2 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -1,285 +1,285 @@ -from typing import cast, List - -from mypy.join import is_similar_callables, combine_similar_callables -from mypy.types import ( - Type, AnyType, TypeVisitor, UnboundType, Void, ErrorType, NoneTyp, TypeVarType, - Instance, CallableType, TupleType, ErasedType, TypeList, UnionType, PartialType, - DeletedType, UninhabitedType, TypeType -) -from mypy.subtypes import is_subtype -from mypy.nodes import TypeInfo - -from mypy import experiments - -# TODO Describe this module. - - -def meet_types(s: Type, t: Type) -> Type: - """Return the greatest lower bound of two types.""" - if isinstance(s, ErasedType): - return s - if isinstance(s, AnyType): - return t - if isinstance(s, UnionType) and not isinstance(t, UnionType): - s, t = t, s - return t.accept(TypeMeetVisitor(s)) - - -def meet_simple(s: Type, t: Type, default_right: bool = True) -> Type: - if s == t: - return s - if isinstance(s, UnionType): - return UnionType.make_simplified_union([meet_types(x, t) for x in s.items]) - elif not is_overlapping_types(s, t, use_promotions=True): - if experiments.STRICT_OPTIONAL: - return UninhabitedType() - else: - return NoneTyp() - else: - if default_right: - return t - else: - return s - - -def is_overlapping_types(t: Type, s: Type, use_promotions: bool = False) -> bool: - """Can a value of type t be a value of type s, or vice versa? - - Note that this effectively checks against erased types, since type - variables are erased at runtime and the overlapping check is based - on runtime behavior. - - If use_promotions is True, also consider type promotions (int and - float would only be overlapping if it's True). - - This does not consider multiple inheritance. For example, A and B in - the following example are not considered overlapping, even though - via C they can be overlapping: - - class A: ... - class B: ... - class C(A, B): ... - - The rationale is that this case is usually very unlikely as multiple - inheritance is rare. Also, we can't reliably determine whether - multiple inheritance actually occurs somewhere in a program, due to - stub files hiding implementation details, dynamic loading etc. - - TODO: Don't consider tuples always overlapping. - TODO: Don't consider callables always overlapping. - TODO: Don't consider type variables with values always overlapping. - """ - # Since we are effectively working with the erased types, we only - # need to handle occurrences of TypeVarType at the top level. - if isinstance(t, TypeVarType): - t = t.erase_to_union_or_bound() - if isinstance(s, TypeVarType): - s = s.erase_to_union_or_bound() - if isinstance(t, Instance): - if isinstance(s, Instance): - # Consider two classes non-disjoint if one is included in the mro - # of another. - if use_promotions: - # Consider cases like int vs float to be overlapping where - # there is only a type promotion relationship but not proper - # subclassing. - if t.type._promote and is_overlapping_types(t.type._promote, s): - return True - if s.type._promote and is_overlapping_types(s.type._promote, t): - return True - return t.type in s.type.mro or s.type in t.type.mro - if isinstance(t, UnionType): - return any(is_overlapping_types(item, s) - for item in t.items) - if isinstance(s, UnionType): - return any(is_overlapping_types(t, item) - for item in s.items) - if isinstance(t, TypeType) and isinstance(s, TypeType): - # If both types are TypeType, compare their inner types. - return is_overlapping_types(t.item, s.item, use_promotions) - elif isinstance(t, TypeType) or isinstance(s, TypeType): - # If exactly only one of t or s is a TypeType, check if one of them - # is an `object` or a `type` and otherwise assume no overlap. - other = s if isinstance(t, TypeType) else t - if isinstance(other, Instance): - return other.type.fullname() in {'builtins.object', 'builtins.type'} - else: - return False - if experiments.STRICT_OPTIONAL: - if isinstance(t, NoneTyp) != isinstance(s, NoneTyp): - # NoneTyp does not overlap with other non-Union types under strict Optional checking - return False - # We conservatively assume that non-instance, non-union, and non-TypeType types can overlap - # any other types. - return True - - -def nearest_builtin_ancestor(type: TypeInfo) -> TypeInfo: - for base in type.mro: - if base.defn.is_builtinclass: - return base - else: - return None - - -class TypeMeetVisitor(TypeVisitor[Type]): - def __init__(self, s: Type) -> None: - self.s = s - - def visit_unbound_type(self, t: UnboundType) -> Type: - if isinstance(self.s, Void) or isinstance(self.s, ErrorType): - return ErrorType() - elif isinstance(self.s, NoneTyp): - if experiments.STRICT_OPTIONAL: - return AnyType() - else: - return self.s - elif isinstance(self.s, UninhabitedType): - return self.s - else: - return AnyType() - - def visit_error_type(self, t: ErrorType) -> Type: - return t - - def visit_type_list(self, t: TypeList) -> Type: - assert False, 'Not supported' - - def visit_any(self, t: AnyType) -> Type: - return self.s - - def visit_union_type(self, t: UnionType) -> Type: - if isinstance(self.s, UnionType): - meets = [] # type: List[Type] - for x in t.items: - for y in self.s.items: - meets.append(meet_types(x, y)) - else: - meets = [meet_types(x, self.s) - for x in t.items] - return UnionType.make_simplified_union(meets) - - def visit_void(self, t: Void) -> Type: - if isinstance(self.s, Void): - return t - else: - return ErrorType() - - def visit_none_type(self, t: NoneTyp) -> Type: - if experiments.STRICT_OPTIONAL: - if isinstance(self.s, NoneTyp) or (isinstance(self.s, Instance) and - self.s.type.fullname() == 'builtins.object'): - return t - else: - return UninhabitedType() - else: - if not isinstance(self.s, Void) and not isinstance(self.s, ErrorType): - return t - else: - return ErrorType() - - def visit_uninhabited_type(self, t: UninhabitedType) -> Type: - if not isinstance(self.s, Void) and not isinstance(self.s, ErrorType): - return t - else: - return ErrorType() - - def visit_deleted_type(self, t: DeletedType) -> Type: - if not isinstance(self.s, Void) and not isinstance(self.s, ErrorType): - if isinstance(self.s, NoneTyp): - if experiments.STRICT_OPTIONAL: - return t - else: - return self.s - elif isinstance(self.s, UninhabitedType): - return self.s - else: - return t - else: - return ErrorType() - - def visit_erased_type(self, t: ErasedType) -> Type: - return self.s - - def visit_type_var(self, t: TypeVarType) -> Type: - if isinstance(self.s, TypeVarType) and self.s.id == t.id: - return self.s - else: - return self.default(self.s) - - def visit_instance(self, t: Instance) -> Type: - if isinstance(self.s, Instance): - si = self.s - if t.type == si.type: - if is_subtype(t, self.s) or is_subtype(self.s, t): - # Combine type arguments. We could have used join below - # equivalently. - args = [] # type: List[Type] - for i in range(len(t.args)): - args.append(self.meet(t.args[i], si.args[i])) - return Instance(t.type, args) - else: - if experiments.STRICT_OPTIONAL: - return UninhabitedType() - else: - return NoneTyp() - else: - if is_subtype(t, self.s): - return t - elif is_subtype(self.s, t): - # See also above comment. - return self.s - else: - if experiments.STRICT_OPTIONAL: - return UninhabitedType() - else: - return NoneTyp() - elif isinstance(self.s, TypeType): - return meet_types(t, self.s) - else: - return self.default(self.s) - - def visit_callable_type(self, t: CallableType) -> Type: - if isinstance(self.s, CallableType) and is_similar_callables(t, self.s): - return combine_similar_callables(t, self.s) - else: - return self.default(self.s) - - def visit_tuple_type(self, t: TupleType) -> Type: - if isinstance(self.s, TupleType) and self.s.length() == t.length(): - items = [] # type: List[Type] - for i in range(t.length()): - items.append(self.meet(t.items[i], self.s.items[i])) - # TODO: What if the fallbacks are different? - return TupleType(items, t.fallback) - else: - return self.default(self.s) - - def visit_partial_type(self, t: PartialType) -> Type: - # We can't determine the meet of partial types. We should never get here. - assert False, 'Internal error' - - def visit_type_type(self, t: TypeType) -> Type: - if isinstance(self.s, TypeType): - typ = self.meet(t.item, self.s.item) - if not isinstance(typ, NoneTyp): - typ = TypeType(typ, line=t.line) - return typ - elif isinstance(self.s, Instance) and self.s.type.fullname() == 'builtins.type': - return t - else: - return self.default(self.s) - - def meet(self, s, t): - return meet_types(s, t) - - def default(self, typ): - if isinstance(typ, UnboundType): - return AnyType() - elif isinstance(typ, Void) or isinstance(typ, ErrorType): - return ErrorType() - else: - if experiments.STRICT_OPTIONAL: - return UninhabitedType() - else: - return NoneTyp() +from typing import cast, List + +from mypy.join import is_similar_callables, combine_similar_callables +from mypy.types import ( + Type, AnyType, TypeVisitor, UnboundType, Void, ErrorType, NoneTyp, TypeVarType, + Instance, CallableType, TupleType, ErasedType, TypeList, UnionType, PartialType, + DeletedType, UninhabitedType, TypeType +) +from mypy.subtypes import is_subtype +from mypy.nodes import TypeInfo + +from mypy import experiments + +# TODO Describe this module. + + +def meet_types(s: Type, t: Type) -> Type: + """Return the greatest lower bound of two types.""" + if isinstance(s, ErasedType): + return s + if isinstance(s, AnyType): + return t + if isinstance(s, UnionType) and not isinstance(t, UnionType): + s, t = t, s + return t.accept(TypeMeetVisitor(s)) + + +def meet_simple(s: Type, t: Type, default_right: bool = True) -> Type: + if s == t: + return s + if isinstance(s, UnionType): + return UnionType.make_simplified_union([meet_types(x, t) for x in s.items]) + elif not is_overlapping_types(s, t, use_promotions=True): + if experiments.STRICT_OPTIONAL: + return UninhabitedType() + else: + return NoneTyp() + else: + if default_right: + return t + else: + return s + + +def is_overlapping_types(t: Type, s: Type, use_promotions: bool = False) -> bool: + """Can a value of type t be a value of type s, or vice versa? + + Note that this effectively checks against erased types, since type + variables are erased at runtime and the overlapping check is based + on runtime behavior. + + If use_promotions is True, also consider type promotions (int and + float would only be overlapping if it's True). + + This does not consider multiple inheritance. For example, A and B in + the following example are not considered overlapping, even though + via C they can be overlapping: + + class A: ... + class B: ... + class C(A, B): ... + + The rationale is that this case is usually very unlikely as multiple + inheritance is rare. Also, we can't reliably determine whether + multiple inheritance actually occurs somewhere in a program, due to + stub files hiding implementation details, dynamic loading etc. + + TODO: Don't consider tuples always overlapping. + TODO: Don't consider callables always overlapping. + TODO: Don't consider type variables with values always overlapping. + """ + # Since we are effectively working with the erased types, we only + # need to handle occurrences of TypeVarType at the top level. + if isinstance(t, TypeVarType): + t = t.erase_to_union_or_bound() + if isinstance(s, TypeVarType): + s = s.erase_to_union_or_bound() + if isinstance(t, Instance): + if isinstance(s, Instance): + # Consider two classes non-disjoint if one is included in the mro + # of another. + if use_promotions: + # Consider cases like int vs float to be overlapping where + # there is only a type promotion relationship but not proper + # subclassing. + if t.type._promote and is_overlapping_types(t.type._promote, s): + return True + if s.type._promote and is_overlapping_types(s.type._promote, t): + return True + return t.type in s.type.mro or s.type in t.type.mro + if isinstance(t, UnionType): + return any(is_overlapping_types(item, s) + for item in t.items) + if isinstance(s, UnionType): + return any(is_overlapping_types(t, item) + for item in s.items) + if isinstance(t, TypeType) and isinstance(s, TypeType): + # If both types are TypeType, compare their inner types. + return is_overlapping_types(t.item, s.item, use_promotions) + elif isinstance(t, TypeType) or isinstance(s, TypeType): + # If exactly only one of t or s is a TypeType, check if one of them + # is an `object` or a `type` and otherwise assume no overlap. + other = s if isinstance(t, TypeType) else t + if isinstance(other, Instance): + return other.type.fullname() in {'builtins.object', 'builtins.type'} + else: + return False + if experiments.STRICT_OPTIONAL: + if isinstance(t, NoneTyp) != isinstance(s, NoneTyp): + # NoneTyp does not overlap with other non-Union types under strict Optional checking + return False + # We conservatively assume that non-instance, non-union, and non-TypeType types can overlap + # any other types. + return True + + +def nearest_builtin_ancestor(type: TypeInfo) -> TypeInfo: + for base in type.mro: + if base.defn.is_builtinclass: + return base + else: + return None + + +class TypeMeetVisitor(TypeVisitor[Type]): + def __init__(self, s: Type) -> None: + self.s = s + + def visit_unbound_type(self, t: UnboundType) -> Type: + if isinstance(self.s, Void) or isinstance(self.s, ErrorType): + return ErrorType() + elif isinstance(self.s, NoneTyp): + if experiments.STRICT_OPTIONAL: + return AnyType() + else: + return self.s + elif isinstance(self.s, UninhabitedType): + return self.s + else: + return AnyType() + + def visit_error_type(self, t: ErrorType) -> Type: + return t + + def visit_type_list(self, t: TypeList) -> Type: + assert False, 'Not supported' + + def visit_any(self, t: AnyType) -> Type: + return self.s + + def visit_union_type(self, t: UnionType) -> Type: + if isinstance(self.s, UnionType): + meets = [] # type: List[Type] + for x in t.items: + for y in self.s.items: + meets.append(meet_types(x, y)) + else: + meets = [meet_types(x, self.s) + for x in t.items] + return UnionType.make_simplified_union(meets) + + def visit_void(self, t: Void) -> Type: + if isinstance(self.s, Void): + return t + else: + return ErrorType() + + def visit_none_type(self, t: NoneTyp) -> Type: + if experiments.STRICT_OPTIONAL: + if isinstance(self.s, NoneTyp) or (isinstance(self.s, Instance) and + self.s.type.fullname() == 'builtins.object'): + return t + else: + return UninhabitedType() + else: + if not isinstance(self.s, Void) and not isinstance(self.s, ErrorType): + return t + else: + return ErrorType() + + def visit_uninhabited_type(self, t: UninhabitedType) -> Type: + if not isinstance(self.s, Void) and not isinstance(self.s, ErrorType): + return t + else: + return ErrorType() + + def visit_deleted_type(self, t: DeletedType) -> Type: + if not isinstance(self.s, Void) and not isinstance(self.s, ErrorType): + if isinstance(self.s, NoneTyp): + if experiments.STRICT_OPTIONAL: + return t + else: + return self.s + elif isinstance(self.s, UninhabitedType): + return self.s + else: + return t + else: + return ErrorType() + + def visit_erased_type(self, t: ErasedType) -> Type: + return self.s + + def visit_type_var(self, t: TypeVarType) -> Type: + if isinstance(self.s, TypeVarType) and self.s.id == t.id: + return self.s + else: + return self.default(self.s) + + def visit_instance(self, t: Instance) -> Type: + if isinstance(self.s, Instance): + si = self.s + if t.type == si.type: + if is_subtype(t, self.s) or is_subtype(self.s, t): + # Combine type arguments. We could have used join below + # equivalently. + args = [] # type: List[Type] + for i in range(len(t.args)): + args.append(self.meet(t.args[i], si.args[i])) + return Instance(t.type, args) + else: + if experiments.STRICT_OPTIONAL: + return UninhabitedType() + else: + return NoneTyp() + else: + if is_subtype(t, self.s): + return t + elif is_subtype(self.s, t): + # See also above comment. + return self.s + else: + if experiments.STRICT_OPTIONAL: + return UninhabitedType() + else: + return NoneTyp() + elif isinstance(self.s, TypeType): + return meet_types(t, self.s) + else: + return self.default(self.s) + + def visit_callable_type(self, t: CallableType) -> Type: + if isinstance(self.s, CallableType) and is_similar_callables(t, self.s): + return combine_similar_callables(t, self.s) + else: + return self.default(self.s) + + def visit_tuple_type(self, t: TupleType) -> Type: + if isinstance(self.s, TupleType) and self.s.length() == t.length(): + items = [] # type: List[Type] + for i in range(t.length()): + items.append(self.meet(t.items[i], self.s.items[i])) + # TODO: What if the fallbacks are different? + return TupleType(items, t.fallback) + else: + return self.default(self.s) + + def visit_partial_type(self, t: PartialType) -> Type: + # We can't determine the meet of partial types. We should never get here. + assert False, 'Internal error' + + def visit_type_type(self, t: TypeType) -> Type: + if isinstance(self.s, TypeType): + typ = self.meet(t.item, self.s.item) + if not isinstance(typ, NoneTyp): + typ = TypeType(typ, line=t.line) + return typ + elif isinstance(self.s, Instance) and self.s.type.fullname() == 'builtins.type': + return t + else: + return self.default(self.s) + + def meet(self, s, t): + return meet_types(s, t) + + def default(self, typ): + if isinstance(typ, UnboundType): + return AnyType() + elif isinstance(typ, Void) or isinstance(typ, ErrorType): + return ErrorType() + else: + if experiments.STRICT_OPTIONAL: + return UninhabitedType() + else: + return NoneTyp() diff --git a/mypy/messages.py b/mypy/messages.py index cc9b4761b703a..5d82c4845782a 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -1,920 +1,920 @@ -"""Facilities and constants for generating error messages during type checking. - -The type checker itself does not deal with message string literals to -improve code clarity and to simplify localization (in the future).""" - -import re -import difflib - -from typing import cast, List, Dict, Any, Sequence, Iterable, Tuple - -from mypy.errors import Errors -from mypy.types import ( - Type, CallableType, Instance, TypeVarType, TupleType, UnionType, Void, NoneTyp, AnyType, - Overloaded, FunctionLike, DeletedType, TypeType -) -from mypy.nodes import ( - TypeInfo, Context, MypyFile, op_methods, FuncDef, reverse_type_aliases, - ARG_STAR, ARG_STAR2 -) - - -# Constants that represent simple type checker error message, i.e. messages -# that do not have any parameters. - -NO_RETURN_VALUE_EXPECTED = 'No return value expected' -INCOMPATIBLE_RETURN_VALUE_TYPE = 'Incompatible return value type' -RETURN_VALUE_EXPECTED = 'Return value expected' -BOOLEAN_VALUE_EXPECTED = 'Boolean value expected' -BOOLEAN_EXPECTED_FOR_IF = 'Boolean value expected for if condition' -BOOLEAN_EXPECTED_FOR_WHILE = 'Boolean value expected for while condition' -BOOLEAN_EXPECTED_FOR_UNTIL = 'Boolean value expected for until condition' -BOOLEAN_EXPECTED_FOR_NOT = 'Boolean value expected for not operand' -INVALID_EXCEPTION = 'Exception must be derived from BaseException' -INVALID_EXCEPTION_TYPE = 'Exception type must be derived from BaseException' -INVALID_RETURN_TYPE_FOR_GENERATOR = \ - 'The return type of a generator function should be "Generator" or one of its supertypes' -INVALID_GENERATOR_RETURN_ITEM_TYPE = \ - 'The return type of a generator function must be None in its third type parameter in Python 2' -YIELD_VALUE_EXPECTED = 'Yield value expected' -INCOMPATIBLE_TYPES = 'Incompatible types' -INCOMPATIBLE_TYPES_IN_ASSIGNMENT = 'Incompatible types in assignment' -INCOMPATIBLE_REDEFINITION = 'Incompatible redefinition' -INCOMPATIBLE_TYPES_IN_YIELD = 'Incompatible types in yield' -INCOMPATIBLE_TYPES_IN_YIELD_FROM = 'Incompatible types in "yield from"' -INCOMPATIBLE_TYPES_IN_STR_INTERPOLATION = 'Incompatible types in string interpolation' -INIT_MUST_HAVE_NONE_RETURN_TYPE = 'The return type of "__init__" must be None' -GETTER_TYPE_INCOMPATIBLE_WITH_SETTER = \ - 'Type of getter incompatible with setter' -TUPLE_INDEX_MUST_BE_AN_INT_LITERAL = 'Tuple index must be an integer literal' -TUPLE_SLICE_MUST_BE_AN_INT_LITERAL = 'Tuple slice must be an integer literal' -TUPLE_INDEX_OUT_OF_RANGE = 'Tuple index out of range' -TYPE_CONSTANT_EXPECTED = 'Type "Constant" or initializer expected' -INCOMPATIBLE_PAIR_ITEM_TYPE = 'Incompatible Pair item type' -INVALID_TYPE_APPLICATION_TARGET_TYPE = 'Invalid type application target type' -INCOMPATIBLE_TUPLE_ITEM_TYPE = 'Incompatible tuple item type' -INCOMPATIBLE_KEY_TYPE = 'Incompatible dictionary key type' -INCOMPATIBLE_VALUE_TYPE = 'Incompatible dictionary value type' -NEED_ANNOTATION_FOR_VAR = 'Need type annotation for variable' -ITERABLE_EXPECTED = 'Iterable expected' -INCOMPATIBLE_TYPES_IN_FOR = 'Incompatible types in for statement' -INCOMPATIBLE_ARRAY_VAR_ARGS = 'Incompatible variable arguments in call' -INVALID_SLICE_INDEX = 'Slice index must be an integer or None' -CANNOT_INFER_LAMBDA_TYPE = 'Cannot infer type of lambda' -CANNOT_INFER_ITEM_TYPE = 'Cannot infer iterable item type' -CANNOT_ACCESS_INIT = 'Cannot access "__init__" directly' -CANNOT_ASSIGN_TO_METHOD = 'Cannot assign to a method' -CANNOT_ASSIGN_TO_TYPE = 'Cannot assign to a type' -INCONSISTENT_ABSTRACT_OVERLOAD = \ - 'Overloaded method has both abstract and non-abstract variants' -READ_ONLY_PROPERTY_OVERRIDES_READ_WRITE = \ - 'Read-only property cannot override read-write property' -INSTANCE_LAYOUT_CONFLICT = 'Instance layout conflict in multiple inheritance' -FORMAT_REQUIRES_MAPPING = 'Format requires a mapping' -GENERIC_TYPE_NOT_VALID_AS_EXPRESSION = \ - "Generic type is prohibited as a runtime expression (use a type alias or '# type:' comment)" -RETURN_TYPE_CANNOT_BE_CONTRAVARIANT = "Cannot use a contravariant type variable as return type" -FUNCTION_PARAMETER_CANNOT_BE_COVARIANT = "Cannot use a covariant type variable as a parameter" -INCOMPATIBLE_IMPORT_OF = "Incompatible import of" -FUNCTION_TYPE_EXPECTED = "Function is missing a type annotation" -RETURN_TYPE_EXPECTED = "Function is missing a return type annotation" -ARGUMENT_TYPE_EXPECTED = "Function is missing a type annotation for one or more arguments" -KEYWORD_ARGUMENT_REQUIRES_STR_KEY_TYPE = \ - 'Keyword argument only valid with "str" key type in call to "dict"' -ALL_MUST_BE_SEQ_STR = 'Type of __all__ must be {}, not {}' - - -class MessageBuilder: - """Helper class for reporting type checker error messages with parameters. - - The methods of this class need to be provided with the context within a - file; the errors member manages the wider context. - - IDEA: Support a 'verbose mode' that includes full information about types - in error messages and that may otherwise produce more detailed error - messages. - """ - - # Report errors using this instance. It knows about the current file and - # import context. - errors = None # type: Errors - - modules = None # type: Dict[str, MypyFile] - - # Number of times errors have been disabled. - disable_count = 0 - - # Hack to deduplicate error messages from union types - disable_type_names = 0 - - def __init__(self, errors: Errors, modules: Dict[str, MypyFile]) -> None: - self.errors = errors - self.modules = modules - self.disable_count = 0 - self.disable_type_names = 0 - - # - # Helpers - # - - def copy(self) -> 'MessageBuilder': - new = MessageBuilder(self.errors.copy(), self.modules) - new.disable_count = self.disable_count - new.disable_type_names = self.disable_type_names - return new - - def add_errors(self, messages: 'MessageBuilder') -> None: - """Add errors in messages to this builder.""" - if self.disable_count <= 0: - for info in messages.errors.error_info: - self.errors.add_error_info(info) - - def disable_errors(self) -> None: - self.disable_count += 1 - - def enable_errors(self) -> None: - self.disable_count -= 1 - - def is_errors(self) -> bool: - return self.errors.is_errors() - - def report(self, msg: str, context: Context, severity: str, file: str = None) -> None: - """Report an error or note (unless disabled).""" - if self.disable_count <= 0: - self.errors.report(context.get_line() if context else -1, - msg.strip(), severity=severity, file=file) - - def fail(self, msg: str, context: Context, file: str = None) -> None: - """Report an error message (unless disabled).""" - # import pdb; pdb.set_trace() - self.report(msg, context, 'error', file=file) - - def note(self, msg: str, context: Context, file: str = None) -> None: - """Report an error message (unless disabled).""" - self.report(msg, context, 'note', file=file) - - def format(self, typ: Type, verbosity: int = 0) -> str: - """Convert a type to a relatively short string that is suitable for error messages. - - Mostly behave like format_simple below, but never return an empty string. - """ - s = self.format_simple(typ, verbosity) - if s != '': - # If format_simple returns a non-trivial result, use that. - return s - elif isinstance(typ, FunctionLike): - func = typ - if func.is_type_obj(): - # The type of a type object type can be derived from the - # return type (this always works). - itype = cast(Instance, func.items()[0].ret_type) - result = self.format(itype) - if verbosity >= 1: - # In some contexts we want to be explicit about the distinction - # between type X and the type of type object X. - result += ' (type object)' - return result - elif isinstance(func, CallableType): - return_type = strip_quotes(self.format(func.ret_type)) - if func.is_ellipsis_args: - return 'Callable[..., {}]'.format(return_type) - arg_types = [strip_quotes(self.format(t)) for t in func.arg_types] - return 'Callable[[{}], {}]'.format(", ".join(arg_types), return_type) - else: - # Use a simple representation for function types; proper - # function types may result in long and difficult-to-read - # error messages. - return 'overloaded function' - else: - # Default case; we simply have to return something meaningful here. - return 'object' - - def format_simple(self, typ: Type, verbosity: int = 0) -> str: - """Convert simple types to string that is suitable for error messages. - - Return "" for complex types. Try to keep the length of the result - relatively short to avoid overly long error messages. - - Examples: - builtins.int -> 'int' - Any type -> 'Any' - None -> None - callable type -> "" (empty string) - """ - if isinstance(typ, Instance): - itype = typ - # Get the short name of the type. - if verbosity >= 2: - base_str = itype.type.fullname() - else: - base_str = itype.type.name() - if itype.args == []: - # No type arguments. Place the type name in quotes to avoid - # potential for confusion: otherwise, the type name could be - # interpreted as a normal word. - return '"{}"'.format(base_str) - elif itype.type.fullname() == 'builtins.tuple': - item_type = strip_quotes(self.format(itype.args[0])) - return 'Tuple[{}, ...]'.format(item_type) - elif itype.type.fullname() in reverse_type_aliases: - alias = reverse_type_aliases[itype.type.fullname()] - alias = alias.split('.')[-1] - items = [strip_quotes(self.format(arg)) for arg in itype.args] - return '{}[{}]'.format(alias, ', '.join(items)) - else: - # There are type arguments. Convert the arguments to strings - # (using format() instead of format_simple() to avoid empty - # strings). If the result is too long, replace arguments - # with [...]. - a = [] # type: List[str] - for arg in itype.args: - a.append(strip_quotes(self.format(arg))) - s = ', '.join(a) - if len((base_str + s)) < 25: - return '{}[{}]'.format(base_str, s) - else: - return '{}[...]'.format(base_str) - elif isinstance(typ, TypeVarType): - # This is similar to non-generic instance types. - return '"{}"'.format(typ.name) - elif isinstance(typ, TupleType): - # Prefer the name of the fallback class (if not tuple), as it's more informative. - if typ.fallback.type.fullname() != 'builtins.tuple': - return self.format_simple(typ.fallback) - items = [] - for t in typ.items: - items.append(strip_quotes(self.format(t))) - s = '"Tuple[{}]"'.format(', '.join(items)) - if len(s) < 40: - return s - else: - return 'tuple(length {})'.format(len(items)) - elif isinstance(typ, UnionType): - # Only print Unions as Optionals if the Optional wouldn't have to contain another Union - print_as_optional = (len(typ.items) - - sum(isinstance(t, NoneTyp) for t in typ.items) == 1) - if print_as_optional: - rest = [t for t in typ.items if not isinstance(t, NoneTyp)] - return '"Optional[{}]"'.format(strip_quotes(self.format(rest[0]))) - else: - items = [] - for t in typ.items: - items.append(strip_quotes(self.format(t))) - s = '"Union[{}]"'.format(', '.join(items)) - if len(s) < 40: - return s - else: - return 'union type ({} items)'.format(len(items)) - elif isinstance(typ, Void): - return 'None' - elif isinstance(typ, NoneTyp): - return 'None' - elif isinstance(typ, AnyType): - return '"Any"' - elif isinstance(typ, DeletedType): - return '' - elif isinstance(typ, TypeType): - return 'Type[{}]'.format( - strip_quotes(self.format_simple(typ.item, verbosity))) - elif typ is None: - raise RuntimeError('Type is None') - else: - # No simple representation for this type that would convey very - # useful information. No need to mention the type explicitly in a - # message. - return '' - - def format_distinctly(self, type1: Type, type2: Type) -> Tuple[str, str]: - """Jointly format a pair of types to distinct strings. - - Increase the verbosity of the type strings until they become distinct. - """ - verbosity = 0 - for verbosity in range(3): - str1 = self.format(type1, verbosity=verbosity) - str2 = self.format(type2, verbosity=verbosity) - if str1 != str2: - return (str1, str2) - return (str1, str2) - - # - # Specific operations - # - - # The following operations are for generating specific error messages. They - # get some information as arguments, and they build an error message based - # on them. - - def has_no_attr(self, typ: Type, member: str, context: Context) -> Type: - """Report a missing or non-accessible member. - - The type argument is the base type. If member corresponds to - an operator, use the corresponding operator name in the - messages. Return type Any. - """ - if (isinstance(typ, Instance) and - typ.type.has_readable_member(member)): - self.fail('Member "{}" is not assignable'.format(member), context) - elif isinstance(typ, Void): - self.check_void(typ, context) - elif member == '__contains__': - self.fail('Unsupported right operand type for in ({})'.format( - self.format(typ)), context) - elif member in op_methods.values(): - # Access to a binary operator member (e.g. _add). This case does - # not handle indexing operations. - for op, method in op_methods.items(): - if method == member: - self.unsupported_left_operand(op, typ, context) - break - elif member == '__neg__': - self.fail('Unsupported operand type for unary - ({})'.format( - self.format(typ)), context) - elif member == '__pos__': - self.fail('Unsupported operand type for unary + ({})'.format( - self.format(typ)), context) - elif member == '__invert__': - self.fail('Unsupported operand type for ~ ({})'.format( - self.format(typ)), context) - elif member == '__getitem__': - # Indexed get. - self.fail('Value of type {} is not indexable'.format( - self.format(typ)), context) - elif member == '__setitem__': - # Indexed set. - self.fail('Unsupported target for indexed assignment', context) - elif member == '__call__': - self.fail('{} not callable'.format(self.format(typ)), context) - else: - # The non-special case: a missing ordinary attribute. - if not self.disable_type_names: - failed = False - if isinstance(typ, Instance) and typ.type.names: - alternatives = set(typ.type.names.keys()) - matches = [m for m in COMMON_MISTAKES.get(member, []) if m in alternatives] - matches.extend(best_matches(member, alternatives)[:3]) - if matches: - self.fail('{} has no attribute "{}"; maybe {}?'.format( - self.format(typ), member, pretty_or(matches)), context) - failed = True - if not failed: - self.fail('{} has no attribute "{}"'.format(self.format(typ), - member), context) - else: - self.fail('Some element of union has no attribute "{}"'.format( - member), context) - return AnyType() - - def unsupported_operand_types(self, op: str, left_type: Any, - right_type: Any, context: Context) -> None: - """Report unsupported operand types for a binary operation. - - Types can be Type objects or strings. - """ - if isinstance(left_type, Void) or isinstance(right_type, Void): - self.check_void(left_type, context) - self.check_void(right_type, context) - return - left_str = '' - if isinstance(left_type, str): - left_str = left_type - else: - left_str = self.format(left_type) - - right_str = '' - if isinstance(right_type, str): - right_str = right_type - else: - right_str = self.format(right_type) - - if self.disable_type_names: - msg = 'Unsupported operand types for {} (likely involving Union)'.format(op) - else: - msg = 'Unsupported operand types for {} ({} and {})'.format( - op, left_str, right_str) - self.fail(msg, context) - - def unsupported_left_operand(self, op: str, typ: Type, - context: Context) -> None: - if not self.check_void(typ, context): - if self.disable_type_names: - msg = 'Unsupported left operand type for {} (some union)'.format(op) - else: - msg = 'Unsupported left operand type for {} ({})'.format( - op, self.format(typ)) - self.fail(msg, context) - - def type_expected_as_right_operand_of_is(self, context: Context) -> None: - self.fail('Type expected as right operand of "is"', context) - - def not_callable(self, typ: Type, context: Context) -> Type: - self.fail('{} not callable'.format(self.format(typ)), context) - return AnyType() - - def untyped_function_call(self, callee: CallableType, context: Context) -> Type: - name = callee.name if callee.name is not None else '(unknown)' - self.fail('call to untyped function {} in typed context'.format(name), context) - return AnyType() - - def incompatible_argument(self, n: int, m: int, callee: CallableType, arg_type: Type, - arg_kind: int, context: Context) -> None: - """Report an error about an incompatible argument type. - - The argument type is arg_type, argument number is n and the - callee type is 'callee'. If the callee represents a method - that corresponds to an operator, use the corresponding - operator name in the messages. - """ - target = '' - if callee.name: - name = callee.name - base = extract_type(name) - - for op, method in op_methods.items(): - for variant in method, '__r' + method[2:]: - if name.startswith('"{}" of'.format(variant)): - if op == 'in' or variant != method: - # Reversed order of base/argument. - self.unsupported_operand_types(op, arg_type, base, - context) - else: - self.unsupported_operand_types(op, base, arg_type, - context) - return - - if name.startswith('"__getitem__" of'): - self.invalid_index_type(arg_type, base, context) - return - - if name.startswith('"__setitem__" of'): - if n == 1: - self.invalid_index_type(arg_type, base, context) - else: - msg = '{} (expression has type {}, target has type {})' - arg_type_str, callee_type_str = self.format_distinctly(arg_type, - callee.arg_types[n - 1]) - self.fail(msg.format(INCOMPATIBLE_TYPES_IN_ASSIGNMENT, - arg_type_str, callee_type_str), - context) - return - - target = 'to {} '.format(name) - - msg = '' - if callee.name == '': - name = callee.name[1:-1] - n -= 1 - msg = '{} item {} has incompatible type {}'.format( - name[0].upper() + name[1:], n, self.format_simple(arg_type)) - elif callee.name == '': - msg = 'List comprehension has incompatible type List[{}]'.format( - strip_quotes(self.format(arg_type))) - elif callee.name == '': - msg = 'Set comprehension has incompatible type Set[{}]'.format( - strip_quotes(self.format(arg_type))) - elif callee.name == '': - msg = ('{} expression in dictionary comprehension has incompatible type {}; ' - 'expected type {}').format( - 'Key' if n == 1 else 'Value', - self.format(arg_type), - self.format(callee.arg_types[n - 1])) - elif callee.name == '': - msg = 'Generator has incompatible item type {}'.format( - self.format_simple(arg_type)) - else: - try: - expected_type = callee.arg_types[m - 1] - except IndexError: # Varargs callees - expected_type = callee.arg_types[-1] - arg_type_str, expected_type_str = self.format_distinctly(arg_type, expected_type) - #import pdb; pdb.set_trace() - if arg_kind == ARG_STAR: - arg_type_str = '*' + arg_type_str - elif arg_kind == ARG_STAR2: - arg_type_str = '**' + arg_type_str - msg = 'Argument {} {}has incompatible type {}; expected {}'.format( - n, target, arg_type_str, expected_type_str) - self.fail(msg, context) - - def invalid_index_type(self, index_type: Type, base_str: str, - context: Context) -> None: - self.fail('Invalid index type {} for {}'.format( - self.format(index_type), base_str), context) - - def too_few_arguments(self, callee: CallableType, context: Context, - argument_names: List[str]) -> None: - if (argument_names is not None and not all(k is None for k in argument_names) - and len(argument_names) >= 1): - diff = [k for k in callee.arg_names if k not in argument_names] - if len(diff) == 1: - msg = 'Missing positional argument' - else: - msg = 'Missing positional arguments' - if callee.name and diff: - msg += ' "{}" in call to {}'.format('", "'.join(diff), callee.name) - else: - msg = 'Too few arguments' - if callee.name: - msg += ' for {}'.format(callee.name) - self.fail(msg, context) - - def too_many_arguments(self, callee: CallableType, context: Context) -> None: - msg = 'Too many arguments' - if callee.name: - msg += ' for {}'.format(callee.name) - self.fail(msg, context) - - def too_many_positional_arguments(self, callee: CallableType, - context: Context) -> None: - msg = 'Too many positional arguments' - if callee.name: - msg += ' for {}'.format(callee.name) - self.fail(msg, context) - - def unexpected_keyword_argument(self, callee: CallableType, name: str, - context: Context) -> None: - msg = 'Unexpected keyword argument "{}"'.format(name) - if callee.name: - msg += ' for {}'.format(callee.name) - self.fail(msg, context) - if callee.definition: - fullname = callee.definition.fullname() - if fullname is not None and '.' in fullname: - module_name = fullname.rsplit('.', 1)[0] - path = self.modules[module_name].path - self.note('{} defined here'.format(callee.name), callee.definition, file=path) - - def duplicate_argument_value(self, callee: CallableType, index: int, - context: Context) -> None: - self.fail('{} gets multiple values for keyword argument "{}"'. - format(capitalize(callable_name(callee)), - callee.arg_names[index]), context) - - def does_not_return_value(self, void_type: Type, context: Context) -> None: - """Report an error about a void type in a non-void context. - - The first argument must be a void type. If the void type has a - source in it, report it in the error message. This allows - giving messages such as 'Foo does not return a value'. - """ - if (cast(Void, void_type)).source is None: - self.fail('Function does not return a value', context) - else: - self.fail('{} does not return a value'.format( - capitalize((cast(Void, void_type)).source)), context) - - def deleted_as_rvalue(self, typ: DeletedType, context: Context) -> None: - """Report an error about using an deleted type as an rvalue.""" - if typ.source is None: - s = "" - else: - s = " '{}'".format(typ.source) - self.fail('Trying to read deleted variable{}'.format(s), context) - - def deleted_as_lvalue(self, typ: DeletedType, context: Context) -> None: - """Report an error about using an deleted type as an lvalue. - - Currently, this only occurs when trying to assign to an - exception variable outside the local except: blocks. - """ - if typ.source is None: - s = "" - else: - s = " '{}'".format(typ.source) - self.fail('Assignment to variable{} outside except: block'.format(s), context) - - def no_variant_matches_arguments(self, overload: Overloaded, arg_types: List[Type], - context: Context) -> None: - if overload.name(): - self.fail('No overload variant of {} matches argument types {}' - .format(overload.name(), arg_types), context) - else: - self.fail('No overload variant matches argument types {}'.format(arg_types), context) - - def function_variants_overlap(self, n1: int, n2: int, - context: Context) -> None: - self.fail('Function signature variants {} and {} overlap'.format( - n1 + 1, n2 + 1), context) - - def invalid_cast(self, target_type: Type, source_type: Type, - context: Context) -> None: - if not self.check_void(source_type, context): - self.fail('Cannot cast from {} to {}'.format( - self.format(source_type), self.format(target_type)), context) - - def wrong_number_values_to_unpack(self, provided: int, expected: int, - context: Context) -> None: - if provided < expected: - if provided == 1: - self.fail('Need more than 1 value to unpack ({} expected)'.format(expected), - context) - else: - self.fail('Need more than {} values to unpack ({} expected)'.format( - provided, expected), context) - elif provided > expected: - self.fail('Too many values to unpack ({} expected, {} provided)'.format( - expected, provided), context) - - def type_not_iterable(self, type: Type, context: Context) -> None: - self.fail('\'{}\' object is not iterable'.format(type), context) - - def incompatible_operator_assignment(self, op: str, - context: Context) -> None: - self.fail('Result type of {} incompatible in assignment'.format(op), - context) - - def incompatible_value_count_in_assignment(self, lvalue_count: int, - rvalue_count: int, - context: Context) -> None: - if rvalue_count < lvalue_count: - self.fail('Need {} values to assign'.format(lvalue_count), context) - elif rvalue_count > lvalue_count: - self.fail('Too many values to assign', context) - - def type_incompatible_with_supertype(self, name: str, supertype: TypeInfo, - context: Context) -> None: - self.fail('Type of "{}" incompatible with supertype "{}"'.format( - name, supertype.name), context) - - def signature_incompatible_with_supertype( - self, name: str, name_in_super: str, supertype: str, - context: Context) -> None: - target = self.override_target(name, name_in_super, supertype) - self.fail('Signature of "{}" incompatible with {}'.format( - name, target), context) - - def argument_incompatible_with_supertype( - self, arg_num: int, name: str, name_in_supertype: str, - supertype: str, context: Context) -> None: - target = self.override_target(name, name_in_supertype, supertype) - self.fail('Argument {} of "{}" incompatible with {}' - .format(arg_num, name, target), context) - - def return_type_incompatible_with_supertype( - self, name: str, name_in_supertype: str, supertype: str, - context: Context) -> None: - target = self.override_target(name, name_in_supertype, supertype) - self.fail('Return type of "{}" incompatible with {}' - .format(name, target), context) - - def override_target(self, name: str, name_in_super: str, - supertype: str) -> str: - target = 'supertype "{}"'.format(supertype) - if name_in_super != name: - target = '"{}" of {}'.format(name_in_super, target) - return target - - def boolean_return_value_expected(self, method: str, - context: Context) -> None: - self.fail('Boolean return value expected for method "{}"'.format( - method), context) - - def incompatible_type_application(self, expected_arg_count: int, - actual_arg_count: int, - context: Context) -> None: - if expected_arg_count == 0: - self.fail('Type application targets a non-generic function', - context) - elif actual_arg_count > expected_arg_count: - self.fail('Type application has too many types ({} expected)' - .format(expected_arg_count), context) - else: - self.fail('Type application has too few types ({} expected)' - .format(expected_arg_count), context) - - def incompatible_array_item_type(self, typ: Type, index: int, - context: Context) -> None: - self.fail('Array item {} has incompatible type {}'.format( - index, self.format(typ)), context) - - def could_not_infer_type_arguments(self, callee_type: CallableType, n: int, - context: Context) -> None: - if callee_type.name and n > 0: - self.fail('Cannot infer type argument {} of {}'.format( - n, callee_type.name), context) - else: - self.fail('Cannot infer function type argument', context) - - def invalid_var_arg(self, typ: Type, context: Context) -> None: - self.fail('List or tuple expected as variable arguments', context) - - def invalid_keyword_var_arg(self, typ: Type, context: Context) -> None: - if isinstance(typ, Instance) and (typ.type.fullname() == 'builtins.dict'): - self.fail('Keywords must be strings', context) - else: - self.fail('Argument after ** must be a dictionary', - context) - - def incomplete_type_var_match(self, member: str, context: Context) -> None: - self.fail('"{}" has incomplete match to supertype type variable' - .format(member), context) - - def not_implemented(self, msg: str, context: Context) -> Type: - self.fail('Feature not implemented yet ({})'.format(msg), context) - return AnyType() - - def undefined_in_superclass(self, member: str, context: Context) -> None: - self.fail('"{}" undefined in superclass'.format(member), context) - - def check_void(self, typ: Type, context: Context) -> bool: - """If type is void, report an error such as '.. does not - return a value' and return True. Otherwise, return False. - """ - if isinstance(typ, Void): - self.does_not_return_value(typ, context) - return True - else: - return False - - def too_few_string_formatting_arguments(self, context: Context) -> None: - self.fail('Not enough arguments for format string', context) - - def too_many_string_formatting_arguments(self, context: Context) -> None: - self.fail('Not all arguments converted during string formatting', context) - - def incomplete_conversion_specifier_format(self, context: Context) -> None: - self.fail('Incomplete format', context) - - def unsupported_placeholder(self, placeholder: str, context: Context) -> None: - self.fail('Unsupported format character \'%s\'' % placeholder, context) - - def string_interpolation_with_star_and_key(self, context: Context) -> None: - self.fail('String interpolation contains both stars and mapping keys', context) - - def requires_int_or_char(self, context: Context) -> None: - self.fail('%c requires int or char', context) - - def key_not_in_mapping(self, key: str, context: Context) -> None: - self.fail('Key \'%s\' not found in mapping' % key, context) - - def string_interpolation_mixing_key_and_non_keys(self, context: Context) -> None: - self.fail('String interpolation mixes specifier with and without mapping keys', context) - - def cannot_determine_type(self, name: str, context: Context) -> None: - self.fail("Cannot determine type of '%s'" % name, context) - - def cannot_determine_type_in_base(self, name: str, base: str, context: Context) -> None: - self.fail("Cannot determine type of '%s' in base class '%s'" % (name, base), context) - - def invalid_method_type(self, sig: CallableType, context: Context) -> None: - self.fail('Invalid method type', context) - - def invalid_class_method_type(self, sig: CallableType, context: Context) -> None: - self.fail('Invalid class method type', context) - - def incompatible_conditional_function_def(self, defn: FuncDef) -> None: - self.fail('All conditional function variants must have identical ' - 'signatures', defn) - - def cannot_instantiate_abstract_class(self, class_name: str, - abstract_attributes: List[str], - context: Context) -> None: - attrs = format_string_list("'%s'" % a for a in abstract_attributes[:5]) - self.fail("Cannot instantiate abstract class '%s' with abstract " - "attribute%s %s" % (class_name, plural_s(abstract_attributes), - attrs), - context) - - def base_class_definitions_incompatible(self, name: str, base1: TypeInfo, - base2: TypeInfo, - context: Context) -> None: - self.fail('Definition of "{}" in base class "{}" is incompatible ' - 'with definition in base class "{}"'.format( - name, base1.name(), base2.name()), context) - - def cant_assign_to_method(self, context: Context) -> None: - self.fail(CANNOT_ASSIGN_TO_METHOD, context) - - def read_only_property(self, name: str, type: TypeInfo, - context: Context) -> None: - self.fail('Property "{}" defined in "{}" is read-only'.format( - name, type.name()), context) - - def incompatible_typevar_value(self, callee: CallableType, index: int, - type: Type, context: Context) -> None: - self.fail('Type argument {} of {} has incompatible value {}'.format( - index, callable_name(callee), self.format(type)), context) - - def overloaded_signatures_overlap(self, index1: int, index2: int, - context: Context) -> None: - self.fail('Overloaded function signatures {} and {} overlap with ' - 'incompatible return types'.format(index1, index2), context) - - def invalid_reverse_operator_signature(self, reverse: str, other: str, - context: Context) -> None: - self.fail('"Any" return type expected since argument to {} ' - 'does not support {}'.format(reverse, other), context) - - def reverse_operator_method_with_any_arg_must_return_any( - self, method: str, context: Context) -> None: - self.fail('"Any" return type expected since argument to {} ' - 'has type "Any"'.format(method), context) - - def operator_method_signatures_overlap( - self, reverse_class: str, reverse_method: str, forward_class: str, - forward_method: str, context: Context) -> None: - self.fail('Signatures of "{}" of "{}" and "{}" of "{}" ' - 'are unsafely overlapping'.format( - reverse_method, reverse_class, - forward_method, forward_class), - context) - - def forward_operator_not_callable( - self, forward_method: str, context: Context) -> None: - self.fail('Forward operator "{}" is not callable'.format( - forward_method), context) - - def signatures_incompatible(self, method: str, other_method: str, - context: Context) -> None: - self.fail('Signatures of "{}" and "{}" are incompatible'.format( - method, other_method), context) - - def yield_from_invalid_operand_type(self, expr: Type, context: Context) -> Type: - text = self.format(expr) if self.format(expr) != 'object' else expr - self.fail('"yield from" can\'t be applied to {}'.format(text), context) - return AnyType() - - def invalid_signature(self, func_type: Type, context: Context) -> None: - self.fail('Invalid signature "{}"'.format(func_type), context) - - def reveal_type(self, typ: Type, context: Context) -> None: - self.fail('Revealed type is \'{}\''.format(typ), context) - - def unsupported_type_type(self, item: Type, context: Context) -> None: - self.fail('Unsupported type Type[{}]'.format(self.format(item)), context) - - def redundant_cast(self, typ: Type, context: Context) -> None: - self.note('Redundant cast to {}'.format(self.format(typ)), context) - - -def capitalize(s: str) -> str: - """Capitalize the first character of a string.""" - if s == '': - return '' - else: - return s[0].upper() + s[1:] - - -def extract_type(name: str) -> str: - """If the argument is the name of a method (of form C.m), return - the type portion in quotes (e.g. "y"). Otherwise, return the string - unmodified. - """ - name = re.sub('^"[a-zA-Z0-9_]+" of ', '', name) - return name - - -def strip_quotes(s: str) -> str: - """Strip a double quote at the beginning and end of the string, if any.""" - s = re.sub('^"', '', s) - s = re.sub('"$', '', s) - return s - - -def plural_s(s: Sequence[Any]) -> str: - if len(s) > 1: - return 's' - else: - return '' - - -def format_string_list(s: Iterable[str]) -> str: - l = list(s) - assert len(l) > 0 - if len(l) == 1: - return l[0] - else: - return '%s and %s' % (', '.join(l[:-1]), l[-1]) - - -def callable_name(type: CallableType) -> str: - if type.name: - return type.name - else: - return 'function' - - -def temp_message_builder() -> MessageBuilder: - """Return a message builder usable for throwaway errors (which may not format properly).""" - return MessageBuilder(Errors(), {}) - - -# For hard-coding suggested missing member alternatives. -COMMON_MISTAKES = { - 'add': ('append', 'extend'), -} # type: Dict[str, Sequence[str]] - - -def best_matches(current: str, options: Iterable[str]) -> List[str]: - ratios = {v: difflib.SequenceMatcher(a=current, b=v).ratio() for v in options} - return sorted((o for o in options if ratios[o] > 0.75), - reverse=True, key=lambda v: (ratios[v], v)) - - -def pretty_or(args: List[str]) -> str: - quoted = ['"' + a + '"' for a in args] - if len(quoted) == 1: - return quoted[0] - if len(quoted) == 2: - return "{} or {}".format(quoted[0], quoted[1]) - return ", ".join(quoted[:-1]) + ", or " + quoted[-1] +"""Facilities and constants for generating error messages during type checking. + +The type checker itself does not deal with message string literals to +improve code clarity and to simplify localization (in the future).""" + +import re +import difflib + +from typing import cast, List, Dict, Any, Sequence, Iterable, Tuple + +from mypy.errors import Errors +from mypy.types import ( + Type, CallableType, Instance, TypeVarType, TupleType, UnionType, Void, NoneTyp, AnyType, + Overloaded, FunctionLike, DeletedType, TypeType +) +from mypy.nodes import ( + TypeInfo, Context, MypyFile, op_methods, FuncDef, reverse_type_aliases, + ARG_STAR, ARG_STAR2 +) + + +# Constants that represent simple type checker error message, i.e. messages +# that do not have any parameters. + +NO_RETURN_VALUE_EXPECTED = 'No return value expected' +INCOMPATIBLE_RETURN_VALUE_TYPE = 'Incompatible return value type' +RETURN_VALUE_EXPECTED = 'Return value expected' +BOOLEAN_VALUE_EXPECTED = 'Boolean value expected' +BOOLEAN_EXPECTED_FOR_IF = 'Boolean value expected for if condition' +BOOLEAN_EXPECTED_FOR_WHILE = 'Boolean value expected for while condition' +BOOLEAN_EXPECTED_FOR_UNTIL = 'Boolean value expected for until condition' +BOOLEAN_EXPECTED_FOR_NOT = 'Boolean value expected for not operand' +INVALID_EXCEPTION = 'Exception must be derived from BaseException' +INVALID_EXCEPTION_TYPE = 'Exception type must be derived from BaseException' +INVALID_RETURN_TYPE_FOR_GENERATOR = \ + 'The return type of a generator function should be "Generator" or one of its supertypes' +INVALID_GENERATOR_RETURN_ITEM_TYPE = \ + 'The return type of a generator function must be None in its third type parameter in Python 2' +YIELD_VALUE_EXPECTED = 'Yield value expected' +INCOMPATIBLE_TYPES = 'Incompatible types' +INCOMPATIBLE_TYPES_IN_ASSIGNMENT = 'Incompatible types in assignment' +INCOMPATIBLE_REDEFINITION = 'Incompatible redefinition' +INCOMPATIBLE_TYPES_IN_YIELD = 'Incompatible types in yield' +INCOMPATIBLE_TYPES_IN_YIELD_FROM = 'Incompatible types in "yield from"' +INCOMPATIBLE_TYPES_IN_STR_INTERPOLATION = 'Incompatible types in string interpolation' +INIT_MUST_HAVE_NONE_RETURN_TYPE = 'The return type of "__init__" must be None' +GETTER_TYPE_INCOMPATIBLE_WITH_SETTER = \ + 'Type of getter incompatible with setter' +TUPLE_INDEX_MUST_BE_AN_INT_LITERAL = 'Tuple index must be an integer literal' +TUPLE_SLICE_MUST_BE_AN_INT_LITERAL = 'Tuple slice must be an integer literal' +TUPLE_INDEX_OUT_OF_RANGE = 'Tuple index out of range' +TYPE_CONSTANT_EXPECTED = 'Type "Constant" or initializer expected' +INCOMPATIBLE_PAIR_ITEM_TYPE = 'Incompatible Pair item type' +INVALID_TYPE_APPLICATION_TARGET_TYPE = 'Invalid type application target type' +INCOMPATIBLE_TUPLE_ITEM_TYPE = 'Incompatible tuple item type' +INCOMPATIBLE_KEY_TYPE = 'Incompatible dictionary key type' +INCOMPATIBLE_VALUE_TYPE = 'Incompatible dictionary value type' +NEED_ANNOTATION_FOR_VAR = 'Need type annotation for variable' +ITERABLE_EXPECTED = 'Iterable expected' +INCOMPATIBLE_TYPES_IN_FOR = 'Incompatible types in for statement' +INCOMPATIBLE_ARRAY_VAR_ARGS = 'Incompatible variable arguments in call' +INVALID_SLICE_INDEX = 'Slice index must be an integer or None' +CANNOT_INFER_LAMBDA_TYPE = 'Cannot infer type of lambda' +CANNOT_INFER_ITEM_TYPE = 'Cannot infer iterable item type' +CANNOT_ACCESS_INIT = 'Cannot access "__init__" directly' +CANNOT_ASSIGN_TO_METHOD = 'Cannot assign to a method' +CANNOT_ASSIGN_TO_TYPE = 'Cannot assign to a type' +INCONSISTENT_ABSTRACT_OVERLOAD = \ + 'Overloaded method has both abstract and non-abstract variants' +READ_ONLY_PROPERTY_OVERRIDES_READ_WRITE = \ + 'Read-only property cannot override read-write property' +INSTANCE_LAYOUT_CONFLICT = 'Instance layout conflict in multiple inheritance' +FORMAT_REQUIRES_MAPPING = 'Format requires a mapping' +GENERIC_TYPE_NOT_VALID_AS_EXPRESSION = \ + "Generic type is prohibited as a runtime expression (use a type alias or '# type:' comment)" +RETURN_TYPE_CANNOT_BE_CONTRAVARIANT = "Cannot use a contravariant type variable as return type" +FUNCTION_PARAMETER_CANNOT_BE_COVARIANT = "Cannot use a covariant type variable as a parameter" +INCOMPATIBLE_IMPORT_OF = "Incompatible import of" +FUNCTION_TYPE_EXPECTED = "Function is missing a type annotation" +RETURN_TYPE_EXPECTED = "Function is missing a return type annotation" +ARGUMENT_TYPE_EXPECTED = "Function is missing a type annotation for one or more arguments" +KEYWORD_ARGUMENT_REQUIRES_STR_KEY_TYPE = \ + 'Keyword argument only valid with "str" key type in call to "dict"' +ALL_MUST_BE_SEQ_STR = 'Type of __all__ must be {}, not {}' + + +class MessageBuilder: + """Helper class for reporting type checker error messages with parameters. + + The methods of this class need to be provided with the context within a + file; the errors member manages the wider context. + + IDEA: Support a 'verbose mode' that includes full information about types + in error messages and that may otherwise produce more detailed error + messages. + """ + + # Report errors using this instance. It knows about the current file and + # import context. + errors = None # type: Errors + + modules = None # type: Dict[str, MypyFile] + + # Number of times errors have been disabled. + disable_count = 0 + + # Hack to deduplicate error messages from union types + disable_type_names = 0 + + def __init__(self, errors: Errors, modules: Dict[str, MypyFile]) -> None: + self.errors = errors + self.modules = modules + self.disable_count = 0 + self.disable_type_names = 0 + + # + # Helpers + # + + def copy(self) -> 'MessageBuilder': + new = MessageBuilder(self.errors.copy(), self.modules) + new.disable_count = self.disable_count + new.disable_type_names = self.disable_type_names + return new + + def add_errors(self, messages: 'MessageBuilder') -> None: + """Add errors in messages to this builder.""" + if self.disable_count <= 0: + for info in messages.errors.error_info: + self.errors.add_error_info(info) + + def disable_errors(self) -> None: + self.disable_count += 1 + + def enable_errors(self) -> None: + self.disable_count -= 1 + + def is_errors(self) -> bool: + return self.errors.is_errors() + + def report(self, msg: str, context: Context, severity: str, file: str = None) -> None: + """Report an error or note (unless disabled).""" + if self.disable_count <= 0: + self.errors.report(context.get_line() if context else -1, + msg.strip(), severity=severity, file=file) + + def fail(self, msg: str, context: Context, file: str = None) -> None: + """Report an error message (unless disabled).""" + # import pdb; pdb.set_trace() + self.report(msg, context, 'error', file=file) + + def note(self, msg: str, context: Context, file: str = None) -> None: + """Report an error message (unless disabled).""" + self.report(msg, context, 'note', file=file) + + def format(self, typ: Type, verbosity: int = 0) -> str: + """Convert a type to a relatively short string that is suitable for error messages. + + Mostly behave like format_simple below, but never return an empty string. + """ + s = self.format_simple(typ, verbosity) + if s != '': + # If format_simple returns a non-trivial result, use that. + return s + elif isinstance(typ, FunctionLike): + func = typ + if func.is_type_obj(): + # The type of a type object type can be derived from the + # return type (this always works). + itype = cast(Instance, func.items()[0].ret_type) + result = self.format(itype) + if verbosity >= 1: + # In some contexts we want to be explicit about the distinction + # between type X and the type of type object X. + result += ' (type object)' + return result + elif isinstance(func, CallableType): + return_type = strip_quotes(self.format(func.ret_type)) + if func.is_ellipsis_args: + return 'Callable[..., {}]'.format(return_type) + arg_types = [strip_quotes(self.format(t)) for t in func.arg_types] + return 'Callable[[{}], {}]'.format(", ".join(arg_types), return_type) + else: + # Use a simple representation for function types; proper + # function types may result in long and difficult-to-read + # error messages. + return 'overloaded function' + else: + # Default case; we simply have to return something meaningful here. + return 'object' + + def format_simple(self, typ: Type, verbosity: int = 0) -> str: + """Convert simple types to string that is suitable for error messages. + + Return "" for complex types. Try to keep the length of the result + relatively short to avoid overly long error messages. + + Examples: + builtins.int -> 'int' + Any type -> 'Any' + None -> None + callable type -> "" (empty string) + """ + if isinstance(typ, Instance): + itype = typ + # Get the short name of the type. + if verbosity >= 2: + base_str = itype.type.fullname() + else: + base_str = itype.type.name() + if itype.args == []: + # No type arguments. Place the type name in quotes to avoid + # potential for confusion: otherwise, the type name could be + # interpreted as a normal word. + return '"{}"'.format(base_str) + elif itype.type.fullname() == 'builtins.tuple': + item_type = strip_quotes(self.format(itype.args[0])) + return 'Tuple[{}, ...]'.format(item_type) + elif itype.type.fullname() in reverse_type_aliases: + alias = reverse_type_aliases[itype.type.fullname()] + alias = alias.split('.')[-1] + items = [strip_quotes(self.format(arg)) for arg in itype.args] + return '{}[{}]'.format(alias, ', '.join(items)) + else: + # There are type arguments. Convert the arguments to strings + # (using format() instead of format_simple() to avoid empty + # strings). If the result is too long, replace arguments + # with [...]. + a = [] # type: List[str] + for arg in itype.args: + a.append(strip_quotes(self.format(arg))) + s = ', '.join(a) + if len((base_str + s)) < 25: + return '{}[{}]'.format(base_str, s) + else: + return '{}[...]'.format(base_str) + elif isinstance(typ, TypeVarType): + # This is similar to non-generic instance types. + return '"{}"'.format(typ.name) + elif isinstance(typ, TupleType): + # Prefer the name of the fallback class (if not tuple), as it's more informative. + if typ.fallback.type.fullname() != 'builtins.tuple': + return self.format_simple(typ.fallback) + items = [] + for t in typ.items: + items.append(strip_quotes(self.format(t))) + s = '"Tuple[{}]"'.format(', '.join(items)) + if len(s) < 40: + return s + else: + return 'tuple(length {})'.format(len(items)) + elif isinstance(typ, UnionType): + # Only print Unions as Optionals if the Optional wouldn't have to contain another Union + print_as_optional = (len(typ.items) - + sum(isinstance(t, NoneTyp) for t in typ.items) == 1) + if print_as_optional: + rest = [t for t in typ.items if not isinstance(t, NoneTyp)] + return '"Optional[{}]"'.format(strip_quotes(self.format(rest[0]))) + else: + items = [] + for t in typ.items: + items.append(strip_quotes(self.format(t))) + s = '"Union[{}]"'.format(', '.join(items)) + if len(s) < 40: + return s + else: + return 'union type ({} items)'.format(len(items)) + elif isinstance(typ, Void): + return 'None' + elif isinstance(typ, NoneTyp): + return 'None' + elif isinstance(typ, AnyType): + return '"Any"' + elif isinstance(typ, DeletedType): + return '' + elif isinstance(typ, TypeType): + return 'Type[{}]'.format( + strip_quotes(self.format_simple(typ.item, verbosity))) + elif typ is None: + raise RuntimeError('Type is None') + else: + # No simple representation for this type that would convey very + # useful information. No need to mention the type explicitly in a + # message. + return '' + + def format_distinctly(self, type1: Type, type2: Type) -> Tuple[str, str]: + """Jointly format a pair of types to distinct strings. + + Increase the verbosity of the type strings until they become distinct. + """ + verbosity = 0 + for verbosity in range(3): + str1 = self.format(type1, verbosity=verbosity) + str2 = self.format(type2, verbosity=verbosity) + if str1 != str2: + return (str1, str2) + return (str1, str2) + + # + # Specific operations + # + + # The following operations are for generating specific error messages. They + # get some information as arguments, and they build an error message based + # on them. + + def has_no_attr(self, typ: Type, member: str, context: Context) -> Type: + """Report a missing or non-accessible member. + + The type argument is the base type. If member corresponds to + an operator, use the corresponding operator name in the + messages. Return type Any. + """ + if (isinstance(typ, Instance) and + typ.type.has_readable_member(member)): + self.fail('Member "{}" is not assignable'.format(member), context) + elif isinstance(typ, Void): + self.check_void(typ, context) + elif member == '__contains__': + self.fail('Unsupported right operand type for in ({})'.format( + self.format(typ)), context) + elif member in op_methods.values(): + # Access to a binary operator member (e.g. _add). This case does + # not handle indexing operations. + for op, method in op_methods.items(): + if method == member: + self.unsupported_left_operand(op, typ, context) + break + elif member == '__neg__': + self.fail('Unsupported operand type for unary - ({})'.format( + self.format(typ)), context) + elif member == '__pos__': + self.fail('Unsupported operand type for unary + ({})'.format( + self.format(typ)), context) + elif member == '__invert__': + self.fail('Unsupported operand type for ~ ({})'.format( + self.format(typ)), context) + elif member == '__getitem__': + # Indexed get. + self.fail('Value of type {} is not indexable'.format( + self.format(typ)), context) + elif member == '__setitem__': + # Indexed set. + self.fail('Unsupported target for indexed assignment', context) + elif member == '__call__': + self.fail('{} not callable'.format(self.format(typ)), context) + else: + # The non-special case: a missing ordinary attribute. + if not self.disable_type_names: + failed = False + if isinstance(typ, Instance) and typ.type.names: + alternatives = set(typ.type.names.keys()) + matches = [m for m in COMMON_MISTAKES.get(member, []) if m in alternatives] + matches.extend(best_matches(member, alternatives)[:3]) + if matches: + self.fail('{} has no attribute "{}"; maybe {}?'.format( + self.format(typ), member, pretty_or(matches)), context) + failed = True + if not failed: + self.fail('{} has no attribute "{}"'.format(self.format(typ), + member), context) + else: + self.fail('Some element of union has no attribute "{}"'.format( + member), context) + return AnyType() + + def unsupported_operand_types(self, op: str, left_type: Any, + right_type: Any, context: Context) -> None: + """Report unsupported operand types for a binary operation. + + Types can be Type objects or strings. + """ + if isinstance(left_type, Void) or isinstance(right_type, Void): + self.check_void(left_type, context) + self.check_void(right_type, context) + return + left_str = '' + if isinstance(left_type, str): + left_str = left_type + else: + left_str = self.format(left_type) + + right_str = '' + if isinstance(right_type, str): + right_str = right_type + else: + right_str = self.format(right_type) + + if self.disable_type_names: + msg = 'Unsupported operand types for {} (likely involving Union)'.format(op) + else: + msg = 'Unsupported operand types for {} ({} and {})'.format( + op, left_str, right_str) + self.fail(msg, context) + + def unsupported_left_operand(self, op: str, typ: Type, + context: Context) -> None: + if not self.check_void(typ, context): + if self.disable_type_names: + msg = 'Unsupported left operand type for {} (some union)'.format(op) + else: + msg = 'Unsupported left operand type for {} ({})'.format( + op, self.format(typ)) + self.fail(msg, context) + + def type_expected_as_right_operand_of_is(self, context: Context) -> None: + self.fail('Type expected as right operand of "is"', context) + + def not_callable(self, typ: Type, context: Context) -> Type: + self.fail('{} not callable'.format(self.format(typ)), context) + return AnyType() + + def untyped_function_call(self, callee: CallableType, context: Context) -> Type: + name = callee.name if callee.name is not None else '(unknown)' + self.fail('call to untyped function {} in typed context'.format(name), context) + return AnyType() + + def incompatible_argument(self, n: int, m: int, callee: CallableType, arg_type: Type, + arg_kind: int, context: Context) -> None: + """Report an error about an incompatible argument type. + + The argument type is arg_type, argument number is n and the + callee type is 'callee'. If the callee represents a method + that corresponds to an operator, use the corresponding + operator name in the messages. + """ + target = '' + if callee.name: + name = callee.name + base = extract_type(name) + + for op, method in op_methods.items(): + for variant in method, '__r' + method[2:]: + if name.startswith('"{}" of'.format(variant)): + if op == 'in' or variant != method: + # Reversed order of base/argument. + self.unsupported_operand_types(op, arg_type, base, + context) + else: + self.unsupported_operand_types(op, base, arg_type, + context) + return + + if name.startswith('"__getitem__" of'): + self.invalid_index_type(arg_type, base, context) + return + + if name.startswith('"__setitem__" of'): + if n == 1: + self.invalid_index_type(arg_type, base, context) + else: + msg = '{} (expression has type {}, target has type {})' + arg_type_str, callee_type_str = self.format_distinctly(arg_type, + callee.arg_types[n - 1]) + self.fail(msg.format(INCOMPATIBLE_TYPES_IN_ASSIGNMENT, + arg_type_str, callee_type_str), + context) + return + + target = 'to {} '.format(name) + + msg = '' + if callee.name == '': + name = callee.name[1:-1] + n -= 1 + msg = '{} item {} has incompatible type {}'.format( + name[0].upper() + name[1:], n, self.format_simple(arg_type)) + elif callee.name == '': + msg = 'List comprehension has incompatible type List[{}]'.format( + strip_quotes(self.format(arg_type))) + elif callee.name == '': + msg = 'Set comprehension has incompatible type Set[{}]'.format( + strip_quotes(self.format(arg_type))) + elif callee.name == '': + msg = ('{} expression in dictionary comprehension has incompatible type {}; ' + 'expected type {}').format( + 'Key' if n == 1 else 'Value', + self.format(arg_type), + self.format(callee.arg_types[n - 1])) + elif callee.name == '': + msg = 'Generator has incompatible item type {}'.format( + self.format_simple(arg_type)) + else: + try: + expected_type = callee.arg_types[m - 1] + except IndexError: # Varargs callees + expected_type = callee.arg_types[-1] + arg_type_str, expected_type_str = self.format_distinctly(arg_type, expected_type) + #import pdb; pdb.set_trace() + if arg_kind == ARG_STAR: + arg_type_str = '*' + arg_type_str + elif arg_kind == ARG_STAR2: + arg_type_str = '**' + arg_type_str + msg = 'Argument {} {}has incompatible type {}; expected {}'.format( + n, target, arg_type_str, expected_type_str) + self.fail(msg, context) + + def invalid_index_type(self, index_type: Type, base_str: str, + context: Context) -> None: + self.fail('Invalid index type {} for {}'.format( + self.format(index_type), base_str), context) + + def too_few_arguments(self, callee: CallableType, context: Context, + argument_names: List[str]) -> None: + if (argument_names is not None and not all(k is None for k in argument_names) + and len(argument_names) >= 1): + diff = [k for k in callee.arg_names if k not in argument_names] + if len(diff) == 1: + msg = 'Missing positional argument' + else: + msg = 'Missing positional arguments' + if callee.name and diff: + msg += ' "{}" in call to {}'.format('", "'.join(diff), callee.name) + else: + msg = 'Too few arguments' + if callee.name: + msg += ' for {}'.format(callee.name) + self.fail(msg, context) + + def too_many_arguments(self, callee: CallableType, context: Context) -> None: + msg = 'Too many arguments' + if callee.name: + msg += ' for {}'.format(callee.name) + self.fail(msg, context) + + def too_many_positional_arguments(self, callee: CallableType, + context: Context) -> None: + msg = 'Too many positional arguments' + if callee.name: + msg += ' for {}'.format(callee.name) + self.fail(msg, context) + + def unexpected_keyword_argument(self, callee: CallableType, name: str, + context: Context) -> None: + msg = 'Unexpected keyword argument "{}"'.format(name) + if callee.name: + msg += ' for {}'.format(callee.name) + self.fail(msg, context) + if callee.definition: + fullname = callee.definition.fullname() + if fullname is not None and '.' in fullname: + module_name = fullname.rsplit('.', 1)[0] + path = self.modules[module_name].path + self.note('{} defined here'.format(callee.name), callee.definition, file=path) + + def duplicate_argument_value(self, callee: CallableType, index: int, + context: Context) -> None: + self.fail('{} gets multiple values for keyword argument "{}"'. + format(capitalize(callable_name(callee)), + callee.arg_names[index]), context) + + def does_not_return_value(self, void_type: Type, context: Context) -> None: + """Report an error about a void type in a non-void context. + + The first argument must be a void type. If the void type has a + source in it, report it in the error message. This allows + giving messages such as 'Foo does not return a value'. + """ + if (cast(Void, void_type)).source is None: + self.fail('Function does not return a value', context) + else: + self.fail('{} does not return a value'.format( + capitalize((cast(Void, void_type)).source)), context) + + def deleted_as_rvalue(self, typ: DeletedType, context: Context) -> None: + """Report an error about using an deleted type as an rvalue.""" + if typ.source is None: + s = "" + else: + s = " '{}'".format(typ.source) + self.fail('Trying to read deleted variable{}'.format(s), context) + + def deleted_as_lvalue(self, typ: DeletedType, context: Context) -> None: + """Report an error about using an deleted type as an lvalue. + + Currently, this only occurs when trying to assign to an + exception variable outside the local except: blocks. + """ + if typ.source is None: + s = "" + else: + s = " '{}'".format(typ.source) + self.fail('Assignment to variable{} outside except: block'.format(s), context) + + def no_variant_matches_arguments(self, overload: Overloaded, arg_types: List[Type], + context: Context) -> None: + if overload.name(): + self.fail('No overload variant of {} matches argument types {}' + .format(overload.name(), arg_types), context) + else: + self.fail('No overload variant matches argument types {}'.format(arg_types), context) + + def function_variants_overlap(self, n1: int, n2: int, + context: Context) -> None: + self.fail('Function signature variants {} and {} overlap'.format( + n1 + 1, n2 + 1), context) + + def invalid_cast(self, target_type: Type, source_type: Type, + context: Context) -> None: + if not self.check_void(source_type, context): + self.fail('Cannot cast from {} to {}'.format( + self.format(source_type), self.format(target_type)), context) + + def wrong_number_values_to_unpack(self, provided: int, expected: int, + context: Context) -> None: + if provided < expected: + if provided == 1: + self.fail('Need more than 1 value to unpack ({} expected)'.format(expected), + context) + else: + self.fail('Need more than {} values to unpack ({} expected)'.format( + provided, expected), context) + elif provided > expected: + self.fail('Too many values to unpack ({} expected, {} provided)'.format( + expected, provided), context) + + def type_not_iterable(self, type: Type, context: Context) -> None: + self.fail('\'{}\' object is not iterable'.format(type), context) + + def incompatible_operator_assignment(self, op: str, + context: Context) -> None: + self.fail('Result type of {} incompatible in assignment'.format(op), + context) + + def incompatible_value_count_in_assignment(self, lvalue_count: int, + rvalue_count: int, + context: Context) -> None: + if rvalue_count < lvalue_count: + self.fail('Need {} values to assign'.format(lvalue_count), context) + elif rvalue_count > lvalue_count: + self.fail('Too many values to assign', context) + + def type_incompatible_with_supertype(self, name: str, supertype: TypeInfo, + context: Context) -> None: + self.fail('Type of "{}" incompatible with supertype "{}"'.format( + name, supertype.name), context) + + def signature_incompatible_with_supertype( + self, name: str, name_in_super: str, supertype: str, + context: Context) -> None: + target = self.override_target(name, name_in_super, supertype) + self.fail('Signature of "{}" incompatible with {}'.format( + name, target), context) + + def argument_incompatible_with_supertype( + self, arg_num: int, name: str, name_in_supertype: str, + supertype: str, context: Context) -> None: + target = self.override_target(name, name_in_supertype, supertype) + self.fail('Argument {} of "{}" incompatible with {}' + .format(arg_num, name, target), context) + + def return_type_incompatible_with_supertype( + self, name: str, name_in_supertype: str, supertype: str, + context: Context) -> None: + target = self.override_target(name, name_in_supertype, supertype) + self.fail('Return type of "{}" incompatible with {}' + .format(name, target), context) + + def override_target(self, name: str, name_in_super: str, + supertype: str) -> str: + target = 'supertype "{}"'.format(supertype) + if name_in_super != name: + target = '"{}" of {}'.format(name_in_super, target) + return target + + def boolean_return_value_expected(self, method: str, + context: Context) -> None: + self.fail('Boolean return value expected for method "{}"'.format( + method), context) + + def incompatible_type_application(self, expected_arg_count: int, + actual_arg_count: int, + context: Context) -> None: + if expected_arg_count == 0: + self.fail('Type application targets a non-generic function', + context) + elif actual_arg_count > expected_arg_count: + self.fail('Type application has too many types ({} expected)' + .format(expected_arg_count), context) + else: + self.fail('Type application has too few types ({} expected)' + .format(expected_arg_count), context) + + def incompatible_array_item_type(self, typ: Type, index: int, + context: Context) -> None: + self.fail('Array item {} has incompatible type {}'.format( + index, self.format(typ)), context) + + def could_not_infer_type_arguments(self, callee_type: CallableType, n: int, + context: Context) -> None: + if callee_type.name and n > 0: + self.fail('Cannot infer type argument {} of {}'.format( + n, callee_type.name), context) + else: + self.fail('Cannot infer function type argument', context) + + def invalid_var_arg(self, typ: Type, context: Context) -> None: + self.fail('List or tuple expected as variable arguments', context) + + def invalid_keyword_var_arg(self, typ: Type, context: Context) -> None: + if isinstance(typ, Instance) and (typ.type.fullname() == 'builtins.dict'): + self.fail('Keywords must be strings', context) + else: + self.fail('Argument after ** must be a dictionary', + context) + + def incomplete_type_var_match(self, member: str, context: Context) -> None: + self.fail('"{}" has incomplete match to supertype type variable' + .format(member), context) + + def not_implemented(self, msg: str, context: Context) -> Type: + self.fail('Feature not implemented yet ({})'.format(msg), context) + return AnyType() + + def undefined_in_superclass(self, member: str, context: Context) -> None: + self.fail('"{}" undefined in superclass'.format(member), context) + + def check_void(self, typ: Type, context: Context) -> bool: + """If type is void, report an error such as '.. does not + return a value' and return True. Otherwise, return False. + """ + if isinstance(typ, Void): + self.does_not_return_value(typ, context) + return True + else: + return False + + def too_few_string_formatting_arguments(self, context: Context) -> None: + self.fail('Not enough arguments for format string', context) + + def too_many_string_formatting_arguments(self, context: Context) -> None: + self.fail('Not all arguments converted during string formatting', context) + + def incomplete_conversion_specifier_format(self, context: Context) -> None: + self.fail('Incomplete format', context) + + def unsupported_placeholder(self, placeholder: str, context: Context) -> None: + self.fail('Unsupported format character \'%s\'' % placeholder, context) + + def string_interpolation_with_star_and_key(self, context: Context) -> None: + self.fail('String interpolation contains both stars and mapping keys', context) + + def requires_int_or_char(self, context: Context) -> None: + self.fail('%c requires int or char', context) + + def key_not_in_mapping(self, key: str, context: Context) -> None: + self.fail('Key \'%s\' not found in mapping' % key, context) + + def string_interpolation_mixing_key_and_non_keys(self, context: Context) -> None: + self.fail('String interpolation mixes specifier with and without mapping keys', context) + + def cannot_determine_type(self, name: str, context: Context) -> None: + self.fail("Cannot determine type of '%s'" % name, context) + + def cannot_determine_type_in_base(self, name: str, base: str, context: Context) -> None: + self.fail("Cannot determine type of '%s' in base class '%s'" % (name, base), context) + + def invalid_method_type(self, sig: CallableType, context: Context) -> None: + self.fail('Invalid method type', context) + + def invalid_class_method_type(self, sig: CallableType, context: Context) -> None: + self.fail('Invalid class method type', context) + + def incompatible_conditional_function_def(self, defn: FuncDef) -> None: + self.fail('All conditional function variants must have identical ' + 'signatures', defn) + + def cannot_instantiate_abstract_class(self, class_name: str, + abstract_attributes: List[str], + context: Context) -> None: + attrs = format_string_list("'%s'" % a for a in abstract_attributes[:5]) + self.fail("Cannot instantiate abstract class '%s' with abstract " + "attribute%s %s" % (class_name, plural_s(abstract_attributes), + attrs), + context) + + def base_class_definitions_incompatible(self, name: str, base1: TypeInfo, + base2: TypeInfo, + context: Context) -> None: + self.fail('Definition of "{}" in base class "{}" is incompatible ' + 'with definition in base class "{}"'.format( + name, base1.name(), base2.name()), context) + + def cant_assign_to_method(self, context: Context) -> None: + self.fail(CANNOT_ASSIGN_TO_METHOD, context) + + def read_only_property(self, name: str, type: TypeInfo, + context: Context) -> None: + self.fail('Property "{}" defined in "{}" is read-only'.format( + name, type.name()), context) + + def incompatible_typevar_value(self, callee: CallableType, index: int, + type: Type, context: Context) -> None: + self.fail('Type argument {} of {} has incompatible value {}'.format( + index, callable_name(callee), self.format(type)), context) + + def overloaded_signatures_overlap(self, index1: int, index2: int, + context: Context) -> None: + self.fail('Overloaded function signatures {} and {} overlap with ' + 'incompatible return types'.format(index1, index2), context) + + def invalid_reverse_operator_signature(self, reverse: str, other: str, + context: Context) -> None: + self.fail('"Any" return type expected since argument to {} ' + 'does not support {}'.format(reverse, other), context) + + def reverse_operator_method_with_any_arg_must_return_any( + self, method: str, context: Context) -> None: + self.fail('"Any" return type expected since argument to {} ' + 'has type "Any"'.format(method), context) + + def operator_method_signatures_overlap( + self, reverse_class: str, reverse_method: str, forward_class: str, + forward_method: str, context: Context) -> None: + self.fail('Signatures of "{}" of "{}" and "{}" of "{}" ' + 'are unsafely overlapping'.format( + reverse_method, reverse_class, + forward_method, forward_class), + context) + + def forward_operator_not_callable( + self, forward_method: str, context: Context) -> None: + self.fail('Forward operator "{}" is not callable'.format( + forward_method), context) + + def signatures_incompatible(self, method: str, other_method: str, + context: Context) -> None: + self.fail('Signatures of "{}" and "{}" are incompatible'.format( + method, other_method), context) + + def yield_from_invalid_operand_type(self, expr: Type, context: Context) -> Type: + text = self.format(expr) if self.format(expr) != 'object' else expr + self.fail('"yield from" can\'t be applied to {}'.format(text), context) + return AnyType() + + def invalid_signature(self, func_type: Type, context: Context) -> None: + self.fail('Invalid signature "{}"'.format(func_type), context) + + def reveal_type(self, typ: Type, context: Context) -> None: + self.fail('Revealed type is \'{}\''.format(typ), context) + + def unsupported_type_type(self, item: Type, context: Context) -> None: + self.fail('Unsupported type Type[{}]'.format(self.format(item)), context) + + def redundant_cast(self, typ: Type, context: Context) -> None: + self.note('Redundant cast to {}'.format(self.format(typ)), context) + + +def capitalize(s: str) -> str: + """Capitalize the first character of a string.""" + if s == '': + return '' + else: + return s[0].upper() + s[1:] + + +def extract_type(name: str) -> str: + """If the argument is the name of a method (of form C.m), return + the type portion in quotes (e.g. "y"). Otherwise, return the string + unmodified. + """ + name = re.sub('^"[a-zA-Z0-9_]+" of ', '', name) + return name + + +def strip_quotes(s: str) -> str: + """Strip a double quote at the beginning and end of the string, if any.""" + s = re.sub('^"', '', s) + s = re.sub('"$', '', s) + return s + + +def plural_s(s: Sequence[Any]) -> str: + if len(s) > 1: + return 's' + else: + return '' + + +def format_string_list(s: Iterable[str]) -> str: + l = list(s) + assert len(l) > 0 + if len(l) == 1: + return l[0] + else: + return '%s and %s' % (', '.join(l[:-1]), l[-1]) + + +def callable_name(type: CallableType) -> str: + if type.name: + return type.name + else: + return 'function' + + +def temp_message_builder() -> MessageBuilder: + """Return a message builder usable for throwaway errors (which may not format properly).""" + return MessageBuilder(Errors(), {}) + + +# For hard-coding suggested missing member alternatives. +COMMON_MISTAKES = { + 'add': ('append', 'extend'), +} # type: Dict[str, Sequence[str]] + + +def best_matches(current: str, options: Iterable[str]) -> List[str]: + ratios = {v: difflib.SequenceMatcher(a=current, b=v).ratio() for v in options} + return sorted((o for o in options if ratios[o] > 0.75), + reverse=True, key=lambda v: (ratios[v], v)) + + +def pretty_or(args: List[str]) -> str: + quoted = ['"' + a + '"' for a in args] + if len(quoted) == 1: + return quoted[0] + if len(quoted) == 2: + return "{} or {}".format(quoted[0], quoted[1]) + return ", ".join(quoted[:-1]) + ", or " + quoted[-1] diff --git a/mypy/nodes.py b/mypy/nodes.py index 02f86b1537b93..60d7b5d18944f 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1,2242 +1,2242 @@ -"""Abstract syntax tree node classes (i.e. parse tree).""" - -import os -import re -from abc import abstractmethod, ABCMeta - -from typing import ( - Any, TypeVar, List, Tuple, cast, Set, Dict, Union, Optional -) - -from mypy.lex import Token -import mypy.strconv -from mypy.visitor import NodeVisitor -from mypy.util import dump_tagged, short_type - - -class Context: - """Base type for objects that are valid as error message locations.""" - @abstractmethod - def get_line(self) -> int: pass - - -if False: - # break import cycle only needed for mypy - import mypy.types - - -T = TypeVar('T') - -JsonDict = Dict[str, Any] - - -# Symbol table node kinds -# -# TODO rename to use more descriptive names - -LDEF = 0 # type: int -GDEF = 1 # type: int -MDEF = 2 # type: int -MODULE_REF = 3 # type: int -# Type variable declared using TypeVar(...) has kind UNBOUND_TVAR. It's not -# valid as a type. A type variable is valid as a type (kind BOUND_TVAR) within -# (1) a generic class that uses the type variable as a type argument or -# (2) a generic function that refers to the type variable in its signature. -UNBOUND_TVAR = 4 # type: int -BOUND_TVAR = 5 # type: int -TYPE_ALIAS = 6 # type: int -# Placeholder for a name imported via 'from ... import'. Second phase of -# semantic will replace this the actual imported reference. This is -# needed so that we can detect whether a name has been imported during -# XXX what? -UNBOUND_IMPORTED = 7 # type: int - - -LITERAL_YES = 2 -LITERAL_TYPE = 1 -LITERAL_NO = 0 - -# Hard coded name of Enum baseclass. -ENUM_BASECLASS = "enum.Enum" - -node_kinds = { - LDEF: 'Ldef', - GDEF: 'Gdef', - MDEF: 'Mdef', - MODULE_REF: 'ModuleRef', - UNBOUND_TVAR: 'UnboundTvar', - BOUND_TVAR: 'Tvar', - TYPE_ALIAS: 'TypeAlias', - UNBOUND_IMPORTED: 'UnboundImported', -} -inverse_node_kinds = {_kind: _name for _name, _kind in node_kinds.items()} - - -implicit_module_attrs = {'__name__': '__builtins__.str', - '__doc__': '__builtins__.str', - '__file__': '__builtins__.str', - '__package__': '__builtins__.str'} - - -type_aliases = { - 'typing.List': '__builtins__.list', - 'typing.Dict': '__builtins__.dict', - 'typing.Set': '__builtins__.set', -} - -reverse_type_aliases = dict((name.replace('__builtins__', 'builtins'), alias) - for alias, name in type_aliases.items()) # type: Dict[str, str] - - -class Node(Context): - """Common base class for all non-type parse tree nodes.""" - - line = -1 - - literal = LITERAL_NO - literal_hash = None # type: Any - - def __str__(self) -> str: - ans = self.accept(mypy.strconv.StrConv()) - if ans is None: - return repr(self) - return ans - - def set_line(self, target: Union[Token, 'Node', int]) -> 'Node': - if isinstance(target, int): - self.line = target - else: - self.line = target.line - return self - - def get_line(self) -> int: - # TODO this should be just 'line' - return self.line - - def accept(self, visitor: NodeVisitor[T]) -> T: - raise RuntimeError('Not implemented') - - -# These are placeholders for a future refactoring; see #1783. -# For now they serve as (unchecked) documentation of what various -# fields of Node subtypes are expected to contain. -Statement = Node -Expression = Node - - -class SymbolNode(Node): - # Nodes that can be stored in a symbol table. - - # TODO do not use methods for these - - @abstractmethod - def name(self) -> str: pass - - @abstractmethod - def fullname(self) -> str: pass - - # NOTE: Can't use @abstractmethod, since many subclasses of Node - # don't implement serialize(). - def serialize(self) -> Any: - raise NotImplementedError('Cannot serialize {} instance'.format(self.__class__.__name__)) - - @classmethod - def deserialize(cls, data: JsonDict) -> 'SymbolNode': - classname = data['.class'] - glo = globals() - if classname in glo: - cl = glo[classname] - if issubclass(cl, cls) and 'deserialize' in cl.__dict__: - return cl.deserialize(data) - raise NotImplementedError('unexpected .class {}'.format(classname)) - - -class MypyFile(SymbolNode, Statement): - """The abstract syntax tree of a single source file.""" - - # Module name ('__main__' for initial file) - _name = None # type: str - # Fully qualified module name - _fullname = None # type: str - # Path to the file (None if not known) - path = '' - # Top-level definitions and statements - defs = None # type: List[Statement] - # Is there a UTF-8 BOM at the start? - is_bom = False - names = None # type: SymbolTable - # All import nodes within the file (also ones within functions etc.) - imports = None # type: List[ImportBase] - # Lines to ignore when checking - ignored_lines = None # type: Set[int] - # Is this file represented by a stub file (.pyi)? - is_stub = False - # Do weak typing globally in the file? - weak_opts = None # type: Set[str] - - def __init__(self, - defs: List[Statement], - imports: List['ImportBase'], - is_bom: bool = False, - ignored_lines: Set[int] = None, - weak_opts: Set[str] = None) -> None: - self.defs = defs - self.line = 1 # Dummy line number - self.imports = imports - self.is_bom = is_bom - self.weak_opts = weak_opts - if ignored_lines: - self.ignored_lines = ignored_lines - else: - self.ignored_lines = set() - - def name(self) -> str: - return self._name - - def fullname(self) -> str: - return self._fullname - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_mypy_file(self) - - def is_package_init_file(self) -> bool: - return not (self.path is None) and len(self.path) != 0 \ - and os.path.basename(self.path).startswith('__init__.') - - def serialize(self) -> JsonDict: - return {'.class': 'MypyFile', - '_name': self._name, - '_fullname': self._fullname, - 'names': self.names.serialize(self._fullname), - 'is_stub': self.is_stub, - 'path': self.path, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'MypyFile': - assert data['.class'] == 'MypyFile', data - tree = MypyFile([], []) - tree._name = data['_name'] - tree._fullname = data['_fullname'] - tree.names = SymbolTable.deserialize(data['names']) - tree.is_stub = data['is_stub'] - tree.path = data['path'] - return tree - - -class ImportBase(Statement): - """Base class for all import statements.""" - is_unreachable = False - is_top_level = False # Set by semanal.FirstPass - # If an import replaces existing definitions, we construct dummy assignment - # statements that assign the imported names to the names in the current scope, - # for type checking purposes. Example: - # - # x = 1 - # from m import x <-- add assignment representing "x = m.x" - assignments = None # type: List[AssignmentStmt] - - def __init__(self) -> None: - self.assignments = [] - - -class Import(ImportBase): - """import m [as n]""" - - ids = None # type: List[Tuple[str, Optional[str]]] # (module id, as id) - - def __init__(self, ids: List[Tuple[str, Optional[str]]]) -> None: - super().__init__() - self.ids = ids - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_import(self) - - -class ImportFrom(ImportBase): - """from m import x [as y], ...""" - - names = None # type: List[Tuple[str, Optional[str]]] # Tuples (name, as name) - - def __init__(self, id: str, relative: int, names: List[Tuple[str, Optional[str]]]) -> None: - super().__init__() - self.id = id - self.names = names - self.relative = relative - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_import_from(self) - - -class ImportAll(ImportBase): - """from m import *""" - - def __init__(self, id: str, relative: int) -> None: - super().__init__() - self.id = id - self.relative = relative - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_import_all(self) - - -class FuncBase(SymbolNode): - """Abstract base class for function-like nodes""" - - # Type signature. This is usually CallableType or Overloaded, but it can be something else for - # decorated functions/ - type = None # type: mypy.types.Type - # If method, reference to TypeInfo - info = None # type: TypeInfo - is_property = False - _fullname = None # type: str # Name with module prefix - - @abstractmethod - def name(self) -> str: pass - - def fullname(self) -> str: - return self._fullname - - def is_method(self) -> bool: - return bool(self.info) - - -class OverloadedFuncDef(FuncBase, Statement): - """A logical node representing all the variants of an overloaded function. - - This node has no explicit representation in the source program. - Overloaded variants must be consecutive in the source file. - """ - - items = None # type: List[Decorator] - - def __init__(self, items: List['Decorator']) -> None: - self.items = items - self.set_line(items[0].line) - - def name(self) -> str: - return self.items[0].func.name() - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_overloaded_func_def(self) - - def serialize(self) -> JsonDict: - return {'.class': 'OverloadedFuncDef', - 'items': [i.serialize() for i in self.items], - 'type': None if self.type is None else self.type.serialize(), - 'fullname': self._fullname, - 'is_property': self.is_property, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'OverloadedFuncDef': - assert data['.class'] == 'OverloadedFuncDef' - res = OverloadedFuncDef([Decorator.deserialize(d) for d in data['items']]) - if data.get('type') is not None: - res.type = mypy.types.Type.deserialize(data['type']) - res._fullname = data['fullname'] - res.is_property = data['is_property'] - # NOTE: res.info will be set in the fixup phase. - return res - - -class Argument(Node): - """A single argument in a FuncItem.""" - - def __init__(self, variable: 'Var', type_annotation: 'Optional[mypy.types.Type]', - initializer: Optional[Expression], kind: int, - initialization_statement: Optional['AssignmentStmt'] = None) -> None: - self.variable = variable - - self.type_annotation = type_annotation - self.initializer = initializer - - self.initialization_statement = initialization_statement - if not self.initialization_statement: - self.initialization_statement = self._initialization_statement() - - self.kind = kind - - def _initialization_statement(self) -> Optional['AssignmentStmt']: - """Convert the initializer into an assignment statement. - """ - if not self.initializer: - return None - - rvalue = self.initializer - lvalue = NameExpr(self.variable.name()) - assign = AssignmentStmt([lvalue], rvalue) - return assign - - def set_line(self, target: Union[Token, Node, int]) -> Node: - super().set_line(target) - - if self.initializer: - self.initializer.set_line(self.line) - - self.variable.set_line(self.line) - - if self.initialization_statement: - self.initialization_statement.set_line(self.line) - self.initialization_statement.lvalues[0].set_line(self.line) - - def serialize(self) -> JsonDict: - data = {'.class': 'Argument', - 'kind': self.kind, - 'variable': self.variable.serialize(), - 'type_annotation': (None if self.type_annotation is None - else self.type_annotation.serialize()), - } # type: JsonDict - # TODO: initializer? - return data - - @classmethod - def deserialize(cls, data: JsonDict) -> 'Argument': - assert data['.class'] == 'Argument' - return Argument(Var.deserialize(data['variable']), - (None if data.get('type_annotation') is None - else mypy.types.Type.deserialize(data['type_annotation'])), - None, # TODO: initializer? - kind=data['kind']) - - -class FuncItem(FuncBase): - arguments = [] # type: List[Argument] - # Minimum number of arguments - min_args = 0 - # Maximum number of positional arguments, -1 if no explicit limit (*args not included) - max_pos = 0 - body = None # type: Block - # Is this an overload variant of function with more than one overload variant? - is_overload = False - is_generator = False # Contains a yield statement? - is_static = False # Uses @staticmethod? - is_class = False # Uses @classmethod? - # Variants of function with type variables with values expanded - expanded = None # type: List[FuncItem] - - def __init__(self, arguments: List[Argument], body: 'Block', - typ: 'mypy.types.FunctionLike' = None) -> None: - self.arguments = arguments - arg_kinds = [arg.kind for arg in self.arguments] - self.max_pos = arg_kinds.count(ARG_POS) + arg_kinds.count(ARG_OPT) - self.body = body - self.type = typ - self.expanded = [] - - self.min_args = 0 - for i in range(len(self.arguments)): - if self.arguments[i] is None and i < self.max_fixed_argc(): - self.min_args = i + 1 - - def max_fixed_argc(self) -> int: - return self.max_pos - - def set_line(self, target: Union[Token, Node, int]) -> Node: - super().set_line(target) - for arg in self.arguments: - arg.set_line(self.line) - return self - - def is_dynamic(self): - return self.type is None - - -class FuncDef(FuncItem, Statement): - """Function definition. - - This is a non-lambda function defined using 'def'. - """ - - is_decorated = False - is_conditional = False # Defined conditionally (within block)? - is_abstract = False - is_property = False - original_def = None # type: Union[None, FuncDef, Var] # Original conditional definition - - def __init__(self, - name: str, # Function name - arguments: List[Argument], - body: 'Block', - typ: 'mypy.types.FunctionLike' = None) -> None: - super().__init__(arguments, body, typ) - self._name = name - - def name(self) -> str: - return self._name - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_func_def(self) - - def is_constructor(self) -> bool: - return self.info is not None and self._name == '__init__' - - def serialize(self) -> JsonDict: - return {'.class': 'FuncDef', - 'name': self._name, - 'fullname': self._fullname, - 'arguments': [a.serialize() for a in self.arguments], - 'type': None if self.type is None else self.type.serialize(), - 'is_property': self.is_property, - 'is_overload': self.is_overload, - 'is_generator': self.is_generator, - 'is_static': self.is_static, - 'is_class': self.is_class, - 'is_decorated': self.is_decorated, - 'is_conditional': self.is_conditional, - 'is_abstract': self.is_abstract, - # TODO: Do we need expanded, original_def? - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'FuncDef': - assert data['.class'] == 'FuncDef' - body = Block([]) - ret = FuncDef(data['name'], - [Argument.deserialize(a) for a in data['arguments']], - body, - (None if data['type'] is None - else mypy.types.FunctionLike.deserialize(data['type']))) - ret._fullname = data['fullname'] - ret.is_property = data['is_property'] - ret.is_overload = data['is_overload'] - ret.is_generator = data['is_generator'] - ret.is_static = data['is_static'] - ret.is_class = data['is_class'] - ret.is_decorated = data['is_decorated'] - ret.is_conditional = data['is_conditional'] - ret.is_abstract = data['is_abstract'] - # NOTE: ret.info is set in the fixup phase. - return ret - - -class Decorator(SymbolNode, Statement): - """A decorated function. - - A single Decorator object can include any number of function decorators. - """ - - func = None # type: FuncDef # Decorated function - decorators = None # type: List[Expression] # Decorators, at least one # XXX Not true - var = None # type: Var # Represents the decorated function obj - is_overload = False - - def __init__(self, func: FuncDef, decorators: List[Expression], - var: 'Var') -> None: - self.func = func - self.decorators = decorators - self.var = var - self.is_overload = False - - def name(self) -> str: - return self.func.name() - - def fullname(self) -> str: - return self.func.fullname() - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_decorator(self) - - def serialize(self) -> JsonDict: - return {'.class': 'Decorator', - 'func': self.func.serialize(), - 'var': self.var.serialize(), - 'is_overload': self.is_overload, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'Decorator': - assert data['.class'] == 'Decorator' - dec = Decorator(FuncDef.deserialize(data['func']), - [], - Var.deserialize(data['var'])) - dec.is_overload = data['is_overload'] - return dec - - -class Var(SymbolNode, Statement): - """A variable. - - It can refer to global/local variable or a data attribute. - """ - - _name = None # type: str # Name without module prefix - _fullname = None # type: str # Name with module prefix - info = None # type: TypeInfo # Defining class (for member variables) - type = None # type: mypy.types.Type # Declared or inferred type, or None - # Is this the first argument to an ordinary method (usually "self")? - is_self = False - is_ready = False # If inferred, is the inferred type available? - # Is this initialized explicitly to a non-None value in class body? - is_initialized_in_class = False - is_staticmethod = False - is_classmethod = False - is_property = False - is_settable_property = False - - def __init__(self, name: str, type: 'mypy.types.Type' = None) -> None: - self._name = name - self.type = type - self.is_self = False - self.is_ready = True - self.is_initialized_in_class = False - - def name(self) -> str: - return self._name - - def fullname(self) -> str: - return self._fullname - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_var(self) - - def serialize(self) -> JsonDict: - # TODO: Leave default values out? - # NOTE: Sometimes self.is_ready is False here, but we don't care. - data = {'.class': 'Var', - 'name': self._name, - 'fullname': self._fullname, - 'type': None if self.type is None else self.type.serialize(), - 'is_self': self.is_self, - 'is_initialized_in_class': self.is_initialized_in_class, - 'is_staticmethod': self.is_staticmethod, - 'is_classmethod': self.is_classmethod, - 'is_property': self.is_property, - 'is_settable_property': self.is_settable_property, - } # type: JsonDict - return data - - @classmethod - def deserialize(cls, data: JsonDict) -> 'Var': - assert data['.class'] == 'Var' - name = data['name'] - type = None if data['type'] is None else mypy.types.Type.deserialize(data['type']) - v = Var(name, type) - v._fullname = data['fullname'] - v.is_self = data['is_self'] - v.is_initialized_in_class = data['is_initialized_in_class'] - v.is_staticmethod = data['is_staticmethod'] - v.is_classmethod = data['is_classmethod'] - v.is_property = data['is_property'] - v.is_settable_property = data['is_settable_property'] - return v - - -class ClassDef(Statement): - """Class definition""" - - name = None # type: str # Name of the class without module prefix - fullname = None # type: str # Fully qualified name of the class - defs = None # type: Block - type_vars = None # type: List[mypy.types.TypeVarDef] - # Base class expressions (not semantically analyzed -- can be arbitrary expressions) - base_type_exprs = None # type: List[Expression] - info = None # type: TypeInfo # Related TypeInfo - metaclass = '' - decorators = None # type: List[Expression] - # Built-in/extension class? (single implementation inheritance only) - is_builtinclass = False - - def __init__(self, - name: str, - defs: 'Block', - type_vars: List['mypy.types.TypeVarDef'] = None, - base_type_exprs: List[Expression] = None, - metaclass: str = None) -> None: - self.name = name - self.defs = defs - self.type_vars = type_vars or [] - self.base_type_exprs = base_type_exprs or [] - self.metaclass = metaclass - self.decorators = [] - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_class_def(self) - - def is_generic(self) -> bool: - return self.info.is_generic() - - def serialize(self) -> JsonDict: - # Not serialized: defs, base_type_exprs, decorators - return {'.class': 'ClassDef', - 'name': self.name, - 'fullname': self.fullname, - 'type_vars': [v.serialize() for v in self.type_vars], - 'metaclass': self.metaclass, - 'is_builtinclass': self.is_builtinclass, - } - - @classmethod - def deserialize(self, data: JsonDict) -> 'ClassDef': - assert data['.class'] == 'ClassDef' - res = ClassDef(data['name'], - Block([]), - [mypy.types.TypeVarDef.deserialize(v) for v in data['type_vars']], - metaclass=data['metaclass'], - ) - res.fullname = data['fullname'] - res.is_builtinclass = data['is_builtinclass'] - return res - - -class GlobalDecl(Statement): - """Declaration global x, y, ...""" - - names = None # type: List[str] - - def __init__(self, names: List[str]) -> None: - self.names = names - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_global_decl(self) - - -class NonlocalDecl(Statement): - """Declaration nonlocal x, y, ...""" - - names = None # type: List[str] - - def __init__(self, names: List[str]) -> None: - self.names = names - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_nonlocal_decl(self) - - -class Block(Statement): - body = None # type: List[Statement] - # True if we can determine that this block is not executed. For example, - # this applies to blocks that are protected by something like "if PY3:" - # when using Python 2. - is_unreachable = False - - def __init__(self, body: List[Statement]) -> None: - self.body = body - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_block(self) - - -# Statements - - -class ExpressionStmt(Statement): - """An expression as a statement, such as print(s).""" - expr = None # type: Expression - - def __init__(self, expr: Expression) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_expression_stmt(self) - - -class AssignmentStmt(Statement): - """Assignment statement - The same node class is used for single assignment, multiple assignment - (e.g. x, y = z) and chained assignment (e.g. x = y = z), assignments - that define new names, and assignments with explicit types (# type). - - An lvalue can be NameExpr, TupleExpr, ListExpr, MemberExpr, IndexExpr. - """ - - lvalues = None # type: List[Expression] - rvalue = None # type: Expression - # Declared type in a comment, may be None. - type = None # type: mypy.types.Type - - def __init__(self, lvalues: List[Expression], rvalue: Expression, - type: 'mypy.types.Type' = None) -> None: - self.lvalues = lvalues - self.rvalue = rvalue - self.type = type - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_assignment_stmt(self) - - -class OperatorAssignmentStmt(Statement): - """Operator assignment statement such as x += 1""" - - op = '' - lvalue = None # type: Expression - rvalue = None # type: Expression - - def __init__(self, op: str, lvalue: Expression, rvalue: Expression) -> None: - self.op = op - self.lvalue = lvalue - self.rvalue = rvalue - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_operator_assignment_stmt(self) - - -class WhileStmt(Statement): - expr = None # type: Expression - body = None # type: Block - else_body = None # type: Block - - def __init__(self, expr: Expression, body: Block, else_body: Block) -> None: - self.expr = expr - self.body = body - self.else_body = else_body - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_while_stmt(self) - - -class ForStmt(Statement): - # Index variables - index = None # type: Expression - # Expression to iterate - expr = None # type: Expression - body = None # type: Block - else_body = None # type: Block - - def __init__(self, index: Expression, expr: Expression, body: Block, - else_body: Block) -> None: - self.index = index - self.expr = expr - self.body = body - self.else_body = else_body - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_for_stmt(self) - - -class ReturnStmt(Statement): - expr = None # type: Optional[Expression] - - def __init__(self, expr: Optional[Expression]) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_return_stmt(self) - - -class AssertStmt(Statement): - expr = None # type: Expression - - def __init__(self, expr: Expression) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_assert_stmt(self) - - -class DelStmt(Statement): - expr = None # type: Expression - - def __init__(self, expr: Expression) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_del_stmt(self) - - -class BreakStmt(Statement): - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_break_stmt(self) - - -class ContinueStmt(Statement): - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_continue_stmt(self) - - -class PassStmt(Statement): - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_pass_stmt(self) - - -class IfStmt(Statement): - expr = None # type: List[Expression] - body = None # type: List[Block] - else_body = None # type: Block - - def __init__(self, expr: List[Expression], body: List[Block], - else_body: Block) -> None: - self.expr = expr - self.body = body - self.else_body = else_body - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_if_stmt(self) - - -class RaiseStmt(Statement): - expr = None # type: Expression - from_expr = None # type: Expression - - def __init__(self, expr: Expression, from_expr: Expression = None) -> None: - self.expr = expr - self.from_expr = from_expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_raise_stmt(self) - - -class TryStmt(Statement): - body = None # type: Block # Try body - types = None # type: List[Expression] # Except type expressions - vars = None # type: List[NameExpr] # Except variable names - handlers = None # type: List[Block] # Except bodies - else_body = None # type: Block - finally_body = None # type: Block - - def __init__(self, body: Block, vars: List['NameExpr'], types: List[Expression], - handlers: List[Block], else_body: Block, - finally_body: Block) -> None: - self.body = body - self.vars = vars - self.types = types - self.handlers = handlers - self.else_body = else_body - self.finally_body = finally_body - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_try_stmt(self) - - -class WithStmt(Statement): - expr = None # type: List[Expression] - target = None # type: List[Expression] - body = None # type: Block - - def __init__(self, expr: List[Expression], target: List[Expression], - body: Block) -> None: - self.expr = expr - self.target = target - self.body = body - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_with_stmt(self) - - -class PrintStmt(Statement): - """Python 2 print statement""" - - args = None # type: List[Expression] - newline = False - # The file-like target object (given using >>). - target = None # type: Optional[Expression] - - def __init__(self, args: List[Expression], newline: bool, target: Expression = None) -> None: - self.args = args - self.newline = newline - self.target = target - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_print_stmt(self) - - -class ExecStmt(Statement): - """Python 2 exec statement""" - - expr = None # type: Expression - variables1 = None # type: Optional[Expression] - variables2 = None # type: Optional[Expression] - - def __init__(self, expr: Expression, - variables1: Optional[Expression], - variables2: Optional[Expression]) -> None: - self.expr = expr - self.variables1 = variables1 - self.variables2 = variables2 - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_exec_stmt(self) - - -# Expressions - - -class IntExpr(Expression): - """Integer literal""" - - value = 0 - literal = LITERAL_YES - - def __init__(self, value: int) -> None: - self.value = value - self.literal_hash = value - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_int_expr(self) - - -# How mypy uses StrExpr, BytesExpr, and UnicodeExpr: -# In Python 2 mode: -# b'x', 'x' -> StrExpr -# u'x' -> UnicodeExpr -# BytesExpr is unused -# -# In Python 3 mode: -# b'x' -> BytesExpr -# 'x', u'x' -> StrExpr -# UnicodeExpr is unused - -class StrExpr(Expression): - """String literal""" - - value = '' - literal = LITERAL_YES - - def __init__(self, value: str) -> None: - self.value = value - self.literal_hash = value - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_str_expr(self) - - -class BytesExpr(Expression): - """Bytes literal""" - - value = '' # TODO use bytes - literal = LITERAL_YES - - def __init__(self, value: str) -> None: - self.value = value - self.literal_hash = value - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_bytes_expr(self) - - -class UnicodeExpr(Expression): - """Unicode literal (Python 2.x)""" - - value = '' # TODO use bytes - literal = LITERAL_YES - - def __init__(self, value: str) -> None: - self.value = value - self.literal_hash = value - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_unicode_expr(self) - - -class FloatExpr(Expression): - """Float literal""" - - value = 0.0 - literal = LITERAL_YES - - def __init__(self, value: float) -> None: - self.value = value - self.literal_hash = value - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_float_expr(self) - - -class ComplexExpr(Expression): - """Complex literal""" - - value = 0.0j - literal = LITERAL_YES - - def __init__(self, value: complex) -> None: - self.value = value - self.literal_hash = value - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_complex_expr(self) - - -class EllipsisExpr(Expression): - """Ellipsis (...)""" - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_ellipsis(self) - - -class StarExpr(Expression): - """Star expression""" - - expr = None # type: Expression - - def __init__(self, expr: Expression) -> None: - self.expr = expr - self.literal = self.expr.literal - self.literal_hash = ('Star', expr.literal_hash,) - - # Whether this starred expression is used in a tuple/list and as lvalue - self.valid = False - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_star_expr(self) - - -class RefExpr(Expression): - """Abstract base class for name-like constructs""" - - kind = None # type: int # LDEF/GDEF/MDEF/... (None if not available) - node = None # type: SymbolNode # Var, FuncDef or TypeInfo that describes this - fullname = None # type: str # Fully qualified name (or name if not global) - - # Does this define a new name with inferred type? - # - # For members, after semantic analysis, this does not take base - # classes into consideration at all; the type checker deals with these. - is_def = False - - -class NameExpr(RefExpr): - """Name expression - - This refers to a local name, global name or a module. - """ - - name = None # type: str # Name referred to (may be qualified) - # TypeInfo of class surrounding expression (may be None) - info = None # type: TypeInfo - - literal = LITERAL_TYPE - - def __init__(self, name: str) -> None: - self.name = name - self.literal_hash = ('Var', name,) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_name_expr(self) - - def serialize(self) -> JsonDict: - # TODO: Find out where and why NameExpr is being serialized (if at all). - assert False, "Serializing NameExpr: %s" % (self,) - return {'.class': 'NameExpr', - 'kind': self.kind, - 'node': None if self.node is None else self.node.serialize(), - 'fullname': self.fullname, - 'is_def': self.is_def, - 'name': self.name, - 'literal': self.literal, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'NameExpr': - assert data['.class'] == 'NameExpr' - ret = NameExpr(data['name']) - ret.kind = data['kind'] - ret.node = None if data['node'] is None else SymbolNode.deserialize(data['node']) - ret.fullname = data['fullname'] - ret.is_def = data['is_def'] - ret.literal = data['literal'] - return ret - - -class MemberExpr(RefExpr): - """Member access expression x.y""" - - expr = None # type: Expression - name = None # type: str - # The variable node related to a definition. - def_var = None # type: Var - - def __init__(self, expr: Expression, name: str) -> None: - self.expr = expr - self.name = name - self.literal = self.expr.literal - self.literal_hash = ('Member', expr.literal_hash, name) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_member_expr(self) - - -# Kinds of arguments - -# Positional argument -ARG_POS = 0 # type: int -# Positional, optional argument (functions only, not calls) -ARG_OPT = 1 # type: int -# *arg argument -ARG_STAR = 2 # type: int -# Keyword argument x=y in call, or keyword-only function arg -ARG_NAMED = 3 # type: int -# **arg argument -ARG_STAR2 = 4 # type: int - - -class CallExpr(Expression): - """Call expression. - - This can also represent several special forms that are syntactically calls - such as cast(...) and None # type: .... - """ - - callee = None # type: Expression - args = None # type: List[Expression] - arg_kinds = None # type: List[int] # ARG_ constants - # Each name can be None if not a keyword argument. - arg_names = None # type: List[str] - # If not None, the node that represents the meaning of the CallExpr. For - # cast(...) this is a CastExpr. - analyzed = None # type: Optional[Expression] - - def __init__(self, callee: Expression, args: List[Expression], arg_kinds: List[int], - arg_names: List[str] = None, analyzed: Expression = None) -> None: - if not arg_names: - arg_names = [None] * len(args) - - self.callee = callee - self.args = args - self.arg_kinds = arg_kinds - self.arg_names = arg_names - self.analyzed = analyzed - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_call_expr(self) - - -class YieldFromExpr(Expression): - expr = None # type: Expression - - def __init__(self, expr: Expression) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_yield_from_expr(self) - - -class YieldExpr(Expression): - expr = None # type: Optional[Expression] - - def __init__(self, expr: Optional[Expression]) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_yield_expr(self) - - -class IndexExpr(Expression): - """Index expression x[y]. - - Also wraps type application such as List[int] as a special form. - """ - - base = None # type: Expression - index = None # type: Expression - # Inferred __getitem__ method type - method_type = None # type: mypy.types.Type - # If not None, this is actually semantically a type application - # Class[type, ...] or a type alias initializer. - analyzed = None # type: Union[TypeApplication, TypeAliasExpr] - - def __init__(self, base: Expression, index: Expression) -> None: - self.base = base - self.index = index - self.analyzed = None - if self.index.literal == LITERAL_YES: - self.literal = self.base.literal - self.literal_hash = ('Member', base.literal_hash, - index.literal_hash) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_index_expr(self) - - -class UnaryExpr(Expression): - """Unary operation""" - - op = '' - expr = None # type: Expression - # Inferred operator method type - method_type = None # type: mypy.types.Type - - def __init__(self, op: str, expr: Expression) -> None: - self.op = op - self.expr = expr - self.literal = self.expr.literal - self.literal_hash = ('Unary', op, expr.literal_hash) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_unary_expr(self) - - -# Map from binary operator id to related method name (in Python 3). -op_methods = { - '+': '__add__', - '-': '__sub__', - '*': '__mul__', - '/': '__truediv__', - '%': '__mod__', - '//': '__floordiv__', - '**': '__pow__', - '&': '__and__', - '|': '__or__', - '^': '__xor__', - '<<': '__lshift__', - '>>': '__rshift__', - '==': '__eq__', - '!=': '__ne__', - '<': '__lt__', - '>=': '__ge__', - '>': '__gt__', - '<=': '__le__', - 'in': '__contains__', -} # type: Dict[str, str] - -ops_with_inplace_method = { - '+', '-', '*', '/', '%', '//', '**', '&', '|', '^', '<<', '>>'} - -inplace_operator_methods = set( - '__i' + op_methods[op][2:] for op in ops_with_inplace_method) - -reverse_op_methods = { - '__add__': '__radd__', - '__sub__': '__rsub__', - '__mul__': '__rmul__', - '__truediv__': '__rtruediv__', - '__mod__': '__rmod__', - '__floordiv__': '__rfloordiv__', - '__pow__': '__rpow__', - '__and__': '__rand__', - '__or__': '__ror__', - '__xor__': '__rxor__', - '__lshift__': '__rlshift__', - '__rshift__': '__rrshift__', - '__eq__': '__eq__', - '__ne__': '__ne__', - '__lt__': '__gt__', - '__ge__': '__le__', - '__gt__': '__lt__', - '__le__': '__ge__', -} - -normal_from_reverse_op = dict((m, n) for n, m in reverse_op_methods.items()) -reverse_op_method_set = set(reverse_op_methods.values()) - - -class OpExpr(Expression): - """Binary operation (other than . or [] or comparison operators, - which have specific nodes).""" - - op = '' - left = None # type: Expression - right = None # type: Expression - # Inferred type for the operator method type (when relevant). - method_type = None # type: mypy.types.Type - - def __init__(self, op: str, left: Expression, right: Expression) -> None: - self.op = op - self.left = left - self.right = right - self.literal = min(self.left.literal, self.right.literal) - self.literal_hash = ('Binary', op, left.literal_hash, right.literal_hash) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_op_expr(self) - - -class ComparisonExpr(Expression): - """Comparison expression (e.g. a < b > c < d).""" - - operators = None # type: List[str] - operands = None # type: List[Expression] - # Inferred type for the operator methods (when relevant; None for 'is'). - method_types = None # type: List[mypy.types.Type] - - def __init__(self, operators: List[str], operands: List[Expression]) -> None: - self.operators = operators - self.operands = operands - self.method_types = [] - self.literal = min(o.literal for o in self.operands) - self.literal_hash = (('Comparison',) + tuple(operators) + - tuple(o.literal_hash for o in operands)) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_comparison_expr(self) - - -class SliceExpr(Expression): - """Slice expression (e.g. 'x:y', 'x:', '::2' or ':'). - - This is only valid as index in index expressions. - """ - - begin_index = None # type: Optional[Expression] - end_index = None # type: Optional[Expression] - stride = None # type: Optional[Expression] - - def __init__(self, begin_index: Optional[Expression], - end_index: Optional[Expression], - stride: Optional[Expression]) -> None: - self.begin_index = begin_index - self.end_index = end_index - self.stride = stride - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_slice_expr(self) - - -class CastExpr(Expression): - """Cast expression cast(type, expr).""" - - expr = None # type: Expression - type = None # type: mypy.types.Type - - def __init__(self, expr: Expression, typ: 'mypy.types.Type') -> None: - self.expr = expr - self.type = typ - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_cast_expr(self) - - -class RevealTypeExpr(Expression): - """Reveal type expression reveal_type(expr).""" - - expr = None # type: Expression - - def __init__(self, expr: Expression) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_reveal_type_expr(self) - - -class SuperExpr(Expression): - """Expression super().name""" - - name = '' - info = None # type: TypeInfo # Type that contains this super expression - - def __init__(self, name: str) -> None: - self.name = name - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_super_expr(self) - - -class FuncExpr(FuncItem, Expression): - """Lambda expression""" - - def name(self) -> str: - return '' - - def expr(self) -> Expression: - """Return the expression (the body) of the lambda.""" - ret = cast(ReturnStmt, self.body.body[-1]) - return ret.expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_func_expr(self) - - -class ListExpr(Expression): - """List literal expression [...].""" - - items = None # type: List[Expression] - - def __init__(self, items: List[Expression]) -> None: - self.items = items - if all(x.literal == LITERAL_YES for x in items): - self.literal = LITERAL_YES - self.literal_hash = ('List',) + tuple(x.literal_hash for x in items) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_list_expr(self) - - -class DictExpr(Expression): - """Dictionary literal expression {key: value, ...}.""" - - items = None # type: List[Tuple[Expression, Expression]] - - def __init__(self, items: List[Tuple[Expression, Expression]]) -> None: - self.items = items - if all(x[0].literal == LITERAL_YES and x[1].literal == LITERAL_YES - for x in items): - self.literal = LITERAL_YES - self.literal_hash = ('Dict',) + tuple( - (x[0].literal_hash, x[1].literal_hash) for x in items) # type: ignore - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_dict_expr(self) - - -class TupleExpr(Expression): - """Tuple literal expression (..., ...)""" - - items = None # type: List[Expression] - - def __init__(self, items: List[Expression]) -> None: - self.items = items - if all(x.literal == LITERAL_YES for x in items): - self.literal = LITERAL_YES - self.literal_hash = ('Tuple',) + tuple(x.literal_hash for x in items) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_tuple_expr(self) - - -class SetExpr(Expression): - """Set literal expression {value, ...}.""" - - items = None # type: List[Expression] - - def __init__(self, items: List[Expression]) -> None: - self.items = items - if all(x.literal == LITERAL_YES for x in items): - self.literal = LITERAL_YES - self.literal_hash = ('Set',) + tuple(x.literal_hash for x in items) - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_set_expr(self) - - -class GeneratorExpr(Expression): - """Generator expression ... for ... in ... [ for ... in ... ] [ if ... ].""" - - left_expr = None # type: Expression - sequences_expr = None # type: List[Expression] - condlists = None # type: List[List[Expression]] - indices = None # type: List[Expression] - - def __init__(self, left_expr: Expression, indices: List[Expression], - sequences: List[Expression], condlists: List[List[Expression]]) -> None: - self.left_expr = left_expr - self.sequences = sequences - self.condlists = condlists - self.indices = indices - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_generator_expr(self) - - -class ListComprehension(Expression): - """List comprehension (e.g. [x + 1 for x in a])""" - - generator = None # type: GeneratorExpr - - def __init__(self, generator: GeneratorExpr) -> None: - self.generator = generator - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_list_comprehension(self) - - -class SetComprehension(Expression): - """Set comprehension (e.g. {x + 1 for x in a})""" - - generator = None # type: GeneratorExpr - - def __init__(self, generator: GeneratorExpr) -> None: - self.generator = generator - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_set_comprehension(self) - - -class DictionaryComprehension(Expression): - """Dictionary comprehension (e.g. {k: v for k, v in a}""" - - key = None # type: Expression - value = None # type: Expression - sequences_expr = None # type: List[Expression] - condlists = None # type: List[List[Expression]] - indices = None # type: List[Expression] - - def __init__(self, key: Expression, value: Expression, indices: List[Expression], - sequences: List[Expression], condlists: List[List[Expression]]) -> None: - self.key = key - self.value = value - self.sequences = sequences - self.condlists = condlists - self.indices = indices - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_dictionary_comprehension(self) - - -class ConditionalExpr(Expression): - """Conditional expression (e.g. x if y else z)""" - - cond = None # type: Expression - if_expr = None # type: Expression - else_expr = None # type: Expression - - def __init__(self, cond: Expression, if_expr: Expression, else_expr: Expression) -> None: - self.cond = cond - self.if_expr = if_expr - self.else_expr = else_expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_conditional_expr(self) - - -class BackquoteExpr(Expression): - """Python 2 expression `...`.""" - - expr = None # type: Expression - - def __init__(self, expr: Expression) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_backquote_expr(self) - - -class TypeApplication(Expression): - """Type application expr[type, ...]""" - - expr = None # type: Expression - types = None # type: List[mypy.types.Type] - - def __init__(self, expr: Expression, types: List['mypy.types.Type']) -> None: - self.expr = expr - self.types = types - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_type_application(self) - - -# Variance of a type variable. For example, T in the definition of -# List[T] is invariant, so List[int] is not a subtype of List[object], -# and also List[object] is not a subtype of List[int]. -# -# The T in Iterable[T] is covariant, so Iterable[int] is a subtype of -# Iterable[object], but not vice versa. -# -# If T is contravariant in Foo[T], Foo[object] is a subtype of -# Foo[int], but not vice versa. -INVARIANT = 0 # type: int -COVARIANT = 1 # type: int -CONTRAVARIANT = 2 # type: int - - -class TypeVarExpr(SymbolNode, Expression): - """Type variable expression TypeVar(...).""" - - _name = '' - _fullname = '' - # Value restriction: only types in the list are valid as values. If the - # list is empty, there is no restriction. - values = None # type: List[mypy.types.Type] - # Upper bound: only subtypes of upper_bound are valid as values. By default - # this is 'object', meaning no restriction. - upper_bound = None # type: mypy.types.Type - # Variance of the type variable. Invariant is the default. - # TypeVar(..., covariant=True) defines a covariant type variable. - # TypeVar(..., contravariant=True) defines a contravariant type - # variable. - variance = INVARIANT - - def __init__(self, name: str, fullname: str, - values: List['mypy.types.Type'], - upper_bound: 'mypy.types.Type', - variance: int=INVARIANT) -> None: - self._name = name - self._fullname = fullname - self.values = values - self.upper_bound = upper_bound - self.variance = variance - - def name(self) -> str: - return self._name - - def fullname(self) -> str: - return self._fullname - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_type_var_expr(self) - - def serialize(self) -> JsonDict: - return {'.class': 'TypeVarExpr', - 'name': self._name, - 'fullname': self._fullname, - 'values': [t.serialize() for t in self.values], - 'upper_bound': self.upper_bound.serialize(), - 'variance': self.variance, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'TypeVarExpr': - assert data['.class'] == 'TypeVarExpr' - return TypeVarExpr(data['name'], - data['fullname'], - [mypy.types.Type.deserialize(v) for v in data['values']], - mypy.types.Type.deserialize(data['upper_bound']), - data['variance']) - - -class TypeAliasExpr(Expression): - """Type alias expression (rvalue).""" - - type = None # type: mypy.types.Type - - def __init__(self, type: 'mypy.types.Type') -> None: - self.type = type - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_type_alias_expr(self) - - -class NamedTupleExpr(Expression): - """Named tuple expression namedtuple(...).""" - - # The class representation of this named tuple (its tuple_type attribute contains - # the tuple item types) - info = None # type: TypeInfo - - def __init__(self, info: 'TypeInfo') -> None: - self.info = info - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_namedtuple_expr(self) - - -class PromoteExpr(Expression): - """Ducktype class decorator expression _promote(...).""" - - type = None # type: mypy.types.Type - - def __init__(self, type: 'mypy.types.Type') -> None: - self.type = type - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit__promote_expr(self) - - -# Constants - - -class TempNode(Expression): - """Temporary dummy node used during type checking. - - This node is not present in the original program; it is just an artifact - of the type checker implementation. It only represents an opaque node with - some fixed type. - """ - - type = None # type: mypy.types.Type - - def __init__(self, typ: 'mypy.types.Type') -> None: - self.type = typ - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_temp_node(self) - - -class TypeInfo(SymbolNode): - """The type structure of a single class. - - Each TypeInfo corresponds one-to-one to a ClassDef, which - represents the AST of the class. - - In type-theory terms, this is a "type constructor", and if the - class is generic then it will be a type constructor of higher kind. - Where the class is used in an actual type, it's in the form of an - Instance, which amounts to a type application of the tycon to - the appropriate number of arguments. - """ - - _fullname = None # type: str # Fully qualified name - defn = None # type: ClassDef # Corresponding ClassDef - # Method Resolution Order: the order of looking up attributes. The first - # value always to refers to this class. - mro = None # type: List[TypeInfo] - subtypes = None # type: Set[TypeInfo] # Direct subclasses encountered so far - names = None # type: SymbolTable # Names defined directly in this type - is_abstract = False # Does the class have any abstract attributes? - abstract_attributes = None # type: List[str] - # Classes inheriting from Enum shadow their true members with a __getattr__, so we - # have to treat them as a special case. - is_enum = False - # If true, any unknown attributes should have type 'Any' instead - # of generating a type error. This would be true if there is a - # base class with type 'Any', but other use cases may be - # possible. This is similar to having __getattr__ that returns Any - # (and __setattr__), but without the __getattr__ method. - fallback_to_any = False - - # Information related to type annotations. - - # Generic type variable names - type_vars = None # type: List[str] - - # Direct base classes. - bases = None # type: List[mypy.types.Instance] - - # Another type which this type will be treated as a subtype of, - # even though it's not a subclass in Python. The non-standard - # `@_promote` decorator introduces this, and there are also - # several builtin examples, in particular `int` -> `float`. - _promote = None # type: mypy.types.Type - - # Representation of a Tuple[...] base class, if the class has any - # (e.g., for named tuples). If this is not None, the actual Type - # object used for this class is not an Instance but a TupleType; - # the corresponding Instance is set as the fallback type of the - # tuple type. - tuple_type = None # type: mypy.types.TupleType - - # Is this a named tuple type? - is_named_tuple = False - - # Is this a dummy from deserialization? - is_dummy = False - - # Alternative to fullname() for 'anonymous' classes. - alt_fullname = None # type: Optional[str] - - def __init__(self, names: 'SymbolTable', defn: ClassDef) -> None: - """Initialize a TypeInfo.""" - self.names = names - self.defn = defn - self.subtypes = set() - self.type_vars = [] - self.bases = [] - # Leave self.mro uninitialized until we compute it for real, - # so we don't accidentally try to use it prematurely. - self._fullname = defn.fullname - self.is_abstract = False - self.abstract_attributes = [] - if defn.type_vars: - for vd in defn.type_vars: - self.type_vars.append(vd.name) - - def name(self) -> str: - """Short name.""" - return self.defn.name - - def fullname(self) -> str: - return self._fullname - - def is_generic(self) -> bool: - """Is the type generic (i.e. does it have type variables)?""" - return len(self.type_vars) > 0 - - def get(self, name: str) -> 'SymbolTableNode': - for cls in self.mro: - n = cls.names.get(name) - if n: - return n - return None - - def __getitem__(self, name: str) -> 'SymbolTableNode': - n = self.get(name) - if n: - return n - else: - raise KeyError(name) - - def __repr__(self) -> str: - return '' % self.fullname() - - # IDEA: Refactor the has* methods to be more consistent and document - # them. - - def has_readable_member(self, name: str) -> bool: - return self.get(name) is not None - - def has_writable_member(self, name: str) -> bool: - return self.has_var(name) - - def has_var(self, name: str) -> bool: - return self.get_var(name) is not None - - def has_method(self, name: str) -> bool: - return self.get_method(name) is not None - - def get_var(self, name: str) -> Var: - for cls in self.mro: - if name in cls.names: - node = cls.names[name].node - if isinstance(node, Var): - return node - else: - return None - return None - - def get_var_or_getter(self, name: str) -> SymbolNode: - # TODO getter - return self.get_var(name) - - def get_var_or_setter(self, name: str) -> SymbolNode: - # TODO setter - return self.get_var(name) - - def get_method(self, name: str) -> FuncBase: - if self.mro is None: # Might be because of a previous error. - return None - for cls in self.mro: - if name in cls.names: - node = cls.names[name].node - if isinstance(node, FuncBase): - return node - else: - return None - return None - - def calculate_mro(self) -> None: - """Calculate and set mro (method resolution order). - - Raise MroError if cannot determine mro. - """ - mro = linearize_hierarchy(self) - assert mro, "Could not produce a MRO at all for %s" % (self,) - self.mro = mro - self.is_enum = self._calculate_is_enum() - - def _calculate_is_enum(self) -> bool: - """ - If this is "enum.Enum" itself, then yes, it's an enum. - If the flag .is_enum has been set on anything in the MRO, it's an enum. - """ - if self.fullname() == ENUM_BASECLASS: - return True - if self.mro: - return any(type_info.is_enum for type_info in self.mro) - return False - - def has_base(self, fullname: str) -> bool: - """Return True if type has a base type with the specified name. - - This can be either via extension or via implementation. - """ - if self.mro: - for cls in self.mro: - if cls.fullname() == fullname: - return True - return False - - def all_subtypes(self) -> 'Set[TypeInfo]': - """Return TypeInfos of all subtypes, including this type, as a set.""" - subtypes = set([self]) - for subt in self.subtypes: - for t in subt.all_subtypes(): - subtypes.add(t) - return subtypes - - def all_base_classes(self) -> 'List[TypeInfo]': - """Return a list of base classes, including indirect bases.""" - assert False - - def direct_base_classes(self) -> 'List[TypeInfo]': - """Return a direct base classes. - - Omit base classes of other base classes. - """ - return [base.type for base in self.bases] - - def __str__(self) -> str: - """Return a string representation of the type. - - This includes the most important information about the type. - """ - base = None # type: str - if self.bases: - base = 'Bases({})'.format(', '.join(str(base) - for base in self.bases)) - return dump_tagged(['Name({})'.format(self.fullname()), - base, - ('Names', sorted(self.names.keys()))], - 'TypeInfo') - - def serialize(self) -> Union[str, JsonDict]: - # NOTE: This is where all ClassDefs originate, so there shouldn't be duplicates. - data = {'.class': 'TypeInfo', - 'fullname': self.fullname(), - 'alt_fullname': self.alt_fullname, - 'names': self.names.serialize(self.alt_fullname or self.fullname()), - 'defn': self.defn.serialize(), - 'is_abstract': self.is_abstract, - 'abstract_attributes': self.abstract_attributes, - 'is_enum': self.is_enum, - 'fallback_to_any': self.fallback_to_any, - 'type_vars': self.type_vars, - 'bases': [b.serialize() for b in self.bases], - '_promote': None if self._promote is None else self._promote.serialize(), - 'tuple_type': None if self.tuple_type is None else self.tuple_type.serialize(), - 'is_named_tuple': self.is_named_tuple, - } - return data - - @classmethod - def deserialize(cls, data: JsonDict) -> 'TypeInfo': - names = SymbolTable.deserialize(data['names']) - defn = ClassDef.deserialize(data['defn']) - ti = TypeInfo(names, defn) - ti._fullname = data['fullname'] - ti.alt_fullname = data['alt_fullname'] - # TODO: Is there a reason to reconstruct ti.subtypes? - ti.is_abstract = data['is_abstract'] - ti.abstract_attributes = data['abstract_attributes'] - ti.is_enum = data['is_enum'] - ti.fallback_to_any = data['fallback_to_any'] - ti.type_vars = data['type_vars'] - ti.bases = [mypy.types.Instance.deserialize(b) for b in data['bases']] - ti._promote = (None if data['_promote'] is None - else mypy.types.Type.deserialize(data['_promote'])) - ti.tuple_type = (None if data['tuple_type'] is None - else mypy.types.TupleType.deserialize(data['tuple_type'])) - ti.is_named_tuple = data['is_named_tuple'] - return ti - - -def NamedTupleTypeInfo(tup: 'mypy.types.NamedTupleType', *args) -> TypeInfo: - info = TypeInfo(*args) - info.tuple_type = tup - info.bases = [tup.fallback] - info.is_named_tuple = True - info.mro = [info] + tup.fallback.type.mro - return info - -class SymbolTableNode: - # Kind of node. Possible values: - # - LDEF: local definition (of any kind) - # - GDEF: global (module-level) definition - # - MDEF: class member definition - # - UNBOUND_TVAR: TypeVar(...) definition, not bound - # - TVAR: type variable in a bound scope (generic function / generic clas) - # - MODULE_REF: reference to a module - # - TYPE_ALIAS: type alias - # - UNBOUND_IMPORTED: temporary kind for imported names - kind = None # type: int - # AST node of definition (FuncDef/Var/TypeInfo/Decorator/TypeVarExpr, - # or None for a bound type variable). - node = None # type: Optional[SymbolNode] - # Type variable definition (for bound type variables only) - tvar_def = None # type: Optional[mypy.types.TypeVarDef] - # Module id (e.g. "foo.bar") or None - mod_id = '' - # If this not None, override the type of the 'node' attribute. - type_override = None # type: Optional[mypy.types.Type] - # If False, this name won't be imported via 'from import *'. - # This has no effect on names within classes. - module_public = True - # For deserialized MODULE_REF nodes, the referenced module name; - # for other nodes, optionally the name of the referenced object. - cross_ref = None # type: Optional[str] - - def __init__(self, kind: int, node: Optional[SymbolNode], mod_id: str = None, - typ: 'mypy.types.Type' = None, - tvar_def: 'mypy.types.TypeVarDef' = None, - module_public: bool = True) -> None: - self.kind = kind - self.node = node - self.type_override = typ - self.mod_id = mod_id - self.tvar_def = tvar_def - self.module_public = module_public - - @property - def fullname(self) -> str: - if self.node is not None: - return self.node.fullname() - else: - return None - - @property - def type(self) -> 'mypy.types.Type': - # IDEA: Get rid of the Any type. - node = self.node # type: Any - if self.type_override is not None: - return self.type_override - elif ((isinstance(node, Var) or isinstance(node, FuncDef)) - and node.type is not None): - return node.type - elif isinstance(node, Decorator): - return node.var.type - else: - return None - - def __str__(self) -> str: - s = '{}/{}'.format(node_kinds[self.kind], short_type(self.node)) - if self.mod_id is not None: - s += ' ({})'.format(self.mod_id) - # Include declared type of variables and functions. - if self.type is not None: - s += ' : {}'.format(self.type) - return s - - def serialize(self, prefix: str, name: str) -> JsonDict: - """Serialize a SymbolTableNode. - - Args: - prefix: full name of the containing module or class; or None - name: name of this object relative to the containing object - """ - data = {'.class': 'SymbolTableNode', - 'kind': node_kinds[self.kind], - } # type: JsonDict - if self.tvar_def: - data['tvar_def'] = self.tvar_def.serialize() - if not self.module_public: - data['module_public'] = False - if self.kind == MODULE_REF: - assert self.node is not None, "Missing module cross ref in %s for %s" % (prefix, name) - data['cross_ref'] = self.node.fullname() - else: - if self.node is not None: - if prefix is not None: - # Check whether this is an alias for another object. - # If the object's canonical full name differs from - # the full name computed from prefix and name, - # it's an alias, and we serialize it as a cross ref. - if isinstance(self.node, TypeInfo): - fullname = self.node.alt_fullname or self.node.fullname() - else: - fullname = self.node.fullname() - if (fullname is not None and '.' in fullname and - fullname != prefix + '.' + name): - data['cross_ref'] = fullname - return data - data['node'] = self.node.serialize() - if self.type_override is not None: - data['type_override'] = self.type_override.serialize() - return data - - @classmethod - def deserialize(cls, data: JsonDict) -> 'SymbolTableNode': - assert data['.class'] == 'SymbolTableNode' - kind = inverse_node_kinds[data['kind']] - if 'cross_ref' in data: - # This will be fixed up later. - stnode = SymbolTableNode(kind, None) - stnode.cross_ref = data['cross_ref'] - else: - node = None - if 'node' in data: - node = SymbolNode.deserialize(data['node']) - typ = None - if 'type_override' in data: - typ = mypy.types.Type.deserialize(data['type_override']) - stnode = SymbolTableNode(kind, node, typ=typ) - if 'tvar_def' in data: - stnode.tvar_def = mypy.types.TypeVarDef.deserialize(data['tvar_def']) - if 'module_public' in data: - stnode.module_public = data['module_public'] - return stnode - - -class SymbolTable(Dict[str, SymbolTableNode]): - def __str__(self) -> str: - a = [] # type: List[str] - for key, value in self.items(): - # Filter out the implicit import of builtins. - if isinstance(value, SymbolTableNode): - if (value.fullname != 'builtins' and - (value.fullname or '').split('.')[-1] not in - implicit_module_attrs): - a.append(' ' + str(key) + ' : ' + str(value)) - else: - a.append(' ') - a = sorted(a) - a.insert(0, 'SymbolTable(') - a[-1] += ')' - return '\n'.join(a) - - def serialize(self, fullname: str) -> JsonDict: - data = {'.class': 'SymbolTable'} # type: JsonDict - for key, value in self.items(): - # Skip __builtins__: it's a reference to the builtins - # module that gets added to every module by - # SemanticAnalyzer.visit_file(), but it shouldn't be - # accessed by users of the module. - if key == '__builtins__': - continue - data[key] = value.serialize(fullname, key) - return data - - @classmethod - def deserialize(cls, data: JsonDict) -> 'SymbolTable': - assert data['.class'] == 'SymbolTable' - st = SymbolTable() - for key, value in data.items(): - if key != '.class': - st[key] = SymbolTableNode.deserialize(value) - return st - - -def function_type(func: FuncBase, fallback: 'mypy.types.Instance') -> 'mypy.types.FunctionLike': - if func.type: - assert isinstance(func.type, mypy.types.FunctionLike) - return func.type - else: - # Implicit type signature with dynamic types. - # Overloaded functions always have a signature, so func must be an ordinary function. - fdef = cast(FuncDef, func) - name = func.name() - if name: - name = '"{}"'.format(name) - names = [] # type: List[str] - for arg in fdef.arguments: - names.append(arg.variable.name()) - - return mypy.types.CallableType( - [mypy.types.AnyType()] * len(fdef.arguments), - [arg.kind for arg in fdef.arguments], - names, - mypy.types.AnyType(), - fallback, - name, - implicit=True, - ) - - -def method_type_with_fallback(func: FuncBase, - fallback: 'mypy.types.Instance') -> 'mypy.types.FunctionLike': - """Return the signature of a method (omit self).""" - return method_type(function_type(func, fallback)) - - -def method_type(sig: 'mypy.types.FunctionLike') -> 'mypy.types.FunctionLike': - if isinstance(sig, mypy.types.CallableType): - return method_callable(sig) - else: - sig = cast(mypy.types.Overloaded, sig) - items = [] # type: List[mypy.types.CallableType] - for c in sig.items(): - items.append(method_callable(c)) - return mypy.types.Overloaded(items) - - -def method_callable(c: 'mypy.types.CallableType') -> 'mypy.types.CallableType': - if c.arg_kinds and c.arg_kinds[0] == ARG_STAR: - # The signature is of the form 'def foo(*args, ...)'. - # In this case we shouldn't drop the first arg, - # since self will be absorbed by the *args. - return c - return c.copy_modified(arg_types=c.arg_types[1:], - arg_kinds=c.arg_kinds[1:], - arg_names=c.arg_names[1:]) - - -class MroError(Exception): - """Raised if a consistent mro cannot be determined for a class.""" - - -def linearize_hierarchy(info: TypeInfo) -> List[TypeInfo]: - # TODO describe - if info.mro: - return info.mro - bases = info.direct_base_classes() - lin_bases = [] - for base in bases: - assert base is not None, "Cannot linearize bases for %s %s" % (info.fullname(), bases) - lin_bases.append(linearize_hierarchy(base)) - lin_bases.append(bases) - return [info] + merge(lin_bases) - - -def merge(seqs: List[List[TypeInfo]]) -> List[TypeInfo]: - seqs = [s[:] for s in seqs] - result = [] # type: List[TypeInfo] - while True: - seqs = [s for s in seqs if s] - if not seqs: - return result - for seq in seqs: - head = seq[0] - if not [s for s in seqs if head in s[1:]]: - break - else: - raise MroError() - result.append(head) - for s in seqs: - if s[0] is head: - del s[0] +"""Abstract syntax tree node classes (i.e. parse tree).""" + +import os +import re +from abc import abstractmethod, ABCMeta + +from typing import ( + Any, TypeVar, List, Tuple, cast, Set, Dict, Union, Optional +) + +from mypy.lex import Token +import mypy.strconv +from mypy.visitor import NodeVisitor +from mypy.util import dump_tagged, short_type + + +class Context: + """Base type for objects that are valid as error message locations.""" + @abstractmethod + def get_line(self) -> int: pass + + +if False: + # break import cycle only needed for mypy + import mypy.types + + +T = TypeVar('T') + +JsonDict = Dict[str, Any] + + +# Symbol table node kinds +# +# TODO rename to use more descriptive names + +LDEF = 0 # type: int +GDEF = 1 # type: int +MDEF = 2 # type: int +MODULE_REF = 3 # type: int +# Type variable declared using TypeVar(...) has kind UNBOUND_TVAR. It's not +# valid as a type. A type variable is valid as a type (kind BOUND_TVAR) within +# (1) a generic class that uses the type variable as a type argument or +# (2) a generic function that refers to the type variable in its signature. +UNBOUND_TVAR = 4 # type: int +BOUND_TVAR = 5 # type: int +TYPE_ALIAS = 6 # type: int +# Placeholder for a name imported via 'from ... import'. Second phase of +# semantic will replace this the actual imported reference. This is +# needed so that we can detect whether a name has been imported during +# XXX what? +UNBOUND_IMPORTED = 7 # type: int + + +LITERAL_YES = 2 +LITERAL_TYPE = 1 +LITERAL_NO = 0 + +# Hard coded name of Enum baseclass. +ENUM_BASECLASS = "enum.Enum" + +node_kinds = { + LDEF: 'Ldef', + GDEF: 'Gdef', + MDEF: 'Mdef', + MODULE_REF: 'ModuleRef', + UNBOUND_TVAR: 'UnboundTvar', + BOUND_TVAR: 'Tvar', + TYPE_ALIAS: 'TypeAlias', + UNBOUND_IMPORTED: 'UnboundImported', +} +inverse_node_kinds = {_kind: _name for _name, _kind in node_kinds.items()} + + +implicit_module_attrs = {'__name__': '__builtins__.str', + '__doc__': '__builtins__.str', + '__file__': '__builtins__.str', + '__package__': '__builtins__.str'} + + +type_aliases = { + 'typing.List': '__builtins__.list', + 'typing.Dict': '__builtins__.dict', + 'typing.Set': '__builtins__.set', +} + +reverse_type_aliases = dict((name.replace('__builtins__', 'builtins'), alias) + for alias, name in type_aliases.items()) # type: Dict[str, str] + + +class Node(Context): + """Common base class for all non-type parse tree nodes.""" + + line = -1 + + literal = LITERAL_NO + literal_hash = None # type: Any + + def __str__(self) -> str: + ans = self.accept(mypy.strconv.StrConv()) + if ans is None: + return repr(self) + return ans + + def set_line(self, target: Union[Token, 'Node', int]) -> 'Node': + if isinstance(target, int): + self.line = target + else: + self.line = target.line + return self + + def get_line(self) -> int: + # TODO this should be just 'line' + return self.line + + def accept(self, visitor: NodeVisitor[T]) -> T: + raise RuntimeError('Not implemented') + + +# These are placeholders for a future refactoring; see #1783. +# For now they serve as (unchecked) documentation of what various +# fields of Node subtypes are expected to contain. +Statement = Node +Expression = Node + + +class SymbolNode(Node): + # Nodes that can be stored in a symbol table. + + # TODO do not use methods for these + + @abstractmethod + def name(self) -> str: pass + + @abstractmethod + def fullname(self) -> str: pass + + # NOTE: Can't use @abstractmethod, since many subclasses of Node + # don't implement serialize(). + def serialize(self) -> Any: + raise NotImplementedError('Cannot serialize {} instance'.format(self.__class__.__name__)) + + @classmethod + def deserialize(cls, data: JsonDict) -> 'SymbolNode': + classname = data['.class'] + glo = globals() + if classname in glo: + cl = glo[classname] + if issubclass(cl, cls) and 'deserialize' in cl.__dict__: + return cl.deserialize(data) + raise NotImplementedError('unexpected .class {}'.format(classname)) + + +class MypyFile(SymbolNode, Statement): + """The abstract syntax tree of a single source file.""" + + # Module name ('__main__' for initial file) + _name = None # type: str + # Fully qualified module name + _fullname = None # type: str + # Path to the file (None if not known) + path = '' + # Top-level definitions and statements + defs = None # type: List[Statement] + # Is there a UTF-8 BOM at the start? + is_bom = False + names = None # type: SymbolTable + # All import nodes within the file (also ones within functions etc.) + imports = None # type: List[ImportBase] + # Lines to ignore when checking + ignored_lines = None # type: Set[int] + # Is this file represented by a stub file (.pyi)? + is_stub = False + # Do weak typing globally in the file? + weak_opts = None # type: Set[str] + + def __init__(self, + defs: List[Statement], + imports: List['ImportBase'], + is_bom: bool = False, + ignored_lines: Set[int] = None, + weak_opts: Set[str] = None) -> None: + self.defs = defs + self.line = 1 # Dummy line number + self.imports = imports + self.is_bom = is_bom + self.weak_opts = weak_opts + if ignored_lines: + self.ignored_lines = ignored_lines + else: + self.ignored_lines = set() + + def name(self) -> str: + return self._name + + def fullname(self) -> str: + return self._fullname + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_mypy_file(self) + + def is_package_init_file(self) -> bool: + return not (self.path is None) and len(self.path) != 0 \ + and os.path.basename(self.path).startswith('__init__.') + + def serialize(self) -> JsonDict: + return {'.class': 'MypyFile', + '_name': self._name, + '_fullname': self._fullname, + 'names': self.names.serialize(self._fullname), + 'is_stub': self.is_stub, + 'path': self.path, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'MypyFile': + assert data['.class'] == 'MypyFile', data + tree = MypyFile([], []) + tree._name = data['_name'] + tree._fullname = data['_fullname'] + tree.names = SymbolTable.deserialize(data['names']) + tree.is_stub = data['is_stub'] + tree.path = data['path'] + return tree + + +class ImportBase(Statement): + """Base class for all import statements.""" + is_unreachable = False + is_top_level = False # Set by semanal.FirstPass + # If an import replaces existing definitions, we construct dummy assignment + # statements that assign the imported names to the names in the current scope, + # for type checking purposes. Example: + # + # x = 1 + # from m import x <-- add assignment representing "x = m.x" + assignments = None # type: List[AssignmentStmt] + + def __init__(self) -> None: + self.assignments = [] + + +class Import(ImportBase): + """import m [as n]""" + + ids = None # type: List[Tuple[str, Optional[str]]] # (module id, as id) + + def __init__(self, ids: List[Tuple[str, Optional[str]]]) -> None: + super().__init__() + self.ids = ids + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_import(self) + + +class ImportFrom(ImportBase): + """from m import x [as y], ...""" + + names = None # type: List[Tuple[str, Optional[str]]] # Tuples (name, as name) + + def __init__(self, id: str, relative: int, names: List[Tuple[str, Optional[str]]]) -> None: + super().__init__() + self.id = id + self.names = names + self.relative = relative + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_import_from(self) + + +class ImportAll(ImportBase): + """from m import *""" + + def __init__(self, id: str, relative: int) -> None: + super().__init__() + self.id = id + self.relative = relative + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_import_all(self) + + +class FuncBase(SymbolNode): + """Abstract base class for function-like nodes""" + + # Type signature. This is usually CallableType or Overloaded, but it can be something else for + # decorated functions/ + type = None # type: mypy.types.Type + # If method, reference to TypeInfo + info = None # type: TypeInfo + is_property = False + _fullname = None # type: str # Name with module prefix + + @abstractmethod + def name(self) -> str: pass + + def fullname(self) -> str: + return self._fullname + + def is_method(self) -> bool: + return bool(self.info) + + +class OverloadedFuncDef(FuncBase, Statement): + """A logical node representing all the variants of an overloaded function. + + This node has no explicit representation in the source program. + Overloaded variants must be consecutive in the source file. + """ + + items = None # type: List[Decorator] + + def __init__(self, items: List['Decorator']) -> None: + self.items = items + self.set_line(items[0].line) + + def name(self) -> str: + return self.items[0].func.name() + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_overloaded_func_def(self) + + def serialize(self) -> JsonDict: + return {'.class': 'OverloadedFuncDef', + 'items': [i.serialize() for i in self.items], + 'type': None if self.type is None else self.type.serialize(), + 'fullname': self._fullname, + 'is_property': self.is_property, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'OverloadedFuncDef': + assert data['.class'] == 'OverloadedFuncDef' + res = OverloadedFuncDef([Decorator.deserialize(d) for d in data['items']]) + if data.get('type') is not None: + res.type = mypy.types.Type.deserialize(data['type']) + res._fullname = data['fullname'] + res.is_property = data['is_property'] + # NOTE: res.info will be set in the fixup phase. + return res + + +class Argument(Node): + """A single argument in a FuncItem.""" + + def __init__(self, variable: 'Var', type_annotation: 'Optional[mypy.types.Type]', + initializer: Optional[Expression], kind: int, + initialization_statement: Optional['AssignmentStmt'] = None) -> None: + self.variable = variable + + self.type_annotation = type_annotation + self.initializer = initializer + + self.initialization_statement = initialization_statement + if not self.initialization_statement: + self.initialization_statement = self._initialization_statement() + + self.kind = kind + + def _initialization_statement(self) -> Optional['AssignmentStmt']: + """Convert the initializer into an assignment statement. + """ + if not self.initializer: + return None + + rvalue = self.initializer + lvalue = NameExpr(self.variable.name()) + assign = AssignmentStmt([lvalue], rvalue) + return assign + + def set_line(self, target: Union[Token, Node, int]) -> Node: + super().set_line(target) + + if self.initializer: + self.initializer.set_line(self.line) + + self.variable.set_line(self.line) + + if self.initialization_statement: + self.initialization_statement.set_line(self.line) + self.initialization_statement.lvalues[0].set_line(self.line) + + def serialize(self) -> JsonDict: + data = {'.class': 'Argument', + 'kind': self.kind, + 'variable': self.variable.serialize(), + 'type_annotation': (None if self.type_annotation is None + else self.type_annotation.serialize()), + } # type: JsonDict + # TODO: initializer? + return data + + @classmethod + def deserialize(cls, data: JsonDict) -> 'Argument': + assert data['.class'] == 'Argument' + return Argument(Var.deserialize(data['variable']), + (None if data.get('type_annotation') is None + else mypy.types.Type.deserialize(data['type_annotation'])), + None, # TODO: initializer? + kind=data['kind']) + + +class FuncItem(FuncBase): + arguments = [] # type: List[Argument] + # Minimum number of arguments + min_args = 0 + # Maximum number of positional arguments, -1 if no explicit limit (*args not included) + max_pos = 0 + body = None # type: Block + # Is this an overload variant of function with more than one overload variant? + is_overload = False + is_generator = False # Contains a yield statement? + is_static = False # Uses @staticmethod? + is_class = False # Uses @classmethod? + # Variants of function with type variables with values expanded + expanded = None # type: List[FuncItem] + + def __init__(self, arguments: List[Argument], body: 'Block', + typ: 'mypy.types.FunctionLike' = None) -> None: + self.arguments = arguments + arg_kinds = [arg.kind for arg in self.arguments] + self.max_pos = arg_kinds.count(ARG_POS) + arg_kinds.count(ARG_OPT) + self.body = body + self.type = typ + self.expanded = [] + + self.min_args = 0 + for i in range(len(self.arguments)): + if self.arguments[i] is None and i < self.max_fixed_argc(): + self.min_args = i + 1 + + def max_fixed_argc(self) -> int: + return self.max_pos + + def set_line(self, target: Union[Token, Node, int]) -> Node: + super().set_line(target) + for arg in self.arguments: + arg.set_line(self.line) + return self + + def is_dynamic(self): + return self.type is None + + +class FuncDef(FuncItem, Statement): + """Function definition. + + This is a non-lambda function defined using 'def'. + """ + + is_decorated = False + is_conditional = False # Defined conditionally (within block)? + is_abstract = False + is_property = False + original_def = None # type: Union[None, FuncDef, Var] # Original conditional definition + + def __init__(self, + name: str, # Function name + arguments: List[Argument], + body: 'Block', + typ: 'mypy.types.FunctionLike' = None) -> None: + super().__init__(arguments, body, typ) + self._name = name + + def name(self) -> str: + return self._name + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_func_def(self) + + def is_constructor(self) -> bool: + return self.info is not None and self._name == '__init__' + + def serialize(self) -> JsonDict: + return {'.class': 'FuncDef', + 'name': self._name, + 'fullname': self._fullname, + 'arguments': [a.serialize() for a in self.arguments], + 'type': None if self.type is None else self.type.serialize(), + 'is_property': self.is_property, + 'is_overload': self.is_overload, + 'is_generator': self.is_generator, + 'is_static': self.is_static, + 'is_class': self.is_class, + 'is_decorated': self.is_decorated, + 'is_conditional': self.is_conditional, + 'is_abstract': self.is_abstract, + # TODO: Do we need expanded, original_def? + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'FuncDef': + assert data['.class'] == 'FuncDef' + body = Block([]) + ret = FuncDef(data['name'], + [Argument.deserialize(a) for a in data['arguments']], + body, + (None if data['type'] is None + else mypy.types.FunctionLike.deserialize(data['type']))) + ret._fullname = data['fullname'] + ret.is_property = data['is_property'] + ret.is_overload = data['is_overload'] + ret.is_generator = data['is_generator'] + ret.is_static = data['is_static'] + ret.is_class = data['is_class'] + ret.is_decorated = data['is_decorated'] + ret.is_conditional = data['is_conditional'] + ret.is_abstract = data['is_abstract'] + # NOTE: ret.info is set in the fixup phase. + return ret + + +class Decorator(SymbolNode, Statement): + """A decorated function. + + A single Decorator object can include any number of function decorators. + """ + + func = None # type: FuncDef # Decorated function + decorators = None # type: List[Expression] # Decorators, at least one # XXX Not true + var = None # type: Var # Represents the decorated function obj + is_overload = False + + def __init__(self, func: FuncDef, decorators: List[Expression], + var: 'Var') -> None: + self.func = func + self.decorators = decorators + self.var = var + self.is_overload = False + + def name(self) -> str: + return self.func.name() + + def fullname(self) -> str: + return self.func.fullname() + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_decorator(self) + + def serialize(self) -> JsonDict: + return {'.class': 'Decorator', + 'func': self.func.serialize(), + 'var': self.var.serialize(), + 'is_overload': self.is_overload, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'Decorator': + assert data['.class'] == 'Decorator' + dec = Decorator(FuncDef.deserialize(data['func']), + [], + Var.deserialize(data['var'])) + dec.is_overload = data['is_overload'] + return dec + + +class Var(SymbolNode, Statement): + """A variable. + + It can refer to global/local variable or a data attribute. + """ + + _name = None # type: str # Name without module prefix + _fullname = None # type: str # Name with module prefix + info = None # type: TypeInfo # Defining class (for member variables) + type = None # type: mypy.types.Type # Declared or inferred type, or None + # Is this the first argument to an ordinary method (usually "self")? + is_self = False + is_ready = False # If inferred, is the inferred type available? + # Is this initialized explicitly to a non-None value in class body? + is_initialized_in_class = False + is_staticmethod = False + is_classmethod = False + is_property = False + is_settable_property = False + + def __init__(self, name: str, type: 'mypy.types.Type' = None) -> None: + self._name = name + self.type = type + self.is_self = False + self.is_ready = True + self.is_initialized_in_class = False + + def name(self) -> str: + return self._name + + def fullname(self) -> str: + return self._fullname + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_var(self) + + def serialize(self) -> JsonDict: + # TODO: Leave default values out? + # NOTE: Sometimes self.is_ready is False here, but we don't care. + data = {'.class': 'Var', + 'name': self._name, + 'fullname': self._fullname, + 'type': None if self.type is None else self.type.serialize(), + 'is_self': self.is_self, + 'is_initialized_in_class': self.is_initialized_in_class, + 'is_staticmethod': self.is_staticmethod, + 'is_classmethod': self.is_classmethod, + 'is_property': self.is_property, + 'is_settable_property': self.is_settable_property, + } # type: JsonDict + return data + + @classmethod + def deserialize(cls, data: JsonDict) -> 'Var': + assert data['.class'] == 'Var' + name = data['name'] + type = None if data['type'] is None else mypy.types.Type.deserialize(data['type']) + v = Var(name, type) + v._fullname = data['fullname'] + v.is_self = data['is_self'] + v.is_initialized_in_class = data['is_initialized_in_class'] + v.is_staticmethod = data['is_staticmethod'] + v.is_classmethod = data['is_classmethod'] + v.is_property = data['is_property'] + v.is_settable_property = data['is_settable_property'] + return v + + +class ClassDef(Statement): + """Class definition""" + + name = None # type: str # Name of the class without module prefix + fullname = None # type: str # Fully qualified name of the class + defs = None # type: Block + type_vars = None # type: List[mypy.types.TypeVarDef] + # Base class expressions (not semantically analyzed -- can be arbitrary expressions) + base_type_exprs = None # type: List[Expression] + info = None # type: TypeInfo # Related TypeInfo + metaclass = '' + decorators = None # type: List[Expression] + # Built-in/extension class? (single implementation inheritance only) + is_builtinclass = False + + def __init__(self, + name: str, + defs: 'Block', + type_vars: List['mypy.types.TypeVarDef'] = None, + base_type_exprs: List[Expression] = None, + metaclass: str = None) -> None: + self.name = name + self.defs = defs + self.type_vars = type_vars or [] + self.base_type_exprs = base_type_exprs or [] + self.metaclass = metaclass + self.decorators = [] + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_class_def(self) + + def is_generic(self) -> bool: + return self.info.is_generic() + + def serialize(self) -> JsonDict: + # Not serialized: defs, base_type_exprs, decorators + return {'.class': 'ClassDef', + 'name': self.name, + 'fullname': self.fullname, + 'type_vars': [v.serialize() for v in self.type_vars], + 'metaclass': self.metaclass, + 'is_builtinclass': self.is_builtinclass, + } + + @classmethod + def deserialize(self, data: JsonDict) -> 'ClassDef': + assert data['.class'] == 'ClassDef' + res = ClassDef(data['name'], + Block([]), + [mypy.types.TypeVarDef.deserialize(v) for v in data['type_vars']], + metaclass=data['metaclass'], + ) + res.fullname = data['fullname'] + res.is_builtinclass = data['is_builtinclass'] + return res + + +class GlobalDecl(Statement): + """Declaration global x, y, ...""" + + names = None # type: List[str] + + def __init__(self, names: List[str]) -> None: + self.names = names + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_global_decl(self) + + +class NonlocalDecl(Statement): + """Declaration nonlocal x, y, ...""" + + names = None # type: List[str] + + def __init__(self, names: List[str]) -> None: + self.names = names + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_nonlocal_decl(self) + + +class Block(Statement): + body = None # type: List[Statement] + # True if we can determine that this block is not executed. For example, + # this applies to blocks that are protected by something like "if PY3:" + # when using Python 2. + is_unreachable = False + + def __init__(self, body: List[Statement]) -> None: + self.body = body + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_block(self) + + +# Statements + + +class ExpressionStmt(Statement): + """An expression as a statement, such as print(s).""" + expr = None # type: Expression + + def __init__(self, expr: Expression) -> None: + self.expr = expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_expression_stmt(self) + + +class AssignmentStmt(Statement): + """Assignment statement + The same node class is used for single assignment, multiple assignment + (e.g. x, y = z) and chained assignment (e.g. x = y = z), assignments + that define new names, and assignments with explicit types (# type). + + An lvalue can be NameExpr, TupleExpr, ListExpr, MemberExpr, IndexExpr. + """ + + lvalues = None # type: List[Expression] + rvalue = None # type: Expression + # Declared type in a comment, may be None. + type = None # type: mypy.types.Type + + def __init__(self, lvalues: List[Expression], rvalue: Expression, + type: 'mypy.types.Type' = None) -> None: + self.lvalues = lvalues + self.rvalue = rvalue + self.type = type + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_assignment_stmt(self) + + +class OperatorAssignmentStmt(Statement): + """Operator assignment statement such as x += 1""" + + op = '' + lvalue = None # type: Expression + rvalue = None # type: Expression + + def __init__(self, op: str, lvalue: Expression, rvalue: Expression) -> None: + self.op = op + self.lvalue = lvalue + self.rvalue = rvalue + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_operator_assignment_stmt(self) + + +class WhileStmt(Statement): + expr = None # type: Expression + body = None # type: Block + else_body = None # type: Block + + def __init__(self, expr: Expression, body: Block, else_body: Block) -> None: + self.expr = expr + self.body = body + self.else_body = else_body + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_while_stmt(self) + + +class ForStmt(Statement): + # Index variables + index = None # type: Expression + # Expression to iterate + expr = None # type: Expression + body = None # type: Block + else_body = None # type: Block + + def __init__(self, index: Expression, expr: Expression, body: Block, + else_body: Block) -> None: + self.index = index + self.expr = expr + self.body = body + self.else_body = else_body + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_for_stmt(self) + + +class ReturnStmt(Statement): + expr = None # type: Optional[Expression] + + def __init__(self, expr: Optional[Expression]) -> None: + self.expr = expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_return_stmt(self) + + +class AssertStmt(Statement): + expr = None # type: Expression + + def __init__(self, expr: Expression) -> None: + self.expr = expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_assert_stmt(self) + + +class DelStmt(Statement): + expr = None # type: Expression + + def __init__(self, expr: Expression) -> None: + self.expr = expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_del_stmt(self) + + +class BreakStmt(Statement): + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_break_stmt(self) + + +class ContinueStmt(Statement): + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_continue_stmt(self) + + +class PassStmt(Statement): + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_pass_stmt(self) + + +class IfStmt(Statement): + expr = None # type: List[Expression] + body = None # type: List[Block] + else_body = None # type: Block + + def __init__(self, expr: List[Expression], body: List[Block], + else_body: Block) -> None: + self.expr = expr + self.body = body + self.else_body = else_body + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_if_stmt(self) + + +class RaiseStmt(Statement): + expr = None # type: Expression + from_expr = None # type: Expression + + def __init__(self, expr: Expression, from_expr: Expression = None) -> None: + self.expr = expr + self.from_expr = from_expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_raise_stmt(self) + + +class TryStmt(Statement): + body = None # type: Block # Try body + types = None # type: List[Expression] # Except type expressions + vars = None # type: List[NameExpr] # Except variable names + handlers = None # type: List[Block] # Except bodies + else_body = None # type: Block + finally_body = None # type: Block + + def __init__(self, body: Block, vars: List['NameExpr'], types: List[Expression], + handlers: List[Block], else_body: Block, + finally_body: Block) -> None: + self.body = body + self.vars = vars + self.types = types + self.handlers = handlers + self.else_body = else_body + self.finally_body = finally_body + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_try_stmt(self) + + +class WithStmt(Statement): + expr = None # type: List[Expression] + target = None # type: List[Expression] + body = None # type: Block + + def __init__(self, expr: List[Expression], target: List[Expression], + body: Block) -> None: + self.expr = expr + self.target = target + self.body = body + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_with_stmt(self) + + +class PrintStmt(Statement): + """Python 2 print statement""" + + args = None # type: List[Expression] + newline = False + # The file-like target object (given using >>). + target = None # type: Optional[Expression] + + def __init__(self, args: List[Expression], newline: bool, target: Expression = None) -> None: + self.args = args + self.newline = newline + self.target = target + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_print_stmt(self) + + +class ExecStmt(Statement): + """Python 2 exec statement""" + + expr = None # type: Expression + variables1 = None # type: Optional[Expression] + variables2 = None # type: Optional[Expression] + + def __init__(self, expr: Expression, + variables1: Optional[Expression], + variables2: Optional[Expression]) -> None: + self.expr = expr + self.variables1 = variables1 + self.variables2 = variables2 + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_exec_stmt(self) + + +# Expressions + + +class IntExpr(Expression): + """Integer literal""" + + value = 0 + literal = LITERAL_YES + + def __init__(self, value: int) -> None: + self.value = value + self.literal_hash = value + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_int_expr(self) + + +# How mypy uses StrExpr, BytesExpr, and UnicodeExpr: +# In Python 2 mode: +# b'x', 'x' -> StrExpr +# u'x' -> UnicodeExpr +# BytesExpr is unused +# +# In Python 3 mode: +# b'x' -> BytesExpr +# 'x', u'x' -> StrExpr +# UnicodeExpr is unused + +class StrExpr(Expression): + """String literal""" + + value = '' + literal = LITERAL_YES + + def __init__(self, value: str) -> None: + self.value = value + self.literal_hash = value + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_str_expr(self) + + +class BytesExpr(Expression): + """Bytes literal""" + + value = '' # TODO use bytes + literal = LITERAL_YES + + def __init__(self, value: str) -> None: + self.value = value + self.literal_hash = value + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_bytes_expr(self) + + +class UnicodeExpr(Expression): + """Unicode literal (Python 2.x)""" + + value = '' # TODO use bytes + literal = LITERAL_YES + + def __init__(self, value: str) -> None: + self.value = value + self.literal_hash = value + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_unicode_expr(self) + + +class FloatExpr(Expression): + """Float literal""" + + value = 0.0 + literal = LITERAL_YES + + def __init__(self, value: float) -> None: + self.value = value + self.literal_hash = value + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_float_expr(self) + + +class ComplexExpr(Expression): + """Complex literal""" + + value = 0.0j + literal = LITERAL_YES + + def __init__(self, value: complex) -> None: + self.value = value + self.literal_hash = value + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_complex_expr(self) + + +class EllipsisExpr(Expression): + """Ellipsis (...)""" + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_ellipsis(self) + + +class StarExpr(Expression): + """Star expression""" + + expr = None # type: Expression + + def __init__(self, expr: Expression) -> None: + self.expr = expr + self.literal = self.expr.literal + self.literal_hash = ('Star', expr.literal_hash,) + + # Whether this starred expression is used in a tuple/list and as lvalue + self.valid = False + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_star_expr(self) + + +class RefExpr(Expression): + """Abstract base class for name-like constructs""" + + kind = None # type: int # LDEF/GDEF/MDEF/... (None if not available) + node = None # type: SymbolNode # Var, FuncDef or TypeInfo that describes this + fullname = None # type: str # Fully qualified name (or name if not global) + + # Does this define a new name with inferred type? + # + # For members, after semantic analysis, this does not take base + # classes into consideration at all; the type checker deals with these. + is_def = False + + +class NameExpr(RefExpr): + """Name expression + + This refers to a local name, global name or a module. + """ + + name = None # type: str # Name referred to (may be qualified) + # TypeInfo of class surrounding expression (may be None) + info = None # type: TypeInfo + + literal = LITERAL_TYPE + + def __init__(self, name: str) -> None: + self.name = name + self.literal_hash = ('Var', name,) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_name_expr(self) + + def serialize(self) -> JsonDict: + # TODO: Find out where and why NameExpr is being serialized (if at all). + assert False, "Serializing NameExpr: %s" % (self,) + return {'.class': 'NameExpr', + 'kind': self.kind, + 'node': None if self.node is None else self.node.serialize(), + 'fullname': self.fullname, + 'is_def': self.is_def, + 'name': self.name, + 'literal': self.literal, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'NameExpr': + assert data['.class'] == 'NameExpr' + ret = NameExpr(data['name']) + ret.kind = data['kind'] + ret.node = None if data['node'] is None else SymbolNode.deserialize(data['node']) + ret.fullname = data['fullname'] + ret.is_def = data['is_def'] + ret.literal = data['literal'] + return ret + + +class MemberExpr(RefExpr): + """Member access expression x.y""" + + expr = None # type: Expression + name = None # type: str + # The variable node related to a definition. + def_var = None # type: Var + + def __init__(self, expr: Expression, name: str) -> None: + self.expr = expr + self.name = name + self.literal = self.expr.literal + self.literal_hash = ('Member', expr.literal_hash, name) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_member_expr(self) + + +# Kinds of arguments + +# Positional argument +ARG_POS = 0 # type: int +# Positional, optional argument (functions only, not calls) +ARG_OPT = 1 # type: int +# *arg argument +ARG_STAR = 2 # type: int +# Keyword argument x=y in call, or keyword-only function arg +ARG_NAMED = 3 # type: int +# **arg argument +ARG_STAR2 = 4 # type: int + + +class CallExpr(Expression): + """Call expression. + + This can also represent several special forms that are syntactically calls + such as cast(...) and None # type: .... + """ + + callee = None # type: Expression + args = None # type: List[Expression] + arg_kinds = None # type: List[int] # ARG_ constants + # Each name can be None if not a keyword argument. + arg_names = None # type: List[str] + # If not None, the node that represents the meaning of the CallExpr. For + # cast(...) this is a CastExpr. + analyzed = None # type: Optional[Expression] + + def __init__(self, callee: Expression, args: List[Expression], arg_kinds: List[int], + arg_names: List[str] = None, analyzed: Expression = None) -> None: + if not arg_names: + arg_names = [None] * len(args) + + self.callee = callee + self.args = args + self.arg_kinds = arg_kinds + self.arg_names = arg_names + self.analyzed = analyzed + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_call_expr(self) + + +class YieldFromExpr(Expression): + expr = None # type: Expression + + def __init__(self, expr: Expression) -> None: + self.expr = expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_yield_from_expr(self) + + +class YieldExpr(Expression): + expr = None # type: Optional[Expression] + + def __init__(self, expr: Optional[Expression]) -> None: + self.expr = expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_yield_expr(self) + + +class IndexExpr(Expression): + """Index expression x[y]. + + Also wraps type application such as List[int] as a special form. + """ + + base = None # type: Expression + index = None # type: Expression + # Inferred __getitem__ method type + method_type = None # type: mypy.types.Type + # If not None, this is actually semantically a type application + # Class[type, ...] or a type alias initializer. + analyzed = None # type: Union[TypeApplication, TypeAliasExpr] + + def __init__(self, base: Expression, index: Expression) -> None: + self.base = base + self.index = index + self.analyzed = None + if self.index.literal == LITERAL_YES: + self.literal = self.base.literal + self.literal_hash = ('Member', base.literal_hash, + index.literal_hash) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_index_expr(self) + + +class UnaryExpr(Expression): + """Unary operation""" + + op = '' + expr = None # type: Expression + # Inferred operator method type + method_type = None # type: mypy.types.Type + + def __init__(self, op: str, expr: Expression) -> None: + self.op = op + self.expr = expr + self.literal = self.expr.literal + self.literal_hash = ('Unary', op, expr.literal_hash) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_unary_expr(self) + + +# Map from binary operator id to related method name (in Python 3). +op_methods = { + '+': '__add__', + '-': '__sub__', + '*': '__mul__', + '/': '__truediv__', + '%': '__mod__', + '//': '__floordiv__', + '**': '__pow__', + '&': '__and__', + '|': '__or__', + '^': '__xor__', + '<<': '__lshift__', + '>>': '__rshift__', + '==': '__eq__', + '!=': '__ne__', + '<': '__lt__', + '>=': '__ge__', + '>': '__gt__', + '<=': '__le__', + 'in': '__contains__', +} # type: Dict[str, str] + +ops_with_inplace_method = { + '+', '-', '*', '/', '%', '//', '**', '&', '|', '^', '<<', '>>'} + +inplace_operator_methods = set( + '__i' + op_methods[op][2:] for op in ops_with_inplace_method) + +reverse_op_methods = { + '__add__': '__radd__', + '__sub__': '__rsub__', + '__mul__': '__rmul__', + '__truediv__': '__rtruediv__', + '__mod__': '__rmod__', + '__floordiv__': '__rfloordiv__', + '__pow__': '__rpow__', + '__and__': '__rand__', + '__or__': '__ror__', + '__xor__': '__rxor__', + '__lshift__': '__rlshift__', + '__rshift__': '__rrshift__', + '__eq__': '__eq__', + '__ne__': '__ne__', + '__lt__': '__gt__', + '__ge__': '__le__', + '__gt__': '__lt__', + '__le__': '__ge__', +} + +normal_from_reverse_op = dict((m, n) for n, m in reverse_op_methods.items()) +reverse_op_method_set = set(reverse_op_methods.values()) + + +class OpExpr(Expression): + """Binary operation (other than . or [] or comparison operators, + which have specific nodes).""" + + op = '' + left = None # type: Expression + right = None # type: Expression + # Inferred type for the operator method type (when relevant). + method_type = None # type: mypy.types.Type + + def __init__(self, op: str, left: Expression, right: Expression) -> None: + self.op = op + self.left = left + self.right = right + self.literal = min(self.left.literal, self.right.literal) + self.literal_hash = ('Binary', op, left.literal_hash, right.literal_hash) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_op_expr(self) + + +class ComparisonExpr(Expression): + """Comparison expression (e.g. a < b > c < d).""" + + operators = None # type: List[str] + operands = None # type: List[Expression] + # Inferred type for the operator methods (when relevant; None for 'is'). + method_types = None # type: List[mypy.types.Type] + + def __init__(self, operators: List[str], operands: List[Expression]) -> None: + self.operators = operators + self.operands = operands + self.method_types = [] + self.literal = min(o.literal for o in self.operands) + self.literal_hash = (('Comparison',) + tuple(operators) + + tuple(o.literal_hash for o in operands)) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_comparison_expr(self) + + +class SliceExpr(Expression): + """Slice expression (e.g. 'x:y', 'x:', '::2' or ':'). + + This is only valid as index in index expressions. + """ + + begin_index = None # type: Optional[Expression] + end_index = None # type: Optional[Expression] + stride = None # type: Optional[Expression] + + def __init__(self, begin_index: Optional[Expression], + end_index: Optional[Expression], + stride: Optional[Expression]) -> None: + self.begin_index = begin_index + self.end_index = end_index + self.stride = stride + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_slice_expr(self) + + +class CastExpr(Expression): + """Cast expression cast(type, expr).""" + + expr = None # type: Expression + type = None # type: mypy.types.Type + + def __init__(self, expr: Expression, typ: 'mypy.types.Type') -> None: + self.expr = expr + self.type = typ + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_cast_expr(self) + + +class RevealTypeExpr(Expression): + """Reveal type expression reveal_type(expr).""" + + expr = None # type: Expression + + def __init__(self, expr: Expression) -> None: + self.expr = expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_reveal_type_expr(self) + + +class SuperExpr(Expression): + """Expression super().name""" + + name = '' + info = None # type: TypeInfo # Type that contains this super expression + + def __init__(self, name: str) -> None: + self.name = name + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_super_expr(self) + + +class FuncExpr(FuncItem, Expression): + """Lambda expression""" + + def name(self) -> str: + return '' + + def expr(self) -> Expression: + """Return the expression (the body) of the lambda.""" + ret = cast(ReturnStmt, self.body.body[-1]) + return ret.expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_func_expr(self) + + +class ListExpr(Expression): + """List literal expression [...].""" + + items = None # type: List[Expression] + + def __init__(self, items: List[Expression]) -> None: + self.items = items + if all(x.literal == LITERAL_YES for x in items): + self.literal = LITERAL_YES + self.literal_hash = ('List',) + tuple(x.literal_hash for x in items) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_list_expr(self) + + +class DictExpr(Expression): + """Dictionary literal expression {key: value, ...}.""" + + items = None # type: List[Tuple[Expression, Expression]] + + def __init__(self, items: List[Tuple[Expression, Expression]]) -> None: + self.items = items + if all(x[0].literal == LITERAL_YES and x[1].literal == LITERAL_YES + for x in items): + self.literal = LITERAL_YES + self.literal_hash = ('Dict',) + tuple( + (x[0].literal_hash, x[1].literal_hash) for x in items) # type: ignore + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_dict_expr(self) + + +class TupleExpr(Expression): + """Tuple literal expression (..., ...)""" + + items = None # type: List[Expression] + + def __init__(self, items: List[Expression]) -> None: + self.items = items + if all(x.literal == LITERAL_YES for x in items): + self.literal = LITERAL_YES + self.literal_hash = ('Tuple',) + tuple(x.literal_hash for x in items) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_tuple_expr(self) + + +class SetExpr(Expression): + """Set literal expression {value, ...}.""" + + items = None # type: List[Expression] + + def __init__(self, items: List[Expression]) -> None: + self.items = items + if all(x.literal == LITERAL_YES for x in items): + self.literal = LITERAL_YES + self.literal_hash = ('Set',) + tuple(x.literal_hash for x in items) + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_set_expr(self) + + +class GeneratorExpr(Expression): + """Generator expression ... for ... in ... [ for ... in ... ] [ if ... ].""" + + left_expr = None # type: Expression + sequences_expr = None # type: List[Expression] + condlists = None # type: List[List[Expression]] + indices = None # type: List[Expression] + + def __init__(self, left_expr: Expression, indices: List[Expression], + sequences: List[Expression], condlists: List[List[Expression]]) -> None: + self.left_expr = left_expr + self.sequences = sequences + self.condlists = condlists + self.indices = indices + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_generator_expr(self) + + +class ListComprehension(Expression): + """List comprehension (e.g. [x + 1 for x in a])""" + + generator = None # type: GeneratorExpr + + def __init__(self, generator: GeneratorExpr) -> None: + self.generator = generator + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_list_comprehension(self) + + +class SetComprehension(Expression): + """Set comprehension (e.g. {x + 1 for x in a})""" + + generator = None # type: GeneratorExpr + + def __init__(self, generator: GeneratorExpr) -> None: + self.generator = generator + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_set_comprehension(self) + + +class DictionaryComprehension(Expression): + """Dictionary comprehension (e.g. {k: v for k, v in a}""" + + key = None # type: Expression + value = None # type: Expression + sequences_expr = None # type: List[Expression] + condlists = None # type: List[List[Expression]] + indices = None # type: List[Expression] + + def __init__(self, key: Expression, value: Expression, indices: List[Expression], + sequences: List[Expression], condlists: List[List[Expression]]) -> None: + self.key = key + self.value = value + self.sequences = sequences + self.condlists = condlists + self.indices = indices + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_dictionary_comprehension(self) + + +class ConditionalExpr(Expression): + """Conditional expression (e.g. x if y else z)""" + + cond = None # type: Expression + if_expr = None # type: Expression + else_expr = None # type: Expression + + def __init__(self, cond: Expression, if_expr: Expression, else_expr: Expression) -> None: + self.cond = cond + self.if_expr = if_expr + self.else_expr = else_expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_conditional_expr(self) + + +class BackquoteExpr(Expression): + """Python 2 expression `...`.""" + + expr = None # type: Expression + + def __init__(self, expr: Expression) -> None: + self.expr = expr + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_backquote_expr(self) + + +class TypeApplication(Expression): + """Type application expr[type, ...]""" + + expr = None # type: Expression + types = None # type: List[mypy.types.Type] + + def __init__(self, expr: Expression, types: List['mypy.types.Type']) -> None: + self.expr = expr + self.types = types + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_type_application(self) + + +# Variance of a type variable. For example, T in the definition of +# List[T] is invariant, so List[int] is not a subtype of List[object], +# and also List[object] is not a subtype of List[int]. +# +# The T in Iterable[T] is covariant, so Iterable[int] is a subtype of +# Iterable[object], but not vice versa. +# +# If T is contravariant in Foo[T], Foo[object] is a subtype of +# Foo[int], but not vice versa. +INVARIANT = 0 # type: int +COVARIANT = 1 # type: int +CONTRAVARIANT = 2 # type: int + + +class TypeVarExpr(SymbolNode, Expression): + """Type variable expression TypeVar(...).""" + + _name = '' + _fullname = '' + # Value restriction: only types in the list are valid as values. If the + # list is empty, there is no restriction. + values = None # type: List[mypy.types.Type] + # Upper bound: only subtypes of upper_bound are valid as values. By default + # this is 'object', meaning no restriction. + upper_bound = None # type: mypy.types.Type + # Variance of the type variable. Invariant is the default. + # TypeVar(..., covariant=True) defines a covariant type variable. + # TypeVar(..., contravariant=True) defines a contravariant type + # variable. + variance = INVARIANT + + def __init__(self, name: str, fullname: str, + values: List['mypy.types.Type'], + upper_bound: 'mypy.types.Type', + variance: int=INVARIANT) -> None: + self._name = name + self._fullname = fullname + self.values = values + self.upper_bound = upper_bound + self.variance = variance + + def name(self) -> str: + return self._name + + def fullname(self) -> str: + return self._fullname + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_type_var_expr(self) + + def serialize(self) -> JsonDict: + return {'.class': 'TypeVarExpr', + 'name': self._name, + 'fullname': self._fullname, + 'values': [t.serialize() for t in self.values], + 'upper_bound': self.upper_bound.serialize(), + 'variance': self.variance, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'TypeVarExpr': + assert data['.class'] == 'TypeVarExpr' + return TypeVarExpr(data['name'], + data['fullname'], + [mypy.types.Type.deserialize(v) for v in data['values']], + mypy.types.Type.deserialize(data['upper_bound']), + data['variance']) + + +class TypeAliasExpr(Expression): + """Type alias expression (rvalue).""" + + type = None # type: mypy.types.Type + + def __init__(self, type: 'mypy.types.Type') -> None: + self.type = type + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_type_alias_expr(self) + + +class NamedTupleExpr(Expression): + """Named tuple expression namedtuple(...).""" + + # The class representation of this named tuple (its tuple_type attribute contains + # the tuple item types) + info = None # type: TypeInfo + + def __init__(self, info: 'TypeInfo') -> None: + self.info = info + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_namedtuple_expr(self) + + +class PromoteExpr(Expression): + """Ducktype class decorator expression _promote(...).""" + + type = None # type: mypy.types.Type + + def __init__(self, type: 'mypy.types.Type') -> None: + self.type = type + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit__promote_expr(self) + + +# Constants + + +class TempNode(Expression): + """Temporary dummy node used during type checking. + + This node is not present in the original program; it is just an artifact + of the type checker implementation. It only represents an opaque node with + some fixed type. + """ + + type = None # type: mypy.types.Type + + def __init__(self, typ: 'mypy.types.Type') -> None: + self.type = typ + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_temp_node(self) + + +class TypeInfo(SymbolNode): + """The type structure of a single class. + + Each TypeInfo corresponds one-to-one to a ClassDef, which + represents the AST of the class. + + In type-theory terms, this is a "type constructor", and if the + class is generic then it will be a type constructor of higher kind. + Where the class is used in an actual type, it's in the form of an + Instance, which amounts to a type application of the tycon to + the appropriate number of arguments. + """ + + _fullname = None # type: str # Fully qualified name + defn = None # type: ClassDef # Corresponding ClassDef + # Method Resolution Order: the order of looking up attributes. The first + # value always to refers to this class. + mro = None # type: List[TypeInfo] + subtypes = None # type: Set[TypeInfo] # Direct subclasses encountered so far + names = None # type: SymbolTable # Names defined directly in this type + is_abstract = False # Does the class have any abstract attributes? + abstract_attributes = None # type: List[str] + # Classes inheriting from Enum shadow their true members with a __getattr__, so we + # have to treat them as a special case. + is_enum = False + # If true, any unknown attributes should have type 'Any' instead + # of generating a type error. This would be true if there is a + # base class with type 'Any', but other use cases may be + # possible. This is similar to having __getattr__ that returns Any + # (and __setattr__), but without the __getattr__ method. + fallback_to_any = False + + # Information related to type annotations. + + # Generic type variable names + type_vars = None # type: List[str] + + # Direct base classes. + bases = None # type: List[mypy.types.Instance] + + # Another type which this type will be treated as a subtype of, + # even though it's not a subclass in Python. The non-standard + # `@_promote` decorator introduces this, and there are also + # several builtin examples, in particular `int` -> `float`. + _promote = None # type: mypy.types.Type + + # Representation of a Tuple[...] base class, if the class has any + # (e.g., for named tuples). If this is not None, the actual Type + # object used for this class is not an Instance but a TupleType; + # the corresponding Instance is set as the fallback type of the + # tuple type. + tuple_type = None # type: mypy.types.TupleType + + # Is this a named tuple type? + is_named_tuple = False + + # Is this a dummy from deserialization? + is_dummy = False + + # Alternative to fullname() for 'anonymous' classes. + alt_fullname = None # type: Optional[str] + + def __init__(self, names: 'SymbolTable', defn: ClassDef) -> None: + """Initialize a TypeInfo.""" + self.names = names + self.defn = defn + self.subtypes = set() + self.type_vars = [] + self.bases = [] + # Leave self.mro uninitialized until we compute it for real, + # so we don't accidentally try to use it prematurely. + self._fullname = defn.fullname + self.is_abstract = False + self.abstract_attributes = [] + if defn.type_vars: + for vd in defn.type_vars: + self.type_vars.append(vd.name) + + def name(self) -> str: + """Short name.""" + return self.defn.name + + def fullname(self) -> str: + return self._fullname + + def is_generic(self) -> bool: + """Is the type generic (i.e. does it have type variables)?""" + return len(self.type_vars) > 0 + + def get(self, name: str) -> 'SymbolTableNode': + for cls in self.mro: + n = cls.names.get(name) + if n: + return n + return None + + def __getitem__(self, name: str) -> 'SymbolTableNode': + n = self.get(name) + if n: + return n + else: + raise KeyError(name) + + def __repr__(self) -> str: + return '' % self.fullname() + + # IDEA: Refactor the has* methods to be more consistent and document + # them. + + def has_readable_member(self, name: str) -> bool: + return self.get(name) is not None + + def has_writable_member(self, name: str) -> bool: + return self.has_var(name) + + def has_var(self, name: str) -> bool: + return self.get_var(name) is not None + + def has_method(self, name: str) -> bool: + return self.get_method(name) is not None + + def get_var(self, name: str) -> Var: + for cls in self.mro: + if name in cls.names: + node = cls.names[name].node + if isinstance(node, Var): + return node + else: + return None + return None + + def get_var_or_getter(self, name: str) -> SymbolNode: + # TODO getter + return self.get_var(name) + + def get_var_or_setter(self, name: str) -> SymbolNode: + # TODO setter + return self.get_var(name) + + def get_method(self, name: str) -> FuncBase: + if self.mro is None: # Might be because of a previous error. + return None + for cls in self.mro: + if name in cls.names: + node = cls.names[name].node + if isinstance(node, FuncBase): + return node + else: + return None + return None + + def calculate_mro(self) -> None: + """Calculate and set mro (method resolution order). + + Raise MroError if cannot determine mro. + """ + mro = linearize_hierarchy(self) + assert mro, "Could not produce a MRO at all for %s" % (self,) + self.mro = mro + self.is_enum = self._calculate_is_enum() + + def _calculate_is_enum(self) -> bool: + """ + If this is "enum.Enum" itself, then yes, it's an enum. + If the flag .is_enum has been set on anything in the MRO, it's an enum. + """ + if self.fullname() == ENUM_BASECLASS: + return True + if self.mro: + return any(type_info.is_enum for type_info in self.mro) + return False + + def has_base(self, fullname: str) -> bool: + """Return True if type has a base type with the specified name. + + This can be either via extension or via implementation. + """ + if self.mro: + for cls in self.mro: + if cls.fullname() == fullname: + return True + return False + + def all_subtypes(self) -> 'Set[TypeInfo]': + """Return TypeInfos of all subtypes, including this type, as a set.""" + subtypes = set([self]) + for subt in self.subtypes: + for t in subt.all_subtypes(): + subtypes.add(t) + return subtypes + + def all_base_classes(self) -> 'List[TypeInfo]': + """Return a list of base classes, including indirect bases.""" + assert False + + def direct_base_classes(self) -> 'List[TypeInfo]': + """Return a direct base classes. + + Omit base classes of other base classes. + """ + return [base.type for base in self.bases] + + def __str__(self) -> str: + """Return a string representation of the type. + + This includes the most important information about the type. + """ + base = None # type: str + if self.bases: + base = 'Bases({})'.format(', '.join(str(base) + for base in self.bases)) + return dump_tagged(['Name({})'.format(self.fullname()), + base, + ('Names', sorted(self.names.keys()))], + 'TypeInfo') + + def serialize(self) -> Union[str, JsonDict]: + # NOTE: This is where all ClassDefs originate, so there shouldn't be duplicates. + data = {'.class': 'TypeInfo', + 'fullname': self.fullname(), + 'alt_fullname': self.alt_fullname, + 'names': self.names.serialize(self.alt_fullname or self.fullname()), + 'defn': self.defn.serialize(), + 'is_abstract': self.is_abstract, + 'abstract_attributes': self.abstract_attributes, + 'is_enum': self.is_enum, + 'fallback_to_any': self.fallback_to_any, + 'type_vars': self.type_vars, + 'bases': [b.serialize() for b in self.bases], + '_promote': None if self._promote is None else self._promote.serialize(), + 'tuple_type': None if self.tuple_type is None else self.tuple_type.serialize(), + 'is_named_tuple': self.is_named_tuple, + } + return data + + @classmethod + def deserialize(cls, data: JsonDict) -> 'TypeInfo': + names = SymbolTable.deserialize(data['names']) + defn = ClassDef.deserialize(data['defn']) + ti = TypeInfo(names, defn) + ti._fullname = data['fullname'] + ti.alt_fullname = data['alt_fullname'] + # TODO: Is there a reason to reconstruct ti.subtypes? + ti.is_abstract = data['is_abstract'] + ti.abstract_attributes = data['abstract_attributes'] + ti.is_enum = data['is_enum'] + ti.fallback_to_any = data['fallback_to_any'] + ti.type_vars = data['type_vars'] + ti.bases = [mypy.types.Instance.deserialize(b) for b in data['bases']] + ti._promote = (None if data['_promote'] is None + else mypy.types.Type.deserialize(data['_promote'])) + ti.tuple_type = (None if data['tuple_type'] is None + else mypy.types.TupleType.deserialize(data['tuple_type'])) + ti.is_named_tuple = data['is_named_tuple'] + return ti + + +def NamedTupleTypeInfo(tup: 'mypy.types.NamedTupleType', *args) -> TypeInfo: + info = TypeInfo(*args) + info.tuple_type = tup + info.bases = [tup.fallback] + info.is_named_tuple = True + info.mro = [info] + tup.fallback.type.mro + return info + +class SymbolTableNode: + # Kind of node. Possible values: + # - LDEF: local definition (of any kind) + # - GDEF: global (module-level) definition + # - MDEF: class member definition + # - UNBOUND_TVAR: TypeVar(...) definition, not bound + # - TVAR: type variable in a bound scope (generic function / generic clas) + # - MODULE_REF: reference to a module + # - TYPE_ALIAS: type alias + # - UNBOUND_IMPORTED: temporary kind for imported names + kind = None # type: int + # AST node of definition (FuncDef/Var/TypeInfo/Decorator/TypeVarExpr, + # or None for a bound type variable). + node = None # type: Optional[SymbolNode] + # Type variable definition (for bound type variables only) + tvar_def = None # type: Optional[mypy.types.TypeVarDef] + # Module id (e.g. "foo.bar") or None + mod_id = '' + # If this not None, override the type of the 'node' attribute. + type_override = None # type: Optional[mypy.types.Type] + # If False, this name won't be imported via 'from import *'. + # This has no effect on names within classes. + module_public = True + # For deserialized MODULE_REF nodes, the referenced module name; + # for other nodes, optionally the name of the referenced object. + cross_ref = None # type: Optional[str] + + def __init__(self, kind: int, node: Optional[SymbolNode], mod_id: str = None, + typ: 'mypy.types.Type' = None, + tvar_def: 'mypy.types.TypeVarDef' = None, + module_public: bool = True) -> None: + self.kind = kind + self.node = node + self.type_override = typ + self.mod_id = mod_id + self.tvar_def = tvar_def + self.module_public = module_public + + @property + def fullname(self) -> str: + if self.node is not None: + return self.node.fullname() + else: + return None + + @property + def type(self) -> 'mypy.types.Type': + # IDEA: Get rid of the Any type. + node = self.node # type: Any + if self.type_override is not None: + return self.type_override + elif ((isinstance(node, Var) or isinstance(node, FuncDef)) + and node.type is not None): + return node.type + elif isinstance(node, Decorator): + return node.var.type + else: + return None + + def __str__(self) -> str: + s = '{}/{}'.format(node_kinds[self.kind], short_type(self.node)) + if self.mod_id is not None: + s += ' ({})'.format(self.mod_id) + # Include declared type of variables and functions. + if self.type is not None: + s += ' : {}'.format(self.type) + return s + + def serialize(self, prefix: str, name: str) -> JsonDict: + """Serialize a SymbolTableNode. + + Args: + prefix: full name of the containing module or class; or None + name: name of this object relative to the containing object + """ + data = {'.class': 'SymbolTableNode', + 'kind': node_kinds[self.kind], + } # type: JsonDict + if self.tvar_def: + data['tvar_def'] = self.tvar_def.serialize() + if not self.module_public: + data['module_public'] = False + if self.kind == MODULE_REF: + assert self.node is not None, "Missing module cross ref in %s for %s" % (prefix, name) + data['cross_ref'] = self.node.fullname() + else: + if self.node is not None: + if prefix is not None: + # Check whether this is an alias for another object. + # If the object's canonical full name differs from + # the full name computed from prefix and name, + # it's an alias, and we serialize it as a cross ref. + if isinstance(self.node, TypeInfo): + fullname = self.node.alt_fullname or self.node.fullname() + else: + fullname = self.node.fullname() + if (fullname is not None and '.' in fullname and + fullname != prefix + '.' + name): + data['cross_ref'] = fullname + return data + data['node'] = self.node.serialize() + if self.type_override is not None: + data['type_override'] = self.type_override.serialize() + return data + + @classmethod + def deserialize(cls, data: JsonDict) -> 'SymbolTableNode': + assert data['.class'] == 'SymbolTableNode' + kind = inverse_node_kinds[data['kind']] + if 'cross_ref' in data: + # This will be fixed up later. + stnode = SymbolTableNode(kind, None) + stnode.cross_ref = data['cross_ref'] + else: + node = None + if 'node' in data: + node = SymbolNode.deserialize(data['node']) + typ = None + if 'type_override' in data: + typ = mypy.types.Type.deserialize(data['type_override']) + stnode = SymbolTableNode(kind, node, typ=typ) + if 'tvar_def' in data: + stnode.tvar_def = mypy.types.TypeVarDef.deserialize(data['tvar_def']) + if 'module_public' in data: + stnode.module_public = data['module_public'] + return stnode + + +class SymbolTable(Dict[str, SymbolTableNode]): + def __str__(self) -> str: + a = [] # type: List[str] + for key, value in self.items(): + # Filter out the implicit import of builtins. + if isinstance(value, SymbolTableNode): + if (value.fullname != 'builtins' and + (value.fullname or '').split('.')[-1] not in + implicit_module_attrs): + a.append(' ' + str(key) + ' : ' + str(value)) + else: + a.append(' ') + a = sorted(a) + a.insert(0, 'SymbolTable(') + a[-1] += ')' + return '\n'.join(a) + + def serialize(self, fullname: str) -> JsonDict: + data = {'.class': 'SymbolTable'} # type: JsonDict + for key, value in self.items(): + # Skip __builtins__: it's a reference to the builtins + # module that gets added to every module by + # SemanticAnalyzer.visit_file(), but it shouldn't be + # accessed by users of the module. + if key == '__builtins__': + continue + data[key] = value.serialize(fullname, key) + return data + + @classmethod + def deserialize(cls, data: JsonDict) -> 'SymbolTable': + assert data['.class'] == 'SymbolTable' + st = SymbolTable() + for key, value in data.items(): + if key != '.class': + st[key] = SymbolTableNode.deserialize(value) + return st + + +def function_type(func: FuncBase, fallback: 'mypy.types.Instance') -> 'mypy.types.FunctionLike': + if func.type: + assert isinstance(func.type, mypy.types.FunctionLike) + return func.type + else: + # Implicit type signature with dynamic types. + # Overloaded functions always have a signature, so func must be an ordinary function. + fdef = cast(FuncDef, func) + name = func.name() + if name: + name = '"{}"'.format(name) + names = [] # type: List[str] + for arg in fdef.arguments: + names.append(arg.variable.name()) + + return mypy.types.CallableType( + [mypy.types.AnyType()] * len(fdef.arguments), + [arg.kind for arg in fdef.arguments], + names, + mypy.types.AnyType(), + fallback, + name, + implicit=True, + ) + + +def method_type_with_fallback(func: FuncBase, + fallback: 'mypy.types.Instance') -> 'mypy.types.FunctionLike': + """Return the signature of a method (omit self).""" + return method_type(function_type(func, fallback)) + + +def method_type(sig: 'mypy.types.FunctionLike') -> 'mypy.types.FunctionLike': + if isinstance(sig, mypy.types.CallableType): + return method_callable(sig) + else: + sig = cast(mypy.types.Overloaded, sig) + items = [] # type: List[mypy.types.CallableType] + for c in sig.items(): + items.append(method_callable(c)) + return mypy.types.Overloaded(items) + + +def method_callable(c: 'mypy.types.CallableType') -> 'mypy.types.CallableType': + if c.arg_kinds and c.arg_kinds[0] == ARG_STAR: + # The signature is of the form 'def foo(*args, ...)'. + # In this case we shouldn't drop the first arg, + # since self will be absorbed by the *args. + return c + return c.copy_modified(arg_types=c.arg_types[1:], + arg_kinds=c.arg_kinds[1:], + arg_names=c.arg_names[1:]) + + +class MroError(Exception): + """Raised if a consistent mro cannot be determined for a class.""" + + +def linearize_hierarchy(info: TypeInfo) -> List[TypeInfo]: + # TODO describe + if info.mro: + return info.mro + bases = info.direct_base_classes() + lin_bases = [] + for base in bases: + assert base is not None, "Cannot linearize bases for %s %s" % (info.fullname(), bases) + lin_bases.append(linearize_hierarchy(base)) + lin_bases.append(bases) + return [info] + merge(lin_bases) + + +def merge(seqs: List[List[TypeInfo]]) -> List[TypeInfo]: + seqs = [s[:] for s in seqs] + result = [] # type: List[TypeInfo] + while True: + seqs = [s for s in seqs if s] + if not seqs: + return result + for seq in seqs: + head = seq[0] + if not [s for s in seqs if head in s[1:]]: + break + else: + raise MroError() + result.append(head) + for s in seqs: + if s[0] is head: + del s[0] diff --git a/mypy/parse.py b/mypy/parse.py index 579f03c57525f..a496f32bebcd3 100644 --- a/mypy/parse.py +++ b/mypy/parse.py @@ -1,2034 +1,2034 @@ -"""Mypy parser. - -Constructs a parse tree (abstract syntax tree) based on a string -representing a source file. Performs only minimal semantic checks. -""" - -import re - -from typing import List, Tuple, Any, Set, cast, Union, Optional - -from mypy import lex, docstring -from mypy.lex import ( - Token, Eof, Bom, Break, Name, Colon, Dedent, IntLit, StrLit, BytesLit, - UnicodeLit, FloatLit, Op, Indent, Keyword, Punct, LexError, ComplexLit, - EllipsisToken -) -import mypy.types -from mypy.nodes import ( - MypyFile, Import, Node, ImportAll, ImportFrom, FuncDef, OverloadedFuncDef, - ClassDef, Decorator, Block, Var, OperatorAssignmentStmt, - ExpressionStmt, AssignmentStmt, ReturnStmt, RaiseStmt, AssertStmt, - DelStmt, BreakStmt, ContinueStmt, PassStmt, GlobalDecl, - WhileStmt, ForStmt, IfStmt, TryStmt, WithStmt, - TupleExpr, GeneratorExpr, ListComprehension, ListExpr, ConditionalExpr, - DictExpr, SetExpr, NameExpr, IntExpr, StrExpr, BytesExpr, UnicodeExpr, - FloatExpr, CallExpr, SuperExpr, MemberExpr, IndexExpr, SliceExpr, OpExpr, - UnaryExpr, FuncExpr, PrintStmt, ImportBase, ComparisonExpr, - StarExpr, YieldFromExpr, NonlocalDecl, DictionaryComprehension, - SetComprehension, ComplexExpr, EllipsisExpr, YieldExpr, ExecStmt, Argument, - BackquoteExpr -) -from mypy import defaults -from mypy import nodes -from mypy.errors import Errors, CompileError -from mypy.types import Type, CallableType, AnyType, UnboundType -from mypy.parsetype import ( - parse_type, parse_types, parse_signature, TypeParseError, parse_str_as_signature -) -from mypy.options import Options - -from mypy import experiments - - -precedence = { - '**': 16, - '-u': 15, '+u': 15, '~': 15, # unary operators (-, + and ~) - '': 14, - '*': 13, '/': 13, '//': 13, '%': 13, - '+': 12, '-': 12, - '>>': 11, '<<': 11, - '&': 10, - '^': 9, - '|': 8, - '==': 7, '!=': 7, '<': 7, '>': 7, '<=': 7, '>=': 7, 'is': 7, 'in': 7, - '*u': 7, # unary * for star expressions - 'not': 6, - 'and': 5, - 'or': 4, - '': 3, # conditional expression - '': 2, # list comprehension - ',': 1} - - -op_assign = set([ - '+=', '-=', '*=', '/=', '//=', '%=', '**=', '|=', '&=', '^=', '>>=', - '<<=']) - -op_comp = set([ - '>', '<', '==', '>=', '<=', '<>', '!=', 'is', 'is', 'in', 'not']) - -none = Token('') # Empty token - - -def parse(source: Union[str, bytes], - fnam: str, - errors: Errors, - options: Options) -> MypyFile: - """Parse a source file, without doing any semantic analysis. - - Return the parse tree. If errors is not provided, raise ParseError - on failure. Otherwise, use the errors object to report parse errors. - - The python_version (major, minor) option determines the Python syntax variant. - """ - if options.fast_parser: - import mypy.fastparse - return mypy.fastparse.parse(source, - fnam=fnam, - errors=errors, - pyversion=options.python_version, - custom_typing_module=options.custom_typing_module) - - is_stub_file = bool(fnam) and fnam.endswith('.pyi') - parser = Parser(fnam, - errors, - options.python_version, - options.custom_typing_module, - is_stub_file=is_stub_file) - tree = parser.parse(source) - tree.path = fnam - tree.is_stub = is_stub_file - return tree - - -class Parser: - """Mypy parser that parses a string into an AST. - - Parses type annotations in addition to basic Python syntax. It supports both Python 2 and 3 - (though Python 2 support is incomplete). - - The AST classes are defined in mypy.nodes and mypy.types. - """ - - tok = None # type: List[Token] - ind = 0 - errors = None # type: Errors - # If True, raise an exception on any parse error. Otherwise, errors are reported via 'errors'. - raise_on_error = False - - # Are we currently parsing the body of a class definition? - is_class_body = False - # All import nodes encountered so far in this parse unit. - imports = None # type: List[ImportBase] - # Names imported from __future__. - future_options = None # type: List[str] - # Lines to ignore (using # type: ignore). - ignored_lines = None # type: Set[int] - - def __init__(self, fnam: str, errors: Errors, pyversion: Tuple[int, int], - custom_typing_module: str = None, is_stub_file: bool = False) -> None: - self.raise_on_error = errors is None - self.pyversion = pyversion - self.custom_typing_module = custom_typing_module - self.is_stub_file = is_stub_file - if errors is not None: - self.errors = errors - else: - self.errors = Errors() - if fnam is not None: - self.errors.set_file(fnam) - else: - self.errors.set_file('') - - def parse(self, s: Union[str, bytes]) -> MypyFile: - self.tok, self.ignored_lines = lex.lex(s, pyversion=self.pyversion, - is_stub_file=self.is_stub_file) - self.ind = 0 - self.imports = [] - self.future_options = [] - file = self.parse_file() - if self.raise_on_error and self.errors.is_errors(): - self.errors.raise_error() - return file - - def weak_opts(self) -> Set[str]: - """Do weak typing if any of the first ten tokens is a comment saying so. - - The comment can be one of: - # mypy: weak=global - # mypy: weak=local - # mypy: weak <- defaults to local - """ - regexp = re.compile(r'^[\s]*# *mypy: *weak(=?)([^\s]*)', re.M) - for t in self.tok[:10]: - for s in [t.string, t.pre]: - m = regexp.search(s) - if m: - opts = set(x for x in m.group(2).split(',') if x) - if not opts: - opts.add('local') - return opts - return set() - - def parse_file(self) -> MypyFile: - """Parse a mypy source file.""" - is_bom = self.parse_bom() - defs = self.parse_defs() - weak_opts = self.weak_opts() - self.expect_type(Eof) - node = MypyFile(defs, self.imports, is_bom, self.ignored_lines, - weak_opts=weak_opts) - return node - - # Parse the initial part - - def parse_bom(self) -> bool: - """Parse the optional byte order mark at the beginning of a file.""" - if isinstance(self.current(), Bom): - self.expect_type(Bom) - if isinstance(self.current(), Break): - self.expect_break() - return True - else: - return False - - def parse_import(self) -> Import: - self.expect('import') - ids = [] - while True: - id = self.parse_qualified_name() - translated = self.translate_module_id(id) - as_id = None - if self.current_str() == 'as': - self.expect('as') - name_tok = self.expect_type(Name) - as_id = name_tok.string - elif translated != id: - as_id = id - ids.append((translated, as_id)) - if self.current_str() != ',': - break - self.expect(',') - node = Import(ids) - self.imports.append(node) - return node - - def translate_module_id(self, id: str) -> str: - """Return the actual, internal module id for a source text id. - - For example, translate '__builtin__' in Python 2 to 'builtins'. - """ - if id == self.custom_typing_module: - return 'typing' - elif id == '__builtin__' and self.pyversion[0] == 2: - # HACK: __builtin__ in Python 2 is aliases to builtins. However, the implementation - # is named __builtin__.py (there is another layer of translation elsewhere). - return 'builtins' - return id - - def parse_import_from(self) -> Node: - self.expect('from') - - # Build the list of beginning relative tokens. - relative = 0 - while self.current_str() in (".", "..."): - relative += len(self.current_str()) - self.skip() - - # Parse qualified name to actually import from. - if self.current_str() == "import": - # Empty/default values. - name = "" - else: - name = self.parse_qualified_name() - - name = self.translate_module_id(name) - - # Parse import list - self.expect('import') - node = None # type: ImportBase - if self.current_str() == '*': - if name == '__future__': - self.parse_error() - # An import all from a module node: - self.skip() - node = ImportAll(name, relative) - else: - is_paren = self.current_str() == '(' - if is_paren: - self.expect('(') - targets = [] # type: List[Tuple[str, str]] - while True: - id, as_id = self.parse_import_name() - if '%s.%s' % (name, id) == self.custom_typing_module: - if targets or self.current_str() == ',': - self.fail('You cannot import any other modules when you ' - 'import a custom typing module', - self.current().line) - node = Import([('typing', as_id)]) - self.skip_until_break() - break - targets.append((id, as_id)) - if self.current_str() != ',': - break - self.expect(',') - if is_paren and self.current_str() == ')': - break - if is_paren: - self.expect(')') - if node is None: - node = ImportFrom(name, relative, targets) - self.imports.append(node) - if name == '__future__': - self.future_options.extend(target[0] for target in targets) - return node - - def parse_import_name(self) -> Tuple[str, Optional[str]]: - tok = self.expect_type(Name) - name = tok.string - if self.current_str() == 'as': - self.skip() - as_name = self.expect_type(Name) - return name, as_name.string - else: - return name, None - - def parse_qualified_name(self) -> str: - """Parse a name with an optional module qualifier. - - Return a tuple with the name as a string and a token array - containing all the components of the name. - """ - tok = self.expect_type(Name) - n = tok.string - while self.current_str() == '.': - self.expect('.') - tok = self.expect_type(Name) - n += '.' + tok.string - return n - - # Parsing global definitions - - def parse_defs(self) -> List[Node]: - defs = [] # type: List[Node] - while not self.eof(): - try: - defn, is_simple = self.parse_statement() - if is_simple: - self.expect_break() - if defn is not None: - if not self.try_combine_overloads(defn, defs): - defs.append(defn) - except ParseError: - pass - return defs - - def parse_class_def(self) -> ClassDef: - old_is_class_body = self.is_class_body - self.is_class_body = True - - self.expect('class') - metaclass = None - - try: - base_types = [] # type: List[Node] - try: - name_tok = self.expect_type(Name) - name = name_tok.string - - self.errors.push_type(name) - - if self.current_str() == '(': - self.skip() - while True: - if self.current_str() == ')': - break - if self.peek().string == '=': - metaclass = self.parse_class_keywords() - break - base_types.append(self.parse_super_type()) - if self.current_str() != ',': - break - self.skip() - self.expect(')') - except ParseError: - pass - - defs, _ = self.parse_block() - - node = ClassDef(name, defs, None, base_types, metaclass=metaclass) - return node - finally: - self.errors.pop_type() - self.is_class_body = old_is_class_body - - def parse_class_keywords(self) -> Optional[str]: - """Parse the class keyword arguments, keeping the metaclass but - ignoring all others. Returns None if the metaclass isn't found. - """ - metaclass = None - while True: - key = self.expect_type(Name) - self.expect('=') - if key.string == 'metaclass': - metaclass = self.parse_qualified_name() - else: - # skip the class value - self.parse_expression(precedence[',']) - if self.current_str() != ',': - break - self.skip() - if self.current_str() == ')': - break - return metaclass - - def parse_super_type(self) -> Node: - return self.parse_expression(precedence[',']) - - def parse_decorated_function_or_class(self) -> Node: - decorators = [] - no_type_checks = False - while self.current_str() == '@': - self.expect('@') - d_exp = self.parse_expression() - if self.is_no_type_check_decorator(d_exp): - no_type_checks = True - decorators.append(d_exp) - self.expect_break() - if self.current_str() != 'class': - func = self.parse_function(no_type_checks) - func.is_decorated = True - var = Var(func.name()) - # Types of decorated functions must always be inferred. - var.is_ready = False - var.set_line(decorators[0].line) - node = Decorator(func, decorators, var) - return node - else: - cls = self.parse_class_def() - cls.decorators = decorators - return cls - - def is_no_type_check_decorator(self, expr: Node) -> bool: - if isinstance(expr, NameExpr): - return expr.name == 'no_type_check' - elif isinstance(expr, MemberExpr): - if isinstance(expr.expr, NameExpr): - return expr.expr.name == 'typing' and expr.name == 'no_type_check' - else: - return False - - def parse_function(self, no_type_checks: bool=False) -> FuncDef: - def_tok = self.expect('def') - is_method = self.is_class_body - self.is_class_body = False - try: - (name, args, typ, is_error, extra_stmts) = self.parse_function_header(no_type_checks) - - arg_kinds = [arg.kind for arg in args] - arg_names = [arg.variable.name() for arg in args] - - body, comment_type = self.parse_block(allow_type=True) - # Potentially insert extra assignment statements to the beginning of the - # body, used to decompose Python 2 tuple arguments. - body.body[:0] = extra_stmts - if comment_type: - # The function has a # type: ... signature. - if typ: - self.errors.report( - def_tok.line, 'Function has duplicate type signatures') - sig = cast(CallableType, comment_type) - if sig.is_ellipsis_args: - # When we encounter an ellipsis, fill in the arg_types with - # a bunch of AnyTypes, emulating Callable[..., T] - arg_types = [AnyType()] * len(arg_kinds) # type: List[Type] - typ = CallableType( - arg_types, - arg_kinds, - arg_names, - sig.ret_type, - None, - line=def_tok.line) - elif is_method and len(sig.arg_kinds) < len(arg_kinds): - self.check_argument_kinds(arg_kinds, - [nodes.ARG_POS] + sig.arg_kinds, - def_tok.line) - # Add implicit 'self' argument to signature. - first_arg = [AnyType()] # type: List[Type] - typ = CallableType( - first_arg + sig.arg_types, - arg_kinds, - arg_names, - sig.ret_type, - None, - line=def_tok.line) - else: - self.check_argument_kinds(arg_kinds, sig.arg_kinds, - def_tok.line) - typ = CallableType( - sig.arg_types, - arg_kinds, - arg_names, - sig.ret_type, - None, - line=def_tok.line) - - # If there was a serious error, we really cannot build a parse tree - # node. - if is_error: - return None - - if typ and isinstance(typ.ret_type, UnboundType): - typ.ret_type.is_ret_type = True - - node = FuncDef(name, args, body, typ) - node.set_line(def_tok) - if typ is not None: - typ.definition = node - return node - finally: - self.errors.pop_function() - self.is_class_body = is_method - - def check_argument_kinds(self, funckinds: List[int], sigkinds: List[int], - line: int) -> None: - """Check that arguments are consistent. - - This verifies that they have the same number and the kinds correspond. - - Arguments: - funckinds: kinds of arguments in function definition - sigkinds: kinds of arguments in signature (after # type:) - """ - if len(funckinds) != len(sigkinds): - if len(funckinds) > len(sigkinds): - self.fail("Type signature has too few arguments", line) - else: - self.fail("Type signature has too many arguments", line) - return - for kind, token in [(nodes.ARG_STAR, '*'), - (nodes.ARG_STAR2, '**')]: - if ((funckinds.count(kind) != sigkinds.count(kind)) or - (kind in funckinds and sigkinds.index(kind) != funckinds.index(kind))): - self.fail( - "Inconsistent use of '{}' in function " - "signature".format(token), line) - - def parse_function_header( - self, no_type_checks: bool=False) -> Tuple[str, - List[Argument], - CallableType, - bool, - List[AssignmentStmt]]: - """Parse function header (a name followed by arguments) - - Return a 5-tuple with the following items: - name - arguments - signature (annotation) - error flag (True if error) - extra statements needed to decompose arguments (usually empty) - - See parse_arg_list for an explanation of the final tuple item. - """ - name = '' - - try: - name_tok = self.expect_type(Name) - name = name_tok.string - - self.errors.push_function(name) - - args, typ, extra_stmts = self.parse_args(no_type_checks) - except ParseError: - if not isinstance(self.current(), Break): - self.ind -= 1 # Kludge: go back to the Break token - # Resynchronise parsing by going back over :, if present. - if isinstance(self.tok[self.ind - 1], Colon): - self.ind -= 1 - return (name, [], None, True, []) - - return (name, args, typ, False, extra_stmts) - - def parse_args(self, no_type_checks: bool=False) -> Tuple[List[Argument], - CallableType, - List[AssignmentStmt]]: - """Parse a function signature (...) [-> t]. - - See parse_arg_list for an explanation of the final tuple item. - """ - lparen = self.expect('(') - - # Parse the argument list (everything within '(' and ')'). - args, extra_stmts = self.parse_arg_list(no_type_checks=no_type_checks) - - self.expect(')') - - if self.current_str() == '->': - self.skip() - if no_type_checks: - self.parse_expression() - ret_type = None - else: - ret_type = self.parse_type() - else: - ret_type = None - - arg_kinds = [arg.kind for arg in args] - self.verify_argument_kinds(arg_kinds, lparen.line) - - annotation = self.build_func_annotation( - ret_type, args, lparen.line) - - return args, annotation, extra_stmts - - def build_func_annotation(self, ret_type: Type, args: List[Argument], - line: int, is_default_ret: bool = False) -> CallableType: - arg_types = [arg.type_annotation for arg in args] - # Are there any type annotations? - if ((ret_type and not is_default_ret) - or arg_types != [None] * len(arg_types)): - # Yes. Construct a type for the function signature. - return self.construct_function_type(args, ret_type, line) - else: - return None - - def parse_arg_list(self, allow_signature: bool = True, - no_type_checks: bool=False) -> Tuple[List[Argument], - List[AssignmentStmt]]: - """Parse function definition argument list. - - This includes everything between '(' and ')' (but not the - parentheses). - - Return tuple (arguments, - extra statements for decomposing arguments). - - The final argument is only used for Python 2 argument lists with - tuples; they contain destructuring assignment statements used to - decompose tuple arguments. For example, consider a header like this: - - . def f((x, y)) - - The actual (sole) argument will be __tuple_arg_1 (a generated - name), whereas the extra statement list will contain a single - assignment statement corresponding to this assignment: - - x, y = __tuple_arg_1 - """ - args = [] # type: List[Argument] - extra_stmts = [] - # This is for checking duplicate argument names. - arg_names = [] # type: List[str] - has_tuple_arg = False - - require_named = False - bare_asterisk_before = -1 - - if self.current_str() != ')' and self.current_str() != ':': - while self.current_str() != ')': - if self.current_str() == '*' and self.peek().string == ',': - self.expect('*') - require_named = True - bare_asterisk_before = len(args) - elif self.current_str() in ['*', '**']: - if bare_asterisk_before == len(args): - # named arguments must follow bare * - self.parse_error() - - arg = self.parse_asterisk_arg( - allow_signature, - no_type_checks, - ) - args.append(arg) - require_named = True - elif self.current_str() == '(': - arg, extra_stmt, names = self.parse_tuple_arg(len(args)) - args.append(arg) - if extra_stmt is not None: - extra_stmts.append(extra_stmt) - has_tuple_arg = True - arg_names.extend(names) - else: - arg, require_named = self.parse_normal_arg( - require_named, - allow_signature, - no_type_checks, - ) - args.append(arg) - arg_names.append(arg.variable.name()) - - if self.current().string != ',': - break - - self.expect(',') - - # Non-tuple argument dupes will be checked elsewhere. Avoid - # generating duplicate errors. - if has_tuple_arg: - self.check_duplicate_argument_names(arg_names) - - return args, extra_stmts - - def check_duplicate_argument_names(self, names: List[str]) -> None: - found = set() # type: Set[str] - for name in names: - if name in found: - self.fail('Duplicate argument name "{}"'.format(name), - self.current().line) - found.add(name) - - def parse_asterisk_arg(self, - allow_signature: bool, - no_type_checks: bool) -> Argument: - asterisk = self.skip() - name = self.expect_type(Name) - variable = Var(name.string) - if asterisk.string == '*': - kind = nodes.ARG_STAR - else: - kind = nodes.ARG_STAR2 - - type = None - if no_type_checks: - self.parse_parameter_annotation() - else: - type = self.parse_arg_type(allow_signature) - - return Argument(variable, type, None, kind) - - def parse_tuple_arg(self, index: int) -> Tuple[Argument, AssignmentStmt, List[str]]: - """Parse a single Python 2 tuple argument. - - Example: def f(x, (y, z)): ... - - The tuple arguments gets transformed into an assignment in the - function body (the second return value). - - Return tuple (argument, decomposing assignment, list of names defined). - - Special case: if the argument is just (x) then it's not a tuple; - we indicate this by returning (argument, None, ['x']). - However, if the argument is (x,) then it *is* a (singleton) tuple. - """ - line = self.current().line - # Generate a new argument name that is very unlikely to clash with anything. - arg_name = '__tuple_arg_{}'.format(index + 1) - if self.pyversion[0] >= 3: - self.fail('Tuples in argument lists only supported in Python 2 mode', line) - paren_arg = self.parse_parentheses() - self.verify_tuple_arg(paren_arg) - if isinstance(paren_arg, NameExpr): - # This isn't a tuple. Revert to a normal argument. - arg_name = paren_arg.name - decompose = None - else: - rvalue = NameExpr(arg_name) - rvalue.set_line(line) - decompose = AssignmentStmt([paren_arg], rvalue) - decompose.set_line(line) - kind = nodes.ARG_POS - initializer = None - if self.current_str() == '=': - self.expect('=') - initializer = self.parse_expression(precedence[',']) - kind = nodes.ARG_OPT - var = Var(arg_name) - arg_names = self.find_tuple_arg_argument_names(paren_arg) - return Argument(var, None, initializer, kind), decompose, arg_names - - def verify_tuple_arg(self, paren_arg: Node) -> None: - if isinstance(paren_arg, TupleExpr): - if not paren_arg.items: - self.fail('Empty tuple not valid as an argument', paren_arg.line) - for item in paren_arg.items: - self.verify_tuple_arg(item) - elif not isinstance(paren_arg, NameExpr): - self.fail('Invalid item in tuple argument', paren_arg.line) - - def find_tuple_arg_argument_names(self, node: Node) -> List[str]: - result = [] # type: List[str] - if isinstance(node, TupleExpr): - for item in node.items: - result.extend(self.find_tuple_arg_argument_names(item)) - elif isinstance(node, NameExpr): - result.append(node.name) - return result - - def parse_normal_arg(self, require_named: bool, - allow_signature: bool, - no_type_checks: bool) -> Tuple[Argument, bool]: - name = self.expect_type(Name) - variable = Var(name.string) - - type = None - if no_type_checks: - self.parse_parameter_annotation() - else: - type = self.parse_arg_type(allow_signature) - - initializer = None # type: Node - if self.current_str() == '=': - self.expect('=') - initializer = self.parse_expression(precedence[',']) - if require_named: - kind = nodes.ARG_NAMED - else: - kind = nodes.ARG_OPT - else: - if require_named: - kind = nodes.ARG_NAMED - else: - kind = nodes.ARG_POS - - self.set_type_optional(type, initializer) - - return Argument(variable, type, initializer, kind), require_named - - def set_type_optional(self, type: Type, initializer: Node) -> None: - if not experiments.STRICT_OPTIONAL: - return - # Indicate that type should be wrapped in an Optional if arg is initialized to None. - optional = isinstance(initializer, NameExpr) and initializer.name == 'None' - if isinstance(type, UnboundType): - type.optional = optional - - def parse_parameter_annotation(self) -> Node: - if self.current_str() == ':': - self.skip() - return self.parse_expression(precedence[',']) - - def parse_arg_type(self, allow_signature: bool) -> Type: - if self.current_str() == ':' and allow_signature: - self.skip() - return self.parse_type() - else: - return None - - def verify_argument_kinds(self, kinds: List[int], line: int) -> None: - found = set() # type: Set[int] - for i, kind in enumerate(kinds): - if kind == nodes.ARG_POS and found & set([nodes.ARG_OPT, - nodes.ARG_STAR, - nodes.ARG_STAR2]): - self.fail('Invalid argument list', line) - elif kind == nodes.ARG_STAR and nodes.ARG_STAR in found: - self.fail('Invalid argument list', line) - elif kind == nodes.ARG_STAR2 and i != len(kinds) - 1: - self.fail('Invalid argument list', line) - found.add(kind) - - def construct_function_type(self, args: List[Argument], ret_type: Type, - line: int) -> CallableType: - # Complete the type annotation by replacing omitted types with 'Any'. - arg_types = [arg.type_annotation for arg in args] - for i in range(len(arg_types)): - if arg_types[i] is None: - arg_types[i] = AnyType(implicit=True) - if ret_type is None: - ret_type = AnyType(implicit=True) - arg_kinds = [arg.kind for arg in args] - arg_names = [arg.variable.name() for arg in args] - return CallableType(arg_types, arg_kinds, arg_names, ret_type, None, name=None, - variables=None, line=line) - - # Parsing statements - - def parse_block(self, allow_type: bool = False) -> Tuple[Block, Type]: - colon = self.expect(':') - if not isinstance(self.current(), Break): - # Block immediately after ':'. - nodes = [] - while True: - ind = self.ind - stmt, is_simple = self.parse_statement() - if not is_simple: - self.parse_error_at(self.tok[ind]) - break - nodes.append(stmt) - brk = self.expect_break() - if brk.string != ';': - break - node = Block(nodes) - node.set_line(colon) - return node, None - else: - # Indented block. - brk = self.expect_break() - type = self.parse_type_comment(brk, signature=True) - self.expect_indent() - stmt_list = [] # type: List[Node] - if allow_type: - cur = self.current() - if type is None and isinstance(cur, StrLit): - ds = docstring.parse_docstring(cur.parsed()) - if ds and False: # TODO: Enable when this is working. - try: - type = parse_str_as_signature(ds.as_type_str(), cur.line) - except TypeParseError: - # We don't require docstrings to be actually correct. - # TODO: Report something here. - type = None - while (not isinstance(self.current(), Dedent) and - not isinstance(self.current(), Eof)): - try: - stmt, is_simple = self.parse_statement() - if is_simple: - self.expect_break() - if stmt is not None: - if not self.try_combine_overloads(stmt, stmt_list): - stmt_list.append(stmt) - except ParseError: - pass - if isinstance(self.current(), Dedent): - self.skip() - node = Block(stmt_list) - node.set_line(colon) - return node, type - - def try_combine_overloads(self, s: Node, stmt: List[Node]) -> bool: - if isinstance(s, Decorator) and stmt: - fdef = s - n = fdef.func.name() - if isinstance(stmt[-1], Decorator) and stmt[-1].func.name() == n: - stmt[-1] = OverloadedFuncDef([stmt[-1], fdef]) - return True - elif isinstance(stmt[-1], OverloadedFuncDef) and stmt[-1].name() == n: - stmt[-1].items.append(fdef) - return True - return False - - def parse_statement(self) -> Tuple[Node, bool]: - stmt = None # type: Node - t = self.current() - ts = self.current_str() - is_simple = True # Is this a non-block statement? - if ts == 'if': - stmt = self.parse_if_stmt() - is_simple = False - elif ts == 'def': - stmt = self.parse_function() - is_simple = False - elif ts == 'while': - stmt = self.parse_while_stmt() - is_simple = False - elif ts == 'return': - stmt = self.parse_return_stmt() - elif ts == 'for': - stmt = self.parse_for_stmt() - is_simple = False - elif ts == 'try': - stmt = self.parse_try_stmt() - is_simple = False - elif ts == 'break': - stmt = self.parse_break_stmt() - elif ts == 'continue': - stmt = self.parse_continue_stmt() - elif ts == 'pass': - stmt = self.parse_pass_stmt() - elif ts == 'raise': - stmt = self.parse_raise_stmt() - elif ts == 'import': - stmt = self.parse_import() - elif ts == 'from': - stmt = self.parse_import_from() - elif ts == 'class': - stmt = self.parse_class_def() - is_simple = False - elif ts == 'global': - stmt = self.parse_global_decl() - elif ts == 'nonlocal' and self.pyversion[0] >= 3: - stmt = self.parse_nonlocal_decl() - elif ts == 'assert': - stmt = self.parse_assert_stmt() - elif ts == 'del': - stmt = self.parse_del_stmt() - elif ts == 'with': - stmt = self.parse_with_stmt() - is_simple = False - elif ts == '@': - stmt = self.parse_decorated_function_or_class() - is_simple = False - elif ts == 'print' and (self.pyversion[0] == 2 and - 'print_function' not in self.future_options): - stmt = self.parse_print_stmt() - elif ts == 'exec' and self.pyversion[0] == 2: - stmt = self.parse_exec_stmt() - else: - stmt = self.parse_expression_or_assignment() - if stmt is not None: - stmt.set_line(t) - return stmt, is_simple - - def parse_expression_or_assignment(self) -> Node: - expr = self.parse_expression(star_expr_allowed=True) - if self.current_str() == '=': - return self.parse_assignment(expr) - elif self.current_str() in op_assign: - # Operator assignment statement. - op = self.current_str()[:-1] - self.skip() - rvalue = self.parse_expression() - return OperatorAssignmentStmt(op, expr, rvalue) - else: - # Expression statement. - return ExpressionStmt(expr) - - def parse_assignment(self, lvalue: Any) -> Node: - """Parse an assignment statement. - - Assume that lvalue has been parsed already, and the current token is '='. - Also parse an optional '# type:' comment. - """ - self.expect('=') - lvalues = [lvalue] - expr = self.parse_expression(star_expr_allowed=True) - while self.current_str() == '=': - self.skip() - lvalues.append(expr) - expr = self.parse_expression(star_expr_allowed=True) - cur = self.current() - if isinstance(cur, Break): - type = self.parse_type_comment(cur, signature=False) - else: - type = None - return AssignmentStmt(lvalues, expr, type) - - def parse_return_stmt(self) -> ReturnStmt: - self.expect('return') - expr = None - current = self.current() - if current.string == 'yield': - self.parse_error() - if not isinstance(current, Break): - expr = self.parse_expression() - node = ReturnStmt(expr) - return node - - def parse_raise_stmt(self) -> RaiseStmt: - self.expect('raise') - expr = None - from_expr = None - if not isinstance(self.current(), Break): - expr = self.parse_expression() - if self.current_str() == 'from': - self.expect('from') - from_expr = self.parse_expression() - node = RaiseStmt(expr, from_expr) - return node - - def parse_assert_stmt(self) -> AssertStmt: - self.expect('assert') - expr = self.parse_expression() - node = AssertStmt(expr) - return node - - def parse_yield_or_yield_from_expr(self) -> Union[YieldFromExpr, YieldExpr]: - self.expect("yield") - expr = None - node = YieldExpr(expr) # type: Union[YieldFromExpr, YieldExpr] - if not isinstance(self.current(), Break): - if self.current_str() == "from": - self.expect("from") - expr = self.parse_expression() # when yield from is assigned to a variable - node = YieldFromExpr(expr) - else: - if self.current_str() == ')': - node = YieldExpr(None) - else: - expr = self.parse_expression() - node = YieldExpr(expr) - return node - - def parse_ellipsis(self) -> EllipsisExpr: - self.expect('...') - node = EllipsisExpr() - return node - - def parse_del_stmt(self) -> DelStmt: - self.expect('del') - expr = self.parse_expression() - node = DelStmt(expr) - return node - - def parse_break_stmt(self) -> BreakStmt: - self.expect('break') - node = BreakStmt() - return node - - def parse_continue_stmt(self) -> ContinueStmt: - self.expect('continue') - node = ContinueStmt() - return node - - def parse_pass_stmt(self) -> PassStmt: - self.expect('pass') - node = PassStmt() - return node - - def parse_global_decl(self) -> GlobalDecl: - self.expect('global') - names = self.parse_identifier_list() - node = GlobalDecl(names) - return node - - def parse_nonlocal_decl(self) -> NonlocalDecl: - self.expect('nonlocal') - names = self.parse_identifier_list() - node = NonlocalDecl(names) - return node - - def parse_identifier_list(self) -> List[str]: - names = [] - while True: - n = self.expect_type(Name) - names.append(n.string) - if self.current_str() != ',': - break - self.skip() - return names - - def parse_while_stmt(self) -> WhileStmt: - is_error = False - self.expect('while') - try: - expr = self.parse_expression() - except ParseError: - is_error = True - body, _ = self.parse_block() - if self.current_str() == 'else': - self.expect('else') - else_body, _ = self.parse_block() - else: - else_body = None - if is_error is not None: - node = WhileStmt(expr, body, else_body) - return node - else: - return None - - def parse_for_stmt(self) -> ForStmt: - self.expect('for') - index = self.parse_for_index_variables() - self.expect('in') - expr = self.parse_expression() - - body, _ = self.parse_block() - - if self.current_str() == 'else': - self.expect('else') - else_body, _ = self.parse_block() - else: - else_body = None - - node = ForStmt(index, expr, body, else_body) - return node - - def parse_for_index_variables(self) -> Node: - # Parse index variables of a 'for' statement. - index_items = [] - force_tuple = False - - while True: - v = self.parse_expression(precedence['in'], - star_expr_allowed=True) # Prevent parsing of for stmt 'in' - index_items.append(v) - if self.current_str() != ',': - break - self.skip() - if self.current_str() == 'in': - force_tuple = True - break - - if len(index_items) == 1 and not force_tuple: - index = index_items[0] - else: - index = TupleExpr(index_items) - index.set_line(index_items[0].get_line()) - - return index - - def parse_if_stmt(self) -> IfStmt: - is_error = False - - self.expect('if') - expr = [] - try: - expr.append(self.parse_expression()) - except ParseError: - is_error = True - - body = [self.parse_block()[0]] - - while self.current_str() == 'elif': - self.expect('elif') - try: - expr.append(self.parse_expression()) - except ParseError: - is_error = True - body.append(self.parse_block()[0]) - - if self.current_str() == 'else': - self.expect('else') - else_body, _ = self.parse_block() - else: - else_body = None - - if not is_error: - node = IfStmt(expr, body, else_body) - return node - else: - return None - - def parse_try_stmt(self) -> Node: - self.expect('try') - body, _ = self.parse_block() - is_error = False - vars = [] # type: List[NameExpr] - types = [] # type: List[Node] - handlers = [] # type: List[Block] - while self.current_str() == 'except': - self.expect('except') - if not isinstance(self.current(), Colon): - try: - t = self.current() - types.append(self.parse_expression(precedence[',']).set_line(t)) - if self.current_str() == 'as': - self.expect('as') - vars.append(self.parse_name_expr()) - elif self.pyversion[0] == 2 and self.current_str() == ',': - self.expect(',') - vars.append(self.parse_name_expr()) - else: - vars.append(None) - except ParseError: - is_error = True - else: - types.append(None) - vars.append(None) - handlers.append(self.parse_block()[0]) - if not is_error: - if self.current_str() == 'else': - self.skip() - else_body, _ = self.parse_block() - else: - else_body = None - if self.current_str() == 'finally': - self.expect('finally') - finally_body, _ = self.parse_block() - else: - finally_body = None - node = TryStmt(body, vars, types, handlers, else_body, - finally_body) - return node - else: - return None - - def parse_with_stmt(self) -> WithStmt: - self.expect('with') - exprs = [] - targets = [] - while True: - expr = self.parse_expression(precedence[',']) - if self.current_str() == 'as': - self.expect('as') - target = self.parse_expression(precedence[',']) - else: - target = None - exprs.append(expr) - targets.append(target) - if self.current_str() != ',': - break - self.expect(',') - body, _ = self.parse_block() - return WithStmt(exprs, targets, body) - - def parse_print_stmt(self) -> PrintStmt: - self.expect('print') - args = [] - target = None - if self.current_str() == '>>': - self.skip() - target = self.parse_expression(precedence[',']) - if self.current_str() == ',': - self.skip() - if isinstance(self.current(), Break): - self.parse_error() - else: - if not isinstance(self.current(), Break): - self.parse_error() - comma = False - while not isinstance(self.current(), Break): - args.append(self.parse_expression(precedence[','])) - if self.current_str() == ',': - comma = True - self.skip() - else: - comma = False - break - return PrintStmt(args, newline=not comma, target=target) - - def parse_exec_stmt(self) -> ExecStmt: - self.expect('exec') - expr = self.parse_expression(precedence['in']) - variables1 = None - variables2 = None - if self.current_str() == 'in': - self.skip() - variables1 = self.parse_expression(precedence[',']) - if self.current_str() == ',': - self.skip() - variables2 = self.parse_expression(precedence[',']) - return ExecStmt(expr, variables1, variables2) - - # Parsing expressions - - def parse_expression(self, prec: int = 0, star_expr_allowed: bool = False) -> Node: - """Parse a subexpression within a specific precedence context.""" - expr = None # type: Node - current = self.current() # Remember token for setting the line number. - - # Parse a "value" expression or unary operator expression and store - # that in expr. - s = self.current_str() - if s == '(': - # Parenthesised expression or cast. - expr = self.parse_parentheses() - elif s == '[': - expr = self.parse_list_expr() - elif s in ['-', '+', 'not', '~']: - # Unary operation. - expr = self.parse_unary_expr() - elif s == 'lambda': - expr = self.parse_lambda_expr() - elif s == '{': - expr = self.parse_dict_or_set_expr() - elif s == '*' and star_expr_allowed: - expr = self.parse_star_expr() - elif s == '`' and self.pyversion[0] == 2: - expr = self.parse_backquote_expr() - else: - if isinstance(current, Name): - # Name expression. - expr = self.parse_name_expr() - elif isinstance(current, IntLit): - expr = self.parse_int_expr() - elif isinstance(current, StrLit): - expr = self.parse_str_expr() - elif isinstance(current, BytesLit): - expr = self.parse_bytes_literal() - elif isinstance(current, UnicodeLit): - expr = self.parse_unicode_literal() - elif isinstance(current, FloatLit): - expr = self.parse_float_expr() - elif isinstance(current, ComplexLit): - expr = self.parse_complex_expr() - elif isinstance(current, Keyword) and s == "yield": - # The expression yield from and yield to assign - expr = self.parse_yield_or_yield_from_expr() - elif isinstance(current, EllipsisToken) and (self.pyversion[0] >= 3 - or self.is_stub_file): - expr = self.parse_ellipsis() - else: - # Invalid expression. - self.parse_error() - - # Set the line of the expression node, if not specified. This - # simplifies recording the line number as not every node type needs to - # deal with it separately. - if expr.line < 0: - expr.set_line(current) - - # Parse operations that require a left argument (stored in expr). - while True: - current = self.current() - s = self.current_str() - if s == '(': - # Call expression. - expr = self.parse_call_expr(expr) - elif s == '.': - # Member access expression. - expr = self.parse_member_expr(expr) - elif s == '[': - # Indexing expression. - expr = self.parse_index_expr(expr) - elif s == ',': - # The comma operator is used to build tuples. Comma also - # separates array items and function arguments, but in this - # case the precedence is too low to build a tuple. - if precedence[','] > prec: - expr = self.parse_tuple_expr(expr) - else: - break - elif s == 'for': - if precedence[''] > prec: - # List comprehension or generator expression. Parse as - # generator expression; it will be converted to list - # comprehension if needed elsewhere. - expr = self.parse_generator_expr(expr) - else: - break - elif s == 'if': - # Conditional expression. - if precedence[''] > prec: - expr = self.parse_conditional_expr(expr) - else: - break - else: - # Binary operation or a special case. - if isinstance(current, Op): - op = self.current_str() - op_prec = precedence[op] - if op == 'not': - # Either "not in" or an error. - op_prec = precedence['in'] - if op_prec > prec: - if op in op_comp: - expr = self.parse_comparison_expr(expr, op_prec) - else: - expr = self.parse_bin_op_expr(expr, op_prec) - else: - # The operation cannot be associated with the - # current left operand due to the precedence - # context; let the caller handle it. - break - else: - # Not an operation that accepts a left argument; let the - # caller handle the rest. - break - - # Set the line of the expression node, if not specified. This - # simplifies recording the line number as not every node type - # needs to deal with it separately. - if expr.line < 0: - expr.set_line(current) - - return expr - - def parse_parentheses(self) -> Node: - self.skip() - if self.current_str() == ')': - # Empty tuple (). - expr = self.parse_empty_tuple_expr() # type: Node - else: - # Parenthesised expression. - expr = self.parse_expression(0, star_expr_allowed=True) - self.expect(')') - return expr - - def parse_star_expr(self) -> Node: - star = self.expect('*') - expr = self.parse_expression(precedence['*u']) - expr = StarExpr(expr) - if expr.line < 0: - expr.set_line(star) - return expr - - def parse_empty_tuple_expr(self) -> TupleExpr: - self.expect(')') - node = TupleExpr([]) - return node - - def parse_list_expr(self) -> Node: - """Parse list literal or list comprehension.""" - items = [] - self.expect('[') - while self.current_str() != ']' and not self.eol(): - items.append(self.parse_expression(precedence[''], star_expr_allowed=True)) - if self.current_str() != ',': - break - self.expect(',') - if self.current_str() == 'for' and len(items) == 1: - items[0] = self.parse_generator_expr(items[0]) - self.expect(']') - if len(items) == 1 and isinstance(items[0], GeneratorExpr): - return ListComprehension(items[0]) - else: - expr = ListExpr(items) - return expr - - def parse_generator_expr(self, left_expr: Node) -> GeneratorExpr: - tok = self.current() - indices, sequences, condlists = self.parse_comp_for() - - gen = GeneratorExpr(left_expr, indices, sequences, condlists) - gen.set_line(tok) - return gen - - def parse_comp_for(self) -> Tuple[List[Node], List[Node], List[List[Node]]]: - indices = [] - sequences = [] - condlists = [] # type: List[List[Node]] - while self.current_str() == 'for': - conds = [] - self.expect('for') - index = self.parse_for_index_variables() - indices.append(index) - self.expect('in') - if self.pyversion[0] >= 3: - sequence = self.parse_expression(precedence['']) - else: - sequence = self.parse_expression_list() - sequences.append(sequence) - while self.current_str() == 'if': - self.skip() - conds.append(self.parse_expression(precedence[''])) - condlists.append(conds) - - return indices, sequences, condlists - - def parse_expression_list(self) -> Node: - prec = precedence[''] - expr = self.parse_expression(prec) - if self.current_str() != ',': - return expr - else: - t = self.current() - return self.parse_tuple_expr(expr, prec).set_line(t) - - def parse_conditional_expr(self, left_expr: Node) -> ConditionalExpr: - self.expect('if') - cond = self.parse_expression(precedence['']) - self.expect('else') - else_expr = self.parse_expression(precedence['']) - return ConditionalExpr(cond, left_expr, else_expr) - - def parse_dict_or_set_expr(self) -> Node: - items = [] # type: List[Tuple[Node, Node]] - self.expect('{') - while self.current_str() != '}' and not self.eol(): - key = self.parse_expression(precedence['']) - if self.current_str() in [',', '}'] and items == []: - return self.parse_set_expr(key) - elif self.current_str() == 'for' and items == []: - return self.parse_set_comprehension(key) - elif self.current_str() != ':': - self.parse_error() - colon = self.expect(':') - value = self.parse_expression(precedence['']) - if self.current_str() == 'for' and items == []: - return self.parse_dict_comprehension(key, value, colon) - items.append((key, value)) - if self.current_str() != ',': - break - self.expect(',') - self.expect('}') - node = DictExpr(items) - return node - - def parse_set_expr(self, first: Node) -> SetExpr: - items = [first] - while self.current_str() != '}' and not self.eol(): - self.expect(',') - if self.current_str() == '}': - break - items.append(self.parse_expression(precedence[','])) - self.expect('}') - expr = SetExpr(items) - return expr - - def parse_set_comprehension(self, expr: Node): - gen = self.parse_generator_expr(expr) - self.expect('}') - set_comp = SetComprehension(gen) - return set_comp - - def parse_dict_comprehension(self, key: Node, value: Node, - colon: Token) -> DictionaryComprehension: - indices, sequences, condlists = self.parse_comp_for() - dic = DictionaryComprehension(key, value, indices, sequences, condlists) - dic.set_line(colon) - self.expect('}') - return dic - - def parse_tuple_expr(self, expr: Node, - prec: int = precedence[',']) -> TupleExpr: - items = [expr] - while True: - self.expect(',') - if (self.current_str() in [')', ']', '=', ':'] or - isinstance(self.current(), Break)): - break - items.append(self.parse_expression(prec, star_expr_allowed=True)) - if self.current_str() != ',': break - node = TupleExpr(items) - return node - - def parse_backquote_expr(self) -> BackquoteExpr: - self.expect('`') - expr = self.parse_expression() - self.expect('`') - return BackquoteExpr(expr) - - def parse_name_expr(self) -> NameExpr: - tok = self.expect_type(Name) - node = NameExpr(tok.string) - node.set_line(tok) - return node - - octal_int = re.compile('0+[1-9]') - - def parse_int_expr(self) -> IntExpr: - tok = self.expect_type(IntLit) - string = tok.string.rstrip('lL') # Strip L prefix (Python 2 long literals) - if self.octal_int.match(string): - value = int(string, 8) - else: - value = int(string, 0) - node = IntExpr(value) - return node - - def parse_str_expr(self) -> Node: - # XXX \uxxxx literals - token = self.expect_type(StrLit) - value = cast(StrLit, token).parsed() - is_unicode = False - while isinstance(self.current(), (StrLit, UnicodeLit)): - token = self.skip() - if isinstance(token, StrLit): - value += token.parsed() - elif isinstance(token, UnicodeLit): - value += token.parsed() - is_unicode = True - if is_unicode or (self.pyversion[0] == 2 and 'unicode_literals' in self.future_options): - node = UnicodeExpr(value) # type: Node - else: - node = StrExpr(value) - return node - - def parse_bytes_literal(self) -> Node: - # XXX \uxxxx literals - tok = [self.expect_type(BytesLit)] - value = (cast(BytesLit, tok[0])).parsed() - while isinstance(self.current(), BytesLit): - t = cast(BytesLit, self.skip()) - value += t.parsed() - if self.pyversion[0] >= 3: - node = BytesExpr(value) # type: Node - else: - node = StrExpr(value) - return node - - def parse_unicode_literal(self) -> Node: - # XXX \uxxxx literals - token = self.expect_type(UnicodeLit) - value = cast(UnicodeLit, token).parsed() - while isinstance(self.current(), (UnicodeLit, StrLit)): - token = cast(Union[UnicodeLit, StrLit], self.skip()) - value += token.parsed() - if self.pyversion[0] >= 3: - # Python 3.3 supports u'...' as an alias of '...'. - node = StrExpr(value) # type: Node - else: - node = UnicodeExpr(value) - return node - - def parse_float_expr(self) -> FloatExpr: - tok = self.expect_type(FloatLit) - node = FloatExpr(float(tok.string)) - return node - - def parse_complex_expr(self) -> ComplexExpr: - tok = self.expect_type(ComplexLit) - node = ComplexExpr(complex(tok.string)) - return node - - def parse_call_expr(self, callee: Any) -> CallExpr: - self.expect('(') - args, kinds, names = self.parse_arg_expr() - self.expect(')') - node = CallExpr(callee, args, kinds, names) - return node - - def parse_arg_expr(self) -> Tuple[List[Node], List[int], List[str]]: - """Parse arguments in a call expression (within '(' and ')'). - - Return a tuple with these items: - argument expressions - argument kinds - argument names (for named arguments; None for ordinary args) - """ - args = [] # type: List[Node] - kinds = [] # type: List[int] - names = [] # type: List[str] - var_arg = False - dict_arg = False - named_args = False - while self.current_str() != ')' and not self.eol() and not dict_arg: - if isinstance(self.current(), Name) and self.peek().string == '=': - # Named argument - name = self.expect_type(Name) - self.expect('=') - kinds.append(nodes.ARG_NAMED) - names.append(name.string) - named_args = True - elif (self.current_str() == '*' and not var_arg and not dict_arg): - # *args - var_arg = True - self.expect('*') - kinds.append(nodes.ARG_STAR) - names.append(None) - elif self.current_str() == '**': - # **kwargs - self.expect('**') - dict_arg = True - kinds.append(nodes.ARG_STAR2) - names.append(None) - elif not var_arg and not named_args: - # Ordinary argument - kinds.append(nodes.ARG_POS) - names.append(None) - else: - self.parse_error() - args.append(self.parse_expression(precedence[','])) - if self.current_str() != ',': - break - self.expect(',') - return args, kinds, names - - def parse_member_expr(self, expr: Any) -> Node: - self.expect('.') - name = self.expect_type(Name) - if (isinstance(expr, CallExpr) and isinstance(expr.callee, NameExpr) - and expr.callee.name == 'super'): - # super() expression - node = SuperExpr(name.string) # type: Node - else: - node = MemberExpr(expr, name.string) - return node - - def parse_index_expr(self, base: Any) -> IndexExpr: - self.expect('[') - index = self.parse_slice_item() - if self.current_str() == ',': - # Extended slicing such as x[1:, :2]. - items = [index] - while self.current_str() == ',': - self.skip() - if self.current_str() == ']' or isinstance(self.current(), Break): - break - items.append(self.parse_slice_item()) - index = TupleExpr(items) - index.set_line(items[0].line) - self.expect(']') - node = IndexExpr(base, index) - return node - - def parse_slice_item(self) -> Node: - if self.current_str() != ':': - if self.current_str() == '...': - # Ellipsis is valid here even in Python 2 (but not elsewhere). - ellipsis = EllipsisExpr() - token = self.skip() - ellipsis.set_line(token) - return ellipsis - else: - item = self.parse_expression(precedence[',']) - else: - item = None - if self.current_str() == ':': - # Slice. - index = item - colon = self.expect(':') - if self.current_str() not in (']', ':', ','): - end_index = self.parse_expression(precedence[',']) - else: - end_index = None - stride = None - if self.current_str() == ':': - self.expect(':') - if self.current_str() not in (']', ','): - stride = self.parse_expression(precedence[',']) - item = SliceExpr(index, end_index, stride).set_line(colon.line) - return item - - def parse_bin_op_expr(self, left: Node, prec: int) -> OpExpr: - op = self.expect_type(Op) - op_str = op.string - if op_str == '~': - self.ind -= 1 - self.parse_error() - right = self.parse_expression(prec) - node = OpExpr(op_str, left, right) - return node - - def parse_comparison_expr(self, left: Node, prec: int) -> ComparisonExpr: - operators_str = [] - operands = [left] - - while True: - op = self.expect_type(Op) - op_str = op.string - if op_str == 'not': - if self.current_str() == 'in': - op_str = 'not in' - self.skip() - else: - self.parse_error() - elif op_str == 'is' and self.current_str() == 'not': - op_str = 'is not' - self.skip() - - operators_str.append(op_str) - operand = self.parse_expression(prec) - operands.append(operand) - - # Continue if next token is a comparison operator - self.current() - s = self.current_str() - if s not in op_comp: - break - - node = ComparisonExpr(operators_str, operands) - return node - - def parse_unary_expr(self) -> UnaryExpr: - op_tok = self.skip() - op = op_tok.string - if op == '-' or op == '+': - prec = precedence['-u'] - else: - prec = precedence[op] - expr = self.parse_expression(prec) - node = UnaryExpr(op, expr) - return node - - def parse_lambda_expr(self) -> FuncExpr: - lambda_tok = self.expect('lambda') - - args, extra_stmts = self.parse_arg_list(allow_signature=False) - - # Use 'object' as the placeholder return type; it will be inferred - # later. We can't use 'Any' since it could make type inference results - # less precise. - ret_type = UnboundType('__builtins__.object') - typ = self.build_func_annotation(ret_type, args, - lambda_tok.line, is_default_ret=True) - - colon = self.expect(':') - - expr = self.parse_expression(precedence[',']) - - nodes = [ReturnStmt(expr).set_line(lambda_tok)] - # Potentially insert extra assignment statements to the beginning of the - # body, used to decompose Python 2 tuple arguments. - nodes[:0] = extra_stmts - body = Block(nodes) - body.set_line(colon) - - return FuncExpr(args, body, typ) - - # Helper methods - - def skip(self) -> Token: - self.ind += 1 - return self.tok[self.ind - 1] - - def expect(self, string: str) -> Token: - if self.current_str() == string: - self.ind += 1 - return self.tok[self.ind - 1] - else: - self.parse_error() - - def expect_indent(self) -> Token: - if isinstance(self.current(), Indent): - return self.expect_type(Indent) - else: - self.fail('Expected an indented block', self.current().line) - return none - - def fail(self, msg: str, line: int) -> None: - self.errors.report(line, msg) - - def expect_type(self, typ: type) -> Token: - current = self.current() - if isinstance(current, typ): - self.ind += 1 - return current - else: - self.parse_error() - - def expect_colon_and_break(self) -> Tuple[Token, Token]: - return self.expect_type(Colon), self.expect_type(Break) - - def expect_break(self) -> Token: - return self.expect_type(Break) - - def current(self) -> Token: - return self.tok[self.ind] - - def current_str(self) -> str: - return self.current().string - - def peek(self) -> Token: - return self.tok[self.ind + 1] - - def parse_error(self) -> None: - self.parse_error_at(self.current()) - raise ParseError() - - def parse_error_at(self, tok: Token, skip: bool = True, reason: Optional[str] = None) -> None: - msg = '' - if isinstance(tok, LexError): - msg = token_repr(tok) - msg = msg[0].upper() + msg[1:] - elif isinstance(tok, Indent) or isinstance(tok, Dedent): - msg = 'Inconsistent indentation' - else: - formatted_reason = ": {}".format(reason) if reason else "" - msg = 'Parse error before {}{}'.format(token_repr(tok), formatted_reason) - - self.errors.report(tok.line, msg) - - if skip: - self.skip_until_next_line() - - def skip_until_break(self) -> None: - n = 0 - while (not isinstance(self.current(), Break) - and not isinstance(self.current(), Eof)): - self.skip() - n += 1 - if isinstance(self.tok[self.ind - 1], Colon) and n > 1: - self.ind -= 1 - - def skip_until_next_line(self) -> None: - self.skip_until_break() - if isinstance(self.current(), Break): - self.skip() - - def eol(self) -> bool: - return isinstance(self.current(), Break) or self.eof() - - def eof(self) -> bool: - return isinstance(self.current(), Eof) - - # Type annotation related functionality - - def parse_type(self) -> Type: - try: - typ, self.ind = parse_type(self.tok, self.ind) - except TypeParseError as e: - self.parse_error_at(e.token, reason=e.message) - raise ParseError() - return typ - - annotation_prefix_re = re.compile(r'#\s*type:') - ignore_prefix_re = re.compile(r'ignore\b') - - def parse_type_comment(self, token: Token, signature: bool) -> Type: - """Parse a '# type: ...' annotation. - - Return None if no annotation found. If signature is True, expect - a type signature of form (...) -> t. - """ - whitespace_or_comments = token.rep().strip() - if self.annotation_prefix_re.match(whitespace_or_comments): - type_as_str = whitespace_or_comments.split(':', 1)[1].strip() - if self.ignore_prefix_re.match(type_as_str): - # Actually a "# type: ignore" annotation -> not a type. - return None - tokens = lex.lex(type_as_str, token.line)[0] - if len(tokens) < 2: - # Empty annotation (only Eof token) - self.errors.report(token.line, 'Empty type annotation') - return None - try: - if not signature: - type, index = parse_types(tokens, 0) - else: - type, index = parse_signature(tokens) - except TypeParseError as e: - self.parse_error_at(e.token, skip=False, reason=e.message) - return None - if index < len(tokens) - 2: - self.parse_error_at(tokens[index], skip=False) - return None - return type - else: - return None - - -class ParseError(Exception): pass - - -def token_repr(tok: Token) -> str: - """Return a representation of a token for use in parse error messages.""" - if isinstance(tok, Break): - return 'end of line' - elif isinstance(tok, Eof): - return 'end of file' - elif isinstance(tok, Keyword) or isinstance(tok, Name): - return '"{}"'.format(tok.string) - elif isinstance(tok, IntLit) or isinstance(tok, FloatLit) or isinstance(tok, ComplexLit): - return 'numeric literal' - elif isinstance(tok, StrLit) or isinstance(tok, UnicodeLit): - return 'string literal' - elif (isinstance(tok, Punct) or isinstance(tok, Op) - or isinstance(tok, Colon)): - return tok.string - elif isinstance(tok, Bom): - return 'byte order mark' - elif isinstance(tok, Indent): - return 'indent' - elif isinstance(tok, Dedent): - return 'dedent' - elif isinstance(tok, EllipsisToken): - return '...' - else: - if isinstance(tok, LexError): - t = tok.type - if t == lex.NUMERIC_LITERAL_ERROR: - return 'invalid numeric literal' - elif t == lex.UNTERMINATED_STRING_LITERAL: - return 'unterminated string literal' - elif t == lex.INVALID_CHARACTER: - msg = 'unrecognized character' - if ord(tok.string) in range(33, 127): - msg += ' ' + tok.string - return msg - elif t == lex.INVALID_DEDENT: - return 'inconsistent indentation' - elif t == lex.DECODE_ERROR: - return tok.message - raise ValueError('Unknown token {}'.format(repr(tok))) - - -if __name__ == '__main__': - # Parse a file and dump the AST (or display errors). - import sys - - def usage(): - print('Usage: parse.py [--py2] [--quiet] FILE [...]', file=sys.stderr) - sys.exit(2) - - args = sys.argv[1:] - pyversion = defaults.PYTHON3_VERSION - quiet = False - while args and args[0].startswith('--'): - if args[0] == '--py2': - pyversion = defaults.PYTHON2_VERSION - elif args[0] == '--quiet': - quiet = True - else: - usage() - args = args[1:] - if len(args) < 1: - usage() - status = 0 - for fnam in args: - s = open(fnam, 'rb').read() - errors = Errors() - try: - options = Options() - options.python_version = pyversion - tree = parse(s, fnam, None, options=options) - if not quiet: - print(tree) - except CompileError as e: - for msg in e.messages: - sys.stderr.write('%s\n' % msg) - status = 1 - sys.exit(status) +"""Mypy parser. + +Constructs a parse tree (abstract syntax tree) based on a string +representing a source file. Performs only minimal semantic checks. +""" + +import re + +from typing import List, Tuple, Any, Set, cast, Union, Optional + +from mypy import lex, docstring +from mypy.lex import ( + Token, Eof, Bom, Break, Name, Colon, Dedent, IntLit, StrLit, BytesLit, + UnicodeLit, FloatLit, Op, Indent, Keyword, Punct, LexError, ComplexLit, + EllipsisToken +) +import mypy.types +from mypy.nodes import ( + MypyFile, Import, Node, ImportAll, ImportFrom, FuncDef, OverloadedFuncDef, + ClassDef, Decorator, Block, Var, OperatorAssignmentStmt, + ExpressionStmt, AssignmentStmt, ReturnStmt, RaiseStmt, AssertStmt, + DelStmt, BreakStmt, ContinueStmt, PassStmt, GlobalDecl, + WhileStmt, ForStmt, IfStmt, TryStmt, WithStmt, + TupleExpr, GeneratorExpr, ListComprehension, ListExpr, ConditionalExpr, + DictExpr, SetExpr, NameExpr, IntExpr, StrExpr, BytesExpr, UnicodeExpr, + FloatExpr, CallExpr, SuperExpr, MemberExpr, IndexExpr, SliceExpr, OpExpr, + UnaryExpr, FuncExpr, PrintStmt, ImportBase, ComparisonExpr, + StarExpr, YieldFromExpr, NonlocalDecl, DictionaryComprehension, + SetComprehension, ComplexExpr, EllipsisExpr, YieldExpr, ExecStmt, Argument, + BackquoteExpr +) +from mypy import defaults +from mypy import nodes +from mypy.errors import Errors, CompileError +from mypy.types import Type, CallableType, AnyType, UnboundType +from mypy.parsetype import ( + parse_type, parse_types, parse_signature, TypeParseError, parse_str_as_signature +) +from mypy.options import Options + +from mypy import experiments + + +precedence = { + '**': 16, + '-u': 15, '+u': 15, '~': 15, # unary operators (-, + and ~) + '': 14, + '*': 13, '/': 13, '//': 13, '%': 13, + '+': 12, '-': 12, + '>>': 11, '<<': 11, + '&': 10, + '^': 9, + '|': 8, + '==': 7, '!=': 7, '<': 7, '>': 7, '<=': 7, '>=': 7, 'is': 7, 'in': 7, + '*u': 7, # unary * for star expressions + 'not': 6, + 'and': 5, + 'or': 4, + '': 3, # conditional expression + '': 2, # list comprehension + ',': 1} + + +op_assign = set([ + '+=', '-=', '*=', '/=', '//=', '%=', '**=', '|=', '&=', '^=', '>>=', + '<<=']) + +op_comp = set([ + '>', '<', '==', '>=', '<=', '<>', '!=', 'is', 'is', 'in', 'not']) + +none = Token('') # Empty token + + +def parse(source: Union[str, bytes], + fnam: str, + errors: Errors, + options: Options) -> MypyFile: + """Parse a source file, without doing any semantic analysis. + + Return the parse tree. If errors is not provided, raise ParseError + on failure. Otherwise, use the errors object to report parse errors. + + The python_version (major, minor) option determines the Python syntax variant. + """ + if options.fast_parser: + import mypy.fastparse + return mypy.fastparse.parse(source, + fnam=fnam, + errors=errors, + pyversion=options.python_version, + custom_typing_module=options.custom_typing_module) + + is_stub_file = bool(fnam) and fnam.endswith('.pyi') + parser = Parser(fnam, + errors, + options.python_version, + options.custom_typing_module, + is_stub_file=is_stub_file) + tree = parser.parse(source) + tree.path = fnam + tree.is_stub = is_stub_file + return tree + + +class Parser: + """Mypy parser that parses a string into an AST. + + Parses type annotations in addition to basic Python syntax. It supports both Python 2 and 3 + (though Python 2 support is incomplete). + + The AST classes are defined in mypy.nodes and mypy.types. + """ + + tok = None # type: List[Token] + ind = 0 + errors = None # type: Errors + # If True, raise an exception on any parse error. Otherwise, errors are reported via 'errors'. + raise_on_error = False + + # Are we currently parsing the body of a class definition? + is_class_body = False + # All import nodes encountered so far in this parse unit. + imports = None # type: List[ImportBase] + # Names imported from __future__. + future_options = None # type: List[str] + # Lines to ignore (using # type: ignore). + ignored_lines = None # type: Set[int] + + def __init__(self, fnam: str, errors: Errors, pyversion: Tuple[int, int], + custom_typing_module: str = None, is_stub_file: bool = False) -> None: + self.raise_on_error = errors is None + self.pyversion = pyversion + self.custom_typing_module = custom_typing_module + self.is_stub_file = is_stub_file + if errors is not None: + self.errors = errors + else: + self.errors = Errors() + if fnam is not None: + self.errors.set_file(fnam) + else: + self.errors.set_file('') + + def parse(self, s: Union[str, bytes]) -> MypyFile: + self.tok, self.ignored_lines = lex.lex(s, pyversion=self.pyversion, + is_stub_file=self.is_stub_file) + self.ind = 0 + self.imports = [] + self.future_options = [] + file = self.parse_file() + if self.raise_on_error and self.errors.is_errors(): + self.errors.raise_error() + return file + + def weak_opts(self) -> Set[str]: + """Do weak typing if any of the first ten tokens is a comment saying so. + + The comment can be one of: + # mypy: weak=global + # mypy: weak=local + # mypy: weak <- defaults to local + """ + regexp = re.compile(r'^[\s]*# *mypy: *weak(=?)([^\s]*)', re.M) + for t in self.tok[:10]: + for s in [t.string, t.pre]: + m = regexp.search(s) + if m: + opts = set(x for x in m.group(2).split(',') if x) + if not opts: + opts.add('local') + return opts + return set() + + def parse_file(self) -> MypyFile: + """Parse a mypy source file.""" + is_bom = self.parse_bom() + defs = self.parse_defs() + weak_opts = self.weak_opts() + self.expect_type(Eof) + node = MypyFile(defs, self.imports, is_bom, self.ignored_lines, + weak_opts=weak_opts) + return node + + # Parse the initial part + + def parse_bom(self) -> bool: + """Parse the optional byte order mark at the beginning of a file.""" + if isinstance(self.current(), Bom): + self.expect_type(Bom) + if isinstance(self.current(), Break): + self.expect_break() + return True + else: + return False + + def parse_import(self) -> Import: + self.expect('import') + ids = [] + while True: + id = self.parse_qualified_name() + translated = self.translate_module_id(id) + as_id = None + if self.current_str() == 'as': + self.expect('as') + name_tok = self.expect_type(Name) + as_id = name_tok.string + elif translated != id: + as_id = id + ids.append((translated, as_id)) + if self.current_str() != ',': + break + self.expect(',') + node = Import(ids) + self.imports.append(node) + return node + + def translate_module_id(self, id: str) -> str: + """Return the actual, internal module id for a source text id. + + For example, translate '__builtin__' in Python 2 to 'builtins'. + """ + if id == self.custom_typing_module: + return 'typing' + elif id == '__builtin__' and self.pyversion[0] == 2: + # HACK: __builtin__ in Python 2 is aliases to builtins. However, the implementation + # is named __builtin__.py (there is another layer of translation elsewhere). + return 'builtins' + return id + + def parse_import_from(self) -> Node: + self.expect('from') + + # Build the list of beginning relative tokens. + relative = 0 + while self.current_str() in (".", "..."): + relative += len(self.current_str()) + self.skip() + + # Parse qualified name to actually import from. + if self.current_str() == "import": + # Empty/default values. + name = "" + else: + name = self.parse_qualified_name() + + name = self.translate_module_id(name) + + # Parse import list + self.expect('import') + node = None # type: ImportBase + if self.current_str() == '*': + if name == '__future__': + self.parse_error() + # An import all from a module node: + self.skip() + node = ImportAll(name, relative) + else: + is_paren = self.current_str() == '(' + if is_paren: + self.expect('(') + targets = [] # type: List[Tuple[str, str]] + while True: + id, as_id = self.parse_import_name() + if '%s.%s' % (name, id) == self.custom_typing_module: + if targets or self.current_str() == ',': + self.fail('You cannot import any other modules when you ' + 'import a custom typing module', + self.current().line) + node = Import([('typing', as_id)]) + self.skip_until_break() + break + targets.append((id, as_id)) + if self.current_str() != ',': + break + self.expect(',') + if is_paren and self.current_str() == ')': + break + if is_paren: + self.expect(')') + if node is None: + node = ImportFrom(name, relative, targets) + self.imports.append(node) + if name == '__future__': + self.future_options.extend(target[0] for target in targets) + return node + + def parse_import_name(self) -> Tuple[str, Optional[str]]: + tok = self.expect_type(Name) + name = tok.string + if self.current_str() == 'as': + self.skip() + as_name = self.expect_type(Name) + return name, as_name.string + else: + return name, None + + def parse_qualified_name(self) -> str: + """Parse a name with an optional module qualifier. + + Return a tuple with the name as a string and a token array + containing all the components of the name. + """ + tok = self.expect_type(Name) + n = tok.string + while self.current_str() == '.': + self.expect('.') + tok = self.expect_type(Name) + n += '.' + tok.string + return n + + # Parsing global definitions + + def parse_defs(self) -> List[Node]: + defs = [] # type: List[Node] + while not self.eof(): + try: + defn, is_simple = self.parse_statement() + if is_simple: + self.expect_break() + if defn is not None: + if not self.try_combine_overloads(defn, defs): + defs.append(defn) + except ParseError: + pass + return defs + + def parse_class_def(self) -> ClassDef: + old_is_class_body = self.is_class_body + self.is_class_body = True + + self.expect('class') + metaclass = None + + try: + base_types = [] # type: List[Node] + try: + name_tok = self.expect_type(Name) + name = name_tok.string + + self.errors.push_type(name) + + if self.current_str() == '(': + self.skip() + while True: + if self.current_str() == ')': + break + if self.peek().string == '=': + metaclass = self.parse_class_keywords() + break + base_types.append(self.parse_super_type()) + if self.current_str() != ',': + break + self.skip() + self.expect(')') + except ParseError: + pass + + defs, _ = self.parse_block() + + node = ClassDef(name, defs, None, base_types, metaclass=metaclass) + return node + finally: + self.errors.pop_type() + self.is_class_body = old_is_class_body + + def parse_class_keywords(self) -> Optional[str]: + """Parse the class keyword arguments, keeping the metaclass but + ignoring all others. Returns None if the metaclass isn't found. + """ + metaclass = None + while True: + key = self.expect_type(Name) + self.expect('=') + if key.string == 'metaclass': + metaclass = self.parse_qualified_name() + else: + # skip the class value + self.parse_expression(precedence[',']) + if self.current_str() != ',': + break + self.skip() + if self.current_str() == ')': + break + return metaclass + + def parse_super_type(self) -> Node: + return self.parse_expression(precedence[',']) + + def parse_decorated_function_or_class(self) -> Node: + decorators = [] + no_type_checks = False + while self.current_str() == '@': + self.expect('@') + d_exp = self.parse_expression() + if self.is_no_type_check_decorator(d_exp): + no_type_checks = True + decorators.append(d_exp) + self.expect_break() + if self.current_str() != 'class': + func = self.parse_function(no_type_checks) + func.is_decorated = True + var = Var(func.name()) + # Types of decorated functions must always be inferred. + var.is_ready = False + var.set_line(decorators[0].line) + node = Decorator(func, decorators, var) + return node + else: + cls = self.parse_class_def() + cls.decorators = decorators + return cls + + def is_no_type_check_decorator(self, expr: Node) -> bool: + if isinstance(expr, NameExpr): + return expr.name == 'no_type_check' + elif isinstance(expr, MemberExpr): + if isinstance(expr.expr, NameExpr): + return expr.expr.name == 'typing' and expr.name == 'no_type_check' + else: + return False + + def parse_function(self, no_type_checks: bool=False) -> FuncDef: + def_tok = self.expect('def') + is_method = self.is_class_body + self.is_class_body = False + try: + (name, args, typ, is_error, extra_stmts) = self.parse_function_header(no_type_checks) + + arg_kinds = [arg.kind for arg in args] + arg_names = [arg.variable.name() for arg in args] + + body, comment_type = self.parse_block(allow_type=True) + # Potentially insert extra assignment statements to the beginning of the + # body, used to decompose Python 2 tuple arguments. + body.body[:0] = extra_stmts + if comment_type: + # The function has a # type: ... signature. + if typ: + self.errors.report( + def_tok.line, 'Function has duplicate type signatures') + sig = cast(CallableType, comment_type) + if sig.is_ellipsis_args: + # When we encounter an ellipsis, fill in the arg_types with + # a bunch of AnyTypes, emulating Callable[..., T] + arg_types = [AnyType()] * len(arg_kinds) # type: List[Type] + typ = CallableType( + arg_types, + arg_kinds, + arg_names, + sig.ret_type, + None, + line=def_tok.line) + elif is_method and len(sig.arg_kinds) < len(arg_kinds): + self.check_argument_kinds(arg_kinds, + [nodes.ARG_POS] + sig.arg_kinds, + def_tok.line) + # Add implicit 'self' argument to signature. + first_arg = [AnyType()] # type: List[Type] + typ = CallableType( + first_arg + sig.arg_types, + arg_kinds, + arg_names, + sig.ret_type, + None, + line=def_tok.line) + else: + self.check_argument_kinds(arg_kinds, sig.arg_kinds, + def_tok.line) + typ = CallableType( + sig.arg_types, + arg_kinds, + arg_names, + sig.ret_type, + None, + line=def_tok.line) + + # If there was a serious error, we really cannot build a parse tree + # node. + if is_error: + return None + + if typ and isinstance(typ.ret_type, UnboundType): + typ.ret_type.is_ret_type = True + + node = FuncDef(name, args, body, typ) + node.set_line(def_tok) + if typ is not None: + typ.definition = node + return node + finally: + self.errors.pop_function() + self.is_class_body = is_method + + def check_argument_kinds(self, funckinds: List[int], sigkinds: List[int], + line: int) -> None: + """Check that arguments are consistent. + + This verifies that they have the same number and the kinds correspond. + + Arguments: + funckinds: kinds of arguments in function definition + sigkinds: kinds of arguments in signature (after # type:) + """ + if len(funckinds) != len(sigkinds): + if len(funckinds) > len(sigkinds): + self.fail("Type signature has too few arguments", line) + else: + self.fail("Type signature has too many arguments", line) + return + for kind, token in [(nodes.ARG_STAR, '*'), + (nodes.ARG_STAR2, '**')]: + if ((funckinds.count(kind) != sigkinds.count(kind)) or + (kind in funckinds and sigkinds.index(kind) != funckinds.index(kind))): + self.fail( + "Inconsistent use of '{}' in function " + "signature".format(token), line) + + def parse_function_header( + self, no_type_checks: bool=False) -> Tuple[str, + List[Argument], + CallableType, + bool, + List[AssignmentStmt]]: + """Parse function header (a name followed by arguments) + + Return a 5-tuple with the following items: + name + arguments + signature (annotation) + error flag (True if error) + extra statements needed to decompose arguments (usually empty) + + See parse_arg_list for an explanation of the final tuple item. + """ + name = '' + + try: + name_tok = self.expect_type(Name) + name = name_tok.string + + self.errors.push_function(name) + + args, typ, extra_stmts = self.parse_args(no_type_checks) + except ParseError: + if not isinstance(self.current(), Break): + self.ind -= 1 # Kludge: go back to the Break token + # Resynchronise parsing by going back over :, if present. + if isinstance(self.tok[self.ind - 1], Colon): + self.ind -= 1 + return (name, [], None, True, []) + + return (name, args, typ, False, extra_stmts) + + def parse_args(self, no_type_checks: bool=False) -> Tuple[List[Argument], + CallableType, + List[AssignmentStmt]]: + """Parse a function signature (...) [-> t]. + + See parse_arg_list for an explanation of the final tuple item. + """ + lparen = self.expect('(') + + # Parse the argument list (everything within '(' and ')'). + args, extra_stmts = self.parse_arg_list(no_type_checks=no_type_checks) + + self.expect(')') + + if self.current_str() == '->': + self.skip() + if no_type_checks: + self.parse_expression() + ret_type = None + else: + ret_type = self.parse_type() + else: + ret_type = None + + arg_kinds = [arg.kind for arg in args] + self.verify_argument_kinds(arg_kinds, lparen.line) + + annotation = self.build_func_annotation( + ret_type, args, lparen.line) + + return args, annotation, extra_stmts + + def build_func_annotation(self, ret_type: Type, args: List[Argument], + line: int, is_default_ret: bool = False) -> CallableType: + arg_types = [arg.type_annotation for arg in args] + # Are there any type annotations? + if ((ret_type and not is_default_ret) + or arg_types != [None] * len(arg_types)): + # Yes. Construct a type for the function signature. + return self.construct_function_type(args, ret_type, line) + else: + return None + + def parse_arg_list(self, allow_signature: bool = True, + no_type_checks: bool=False) -> Tuple[List[Argument], + List[AssignmentStmt]]: + """Parse function definition argument list. + + This includes everything between '(' and ')' (but not the + parentheses). + + Return tuple (arguments, + extra statements for decomposing arguments). + + The final argument is only used for Python 2 argument lists with + tuples; they contain destructuring assignment statements used to + decompose tuple arguments. For example, consider a header like this: + + . def f((x, y)) + + The actual (sole) argument will be __tuple_arg_1 (a generated + name), whereas the extra statement list will contain a single + assignment statement corresponding to this assignment: + + x, y = __tuple_arg_1 + """ + args = [] # type: List[Argument] + extra_stmts = [] + # This is for checking duplicate argument names. + arg_names = [] # type: List[str] + has_tuple_arg = False + + require_named = False + bare_asterisk_before = -1 + + if self.current_str() != ')' and self.current_str() != ':': + while self.current_str() != ')': + if self.current_str() == '*' and self.peek().string == ',': + self.expect('*') + require_named = True + bare_asterisk_before = len(args) + elif self.current_str() in ['*', '**']: + if bare_asterisk_before == len(args): + # named arguments must follow bare * + self.parse_error() + + arg = self.parse_asterisk_arg( + allow_signature, + no_type_checks, + ) + args.append(arg) + require_named = True + elif self.current_str() == '(': + arg, extra_stmt, names = self.parse_tuple_arg(len(args)) + args.append(arg) + if extra_stmt is not None: + extra_stmts.append(extra_stmt) + has_tuple_arg = True + arg_names.extend(names) + else: + arg, require_named = self.parse_normal_arg( + require_named, + allow_signature, + no_type_checks, + ) + args.append(arg) + arg_names.append(arg.variable.name()) + + if self.current().string != ',': + break + + self.expect(',') + + # Non-tuple argument dupes will be checked elsewhere. Avoid + # generating duplicate errors. + if has_tuple_arg: + self.check_duplicate_argument_names(arg_names) + + return args, extra_stmts + + def check_duplicate_argument_names(self, names: List[str]) -> None: + found = set() # type: Set[str] + for name in names: + if name in found: + self.fail('Duplicate argument name "{}"'.format(name), + self.current().line) + found.add(name) + + def parse_asterisk_arg(self, + allow_signature: bool, + no_type_checks: bool) -> Argument: + asterisk = self.skip() + name = self.expect_type(Name) + variable = Var(name.string) + if asterisk.string == '*': + kind = nodes.ARG_STAR + else: + kind = nodes.ARG_STAR2 + + type = None + if no_type_checks: + self.parse_parameter_annotation() + else: + type = self.parse_arg_type(allow_signature) + + return Argument(variable, type, None, kind) + + def parse_tuple_arg(self, index: int) -> Tuple[Argument, AssignmentStmt, List[str]]: + """Parse a single Python 2 tuple argument. + + Example: def f(x, (y, z)): ... + + The tuple arguments gets transformed into an assignment in the + function body (the second return value). + + Return tuple (argument, decomposing assignment, list of names defined). + + Special case: if the argument is just (x) then it's not a tuple; + we indicate this by returning (argument, None, ['x']). + However, if the argument is (x,) then it *is* a (singleton) tuple. + """ + line = self.current().line + # Generate a new argument name that is very unlikely to clash with anything. + arg_name = '__tuple_arg_{}'.format(index + 1) + if self.pyversion[0] >= 3: + self.fail('Tuples in argument lists only supported in Python 2 mode', line) + paren_arg = self.parse_parentheses() + self.verify_tuple_arg(paren_arg) + if isinstance(paren_arg, NameExpr): + # This isn't a tuple. Revert to a normal argument. + arg_name = paren_arg.name + decompose = None + else: + rvalue = NameExpr(arg_name) + rvalue.set_line(line) + decompose = AssignmentStmt([paren_arg], rvalue) + decompose.set_line(line) + kind = nodes.ARG_POS + initializer = None + if self.current_str() == '=': + self.expect('=') + initializer = self.parse_expression(precedence[',']) + kind = nodes.ARG_OPT + var = Var(arg_name) + arg_names = self.find_tuple_arg_argument_names(paren_arg) + return Argument(var, None, initializer, kind), decompose, arg_names + + def verify_tuple_arg(self, paren_arg: Node) -> None: + if isinstance(paren_arg, TupleExpr): + if not paren_arg.items: + self.fail('Empty tuple not valid as an argument', paren_arg.line) + for item in paren_arg.items: + self.verify_tuple_arg(item) + elif not isinstance(paren_arg, NameExpr): + self.fail('Invalid item in tuple argument', paren_arg.line) + + def find_tuple_arg_argument_names(self, node: Node) -> List[str]: + result = [] # type: List[str] + if isinstance(node, TupleExpr): + for item in node.items: + result.extend(self.find_tuple_arg_argument_names(item)) + elif isinstance(node, NameExpr): + result.append(node.name) + return result + + def parse_normal_arg(self, require_named: bool, + allow_signature: bool, + no_type_checks: bool) -> Tuple[Argument, bool]: + name = self.expect_type(Name) + variable = Var(name.string) + + type = None + if no_type_checks: + self.parse_parameter_annotation() + else: + type = self.parse_arg_type(allow_signature) + + initializer = None # type: Node + if self.current_str() == '=': + self.expect('=') + initializer = self.parse_expression(precedence[',']) + if require_named: + kind = nodes.ARG_NAMED + else: + kind = nodes.ARG_OPT + else: + if require_named: + kind = nodes.ARG_NAMED + else: + kind = nodes.ARG_POS + + self.set_type_optional(type, initializer) + + return Argument(variable, type, initializer, kind), require_named + + def set_type_optional(self, type: Type, initializer: Node) -> None: + if not experiments.STRICT_OPTIONAL: + return + # Indicate that type should be wrapped in an Optional if arg is initialized to None. + optional = isinstance(initializer, NameExpr) and initializer.name == 'None' + if isinstance(type, UnboundType): + type.optional = optional + + def parse_parameter_annotation(self) -> Node: + if self.current_str() == ':': + self.skip() + return self.parse_expression(precedence[',']) + + def parse_arg_type(self, allow_signature: bool) -> Type: + if self.current_str() == ':' and allow_signature: + self.skip() + return self.parse_type() + else: + return None + + def verify_argument_kinds(self, kinds: List[int], line: int) -> None: + found = set() # type: Set[int] + for i, kind in enumerate(kinds): + if kind == nodes.ARG_POS and found & set([nodes.ARG_OPT, + nodes.ARG_STAR, + nodes.ARG_STAR2]): + self.fail('Invalid argument list', line) + elif kind == nodes.ARG_STAR and nodes.ARG_STAR in found: + self.fail('Invalid argument list', line) + elif kind == nodes.ARG_STAR2 and i != len(kinds) - 1: + self.fail('Invalid argument list', line) + found.add(kind) + + def construct_function_type(self, args: List[Argument], ret_type: Type, + line: int) -> CallableType: + # Complete the type annotation by replacing omitted types with 'Any'. + arg_types = [arg.type_annotation for arg in args] + for i in range(len(arg_types)): + if arg_types[i] is None: + arg_types[i] = AnyType(implicit=True) + if ret_type is None: + ret_type = AnyType(implicit=True) + arg_kinds = [arg.kind for arg in args] + arg_names = [arg.variable.name() for arg in args] + return CallableType(arg_types, arg_kinds, arg_names, ret_type, None, name=None, + variables=None, line=line) + + # Parsing statements + + def parse_block(self, allow_type: bool = False) -> Tuple[Block, Type]: + colon = self.expect(':') + if not isinstance(self.current(), Break): + # Block immediately after ':'. + nodes = [] + while True: + ind = self.ind + stmt, is_simple = self.parse_statement() + if not is_simple: + self.parse_error_at(self.tok[ind]) + break + nodes.append(stmt) + brk = self.expect_break() + if brk.string != ';': + break + node = Block(nodes) + node.set_line(colon) + return node, None + else: + # Indented block. + brk = self.expect_break() + type = self.parse_type_comment(brk, signature=True) + self.expect_indent() + stmt_list = [] # type: List[Node] + if allow_type: + cur = self.current() + if type is None and isinstance(cur, StrLit): + ds = docstring.parse_docstring(cur.parsed()) + if ds and False: # TODO: Enable when this is working. + try: + type = parse_str_as_signature(ds.as_type_str(), cur.line) + except TypeParseError: + # We don't require docstrings to be actually correct. + # TODO: Report something here. + type = None + while (not isinstance(self.current(), Dedent) and + not isinstance(self.current(), Eof)): + try: + stmt, is_simple = self.parse_statement() + if is_simple: + self.expect_break() + if stmt is not None: + if not self.try_combine_overloads(stmt, stmt_list): + stmt_list.append(stmt) + except ParseError: + pass + if isinstance(self.current(), Dedent): + self.skip() + node = Block(stmt_list) + node.set_line(colon) + return node, type + + def try_combine_overloads(self, s: Node, stmt: List[Node]) -> bool: + if isinstance(s, Decorator) and stmt: + fdef = s + n = fdef.func.name() + if isinstance(stmt[-1], Decorator) and stmt[-1].func.name() == n: + stmt[-1] = OverloadedFuncDef([stmt[-1], fdef]) + return True + elif isinstance(stmt[-1], OverloadedFuncDef) and stmt[-1].name() == n: + stmt[-1].items.append(fdef) + return True + return False + + def parse_statement(self) -> Tuple[Node, bool]: + stmt = None # type: Node + t = self.current() + ts = self.current_str() + is_simple = True # Is this a non-block statement? + if ts == 'if': + stmt = self.parse_if_stmt() + is_simple = False + elif ts == 'def': + stmt = self.parse_function() + is_simple = False + elif ts == 'while': + stmt = self.parse_while_stmt() + is_simple = False + elif ts == 'return': + stmt = self.parse_return_stmt() + elif ts == 'for': + stmt = self.parse_for_stmt() + is_simple = False + elif ts == 'try': + stmt = self.parse_try_stmt() + is_simple = False + elif ts == 'break': + stmt = self.parse_break_stmt() + elif ts == 'continue': + stmt = self.parse_continue_stmt() + elif ts == 'pass': + stmt = self.parse_pass_stmt() + elif ts == 'raise': + stmt = self.parse_raise_stmt() + elif ts == 'import': + stmt = self.parse_import() + elif ts == 'from': + stmt = self.parse_import_from() + elif ts == 'class': + stmt = self.parse_class_def() + is_simple = False + elif ts == 'global': + stmt = self.parse_global_decl() + elif ts == 'nonlocal' and self.pyversion[0] >= 3: + stmt = self.parse_nonlocal_decl() + elif ts == 'assert': + stmt = self.parse_assert_stmt() + elif ts == 'del': + stmt = self.parse_del_stmt() + elif ts == 'with': + stmt = self.parse_with_stmt() + is_simple = False + elif ts == '@': + stmt = self.parse_decorated_function_or_class() + is_simple = False + elif ts == 'print' and (self.pyversion[0] == 2 and + 'print_function' not in self.future_options): + stmt = self.parse_print_stmt() + elif ts == 'exec' and self.pyversion[0] == 2: + stmt = self.parse_exec_stmt() + else: + stmt = self.parse_expression_or_assignment() + if stmt is not None: + stmt.set_line(t) + return stmt, is_simple + + def parse_expression_or_assignment(self) -> Node: + expr = self.parse_expression(star_expr_allowed=True) + if self.current_str() == '=': + return self.parse_assignment(expr) + elif self.current_str() in op_assign: + # Operator assignment statement. + op = self.current_str()[:-1] + self.skip() + rvalue = self.parse_expression() + return OperatorAssignmentStmt(op, expr, rvalue) + else: + # Expression statement. + return ExpressionStmt(expr) + + def parse_assignment(self, lvalue: Any) -> Node: + """Parse an assignment statement. + + Assume that lvalue has been parsed already, and the current token is '='. + Also parse an optional '# type:' comment. + """ + self.expect('=') + lvalues = [lvalue] + expr = self.parse_expression(star_expr_allowed=True) + while self.current_str() == '=': + self.skip() + lvalues.append(expr) + expr = self.parse_expression(star_expr_allowed=True) + cur = self.current() + if isinstance(cur, Break): + type = self.parse_type_comment(cur, signature=False) + else: + type = None + return AssignmentStmt(lvalues, expr, type) + + def parse_return_stmt(self) -> ReturnStmt: + self.expect('return') + expr = None + current = self.current() + if current.string == 'yield': + self.parse_error() + if not isinstance(current, Break): + expr = self.parse_expression() + node = ReturnStmt(expr) + return node + + def parse_raise_stmt(self) -> RaiseStmt: + self.expect('raise') + expr = None + from_expr = None + if not isinstance(self.current(), Break): + expr = self.parse_expression() + if self.current_str() == 'from': + self.expect('from') + from_expr = self.parse_expression() + node = RaiseStmt(expr, from_expr) + return node + + def parse_assert_stmt(self) -> AssertStmt: + self.expect('assert') + expr = self.parse_expression() + node = AssertStmt(expr) + return node + + def parse_yield_or_yield_from_expr(self) -> Union[YieldFromExpr, YieldExpr]: + self.expect("yield") + expr = None + node = YieldExpr(expr) # type: Union[YieldFromExpr, YieldExpr] + if not isinstance(self.current(), Break): + if self.current_str() == "from": + self.expect("from") + expr = self.parse_expression() # when yield from is assigned to a variable + node = YieldFromExpr(expr) + else: + if self.current_str() == ')': + node = YieldExpr(None) + else: + expr = self.parse_expression() + node = YieldExpr(expr) + return node + + def parse_ellipsis(self) -> EllipsisExpr: + self.expect('...') + node = EllipsisExpr() + return node + + def parse_del_stmt(self) -> DelStmt: + self.expect('del') + expr = self.parse_expression() + node = DelStmt(expr) + return node + + def parse_break_stmt(self) -> BreakStmt: + self.expect('break') + node = BreakStmt() + return node + + def parse_continue_stmt(self) -> ContinueStmt: + self.expect('continue') + node = ContinueStmt() + return node + + def parse_pass_stmt(self) -> PassStmt: + self.expect('pass') + node = PassStmt() + return node + + def parse_global_decl(self) -> GlobalDecl: + self.expect('global') + names = self.parse_identifier_list() + node = GlobalDecl(names) + return node + + def parse_nonlocal_decl(self) -> NonlocalDecl: + self.expect('nonlocal') + names = self.parse_identifier_list() + node = NonlocalDecl(names) + return node + + def parse_identifier_list(self) -> List[str]: + names = [] + while True: + n = self.expect_type(Name) + names.append(n.string) + if self.current_str() != ',': + break + self.skip() + return names + + def parse_while_stmt(self) -> WhileStmt: + is_error = False + self.expect('while') + try: + expr = self.parse_expression() + except ParseError: + is_error = True + body, _ = self.parse_block() + if self.current_str() == 'else': + self.expect('else') + else_body, _ = self.parse_block() + else: + else_body = None + if is_error is not None: + node = WhileStmt(expr, body, else_body) + return node + else: + return None + + def parse_for_stmt(self) -> ForStmt: + self.expect('for') + index = self.parse_for_index_variables() + self.expect('in') + expr = self.parse_expression() + + body, _ = self.parse_block() + + if self.current_str() == 'else': + self.expect('else') + else_body, _ = self.parse_block() + else: + else_body = None + + node = ForStmt(index, expr, body, else_body) + return node + + def parse_for_index_variables(self) -> Node: + # Parse index variables of a 'for' statement. + index_items = [] + force_tuple = False + + while True: + v = self.parse_expression(precedence['in'], + star_expr_allowed=True) # Prevent parsing of for stmt 'in' + index_items.append(v) + if self.current_str() != ',': + break + self.skip() + if self.current_str() == 'in': + force_tuple = True + break + + if len(index_items) == 1 and not force_tuple: + index = index_items[0] + else: + index = TupleExpr(index_items) + index.set_line(index_items[0].get_line()) + + return index + + def parse_if_stmt(self) -> IfStmt: + is_error = False + + self.expect('if') + expr = [] + try: + expr.append(self.parse_expression()) + except ParseError: + is_error = True + + body = [self.parse_block()[0]] + + while self.current_str() == 'elif': + self.expect('elif') + try: + expr.append(self.parse_expression()) + except ParseError: + is_error = True + body.append(self.parse_block()[0]) + + if self.current_str() == 'else': + self.expect('else') + else_body, _ = self.parse_block() + else: + else_body = None + + if not is_error: + node = IfStmt(expr, body, else_body) + return node + else: + return None + + def parse_try_stmt(self) -> Node: + self.expect('try') + body, _ = self.parse_block() + is_error = False + vars = [] # type: List[NameExpr] + types = [] # type: List[Node] + handlers = [] # type: List[Block] + while self.current_str() == 'except': + self.expect('except') + if not isinstance(self.current(), Colon): + try: + t = self.current() + types.append(self.parse_expression(precedence[',']).set_line(t)) + if self.current_str() == 'as': + self.expect('as') + vars.append(self.parse_name_expr()) + elif self.pyversion[0] == 2 and self.current_str() == ',': + self.expect(',') + vars.append(self.parse_name_expr()) + else: + vars.append(None) + except ParseError: + is_error = True + else: + types.append(None) + vars.append(None) + handlers.append(self.parse_block()[0]) + if not is_error: + if self.current_str() == 'else': + self.skip() + else_body, _ = self.parse_block() + else: + else_body = None + if self.current_str() == 'finally': + self.expect('finally') + finally_body, _ = self.parse_block() + else: + finally_body = None + node = TryStmt(body, vars, types, handlers, else_body, + finally_body) + return node + else: + return None + + def parse_with_stmt(self) -> WithStmt: + self.expect('with') + exprs = [] + targets = [] + while True: + expr = self.parse_expression(precedence[',']) + if self.current_str() == 'as': + self.expect('as') + target = self.parse_expression(precedence[',']) + else: + target = None + exprs.append(expr) + targets.append(target) + if self.current_str() != ',': + break + self.expect(',') + body, _ = self.parse_block() + return WithStmt(exprs, targets, body) + + def parse_print_stmt(self) -> PrintStmt: + self.expect('print') + args = [] + target = None + if self.current_str() == '>>': + self.skip() + target = self.parse_expression(precedence[',']) + if self.current_str() == ',': + self.skip() + if isinstance(self.current(), Break): + self.parse_error() + else: + if not isinstance(self.current(), Break): + self.parse_error() + comma = False + while not isinstance(self.current(), Break): + args.append(self.parse_expression(precedence[','])) + if self.current_str() == ',': + comma = True + self.skip() + else: + comma = False + break + return PrintStmt(args, newline=not comma, target=target) + + def parse_exec_stmt(self) -> ExecStmt: + self.expect('exec') + expr = self.parse_expression(precedence['in']) + variables1 = None + variables2 = None + if self.current_str() == 'in': + self.skip() + variables1 = self.parse_expression(precedence[',']) + if self.current_str() == ',': + self.skip() + variables2 = self.parse_expression(precedence[',']) + return ExecStmt(expr, variables1, variables2) + + # Parsing expressions + + def parse_expression(self, prec: int = 0, star_expr_allowed: bool = False) -> Node: + """Parse a subexpression within a specific precedence context.""" + expr = None # type: Node + current = self.current() # Remember token for setting the line number. + + # Parse a "value" expression or unary operator expression and store + # that in expr. + s = self.current_str() + if s == '(': + # Parenthesised expression or cast. + expr = self.parse_parentheses() + elif s == '[': + expr = self.parse_list_expr() + elif s in ['-', '+', 'not', '~']: + # Unary operation. + expr = self.parse_unary_expr() + elif s == 'lambda': + expr = self.parse_lambda_expr() + elif s == '{': + expr = self.parse_dict_or_set_expr() + elif s == '*' and star_expr_allowed: + expr = self.parse_star_expr() + elif s == '`' and self.pyversion[0] == 2: + expr = self.parse_backquote_expr() + else: + if isinstance(current, Name): + # Name expression. + expr = self.parse_name_expr() + elif isinstance(current, IntLit): + expr = self.parse_int_expr() + elif isinstance(current, StrLit): + expr = self.parse_str_expr() + elif isinstance(current, BytesLit): + expr = self.parse_bytes_literal() + elif isinstance(current, UnicodeLit): + expr = self.parse_unicode_literal() + elif isinstance(current, FloatLit): + expr = self.parse_float_expr() + elif isinstance(current, ComplexLit): + expr = self.parse_complex_expr() + elif isinstance(current, Keyword) and s == "yield": + # The expression yield from and yield to assign + expr = self.parse_yield_or_yield_from_expr() + elif isinstance(current, EllipsisToken) and (self.pyversion[0] >= 3 + or self.is_stub_file): + expr = self.parse_ellipsis() + else: + # Invalid expression. + self.parse_error() + + # Set the line of the expression node, if not specified. This + # simplifies recording the line number as not every node type needs to + # deal with it separately. + if expr.line < 0: + expr.set_line(current) + + # Parse operations that require a left argument (stored in expr). + while True: + current = self.current() + s = self.current_str() + if s == '(': + # Call expression. + expr = self.parse_call_expr(expr) + elif s == '.': + # Member access expression. + expr = self.parse_member_expr(expr) + elif s == '[': + # Indexing expression. + expr = self.parse_index_expr(expr) + elif s == ',': + # The comma operator is used to build tuples. Comma also + # separates array items and function arguments, but in this + # case the precedence is too low to build a tuple. + if precedence[','] > prec: + expr = self.parse_tuple_expr(expr) + else: + break + elif s == 'for': + if precedence[''] > prec: + # List comprehension or generator expression. Parse as + # generator expression; it will be converted to list + # comprehension if needed elsewhere. + expr = self.parse_generator_expr(expr) + else: + break + elif s == 'if': + # Conditional expression. + if precedence[''] > prec: + expr = self.parse_conditional_expr(expr) + else: + break + else: + # Binary operation or a special case. + if isinstance(current, Op): + op = self.current_str() + op_prec = precedence[op] + if op == 'not': + # Either "not in" or an error. + op_prec = precedence['in'] + if op_prec > prec: + if op in op_comp: + expr = self.parse_comparison_expr(expr, op_prec) + else: + expr = self.parse_bin_op_expr(expr, op_prec) + else: + # The operation cannot be associated with the + # current left operand due to the precedence + # context; let the caller handle it. + break + else: + # Not an operation that accepts a left argument; let the + # caller handle the rest. + break + + # Set the line of the expression node, if not specified. This + # simplifies recording the line number as not every node type + # needs to deal with it separately. + if expr.line < 0: + expr.set_line(current) + + return expr + + def parse_parentheses(self) -> Node: + self.skip() + if self.current_str() == ')': + # Empty tuple (). + expr = self.parse_empty_tuple_expr() # type: Node + else: + # Parenthesised expression. + expr = self.parse_expression(0, star_expr_allowed=True) + self.expect(')') + return expr + + def parse_star_expr(self) -> Node: + star = self.expect('*') + expr = self.parse_expression(precedence['*u']) + expr = StarExpr(expr) + if expr.line < 0: + expr.set_line(star) + return expr + + def parse_empty_tuple_expr(self) -> TupleExpr: + self.expect(')') + node = TupleExpr([]) + return node + + def parse_list_expr(self) -> Node: + """Parse list literal or list comprehension.""" + items = [] + self.expect('[') + while self.current_str() != ']' and not self.eol(): + items.append(self.parse_expression(precedence[''], star_expr_allowed=True)) + if self.current_str() != ',': + break + self.expect(',') + if self.current_str() == 'for' and len(items) == 1: + items[0] = self.parse_generator_expr(items[0]) + self.expect(']') + if len(items) == 1 and isinstance(items[0], GeneratorExpr): + return ListComprehension(items[0]) + else: + expr = ListExpr(items) + return expr + + def parse_generator_expr(self, left_expr: Node) -> GeneratorExpr: + tok = self.current() + indices, sequences, condlists = self.parse_comp_for() + + gen = GeneratorExpr(left_expr, indices, sequences, condlists) + gen.set_line(tok) + return gen + + def parse_comp_for(self) -> Tuple[List[Node], List[Node], List[List[Node]]]: + indices = [] + sequences = [] + condlists = [] # type: List[List[Node]] + while self.current_str() == 'for': + conds = [] + self.expect('for') + index = self.parse_for_index_variables() + indices.append(index) + self.expect('in') + if self.pyversion[0] >= 3: + sequence = self.parse_expression(precedence['']) + else: + sequence = self.parse_expression_list() + sequences.append(sequence) + while self.current_str() == 'if': + self.skip() + conds.append(self.parse_expression(precedence[''])) + condlists.append(conds) + + return indices, sequences, condlists + + def parse_expression_list(self) -> Node: + prec = precedence[''] + expr = self.parse_expression(prec) + if self.current_str() != ',': + return expr + else: + t = self.current() + return self.parse_tuple_expr(expr, prec).set_line(t) + + def parse_conditional_expr(self, left_expr: Node) -> ConditionalExpr: + self.expect('if') + cond = self.parse_expression(precedence['']) + self.expect('else') + else_expr = self.parse_expression(precedence['']) + return ConditionalExpr(cond, left_expr, else_expr) + + def parse_dict_or_set_expr(self) -> Node: + items = [] # type: List[Tuple[Node, Node]] + self.expect('{') + while self.current_str() != '}' and not self.eol(): + key = self.parse_expression(precedence['']) + if self.current_str() in [',', '}'] and items == []: + return self.parse_set_expr(key) + elif self.current_str() == 'for' and items == []: + return self.parse_set_comprehension(key) + elif self.current_str() != ':': + self.parse_error() + colon = self.expect(':') + value = self.parse_expression(precedence['']) + if self.current_str() == 'for' and items == []: + return self.parse_dict_comprehension(key, value, colon) + items.append((key, value)) + if self.current_str() != ',': + break + self.expect(',') + self.expect('}') + node = DictExpr(items) + return node + + def parse_set_expr(self, first: Node) -> SetExpr: + items = [first] + while self.current_str() != '}' and not self.eol(): + self.expect(',') + if self.current_str() == '}': + break + items.append(self.parse_expression(precedence[','])) + self.expect('}') + expr = SetExpr(items) + return expr + + def parse_set_comprehension(self, expr: Node): + gen = self.parse_generator_expr(expr) + self.expect('}') + set_comp = SetComprehension(gen) + return set_comp + + def parse_dict_comprehension(self, key: Node, value: Node, + colon: Token) -> DictionaryComprehension: + indices, sequences, condlists = self.parse_comp_for() + dic = DictionaryComprehension(key, value, indices, sequences, condlists) + dic.set_line(colon) + self.expect('}') + return dic + + def parse_tuple_expr(self, expr: Node, + prec: int = precedence[',']) -> TupleExpr: + items = [expr] + while True: + self.expect(',') + if (self.current_str() in [')', ']', '=', ':'] or + isinstance(self.current(), Break)): + break + items.append(self.parse_expression(prec, star_expr_allowed=True)) + if self.current_str() != ',': break + node = TupleExpr(items) + return node + + def parse_backquote_expr(self) -> BackquoteExpr: + self.expect('`') + expr = self.parse_expression() + self.expect('`') + return BackquoteExpr(expr) + + def parse_name_expr(self) -> NameExpr: + tok = self.expect_type(Name) + node = NameExpr(tok.string) + node.set_line(tok) + return node + + octal_int = re.compile('0+[1-9]') + + def parse_int_expr(self) -> IntExpr: + tok = self.expect_type(IntLit) + string = tok.string.rstrip('lL') # Strip L prefix (Python 2 long literals) + if self.octal_int.match(string): + value = int(string, 8) + else: + value = int(string, 0) + node = IntExpr(value) + return node + + def parse_str_expr(self) -> Node: + # XXX \uxxxx literals + token = self.expect_type(StrLit) + value = cast(StrLit, token).parsed() + is_unicode = False + while isinstance(self.current(), (StrLit, UnicodeLit)): + token = self.skip() + if isinstance(token, StrLit): + value += token.parsed() + elif isinstance(token, UnicodeLit): + value += token.parsed() + is_unicode = True + if is_unicode or (self.pyversion[0] == 2 and 'unicode_literals' in self.future_options): + node = UnicodeExpr(value) # type: Node + else: + node = StrExpr(value) + return node + + def parse_bytes_literal(self) -> Node: + # XXX \uxxxx literals + tok = [self.expect_type(BytesLit)] + value = (cast(BytesLit, tok[0])).parsed() + while isinstance(self.current(), BytesLit): + t = cast(BytesLit, self.skip()) + value += t.parsed() + if self.pyversion[0] >= 3: + node = BytesExpr(value) # type: Node + else: + node = StrExpr(value) + return node + + def parse_unicode_literal(self) -> Node: + # XXX \uxxxx literals + token = self.expect_type(UnicodeLit) + value = cast(UnicodeLit, token).parsed() + while isinstance(self.current(), (UnicodeLit, StrLit)): + token = cast(Union[UnicodeLit, StrLit], self.skip()) + value += token.parsed() + if self.pyversion[0] >= 3: + # Python 3.3 supports u'...' as an alias of '...'. + node = StrExpr(value) # type: Node + else: + node = UnicodeExpr(value) + return node + + def parse_float_expr(self) -> FloatExpr: + tok = self.expect_type(FloatLit) + node = FloatExpr(float(tok.string)) + return node + + def parse_complex_expr(self) -> ComplexExpr: + tok = self.expect_type(ComplexLit) + node = ComplexExpr(complex(tok.string)) + return node + + def parse_call_expr(self, callee: Any) -> CallExpr: + self.expect('(') + args, kinds, names = self.parse_arg_expr() + self.expect(')') + node = CallExpr(callee, args, kinds, names) + return node + + def parse_arg_expr(self) -> Tuple[List[Node], List[int], List[str]]: + """Parse arguments in a call expression (within '(' and ')'). + + Return a tuple with these items: + argument expressions + argument kinds + argument names (for named arguments; None for ordinary args) + """ + args = [] # type: List[Node] + kinds = [] # type: List[int] + names = [] # type: List[str] + var_arg = False + dict_arg = False + named_args = False + while self.current_str() != ')' and not self.eol() and not dict_arg: + if isinstance(self.current(), Name) and self.peek().string == '=': + # Named argument + name = self.expect_type(Name) + self.expect('=') + kinds.append(nodes.ARG_NAMED) + names.append(name.string) + named_args = True + elif (self.current_str() == '*' and not var_arg and not dict_arg): + # *args + var_arg = True + self.expect('*') + kinds.append(nodes.ARG_STAR) + names.append(None) + elif self.current_str() == '**': + # **kwargs + self.expect('**') + dict_arg = True + kinds.append(nodes.ARG_STAR2) + names.append(None) + elif not var_arg and not named_args: + # Ordinary argument + kinds.append(nodes.ARG_POS) + names.append(None) + else: + self.parse_error() + args.append(self.parse_expression(precedence[','])) + if self.current_str() != ',': + break + self.expect(',') + return args, kinds, names + + def parse_member_expr(self, expr: Any) -> Node: + self.expect('.') + name = self.expect_type(Name) + if (isinstance(expr, CallExpr) and isinstance(expr.callee, NameExpr) + and expr.callee.name == 'super'): + # super() expression + node = SuperExpr(name.string) # type: Node + else: + node = MemberExpr(expr, name.string) + return node + + def parse_index_expr(self, base: Any) -> IndexExpr: + self.expect('[') + index = self.parse_slice_item() + if self.current_str() == ',': + # Extended slicing such as x[1:, :2]. + items = [index] + while self.current_str() == ',': + self.skip() + if self.current_str() == ']' or isinstance(self.current(), Break): + break + items.append(self.parse_slice_item()) + index = TupleExpr(items) + index.set_line(items[0].line) + self.expect(']') + node = IndexExpr(base, index) + return node + + def parse_slice_item(self) -> Node: + if self.current_str() != ':': + if self.current_str() == '...': + # Ellipsis is valid here even in Python 2 (but not elsewhere). + ellipsis = EllipsisExpr() + token = self.skip() + ellipsis.set_line(token) + return ellipsis + else: + item = self.parse_expression(precedence[',']) + else: + item = None + if self.current_str() == ':': + # Slice. + index = item + colon = self.expect(':') + if self.current_str() not in (']', ':', ','): + end_index = self.parse_expression(precedence[',']) + else: + end_index = None + stride = None + if self.current_str() == ':': + self.expect(':') + if self.current_str() not in (']', ','): + stride = self.parse_expression(precedence[',']) + item = SliceExpr(index, end_index, stride).set_line(colon.line) + return item + + def parse_bin_op_expr(self, left: Node, prec: int) -> OpExpr: + op = self.expect_type(Op) + op_str = op.string + if op_str == '~': + self.ind -= 1 + self.parse_error() + right = self.parse_expression(prec) + node = OpExpr(op_str, left, right) + return node + + def parse_comparison_expr(self, left: Node, prec: int) -> ComparisonExpr: + operators_str = [] + operands = [left] + + while True: + op = self.expect_type(Op) + op_str = op.string + if op_str == 'not': + if self.current_str() == 'in': + op_str = 'not in' + self.skip() + else: + self.parse_error() + elif op_str == 'is' and self.current_str() == 'not': + op_str = 'is not' + self.skip() + + operators_str.append(op_str) + operand = self.parse_expression(prec) + operands.append(operand) + + # Continue if next token is a comparison operator + self.current() + s = self.current_str() + if s not in op_comp: + break + + node = ComparisonExpr(operators_str, operands) + return node + + def parse_unary_expr(self) -> UnaryExpr: + op_tok = self.skip() + op = op_tok.string + if op == '-' or op == '+': + prec = precedence['-u'] + else: + prec = precedence[op] + expr = self.parse_expression(prec) + node = UnaryExpr(op, expr) + return node + + def parse_lambda_expr(self) -> FuncExpr: + lambda_tok = self.expect('lambda') + + args, extra_stmts = self.parse_arg_list(allow_signature=False) + + # Use 'object' as the placeholder return type; it will be inferred + # later. We can't use 'Any' since it could make type inference results + # less precise. + ret_type = UnboundType('__builtins__.object') + typ = self.build_func_annotation(ret_type, args, + lambda_tok.line, is_default_ret=True) + + colon = self.expect(':') + + expr = self.parse_expression(precedence[',']) + + nodes = [ReturnStmt(expr).set_line(lambda_tok)] + # Potentially insert extra assignment statements to the beginning of the + # body, used to decompose Python 2 tuple arguments. + nodes[:0] = extra_stmts + body = Block(nodes) + body.set_line(colon) + + return FuncExpr(args, body, typ) + + # Helper methods + + def skip(self) -> Token: + self.ind += 1 + return self.tok[self.ind - 1] + + def expect(self, string: str) -> Token: + if self.current_str() == string: + self.ind += 1 + return self.tok[self.ind - 1] + else: + self.parse_error() + + def expect_indent(self) -> Token: + if isinstance(self.current(), Indent): + return self.expect_type(Indent) + else: + self.fail('Expected an indented block', self.current().line) + return none + + def fail(self, msg: str, line: int) -> None: + self.errors.report(line, msg) + + def expect_type(self, typ: type) -> Token: + current = self.current() + if isinstance(current, typ): + self.ind += 1 + return current + else: + self.parse_error() + + def expect_colon_and_break(self) -> Tuple[Token, Token]: + return self.expect_type(Colon), self.expect_type(Break) + + def expect_break(self) -> Token: + return self.expect_type(Break) + + def current(self) -> Token: + return self.tok[self.ind] + + def current_str(self) -> str: + return self.current().string + + def peek(self) -> Token: + return self.tok[self.ind + 1] + + def parse_error(self) -> None: + self.parse_error_at(self.current()) + raise ParseError() + + def parse_error_at(self, tok: Token, skip: bool = True, reason: Optional[str] = None) -> None: + msg = '' + if isinstance(tok, LexError): + msg = token_repr(tok) + msg = msg[0].upper() + msg[1:] + elif isinstance(tok, Indent) or isinstance(tok, Dedent): + msg = 'Inconsistent indentation' + else: + formatted_reason = ": {}".format(reason) if reason else "" + msg = 'Parse error before {}{}'.format(token_repr(tok), formatted_reason) + + self.errors.report(tok.line, msg) + + if skip: + self.skip_until_next_line() + + def skip_until_break(self) -> None: + n = 0 + while (not isinstance(self.current(), Break) + and not isinstance(self.current(), Eof)): + self.skip() + n += 1 + if isinstance(self.tok[self.ind - 1], Colon) and n > 1: + self.ind -= 1 + + def skip_until_next_line(self) -> None: + self.skip_until_break() + if isinstance(self.current(), Break): + self.skip() + + def eol(self) -> bool: + return isinstance(self.current(), Break) or self.eof() + + def eof(self) -> bool: + return isinstance(self.current(), Eof) + + # Type annotation related functionality + + def parse_type(self) -> Type: + try: + typ, self.ind = parse_type(self.tok, self.ind) + except TypeParseError as e: + self.parse_error_at(e.token, reason=e.message) + raise ParseError() + return typ + + annotation_prefix_re = re.compile(r'#\s*type:') + ignore_prefix_re = re.compile(r'ignore\b') + + def parse_type_comment(self, token: Token, signature: bool) -> Type: + """Parse a '# type: ...' annotation. + + Return None if no annotation found. If signature is True, expect + a type signature of form (...) -> t. + """ + whitespace_or_comments = token.rep().strip() + if self.annotation_prefix_re.match(whitespace_or_comments): + type_as_str = whitespace_or_comments.split(':', 1)[1].strip() + if self.ignore_prefix_re.match(type_as_str): + # Actually a "# type: ignore" annotation -> not a type. + return None + tokens = lex.lex(type_as_str, token.line)[0] + if len(tokens) < 2: + # Empty annotation (only Eof token) + self.errors.report(token.line, 'Empty type annotation') + return None + try: + if not signature: + type, index = parse_types(tokens, 0) + else: + type, index = parse_signature(tokens) + except TypeParseError as e: + self.parse_error_at(e.token, skip=False, reason=e.message) + return None + if index < len(tokens) - 2: + self.parse_error_at(tokens[index], skip=False) + return None + return type + else: + return None + + +class ParseError(Exception): pass + + +def token_repr(tok: Token) -> str: + """Return a representation of a token for use in parse error messages.""" + if isinstance(tok, Break): + return 'end of line' + elif isinstance(tok, Eof): + return 'end of file' + elif isinstance(tok, Keyword) or isinstance(tok, Name): + return '"{}"'.format(tok.string) + elif isinstance(tok, IntLit) or isinstance(tok, FloatLit) or isinstance(tok, ComplexLit): + return 'numeric literal' + elif isinstance(tok, StrLit) or isinstance(tok, UnicodeLit): + return 'string literal' + elif (isinstance(tok, Punct) or isinstance(tok, Op) + or isinstance(tok, Colon)): + return tok.string + elif isinstance(tok, Bom): + return 'byte order mark' + elif isinstance(tok, Indent): + return 'indent' + elif isinstance(tok, Dedent): + return 'dedent' + elif isinstance(tok, EllipsisToken): + return '...' + else: + if isinstance(tok, LexError): + t = tok.type + if t == lex.NUMERIC_LITERAL_ERROR: + return 'invalid numeric literal' + elif t == lex.UNTERMINATED_STRING_LITERAL: + return 'unterminated string literal' + elif t == lex.INVALID_CHARACTER: + msg = 'unrecognized character' + if ord(tok.string) in range(33, 127): + msg += ' ' + tok.string + return msg + elif t == lex.INVALID_DEDENT: + return 'inconsistent indentation' + elif t == lex.DECODE_ERROR: + return tok.message + raise ValueError('Unknown token {}'.format(repr(tok))) + + +if __name__ == '__main__': + # Parse a file and dump the AST (or display errors). + import sys + + def usage(): + print('Usage: parse.py [--py2] [--quiet] FILE [...]', file=sys.stderr) + sys.exit(2) + + args = sys.argv[1:] + pyversion = defaults.PYTHON3_VERSION + quiet = False + while args and args[0].startswith('--'): + if args[0] == '--py2': + pyversion = defaults.PYTHON2_VERSION + elif args[0] == '--quiet': + quiet = True + else: + usage() + args = args[1:] + if len(args) < 1: + usage() + status = 0 + for fnam in args: + s = open(fnam, 'rb').read() + errors = Errors() + try: + options = Options() + options.python_version = pyversion + tree = parse(s, fnam, None, options=options) + if not quiet: + print(tree) + except CompileError as e: + for msg in e.messages: + sys.stderr.write('%s\n' % msg) + status = 1 + sys.exit(status) diff --git a/mypy/semanal.py b/mypy/semanal.py index 8ab29f560a4e7..0f4331499d7cc 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1,2839 +1,2839 @@ -"""The semantic analyzer. - -Bind names to definitions and do various other simple consistency -checks. For example, consider this program: - - x = 1 - y = x - -Here semantic analysis would detect that the assignment 'x = 1' -defines a new variable, the type of which is to be inferred (in a -later pass; type inference or type checking is not part of semantic -analysis). Also, it would bind both references to 'x' to the same -module-level variable node. The second assignment would also be -analyzed, and the type of 'y' marked as being inferred. - -Semantic analysis is the first analysis pass after parsing, and it is -subdivided into three passes: - - * FirstPass looks up externally visible names defined in a module but - ignores imports and local definitions. It helps enable (some) - cyclic references between modules, such as module 'a' that imports - module 'b' and used names defined in b *and* vice versa. The first - pass can be performed before dependent modules have been processed. - - * SemanticAnalyzer is the second pass. It does the bulk of the work. - It assumes that dependent modules have been semantically analyzed, - up to the second pass, unless there is a import cycle. - - * ThirdPass checks that type argument counts are valid; for example, - it will reject Dict[int]. We don't do this in the second pass, - since we infer the type argument counts of classes during this - pass, and it is possible to refer to classes defined later in a - file, which would not have the type argument count set yet. This - pass also recomputes the method resolution order of each class, in - case one of its bases belongs to a module involved in an import - loop. - -Semantic analysis of types is implemented in module mypy.typeanal. - -TODO: Check if the third pass slows down type checking significantly. - We could probably get rid of it -- for example, we could collect all - analyzed types in a collection and check them without having to - traverse the entire AST. -""" - -from typing import ( - List, Dict, Set, Tuple, cast, Any, overload, TypeVar, Union, Optional, Callable -) - -from mypy.nodes import ( - MypyFile, TypeInfo, Node, AssignmentStmt, FuncDef, OverloadedFuncDef, - ClassDef, Var, GDEF, MODULE_REF, FuncItem, Import, - ImportFrom, ImportAll, Block, LDEF, NameExpr, MemberExpr, - IndexExpr, TupleExpr, ListExpr, ExpressionStmt, ReturnStmt, - RaiseStmt, AssertStmt, OperatorAssignmentStmt, WhileStmt, - ForStmt, BreakStmt, ContinueStmt, IfStmt, TryStmt, WithStmt, DelStmt, - GlobalDecl, SuperExpr, DictExpr, CallExpr, RefExpr, OpExpr, UnaryExpr, - SliceExpr, CastExpr, RevealTypeExpr, TypeApplication, Context, SymbolTable, - SymbolTableNode, BOUND_TVAR, UNBOUND_TVAR, ListComprehension, GeneratorExpr, - FuncExpr, MDEF, FuncBase, Decorator, SetExpr, TypeVarExpr, - StrExpr, BytesExpr, PrintStmt, ConditionalExpr, PromoteExpr, - ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, MroError, type_aliases, - YieldFromExpr, NamedTupleExpr, NonlocalDecl, - SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr, - YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, COVARIANT, CONTRAVARIANT, - IntExpr, FloatExpr, UnicodeExpr, - INVARIANT, UNBOUND_IMPORTED, Expression, EllipsisExpr, NamedTupleTypeInfo -) -from mypy.visitor import NodeVisitor -from mypy.traverser import TraverserVisitor -from mypy.errors import Errors, report_internal_error -from mypy.types import ( - NoneTyp, CallableType, Overloaded, Instance, Type, TypeVarType, AnyType, - FunctionLike, UnboundType, TypeList, ErrorType, TypeVarDef, - replace_leading_arg_type, TupleType, UnionType, StarType, EllipsisType, - NamedTupleType -) -from mypy.nodes import function_type, implicit_module_attrs -from mypy.typeanal import TypeAnalyser, TypeAnalyserPass3, analyze_type_alias -from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError -from mypy.lex import lex -from mypy.parsetype import parse_type -from mypy.sametypes import is_same_type -from mypy.erasetype import erase_typevars -from mypy import defaults -from mypy.options import Options - - -T = TypeVar('T') - - -# Inferred value of an expression. -ALWAYS_TRUE = 0 -ALWAYS_FALSE = 1 -TRUTH_VALUE_UNKNOWN = 2 - -# Map from obsolete name to the current spelling. -obsolete_name_mapping = { - 'typing.Function': 'typing.Callable', - 'typing.typevar': 'typing.TypeVar', -} - -# Hard coded type promotions (shared between all Python versions). -# These add extra ad-hoc edges to the subtyping relation. For example, -# int is considered a subtype of float, even though there is no -# subclass relationship. -TYPE_PROMOTIONS = { - 'builtins.int': 'builtins.float', - 'builtins.float': 'builtins.complex', -} - -# Hard coded type promotions for Python 3. -# -# Note that the bytearray -> bytes promotion is a little unsafe -# as some functions only accept bytes objects. Here convenience -# trumps safety. -TYPE_PROMOTIONS_PYTHON3 = TYPE_PROMOTIONS.copy() -TYPE_PROMOTIONS_PYTHON3.update({ - 'builtins.bytearray': 'builtins.bytes', -}) - -# Hard coded type promotions for Python 2. -# -# These promotions are unsafe, but we are doing them anyway -# for convenience and also for Python 3 compatibility -# (bytearray -> str). -TYPE_PROMOTIONS_PYTHON2 = TYPE_PROMOTIONS.copy() -TYPE_PROMOTIONS_PYTHON2.update({ - 'builtins.str': 'builtins.unicode', - 'builtins.bytearray': 'builtins.str', -}) - -# When analyzing a function, should we analyze the whole function in one go, or -# should we only perform one phase of the analysis? The latter is used for -# nested functions. In the first phase we add the function to the symbol table -# but don't process body. In the second phase we process function body. This -# way we can have mutually recursive nested functions. -FUNCTION_BOTH_PHASES = 0 # Everthing in one go -FUNCTION_FIRST_PHASE_POSTPONE_SECOND = 1 # Add to symbol table but postpone body -FUNCTION_SECOND_PHASE = 2 # Only analyze body - - -class SemanticAnalyzer(NodeVisitor): - """Semantically analyze parsed mypy files. - - The analyzer binds names and does various consistency checks for a - parse tree. Note that type checking is performed as a separate - pass. - - This is the second phase of semantic analysis. - """ - - # Library search paths - lib_path = None # type: List[str] - # Module name space - modules = None # type: Dict[str, MypyFile] - # Global name space for current module - globals = None # type: SymbolTable - # Names declared using "global" (separate set for each scope) - global_decls = None # type: List[Set[str]] - # Names declated using "nonlocal" (separate set for each scope) - nonlocal_decls = None # type: List[Set[str]] - # Local names of function scopes; None for non-function scopes. - locals = None # type: List[SymbolTable] - # Nested block depths of scopes - block_depth = None # type: List[int] - # TypeInfo of directly enclosing class (or None) - type = None # type: TypeInfo - # Stack of outer classes (the second tuple item contains tvars). - type_stack = None # type: List[TypeInfo] - # Type variables that are bound by the directly enclosing class - bound_tvars = None # type: List[SymbolTableNode] - # Stack of type variables that were bound by outer classess - tvar_stack = None # type: List[List[SymbolTableNode]] - # Do weak type checking in this file - weak_opts = set() # type: Set[str] - - # Stack of functions being analyzed - function_stack = None # type: List[FuncItem] - # Stack of next available function type variable ids - next_function_tvar_id_stack = None # type: List[int] - - # Status of postponing analysis of nested function bodies. By using this we - # can have mutually recursive nested functions. Values are FUNCTION_x - # constants. Note that separate phasea are not used for methods. - postpone_nested_functions_stack = None # type: List[int] - # Postponed functions collected if - # postpone_nested_functions_stack[-1] == FUNCTION_FIRST_PHASE_POSTPONE_SECOND. - postponed_functions_stack = None # type: List[List[Node]] - - loop_depth = 0 # Depth of breakable loops - cur_mod_id = '' # Current module id (or None) (phase 2) - is_stub_file = False # Are we analyzing a stub file? - imports = None # type: Set[str] # Imported modules (during phase 2 analysis) - errors = None # type: Errors # Keeps track of generated errors - - def __init__(self, - lib_path: List[str], - errors: Errors, - options: Options) -> None: - """Construct semantic analyzer. - - Use lib_path to search for modules, and report analysis errors - using the Errors instance. - """ - self.locals = [None] - self.imports = set() - self.type = None - self.type_stack = [] - self.bound_tvars = None - self.tvar_stack = [] - self.function_stack = [] - self.next_function_tvar_id_stack = [-1] - self.block_depth = [0] - self.loop_depth = 0 - self.lib_path = lib_path - self.errors = errors - self.modules = {} - self.options = options - self.postpone_nested_functions_stack = [FUNCTION_BOTH_PHASES] - self.postponed_functions_stack = [] - self.all_exports = set() # type: Set[str] - - def visit_file(self, file_node: MypyFile, fnam: str) -> None: - self.errors.set_file(fnam) - self.cur_mod_node = file_node - self.cur_mod_id = file_node.fullname() - self.is_stub_file = fnam.lower().endswith('.pyi') - self.globals = file_node.names - self.weak_opts = file_node.weak_opts - - if 'builtins' in self.modules: - self.globals['__builtins__'] = SymbolTableNode( - MODULE_REF, self.modules['builtins'], self.cur_mod_id) - - for name in implicit_module_attrs: - v = self.globals[name].node - if isinstance(v, Var): - v.type = self.anal_type(v.type) - v.is_ready = True - - defs = file_node.defs - for d in defs: - self.accept(d) - - if self.cur_mod_id == 'builtins': - remove_imported_names_from_symtable(self.globals, 'builtins') - - if '__all__' in self.globals: - for name, g in self.globals.items(): - if name not in self.all_exports: - g.module_public = False - - def visit_func_def(self, defn: FuncDef) -> None: - phase_info = self.postpone_nested_functions_stack[-1] - if phase_info != FUNCTION_SECOND_PHASE: - self.function_stack.append(defn) - # First phase of analysis for function. - self.errors.push_function(defn.name()) - self.update_function_type_variables(defn) - self.errors.pop_function() - self.function_stack.pop() - - defn.is_conditional = self.block_depth[-1] > 0 - - # TODO(jukka): Figure out how to share the various cases. It doesn't - # make sense to have (almost) duplicate code (here and elsewhere) for - # 3 cases: module-level, class-level and local names. Maybe implement - # a common stack of namespaces. As the 3 kinds of namespaces have - # different semantics, this wouldn't always work, but it might still - # be a win. - if self.is_class_scope(): - # Method definition - defn.info = self.type - if not defn.is_decorated and not defn.is_overload: - if defn.name() in self.type.names: - # Redefinition. Conditional redefinition is okay. - n = self.type.names[defn.name()].node - if self.is_conditional_func(n, defn): - defn.original_def = cast(FuncDef, n) - else: - self.name_already_defined(defn.name(), defn) - self.type.names[defn.name()] = SymbolTableNode(MDEF, defn) - self.prepare_method_signature(defn) - elif self.is_func_scope(): - # Nested function - if not defn.is_decorated and not defn.is_overload: - if defn.name() in self.locals[-1]: - # Redefinition. Conditional redefinition is okay. - n = self.locals[-1][defn.name()].node - if self.is_conditional_func(n, defn): - defn.original_def = cast(FuncDef, n) - else: - self.name_already_defined(defn.name(), defn) - else: - self.add_local(defn, defn) - else: - # Top-level function - if not defn.is_decorated and not defn.is_overload: - symbol = self.globals.get(defn.name()) - if isinstance(symbol.node, FuncDef) and symbol.node != defn: - # This is redefinition. Conditional redefinition is okay. - original_def = symbol.node - if self.is_conditional_func(original_def, defn): - # Conditional function definition -- multiple defs are ok. - defn.original_def = original_def - else: - # Report error. - self.check_no_global(defn.name(), defn, True) - if phase_info == FUNCTION_FIRST_PHASE_POSTPONE_SECOND: - # Postpone this function (for the second phase). - self.postponed_functions_stack[-1].append(defn) - return - if phase_info != FUNCTION_FIRST_PHASE_POSTPONE_SECOND: - # Second phase of analysis for function. - self.errors.push_function(defn.name()) - self.analyze_function(defn) - self.errors.pop_function() - - def prepare_method_signature(self, func: FuncDef) -> None: - """Check basic signature validity and tweak annotation of self/cls argument.""" - # Only non-static methods are special. - if not func.is_static: - if not func.arguments: - self.fail('Method must have at least one argument', func) - elif func.type: - sig = cast(FunctionLike, func.type) - if func.is_class: - leading_type = self.class_type(self.type) - else: - leading_type = self_type(self.type) - func.type = replace_implicit_first_type(sig, leading_type) - - def is_conditional_func(self, previous: Node, new: FuncDef) -> bool: - """Does 'new' conditionally redefine 'previous'? - - We reject straight redefinitions of functions, as they are usually - a programming error. For example: - - . def f(): ... - . def f(): ... # Error: 'f' redefined - """ - return isinstance(previous, (FuncDef, Var)) and new.is_conditional - - def update_function_type_variables(self, defn: FuncDef) -> None: - """Make any type variables in the signature of defn explicit. - - Update the signature of defn to contain type variable definitions - if defn is generic. - """ - if defn.type: - functype = cast(CallableType, defn.type) - typevars = self.infer_type_variables(functype) - # Do not define a new type variable if already defined in scope. - typevars = [(name, tvar) for name, tvar in typevars - if not self.is_defined_type_var(name, defn)] - if typevars: - next_tvar_id = self.next_function_tvar_id() - defs = [TypeVarDef(tvar[0], next_tvar_id - i, - tvar[1].values, tvar[1].upper_bound, - tvar[1].variance) - for i, tvar in enumerate(typevars)] - functype.variables = defs - - def infer_type_variables(self, - type: CallableType) -> List[Tuple[str, TypeVarExpr]]: - """Return list of unique type variables referred to in a callable.""" - names = [] # type: List[str] - tvars = [] # type: List[TypeVarExpr] - for arg in type.arg_types + [type.ret_type]: - for name, tvar_expr in self.find_type_variables_in_type(arg): - if name not in names: - names.append(name) - tvars.append(tvar_expr) - return list(zip(names, tvars)) - - def find_type_variables_in_type( - self, type: Type) -> List[Tuple[str, TypeVarExpr]]: - """Return a list of all unique type variable references in type. - - This effectively does partial name binding, results of which are mostly thrown away. - """ - result = [] # type: List[Tuple[str, TypeVarExpr]] - if isinstance(type, UnboundType): - name = type.name - node = self.lookup_qualified(name, type) - if node and node.kind == UNBOUND_TVAR: - result.append((name, cast(TypeVarExpr, node.node))) - for arg in type.args: - result.extend(self.find_type_variables_in_type(arg)) - elif isinstance(type, TypeList): - for item in type.items: - result.extend(self.find_type_variables_in_type(item)) - elif isinstance(type, UnionType): - for item in type.items: - result.extend(self.find_type_variables_in_type(item)) - elif isinstance(type, AnyType): - pass - elif isinstance(type, EllipsisType) or isinstance(type, TupleType): - pass - else: - assert False, 'Unsupported type %s' % type - return result - - def is_defined_type_var(self, tvar: str, context: Node) -> bool: - return self.lookup_qualified(tvar, context).kind == BOUND_TVAR - - def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: - t = [] # type: List[CallableType] - for i, item in enumerate(defn.items): - # TODO support decorated overloaded functions properly - item.is_overload = True - item.func.is_overload = True - item.accept(self) - t.append(cast(CallableType, function_type(item.func, - self.builtin_type('builtins.function')))) - if item.func.is_property and i == 0: - # This defines a property, probably with a setter and/or deleter. - self.analyze_property_with_multi_part_definition(defn) - break - if not [dec for dec in item.decorators - if refers_to_fullname(dec, 'typing.overload')]: - self.fail("'overload' decorator expected", item) - - defn.type = Overloaded(t) - defn.type.line = defn.line - - if self.is_class_scope(): - self.type.names[defn.name()] = SymbolTableNode(MDEF, defn, - typ=defn.type) - defn.info = self.type - elif self.is_func_scope(): - self.add_local(defn, defn) - - def analyze_property_with_multi_part_definition(self, defn: OverloadedFuncDef) -> None: - """Analyze a property defined using multiple methods (e.g., using @x.setter). - - Assume that the first method (@property) has already been analyzed. - """ - defn.is_property = True - items = defn.items - for item in items[1:]: - if len(item.decorators) == 1: - node = item.decorators[0] - if isinstance(node, MemberExpr): - if node.name == 'setter': - # The first item represents the entire property. - defn.items[0].var.is_settable_property = True - # Get abstractness from the original definition. - item.func.is_abstract = items[0].func.is_abstract - else: - self.fail("Decorated property not supported", item) - item.func.accept(self) - - def next_function_tvar_id(self) -> int: - return self.next_function_tvar_id_stack[-1] - - def analyze_function(self, defn: FuncItem) -> None: - is_method = self.is_class_scope() - - tvarnodes = self.add_func_type_variables_to_symbol_table(defn) - next_function_tvar_id = min([self.next_function_tvar_id()] + - [n.tvar_def.id.raw_id - 1 for n in tvarnodes]) - self.next_function_tvar_id_stack.append(next_function_tvar_id) - - if defn.type: - # Signature must be analyzed in the surrounding scope so that - # class-level imported names and type variables are in scope. - defn.type = self.anal_type(defn.type) - self.check_function_signature(defn) - if isinstance(defn, FuncDef): - defn.type = set_callable_name(defn.type, defn) - for arg in defn.arguments: - if arg.initializer: - arg.initializer.accept(self) - self.function_stack.append(defn) - self.enter() - for arg in defn.arguments: - self.add_local(arg.variable, defn) - for arg in defn.arguments: - if arg.initialization_statement: - lvalue = arg.initialization_statement.lvalues[0] - lvalue.accept(self) - - # The first argument of a non-static, non-class method is like 'self' - # (though the name could be different), having the enclosing class's - # instance type. - if is_method and not defn.is_static and not defn.is_class and defn.arguments: - defn.arguments[0].variable.is_self = True - - # First analyze body of the function but ignore nested functions. - self.postpone_nested_functions_stack.append(FUNCTION_FIRST_PHASE_POSTPONE_SECOND) - self.postponed_functions_stack.append([]) - defn.body.accept(self) - - # Analyze nested functions (if any) as a second phase. - self.postpone_nested_functions_stack[-1] = FUNCTION_SECOND_PHASE - for postponed in self.postponed_functions_stack[-1]: - postponed.accept(self) - self.postpone_nested_functions_stack.pop() - self.postponed_functions_stack.pop() - - self.next_function_tvar_id_stack.pop() - disable_typevars(tvarnodes) - - self.leave() - self.function_stack.pop() - - def add_func_type_variables_to_symbol_table( - self, defn: FuncItem) -> List[SymbolTableNode]: - nodes = [] # type: List[SymbolTableNode] - if defn.type: - tt = defn.type - names = self.type_var_names() - items = cast(CallableType, tt).variables - for item in items: - name = item.name - if name in names: - self.name_already_defined(name, defn) - node = self.bind_type_var(name, item, defn) - nodes.append(node) - names.add(name) - return nodes - - def type_var_names(self) -> Set[str]: - if not self.type: - return set() - else: - return set(self.type.type_vars) - - def bind_type_var(self, fullname: str, tvar_def: TypeVarDef, - context: Context) -> SymbolTableNode: - node = self.lookup_qualified(fullname, context) - node.kind = BOUND_TVAR - node.tvar_def = tvar_def - return node - - def check_function_signature(self, fdef: FuncItem) -> None: - sig = cast(CallableType, fdef.type) - if len(sig.arg_types) < len(fdef.arguments): - self.fail('Type signature has too few arguments', fdef) - # Add dummy Any arguments to prevent crashes later. - extra_anys = [AnyType()] * (len(fdef.arguments) - len(sig.arg_types)) - sig.arg_types.extend(extra_anys) - elif len(sig.arg_types) > len(fdef.arguments): - self.fail('Type signature has too many arguments', fdef) - - def visit_class_def(self, defn: ClassDef) -> None: - self.clean_up_bases_and_infer_type_variables(defn) - self.setup_class_def_analysis(defn) - - self.bind_class_type_vars(defn) - - self.analyze_base_classes(defn) - self.analyze_metaclass(defn) - - for decorator in defn.decorators: - self.analyze_class_decorator(defn, decorator) - - self.enter_class(defn) - - self.setup_is_builtinclass(defn) - - # Analyze class body. - defn.defs.accept(self) - - self.calculate_abstract_status(defn.info) - self.setup_type_promotion(defn) - - self.leave_class() - self.unbind_class_type_vars() - - def enter_class(self, defn: ClassDef) -> None: - # Remember previous active class - self.type_stack.append(self.type) - self.locals.append(None) # Add class scope - self.block_depth.append(-1) # The class body increments this to 0 - self.postpone_nested_functions_stack.append(FUNCTION_BOTH_PHASES) - self.type = defn.info - - def leave_class(self) -> None: - """ Restore analyzer state. """ - self.postpone_nested_functions_stack.pop() - self.block_depth.pop() - self.locals.pop() - self.type = self.type_stack.pop() - - def bind_class_type_vars(self, defn: ClassDef) -> None: - """ Unbind type variables of previously active class and bind - the type variables for the active class. - """ - if self.bound_tvars: - disable_typevars(self.bound_tvars) - self.tvar_stack.append(self.bound_tvars) - self.bound_tvars = self.bind_class_type_variables_in_symbol_table(defn.info) - - def unbind_class_type_vars(self) -> None: - """ Unbind the active class' type vars and rebind the - type vars of the previously active class. - """ - disable_typevars(self.bound_tvars) - self.bound_tvars = self.tvar_stack.pop() - if self.bound_tvars: - enable_typevars(self.bound_tvars) - - def analyze_class_decorator(self, defn: ClassDef, decorator: Node) -> None: - decorator.accept(self) - - def setup_is_builtinclass(self, defn: ClassDef): - for decorator in defn.decorators: - if refers_to_fullname(decorator, 'typing.builtinclass'): - defn.is_builtinclass = True - if defn.fullname == 'builtins.object': - # Only 'object' is marked as a built-in class, as otherwise things elsewhere - # would break. We need a better way of dealing with built-in classes. - defn.is_builtinclass = True - - def calculate_abstract_status(self, typ: TypeInfo) -> None: - """Calculate abstract status of a class. - - Set is_abstract of the type to True if the type has an unimplemented - abstract attribute. Also compute a list of abstract attributes. - """ - concrete = set() # type: Set[str] - abstract = [] # type: List[str] - for base in typ.mro: - for name, symnode in base.names.items(): - node = symnode.node - if isinstance(node, OverloadedFuncDef): - # Unwrap an overloaded function definition. We can just - # check arbitrarily the first overload item. If the - # different items have a different abstract status, there - # should be an error reported elsewhere. - func = node.items[0] # type: Node - else: - func = node - if isinstance(func, Decorator): - fdef = func.func - if fdef.is_abstract and name not in concrete: - typ.is_abstract = True - abstract.append(name) - concrete.add(name) - typ.abstract_attributes = sorted(abstract) - - def setup_type_promotion(self, defn: ClassDef) -> None: - """Setup extra, ad-hoc subtyping relationships between classes (promotion). - - This includes things like 'int' being compatible with 'float'. - """ - promote_target = None # type: Type - for decorator in defn.decorators: - if isinstance(decorator, CallExpr): - analyzed = decorator.analyzed - if isinstance(analyzed, PromoteExpr): - # _promote class decorator (undocumented faeture). - promote_target = analyzed.type - if not promote_target: - promotions = (TYPE_PROMOTIONS_PYTHON3 if self.options.python_version[0] >= 3 - else TYPE_PROMOTIONS_PYTHON2) - if defn.fullname in promotions: - promote_target = self.named_type_or_none(promotions[defn.fullname]) - defn.info._promote = promote_target - - def clean_up_bases_and_infer_type_variables(self, defn: ClassDef) -> None: - """Remove extra base classes such as Generic and infer type vars. - - For example, consider this class: - - . class Foo(Bar, Generic[T]): ... - - Now we will remove Generic[T] from bases of Foo and infer that the - type variable 'T' is a type argument of Foo. - - Note that this is performed *before* semantic analysis. - """ - removed = [] # type: List[int] - type_vars = [] # type: List[TypeVarDef] - for i, base_expr in enumerate(defn.base_type_exprs): - try: - base = expr_to_unanalyzed_type(base_expr) - except TypeTranslationError: - # This error will be caught later. - continue - tvars = self.analyze_typevar_declaration(base) - if tvars is not None: - if type_vars: - self.fail('Duplicate Generic in bases', defn) - removed.append(i) - for j, (name, tvar_expr) in enumerate(tvars): - type_vars.append(TypeVarDef(name, j + 1, tvar_expr.values, - tvar_expr.upper_bound, tvar_expr.variance)) - if type_vars: - defn.type_vars = type_vars - if defn.info: - defn.info.type_vars = [tv.name for tv in type_vars] - for i in reversed(removed): - del defn.base_type_exprs[i] - - def analyze_typevar_declaration(self, t: Type) -> Optional[List[Tuple[str, TypeVarExpr]]]: - if not isinstance(t, UnboundType): - return None - unbound = t - sym = self.lookup_qualified(unbound.name, unbound) - if sym is None or sym.node is None: - return None - if sym.node.fullname() == 'typing.Generic': - tvars = [] # type: List[Tuple[str, TypeVarExpr]] - for arg in unbound.args: - tvar = self.analyze_unbound_tvar(arg) - if tvar: - tvars.append(tvar) - else: - self.fail('Free type variable expected in %s[...]' % - sym.node.name(), t) - return tvars - return None - - def analyze_unbound_tvar(self, t: Type) -> Tuple[str, TypeVarExpr]: - if not isinstance(t, UnboundType): - return None - unbound = t - sym = self.lookup_qualified(unbound.name, unbound) - if sym is not None and sym.kind == UNBOUND_TVAR: - return unbound.name, cast(TypeVarExpr, sym.node) - return None - - def setup_class_def_analysis(self, defn: ClassDef) -> None: - """Prepare for the analysis of a class definition.""" - if not defn.info: - defn.info = TypeInfo(SymbolTable(), defn) - defn.info._fullname = defn.info.name() - if self.is_func_scope() or self.type: - kind = MDEF - if self.is_func_scope(): - kind = LDEF - self.add_symbol(defn.name, SymbolTableNode(kind, defn.info), defn) - - def analyze_base_classes(self, defn: ClassDef) -> None: - """Analyze and set up base classes. - - This computes several attributes on the corresponding TypeInfo defn.info - related to the base classes: defn.info.bases, defn.info.mro, and - miscellaneous others (at least tuple_type, fallback_to_any, and is_enum.) - """ - - base_types = [] # type: List[Instance] - info = defn.info - for base_expr in defn.base_type_exprs: - try: - base = self.expr_to_analyzed_type(base_expr) - except TypeTranslationError: - self.fail('Invalid base class', base_expr) - info.fallback_to_any = True - continue - - if isinstance(base, TupleType): - if info.tuple_type: - self.fail("Class has two incompatible bases derived from tuple", defn) - defn.incompatible_baseclass = True - if (not self.is_stub_file - and not info.is_named_tuple - and base.fallback.type.fullname() == 'builtins.tuple'): - self.fail("Tuple[...] not supported as a base class outside a stub file", defn) - info.tuple_type = base - base_types.append(base.fallback) - elif isinstance(base, Instance): - base_types.append(base) - elif isinstance(base, AnyType): - info.fallback_to_any = True - else: - self.fail('Invalid base class', base_expr) - info.fallback_to_any = True - - # Add 'object' as implicit base if there is no other base class. - if (not base_types and defn.fullname != 'builtins.object'): - base_types.append(self.object_type()) - - info.bases = base_types - - # Calculate the MRO. It might be incomplete at this point if - # the bases of defn include classes imported from other - # modules in an import loop. We'll recompute it in ThirdPass. - if not self.verify_base_classes(defn): - info.mro = [] - return - calculate_class_mro(defn, self.fail) - # If there are cyclic imports, we may be missing 'object' in - # the MRO. Fix MRO if needed. - if info.mro and info.mro[-1].fullname() != 'builtins.object': - info.mro.append(self.object_type().type) - - def expr_to_analyzed_type(self, expr: Node) -> Type: - if isinstance(expr, CallExpr): - expr.accept(self) - info = self.check_namedtuple(expr) - if info is None: - # Some form of namedtuple is the only valid type that looks like a call - # expression. This isn't a valid type. - raise TypeTranslationError() - fallback = Instance(info, []) - return TupleType(info.tuple_type.items, fallback=fallback) - typ = expr_to_unanalyzed_type(expr) - return self.anal_type(typ) - - def verify_base_classes(self, defn: ClassDef) -> bool: - info = defn.info - for base in info.bases: - baseinfo = base.type - if self.is_base_class(info, baseinfo): - self.fail('Cycle in inheritance hierarchy', defn) - # Clear bases to forcefully get rid of the cycle. - info.bases = [] - if baseinfo.fullname() == 'builtins.bool': - self.fail("'%s' is not a valid base class" % - baseinfo.name(), defn) - return False - dup = find_duplicate(info.direct_base_classes()) - if dup: - self.fail('Duplicate base class "%s"' % dup.name(), defn) - return False - return True - - def is_base_class(self, t: TypeInfo, s: TypeInfo) -> bool: - """Determine if t is a base class of s (but do not use mro).""" - # Search the base class graph for t, starting from s. - worklist = [s] - visited = {s} - while worklist: - nxt = worklist.pop() - if nxt == t: - return True - for base in nxt.bases: - if base.type not in visited: - worklist.append(base.type) - visited.add(base.type) - return False - - def analyze_metaclass(self, defn: ClassDef) -> None: - if defn.metaclass: - sym = self.lookup_qualified(defn.metaclass, defn) - if sym is not None and not isinstance(sym.node, TypeInfo): - self.fail("Invalid metaclass '%s'" % defn.metaclass, defn) - - def object_type(self) -> Instance: - return self.named_type('__builtins__.object') - - def class_type(self, info: TypeInfo) -> Type: - # Construct a function type whose fallback is cls. - from mypy import checkmember # To avoid import cycle. - leading_type = checkmember.type_object_type(info, self.builtin_type) - if isinstance(leading_type, Overloaded): - # Overloaded __init__ is too complex to handle. Plus it's stubs only. - return AnyType() - else: - return leading_type - - def named_type(self, qualified_name: str, args: List[Type] = None) -> Instance: - sym = self.lookup_qualified(qualified_name, None) - return Instance(cast(TypeInfo, sym.node), args or []) - - def named_type_or_none(self, qualified_name: str) -> Instance: - sym = self.lookup_fully_qualified_or_none(qualified_name) - if not sym: - return None - return Instance(cast(TypeInfo, sym.node), []) - - def is_instance_type(self, t: Type) -> bool: - return isinstance(t, Instance) - - def bind_class_type_variables_in_symbol_table( - self, info: TypeInfo) -> List[SymbolTableNode]: - nodes = [] # type: List[SymbolTableNode] - for var, binder in zip(info.type_vars, info.defn.type_vars): - node = self.bind_type_var(var, binder, info) - nodes.append(node) - return nodes - - def visit_import(self, i: Import) -> None: - for id, as_id in i.ids: - if as_id is not None: - self.add_module_symbol(id, as_id, module_public=True, context=i) - else: - # Modules imported in a stub file without using 'as x' won't get exported when - # doing 'from m import *'. - module_public = not self.is_stub_file - base = id.split('.')[0] - self.add_module_symbol(base, base, module_public=module_public, - context=i) - - def add_module_symbol(self, id: str, as_id: str, module_public: bool, - context: Context) -> None: - if id in self.modules: - m = self.modules[id] - self.add_symbol(as_id, SymbolTableNode(MODULE_REF, m, self.cur_mod_id, - module_public=module_public), context) - else: - self.add_unknown_symbol(as_id, context) - - def visit_import_from(self, imp: ImportFrom) -> None: - import_id = self.correct_relative_import(imp) - if import_id in self.modules: - module = self.modules[import_id] - for id, as_id in imp.names: - node = module.names.get(id) - if node and node.kind != UNBOUND_IMPORTED: - node = self.normalize_type_alias(node, imp) - if not node: - return - imported_id = as_id or id - existing_symbol = self.globals.get(imported_id) - if existing_symbol: - # Import can redefine a variable. They get special treatment. - if self.process_import_over_existing_name( - imported_id, existing_symbol, node, imp): - continue - # 'from m import x as x' exports x in a stub file. - module_public = not self.is_stub_file or as_id is not None - symbol = SymbolTableNode(node.kind, node.node, - self.cur_mod_id, - node.type_override, - module_public=module_public) - self.add_symbol(imported_id, symbol, imp) - else: - message = "Module has no attribute '{}'".format(id) - extra = self.undefined_name_extra_info('{}.{}'.format(import_id, id)) - if extra: - message += " {}".format(extra) - self.fail(message, imp) - else: - # Missing module. - for id, as_id in imp.names: - self.add_unknown_symbol(as_id or id, imp) - - def process_import_over_existing_name(self, - imported_id: str, existing_symbol: SymbolTableNode, - module_symbol: SymbolTableNode, - import_node: ImportBase) -> bool: - if (existing_symbol.kind in (LDEF, GDEF, MDEF) and - isinstance(existing_symbol.node, (Var, FuncDef))): - # This is a valid import over an existing definition in the file. Construct a dummy - # assignment that we'll use to type check the import. - lvalue = NameExpr(imported_id) - lvalue.kind = existing_symbol.kind - lvalue.node = existing_symbol.node - rvalue = NameExpr(imported_id) - rvalue.kind = module_symbol.kind - rvalue.node = module_symbol.node - assignment = AssignmentStmt([lvalue], rvalue) - for node in assignment, lvalue, rvalue: - node.set_line(import_node) - import_node.assignments.append(assignment) - return True - return False - - def normalize_type_alias(self, node: SymbolTableNode, - ctx: Context) -> SymbolTableNode: - if node.fullname in type_aliases: - # Node refers to an aliased type such as typing.List; normalize. - node = self.lookup_qualified(type_aliases[node.fullname], ctx) - if node.fullname == 'typing.DefaultDict': - self.add_module_symbol('collections', '__mypy_collections__', False, ctx) - node = self.lookup_qualified('__mypy_collections__.defaultdict', ctx) - return node - - def correct_relative_import(self, node: Union[ImportFrom, ImportAll]) -> str: - if node.relative == 0: - return node.id - - parts = self.cur_mod_id.split(".") - cur_mod_id = self.cur_mod_id - - rel = node.relative - if self.cur_mod_node.is_package_init_file(): - rel -= 1 - if len(parts) < rel: - self.fail("Relative import climbs too many namespaces", node) - if rel != 0: - cur_mod_id = ".".join(parts[:-rel]) - - return cur_mod_id + (("." + node.id) if node.id else "") - - def visit_import_all(self, i: ImportAll) -> None: - i_id = self.correct_relative_import(i) - if i_id in self.modules: - m = self.modules[i_id] - for name, node in m.names.items(): - node = self.normalize_type_alias(node, i) - if not name.startswith('_') and node.module_public: - existing_symbol = self.globals.get(name) - if existing_symbol: - # Import can redefine a variable. They get special treatment. - if self.process_import_over_existing_name( - name, existing_symbol, node, i): - continue - self.add_symbol(name, SymbolTableNode(node.kind, node.node, - self.cur_mod_id), i) - else: - # Don't add any dummy symbols for 'from x import *' if 'x' is unknown. - pass - - def add_unknown_symbol(self, name: str, context: Context) -> None: - var = Var(name) - var._fullname = self.qualified_name(name) - var.is_ready = True - var.type = AnyType() - self.add_symbol(name, SymbolTableNode(GDEF, var, self.cur_mod_id), context) - - # - # Statements - # - - def visit_block(self, b: Block) -> None: - if b.is_unreachable: - return - self.block_depth[-1] += 1 - for s in b.body: - self.accept(s) - self.block_depth[-1] -= 1 - - def visit_block_maybe(self, b: Block) -> None: - if b: - self.visit_block(b) - - def anal_type(self, t: Type, allow_tuple_literal: bool = False) -> Type: - if t: - if allow_tuple_literal: - # Types such as (t1, t2, ...) only allowed in assignment statements. They'll - # generate errors elsewhere, and Tuple[t1, t2, ...] must be used instead. - if isinstance(t, TupleType): - # Unlike TypeAnalyser, also allow implicit tuple types (without Tuple[...]). - star_count = sum(1 for item in t.items if isinstance(item, StarType)) - if star_count > 1: - self.fail('At most one star type allowed in a tuple', t) - return TupleType([AnyType() for _ in t.items], - self.builtin_type('builtins.tuple'), t.line) - items = [self.anal_type(item, True) - for item in t.items] - return TupleType(items, self.builtin_type('builtins.tuple'), t.line) - a = TypeAnalyser(self.lookup_qualified, - self.lookup_fully_qualified, - self.fail) - return t.accept(a) - else: - return None - - def visit_assignment_stmt(self, s: AssignmentStmt) -> None: - for lval in s.lvalues: - self.analyze_lvalue(lval, explicit_type=s.type is not None) - s.rvalue.accept(self) - if s.type: - allow_tuple_literal = isinstance(s.lvalues[-1], (TupleExpr, ListExpr)) - s.type = self.anal_type(s.type, allow_tuple_literal) - else: - # For simple assignments, allow binding type aliases. - # Also set the type if the rvalue is a simple literal. - if (s.type is None and len(s.lvalues) == 1 and - isinstance(s.lvalues[0], NameExpr)): - if s.lvalues[0].is_def: - s.type = self.analyze_simple_literal_type(s.rvalue) - res = analyze_type_alias(s.rvalue, - self.lookup_qualified, - self.lookup_fully_qualified, - self.fail) - if res and (not isinstance(res, Instance) or cast(Instance, res).args): - # TODO: What if this gets reassigned? - name = s.lvalues[0] - node = self.lookup(name.name, name) - node.kind = TYPE_ALIAS - node.type_override = res - if isinstance(s.rvalue, IndexExpr): - s.rvalue.analyzed = TypeAliasExpr(res) - if s.type: - # Store type into nodes. - for lvalue in s.lvalues: - self.store_declared_types(lvalue, s.type) - self.check_and_set_up_type_alias(s) - self.process_typevar_declaration(s) - self.process_namedtuple_definition(s) - - if (len(s.lvalues) == 1 and isinstance(s.lvalues[0], NameExpr) and - s.lvalues[0].name == '__all__' and s.lvalues[0].kind == GDEF and - isinstance(s.rvalue, (ListExpr, TupleExpr))): - self.add_exports(*s.rvalue.items) - - def analyze_simple_literal_type(self, rvalue: Node) -> Optional[Type]: - """Return builtins.int if rvalue is an int literal, etc.""" - if self.weak_opts or self.options.semantic_analysis_only or self.function_stack: - # Skip this if any weak options are set. - # Also skip if we're only doing the semantic analysis pass. - # This is mostly to avoid breaking unit tests. - # Also skip inside a function; this is to avoid confusing - # the code that handles dead code due to isinstance() - # inside type variables with value restrictions (like - # AnyStr). - return None - if isinstance(rvalue, IntExpr): - return self.named_type_or_none('builtins.int') - if isinstance(rvalue, FloatExpr): - return self.named_type_or_none('builtins.float') - if isinstance(rvalue, StrExpr): - return self.named_type_or_none('builtins.str') - if isinstance(rvalue, BytesExpr): - return self.named_type_or_none('builtins.bytes') - if isinstance(rvalue, UnicodeExpr): - return self.named_type_or_none('builtins.unicode') - return None - - def check_and_set_up_type_alias(self, s: AssignmentStmt) -> None: - """Check if assignment creates a type alias and set it up as needed.""" - # For now, type aliases only work at the top level of a module. - if (len(s.lvalues) == 1 and not self.is_func_scope() and not self.type - and not s.type): - lvalue = s.lvalues[0] - if isinstance(lvalue, NameExpr): - if not lvalue.is_def: - # Only a definition can create a type alias, not regular assignment. - return - rvalue = s.rvalue - if isinstance(rvalue, RefExpr): - node = rvalue.node - if isinstance(node, TypeInfo): - # TODO: We should record the fact that this is a variable - # that refers to a type, rather than making this - # just an alias for the type. - self.globals[lvalue.name].node = node - - def analyze_lvalue(self, lval: Node, nested: bool = False, - add_global: bool = False, - explicit_type: bool = False) -> None: - """Analyze an lvalue or assignment target. - - Only if add_global is True, add name to globals table. If nested - is true, the lvalue is within a tuple or list lvalue expression. - """ - - if isinstance(lval, NameExpr): - # Top-level definitions within some statements (at least while) are - # not handled in the first pass, so they have to be added now. - nested_global = (not self.is_func_scope() and - self.block_depth[-1] > 0 and - not self.type) - if (add_global or nested_global) and lval.name not in self.globals: - # Define new global name. - v = Var(lval.name) - v._fullname = self.qualified_name(lval.name) - v.is_ready = False # Type not inferred yet - lval.node = v - lval.is_def = True - lval.kind = GDEF - lval.fullname = v._fullname - self.globals[lval.name] = SymbolTableNode(GDEF, v, - self.cur_mod_id) - elif isinstance(lval.node, Var) and lval.is_def: - # Since the is_def flag is set, this must have been analyzed - # already in the first pass and added to the symbol table. - assert lval.node.name() in self.globals - elif (self.is_func_scope() and lval.name not in self.locals[-1] and - lval.name not in self.global_decls[-1] and - lval.name not in self.nonlocal_decls[-1]): - # Define new local name. - v = Var(lval.name) - lval.node = v - lval.is_def = True - lval.kind = LDEF - lval.fullname = lval.name - self.add_local(v, lval) - elif not self.is_func_scope() and (self.type and - lval.name not in self.type.names): - # Define a new attribute within class body. - v = Var(lval.name) - v.info = self.type - v.is_initialized_in_class = True - v.set_line(lval) - lval.node = v - lval.is_def = True - lval.kind = MDEF - lval.fullname = lval.name - self.type.names[lval.name] = SymbolTableNode(MDEF, v) - else: - # Bind to an existing name. - if explicit_type: - self.name_already_defined(lval.name, lval) - lval.accept(self) - self.check_lvalue_validity(lval.node, lval) - elif isinstance(lval, MemberExpr): - if not add_global: - self.analyze_member_lvalue(lval) - if explicit_type and not self.is_self_member_ref(lval): - self.fail('Type cannot be declared in assignment to non-self ' - 'attribute', lval) - elif isinstance(lval, IndexExpr): - if explicit_type: - self.fail('Unexpected type declaration', lval) - if not add_global: - lval.accept(self) - elif (isinstance(lval, TupleExpr) or - isinstance(lval, ListExpr)): - items = cast(Any, lval).items - if len(items) == 0 and isinstance(lval, TupleExpr): - self.fail("Can't assign to ()", lval) - self.analyze_tuple_or_list_lvalue(cast(Union[ListExpr, TupleExpr], lval), - add_global, explicit_type) - elif isinstance(lval, StarExpr): - if nested: - self.analyze_lvalue(lval.expr, nested, add_global, explicit_type) - else: - self.fail('Starred assignment target must be in a list or tuple', lval) - else: - self.fail('Invalid assignment target', lval) - - def analyze_tuple_or_list_lvalue(self, lval: Union[ListExpr, TupleExpr], - add_global: bool = False, - explicit_type: bool = False) -> None: - """Analyze an lvalue or assignment target that is a list or tuple.""" - items = lval.items - star_exprs = [cast(StarExpr, item) - for item in items - if isinstance(item, StarExpr)] - - if len(star_exprs) > 1: - self.fail('Two starred expressions in assignment', lval) - else: - if len(star_exprs) == 1: - star_exprs[0].valid = True - for i in items: - self.analyze_lvalue(i, nested=True, add_global=add_global, - explicit_type = explicit_type) - - def analyze_member_lvalue(self, lval: MemberExpr) -> None: - lval.accept(self) - if (self.is_self_member_ref(lval) and - self.type.get(lval.name) is None): - # Implicit attribute definition in __init__. - lval.is_def = True - v = Var(lval.name) - v.info = self.type - v.is_ready = False - lval.def_var = v - lval.node = v - self.type.names[lval.name] = SymbolTableNode(MDEF, v) - self.check_lvalue_validity(lval.node, lval) - - def is_self_member_ref(self, memberexpr: MemberExpr) -> bool: - """Does memberexpr to refer to an attribute of self?""" - if not isinstance(memberexpr.expr, NameExpr): - return False - node = memberexpr.expr.node - return isinstance(node, Var) and node.is_self - - def check_lvalue_validity(self, node: Node, ctx: Context) -> None: - if isinstance(node, (TypeInfo, TypeVarExpr)): - self.fail('Invalid assignment target', ctx) - - def store_declared_types(self, lvalue: Node, typ: Type) -> None: - if isinstance(typ, StarType) and not isinstance(lvalue, StarExpr): - self.fail('Star type only allowed for starred expressions', lvalue) - if isinstance(lvalue, RefExpr): - lvalue.is_def = False - if isinstance(lvalue.node, Var): - var = lvalue.node - var.type = typ - var.is_ready = True - # If node is not a variable, we'll catch it elsewhere. - elif isinstance(lvalue, TupleExpr): - if isinstance(typ, TupleType): - if len(lvalue.items) != len(typ.items): - self.fail('Incompatible number of tuple items', lvalue) - return - for item, itemtype in zip(lvalue.items, typ.items): - self.store_declared_types(item, itemtype) - else: - self.fail('Tuple type expected for multiple variables', - lvalue) - elif isinstance(lvalue, StarExpr): - if isinstance(typ, StarType): - self.store_declared_types(lvalue.expr, typ.type) - else: - self.fail('Star type expected for starred expression', lvalue) - else: - # This has been flagged elsewhere as an error, so just ignore here. - pass - - def process_typevar_declaration(self, s: AssignmentStmt) -> None: - """Check if s declares a TypeVar; it yes, store it in symbol table.""" - call = self.get_typevar_declaration(s) - if not call: - return - - lvalue = cast(NameExpr, s.lvalues[0]) - name = lvalue.name - if not lvalue.is_def: - if s.type: - self.fail("Cannot declare the type of a type variable", s) - else: - self.fail("Cannot redefine '%s' as a type variable" % name, s) - return - - if not self.check_typevar_name(call, name, s): - return - - # Constraining types - n_values = call.arg_kinds[1:].count(ARG_POS) - values = self.analyze_types(call.args[1:1 + n_values]) - - res = self.process_typevar_parameters(call.args[1 + n_values:], - call.arg_names[1 + n_values:], - call.arg_kinds[1 + n_values:], - bool(values), - s) - if res is None: - return - variance, upper_bound = res - - # Yes, it's a valid type variable definition! Add it to the symbol table. - node = self.lookup(name, s) - node.kind = UNBOUND_TVAR - TypeVar = TypeVarExpr(name, node.fullname, values, upper_bound, variance) - TypeVar.line = call.line - call.analyzed = TypeVar - node.node = TypeVar - - def check_typevar_name(self, call: CallExpr, name: str, context: Context) -> bool: - if len(call.args) < 1: - self.fail("Too few arguments for TypeVar()", context) - return False - if not isinstance(call.args[0], (StrExpr, BytesExpr)) or not call.arg_kinds[0] == ARG_POS: - self.fail("TypeVar() expects a string literal as first argument", context) - return False - if cast(StrExpr, call.args[0]).value != name: - self.fail("Unexpected TypeVar() argument value", context) - return False - return True - - def get_typevar_declaration(self, s: AssignmentStmt) -> Optional[CallExpr]: - """Returns the TypeVar() call expression if `s` is a type var declaration - or None otherwise. - """ - if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): - return None - if not isinstance(s.rvalue, CallExpr): - return None - call = s.rvalue - if not isinstance(call.callee, RefExpr): - return None - callee = call.callee - if callee.fullname != 'typing.TypeVar': - return None - return call - - def process_typevar_parameters(self, args: List[Node], - names: List[Optional[str]], - kinds: List[int], - has_values: bool, - context: Context) -> Optional[Tuple[int, Type]]: - covariant = False - contravariant = False - upper_bound = self.object_type() # type: Type - for param_value, param_name, param_kind in zip(args, names, kinds): - if not param_kind == ARG_NAMED: - self.fail("Unexpected argument to TypeVar()", context) - return None - if param_name == 'covariant': - if isinstance(param_value, NameExpr): - if param_value.name == 'True': - covariant = True - else: - self.fail("TypeVar 'covariant' may only be 'True'", context) - return None - else: - self.fail("TypeVar 'covariant' may only be 'True'", context) - return None - elif param_name == 'contravariant': - if isinstance(param_value, NameExpr): - if param_value.name == 'True': - contravariant = True - else: - self.fail("TypeVar 'contravariant' may only be 'True'", context) - return None - else: - self.fail("TypeVar 'contravariant' may only be 'True'", context) - return None - elif param_name == 'bound': - if has_values: - self.fail("TypeVar cannot have both values and an upper bound", context) - return None - try: - upper_bound = self.expr_to_analyzed_type(param_value) - except TypeTranslationError: - self.fail("TypeVar 'bound' must be a type", param_value) - return None - elif param_name == 'values': - # Probably using obsolete syntax with values=(...). Explain the current syntax. - self.fail("TypeVar 'values' argument not supported", context) - self.fail("Use TypeVar('T', t, ...) instead of TypeVar('T', values=(t, ...))", - context) - return None - else: - self.fail("Unexpected argument to TypeVar(): {}".format(param_name), context) - return None - - if covariant and contravariant: - self.fail("TypeVar cannot be both covariant and contravariant", context) - return None - elif covariant: - variance = COVARIANT - elif contravariant: - variance = CONTRAVARIANT - else: - variance = INVARIANT - return (variance, upper_bound) - - def process_namedtuple_definition(self, s: AssignmentStmt) -> None: - """Check if s defines a namedtuple; if yes, store the definition in symbol table.""" - if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): - return - lvalue = cast(NameExpr, s.lvalues[0]) - name = lvalue.name - named_tuple = self.check_namedtuple(s.rvalue, name) - if named_tuple is None: - return - # Yes, it's a valid namedtuple definition. Add it to the symbol table. - node = self.lookup(name, s) - node.kind = GDEF # TODO locally defined namedtuple - # TODO call.analyzed - node.node = named_tuple - - def check_namedtuple(self, node: Node, var_name: str = None) -> TypeInfo: - """Check if a call defines a namedtuple. - - The optional var_name argument is the name of the variable to - which this is assigned, if any. - - If it does, return the corresponding TypeInfo. Return None otherwise. - - If the definition is invalid but looks like a namedtuple, - report errors but return (some) TypeInfo. - """ - if not isinstance(node, CallExpr): - return None - call = node - if not isinstance(call.callee, RefExpr): - return None - callee = call.callee - fullname = callee.fullname - if fullname not in ('collections.namedtuple', 'typing.NamedTuple'): - return None - items, types, ok = self.parse_namedtuple_args(call, fullname) - if not ok: - # Error. Construct dummy return value. - return self.build_namedtuple_typeinfo('namedtuple', [], []) - else: - # Give it a unique name derived from the line number. - name = cast(StrExpr, call.args[0]).value - if name != var_name: - name += '@' + str(call.line) - info = self.build_namedtuple_typeinfo(name, items, types) - # Store it as a global just in case it would remain anonymous. - self.globals[name] = SymbolTableNode(GDEF, info, self.cur_mod_id) - call.analyzed = NamedTupleExpr(info).set_line(call.line) - return info - - def parse_namedtuple_args(self, call: CallExpr, - fullname: str) -> Tuple[List[str], List[Type], bool]: - # TODO Share code with check_argument_count in checkexpr.py? - args = call.args - if len(args) < 2: - return self.fail_namedtuple_arg("Too few arguments for namedtuple()", call) - if len(args) > 2: - return self.fail_namedtuple_arg("Too many arguments for namedtuple()", call) - if call.arg_kinds != [ARG_POS, ARG_POS]: - return self.fail_namedtuple_arg("Unexpected arguments to namedtuple()", call) - if not isinstance(args[0], (StrExpr, BytesExpr)): - return self.fail_namedtuple_arg( - "namedtuple() expects a string literal as the first argument", call) - types = [] # type: List[Type] - ok = True - if not isinstance(args[1], ListExpr): - if fullname == 'collections.namedtuple' and isinstance(args[1], (StrExpr, BytesExpr)): - str_expr = cast(StrExpr, args[1]) - items = str_expr.value.split() - else: - return self.fail_namedtuple_arg( - "List literal expected as the second argument to namedtuple()", call) - else: - listexpr = args[1] - if fullname == 'collections.namedtuple': - # The fields argument contains just names, with implicit Any types. - if any(not isinstance(item, (StrExpr, BytesExpr)) for item in listexpr.items): - return self.fail_namedtuple_arg("String literal expected as namedtuple() item", - call) - items = [cast(StrExpr, item).value for item in listexpr.items] - else: - # The fields argument contains (name, type) tuples. - items, types, ok = self.parse_namedtuple_fields_with_types(listexpr.items, call) - if not types: - types = [AnyType() for _ in items] - return items, types, ok - - def parse_namedtuple_fields_with_types(self, nodes: List[Node], - context: Context) -> Tuple[List[str], List[Type], bool]: - items = [] # type: List[str] - types = [] # type: List[Type] - for item in nodes: - if isinstance(item, TupleExpr): - if len(item.items) != 2: - return self.fail_namedtuple_arg("Invalid NamedTuple field definition", - item) - name, type_node = item.items - if isinstance(name, (StrExpr, BytesExpr)): - items.append(name.value) - else: - return self.fail_namedtuple_arg("Invalid NamedTuple() field name", item) - try: - type = expr_to_unanalyzed_type(type_node) - except TypeTranslationError: - return self.fail_namedtuple_arg('Invalid field type', type_node) - types.append(self.anal_type(type)) - else: - return self.fail_namedtuple_arg("Tuple expected as NamedTuple() field", item) - return items, types, True - - def fail_namedtuple_arg(self, message: str, - context: Context) -> Tuple[List[str], List[Type], bool]: - self.fail(message, context) - return [], [], False - - def build_namedtuple_typeinfo(self, name: str, items: List[str], - types: List[Type]) -> TypeInfo: - symbols = SymbolTable() - tup = NamedTupleType(name, items, types, self.named_type('__builtins__.tuple', types)) - class_def = ClassDef(name, Block([])) - class_def.fullname = self.qualified_name(name) - info = NamedTupleTypeInfo(tup, symbols, class_def) - info.attrs = items - vars = [Var(item, typ) for item, typ in zip(items, types)] - # Add named tuple items as attributes. - # TODO: Make them read-only. - for var in vars: - var.info = info - symbols[var.name()] = SymbolTableNode(MDEF, var) - this_type = self_type(info) - # Add __init__, _replace, _asdict methods and _fields static field - self.add_namedtuple_method('_replace', symbols, info, this_type, this_type, - self.make_factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) - self.add_namedtuple_method('__init__', symbols, info, this_type, NoneTyp(), - self.make_factory_args(vars, ARG_POS), info.name()) - # TODO: refine to OrderedDict[Union[types]] - self.add_namedtuple_method('_asdict', symbols, info, this_type, AnyType(), []) - return info - - def make_factory_args(self, vars: List[Var], kind: int, - initializer: Expression = None) -> List[Argument]: - return [Argument(var, var.type, initializer, kind) for var in vars] - - def add_namedtuple_method(self, funcname: str, symbols: SymbolTable, info: TypeInfo, - this_type: Type, ret: Type, args: List[Argument], name=None) -> None: - args = [Argument(Var('self'), this_type, None, ARG_POS)] + args - types = [arg.type_annotation for arg in args] - items = [arg.variable.name() for arg in args] - arg_kinds = [arg.kind for arg in args] - # TODO: Make sure that the self argument name is not visible? - signature = CallableType(types, arg_kinds, items, ret, - self.named_type('__builtins__.function'), - name=name or info.name() + '.' + funcname) - func = FuncDef(funcname, - args, - Block([]), - typ=signature) - func.info = info - symbols[funcname] = SymbolTableNode(MDEF, func) - - def analyze_types(self, items: List[Node]) -> List[Type]: - result = [] # type: List[Type] - for node in items: - try: - result.append(self.anal_type(expr_to_unanalyzed_type(node))) - except TypeTranslationError: - self.fail('Type expected', node) - result.append(AnyType()) - return result - - def visit_decorator(self, dec: Decorator) -> None: - for d in dec.decorators: - d.accept(self) - removed = [] # type: List[int] - no_type_check = False - for i, d in enumerate(dec.decorators): - # A bunch of decorators are special cased here. - if refers_to_fullname(d, 'abc.abstractmethod'): - removed.append(i) - dec.func.is_abstract = True - self.check_decorated_function_is_method('abstractmethod', dec) - elif refers_to_fullname(d, 'asyncio.tasks.coroutine'): - removed.append(i) - elif refers_to_fullname(d, 'builtins.staticmethod'): - removed.append(i) - dec.func.is_static = True - dec.var.is_staticmethod = True - self.check_decorated_function_is_method('staticmethod', dec) - elif refers_to_fullname(d, 'builtins.classmethod'): - removed.append(i) - dec.func.is_class = True - dec.var.is_classmethod = True - self.check_decorated_function_is_method('classmethod', dec) - elif (refers_to_fullname(d, 'builtins.property') or - refers_to_fullname(d, 'abc.abstractproperty')): - removed.append(i) - dec.func.is_property = True - dec.var.is_property = True - if refers_to_fullname(d, 'abc.abstractproperty'): - dec.func.is_abstract = True - self.check_decorated_function_is_method('property', dec) - if len(dec.func.arguments) > 1: - self.fail('Too many arguments', dec.func) - elif refers_to_fullname(d, 'typing.no_type_check'): - dec.var.type = AnyType() - no_type_check = True - for i in reversed(removed): - del dec.decorators[i] - if not dec.is_overload or dec.var.is_property: - if self.is_func_scope(): - self.add_symbol(dec.var.name(), SymbolTableNode(LDEF, dec), - dec) - elif self.type: - dec.var.info = self.type - dec.var.is_initialized_in_class = True - self.add_symbol(dec.var.name(), SymbolTableNode(MDEF, dec), - dec) - if not no_type_check: - dec.func.accept(self) - if dec.decorators and dec.var.is_property: - self.fail('Decorated property not supported', dec) - - def check_decorated_function_is_method(self, decorator: str, - context: Context) -> None: - if not self.type or self.is_func_scope(): - self.fail("'%s' used with a non-method" % decorator, context) - - def visit_expression_stmt(self, s: ExpressionStmt) -> None: - s.expr.accept(self) - - def visit_return_stmt(self, s: ReturnStmt) -> None: - if not self.is_func_scope(): - self.fail("'return' outside function", s) - if s.expr: - s.expr.accept(self) - - def visit_raise_stmt(self, s: RaiseStmt) -> None: - if s.expr: - s.expr.accept(self) - if s.from_expr: - s.from_expr.accept(self) - - def visit_assert_stmt(self, s: AssertStmt) -> None: - if s.expr: - s.expr.accept(self) - - def visit_operator_assignment_stmt(self, - s: OperatorAssignmentStmt) -> None: - s.lvalue.accept(self) - s.rvalue.accept(self) - if (isinstance(s.lvalue, NameExpr) and s.lvalue.name == '__all__' and - s.lvalue.kind == GDEF and isinstance(s.rvalue, (ListExpr, TupleExpr))): - self.add_exports(*s.rvalue.items) - - def visit_while_stmt(self, s: WhileStmt) -> None: - s.expr.accept(self) - self.loop_depth += 1 - s.body.accept(self) - self.loop_depth -= 1 - self.visit_block_maybe(s.else_body) - - def visit_for_stmt(self, s: ForStmt) -> None: - s.expr.accept(self) - - # Bind index variables and check if they define new names. - self.analyze_lvalue(s.index) - - self.loop_depth += 1 - self.visit_block(s.body) - self.loop_depth -= 1 - - self.visit_block_maybe(s.else_body) - - def visit_break_stmt(self, s: BreakStmt) -> None: - if self.loop_depth == 0: - self.fail("'break' outside loop", s, True, blocker=True) - - def visit_continue_stmt(self, s: ContinueStmt) -> None: - if self.loop_depth == 0: - self.fail("'continue' outside loop", s, True, blocker=True) - - def visit_if_stmt(self, s: IfStmt) -> None: - infer_reachability_of_if_statement(s, pyversion=self.options.python_version) - for i in range(len(s.expr)): - s.expr[i].accept(self) - self.visit_block(s.body[i]) - self.visit_block_maybe(s.else_body) - - def visit_try_stmt(self, s: TryStmt) -> None: - self.analyze_try_stmt(s, self) - - def analyze_try_stmt(self, s: TryStmt, visitor: NodeVisitor, - add_global: bool = False) -> None: - s.body.accept(visitor) - for type, var, handler in zip(s.types, s.vars, s.handlers): - if type: - type.accept(visitor) - if var: - self.analyze_lvalue(var, add_global=add_global) - handler.accept(visitor) - if s.else_body: - s.else_body.accept(visitor) - if s.finally_body: - s.finally_body.accept(visitor) - - def visit_with_stmt(self, s: WithStmt) -> None: - for e, n in zip(s.expr, s.target): - e.accept(self) - if n: - self.analyze_lvalue(n) - self.visit_block(s.body) - - def visit_del_stmt(self, s: DelStmt) -> None: - s.expr.accept(self) - if not self.is_valid_del_target(s.expr): - self.fail('Invalid delete target', s) - - def is_valid_del_target(self, s: Node) -> bool: - if isinstance(s, (IndexExpr, NameExpr, MemberExpr)): - return True - elif isinstance(s, TupleExpr): - return all(self.is_valid_del_target(item) for item in s.items) - - def visit_global_decl(self, g: GlobalDecl) -> None: - for name in g.names: - if name in self.nonlocal_decls[-1]: - self.fail("Name '{}' is nonlocal and global".format(name), g) - self.global_decls[-1].add(name) - - def visit_nonlocal_decl(self, d: NonlocalDecl) -> None: - if not self.is_func_scope(): - self.fail("nonlocal declaration not allowed at module level", d) - else: - for name in d.names: - for table in reversed(self.locals[:-1]): - if table is not None and name in table: - break - else: - self.fail("No binding for nonlocal '{}' found".format(name), d) - - if self.locals[-1] is not None and name in self.locals[-1]: - self.fail("Name '{}' is already defined in local " - "scope before nonlocal declaration".format(name), d) - - if name in self.global_decls[-1]: - self.fail("Name '{}' is nonlocal and global".format(name), d) - self.nonlocal_decls[-1].add(name) - - def visit_print_stmt(self, s: PrintStmt) -> None: - for arg in s.args: - arg.accept(self) - if s.target: - s.target.accept(self) - - def visit_exec_stmt(self, s: ExecStmt) -> None: - s.expr.accept(self) - if s.variables1: - s.variables1.accept(self) - if s.variables2: - s.variables2.accept(self) - - # - # Expressions - # - - def visit_name_expr(self, expr: NameExpr) -> None: - n = self.lookup(expr.name, expr) - if n: - if n.kind == BOUND_TVAR: - self.fail("'{}' is a type variable and only valid in type " - "context".format(expr.name), expr) - else: - expr.kind = n.kind - expr.node = n.node - expr.fullname = n.fullname - - def visit_super_expr(self, expr: SuperExpr) -> None: - if not self.type: - self.fail('"super" used outside class', expr) - return - expr.info = self.type - - def visit_tuple_expr(self, expr: TupleExpr) -> None: - for item in expr.items: - item.accept(self) - - def visit_list_expr(self, expr: ListExpr) -> None: - for item in expr.items: - item.accept(self) - - def visit_set_expr(self, expr: SetExpr) -> None: - for item in expr.items: - item.accept(self) - - def visit_dict_expr(self, expr: DictExpr) -> None: - for key, value in expr.items: - key.accept(self) - value.accept(self) - - def visit_star_expr(self, expr: StarExpr) -> None: - if not expr.valid: - self.fail('Can use starred expression only as assignment target', expr) - else: - expr.expr.accept(self) - - def visit_yield_from_expr(self, e: YieldFromExpr) -> None: - if not self.is_func_scope(): # not sure - self.fail("'yield from' outside function", e, True, blocker=True) - else: - self.function_stack[-1].is_generator = True - if e.expr: - e.expr.accept(self) - - def visit_call_expr(self, expr: CallExpr) -> None: - """Analyze a call expression. - - Some call expressions are recognized as special forms, including - cast(...) and Any(...). - """ - expr.callee.accept(self) - if refers_to_fullname(expr.callee, 'typing.cast'): - # Special form cast(...). - if not self.check_fixed_args(expr, 2, 'cast'): - return - # Translate first argument to an unanalyzed type. - try: - target = expr_to_unanalyzed_type(expr.args[0]) - except TypeTranslationError: - self.fail('Cast target is not a type', expr) - return - # Piggyback CastExpr object to the CallExpr object; it takes - # precedence over the CallExpr semantics. - expr.analyzed = CastExpr(expr.args[1], target) - expr.analyzed.line = expr.line - expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'builtins.reveal_type'): - if not self.check_fixed_args(expr, 1, 'reveal_type'): - return - expr.analyzed = RevealTypeExpr(expr.args[0]) - expr.analyzed.line = expr.line - expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'typing.Any'): - # Special form Any(...). - if not self.check_fixed_args(expr, 1, 'Any'): - return - expr.analyzed = CastExpr(expr.args[0], AnyType()) - expr.analyzed.line = expr.line - expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'typing._promote'): - # Special form _promote(...). - if not self.check_fixed_args(expr, 1, '_promote'): - return - # Translate first argument to an unanalyzed type. - try: - target = expr_to_unanalyzed_type(expr.args[0]) - except TypeTranslationError: - self.fail('Argument 1 to _promote is not a type', expr) - return - expr.analyzed = PromoteExpr(target) - expr.analyzed.line = expr.line - expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, 'builtins.dict'): - expr.analyzed = self.translate_dict_call(expr) - else: - # Normal call expression. - for a in expr.args: - a.accept(self) - - if (isinstance(expr.callee, MemberExpr) and - isinstance(expr.callee.expr, NameExpr) and - expr.callee.expr.name == '__all__' and - expr.callee.expr.kind == GDEF and - expr.callee.name in ('append', 'extend')): - if expr.callee.name == 'append' and expr.args: - self.add_exports(expr.args[0]) - elif (expr.callee.name == 'extend' and expr.args and - isinstance(expr.args[0], (ListExpr, TupleExpr))): - self.add_exports(*expr.args[0].items) - - def translate_dict_call(self, call: CallExpr) -> Optional[DictExpr]: - """Translate 'dict(x=y, ...)' to {'x': y, ...}. - - For other variants of dict(...), return None. - """ - if not call.args: - return None - if not all(kind == ARG_NAMED for kind in call.arg_kinds): - # Must still accept those args. - for a in call.args: - a.accept(self) - return None - expr = DictExpr([(StrExpr(key), value) - for key, value in zip(call.arg_names, call.args)]) - expr.set_line(call) - expr.accept(self) - return expr - - def check_fixed_args(self, expr: CallExpr, numargs: int, - name: str) -> bool: - """Verify that expr has specified number of positional args. - - Return True if the arguments are valid. - """ - s = 's' - if numargs == 1: - s = '' - if len(expr.args) != numargs: - self.fail("'%s' expects %d argument%s" % (name, numargs, s), - expr) - return False - if expr.arg_kinds != [ARG_POS] * numargs: - self.fail("'%s' must be called with %s positional argument%s" % - (name, numargs, s), expr) - return False - return True - - def visit_member_expr(self, expr: MemberExpr) -> None: - base = expr.expr - base.accept(self) - # Bind references to module attributes. - if isinstance(base, RefExpr) and base.kind == MODULE_REF: - file = cast(MypyFile, base.node) - n = file.names.get(expr.name, None) if file is not None else None - if n: - n = self.normalize_type_alias(n, expr) - if not n: - return - expr.kind = n.kind - expr.fullname = n.fullname - expr.node = n.node - else: - # We only catch some errors here; the rest will be - # catched during type checking. - # - # This way we can report a larger number of errors in - # one type checker run. If we reported errors here, - # the build would terminate after semantic analysis - # and we wouldn't be able to report any type errors. - full_name = '%s.%s' % (file.fullname() if file is not None else None, expr.name) - if full_name in obsolete_name_mapping: - self.fail("Module has no attribute %r (it's now called %r)" % ( - expr.name, obsolete_name_mapping[full_name]), expr) - - def visit_op_expr(self, expr: OpExpr) -> None: - expr.left.accept(self) - expr.right.accept(self) - - def visit_comparison_expr(self, expr: ComparisonExpr) -> None: - for operand in expr.operands: - operand.accept(self) - - def visit_unary_expr(self, expr: UnaryExpr) -> None: - expr.expr.accept(self) - - def visit_index_expr(self, expr: IndexExpr) -> None: - expr.base.accept(self) - if refers_to_class_or_function(expr.base): - # Special form -- type application. - # Translate index to an unanalyzed type. - types = [] # type: List[Type] - if isinstance(expr.index, TupleExpr): - items = expr.index.items - else: - items = [expr.index] - for item in items: - try: - typearg = expr_to_unanalyzed_type(item) - except TypeTranslationError: - self.fail('Type expected within [...]', expr) - return - typearg = self.anal_type(typearg) - types.append(typearg) - expr.analyzed = TypeApplication(expr.base, types) - expr.analyzed.line = expr.line - else: - expr.index.accept(self) - - def visit_slice_expr(self, expr: SliceExpr) -> None: - if expr.begin_index: - expr.begin_index.accept(self) - if expr.end_index: - expr.end_index.accept(self) - if expr.stride: - expr.stride.accept(self) - - def visit_cast_expr(self, expr: CastExpr) -> None: - expr.expr.accept(self) - expr.type = self.anal_type(expr.type) - - def visit_reveal_type_expr(self, expr: RevealTypeExpr) -> None: - expr.expr.accept(self) - - def visit_type_application(self, expr: TypeApplication) -> None: - expr.expr.accept(self) - for i in range(len(expr.types)): - expr.types[i] = self.anal_type(expr.types[i]) - - def visit_list_comprehension(self, expr: ListComprehension) -> None: - expr.generator.accept(self) - - def visit_set_comprehension(self, expr: SetComprehension) -> None: - expr.generator.accept(self) - - def visit_dictionary_comprehension(self, expr: DictionaryComprehension) -> None: - self.enter() - self.analyze_comp_for(expr) - expr.key.accept(self) - expr.value.accept(self) - self.leave() - self.analyze_comp_for_2(expr) - - def visit_generator_expr(self, expr: GeneratorExpr) -> None: - self.enter() - self.analyze_comp_for(expr) - expr.left_expr.accept(self) - self.leave() - self.analyze_comp_for_2(expr) - - def analyze_comp_for(self, expr: Union[GeneratorExpr, - DictionaryComprehension]) -> None: - """Analyses the 'comp_for' part of comprehensions (part 1). - - That is the part after 'for' in (x for x in l if p). This analyzes - variables and conditions which are analyzed in a local scope. - """ - for i, (index, sequence, conditions) in enumerate(zip(expr.indices, - expr.sequences, - expr.condlists)): - if i > 0: - sequence.accept(self) - # Bind index variables. - self.analyze_lvalue(index) - for cond in conditions: - cond.accept(self) - - def analyze_comp_for_2(self, expr: Union[GeneratorExpr, - DictionaryComprehension]) -> None: - """Analyses the 'comp_for' part of comprehensions (part 2). - - That is the part after 'for' in (x for x in l if p). This analyzes - the 'l' part which is analyzed in the surrounding scope. - """ - expr.sequences[0].accept(self) - - def visit_func_expr(self, expr: FuncExpr) -> None: - self.analyze_function(expr) - - def visit_conditional_expr(self, expr: ConditionalExpr) -> None: - expr.if_expr.accept(self) - expr.cond.accept(self) - expr.else_expr.accept(self) - - def visit_backquote_expr(self, expr: BackquoteExpr) -> None: - expr.expr.accept(self) - - def visit__promote_expr(self, expr: PromoteExpr) -> None: - expr.type = self.anal_type(expr.type) - - def visit_yield_expr(self, expr: YieldExpr) -> None: - if not self.is_func_scope(): - self.fail("'yield' outside function", expr, True, blocker=True) - else: - self.function_stack[-1].is_generator = True - if expr.expr: - expr.expr.accept(self) - - # - # Helpers - # - - def lookup(self, name: str, ctx: Context) -> SymbolTableNode: - """Look up an unqualified name in all active namespaces.""" - # 1a. Name declared using 'global x' takes precedence - if name in self.global_decls[-1]: - if name in self.globals: - return self.globals[name] - else: - self.name_not_defined(name, ctx) - return None - # 1b. Name declared using 'nonlocal x' takes precedence - if name in self.nonlocal_decls[-1]: - for table in reversed(self.locals[:-1]): - if table is not None and name in table: - return table[name] - else: - self.name_not_defined(name, ctx) - return None - # 2. Class attributes (if within class definition) - if self.is_class_scope() and name in self.type.names: - return self.type[name] - # 3. Local (function) scopes - for table in reversed(self.locals): - if table is not None and name in table: - return table[name] - # 4. Current file global scope - if name in self.globals: - return self.globals[name] - # 5. Builtins - b = self.globals.get('__builtins__', None) - if b: - table = cast(MypyFile, b.node).names - if name in table: - if name[0] == "_" and name[1] != "_": - self.name_not_defined(name, ctx) - return None - node = table[name] - return node - # Give up. - self.name_not_defined(name, ctx) - self.check_for_obsolete_short_name(name, ctx) - return None - - def check_for_obsolete_short_name(self, name: str, ctx: Context) -> None: - matches = [obsolete_name - for obsolete_name in obsolete_name_mapping - if obsolete_name.rsplit('.', 1)[-1] == name] - if len(matches) == 1: - self.note("(Did you mean '{}'?)".format(obsolete_name_mapping[matches[0]]), ctx) - - def lookup_qualified(self, name: str, ctx: Context) -> SymbolTableNode: - if '.' not in name: - return self.lookup(name, ctx) - else: - parts = name.split('.') - n = self.lookup(parts[0], ctx) # type: SymbolTableNode - if n: - for i in range(1, len(parts)): - if isinstance(n.node, TypeInfo): - if n.node.mro is None: - # We haven't yet analyzed the class `n.node`. Fall back to direct - # lookup in the names declared directly under it, without its base - # classes. This can happen when we have a forward reference to a - # nested class, and the reference is bound before the outer class - # has been fully semantically analyzed. - # - # A better approach would be to introduce a new analysis pass or - # to move things around between passes, but this unblocks a common - # use case even though this is a little limited in case there is - # inheritance involved. - result = n.node.names.get(parts[i]) - else: - result = n.node.get(parts[i]) - n = result - elif isinstance(n.node, MypyFile): - n = n.node.names.get(parts[i], None) - # TODO: What if node is Var or FuncDef? - if not n: - self.name_not_defined(name, ctx) - break - if n: - n = self.normalize_type_alias(n, ctx) - return n - - def builtin_type(self, fully_qualified_name: str) -> Instance: - node = self.lookup_fully_qualified(fully_qualified_name) - info = cast(TypeInfo, node.node) - return Instance(info, []) - - def lookup_fully_qualified(self, name: str) -> SymbolTableNode: - """Lookup a fully qualified name. - - Assume that the name is defined. This happens in the global namespace -- the local - module namespace is ignored. - """ - assert '.' in name - parts = name.split('.') - n = self.modules[parts[0]] - for i in range(1, len(parts) - 1): - n = cast(MypyFile, n.names[parts[i]].node) - return n.names[parts[-1]] - - def lookup_fully_qualified_or_none(self, name: str) -> SymbolTableNode: - """Lookup a fully qualified name. - - Assume that the name is defined. This happens in the global namespace -- the local - module namespace is ignored. - """ - assert '.' in name - parts = name.split('.') - n = self.modules[parts[0]] - for i in range(1, len(parts) - 1): - next_sym = n.names.get(parts[i]) - if not next_sym: - return None - n = cast(MypyFile, next_sym.node) - return n.names.get(parts[-1]) - - def qualified_name(self, n: str) -> str: - return self.cur_mod_id + '.' + n - - def enter(self) -> None: - self.locals.append(SymbolTable()) - self.global_decls.append(set()) - self.nonlocal_decls.append(set()) - # -1 since entering block will increment this to 0. - self.block_depth.append(-1) - - def leave(self) -> None: - self.locals.pop() - self.global_decls.pop() - self.nonlocal_decls.pop() - self.block_depth.pop() - - def is_func_scope(self) -> bool: - return self.locals[-1] is not None - - def is_class_scope(self) -> bool: - return self.type is not None and not self.is_func_scope() - - def add_symbol(self, name: str, node: SymbolTableNode, - context: Context) -> None: - if self.is_func_scope(): - if name in self.locals[-1]: - # Flag redefinition unless this is a reimport of a module. - if not (node.kind == MODULE_REF and - self.locals[-1][name].node == node.node): - self.name_already_defined(name, context) - self.locals[-1][name] = node - elif self.type: - self.type.names[name] = node - else: - existing = self.globals.get(name) - if existing and (not isinstance(node.node, MypyFile) or - existing.node != node.node) and existing.kind != UNBOUND_IMPORTED: - # Modules can be imported multiple times to support import - # of multiple submodules of a package (e.g. a.x and a.y). - ok = False - # Only report an error if the symbol collision provides a different type. - if existing.type and node.type and is_same_type(existing.type, node.type): - ok = True - if not ok: - self.name_already_defined(name, context) - self.globals[name] = node - - def add_var(self, v: Var, ctx: Context) -> None: - if self.is_func_scope(): - self.add_local(v, ctx) - else: - self.globals[v.name()] = SymbolTableNode(GDEF, v, self.cur_mod_id) - v._fullname = self.qualified_name(v.name()) - - def add_local(self, node: Union[Var, FuncBase], ctx: Context) -> None: - name = node.name() - if name in self.locals[-1]: - self.name_already_defined(name, ctx) - node._fullname = name - self.locals[-1][name] = SymbolTableNode(LDEF, node) - - def add_exports(self, *exps: Node) -> None: - for exp in exps: - if isinstance(exp, StrExpr): - self.all_exports.add(exp.value) - - def check_no_global(self, n: str, ctx: Context, - is_overloaded_func: bool = False) -> None: - if n in self.globals: - prev_is_overloaded = isinstance(self.globals[n], OverloadedFuncDef) - if is_overloaded_func and prev_is_overloaded: - self.fail("Nonconsecutive overload {} found".format(n), ctx) - elif prev_is_overloaded: - self.fail("Definition of '{}' missing 'overload'".format(n), ctx) - else: - self.name_already_defined(n, ctx) - - def name_not_defined(self, name: str, ctx: Context) -> None: - message = "Name '{}' is not defined".format(name) - extra = self.undefined_name_extra_info(name) - if extra: - message += ' {}'.format(extra) - self.fail(message, ctx) - - def name_already_defined(self, name: str, ctx: Context) -> None: - self.fail("Name '{}' already defined".format(name), ctx) - - def fail(self, msg: str, ctx: Context, serious: bool = False, *, - blocker: bool = False) -> None: - if (not serious and - not self.options.check_untyped_defs and - self.function_stack and - self.function_stack[-1].is_dynamic()): - return - self.errors.report(ctx.get_line(), msg, blocker=blocker) - - def note(self, msg: str, ctx: Context) -> None: - if (not self.options.check_untyped_defs and - self.function_stack and - self.function_stack[-1].is_dynamic()): - return - self.errors.report(ctx.get_line(), msg, severity='note') - - def undefined_name_extra_info(self, fullname: str) -> Optional[str]: - if fullname in obsolete_name_mapping: - return "(it's now called '{}')".format(obsolete_name_mapping[fullname]) - else: - return None - - def accept(self, node: Node) -> None: - try: - node.accept(self) - except Exception as err: - report_internal_error(err, self.errors.file, node.line) - - -class FirstPass(NodeVisitor): - """First phase of semantic analysis. - - See docstring of 'analyze()' below for a description of what this does. - """ - - def __init__(self, sem: SemanticAnalyzer) -> None: - self.sem = sem - self.pyversion = sem.options.python_version - - def analyze(self, file: MypyFile, fnam: str, mod_id: str) -> None: - """Perform the first analysis pass. - - Populate module global table. Resolve the full names of - definitions not nested within functions and construct type - info structures, but do not resolve inter-definition - references such as base classes. - - Also add implicit definitions such as __name__. - - In this phase we don't resolve imports. For 'from ... import', - we generate dummy symbol table nodes for the imported names, - and these will get resolved in later phases of semantic - analysis. - """ - sem = self.sem - sem.cur_mod_id = mod_id - sem.errors.set_file(fnam) - sem.globals = SymbolTable() - sem.global_decls = [set()] - sem.nonlocal_decls = [set()] - sem.block_depth = [0] - - defs = file.defs - - # Add implicit definitions of module '__name__' etc. - for name, t in implicit_module_attrs.items(): - v = Var(name, UnboundType(t)) - v._fullname = self.sem.qualified_name(name) - self.sem.globals[name] = SymbolTableNode(GDEF, v, self.sem.cur_mod_id) - - for d in defs: - d.accept(self) - - # Add implicit definition of literals/keywords to builtins, as we - # cannot define a variable with them explicitly. - if mod_id == 'builtins': - literal_types = [ - ('None', NoneTyp()), - # reveal_type is a mypy-only function that gives an error with the type of its arg - ('reveal_type', AnyType()), - ] # type: List[Tuple[str, Type]] - - # TODO(ddfisher): This guard is only needed because mypy defines - # fake builtins for its tests which often don't define bool. If - # mypy is fast enough that we no longer need those, this - # conditional check should be removed. - if 'bool' in self.sem.globals: - bool_type = self.sem.named_type('bool') - literal_types.extend([ - ('True', bool_type), - ('False', bool_type), - ('__debug__', bool_type), - ]) - - for name, typ in literal_types: - v = Var(name, typ) - v._fullname = self.sem.qualified_name(name) - self.sem.globals[name] = SymbolTableNode(GDEF, v, self.sem.cur_mod_id) - - def visit_block(self, b: Block) -> None: - if b.is_unreachable: - return - self.sem.block_depth[-1] += 1 - for node in b.body: - node.accept(self) - self.sem.block_depth[-1] -= 1 - - def visit_assignment_stmt(self, s: AssignmentStmt) -> None: - for lval in s.lvalues: - self.analyze_lvalue(lval, explicit_type=s.type is not None) - - def visit_func_def(self, func: FuncDef) -> None: - sem = self.sem - func.is_conditional = sem.block_depth[-1] > 0 - func._fullname = sem.qualified_name(func.name()) - if func.name() in sem.globals: - # Already defined in this module. - original_sym = sem.globals[func.name()] - if original_sym.kind == UNBOUND_IMPORTED: - # Ah this is an imported name. We can't resolve them now, so we'll postpone - # this until the main phase of semantic analysis. - return - original_def = original_sym.node - if sem.is_conditional_func(original_def, func): - # Conditional function definition -- multiple defs are ok. - func.original_def = cast(FuncDef, original_def) - else: - # Report error. - sem.check_no_global(func.name(), func) - else: - sem.globals[func.name()] = SymbolTableNode(GDEF, func, sem.cur_mod_id) - - def visit_overloaded_func_def(self, func: OverloadedFuncDef) -> None: - self.sem.check_no_global(func.name(), func, True) - func._fullname = self.sem.qualified_name(func.name()) - self.sem.globals[func.name()] = SymbolTableNode(GDEF, func, - self.sem.cur_mod_id) - - def visit_class_def(self, cdef: ClassDef) -> None: - self.sem.check_no_global(cdef.name, cdef) - cdef.fullname = self.sem.qualified_name(cdef.name) - info = TypeInfo(SymbolTable(), cdef) - info.set_line(cdef.line) - cdef.info = info - self.sem.globals[cdef.name] = SymbolTableNode(GDEF, info, - self.sem.cur_mod_id) - self.process_nested_classes(cdef) - - def process_nested_classes(self, outer_def: ClassDef) -> None: - for node in outer_def.defs.body: - if isinstance(node, ClassDef): - node.info = TypeInfo(SymbolTable(), node) - if outer_def.fullname: - node.info._fullname = outer_def.fullname + '.' + node.info.name() - else: - node.info._fullname = node.info.name() - symbol = SymbolTableNode(MDEF, node.info) - outer_def.info.names[node.name] = symbol - self.process_nested_classes(node) - - def visit_import_from(self, node: ImportFrom) -> None: - # We can't bind module names during the first pass, as the target module might be - # unprocessed. However, we add dummy unbound imported names to the symbol table so - # that we at least know that the name refers to a module. - node.is_top_level = True - for name, as_name in node.names: - imported_name = as_name or name - if imported_name not in self.sem.globals: - self.sem.add_symbol(imported_name, SymbolTableNode(UNBOUND_IMPORTED, None), node) - - def visit_import(self, node: Import) -> None: - node.is_top_level = True - # This is similar to visit_import_from -- see the comment there. - for id, as_id in node.ids: - imported_id = as_id or id - if imported_id not in self.sem.globals: - self.sem.add_symbol(imported_id, SymbolTableNode(UNBOUND_IMPORTED, None), node) - else: - # If the previous symbol is a variable, this should take precedence. - self.sem.globals[imported_id] = SymbolTableNode(UNBOUND_IMPORTED, None) - - def visit_import_all(self, node: ImportAll) -> None: - node.is_top_level = True - - def visit_while_stmt(self, s: WhileStmt) -> None: - s.body.accept(self) - if s.else_body: - s.else_body.accept(self) - - def visit_for_stmt(self, s: ForStmt) -> None: - self.analyze_lvalue(s.index) - s.body.accept(self) - if s.else_body: - s.else_body.accept(self) - - def visit_with_stmt(self, s: WithStmt) -> None: - for n in s.target: - if n: - self.analyze_lvalue(n) - s.body.accept(self) - - def visit_decorator(self, d: Decorator) -> None: - d.var._fullname = self.sem.qualified_name(d.var.name()) - self.sem.add_symbol(d.var.name(), SymbolTableNode(GDEF, d.var), d) - - def visit_if_stmt(self, s: IfStmt) -> None: - infer_reachability_of_if_statement(s, pyversion=self.pyversion) - for node in s.body: - node.accept(self) - if s.else_body: - s.else_body.accept(self) - - def visit_try_stmt(self, s: TryStmt) -> None: - self.sem.analyze_try_stmt(s, self, add_global=True) - - def analyze_lvalue(self, lvalue: Node, explicit_type: bool = False) -> None: - self.sem.analyze_lvalue(lvalue, add_global=True, explicit_type=explicit_type) - - -class ThirdPass(TraverserVisitor): - """The third and final pass of semantic analysis. - - Check type argument counts and values of generic types, and perform some - straightforward type inference. - """ - - def __init__(self, modules: Dict[str, MypyFile], errors: Errors) -> None: - self.modules = modules - self.errors = errors - - def visit_file(self, file_node: MypyFile, fnam: str) -> None: - self.errors.set_file(fnam) - self.accept(file_node) - - def accept(self, node: Node) -> None: - try: - node.accept(self) - except Exception as err: - report_internal_error(err, self.errors.file, node.line) - - def visit_block(self, b: Block) -> None: - if b.is_unreachable: - return - super().visit_block(b) - - def visit_func_def(self, fdef: FuncDef) -> None: - self.errors.push_function(fdef.name()) - self.analyze(fdef.type) - super().visit_func_def(fdef) - self.errors.pop_function() - - def visit_class_def(self, tdef: ClassDef) -> None: - for type in tdef.info.bases: - self.analyze(type) - # Recompute MRO now that we have analyzed all modules, to pick - # up superclasses of bases imported from other modules in an - # import loop. (Only do so if we succeeded the first time.) - if tdef.info.mro: - tdef.info.mro = [] # Force recomputation - calculate_class_mro(tdef, self.fail) - super().visit_class_def(tdef) - - def visit_decorator(self, dec: Decorator) -> None: - """Try to infer the type of the decorated function. - - This lets us resolve references to decorated functions during - type checking when there are cyclic imports, as otherwise the - type might not be available when we need it. - - This basically uses a simple special-purpose type inference - engine just for decorators. - """ - super().visit_decorator(dec) - if dec.var.is_property: - # Decorators are expected to have a callable type (it's a little odd). - if dec.func.type is None: - dec.var.type = CallableType( - [AnyType()], - [ARG_POS], - [None], - AnyType(), - self.builtin_type('function'), - name=dec.var.name()) - elif isinstance(dec.func.type, CallableType): - dec.var.type = dec.func.type - return - decorator_preserves_type = True - for expr in dec.decorators: - preserve_type = False - if isinstance(expr, RefExpr) and isinstance(expr.node, FuncDef): - if is_identity_signature(expr.node.type): - preserve_type = True - if not preserve_type: - decorator_preserves_type = False - break - if decorator_preserves_type: - # No non-identity decorators left. We can trivially infer the type - # of the function here. - dec.var.type = function_type(dec.func, self.builtin_type('function')) - if dec.decorators: - if returns_any_if_called(dec.decorators[0]): - # The outermost decorator will return Any so we know the type of the - # decorated function. - dec.var.type = AnyType() - sig = find_fixed_callable_return(dec.decorators[0]) - if sig: - # The outermost decorator always returns the same kind of function, - # so we know that this is the type of the decoratored function. - orig_sig = function_type(dec.func, self.builtin_type('function')) - sig.name = orig_sig.items()[0].name - dec.var.type = sig - - def visit_assignment_stmt(self, s: AssignmentStmt) -> None: - self.analyze(s.type) - super().visit_assignment_stmt(s) - - def visit_cast_expr(self, e: CastExpr) -> None: - self.analyze(e.type) - super().visit_cast_expr(e) - - def visit_reveal_type_expr(self, e: RevealTypeExpr) -> None: - super().visit_reveal_type_expr(e) - - def visit_type_application(self, e: TypeApplication) -> None: - for type in e.types: - self.analyze(type) - super().visit_type_application(e) - - # Helpers - - def analyze(self, type: Type) -> None: - if type: - analyzer = TypeAnalyserPass3(self.fail) - type.accept(analyzer) - - def fail(self, msg: str, ctx: Context) -> None: - self.errors.report(ctx.get_line(), msg) - - def builtin_type(self, name: str, args: List[Type] = None) -> Instance: - names = self.modules['builtins'] - sym = names.names[name] - assert isinstance(sym.node, TypeInfo) - return Instance(sym.node, args or []) - - -def self_type(typ: TypeInfo) -> Union[Instance, TupleType]: - """For a non-generic type, return instance type representing the type. - For a generic G type with parameters T1, .., Tn, return G[T1, ..., Tn]. - """ - tv = [] # type: List[Type] - for i in range(len(typ.type_vars)): - tv.append(TypeVarType(typ.defn.type_vars[i])) - inst = Instance(typ, tv) - if typ.tuple_type is None: - return inst - return typ.tuple_type.self_type(inst) - - -def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: - if isinstance(sig, CallableType): - return replace_leading_arg_type(sig, new) - else: - sig = cast(Overloaded, sig) - return Overloaded([cast(CallableType, replace_implicit_first_type(i, new)) - for i in sig.items()]) - - -def set_callable_name(sig: Type, fdef: FuncDef) -> Type: - if isinstance(sig, FunctionLike): - if fdef.info: - return sig.with_name( - '"{}" of "{}"'.format(fdef.name(), fdef.info.name())) - else: - return sig.with_name('"{}"'.format(fdef.name())) - else: - return sig - - -def refers_to_fullname(node: Node, fullname: str) -> bool: - """Is node a name or member expression with the given full name?""" - return isinstance(node, RefExpr) and node.fullname == fullname - - -def refers_to_class_or_function(node: Node) -> bool: - """Does semantically analyzed node refer to a class?""" - return (isinstance(node, RefExpr) and - isinstance(node.node, (TypeInfo, FuncDef, OverloadedFuncDef))) - - -def calculate_class_mro(defn: ClassDef, fail: Callable[[str, Context], None]) -> None: - try: - defn.info.calculate_mro() - except MroError: - fail("Cannot determine consistent method resolution order " - '(MRO) for "%s"' % defn.name, defn) - defn.info.mro = [] - # The property of falling back to Any is inherited. - defn.info.fallback_to_any = any(baseinfo.fallback_to_any for baseinfo in defn.info.mro) - - -def find_duplicate(list: List[T]) -> T: - """If the list has duplicates, return one of the duplicates. - - Otherwise, return None. - """ - for i in range(1, len(list)): - if list[i] in list[:i]: - return list[i] - return None - - -def disable_typevars(nodes: List[SymbolTableNode]) -> None: - for node in nodes: - assert node.kind in (BOUND_TVAR, UNBOUND_TVAR) - node.kind = UNBOUND_TVAR - - -def enable_typevars(nodes: List[SymbolTableNode]) -> None: - for node in nodes: - assert node.kind in (BOUND_TVAR, UNBOUND_TVAR) - node.kind = BOUND_TVAR - - -def remove_imported_names_from_symtable(names: SymbolTable, - module: str) -> None: - """Remove all imported names from the symbol table of a module.""" - removed = [] # type: List[str] - for name, node in names.items(): - if node.node is None: - continue - fullname = node.node.fullname() - prefix = fullname[:fullname.rfind('.')] - if prefix != module: - removed.append(name) - for name in removed: - del names[name] - - -def infer_reachability_of_if_statement(s: IfStmt, - pyversion: Tuple[int, int]) -> None: - for i in range(len(s.expr)): - result = infer_if_condition_value(s.expr[i], pyversion) - if result == ALWAYS_FALSE: - # The condition is always false, so we skip the if/elif body. - mark_block_unreachable(s.body[i]) - elif result == ALWAYS_TRUE: - # This condition is always true, so all of the remaining - # elif/else bodies will never be executed. - for body in s.body[i + 1:]: - mark_block_unreachable(body) - if s.else_body: - mark_block_unreachable(s.else_body) - break - - -def infer_if_condition_value(expr: Node, pyversion: Tuple[int, int]) -> int: - """Infer whether if condition is always true/false. - - Return ALWAYS_TRUE if always true, ALWAYS_FALSE if always false, - and TRUTH_VALUE_UNKNOWN otherwise. - """ - name = '' - negated = False - alias = expr - if isinstance(alias, UnaryExpr): - if alias.op == 'not': - expr = alias.expr - negated = True - if isinstance(expr, NameExpr): - name = expr.name - elif isinstance(expr, MemberExpr): - name = expr.name - result = TRUTH_VALUE_UNKNOWN - if name == 'PY2': - result = ALWAYS_TRUE if pyversion[0] == 2 else ALWAYS_FALSE - elif name == 'PY3': - result = ALWAYS_TRUE if pyversion[0] == 3 else ALWAYS_FALSE - elif name == 'MYPY': - result = ALWAYS_TRUE - if negated: - if result == ALWAYS_TRUE: - result = ALWAYS_FALSE - elif result == ALWAYS_FALSE: - result = ALWAYS_TRUE - return result - - -def mark_block_unreachable(block: Block) -> None: - block.is_unreachable = True - block.accept(MarkImportsUnreachableVisitor()) - - -class MarkImportsUnreachableVisitor(TraverserVisitor): - """Visitor that flags all imports nested within a node as unreachable.""" - - def visit_import(self, node: Import) -> None: - node.is_unreachable = True - - def visit_import_from(self, node: ImportFrom) -> None: - node.is_unreachable = True - - def visit_import_all(self, node: ImportAll) -> None: - node.is_unreachable = True - - -def is_identity_signature(sig: Type) -> bool: - """Is type a callable of form T -> T (where T is a type variable)?""" - if isinstance(sig, CallableType) and sig.arg_kinds == [ARG_POS]: - if isinstance(sig.arg_types[0], TypeVarType) and isinstance(sig.ret_type, TypeVarType): - return sig.arg_types[0].id == sig.ret_type.id - return False - - -def returns_any_if_called(expr: Node) -> bool: - """Return True if we can predict that expr will return Any if called. - - This only uses information available during semantic analysis so this - will sometimes return False because of insufficient information (as - type inference hasn't run yet). - """ - if isinstance(expr, RefExpr): - if isinstance(expr.node, FuncDef): - typ = expr.node.type - if typ is None: - # No signature -> default to Any. - return True - # Explicit Any return? - return isinstance(typ, CallableType) and isinstance(typ.ret_type, AnyType) - elif isinstance(expr.node, Var): - typ = expr.node.type - return typ is None or isinstance(typ, AnyType) - elif isinstance(expr, CallExpr): - return returns_any_if_called(expr.callee) - return False - - -def find_fixed_callable_return(expr: Node) -> Optional[CallableType]: - if isinstance(expr, RefExpr): - if isinstance(expr.node, FuncDef): - typ = expr.node.type - if typ: - if isinstance(typ, CallableType) and has_no_typevars(typ.ret_type): - if isinstance(typ.ret_type, CallableType): - return typ.ret_type - elif isinstance(expr, CallExpr): - t = find_fixed_callable_return(expr.callee) - if t: - if isinstance(t.ret_type, CallableType): - return t.ret_type - return None - - -def has_no_typevars(typ: Type) -> bool: - return is_same_type(typ, erase_typevars(typ)) +"""The semantic analyzer. + +Bind names to definitions and do various other simple consistency +checks. For example, consider this program: + + x = 1 + y = x + +Here semantic analysis would detect that the assignment 'x = 1' +defines a new variable, the type of which is to be inferred (in a +later pass; type inference or type checking is not part of semantic +analysis). Also, it would bind both references to 'x' to the same +module-level variable node. The second assignment would also be +analyzed, and the type of 'y' marked as being inferred. + +Semantic analysis is the first analysis pass after parsing, and it is +subdivided into three passes: + + * FirstPass looks up externally visible names defined in a module but + ignores imports and local definitions. It helps enable (some) + cyclic references between modules, such as module 'a' that imports + module 'b' and used names defined in b *and* vice versa. The first + pass can be performed before dependent modules have been processed. + + * SemanticAnalyzer is the second pass. It does the bulk of the work. + It assumes that dependent modules have been semantically analyzed, + up to the second pass, unless there is a import cycle. + + * ThirdPass checks that type argument counts are valid; for example, + it will reject Dict[int]. We don't do this in the second pass, + since we infer the type argument counts of classes during this + pass, and it is possible to refer to classes defined later in a + file, which would not have the type argument count set yet. This + pass also recomputes the method resolution order of each class, in + case one of its bases belongs to a module involved in an import + loop. + +Semantic analysis of types is implemented in module mypy.typeanal. + +TODO: Check if the third pass slows down type checking significantly. + We could probably get rid of it -- for example, we could collect all + analyzed types in a collection and check them without having to + traverse the entire AST. +""" + +from typing import ( + List, Dict, Set, Tuple, cast, Any, overload, TypeVar, Union, Optional, Callable +) + +from mypy.nodes import ( + MypyFile, TypeInfo, Node, AssignmentStmt, FuncDef, OverloadedFuncDef, + ClassDef, Var, GDEF, MODULE_REF, FuncItem, Import, + ImportFrom, ImportAll, Block, LDEF, NameExpr, MemberExpr, + IndexExpr, TupleExpr, ListExpr, ExpressionStmt, ReturnStmt, + RaiseStmt, AssertStmt, OperatorAssignmentStmt, WhileStmt, + ForStmt, BreakStmt, ContinueStmt, IfStmt, TryStmt, WithStmt, DelStmt, + GlobalDecl, SuperExpr, DictExpr, CallExpr, RefExpr, OpExpr, UnaryExpr, + SliceExpr, CastExpr, RevealTypeExpr, TypeApplication, Context, SymbolTable, + SymbolTableNode, BOUND_TVAR, UNBOUND_TVAR, ListComprehension, GeneratorExpr, + FuncExpr, MDEF, FuncBase, Decorator, SetExpr, TypeVarExpr, + StrExpr, BytesExpr, PrintStmt, ConditionalExpr, PromoteExpr, + ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, MroError, type_aliases, + YieldFromExpr, NamedTupleExpr, NonlocalDecl, + SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr, + YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, COVARIANT, CONTRAVARIANT, + IntExpr, FloatExpr, UnicodeExpr, + INVARIANT, UNBOUND_IMPORTED, Expression, EllipsisExpr, NamedTupleTypeInfo +) +from mypy.visitor import NodeVisitor +from mypy.traverser import TraverserVisitor +from mypy.errors import Errors, report_internal_error +from mypy.types import ( + NoneTyp, CallableType, Overloaded, Instance, Type, TypeVarType, AnyType, + FunctionLike, UnboundType, TypeList, ErrorType, TypeVarDef, + replace_leading_arg_type, TupleType, UnionType, StarType, EllipsisType, + NamedTupleType +) +from mypy.nodes import function_type, implicit_module_attrs +from mypy.typeanal import TypeAnalyser, TypeAnalyserPass3, analyze_type_alias +from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError +from mypy.lex import lex +from mypy.parsetype import parse_type +from mypy.sametypes import is_same_type +from mypy.erasetype import erase_typevars +from mypy import defaults +from mypy.options import Options + + +T = TypeVar('T') + + +# Inferred value of an expression. +ALWAYS_TRUE = 0 +ALWAYS_FALSE = 1 +TRUTH_VALUE_UNKNOWN = 2 + +# Map from obsolete name to the current spelling. +obsolete_name_mapping = { + 'typing.Function': 'typing.Callable', + 'typing.typevar': 'typing.TypeVar', +} + +# Hard coded type promotions (shared between all Python versions). +# These add extra ad-hoc edges to the subtyping relation. For example, +# int is considered a subtype of float, even though there is no +# subclass relationship. +TYPE_PROMOTIONS = { + 'builtins.int': 'builtins.float', + 'builtins.float': 'builtins.complex', +} + +# Hard coded type promotions for Python 3. +# +# Note that the bytearray -> bytes promotion is a little unsafe +# as some functions only accept bytes objects. Here convenience +# trumps safety. +TYPE_PROMOTIONS_PYTHON3 = TYPE_PROMOTIONS.copy() +TYPE_PROMOTIONS_PYTHON3.update({ + 'builtins.bytearray': 'builtins.bytes', +}) + +# Hard coded type promotions for Python 2. +# +# These promotions are unsafe, but we are doing them anyway +# for convenience and also for Python 3 compatibility +# (bytearray -> str). +TYPE_PROMOTIONS_PYTHON2 = TYPE_PROMOTIONS.copy() +TYPE_PROMOTIONS_PYTHON2.update({ + 'builtins.str': 'builtins.unicode', + 'builtins.bytearray': 'builtins.str', +}) + +# When analyzing a function, should we analyze the whole function in one go, or +# should we only perform one phase of the analysis? The latter is used for +# nested functions. In the first phase we add the function to the symbol table +# but don't process body. In the second phase we process function body. This +# way we can have mutually recursive nested functions. +FUNCTION_BOTH_PHASES = 0 # Everthing in one go +FUNCTION_FIRST_PHASE_POSTPONE_SECOND = 1 # Add to symbol table but postpone body +FUNCTION_SECOND_PHASE = 2 # Only analyze body + + +class SemanticAnalyzer(NodeVisitor): + """Semantically analyze parsed mypy files. + + The analyzer binds names and does various consistency checks for a + parse tree. Note that type checking is performed as a separate + pass. + + This is the second phase of semantic analysis. + """ + + # Library search paths + lib_path = None # type: List[str] + # Module name space + modules = None # type: Dict[str, MypyFile] + # Global name space for current module + globals = None # type: SymbolTable + # Names declared using "global" (separate set for each scope) + global_decls = None # type: List[Set[str]] + # Names declated using "nonlocal" (separate set for each scope) + nonlocal_decls = None # type: List[Set[str]] + # Local names of function scopes; None for non-function scopes. + locals = None # type: List[SymbolTable] + # Nested block depths of scopes + block_depth = None # type: List[int] + # TypeInfo of directly enclosing class (or None) + type = None # type: TypeInfo + # Stack of outer classes (the second tuple item contains tvars). + type_stack = None # type: List[TypeInfo] + # Type variables that are bound by the directly enclosing class + bound_tvars = None # type: List[SymbolTableNode] + # Stack of type variables that were bound by outer classess + tvar_stack = None # type: List[List[SymbolTableNode]] + # Do weak type checking in this file + weak_opts = set() # type: Set[str] + + # Stack of functions being analyzed + function_stack = None # type: List[FuncItem] + # Stack of next available function type variable ids + next_function_tvar_id_stack = None # type: List[int] + + # Status of postponing analysis of nested function bodies. By using this we + # can have mutually recursive nested functions. Values are FUNCTION_x + # constants. Note that separate phasea are not used for methods. + postpone_nested_functions_stack = None # type: List[int] + # Postponed functions collected if + # postpone_nested_functions_stack[-1] == FUNCTION_FIRST_PHASE_POSTPONE_SECOND. + postponed_functions_stack = None # type: List[List[Node]] + + loop_depth = 0 # Depth of breakable loops + cur_mod_id = '' # Current module id (or None) (phase 2) + is_stub_file = False # Are we analyzing a stub file? + imports = None # type: Set[str] # Imported modules (during phase 2 analysis) + errors = None # type: Errors # Keeps track of generated errors + + def __init__(self, + lib_path: List[str], + errors: Errors, + options: Options) -> None: + """Construct semantic analyzer. + + Use lib_path to search for modules, and report analysis errors + using the Errors instance. + """ + self.locals = [None] + self.imports = set() + self.type = None + self.type_stack = [] + self.bound_tvars = None + self.tvar_stack = [] + self.function_stack = [] + self.next_function_tvar_id_stack = [-1] + self.block_depth = [0] + self.loop_depth = 0 + self.lib_path = lib_path + self.errors = errors + self.modules = {} + self.options = options + self.postpone_nested_functions_stack = [FUNCTION_BOTH_PHASES] + self.postponed_functions_stack = [] + self.all_exports = set() # type: Set[str] + + def visit_file(self, file_node: MypyFile, fnam: str) -> None: + self.errors.set_file(fnam) + self.cur_mod_node = file_node + self.cur_mod_id = file_node.fullname() + self.is_stub_file = fnam.lower().endswith('.pyi') + self.globals = file_node.names + self.weak_opts = file_node.weak_opts + + if 'builtins' in self.modules: + self.globals['__builtins__'] = SymbolTableNode( + MODULE_REF, self.modules['builtins'], self.cur_mod_id) + + for name in implicit_module_attrs: + v = self.globals[name].node + if isinstance(v, Var): + v.type = self.anal_type(v.type) + v.is_ready = True + + defs = file_node.defs + for d in defs: + self.accept(d) + + if self.cur_mod_id == 'builtins': + remove_imported_names_from_symtable(self.globals, 'builtins') + + if '__all__' in self.globals: + for name, g in self.globals.items(): + if name not in self.all_exports: + g.module_public = False + + def visit_func_def(self, defn: FuncDef) -> None: + phase_info = self.postpone_nested_functions_stack[-1] + if phase_info != FUNCTION_SECOND_PHASE: + self.function_stack.append(defn) + # First phase of analysis for function. + self.errors.push_function(defn.name()) + self.update_function_type_variables(defn) + self.errors.pop_function() + self.function_stack.pop() + + defn.is_conditional = self.block_depth[-1] > 0 + + # TODO(jukka): Figure out how to share the various cases. It doesn't + # make sense to have (almost) duplicate code (here and elsewhere) for + # 3 cases: module-level, class-level and local names. Maybe implement + # a common stack of namespaces. As the 3 kinds of namespaces have + # different semantics, this wouldn't always work, but it might still + # be a win. + if self.is_class_scope(): + # Method definition + defn.info = self.type + if not defn.is_decorated and not defn.is_overload: + if defn.name() in self.type.names: + # Redefinition. Conditional redefinition is okay. + n = self.type.names[defn.name()].node + if self.is_conditional_func(n, defn): + defn.original_def = cast(FuncDef, n) + else: + self.name_already_defined(defn.name(), defn) + self.type.names[defn.name()] = SymbolTableNode(MDEF, defn) + self.prepare_method_signature(defn) + elif self.is_func_scope(): + # Nested function + if not defn.is_decorated and not defn.is_overload: + if defn.name() in self.locals[-1]: + # Redefinition. Conditional redefinition is okay. + n = self.locals[-1][defn.name()].node + if self.is_conditional_func(n, defn): + defn.original_def = cast(FuncDef, n) + else: + self.name_already_defined(defn.name(), defn) + else: + self.add_local(defn, defn) + else: + # Top-level function + if not defn.is_decorated and not defn.is_overload: + symbol = self.globals.get(defn.name()) + if isinstance(symbol.node, FuncDef) and symbol.node != defn: + # This is redefinition. Conditional redefinition is okay. + original_def = symbol.node + if self.is_conditional_func(original_def, defn): + # Conditional function definition -- multiple defs are ok. + defn.original_def = original_def + else: + # Report error. + self.check_no_global(defn.name(), defn, True) + if phase_info == FUNCTION_FIRST_PHASE_POSTPONE_SECOND: + # Postpone this function (for the second phase). + self.postponed_functions_stack[-1].append(defn) + return + if phase_info != FUNCTION_FIRST_PHASE_POSTPONE_SECOND: + # Second phase of analysis for function. + self.errors.push_function(defn.name()) + self.analyze_function(defn) + self.errors.pop_function() + + def prepare_method_signature(self, func: FuncDef) -> None: + """Check basic signature validity and tweak annotation of self/cls argument.""" + # Only non-static methods are special. + if not func.is_static: + if not func.arguments: + self.fail('Method must have at least one argument', func) + elif func.type: + sig = cast(FunctionLike, func.type) + if func.is_class: + leading_type = self.class_type(self.type) + else: + leading_type = self_type(self.type) + func.type = replace_implicit_first_type(sig, leading_type) + + def is_conditional_func(self, previous: Node, new: FuncDef) -> bool: + """Does 'new' conditionally redefine 'previous'? + + We reject straight redefinitions of functions, as they are usually + a programming error. For example: + + . def f(): ... + . def f(): ... # Error: 'f' redefined + """ + return isinstance(previous, (FuncDef, Var)) and new.is_conditional + + def update_function_type_variables(self, defn: FuncDef) -> None: + """Make any type variables in the signature of defn explicit. + + Update the signature of defn to contain type variable definitions + if defn is generic. + """ + if defn.type: + functype = cast(CallableType, defn.type) + typevars = self.infer_type_variables(functype) + # Do not define a new type variable if already defined in scope. + typevars = [(name, tvar) for name, tvar in typevars + if not self.is_defined_type_var(name, defn)] + if typevars: + next_tvar_id = self.next_function_tvar_id() + defs = [TypeVarDef(tvar[0], next_tvar_id - i, + tvar[1].values, tvar[1].upper_bound, + tvar[1].variance) + for i, tvar in enumerate(typevars)] + functype.variables = defs + + def infer_type_variables(self, + type: CallableType) -> List[Tuple[str, TypeVarExpr]]: + """Return list of unique type variables referred to in a callable.""" + names = [] # type: List[str] + tvars = [] # type: List[TypeVarExpr] + for arg in type.arg_types + [type.ret_type]: + for name, tvar_expr in self.find_type_variables_in_type(arg): + if name not in names: + names.append(name) + tvars.append(tvar_expr) + return list(zip(names, tvars)) + + def find_type_variables_in_type( + self, type: Type) -> List[Tuple[str, TypeVarExpr]]: + """Return a list of all unique type variable references in type. + + This effectively does partial name binding, results of which are mostly thrown away. + """ + result = [] # type: List[Tuple[str, TypeVarExpr]] + if isinstance(type, UnboundType): + name = type.name + node = self.lookup_qualified(name, type) + if node and node.kind == UNBOUND_TVAR: + result.append((name, cast(TypeVarExpr, node.node))) + for arg in type.args: + result.extend(self.find_type_variables_in_type(arg)) + elif isinstance(type, TypeList): + for item in type.items: + result.extend(self.find_type_variables_in_type(item)) + elif isinstance(type, UnionType): + for item in type.items: + result.extend(self.find_type_variables_in_type(item)) + elif isinstance(type, AnyType): + pass + elif isinstance(type, EllipsisType) or isinstance(type, TupleType): + pass + else: + assert False, 'Unsupported type %s' % type + return result + + def is_defined_type_var(self, tvar: str, context: Node) -> bool: + return self.lookup_qualified(tvar, context).kind == BOUND_TVAR + + def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: + t = [] # type: List[CallableType] + for i, item in enumerate(defn.items): + # TODO support decorated overloaded functions properly + item.is_overload = True + item.func.is_overload = True + item.accept(self) + t.append(cast(CallableType, function_type(item.func, + self.builtin_type('builtins.function')))) + if item.func.is_property and i == 0: + # This defines a property, probably with a setter and/or deleter. + self.analyze_property_with_multi_part_definition(defn) + break + if not [dec for dec in item.decorators + if refers_to_fullname(dec, 'typing.overload')]: + self.fail("'overload' decorator expected", item) + + defn.type = Overloaded(t) + defn.type.line = defn.line + + if self.is_class_scope(): + self.type.names[defn.name()] = SymbolTableNode(MDEF, defn, + typ=defn.type) + defn.info = self.type + elif self.is_func_scope(): + self.add_local(defn, defn) + + def analyze_property_with_multi_part_definition(self, defn: OverloadedFuncDef) -> None: + """Analyze a property defined using multiple methods (e.g., using @x.setter). + + Assume that the first method (@property) has already been analyzed. + """ + defn.is_property = True + items = defn.items + for item in items[1:]: + if len(item.decorators) == 1: + node = item.decorators[0] + if isinstance(node, MemberExpr): + if node.name == 'setter': + # The first item represents the entire property. + defn.items[0].var.is_settable_property = True + # Get abstractness from the original definition. + item.func.is_abstract = items[0].func.is_abstract + else: + self.fail("Decorated property not supported", item) + item.func.accept(self) + + def next_function_tvar_id(self) -> int: + return self.next_function_tvar_id_stack[-1] + + def analyze_function(self, defn: FuncItem) -> None: + is_method = self.is_class_scope() + + tvarnodes = self.add_func_type_variables_to_symbol_table(defn) + next_function_tvar_id = min([self.next_function_tvar_id()] + + [n.tvar_def.id.raw_id - 1 for n in tvarnodes]) + self.next_function_tvar_id_stack.append(next_function_tvar_id) + + if defn.type: + # Signature must be analyzed in the surrounding scope so that + # class-level imported names and type variables are in scope. + defn.type = self.anal_type(defn.type) + self.check_function_signature(defn) + if isinstance(defn, FuncDef): + defn.type = set_callable_name(defn.type, defn) + for arg in defn.arguments: + if arg.initializer: + arg.initializer.accept(self) + self.function_stack.append(defn) + self.enter() + for arg in defn.arguments: + self.add_local(arg.variable, defn) + for arg in defn.arguments: + if arg.initialization_statement: + lvalue = arg.initialization_statement.lvalues[0] + lvalue.accept(self) + + # The first argument of a non-static, non-class method is like 'self' + # (though the name could be different), having the enclosing class's + # instance type. + if is_method and not defn.is_static and not defn.is_class and defn.arguments: + defn.arguments[0].variable.is_self = True + + # First analyze body of the function but ignore nested functions. + self.postpone_nested_functions_stack.append(FUNCTION_FIRST_PHASE_POSTPONE_SECOND) + self.postponed_functions_stack.append([]) + defn.body.accept(self) + + # Analyze nested functions (if any) as a second phase. + self.postpone_nested_functions_stack[-1] = FUNCTION_SECOND_PHASE + for postponed in self.postponed_functions_stack[-1]: + postponed.accept(self) + self.postpone_nested_functions_stack.pop() + self.postponed_functions_stack.pop() + + self.next_function_tvar_id_stack.pop() + disable_typevars(tvarnodes) + + self.leave() + self.function_stack.pop() + + def add_func_type_variables_to_symbol_table( + self, defn: FuncItem) -> List[SymbolTableNode]: + nodes = [] # type: List[SymbolTableNode] + if defn.type: + tt = defn.type + names = self.type_var_names() + items = cast(CallableType, tt).variables + for item in items: + name = item.name + if name in names: + self.name_already_defined(name, defn) + node = self.bind_type_var(name, item, defn) + nodes.append(node) + names.add(name) + return nodes + + def type_var_names(self) -> Set[str]: + if not self.type: + return set() + else: + return set(self.type.type_vars) + + def bind_type_var(self, fullname: str, tvar_def: TypeVarDef, + context: Context) -> SymbolTableNode: + node = self.lookup_qualified(fullname, context) + node.kind = BOUND_TVAR + node.tvar_def = tvar_def + return node + + def check_function_signature(self, fdef: FuncItem) -> None: + sig = cast(CallableType, fdef.type) + if len(sig.arg_types) < len(fdef.arguments): + self.fail('Type signature has too few arguments', fdef) + # Add dummy Any arguments to prevent crashes later. + extra_anys = [AnyType()] * (len(fdef.arguments) - len(sig.arg_types)) + sig.arg_types.extend(extra_anys) + elif len(sig.arg_types) > len(fdef.arguments): + self.fail('Type signature has too many arguments', fdef) + + def visit_class_def(self, defn: ClassDef) -> None: + self.clean_up_bases_and_infer_type_variables(defn) + self.setup_class_def_analysis(defn) + + self.bind_class_type_vars(defn) + + self.analyze_base_classes(defn) + self.analyze_metaclass(defn) + + for decorator in defn.decorators: + self.analyze_class_decorator(defn, decorator) + + self.enter_class(defn) + + self.setup_is_builtinclass(defn) + + # Analyze class body. + defn.defs.accept(self) + + self.calculate_abstract_status(defn.info) + self.setup_type_promotion(defn) + + self.leave_class() + self.unbind_class_type_vars() + + def enter_class(self, defn: ClassDef) -> None: + # Remember previous active class + self.type_stack.append(self.type) + self.locals.append(None) # Add class scope + self.block_depth.append(-1) # The class body increments this to 0 + self.postpone_nested_functions_stack.append(FUNCTION_BOTH_PHASES) + self.type = defn.info + + def leave_class(self) -> None: + """ Restore analyzer state. """ + self.postpone_nested_functions_stack.pop() + self.block_depth.pop() + self.locals.pop() + self.type = self.type_stack.pop() + + def bind_class_type_vars(self, defn: ClassDef) -> None: + """ Unbind type variables of previously active class and bind + the type variables for the active class. + """ + if self.bound_tvars: + disable_typevars(self.bound_tvars) + self.tvar_stack.append(self.bound_tvars) + self.bound_tvars = self.bind_class_type_variables_in_symbol_table(defn.info) + + def unbind_class_type_vars(self) -> None: + """ Unbind the active class' type vars and rebind the + type vars of the previously active class. + """ + disable_typevars(self.bound_tvars) + self.bound_tvars = self.tvar_stack.pop() + if self.bound_tvars: + enable_typevars(self.bound_tvars) + + def analyze_class_decorator(self, defn: ClassDef, decorator: Node) -> None: + decorator.accept(self) + + def setup_is_builtinclass(self, defn: ClassDef): + for decorator in defn.decorators: + if refers_to_fullname(decorator, 'typing.builtinclass'): + defn.is_builtinclass = True + if defn.fullname == 'builtins.object': + # Only 'object' is marked as a built-in class, as otherwise things elsewhere + # would break. We need a better way of dealing with built-in classes. + defn.is_builtinclass = True + + def calculate_abstract_status(self, typ: TypeInfo) -> None: + """Calculate abstract status of a class. + + Set is_abstract of the type to True if the type has an unimplemented + abstract attribute. Also compute a list of abstract attributes. + """ + concrete = set() # type: Set[str] + abstract = [] # type: List[str] + for base in typ.mro: + for name, symnode in base.names.items(): + node = symnode.node + if isinstance(node, OverloadedFuncDef): + # Unwrap an overloaded function definition. We can just + # check arbitrarily the first overload item. If the + # different items have a different abstract status, there + # should be an error reported elsewhere. + func = node.items[0] # type: Node + else: + func = node + if isinstance(func, Decorator): + fdef = func.func + if fdef.is_abstract and name not in concrete: + typ.is_abstract = True + abstract.append(name) + concrete.add(name) + typ.abstract_attributes = sorted(abstract) + + def setup_type_promotion(self, defn: ClassDef) -> None: + """Setup extra, ad-hoc subtyping relationships between classes (promotion). + + This includes things like 'int' being compatible with 'float'. + """ + promote_target = None # type: Type + for decorator in defn.decorators: + if isinstance(decorator, CallExpr): + analyzed = decorator.analyzed + if isinstance(analyzed, PromoteExpr): + # _promote class decorator (undocumented faeture). + promote_target = analyzed.type + if not promote_target: + promotions = (TYPE_PROMOTIONS_PYTHON3 if self.options.python_version[0] >= 3 + else TYPE_PROMOTIONS_PYTHON2) + if defn.fullname in promotions: + promote_target = self.named_type_or_none(promotions[defn.fullname]) + defn.info._promote = promote_target + + def clean_up_bases_and_infer_type_variables(self, defn: ClassDef) -> None: + """Remove extra base classes such as Generic and infer type vars. + + For example, consider this class: + + . class Foo(Bar, Generic[T]): ... + + Now we will remove Generic[T] from bases of Foo and infer that the + type variable 'T' is a type argument of Foo. + + Note that this is performed *before* semantic analysis. + """ + removed = [] # type: List[int] + type_vars = [] # type: List[TypeVarDef] + for i, base_expr in enumerate(defn.base_type_exprs): + try: + base = expr_to_unanalyzed_type(base_expr) + except TypeTranslationError: + # This error will be caught later. + continue + tvars = self.analyze_typevar_declaration(base) + if tvars is not None: + if type_vars: + self.fail('Duplicate Generic in bases', defn) + removed.append(i) + for j, (name, tvar_expr) in enumerate(tvars): + type_vars.append(TypeVarDef(name, j + 1, tvar_expr.values, + tvar_expr.upper_bound, tvar_expr.variance)) + if type_vars: + defn.type_vars = type_vars + if defn.info: + defn.info.type_vars = [tv.name for tv in type_vars] + for i in reversed(removed): + del defn.base_type_exprs[i] + + def analyze_typevar_declaration(self, t: Type) -> Optional[List[Tuple[str, TypeVarExpr]]]: + if not isinstance(t, UnboundType): + return None + unbound = t + sym = self.lookup_qualified(unbound.name, unbound) + if sym is None or sym.node is None: + return None + if sym.node.fullname() == 'typing.Generic': + tvars = [] # type: List[Tuple[str, TypeVarExpr]] + for arg in unbound.args: + tvar = self.analyze_unbound_tvar(arg) + if tvar: + tvars.append(tvar) + else: + self.fail('Free type variable expected in %s[...]' % + sym.node.name(), t) + return tvars + return None + + def analyze_unbound_tvar(self, t: Type) -> Tuple[str, TypeVarExpr]: + if not isinstance(t, UnboundType): + return None + unbound = t + sym = self.lookup_qualified(unbound.name, unbound) + if sym is not None and sym.kind == UNBOUND_TVAR: + return unbound.name, cast(TypeVarExpr, sym.node) + return None + + def setup_class_def_analysis(self, defn: ClassDef) -> None: + """Prepare for the analysis of a class definition.""" + if not defn.info: + defn.info = TypeInfo(SymbolTable(), defn) + defn.info._fullname = defn.info.name() + if self.is_func_scope() or self.type: + kind = MDEF + if self.is_func_scope(): + kind = LDEF + self.add_symbol(defn.name, SymbolTableNode(kind, defn.info), defn) + + def analyze_base_classes(self, defn: ClassDef) -> None: + """Analyze and set up base classes. + + This computes several attributes on the corresponding TypeInfo defn.info + related to the base classes: defn.info.bases, defn.info.mro, and + miscellaneous others (at least tuple_type, fallback_to_any, and is_enum.) + """ + + base_types = [] # type: List[Instance] + info = defn.info + for base_expr in defn.base_type_exprs: + try: + base = self.expr_to_analyzed_type(base_expr) + except TypeTranslationError: + self.fail('Invalid base class', base_expr) + info.fallback_to_any = True + continue + + if isinstance(base, TupleType): + if info.tuple_type: + self.fail("Class has two incompatible bases derived from tuple", defn) + defn.incompatible_baseclass = True + if (not self.is_stub_file + and not info.is_named_tuple + and base.fallback.type.fullname() == 'builtins.tuple'): + self.fail("Tuple[...] not supported as a base class outside a stub file", defn) + info.tuple_type = base + base_types.append(base.fallback) + elif isinstance(base, Instance): + base_types.append(base) + elif isinstance(base, AnyType): + info.fallback_to_any = True + else: + self.fail('Invalid base class', base_expr) + info.fallback_to_any = True + + # Add 'object' as implicit base if there is no other base class. + if (not base_types and defn.fullname != 'builtins.object'): + base_types.append(self.object_type()) + + info.bases = base_types + + # Calculate the MRO. It might be incomplete at this point if + # the bases of defn include classes imported from other + # modules in an import loop. We'll recompute it in ThirdPass. + if not self.verify_base_classes(defn): + info.mro = [] + return + calculate_class_mro(defn, self.fail) + # If there are cyclic imports, we may be missing 'object' in + # the MRO. Fix MRO if needed. + if info.mro and info.mro[-1].fullname() != 'builtins.object': + info.mro.append(self.object_type().type) + + def expr_to_analyzed_type(self, expr: Node) -> Type: + if isinstance(expr, CallExpr): + expr.accept(self) + info = self.check_namedtuple(expr) + if info is None: + # Some form of namedtuple is the only valid type that looks like a call + # expression. This isn't a valid type. + raise TypeTranslationError() + fallback = Instance(info, []) + return TupleType(info.tuple_type.items, fallback=fallback) + typ = expr_to_unanalyzed_type(expr) + return self.anal_type(typ) + + def verify_base_classes(self, defn: ClassDef) -> bool: + info = defn.info + for base in info.bases: + baseinfo = base.type + if self.is_base_class(info, baseinfo): + self.fail('Cycle in inheritance hierarchy', defn) + # Clear bases to forcefully get rid of the cycle. + info.bases = [] + if baseinfo.fullname() == 'builtins.bool': + self.fail("'%s' is not a valid base class" % + baseinfo.name(), defn) + return False + dup = find_duplicate(info.direct_base_classes()) + if dup: + self.fail('Duplicate base class "%s"' % dup.name(), defn) + return False + return True + + def is_base_class(self, t: TypeInfo, s: TypeInfo) -> bool: + """Determine if t is a base class of s (but do not use mro).""" + # Search the base class graph for t, starting from s. + worklist = [s] + visited = {s} + while worklist: + nxt = worklist.pop() + if nxt == t: + return True + for base in nxt.bases: + if base.type not in visited: + worklist.append(base.type) + visited.add(base.type) + return False + + def analyze_metaclass(self, defn: ClassDef) -> None: + if defn.metaclass: + sym = self.lookup_qualified(defn.metaclass, defn) + if sym is not None and not isinstance(sym.node, TypeInfo): + self.fail("Invalid metaclass '%s'" % defn.metaclass, defn) + + def object_type(self) -> Instance: + return self.named_type('__builtins__.object') + + def class_type(self, info: TypeInfo) -> Type: + # Construct a function type whose fallback is cls. + from mypy import checkmember # To avoid import cycle. + leading_type = checkmember.type_object_type(info, self.builtin_type) + if isinstance(leading_type, Overloaded): + # Overloaded __init__ is too complex to handle. Plus it's stubs only. + return AnyType() + else: + return leading_type + + def named_type(self, qualified_name: str, args: List[Type] = None) -> Instance: + sym = self.lookup_qualified(qualified_name, None) + return Instance(cast(TypeInfo, sym.node), args or []) + + def named_type_or_none(self, qualified_name: str) -> Instance: + sym = self.lookup_fully_qualified_or_none(qualified_name) + if not sym: + return None + return Instance(cast(TypeInfo, sym.node), []) + + def is_instance_type(self, t: Type) -> bool: + return isinstance(t, Instance) + + def bind_class_type_variables_in_symbol_table( + self, info: TypeInfo) -> List[SymbolTableNode]: + nodes = [] # type: List[SymbolTableNode] + for var, binder in zip(info.type_vars, info.defn.type_vars): + node = self.bind_type_var(var, binder, info) + nodes.append(node) + return nodes + + def visit_import(self, i: Import) -> None: + for id, as_id in i.ids: + if as_id is not None: + self.add_module_symbol(id, as_id, module_public=True, context=i) + else: + # Modules imported in a stub file without using 'as x' won't get exported when + # doing 'from m import *'. + module_public = not self.is_stub_file + base = id.split('.')[0] + self.add_module_symbol(base, base, module_public=module_public, + context=i) + + def add_module_symbol(self, id: str, as_id: str, module_public: bool, + context: Context) -> None: + if id in self.modules: + m = self.modules[id] + self.add_symbol(as_id, SymbolTableNode(MODULE_REF, m, self.cur_mod_id, + module_public=module_public), context) + else: + self.add_unknown_symbol(as_id, context) + + def visit_import_from(self, imp: ImportFrom) -> None: + import_id = self.correct_relative_import(imp) + if import_id in self.modules: + module = self.modules[import_id] + for id, as_id in imp.names: + node = module.names.get(id) + if node and node.kind != UNBOUND_IMPORTED: + node = self.normalize_type_alias(node, imp) + if not node: + return + imported_id = as_id or id + existing_symbol = self.globals.get(imported_id) + if existing_symbol: + # Import can redefine a variable. They get special treatment. + if self.process_import_over_existing_name( + imported_id, existing_symbol, node, imp): + continue + # 'from m import x as x' exports x in a stub file. + module_public = not self.is_stub_file or as_id is not None + symbol = SymbolTableNode(node.kind, node.node, + self.cur_mod_id, + node.type_override, + module_public=module_public) + self.add_symbol(imported_id, symbol, imp) + else: + message = "Module has no attribute '{}'".format(id) + extra = self.undefined_name_extra_info('{}.{}'.format(import_id, id)) + if extra: + message += " {}".format(extra) + self.fail(message, imp) + else: + # Missing module. + for id, as_id in imp.names: + self.add_unknown_symbol(as_id or id, imp) + + def process_import_over_existing_name(self, + imported_id: str, existing_symbol: SymbolTableNode, + module_symbol: SymbolTableNode, + import_node: ImportBase) -> bool: + if (existing_symbol.kind in (LDEF, GDEF, MDEF) and + isinstance(existing_symbol.node, (Var, FuncDef))): + # This is a valid import over an existing definition in the file. Construct a dummy + # assignment that we'll use to type check the import. + lvalue = NameExpr(imported_id) + lvalue.kind = existing_symbol.kind + lvalue.node = existing_symbol.node + rvalue = NameExpr(imported_id) + rvalue.kind = module_symbol.kind + rvalue.node = module_symbol.node + assignment = AssignmentStmt([lvalue], rvalue) + for node in assignment, lvalue, rvalue: + node.set_line(import_node) + import_node.assignments.append(assignment) + return True + return False + + def normalize_type_alias(self, node: SymbolTableNode, + ctx: Context) -> SymbolTableNode: + if node.fullname in type_aliases: + # Node refers to an aliased type such as typing.List; normalize. + node = self.lookup_qualified(type_aliases[node.fullname], ctx) + if node.fullname == 'typing.DefaultDict': + self.add_module_symbol('collections', '__mypy_collections__', False, ctx) + node = self.lookup_qualified('__mypy_collections__.defaultdict', ctx) + return node + + def correct_relative_import(self, node: Union[ImportFrom, ImportAll]) -> str: + if node.relative == 0: + return node.id + + parts = self.cur_mod_id.split(".") + cur_mod_id = self.cur_mod_id + + rel = node.relative + if self.cur_mod_node.is_package_init_file(): + rel -= 1 + if len(parts) < rel: + self.fail("Relative import climbs too many namespaces", node) + if rel != 0: + cur_mod_id = ".".join(parts[:-rel]) + + return cur_mod_id + (("." + node.id) if node.id else "") + + def visit_import_all(self, i: ImportAll) -> None: + i_id = self.correct_relative_import(i) + if i_id in self.modules: + m = self.modules[i_id] + for name, node in m.names.items(): + node = self.normalize_type_alias(node, i) + if not name.startswith('_') and node.module_public: + existing_symbol = self.globals.get(name) + if existing_symbol: + # Import can redefine a variable. They get special treatment. + if self.process_import_over_existing_name( + name, existing_symbol, node, i): + continue + self.add_symbol(name, SymbolTableNode(node.kind, node.node, + self.cur_mod_id), i) + else: + # Don't add any dummy symbols for 'from x import *' if 'x' is unknown. + pass + + def add_unknown_symbol(self, name: str, context: Context) -> None: + var = Var(name) + var._fullname = self.qualified_name(name) + var.is_ready = True + var.type = AnyType() + self.add_symbol(name, SymbolTableNode(GDEF, var, self.cur_mod_id), context) + + # + # Statements + # + + def visit_block(self, b: Block) -> None: + if b.is_unreachable: + return + self.block_depth[-1] += 1 + for s in b.body: + self.accept(s) + self.block_depth[-1] -= 1 + + def visit_block_maybe(self, b: Block) -> None: + if b: + self.visit_block(b) + + def anal_type(self, t: Type, allow_tuple_literal: bool = False) -> Type: + if t: + if allow_tuple_literal: + # Types such as (t1, t2, ...) only allowed in assignment statements. They'll + # generate errors elsewhere, and Tuple[t1, t2, ...] must be used instead. + if isinstance(t, TupleType): + # Unlike TypeAnalyser, also allow implicit tuple types (without Tuple[...]). + star_count = sum(1 for item in t.items if isinstance(item, StarType)) + if star_count > 1: + self.fail('At most one star type allowed in a tuple', t) + return TupleType([AnyType() for _ in t.items], + self.builtin_type('builtins.tuple'), t.line) + items = [self.anal_type(item, True) + for item in t.items] + return TupleType(items, self.builtin_type('builtins.tuple'), t.line) + a = TypeAnalyser(self.lookup_qualified, + self.lookup_fully_qualified, + self.fail) + return t.accept(a) + else: + return None + + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + for lval in s.lvalues: + self.analyze_lvalue(lval, explicit_type=s.type is not None) + s.rvalue.accept(self) + if s.type: + allow_tuple_literal = isinstance(s.lvalues[-1], (TupleExpr, ListExpr)) + s.type = self.anal_type(s.type, allow_tuple_literal) + else: + # For simple assignments, allow binding type aliases. + # Also set the type if the rvalue is a simple literal. + if (s.type is None and len(s.lvalues) == 1 and + isinstance(s.lvalues[0], NameExpr)): + if s.lvalues[0].is_def: + s.type = self.analyze_simple_literal_type(s.rvalue) + res = analyze_type_alias(s.rvalue, + self.lookup_qualified, + self.lookup_fully_qualified, + self.fail) + if res and (not isinstance(res, Instance) or cast(Instance, res).args): + # TODO: What if this gets reassigned? + name = s.lvalues[0] + node = self.lookup(name.name, name) + node.kind = TYPE_ALIAS + node.type_override = res + if isinstance(s.rvalue, IndexExpr): + s.rvalue.analyzed = TypeAliasExpr(res) + if s.type: + # Store type into nodes. + for lvalue in s.lvalues: + self.store_declared_types(lvalue, s.type) + self.check_and_set_up_type_alias(s) + self.process_typevar_declaration(s) + self.process_namedtuple_definition(s) + + if (len(s.lvalues) == 1 and isinstance(s.lvalues[0], NameExpr) and + s.lvalues[0].name == '__all__' and s.lvalues[0].kind == GDEF and + isinstance(s.rvalue, (ListExpr, TupleExpr))): + self.add_exports(*s.rvalue.items) + + def analyze_simple_literal_type(self, rvalue: Node) -> Optional[Type]: + """Return builtins.int if rvalue is an int literal, etc.""" + if self.weak_opts or self.options.semantic_analysis_only or self.function_stack: + # Skip this if any weak options are set. + # Also skip if we're only doing the semantic analysis pass. + # This is mostly to avoid breaking unit tests. + # Also skip inside a function; this is to avoid confusing + # the code that handles dead code due to isinstance() + # inside type variables with value restrictions (like + # AnyStr). + return None + if isinstance(rvalue, IntExpr): + return self.named_type_or_none('builtins.int') + if isinstance(rvalue, FloatExpr): + return self.named_type_or_none('builtins.float') + if isinstance(rvalue, StrExpr): + return self.named_type_or_none('builtins.str') + if isinstance(rvalue, BytesExpr): + return self.named_type_or_none('builtins.bytes') + if isinstance(rvalue, UnicodeExpr): + return self.named_type_or_none('builtins.unicode') + return None + + def check_and_set_up_type_alias(self, s: AssignmentStmt) -> None: + """Check if assignment creates a type alias and set it up as needed.""" + # For now, type aliases only work at the top level of a module. + if (len(s.lvalues) == 1 and not self.is_func_scope() and not self.type + and not s.type): + lvalue = s.lvalues[0] + if isinstance(lvalue, NameExpr): + if not lvalue.is_def: + # Only a definition can create a type alias, not regular assignment. + return + rvalue = s.rvalue + if isinstance(rvalue, RefExpr): + node = rvalue.node + if isinstance(node, TypeInfo): + # TODO: We should record the fact that this is a variable + # that refers to a type, rather than making this + # just an alias for the type. + self.globals[lvalue.name].node = node + + def analyze_lvalue(self, lval: Node, nested: bool = False, + add_global: bool = False, + explicit_type: bool = False) -> None: + """Analyze an lvalue or assignment target. + + Only if add_global is True, add name to globals table. If nested + is true, the lvalue is within a tuple or list lvalue expression. + """ + + if isinstance(lval, NameExpr): + # Top-level definitions within some statements (at least while) are + # not handled in the first pass, so they have to be added now. + nested_global = (not self.is_func_scope() and + self.block_depth[-1] > 0 and + not self.type) + if (add_global or nested_global) and lval.name not in self.globals: + # Define new global name. + v = Var(lval.name) + v._fullname = self.qualified_name(lval.name) + v.is_ready = False # Type not inferred yet + lval.node = v + lval.is_def = True + lval.kind = GDEF + lval.fullname = v._fullname + self.globals[lval.name] = SymbolTableNode(GDEF, v, + self.cur_mod_id) + elif isinstance(lval.node, Var) and lval.is_def: + # Since the is_def flag is set, this must have been analyzed + # already in the first pass and added to the symbol table. + assert lval.node.name() in self.globals + elif (self.is_func_scope() and lval.name not in self.locals[-1] and + lval.name not in self.global_decls[-1] and + lval.name not in self.nonlocal_decls[-1]): + # Define new local name. + v = Var(lval.name) + lval.node = v + lval.is_def = True + lval.kind = LDEF + lval.fullname = lval.name + self.add_local(v, lval) + elif not self.is_func_scope() and (self.type and + lval.name not in self.type.names): + # Define a new attribute within class body. + v = Var(lval.name) + v.info = self.type + v.is_initialized_in_class = True + v.set_line(lval) + lval.node = v + lval.is_def = True + lval.kind = MDEF + lval.fullname = lval.name + self.type.names[lval.name] = SymbolTableNode(MDEF, v) + else: + # Bind to an existing name. + if explicit_type: + self.name_already_defined(lval.name, lval) + lval.accept(self) + self.check_lvalue_validity(lval.node, lval) + elif isinstance(lval, MemberExpr): + if not add_global: + self.analyze_member_lvalue(lval) + if explicit_type and not self.is_self_member_ref(lval): + self.fail('Type cannot be declared in assignment to non-self ' + 'attribute', lval) + elif isinstance(lval, IndexExpr): + if explicit_type: + self.fail('Unexpected type declaration', lval) + if not add_global: + lval.accept(self) + elif (isinstance(lval, TupleExpr) or + isinstance(lval, ListExpr)): + items = cast(Any, lval).items + if len(items) == 0 and isinstance(lval, TupleExpr): + self.fail("Can't assign to ()", lval) + self.analyze_tuple_or_list_lvalue(cast(Union[ListExpr, TupleExpr], lval), + add_global, explicit_type) + elif isinstance(lval, StarExpr): + if nested: + self.analyze_lvalue(lval.expr, nested, add_global, explicit_type) + else: + self.fail('Starred assignment target must be in a list or tuple', lval) + else: + self.fail('Invalid assignment target', lval) + + def analyze_tuple_or_list_lvalue(self, lval: Union[ListExpr, TupleExpr], + add_global: bool = False, + explicit_type: bool = False) -> None: + """Analyze an lvalue or assignment target that is a list or tuple.""" + items = lval.items + star_exprs = [cast(StarExpr, item) + for item in items + if isinstance(item, StarExpr)] + + if len(star_exprs) > 1: + self.fail('Two starred expressions in assignment', lval) + else: + if len(star_exprs) == 1: + star_exprs[0].valid = True + for i in items: + self.analyze_lvalue(i, nested=True, add_global=add_global, + explicit_type = explicit_type) + + def analyze_member_lvalue(self, lval: MemberExpr) -> None: + lval.accept(self) + if (self.is_self_member_ref(lval) and + self.type.get(lval.name) is None): + # Implicit attribute definition in __init__. + lval.is_def = True + v = Var(lval.name) + v.info = self.type + v.is_ready = False + lval.def_var = v + lval.node = v + self.type.names[lval.name] = SymbolTableNode(MDEF, v) + self.check_lvalue_validity(lval.node, lval) + + def is_self_member_ref(self, memberexpr: MemberExpr) -> bool: + """Does memberexpr to refer to an attribute of self?""" + if not isinstance(memberexpr.expr, NameExpr): + return False + node = memberexpr.expr.node + return isinstance(node, Var) and node.is_self + + def check_lvalue_validity(self, node: Node, ctx: Context) -> None: + if isinstance(node, (TypeInfo, TypeVarExpr)): + self.fail('Invalid assignment target', ctx) + + def store_declared_types(self, lvalue: Node, typ: Type) -> None: + if isinstance(typ, StarType) and not isinstance(lvalue, StarExpr): + self.fail('Star type only allowed for starred expressions', lvalue) + if isinstance(lvalue, RefExpr): + lvalue.is_def = False + if isinstance(lvalue.node, Var): + var = lvalue.node + var.type = typ + var.is_ready = True + # If node is not a variable, we'll catch it elsewhere. + elif isinstance(lvalue, TupleExpr): + if isinstance(typ, TupleType): + if len(lvalue.items) != len(typ.items): + self.fail('Incompatible number of tuple items', lvalue) + return + for item, itemtype in zip(lvalue.items, typ.items): + self.store_declared_types(item, itemtype) + else: + self.fail('Tuple type expected for multiple variables', + lvalue) + elif isinstance(lvalue, StarExpr): + if isinstance(typ, StarType): + self.store_declared_types(lvalue.expr, typ.type) + else: + self.fail('Star type expected for starred expression', lvalue) + else: + # This has been flagged elsewhere as an error, so just ignore here. + pass + + def process_typevar_declaration(self, s: AssignmentStmt) -> None: + """Check if s declares a TypeVar; it yes, store it in symbol table.""" + call = self.get_typevar_declaration(s) + if not call: + return + + lvalue = cast(NameExpr, s.lvalues[0]) + name = lvalue.name + if not lvalue.is_def: + if s.type: + self.fail("Cannot declare the type of a type variable", s) + else: + self.fail("Cannot redefine '%s' as a type variable" % name, s) + return + + if not self.check_typevar_name(call, name, s): + return + + # Constraining types + n_values = call.arg_kinds[1:].count(ARG_POS) + values = self.analyze_types(call.args[1:1 + n_values]) + + res = self.process_typevar_parameters(call.args[1 + n_values:], + call.arg_names[1 + n_values:], + call.arg_kinds[1 + n_values:], + bool(values), + s) + if res is None: + return + variance, upper_bound = res + + # Yes, it's a valid type variable definition! Add it to the symbol table. + node = self.lookup(name, s) + node.kind = UNBOUND_TVAR + TypeVar = TypeVarExpr(name, node.fullname, values, upper_bound, variance) + TypeVar.line = call.line + call.analyzed = TypeVar + node.node = TypeVar + + def check_typevar_name(self, call: CallExpr, name: str, context: Context) -> bool: + if len(call.args) < 1: + self.fail("Too few arguments for TypeVar()", context) + return False + if not isinstance(call.args[0], (StrExpr, BytesExpr)) or not call.arg_kinds[0] == ARG_POS: + self.fail("TypeVar() expects a string literal as first argument", context) + return False + if cast(StrExpr, call.args[0]).value != name: + self.fail("Unexpected TypeVar() argument value", context) + return False + return True + + def get_typevar_declaration(self, s: AssignmentStmt) -> Optional[CallExpr]: + """Returns the TypeVar() call expression if `s` is a type var declaration + or None otherwise. + """ + if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): + return None + if not isinstance(s.rvalue, CallExpr): + return None + call = s.rvalue + if not isinstance(call.callee, RefExpr): + return None + callee = call.callee + if callee.fullname != 'typing.TypeVar': + return None + return call + + def process_typevar_parameters(self, args: List[Node], + names: List[Optional[str]], + kinds: List[int], + has_values: bool, + context: Context) -> Optional[Tuple[int, Type]]: + covariant = False + contravariant = False + upper_bound = self.object_type() # type: Type + for param_value, param_name, param_kind in zip(args, names, kinds): + if not param_kind == ARG_NAMED: + self.fail("Unexpected argument to TypeVar()", context) + return None + if param_name == 'covariant': + if isinstance(param_value, NameExpr): + if param_value.name == 'True': + covariant = True + else: + self.fail("TypeVar 'covariant' may only be 'True'", context) + return None + else: + self.fail("TypeVar 'covariant' may only be 'True'", context) + return None + elif param_name == 'contravariant': + if isinstance(param_value, NameExpr): + if param_value.name == 'True': + contravariant = True + else: + self.fail("TypeVar 'contravariant' may only be 'True'", context) + return None + else: + self.fail("TypeVar 'contravariant' may only be 'True'", context) + return None + elif param_name == 'bound': + if has_values: + self.fail("TypeVar cannot have both values and an upper bound", context) + return None + try: + upper_bound = self.expr_to_analyzed_type(param_value) + except TypeTranslationError: + self.fail("TypeVar 'bound' must be a type", param_value) + return None + elif param_name == 'values': + # Probably using obsolete syntax with values=(...). Explain the current syntax. + self.fail("TypeVar 'values' argument not supported", context) + self.fail("Use TypeVar('T', t, ...) instead of TypeVar('T', values=(t, ...))", + context) + return None + else: + self.fail("Unexpected argument to TypeVar(): {}".format(param_name), context) + return None + + if covariant and contravariant: + self.fail("TypeVar cannot be both covariant and contravariant", context) + return None + elif covariant: + variance = COVARIANT + elif contravariant: + variance = CONTRAVARIANT + else: + variance = INVARIANT + return (variance, upper_bound) + + def process_namedtuple_definition(self, s: AssignmentStmt) -> None: + """Check if s defines a namedtuple; if yes, store the definition in symbol table.""" + if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): + return + lvalue = cast(NameExpr, s.lvalues[0]) + name = lvalue.name + named_tuple = self.check_namedtuple(s.rvalue, name) + if named_tuple is None: + return + # Yes, it's a valid namedtuple definition. Add it to the symbol table. + node = self.lookup(name, s) + node.kind = GDEF # TODO locally defined namedtuple + # TODO call.analyzed + node.node = named_tuple + + def check_namedtuple(self, node: Node, var_name: str = None) -> TypeInfo: + """Check if a call defines a namedtuple. + + The optional var_name argument is the name of the variable to + which this is assigned, if any. + + If it does, return the corresponding TypeInfo. Return None otherwise. + + If the definition is invalid but looks like a namedtuple, + report errors but return (some) TypeInfo. + """ + if not isinstance(node, CallExpr): + return None + call = node + if not isinstance(call.callee, RefExpr): + return None + callee = call.callee + fullname = callee.fullname + if fullname not in ('collections.namedtuple', 'typing.NamedTuple'): + return None + items, types, ok = self.parse_namedtuple_args(call, fullname) + if not ok: + # Error. Construct dummy return value. + return self.build_namedtuple_typeinfo('namedtuple', [], []) + else: + # Give it a unique name derived from the line number. + name = cast(StrExpr, call.args[0]).value + if name != var_name: + name += '@' + str(call.line) + info = self.build_namedtuple_typeinfo(name, items, types) + # Store it as a global just in case it would remain anonymous. + self.globals[name] = SymbolTableNode(GDEF, info, self.cur_mod_id) + call.analyzed = NamedTupleExpr(info).set_line(call.line) + return info + + def parse_namedtuple_args(self, call: CallExpr, + fullname: str) -> Tuple[List[str], List[Type], bool]: + # TODO Share code with check_argument_count in checkexpr.py? + args = call.args + if len(args) < 2: + return self.fail_namedtuple_arg("Too few arguments for namedtuple()", call) + if len(args) > 2: + return self.fail_namedtuple_arg("Too many arguments for namedtuple()", call) + if call.arg_kinds != [ARG_POS, ARG_POS]: + return self.fail_namedtuple_arg("Unexpected arguments to namedtuple()", call) + if not isinstance(args[0], (StrExpr, BytesExpr)): + return self.fail_namedtuple_arg( + "namedtuple() expects a string literal as the first argument", call) + types = [] # type: List[Type] + ok = True + if not isinstance(args[1], ListExpr): + if fullname == 'collections.namedtuple' and isinstance(args[1], (StrExpr, BytesExpr)): + str_expr = cast(StrExpr, args[1]) + items = str_expr.value.split() + else: + return self.fail_namedtuple_arg( + "List literal expected as the second argument to namedtuple()", call) + else: + listexpr = args[1] + if fullname == 'collections.namedtuple': + # The fields argument contains just names, with implicit Any types. + if any(not isinstance(item, (StrExpr, BytesExpr)) for item in listexpr.items): + return self.fail_namedtuple_arg("String literal expected as namedtuple() item", + call) + items = [cast(StrExpr, item).value for item in listexpr.items] + else: + # The fields argument contains (name, type) tuples. + items, types, ok = self.parse_namedtuple_fields_with_types(listexpr.items, call) + if not types: + types = [AnyType() for _ in items] + return items, types, ok + + def parse_namedtuple_fields_with_types(self, nodes: List[Node], + context: Context) -> Tuple[List[str], List[Type], bool]: + items = [] # type: List[str] + types = [] # type: List[Type] + for item in nodes: + if isinstance(item, TupleExpr): + if len(item.items) != 2: + return self.fail_namedtuple_arg("Invalid NamedTuple field definition", + item) + name, type_node = item.items + if isinstance(name, (StrExpr, BytesExpr)): + items.append(name.value) + else: + return self.fail_namedtuple_arg("Invalid NamedTuple() field name", item) + try: + type = expr_to_unanalyzed_type(type_node) + except TypeTranslationError: + return self.fail_namedtuple_arg('Invalid field type', type_node) + types.append(self.anal_type(type)) + else: + return self.fail_namedtuple_arg("Tuple expected as NamedTuple() field", item) + return items, types, True + + def fail_namedtuple_arg(self, message: str, + context: Context) -> Tuple[List[str], List[Type], bool]: + self.fail(message, context) + return [], [], False + + def build_namedtuple_typeinfo(self, name: str, items: List[str], + types: List[Type]) -> TypeInfo: + symbols = SymbolTable() + tup = NamedTupleType(name, items, types, self.named_type('__builtins__.tuple', types)) + class_def = ClassDef(name, Block([])) + class_def.fullname = self.qualified_name(name) + info = NamedTupleTypeInfo(tup, symbols, class_def) + info.attrs = items + vars = [Var(item, typ) for item, typ in zip(items, types)] + # Add named tuple items as attributes. + # TODO: Make them read-only. + for var in vars: + var.info = info + symbols[var.name()] = SymbolTableNode(MDEF, var) + this_type = self_type(info) + # Add __init__, _replace, _asdict methods and _fields static field + self.add_namedtuple_method('_replace', symbols, info, this_type, this_type, + self.make_factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) + self.add_namedtuple_method('__init__', symbols, info, this_type, NoneTyp(), + self.make_factory_args(vars, ARG_POS), info.name()) + # TODO: refine to OrderedDict[Union[types]] + self.add_namedtuple_method('_asdict', symbols, info, this_type, AnyType(), []) + return info + + def make_factory_args(self, vars: List[Var], kind: int, + initializer: Expression = None) -> List[Argument]: + return [Argument(var, var.type, initializer, kind) for var in vars] + + def add_namedtuple_method(self, funcname: str, symbols: SymbolTable, info: TypeInfo, + this_type: Type, ret: Type, args: List[Argument], name=None) -> None: + args = [Argument(Var('self'), this_type, None, ARG_POS)] + args + types = [arg.type_annotation for arg in args] + items = [arg.variable.name() for arg in args] + arg_kinds = [arg.kind for arg in args] + # TODO: Make sure that the self argument name is not visible? + signature = CallableType(types, arg_kinds, items, ret, + self.named_type('__builtins__.function'), + name=name or info.name() + '.' + funcname) + func = FuncDef(funcname, + args, + Block([]), + typ=signature) + func.info = info + symbols[funcname] = SymbolTableNode(MDEF, func) + + def analyze_types(self, items: List[Node]) -> List[Type]: + result = [] # type: List[Type] + for node in items: + try: + result.append(self.anal_type(expr_to_unanalyzed_type(node))) + except TypeTranslationError: + self.fail('Type expected', node) + result.append(AnyType()) + return result + + def visit_decorator(self, dec: Decorator) -> None: + for d in dec.decorators: + d.accept(self) + removed = [] # type: List[int] + no_type_check = False + for i, d in enumerate(dec.decorators): + # A bunch of decorators are special cased here. + if refers_to_fullname(d, 'abc.abstractmethod'): + removed.append(i) + dec.func.is_abstract = True + self.check_decorated_function_is_method('abstractmethod', dec) + elif refers_to_fullname(d, 'asyncio.tasks.coroutine'): + removed.append(i) + elif refers_to_fullname(d, 'builtins.staticmethod'): + removed.append(i) + dec.func.is_static = True + dec.var.is_staticmethod = True + self.check_decorated_function_is_method('staticmethod', dec) + elif refers_to_fullname(d, 'builtins.classmethod'): + removed.append(i) + dec.func.is_class = True + dec.var.is_classmethod = True + self.check_decorated_function_is_method('classmethod', dec) + elif (refers_to_fullname(d, 'builtins.property') or + refers_to_fullname(d, 'abc.abstractproperty')): + removed.append(i) + dec.func.is_property = True + dec.var.is_property = True + if refers_to_fullname(d, 'abc.abstractproperty'): + dec.func.is_abstract = True + self.check_decorated_function_is_method('property', dec) + if len(dec.func.arguments) > 1: + self.fail('Too many arguments', dec.func) + elif refers_to_fullname(d, 'typing.no_type_check'): + dec.var.type = AnyType() + no_type_check = True + for i in reversed(removed): + del dec.decorators[i] + if not dec.is_overload or dec.var.is_property: + if self.is_func_scope(): + self.add_symbol(dec.var.name(), SymbolTableNode(LDEF, dec), + dec) + elif self.type: + dec.var.info = self.type + dec.var.is_initialized_in_class = True + self.add_symbol(dec.var.name(), SymbolTableNode(MDEF, dec), + dec) + if not no_type_check: + dec.func.accept(self) + if dec.decorators and dec.var.is_property: + self.fail('Decorated property not supported', dec) + + def check_decorated_function_is_method(self, decorator: str, + context: Context) -> None: + if not self.type or self.is_func_scope(): + self.fail("'%s' used with a non-method" % decorator, context) + + def visit_expression_stmt(self, s: ExpressionStmt) -> None: + s.expr.accept(self) + + def visit_return_stmt(self, s: ReturnStmt) -> None: + if not self.is_func_scope(): + self.fail("'return' outside function", s) + if s.expr: + s.expr.accept(self) + + def visit_raise_stmt(self, s: RaiseStmt) -> None: + if s.expr: + s.expr.accept(self) + if s.from_expr: + s.from_expr.accept(self) + + def visit_assert_stmt(self, s: AssertStmt) -> None: + if s.expr: + s.expr.accept(self) + + def visit_operator_assignment_stmt(self, + s: OperatorAssignmentStmt) -> None: + s.lvalue.accept(self) + s.rvalue.accept(self) + if (isinstance(s.lvalue, NameExpr) and s.lvalue.name == '__all__' and + s.lvalue.kind == GDEF and isinstance(s.rvalue, (ListExpr, TupleExpr))): + self.add_exports(*s.rvalue.items) + + def visit_while_stmt(self, s: WhileStmt) -> None: + s.expr.accept(self) + self.loop_depth += 1 + s.body.accept(self) + self.loop_depth -= 1 + self.visit_block_maybe(s.else_body) + + def visit_for_stmt(self, s: ForStmt) -> None: + s.expr.accept(self) + + # Bind index variables and check if they define new names. + self.analyze_lvalue(s.index) + + self.loop_depth += 1 + self.visit_block(s.body) + self.loop_depth -= 1 + + self.visit_block_maybe(s.else_body) + + def visit_break_stmt(self, s: BreakStmt) -> None: + if self.loop_depth == 0: + self.fail("'break' outside loop", s, True, blocker=True) + + def visit_continue_stmt(self, s: ContinueStmt) -> None: + if self.loop_depth == 0: + self.fail("'continue' outside loop", s, True, blocker=True) + + def visit_if_stmt(self, s: IfStmt) -> None: + infer_reachability_of_if_statement(s, pyversion=self.options.python_version) + for i in range(len(s.expr)): + s.expr[i].accept(self) + self.visit_block(s.body[i]) + self.visit_block_maybe(s.else_body) + + def visit_try_stmt(self, s: TryStmt) -> None: + self.analyze_try_stmt(s, self) + + def analyze_try_stmt(self, s: TryStmt, visitor: NodeVisitor, + add_global: bool = False) -> None: + s.body.accept(visitor) + for type, var, handler in zip(s.types, s.vars, s.handlers): + if type: + type.accept(visitor) + if var: + self.analyze_lvalue(var, add_global=add_global) + handler.accept(visitor) + if s.else_body: + s.else_body.accept(visitor) + if s.finally_body: + s.finally_body.accept(visitor) + + def visit_with_stmt(self, s: WithStmt) -> None: + for e, n in zip(s.expr, s.target): + e.accept(self) + if n: + self.analyze_lvalue(n) + self.visit_block(s.body) + + def visit_del_stmt(self, s: DelStmt) -> None: + s.expr.accept(self) + if not self.is_valid_del_target(s.expr): + self.fail('Invalid delete target', s) + + def is_valid_del_target(self, s: Node) -> bool: + if isinstance(s, (IndexExpr, NameExpr, MemberExpr)): + return True + elif isinstance(s, TupleExpr): + return all(self.is_valid_del_target(item) for item in s.items) + + def visit_global_decl(self, g: GlobalDecl) -> None: + for name in g.names: + if name in self.nonlocal_decls[-1]: + self.fail("Name '{}' is nonlocal and global".format(name), g) + self.global_decls[-1].add(name) + + def visit_nonlocal_decl(self, d: NonlocalDecl) -> None: + if not self.is_func_scope(): + self.fail("nonlocal declaration not allowed at module level", d) + else: + for name in d.names: + for table in reversed(self.locals[:-1]): + if table is not None and name in table: + break + else: + self.fail("No binding for nonlocal '{}' found".format(name), d) + + if self.locals[-1] is not None and name in self.locals[-1]: + self.fail("Name '{}' is already defined in local " + "scope before nonlocal declaration".format(name), d) + + if name in self.global_decls[-1]: + self.fail("Name '{}' is nonlocal and global".format(name), d) + self.nonlocal_decls[-1].add(name) + + def visit_print_stmt(self, s: PrintStmt) -> None: + for arg in s.args: + arg.accept(self) + if s.target: + s.target.accept(self) + + def visit_exec_stmt(self, s: ExecStmt) -> None: + s.expr.accept(self) + if s.variables1: + s.variables1.accept(self) + if s.variables2: + s.variables2.accept(self) + + # + # Expressions + # + + def visit_name_expr(self, expr: NameExpr) -> None: + n = self.lookup(expr.name, expr) + if n: + if n.kind == BOUND_TVAR: + self.fail("'{}' is a type variable and only valid in type " + "context".format(expr.name), expr) + else: + expr.kind = n.kind + expr.node = n.node + expr.fullname = n.fullname + + def visit_super_expr(self, expr: SuperExpr) -> None: + if not self.type: + self.fail('"super" used outside class', expr) + return + expr.info = self.type + + def visit_tuple_expr(self, expr: TupleExpr) -> None: + for item in expr.items: + item.accept(self) + + def visit_list_expr(self, expr: ListExpr) -> None: + for item in expr.items: + item.accept(self) + + def visit_set_expr(self, expr: SetExpr) -> None: + for item in expr.items: + item.accept(self) + + def visit_dict_expr(self, expr: DictExpr) -> None: + for key, value in expr.items: + key.accept(self) + value.accept(self) + + def visit_star_expr(self, expr: StarExpr) -> None: + if not expr.valid: + self.fail('Can use starred expression only as assignment target', expr) + else: + expr.expr.accept(self) + + def visit_yield_from_expr(self, e: YieldFromExpr) -> None: + if not self.is_func_scope(): # not sure + self.fail("'yield from' outside function", e, True, blocker=True) + else: + self.function_stack[-1].is_generator = True + if e.expr: + e.expr.accept(self) + + def visit_call_expr(self, expr: CallExpr) -> None: + """Analyze a call expression. + + Some call expressions are recognized as special forms, including + cast(...) and Any(...). + """ + expr.callee.accept(self) + if refers_to_fullname(expr.callee, 'typing.cast'): + # Special form cast(...). + if not self.check_fixed_args(expr, 2, 'cast'): + return + # Translate first argument to an unanalyzed type. + try: + target = expr_to_unanalyzed_type(expr.args[0]) + except TypeTranslationError: + self.fail('Cast target is not a type', expr) + return + # Piggyback CastExpr object to the CallExpr object; it takes + # precedence over the CallExpr semantics. + expr.analyzed = CastExpr(expr.args[1], target) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, 'builtins.reveal_type'): + if not self.check_fixed_args(expr, 1, 'reveal_type'): + return + expr.analyzed = RevealTypeExpr(expr.args[0]) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, 'typing.Any'): + # Special form Any(...). + if not self.check_fixed_args(expr, 1, 'Any'): + return + expr.analyzed = CastExpr(expr.args[0], AnyType()) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, 'typing._promote'): + # Special form _promote(...). + if not self.check_fixed_args(expr, 1, '_promote'): + return + # Translate first argument to an unanalyzed type. + try: + target = expr_to_unanalyzed_type(expr.args[0]) + except TypeTranslationError: + self.fail('Argument 1 to _promote is not a type', expr) + return + expr.analyzed = PromoteExpr(target) + expr.analyzed.line = expr.line + expr.analyzed.accept(self) + elif refers_to_fullname(expr.callee, 'builtins.dict'): + expr.analyzed = self.translate_dict_call(expr) + else: + # Normal call expression. + for a in expr.args: + a.accept(self) + + if (isinstance(expr.callee, MemberExpr) and + isinstance(expr.callee.expr, NameExpr) and + expr.callee.expr.name == '__all__' and + expr.callee.expr.kind == GDEF and + expr.callee.name in ('append', 'extend')): + if expr.callee.name == 'append' and expr.args: + self.add_exports(expr.args[0]) + elif (expr.callee.name == 'extend' and expr.args and + isinstance(expr.args[0], (ListExpr, TupleExpr))): + self.add_exports(*expr.args[0].items) + + def translate_dict_call(self, call: CallExpr) -> Optional[DictExpr]: + """Translate 'dict(x=y, ...)' to {'x': y, ...}. + + For other variants of dict(...), return None. + """ + if not call.args: + return None + if not all(kind == ARG_NAMED for kind in call.arg_kinds): + # Must still accept those args. + for a in call.args: + a.accept(self) + return None + expr = DictExpr([(StrExpr(key), value) + for key, value in zip(call.arg_names, call.args)]) + expr.set_line(call) + expr.accept(self) + return expr + + def check_fixed_args(self, expr: CallExpr, numargs: int, + name: str) -> bool: + """Verify that expr has specified number of positional args. + + Return True if the arguments are valid. + """ + s = 's' + if numargs == 1: + s = '' + if len(expr.args) != numargs: + self.fail("'%s' expects %d argument%s" % (name, numargs, s), + expr) + return False + if expr.arg_kinds != [ARG_POS] * numargs: + self.fail("'%s' must be called with %s positional argument%s" % + (name, numargs, s), expr) + return False + return True + + def visit_member_expr(self, expr: MemberExpr) -> None: + base = expr.expr + base.accept(self) + # Bind references to module attributes. + if isinstance(base, RefExpr) and base.kind == MODULE_REF: + file = cast(MypyFile, base.node) + n = file.names.get(expr.name, None) if file is not None else None + if n: + n = self.normalize_type_alias(n, expr) + if not n: + return + expr.kind = n.kind + expr.fullname = n.fullname + expr.node = n.node + else: + # We only catch some errors here; the rest will be + # catched during type checking. + # + # This way we can report a larger number of errors in + # one type checker run. If we reported errors here, + # the build would terminate after semantic analysis + # and we wouldn't be able to report any type errors. + full_name = '%s.%s' % (file.fullname() if file is not None else None, expr.name) + if full_name in obsolete_name_mapping: + self.fail("Module has no attribute %r (it's now called %r)" % ( + expr.name, obsolete_name_mapping[full_name]), expr) + + def visit_op_expr(self, expr: OpExpr) -> None: + expr.left.accept(self) + expr.right.accept(self) + + def visit_comparison_expr(self, expr: ComparisonExpr) -> None: + for operand in expr.operands: + operand.accept(self) + + def visit_unary_expr(self, expr: UnaryExpr) -> None: + expr.expr.accept(self) + + def visit_index_expr(self, expr: IndexExpr) -> None: + expr.base.accept(self) + if refers_to_class_or_function(expr.base): + # Special form -- type application. + # Translate index to an unanalyzed type. + types = [] # type: List[Type] + if isinstance(expr.index, TupleExpr): + items = expr.index.items + else: + items = [expr.index] + for item in items: + try: + typearg = expr_to_unanalyzed_type(item) + except TypeTranslationError: + self.fail('Type expected within [...]', expr) + return + typearg = self.anal_type(typearg) + types.append(typearg) + expr.analyzed = TypeApplication(expr.base, types) + expr.analyzed.line = expr.line + else: + expr.index.accept(self) + + def visit_slice_expr(self, expr: SliceExpr) -> None: + if expr.begin_index: + expr.begin_index.accept(self) + if expr.end_index: + expr.end_index.accept(self) + if expr.stride: + expr.stride.accept(self) + + def visit_cast_expr(self, expr: CastExpr) -> None: + expr.expr.accept(self) + expr.type = self.anal_type(expr.type) + + def visit_reveal_type_expr(self, expr: RevealTypeExpr) -> None: + expr.expr.accept(self) + + def visit_type_application(self, expr: TypeApplication) -> None: + expr.expr.accept(self) + for i in range(len(expr.types)): + expr.types[i] = self.anal_type(expr.types[i]) + + def visit_list_comprehension(self, expr: ListComprehension) -> None: + expr.generator.accept(self) + + def visit_set_comprehension(self, expr: SetComprehension) -> None: + expr.generator.accept(self) + + def visit_dictionary_comprehension(self, expr: DictionaryComprehension) -> None: + self.enter() + self.analyze_comp_for(expr) + expr.key.accept(self) + expr.value.accept(self) + self.leave() + self.analyze_comp_for_2(expr) + + def visit_generator_expr(self, expr: GeneratorExpr) -> None: + self.enter() + self.analyze_comp_for(expr) + expr.left_expr.accept(self) + self.leave() + self.analyze_comp_for_2(expr) + + def analyze_comp_for(self, expr: Union[GeneratorExpr, + DictionaryComprehension]) -> None: + """Analyses the 'comp_for' part of comprehensions (part 1). + + That is the part after 'for' in (x for x in l if p). This analyzes + variables and conditions which are analyzed in a local scope. + """ + for i, (index, sequence, conditions) in enumerate(zip(expr.indices, + expr.sequences, + expr.condlists)): + if i > 0: + sequence.accept(self) + # Bind index variables. + self.analyze_lvalue(index) + for cond in conditions: + cond.accept(self) + + def analyze_comp_for_2(self, expr: Union[GeneratorExpr, + DictionaryComprehension]) -> None: + """Analyses the 'comp_for' part of comprehensions (part 2). + + That is the part after 'for' in (x for x in l if p). This analyzes + the 'l' part which is analyzed in the surrounding scope. + """ + expr.sequences[0].accept(self) + + def visit_func_expr(self, expr: FuncExpr) -> None: + self.analyze_function(expr) + + def visit_conditional_expr(self, expr: ConditionalExpr) -> None: + expr.if_expr.accept(self) + expr.cond.accept(self) + expr.else_expr.accept(self) + + def visit_backquote_expr(self, expr: BackquoteExpr) -> None: + expr.expr.accept(self) + + def visit__promote_expr(self, expr: PromoteExpr) -> None: + expr.type = self.anal_type(expr.type) + + def visit_yield_expr(self, expr: YieldExpr) -> None: + if not self.is_func_scope(): + self.fail("'yield' outside function", expr, True, blocker=True) + else: + self.function_stack[-1].is_generator = True + if expr.expr: + expr.expr.accept(self) + + # + # Helpers + # + + def lookup(self, name: str, ctx: Context) -> SymbolTableNode: + """Look up an unqualified name in all active namespaces.""" + # 1a. Name declared using 'global x' takes precedence + if name in self.global_decls[-1]: + if name in self.globals: + return self.globals[name] + else: + self.name_not_defined(name, ctx) + return None + # 1b. Name declared using 'nonlocal x' takes precedence + if name in self.nonlocal_decls[-1]: + for table in reversed(self.locals[:-1]): + if table is not None and name in table: + return table[name] + else: + self.name_not_defined(name, ctx) + return None + # 2. Class attributes (if within class definition) + if self.is_class_scope() and name in self.type.names: + return self.type[name] + # 3. Local (function) scopes + for table in reversed(self.locals): + if table is not None and name in table: + return table[name] + # 4. Current file global scope + if name in self.globals: + return self.globals[name] + # 5. Builtins + b = self.globals.get('__builtins__', None) + if b: + table = cast(MypyFile, b.node).names + if name in table: + if name[0] == "_" and name[1] != "_": + self.name_not_defined(name, ctx) + return None + node = table[name] + return node + # Give up. + self.name_not_defined(name, ctx) + self.check_for_obsolete_short_name(name, ctx) + return None + + def check_for_obsolete_short_name(self, name: str, ctx: Context) -> None: + matches = [obsolete_name + for obsolete_name in obsolete_name_mapping + if obsolete_name.rsplit('.', 1)[-1] == name] + if len(matches) == 1: + self.note("(Did you mean '{}'?)".format(obsolete_name_mapping[matches[0]]), ctx) + + def lookup_qualified(self, name: str, ctx: Context) -> SymbolTableNode: + if '.' not in name: + return self.lookup(name, ctx) + else: + parts = name.split('.') + n = self.lookup(parts[0], ctx) # type: SymbolTableNode + if n: + for i in range(1, len(parts)): + if isinstance(n.node, TypeInfo): + if n.node.mro is None: + # We haven't yet analyzed the class `n.node`. Fall back to direct + # lookup in the names declared directly under it, without its base + # classes. This can happen when we have a forward reference to a + # nested class, and the reference is bound before the outer class + # has been fully semantically analyzed. + # + # A better approach would be to introduce a new analysis pass or + # to move things around between passes, but this unblocks a common + # use case even though this is a little limited in case there is + # inheritance involved. + result = n.node.names.get(parts[i]) + else: + result = n.node.get(parts[i]) + n = result + elif isinstance(n.node, MypyFile): + n = n.node.names.get(parts[i], None) + # TODO: What if node is Var or FuncDef? + if not n: + self.name_not_defined(name, ctx) + break + if n: + n = self.normalize_type_alias(n, ctx) + return n + + def builtin_type(self, fully_qualified_name: str) -> Instance: + node = self.lookup_fully_qualified(fully_qualified_name) + info = cast(TypeInfo, node.node) + return Instance(info, []) + + def lookup_fully_qualified(self, name: str) -> SymbolTableNode: + """Lookup a fully qualified name. + + Assume that the name is defined. This happens in the global namespace -- the local + module namespace is ignored. + """ + assert '.' in name + parts = name.split('.') + n = self.modules[parts[0]] + for i in range(1, len(parts) - 1): + n = cast(MypyFile, n.names[parts[i]].node) + return n.names[parts[-1]] + + def lookup_fully_qualified_or_none(self, name: str) -> SymbolTableNode: + """Lookup a fully qualified name. + + Assume that the name is defined. This happens in the global namespace -- the local + module namespace is ignored. + """ + assert '.' in name + parts = name.split('.') + n = self.modules[parts[0]] + for i in range(1, len(parts) - 1): + next_sym = n.names.get(parts[i]) + if not next_sym: + return None + n = cast(MypyFile, next_sym.node) + return n.names.get(parts[-1]) + + def qualified_name(self, n: str) -> str: + return self.cur_mod_id + '.' + n + + def enter(self) -> None: + self.locals.append(SymbolTable()) + self.global_decls.append(set()) + self.nonlocal_decls.append(set()) + # -1 since entering block will increment this to 0. + self.block_depth.append(-1) + + def leave(self) -> None: + self.locals.pop() + self.global_decls.pop() + self.nonlocal_decls.pop() + self.block_depth.pop() + + def is_func_scope(self) -> bool: + return self.locals[-1] is not None + + def is_class_scope(self) -> bool: + return self.type is not None and not self.is_func_scope() + + def add_symbol(self, name: str, node: SymbolTableNode, + context: Context) -> None: + if self.is_func_scope(): + if name in self.locals[-1]: + # Flag redefinition unless this is a reimport of a module. + if not (node.kind == MODULE_REF and + self.locals[-1][name].node == node.node): + self.name_already_defined(name, context) + self.locals[-1][name] = node + elif self.type: + self.type.names[name] = node + else: + existing = self.globals.get(name) + if existing and (not isinstance(node.node, MypyFile) or + existing.node != node.node) and existing.kind != UNBOUND_IMPORTED: + # Modules can be imported multiple times to support import + # of multiple submodules of a package (e.g. a.x and a.y). + ok = False + # Only report an error if the symbol collision provides a different type. + if existing.type and node.type and is_same_type(existing.type, node.type): + ok = True + if not ok: + self.name_already_defined(name, context) + self.globals[name] = node + + def add_var(self, v: Var, ctx: Context) -> None: + if self.is_func_scope(): + self.add_local(v, ctx) + else: + self.globals[v.name()] = SymbolTableNode(GDEF, v, self.cur_mod_id) + v._fullname = self.qualified_name(v.name()) + + def add_local(self, node: Union[Var, FuncBase], ctx: Context) -> None: + name = node.name() + if name in self.locals[-1]: + self.name_already_defined(name, ctx) + node._fullname = name + self.locals[-1][name] = SymbolTableNode(LDEF, node) + + def add_exports(self, *exps: Node) -> None: + for exp in exps: + if isinstance(exp, StrExpr): + self.all_exports.add(exp.value) + + def check_no_global(self, n: str, ctx: Context, + is_overloaded_func: bool = False) -> None: + if n in self.globals: + prev_is_overloaded = isinstance(self.globals[n], OverloadedFuncDef) + if is_overloaded_func and prev_is_overloaded: + self.fail("Nonconsecutive overload {} found".format(n), ctx) + elif prev_is_overloaded: + self.fail("Definition of '{}' missing 'overload'".format(n), ctx) + else: + self.name_already_defined(n, ctx) + + def name_not_defined(self, name: str, ctx: Context) -> None: + message = "Name '{}' is not defined".format(name) + extra = self.undefined_name_extra_info(name) + if extra: + message += ' {}'.format(extra) + self.fail(message, ctx) + + def name_already_defined(self, name: str, ctx: Context) -> None: + self.fail("Name '{}' already defined".format(name), ctx) + + def fail(self, msg: str, ctx: Context, serious: bool = False, *, + blocker: bool = False) -> None: + if (not serious and + not self.options.check_untyped_defs and + self.function_stack and + self.function_stack[-1].is_dynamic()): + return + self.errors.report(ctx.get_line(), msg, blocker=blocker) + + def note(self, msg: str, ctx: Context) -> None: + if (not self.options.check_untyped_defs and + self.function_stack and + self.function_stack[-1].is_dynamic()): + return + self.errors.report(ctx.get_line(), msg, severity='note') + + def undefined_name_extra_info(self, fullname: str) -> Optional[str]: + if fullname in obsolete_name_mapping: + return "(it's now called '{}')".format(obsolete_name_mapping[fullname]) + else: + return None + + def accept(self, node: Node) -> None: + try: + node.accept(self) + except Exception as err: + report_internal_error(err, self.errors.file, node.line) + + +class FirstPass(NodeVisitor): + """First phase of semantic analysis. + + See docstring of 'analyze()' below for a description of what this does. + """ + + def __init__(self, sem: SemanticAnalyzer) -> None: + self.sem = sem + self.pyversion = sem.options.python_version + + def analyze(self, file: MypyFile, fnam: str, mod_id: str) -> None: + """Perform the first analysis pass. + + Populate module global table. Resolve the full names of + definitions not nested within functions and construct type + info structures, but do not resolve inter-definition + references such as base classes. + + Also add implicit definitions such as __name__. + + In this phase we don't resolve imports. For 'from ... import', + we generate dummy symbol table nodes for the imported names, + and these will get resolved in later phases of semantic + analysis. + """ + sem = self.sem + sem.cur_mod_id = mod_id + sem.errors.set_file(fnam) + sem.globals = SymbolTable() + sem.global_decls = [set()] + sem.nonlocal_decls = [set()] + sem.block_depth = [0] + + defs = file.defs + + # Add implicit definitions of module '__name__' etc. + for name, t in implicit_module_attrs.items(): + v = Var(name, UnboundType(t)) + v._fullname = self.sem.qualified_name(name) + self.sem.globals[name] = SymbolTableNode(GDEF, v, self.sem.cur_mod_id) + + for d in defs: + d.accept(self) + + # Add implicit definition of literals/keywords to builtins, as we + # cannot define a variable with them explicitly. + if mod_id == 'builtins': + literal_types = [ + ('None', NoneTyp()), + # reveal_type is a mypy-only function that gives an error with the type of its arg + ('reveal_type', AnyType()), + ] # type: List[Tuple[str, Type]] + + # TODO(ddfisher): This guard is only needed because mypy defines + # fake builtins for its tests which often don't define bool. If + # mypy is fast enough that we no longer need those, this + # conditional check should be removed. + if 'bool' in self.sem.globals: + bool_type = self.sem.named_type('bool') + literal_types.extend([ + ('True', bool_type), + ('False', bool_type), + ('__debug__', bool_type), + ]) + + for name, typ in literal_types: + v = Var(name, typ) + v._fullname = self.sem.qualified_name(name) + self.sem.globals[name] = SymbolTableNode(GDEF, v, self.sem.cur_mod_id) + + def visit_block(self, b: Block) -> None: + if b.is_unreachable: + return + self.sem.block_depth[-1] += 1 + for node in b.body: + node.accept(self) + self.sem.block_depth[-1] -= 1 + + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + for lval in s.lvalues: + self.analyze_lvalue(lval, explicit_type=s.type is not None) + + def visit_func_def(self, func: FuncDef) -> None: + sem = self.sem + func.is_conditional = sem.block_depth[-1] > 0 + func._fullname = sem.qualified_name(func.name()) + if func.name() in sem.globals: + # Already defined in this module. + original_sym = sem.globals[func.name()] + if original_sym.kind == UNBOUND_IMPORTED: + # Ah this is an imported name. We can't resolve them now, so we'll postpone + # this until the main phase of semantic analysis. + return + original_def = original_sym.node + if sem.is_conditional_func(original_def, func): + # Conditional function definition -- multiple defs are ok. + func.original_def = cast(FuncDef, original_def) + else: + # Report error. + sem.check_no_global(func.name(), func) + else: + sem.globals[func.name()] = SymbolTableNode(GDEF, func, sem.cur_mod_id) + + def visit_overloaded_func_def(self, func: OverloadedFuncDef) -> None: + self.sem.check_no_global(func.name(), func, True) + func._fullname = self.sem.qualified_name(func.name()) + self.sem.globals[func.name()] = SymbolTableNode(GDEF, func, + self.sem.cur_mod_id) + + def visit_class_def(self, cdef: ClassDef) -> None: + self.sem.check_no_global(cdef.name, cdef) + cdef.fullname = self.sem.qualified_name(cdef.name) + info = TypeInfo(SymbolTable(), cdef) + info.set_line(cdef.line) + cdef.info = info + self.sem.globals[cdef.name] = SymbolTableNode(GDEF, info, + self.sem.cur_mod_id) + self.process_nested_classes(cdef) + + def process_nested_classes(self, outer_def: ClassDef) -> None: + for node in outer_def.defs.body: + if isinstance(node, ClassDef): + node.info = TypeInfo(SymbolTable(), node) + if outer_def.fullname: + node.info._fullname = outer_def.fullname + '.' + node.info.name() + else: + node.info._fullname = node.info.name() + symbol = SymbolTableNode(MDEF, node.info) + outer_def.info.names[node.name] = symbol + self.process_nested_classes(node) + + def visit_import_from(self, node: ImportFrom) -> None: + # We can't bind module names during the first pass, as the target module might be + # unprocessed. However, we add dummy unbound imported names to the symbol table so + # that we at least know that the name refers to a module. + node.is_top_level = True + for name, as_name in node.names: + imported_name = as_name or name + if imported_name not in self.sem.globals: + self.sem.add_symbol(imported_name, SymbolTableNode(UNBOUND_IMPORTED, None), node) + + def visit_import(self, node: Import) -> None: + node.is_top_level = True + # This is similar to visit_import_from -- see the comment there. + for id, as_id in node.ids: + imported_id = as_id or id + if imported_id not in self.sem.globals: + self.sem.add_symbol(imported_id, SymbolTableNode(UNBOUND_IMPORTED, None), node) + else: + # If the previous symbol is a variable, this should take precedence. + self.sem.globals[imported_id] = SymbolTableNode(UNBOUND_IMPORTED, None) + + def visit_import_all(self, node: ImportAll) -> None: + node.is_top_level = True + + def visit_while_stmt(self, s: WhileStmt) -> None: + s.body.accept(self) + if s.else_body: + s.else_body.accept(self) + + def visit_for_stmt(self, s: ForStmt) -> None: + self.analyze_lvalue(s.index) + s.body.accept(self) + if s.else_body: + s.else_body.accept(self) + + def visit_with_stmt(self, s: WithStmt) -> None: + for n in s.target: + if n: + self.analyze_lvalue(n) + s.body.accept(self) + + def visit_decorator(self, d: Decorator) -> None: + d.var._fullname = self.sem.qualified_name(d.var.name()) + self.sem.add_symbol(d.var.name(), SymbolTableNode(GDEF, d.var), d) + + def visit_if_stmt(self, s: IfStmt) -> None: + infer_reachability_of_if_statement(s, pyversion=self.pyversion) + for node in s.body: + node.accept(self) + if s.else_body: + s.else_body.accept(self) + + def visit_try_stmt(self, s: TryStmt) -> None: + self.sem.analyze_try_stmt(s, self, add_global=True) + + def analyze_lvalue(self, lvalue: Node, explicit_type: bool = False) -> None: + self.sem.analyze_lvalue(lvalue, add_global=True, explicit_type=explicit_type) + + +class ThirdPass(TraverserVisitor): + """The third and final pass of semantic analysis. + + Check type argument counts and values of generic types, and perform some + straightforward type inference. + """ + + def __init__(self, modules: Dict[str, MypyFile], errors: Errors) -> None: + self.modules = modules + self.errors = errors + + def visit_file(self, file_node: MypyFile, fnam: str) -> None: + self.errors.set_file(fnam) + self.accept(file_node) + + def accept(self, node: Node) -> None: + try: + node.accept(self) + except Exception as err: + report_internal_error(err, self.errors.file, node.line) + + def visit_block(self, b: Block) -> None: + if b.is_unreachable: + return + super().visit_block(b) + + def visit_func_def(self, fdef: FuncDef) -> None: + self.errors.push_function(fdef.name()) + self.analyze(fdef.type) + super().visit_func_def(fdef) + self.errors.pop_function() + + def visit_class_def(self, tdef: ClassDef) -> None: + for type in tdef.info.bases: + self.analyze(type) + # Recompute MRO now that we have analyzed all modules, to pick + # up superclasses of bases imported from other modules in an + # import loop. (Only do so if we succeeded the first time.) + if tdef.info.mro: + tdef.info.mro = [] # Force recomputation + calculate_class_mro(tdef, self.fail) + super().visit_class_def(tdef) + + def visit_decorator(self, dec: Decorator) -> None: + """Try to infer the type of the decorated function. + + This lets us resolve references to decorated functions during + type checking when there are cyclic imports, as otherwise the + type might not be available when we need it. + + This basically uses a simple special-purpose type inference + engine just for decorators. + """ + super().visit_decorator(dec) + if dec.var.is_property: + # Decorators are expected to have a callable type (it's a little odd). + if dec.func.type is None: + dec.var.type = CallableType( + [AnyType()], + [ARG_POS], + [None], + AnyType(), + self.builtin_type('function'), + name=dec.var.name()) + elif isinstance(dec.func.type, CallableType): + dec.var.type = dec.func.type + return + decorator_preserves_type = True + for expr in dec.decorators: + preserve_type = False + if isinstance(expr, RefExpr) and isinstance(expr.node, FuncDef): + if is_identity_signature(expr.node.type): + preserve_type = True + if not preserve_type: + decorator_preserves_type = False + break + if decorator_preserves_type: + # No non-identity decorators left. We can trivially infer the type + # of the function here. + dec.var.type = function_type(dec.func, self.builtin_type('function')) + if dec.decorators: + if returns_any_if_called(dec.decorators[0]): + # The outermost decorator will return Any so we know the type of the + # decorated function. + dec.var.type = AnyType() + sig = find_fixed_callable_return(dec.decorators[0]) + if sig: + # The outermost decorator always returns the same kind of function, + # so we know that this is the type of the decoratored function. + orig_sig = function_type(dec.func, self.builtin_type('function')) + sig.name = orig_sig.items()[0].name + dec.var.type = sig + + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + self.analyze(s.type) + super().visit_assignment_stmt(s) + + def visit_cast_expr(self, e: CastExpr) -> None: + self.analyze(e.type) + super().visit_cast_expr(e) + + def visit_reveal_type_expr(self, e: RevealTypeExpr) -> None: + super().visit_reveal_type_expr(e) + + def visit_type_application(self, e: TypeApplication) -> None: + for type in e.types: + self.analyze(type) + super().visit_type_application(e) + + # Helpers + + def analyze(self, type: Type) -> None: + if type: + analyzer = TypeAnalyserPass3(self.fail) + type.accept(analyzer) + + def fail(self, msg: str, ctx: Context) -> None: + self.errors.report(ctx.get_line(), msg) + + def builtin_type(self, name: str, args: List[Type] = None) -> Instance: + names = self.modules['builtins'] + sym = names.names[name] + assert isinstance(sym.node, TypeInfo) + return Instance(sym.node, args or []) + + +def self_type(typ: TypeInfo) -> Union[Instance, TupleType]: + """For a non-generic type, return instance type representing the type. + For a generic G type with parameters T1, .., Tn, return G[T1, ..., Tn]. + """ + tv = [] # type: List[Type] + for i in range(len(typ.type_vars)): + tv.append(TypeVarType(typ.defn.type_vars[i])) + inst = Instance(typ, tv) + if typ.tuple_type is None: + return inst + return typ.tuple_type.self_type(inst) + + +def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: + if isinstance(sig, CallableType): + return replace_leading_arg_type(sig, new) + else: + sig = cast(Overloaded, sig) + return Overloaded([cast(CallableType, replace_implicit_first_type(i, new)) + for i in sig.items()]) + + +def set_callable_name(sig: Type, fdef: FuncDef) -> Type: + if isinstance(sig, FunctionLike): + if fdef.info: + return sig.with_name( + '"{}" of "{}"'.format(fdef.name(), fdef.info.name())) + else: + return sig.with_name('"{}"'.format(fdef.name())) + else: + return sig + + +def refers_to_fullname(node: Node, fullname: str) -> bool: + """Is node a name or member expression with the given full name?""" + return isinstance(node, RefExpr) and node.fullname == fullname + + +def refers_to_class_or_function(node: Node) -> bool: + """Does semantically analyzed node refer to a class?""" + return (isinstance(node, RefExpr) and + isinstance(node.node, (TypeInfo, FuncDef, OverloadedFuncDef))) + + +def calculate_class_mro(defn: ClassDef, fail: Callable[[str, Context], None]) -> None: + try: + defn.info.calculate_mro() + except MroError: + fail("Cannot determine consistent method resolution order " + '(MRO) for "%s"' % defn.name, defn) + defn.info.mro = [] + # The property of falling back to Any is inherited. + defn.info.fallback_to_any = any(baseinfo.fallback_to_any for baseinfo in defn.info.mro) + + +def find_duplicate(list: List[T]) -> T: + """If the list has duplicates, return one of the duplicates. + + Otherwise, return None. + """ + for i in range(1, len(list)): + if list[i] in list[:i]: + return list[i] + return None + + +def disable_typevars(nodes: List[SymbolTableNode]) -> None: + for node in nodes: + assert node.kind in (BOUND_TVAR, UNBOUND_TVAR) + node.kind = UNBOUND_TVAR + + +def enable_typevars(nodes: List[SymbolTableNode]) -> None: + for node in nodes: + assert node.kind in (BOUND_TVAR, UNBOUND_TVAR) + node.kind = BOUND_TVAR + + +def remove_imported_names_from_symtable(names: SymbolTable, + module: str) -> None: + """Remove all imported names from the symbol table of a module.""" + removed = [] # type: List[str] + for name, node in names.items(): + if node.node is None: + continue + fullname = node.node.fullname() + prefix = fullname[:fullname.rfind('.')] + if prefix != module: + removed.append(name) + for name in removed: + del names[name] + + +def infer_reachability_of_if_statement(s: IfStmt, + pyversion: Tuple[int, int]) -> None: + for i in range(len(s.expr)): + result = infer_if_condition_value(s.expr[i], pyversion) + if result == ALWAYS_FALSE: + # The condition is always false, so we skip the if/elif body. + mark_block_unreachable(s.body[i]) + elif result == ALWAYS_TRUE: + # This condition is always true, so all of the remaining + # elif/else bodies will never be executed. + for body in s.body[i + 1:]: + mark_block_unreachable(body) + if s.else_body: + mark_block_unreachable(s.else_body) + break + + +def infer_if_condition_value(expr: Node, pyversion: Tuple[int, int]) -> int: + """Infer whether if condition is always true/false. + + Return ALWAYS_TRUE if always true, ALWAYS_FALSE if always false, + and TRUTH_VALUE_UNKNOWN otherwise. + """ + name = '' + negated = False + alias = expr + if isinstance(alias, UnaryExpr): + if alias.op == 'not': + expr = alias.expr + negated = True + if isinstance(expr, NameExpr): + name = expr.name + elif isinstance(expr, MemberExpr): + name = expr.name + result = TRUTH_VALUE_UNKNOWN + if name == 'PY2': + result = ALWAYS_TRUE if pyversion[0] == 2 else ALWAYS_FALSE + elif name == 'PY3': + result = ALWAYS_TRUE if pyversion[0] == 3 else ALWAYS_FALSE + elif name == 'MYPY': + result = ALWAYS_TRUE + if negated: + if result == ALWAYS_TRUE: + result = ALWAYS_FALSE + elif result == ALWAYS_FALSE: + result = ALWAYS_TRUE + return result + + +def mark_block_unreachable(block: Block) -> None: + block.is_unreachable = True + block.accept(MarkImportsUnreachableVisitor()) + + +class MarkImportsUnreachableVisitor(TraverserVisitor): + """Visitor that flags all imports nested within a node as unreachable.""" + + def visit_import(self, node: Import) -> None: + node.is_unreachable = True + + def visit_import_from(self, node: ImportFrom) -> None: + node.is_unreachable = True + + def visit_import_all(self, node: ImportAll) -> None: + node.is_unreachable = True + + +def is_identity_signature(sig: Type) -> bool: + """Is type a callable of form T -> T (where T is a type variable)?""" + if isinstance(sig, CallableType) and sig.arg_kinds == [ARG_POS]: + if isinstance(sig.arg_types[0], TypeVarType) and isinstance(sig.ret_type, TypeVarType): + return sig.arg_types[0].id == sig.ret_type.id + return False + + +def returns_any_if_called(expr: Node) -> bool: + """Return True if we can predict that expr will return Any if called. + + This only uses information available during semantic analysis so this + will sometimes return False because of insufficient information (as + type inference hasn't run yet). + """ + if isinstance(expr, RefExpr): + if isinstance(expr.node, FuncDef): + typ = expr.node.type + if typ is None: + # No signature -> default to Any. + return True + # Explicit Any return? + return isinstance(typ, CallableType) and isinstance(typ.ret_type, AnyType) + elif isinstance(expr.node, Var): + typ = expr.node.type + return typ is None or isinstance(typ, AnyType) + elif isinstance(expr, CallExpr): + return returns_any_if_called(expr.callee) + return False + + +def find_fixed_callable_return(expr: Node) -> Optional[CallableType]: + if isinstance(expr, RefExpr): + if isinstance(expr.node, FuncDef): + typ = expr.node.type + if typ: + if isinstance(typ, CallableType) and has_no_typevars(typ.ret_type): + if isinstance(typ.ret_type, CallableType): + return typ.ret_type + elif isinstance(expr, CallExpr): + t = find_fixed_callable_return(expr.callee) + if t: + if isinstance(t.ret_type, CallableType): + return t.ret_type + return None + + +def has_no_typevars(typ: Type) -> bool: + return is_same_type(typ, erase_typevars(typ)) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 42405695d5aed..6872e297c3a64 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -1,409 +1,409 @@ -from typing import cast, List, Dict, Callable - -from mypy.types import ( - Type, AnyType, UnboundType, TypeVisitor, ErrorType, Void, NoneTyp, - Instance, TypeVarType, CallableType, TupleType, NamedTupleType, UnionType, Overloaded, ErasedType, TypeList, - PartialType, DeletedType, UninhabitedType, TypeType, is_named_instance -) -import mypy.applytype -import mypy.constraints -# Circular import; done in the function instead. -# import mypy.solve -from mypy import messages, sametypes -from mypy.nodes import CONTRAVARIANT, COVARIANT -from mypy.maptype import map_instance_to_supertype - -from mypy import experiments - - -TypeParameterChecker = Callable[[Type, Type, int], bool] - - -def check_type_parameter(lefta: Type, righta: Type, variance: int) -> bool: - if variance == COVARIANT: - return is_subtype(lefta, righta, check_type_parameter) - elif variance == CONTRAVARIANT: - return is_subtype(righta, lefta, check_type_parameter) - else: - return is_equivalent(lefta, righta, check_type_parameter) - - -def is_subtype(left: Type, right: Type, - type_parameter_checker: TypeParameterChecker = check_type_parameter) -> bool: - """Is 'left' subtype of 'right'? - - Also consider Any to be a subtype of any type, and vice versa. This - recursively applies to components of composite types (List[int] is subtype - of List[Any], for example). - - type_parameter_checker is used to check the type parameters (for example, - A with B in is_subtype(C[A], C[B]). The default checks for subtype relation - between the type arguments (e.g., A and B), taking the variance of the - type var into account. - """ - if (isinstance(right, AnyType) or isinstance(right, UnboundType) - or isinstance(right, ErasedType)): - return True - elif isinstance(right, UnionType) and not isinstance(left, UnionType): - return any(is_subtype(left, item, type_parameter_checker) - for item in right.items) - else: - return left.accept(SubtypeVisitor(right, type_parameter_checker)) - - -def is_subtype_ignoring_tvars(left: Type, right: Type) -> bool: - def ignore_tvars(s: Type, t: Type, v: int) -> bool: - return True - return is_subtype(left, right, ignore_tvars) - - -def is_equivalent(a: Type, b: Type, - type_parameter_checker=check_type_parameter) -> bool: - return is_subtype(a, b, type_parameter_checker) and is_subtype(b, a, type_parameter_checker) - - -def satisfies_upper_bound(a: Type, upper_bound: Type) -> bool: - """Is 'a' valid value for a type variable with the given 'upper_bound'? - - Same as is_subtype except that Void is considered to be a subtype of - any upper_bound. This is needed in a case like - - def f(g: Callable[[], T]) -> T: ... - def h() -> None: ... - f(h) - """ - return isinstance(a, Void) or is_subtype(a, upper_bound) - - -class SubtypeVisitor(TypeVisitor[bool]): - - def __init__(self, right: Type, - type_parameter_checker: TypeParameterChecker) -> None: - self.right = right - self.check_type_parameter = type_parameter_checker - - # visit_x(left) means: is left (which is an instance of X) a subtype of - # right? - - def visit_unbound_type(self, left: UnboundType) -> bool: - return True - - def visit_error_type(self, left: ErrorType) -> bool: - return False - - def visit_type_list(self, t: TypeList) -> bool: - assert False, 'Not supported' - - def visit_any(self, left: AnyType) -> bool: - return True - - def visit_void(self, left: Void) -> bool: - return isinstance(self.right, Void) - - def visit_none_type(self, left: NoneTyp) -> bool: - if experiments.STRICT_OPTIONAL: - return (isinstance(self.right, NoneTyp) or - is_named_instance(self.right, 'builtins.object')) - else: - return not isinstance(self.right, Void) - - def visit_uninhabited_type(self, left: UninhabitedType) -> bool: - return not isinstance(self.right, Void) - - def visit_erased_type(self, left: ErasedType) -> bool: - return True - - def visit_deleted_type(self, left: DeletedType) -> bool: - return True - - def visit_instance(self, left: Instance) -> bool: - if left.type.fallback_to_any: - return True - right = self.right - if isinstance(right, Instance): - if left.type._promote and is_subtype(left.type._promote, - self.right, - self.check_type_parameter): - return True - rname = right.type.fullname() - if not left.type.has_base(rname) and rname != 'builtins.object': - return False - - # Map left type to corresponding right instances. - t = map_instance_to_supertype(left, right.type) - - return all(self.check_type_parameter(lefta, righta, tvar.variance) - for lefta, righta, tvar in - zip(t.args, right.args, right.type.defn.type_vars)) - else: - return False - - def visit_type_var(self, left: TypeVarType) -> bool: - right = self.right - if isinstance(right, TypeVarType): - return left.id == right.id - else: - return is_subtype(left.upper_bound, self.right) - - def visit_callable_type(self, left: CallableType) -> bool: - right = self.right - if isinstance(right, CallableType): - return is_callable_subtype(left, right) - elif isinstance(right, Overloaded): - return all(is_subtype(left, item, self.check_type_parameter) - for item in right.items()) - elif isinstance(right, Instance): - return is_subtype(left.fallback, right) - elif isinstance(right, TypeType): - # This is unsound, we don't check the __init__ signature. - return left.is_type_obj() and is_subtype(left.ret_type, right.item) - else: - return False - - def visit_tuple_type(self, left: TupleType) -> bool: - right = self.right - if isinstance(right, Instance): - if is_named_instance(right, 'builtins.object'): - return True - if is_named_instance(right, 'builtins.tuple'): - target_item_type = right.args[0] - return all(is_subtype(item, target_item_type) - for item in left.items) - elif is_named_instance(right, 'typing.Sized'): - return True - elif (is_named_instance(right, 'typing.Iterable') or - is_named_instance(right, 'typing.Container') or - is_named_instance(right, 'typing.Sequence') or - is_named_instance(right, 'typing.Reversible')): - iter_type = right.args[0] - return all(is_subtype(li, iter_type) for li in left.items) - return False - elif isinstance(right, TupleType): - if len(left.items) != len(right.items): - return False - for l, r in zip(left.items, right.items): - if not is_subtype(l, r, self.check_type_parameter): - return False - if not is_subtype(left.fallback, right.fallback, self.check_type_parameter): - return False - return True - else: - return False - - def visit_namedtuple_type(self, left: NamedTupleType) -> bool: - if isinstance(self.right, NamedTupleType): - # left.attrs.issubset(right.attrs) is insufficient - # (a: int, b: str) is not subtype of (b: int, a: str) - if any(l != r for l, r in zip(left.attrs, self.right.attrs)): - return False - return self.visit_tuple_type(left) - - def visit_overloaded(self, left: Overloaded) -> bool: - right = self.right - if isinstance(right, Instance): - return is_subtype(left.fallback, right) - elif isinstance(right, CallableType) or is_named_instance( - right, 'builtins.type'): - for item in left.items(): - if is_subtype(item, right, self.check_type_parameter): - return True - return False - elif isinstance(right, Overloaded): - # TODO: this may be too restrictive - if len(left.items()) != len(right.items()): - return False - for i in range(len(left.items())): - if not is_subtype(left.items()[i], right.items()[i], self.check_type_parameter): - return False - return True - elif isinstance(right, UnboundType): - return True - elif isinstance(right, TypeType): - # All the items must have the same type object status, so - # it's sufficient to query only (any) one of them. - # This is unsound, we don't check the __init__ signature. - return left.is_type_obj() and is_subtype(left.items()[0].ret_type, right.item) - else: - return False - - def visit_union_type(self, left: UnionType) -> bool: - return all(is_subtype(item, self.right, self.check_type_parameter) - for item in left.items) - - def visit_partial_type(self, left: PartialType) -> bool: - # This is indeterminate as we don't really know the complete type yet. - raise RuntimeError - - def visit_type_type(self, left: TypeType) -> bool: - right = self.right - if isinstance(right, TypeType): - return is_subtype(left.item, right.item) - if isinstance(right, CallableType): - # This is unsound, we don't check the __init__ signature. - return right.is_type_obj() and is_subtype(left.item, right.ret_type) - if (isinstance(right, Instance) and - right.type.fullname() in ('builtins.type', 'builtins.object')): - # Treat builtins.type the same as Type[Any]; - # treat builtins.object the same as Any. - return True - return False - - -def is_callable_subtype(left: CallableType, right: CallableType, - ignore_return: bool = False) -> bool: - """Is left a subtype of right?""" - # TODO: Support named arguments, **args, etc. - # Non-type cannot be a subtype of type. - if right.is_type_obj() and not left.is_type_obj(): - return False - - # A callable L is a subtype of a generic callable R if L is a - # subtype of every type obtained from R by substituting types for - # the variables of R. We can check this by simply leaving the - # generic variables of R as type variables, effectively varying - # over all possible values. - - # It's okay even if these variables share ids with generic - # type variables of L, because generating and solving - # constraints for the variables of L to make L a subtype of R - # (below) treats type variables on the two sides as independent. - - if left.variables: - # Apply generic type variables away in left via type inference. - left = unify_generic_callable(left, right, ignore_return=ignore_return) - if left is None: - return False - - # Check return types. - if not ignore_return and not is_subtype(left.ret_type, right.ret_type): - return False - - if right.is_ellipsis_args: - return True - - # Check argument types. - if left.min_args > right.min_args: - return False - if left.is_var_arg: - return is_var_arg_callable_subtype_helper(left, right) - if right.is_var_arg: - return False - if len(left.arg_types) < len(right.arg_types): - return False - for i in range(len(right.arg_types)): - if not is_subtype(right.arg_types[i], left.arg_types[i]): - return False - return True - - -def is_var_arg_callable_subtype_helper(left: CallableType, right: CallableType) -> bool: - """Is left a subtype of right, assuming left has *args? - - See also is_callable_subtype for additional assumptions we can make. - """ - left_fixed = left.max_fixed_args() - right_fixed = right.max_fixed_args() - num_fixed_matching = min(left_fixed, right_fixed) - for i in range(num_fixed_matching): - if not is_subtype(right.arg_types[i], left.arg_types[i]): - return False - if not right.is_var_arg: - for i in range(num_fixed_matching, len(right.arg_types)): - if not is_subtype(right.arg_types[i], left.arg_types[-1]): - return False - return True - else: - for i in range(left_fixed, right_fixed): - if not is_subtype(right.arg_types[i], left.arg_types[-1]): - return False - for i in range(right_fixed, left_fixed): - if not is_subtype(right.arg_types[-1], left.arg_types[i]): - return False - return is_subtype(right.arg_types[-1], left.arg_types[-1]) - - -def unify_generic_callable(type: CallableType, target: CallableType, - ignore_return: bool) -> CallableType: - """Try to unify a generic callable type with another callable type. - - Return unified CallableType if successful; otherwise, return None. - """ - import mypy.solve - constraints = [] # type: List[mypy.constraints.Constraint] - for arg_type, target_arg_type in zip(type.arg_types, target.arg_types): - c = mypy.constraints.infer_constraints( - arg_type, target_arg_type, mypy.constraints.SUPERTYPE_OF) - constraints.extend(c) - if not ignore_return: - c = mypy.constraints.infer_constraints( - type.ret_type, target.ret_type, mypy.constraints.SUBTYPE_OF) - constraints.extend(c) - type_var_ids = [tvar.id for tvar in type.variables] - inferred_vars = mypy.solve.solve_constraints(type_var_ids, constraints) - if None in inferred_vars: - return None - msg = messages.temp_message_builder() - applied = mypy.applytype.apply_generic_arguments(type, inferred_vars, msg, context=target) - if msg.is_errors() or not isinstance(applied, CallableType): - return None - return cast(CallableType, applied) - - -def restrict_subtype_away(t: Type, s: Type) -> Type: - """Return a supertype of (t intersect not s) - - Currently just remove elements of a union type. - """ - if isinstance(t, UnionType): - new_items = [item for item in t.items if not is_subtype(item, s)] - return UnionType.make_union(new_items) - else: - return t - - -def is_proper_subtype(t: Type, s: Type) -> bool: - """Check if t is a proper subtype of s? - - For proper subtypes, there's no need to rely on compatibility due to - Any types. Any instance type t is also a proper subtype of t. - """ - # FIX tuple types - if isinstance(t, Instance): - if isinstance(s, Instance): - if not t.type.has_base(s.type.fullname()): - return False - - def check_argument(left: Type, right: Type, variance: int) -> bool: - if variance == COVARIANT: - return is_proper_subtype(left, right) - elif variance == CONTRAVARIANT: - return is_proper_subtype(right, left) - else: - return sametypes.is_same_type(left, right) - - # Map left type to corresponding right instances. - t = map_instance_to_supertype(t, s.type) - - return all(check_argument(ta, ra, tvar.variance) for ta, ra, tvar in - zip(t.args, s.args, s.type.defn.type_vars)) - return False - else: - return sametypes.is_same_type(t, s) - - -def is_more_precise(t: Type, s: Type) -> bool: - """Check if t is a more precise type than s. - - A t is a proper subtype of s, t is also more precise than s. Also, if - s is Any, t is more precise than s for any t. Finally, if t is the same - type as s, t is more precise than s. - """ - # TODO Should List[int] be more precise than List[Any]? - if isinstance(s, AnyType): - return True - if isinstance(s, Instance): - if isinstance(t, CallableType): - # Fall back to subclass check and ignore other properties of the callable. - return is_proper_subtype(t.fallback, s) - return is_proper_subtype(t, s) - return sametypes.is_same_type(t, s) +from typing import cast, List, Dict, Callable + +from mypy.types import ( + Type, AnyType, UnboundType, TypeVisitor, ErrorType, Void, NoneTyp, + Instance, TypeVarType, CallableType, TupleType, NamedTupleType, UnionType, Overloaded, ErasedType, TypeList, + PartialType, DeletedType, UninhabitedType, TypeType, is_named_instance +) +import mypy.applytype +import mypy.constraints +# Circular import; done in the function instead. +# import mypy.solve +from mypy import messages, sametypes +from mypy.nodes import CONTRAVARIANT, COVARIANT +from mypy.maptype import map_instance_to_supertype + +from mypy import experiments + + +TypeParameterChecker = Callable[[Type, Type, int], bool] + + +def check_type_parameter(lefta: Type, righta: Type, variance: int) -> bool: + if variance == COVARIANT: + return is_subtype(lefta, righta, check_type_parameter) + elif variance == CONTRAVARIANT: + return is_subtype(righta, lefta, check_type_parameter) + else: + return is_equivalent(lefta, righta, check_type_parameter) + + +def is_subtype(left: Type, right: Type, + type_parameter_checker: TypeParameterChecker = check_type_parameter) -> bool: + """Is 'left' subtype of 'right'? + + Also consider Any to be a subtype of any type, and vice versa. This + recursively applies to components of composite types (List[int] is subtype + of List[Any], for example). + + type_parameter_checker is used to check the type parameters (for example, + A with B in is_subtype(C[A], C[B]). The default checks for subtype relation + between the type arguments (e.g., A and B), taking the variance of the + type var into account. + """ + if (isinstance(right, AnyType) or isinstance(right, UnboundType) + or isinstance(right, ErasedType)): + return True + elif isinstance(right, UnionType) and not isinstance(left, UnionType): + return any(is_subtype(left, item, type_parameter_checker) + for item in right.items) + else: + return left.accept(SubtypeVisitor(right, type_parameter_checker)) + + +def is_subtype_ignoring_tvars(left: Type, right: Type) -> bool: + def ignore_tvars(s: Type, t: Type, v: int) -> bool: + return True + return is_subtype(left, right, ignore_tvars) + + +def is_equivalent(a: Type, b: Type, + type_parameter_checker=check_type_parameter) -> bool: + return is_subtype(a, b, type_parameter_checker) and is_subtype(b, a, type_parameter_checker) + + +def satisfies_upper_bound(a: Type, upper_bound: Type) -> bool: + """Is 'a' valid value for a type variable with the given 'upper_bound'? + + Same as is_subtype except that Void is considered to be a subtype of + any upper_bound. This is needed in a case like + + def f(g: Callable[[], T]) -> T: ... + def h() -> None: ... + f(h) + """ + return isinstance(a, Void) or is_subtype(a, upper_bound) + + +class SubtypeVisitor(TypeVisitor[bool]): + + def __init__(self, right: Type, + type_parameter_checker: TypeParameterChecker) -> None: + self.right = right + self.check_type_parameter = type_parameter_checker + + # visit_x(left) means: is left (which is an instance of X) a subtype of + # right? + + def visit_unbound_type(self, left: UnboundType) -> bool: + return True + + def visit_error_type(self, left: ErrorType) -> bool: + return False + + def visit_type_list(self, t: TypeList) -> bool: + assert False, 'Not supported' + + def visit_any(self, left: AnyType) -> bool: + return True + + def visit_void(self, left: Void) -> bool: + return isinstance(self.right, Void) + + def visit_none_type(self, left: NoneTyp) -> bool: + if experiments.STRICT_OPTIONAL: + return (isinstance(self.right, NoneTyp) or + is_named_instance(self.right, 'builtins.object')) + else: + return not isinstance(self.right, Void) + + def visit_uninhabited_type(self, left: UninhabitedType) -> bool: + return not isinstance(self.right, Void) + + def visit_erased_type(self, left: ErasedType) -> bool: + return True + + def visit_deleted_type(self, left: DeletedType) -> bool: + return True + + def visit_instance(self, left: Instance) -> bool: + if left.type.fallback_to_any: + return True + right = self.right + if isinstance(right, Instance): + if left.type._promote and is_subtype(left.type._promote, + self.right, + self.check_type_parameter): + return True + rname = right.type.fullname() + if not left.type.has_base(rname) and rname != 'builtins.object': + return False + + # Map left type to corresponding right instances. + t = map_instance_to_supertype(left, right.type) + + return all(self.check_type_parameter(lefta, righta, tvar.variance) + for lefta, righta, tvar in + zip(t.args, right.args, right.type.defn.type_vars)) + else: + return False + + def visit_type_var(self, left: TypeVarType) -> bool: + right = self.right + if isinstance(right, TypeVarType): + return left.id == right.id + else: + return is_subtype(left.upper_bound, self.right) + + def visit_callable_type(self, left: CallableType) -> bool: + right = self.right + if isinstance(right, CallableType): + return is_callable_subtype(left, right) + elif isinstance(right, Overloaded): + return all(is_subtype(left, item, self.check_type_parameter) + for item in right.items()) + elif isinstance(right, Instance): + return is_subtype(left.fallback, right) + elif isinstance(right, TypeType): + # This is unsound, we don't check the __init__ signature. + return left.is_type_obj() and is_subtype(left.ret_type, right.item) + else: + return False + + def visit_tuple_type(self, left: TupleType) -> bool: + right = self.right + if isinstance(right, Instance): + if is_named_instance(right, 'builtins.object'): + return True + if is_named_instance(right, 'builtins.tuple'): + target_item_type = right.args[0] + return all(is_subtype(item, target_item_type) + for item in left.items) + elif is_named_instance(right, 'typing.Sized'): + return True + elif (is_named_instance(right, 'typing.Iterable') or + is_named_instance(right, 'typing.Container') or + is_named_instance(right, 'typing.Sequence') or + is_named_instance(right, 'typing.Reversible')): + iter_type = right.args[0] + return all(is_subtype(li, iter_type) for li in left.items) + return False + elif isinstance(right, TupleType): + if len(left.items) != len(right.items): + return False + for l, r in zip(left.items, right.items): + if not is_subtype(l, r, self.check_type_parameter): + return False + if not is_subtype(left.fallback, right.fallback, self.check_type_parameter): + return False + return True + else: + return False + + def visit_namedtuple_type(self, left: NamedTupleType) -> bool: + if isinstance(self.right, NamedTupleType): + # left.attrs.issubset(right.attrs) is insufficient + # (a: int, b: str) is not subtype of (b: int, a: str) + if any(l != r for l, r in zip(left.attrs, self.right.attrs)): + return False + return self.visit_tuple_type(left) + + def visit_overloaded(self, left: Overloaded) -> bool: + right = self.right + if isinstance(right, Instance): + return is_subtype(left.fallback, right) + elif isinstance(right, CallableType) or is_named_instance( + right, 'builtins.type'): + for item in left.items(): + if is_subtype(item, right, self.check_type_parameter): + return True + return False + elif isinstance(right, Overloaded): + # TODO: this may be too restrictive + if len(left.items()) != len(right.items()): + return False + for i in range(len(left.items())): + if not is_subtype(left.items()[i], right.items()[i], self.check_type_parameter): + return False + return True + elif isinstance(right, UnboundType): + return True + elif isinstance(right, TypeType): + # All the items must have the same type object status, so + # it's sufficient to query only (any) one of them. + # This is unsound, we don't check the __init__ signature. + return left.is_type_obj() and is_subtype(left.items()[0].ret_type, right.item) + else: + return False + + def visit_union_type(self, left: UnionType) -> bool: + return all(is_subtype(item, self.right, self.check_type_parameter) + for item in left.items) + + def visit_partial_type(self, left: PartialType) -> bool: + # This is indeterminate as we don't really know the complete type yet. + raise RuntimeError + + def visit_type_type(self, left: TypeType) -> bool: + right = self.right + if isinstance(right, TypeType): + return is_subtype(left.item, right.item) + if isinstance(right, CallableType): + # This is unsound, we don't check the __init__ signature. + return right.is_type_obj() and is_subtype(left.item, right.ret_type) + if (isinstance(right, Instance) and + right.type.fullname() in ('builtins.type', 'builtins.object')): + # Treat builtins.type the same as Type[Any]; + # treat builtins.object the same as Any. + return True + return False + + +def is_callable_subtype(left: CallableType, right: CallableType, + ignore_return: bool = False) -> bool: + """Is left a subtype of right?""" + # TODO: Support named arguments, **args, etc. + # Non-type cannot be a subtype of type. + if right.is_type_obj() and not left.is_type_obj(): + return False + + # A callable L is a subtype of a generic callable R if L is a + # subtype of every type obtained from R by substituting types for + # the variables of R. We can check this by simply leaving the + # generic variables of R as type variables, effectively varying + # over all possible values. + + # It's okay even if these variables share ids with generic + # type variables of L, because generating and solving + # constraints for the variables of L to make L a subtype of R + # (below) treats type variables on the two sides as independent. + + if left.variables: + # Apply generic type variables away in left via type inference. + left = unify_generic_callable(left, right, ignore_return=ignore_return) + if left is None: + return False + + # Check return types. + if not ignore_return and not is_subtype(left.ret_type, right.ret_type): + return False + + if right.is_ellipsis_args: + return True + + # Check argument types. + if left.min_args > right.min_args: + return False + if left.is_var_arg: + return is_var_arg_callable_subtype_helper(left, right) + if right.is_var_arg: + return False + if len(left.arg_types) < len(right.arg_types): + return False + for i in range(len(right.arg_types)): + if not is_subtype(right.arg_types[i], left.arg_types[i]): + return False + return True + + +def is_var_arg_callable_subtype_helper(left: CallableType, right: CallableType) -> bool: + """Is left a subtype of right, assuming left has *args? + + See also is_callable_subtype for additional assumptions we can make. + """ + left_fixed = left.max_fixed_args() + right_fixed = right.max_fixed_args() + num_fixed_matching = min(left_fixed, right_fixed) + for i in range(num_fixed_matching): + if not is_subtype(right.arg_types[i], left.arg_types[i]): + return False + if not right.is_var_arg: + for i in range(num_fixed_matching, len(right.arg_types)): + if not is_subtype(right.arg_types[i], left.arg_types[-1]): + return False + return True + else: + for i in range(left_fixed, right_fixed): + if not is_subtype(right.arg_types[i], left.arg_types[-1]): + return False + for i in range(right_fixed, left_fixed): + if not is_subtype(right.arg_types[-1], left.arg_types[i]): + return False + return is_subtype(right.arg_types[-1], left.arg_types[-1]) + + +def unify_generic_callable(type: CallableType, target: CallableType, + ignore_return: bool) -> CallableType: + """Try to unify a generic callable type with another callable type. + + Return unified CallableType if successful; otherwise, return None. + """ + import mypy.solve + constraints = [] # type: List[mypy.constraints.Constraint] + for arg_type, target_arg_type in zip(type.arg_types, target.arg_types): + c = mypy.constraints.infer_constraints( + arg_type, target_arg_type, mypy.constraints.SUPERTYPE_OF) + constraints.extend(c) + if not ignore_return: + c = mypy.constraints.infer_constraints( + type.ret_type, target.ret_type, mypy.constraints.SUBTYPE_OF) + constraints.extend(c) + type_var_ids = [tvar.id for tvar in type.variables] + inferred_vars = mypy.solve.solve_constraints(type_var_ids, constraints) + if None in inferred_vars: + return None + msg = messages.temp_message_builder() + applied = mypy.applytype.apply_generic_arguments(type, inferred_vars, msg, context=target) + if msg.is_errors() or not isinstance(applied, CallableType): + return None + return cast(CallableType, applied) + + +def restrict_subtype_away(t: Type, s: Type) -> Type: + """Return a supertype of (t intersect not s) + + Currently just remove elements of a union type. + """ + if isinstance(t, UnionType): + new_items = [item for item in t.items if not is_subtype(item, s)] + return UnionType.make_union(new_items) + else: + return t + + +def is_proper_subtype(t: Type, s: Type) -> bool: + """Check if t is a proper subtype of s? + + For proper subtypes, there's no need to rely on compatibility due to + Any types. Any instance type t is also a proper subtype of t. + """ + # FIX tuple types + if isinstance(t, Instance): + if isinstance(s, Instance): + if not t.type.has_base(s.type.fullname()): + return False + + def check_argument(left: Type, right: Type, variance: int) -> bool: + if variance == COVARIANT: + return is_proper_subtype(left, right) + elif variance == CONTRAVARIANT: + return is_proper_subtype(right, left) + else: + return sametypes.is_same_type(left, right) + + # Map left type to corresponding right instances. + t = map_instance_to_supertype(t, s.type) + + return all(check_argument(ta, ra, tvar.variance) for ta, ra, tvar in + zip(t.args, s.args, s.type.defn.type_vars)) + return False + else: + return sametypes.is_same_type(t, s) + + +def is_more_precise(t: Type, s: Type) -> bool: + """Check if t is a more precise type than s. + + A t is a proper subtype of s, t is also more precise than s. Also, if + s is Any, t is more precise than s for any t. Finally, if t is the same + type as s, t is more precise than s. + """ + # TODO Should List[int] be more precise than List[Any]? + if isinstance(s, AnyType): + return True + if isinstance(s, Instance): + if isinstance(t, CallableType): + # Fall back to subclass check and ignore other properties of the callable. + return is_proper_subtype(t.fallback, s) + return is_proper_subtype(t, s) + return sametypes.is_same_type(t, s) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index f10048c4f0fe4..1841f283a19e1 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -1,430 +1,430 @@ -"""Semantic analysis of types""" - -from typing import Callable, cast, List, Tuple - -from mypy.types import ( - Type, UnboundType, TypeVarType, TupleType, NamedTupleType, UnionType, Instance, AnyType, CallableType, - Void, NoneTyp, DeletedType, TypeList, TypeVarDef, TypeVisitor, StarType, PartialType, - EllipsisType, UninhabitedType, TypeType -) -from mypy.nodes import ( - BOUND_TVAR, TYPE_ALIAS, UNBOUND_IMPORTED, - TypeInfo, Context, SymbolTableNode, TypeVarExpr, Var, Node, - IndexExpr, RefExpr -) -from mypy.sametypes import is_same_type -from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError -from mypy.subtypes import satisfies_upper_bound -from mypy import nodes -from mypy import experiments - - -type_constructors = {'typing.Tuple', 'typing.Union', 'typing.Callable', 'typing.Type'} - - -def analyze_type_alias(node: Node, - lookup_func: Callable[[str, Context], SymbolTableNode], - lookup_fqn_func: Callable[[str], SymbolTableNode], - fail_func: Callable[[str, Context], None]) -> Type: - """Return type if node is valid as a type alias rvalue. - - Return None otherwise. 'node' must have been semantically analyzed. - """ - # Quickly return None if the expression doesn't look like a type. Note - # that we don't support straight string literals as type aliases - # (only string literals within index expressions). - if isinstance(node, RefExpr): - if not (isinstance(node.node, TypeInfo) or - node.fullname == 'typing.Any' or - node.kind == TYPE_ALIAS): - return None - elif isinstance(node, IndexExpr): - base = node.base - if isinstance(base, RefExpr): - if not (isinstance(base.node, TypeInfo) or - base.fullname in type_constructors): - return None - else: - return None - else: - return None - - # It's a type alias (though it may be an invalid one). - try: - type = expr_to_unanalyzed_type(node) - except TypeTranslationError: - fail_func('Invalid type alias', node) - return None - analyzer = TypeAnalyser(lookup_func, lookup_fqn_func, fail_func) - return type.accept(analyzer) - - -class TypeAnalyser(TypeVisitor[Type]): - """Semantic analyzer for types (semantic analysis pass 2). - - Converts unbound types into bound types. - """ - - def __init__(self, - lookup_func: Callable[[str, Context], SymbolTableNode], - lookup_fqn_func: Callable[[str], SymbolTableNode], - fail_func: Callable[[str, Context], None]) -> None: - self.lookup = lookup_func - self.lookup_fqn_func = lookup_fqn_func - self.fail = fail_func - - def visit_unbound_type(self, t: UnboundType) -> Type: - if t.optional: - t.optional = False - # We don't need to worry about double-wrapping Optionals or - # wrapping Anys: Union simplification will take care of that. - return UnionType.make_simplified_union([self.visit_unbound_type(t), NoneTyp()]) - sym = self.lookup(t.name, t) - if sym is not None: - if sym.node is None: - # UNBOUND_IMPORTED can happen if an unknown name was imported. - if sym.kind != UNBOUND_IMPORTED: - self.fail('Internal error (node is None, kind={})'.format(sym.kind), t) - return AnyType() - fullname = sym.node.fullname() - if sym.kind == BOUND_TVAR: - if len(t.args) > 0: - self.fail('Type variable "{}" used with arguments'.format( - t.name), t) - assert sym.tvar_def is not None - return TypeVarType(sym.tvar_def, t.line) - elif fullname == 'builtins.None': - if experiments.STRICT_OPTIONAL: - if t.is_ret_type: - return Void() - else: - return NoneTyp() - else: - return Void() - elif fullname == 'typing.Any': - return AnyType() - elif fullname == 'typing.Tuple': - if len(t.args) == 2 and isinstance(t.args[1], EllipsisType): - # Tuple[T, ...] (uniform, variable-length tuple) - node = self.lookup_fqn_func('builtins.tuple') - tuple_info = cast(TypeInfo, node.node) - return Instance(tuple_info, [t.args[0].accept(self)], t.line) - return self.tuple_type(self.anal_array(t.args)) - elif fullname == 'typing.Union': - items = self.anal_array(t.args) - items = [item for item in items if not isinstance(item, Void)] - return UnionType.make_union(items) - elif fullname == 'typing.Optional': - if len(t.args) != 1: - self.fail('Optional[...] must have exactly one type argument', t) - return AnyType() - items = self.anal_array(t.args) - if experiments.STRICT_OPTIONAL: - return UnionType.make_simplified_union([items[0], NoneTyp()]) - else: - # Without strict Optional checking Optional[t] is just an alias for t. - return items[0] - elif fullname == 'typing.Callable': - return self.analyze_callable_type(t) - elif fullname == 'typing.Type': - if len(t.args) == 0: - return TypeType(AnyType(), line=t.line) - if len(t.args) != 1: - self.fail('Type[...] must have exactly one type argument', t) - items = self.anal_array(t.args) - item = items[0] - return TypeType(item, line=t.line) - elif sym.kind == TYPE_ALIAS: - # TODO: Generic type aliases. - return sym.type_override - elif not isinstance(sym.node, TypeInfo): - name = sym.fullname - if name is None: - name = sym.node.name() - if isinstance(sym.node, Var) and isinstance(sym.node.type, AnyType): - # Something with an Any type -- make it an alias for Any in a type - # context. This is slightly problematic as it allows using the type 'Any' - # as a base class -- however, this will fail soon at runtime so the problem - # is pretty minor. - return AnyType() - self.fail('Invalid type "{}"'.format(name), t) - return t - info = sym.node # type: TypeInfo - if len(t.args) > 0 and info.fullname() == 'builtins.tuple': - return TupleType(self.anal_array(t.args), - Instance(info, [AnyType()], t.line), - t.line) - else: - # Analyze arguments and construct Instance type. The - # number of type arguments and their values are - # checked only later, since we do not always know the - # valid count at this point. Thus we may construct an - # Instance with an invalid number of type arguments. - instance = Instance(info, self.anal_array(t.args), t.line) - if info.tuple_type is None: - return instance - else: - # The class has a Tuple[...] base class so it will be - # represented as a tuple type. - if t.args: - self.fail('Generic tuple types not supported', t) - return AnyType() - # import pdb; pdb.set_trace() - if info.is_named_tuple: - return NamedTupleType(info.name(), info.attrs, - self.anal_array(info.tuple_type.items), - fallback=instance, - line=t.line) - return TupleType(self.anal_array(info.tuple_type.items), - fallback=instance, - line=t.line) - else: - return AnyType() - - def visit_any(self, t: AnyType) -> Type: - return t - - def visit_void(self, t: Void) -> Type: - return t - - def visit_none_type(self, t: NoneTyp) -> Type: - return t - - def visit_uninhabited_type(self, t: UninhabitedType) -> Type: - return t - - def visit_deleted_type(self, t: DeletedType) -> Type: - return t - - def visit_type_list(self, t: TypeList) -> Type: - self.fail('Invalid type', t) - return AnyType() - - def visit_instance(self, t: Instance) -> Type: - return t - - def visit_type_var(self, t: TypeVarType) -> Type: - raise RuntimeError('TypeVarType is already analyzed') - - def visit_callable_type(self, t: CallableType) -> Type: - return t.copy_modified(arg_types=self.anal_array(t.arg_types), - ret_type=t.ret_type.accept(self), - fallback=t.fallback or self.builtin_type('builtins.function'), - variables=self.anal_var_defs(t.variables)) - - def visit_tuple_type(self, t: TupleType) -> Type: - if t.implicit: - self.fail('Invalid tuple literal type', t) - return AnyType() - star_count = sum(1 for item in t.items if isinstance(item, StarType)) - if star_count > 1: - self.fail('At most one star type allowed in a tuple', t) - return AnyType() - fallback = t.fallback if t.fallback else self.builtin_type('builtins.tuple', [AnyType()]) - return TupleType(self.anal_array(t.items), fallback, t.line) - - def visit_namedtuple_type(self, t: NamedTupleType) -> Type: - return NamedTupleType(t.name, t.attrs, - self.anal_array(t.items), - fallback=t.fallback, - line=t.line) - - def visit_star_type(self, t: StarType) -> Type: - return StarType(t.type.accept(self), t.line) - - def visit_union_type(self, t: UnionType) -> Type: - return UnionType(self.anal_array(t.items), t.line) - - def visit_partial_type(self, t: PartialType) -> Type: - assert False, "Internal error: Unexpected partial type" - - def visit_ellipsis_type(self, t: EllipsisType) -> Type: - self.fail("Unexpected '...'", t) - return AnyType() - - def visit_type_type(self, t: TypeType) -> Type: - return TypeType(t.item.accept(self), line=t.line) - - def analyze_callable_type(self, t: UnboundType) -> Type: - fallback = self.builtin_type('builtins.function') - if len(t.args) == 0: - # Callable (bare). Treat as Callable[..., Any]. - return CallableType([AnyType(), AnyType()], - [nodes.ARG_STAR, nodes.ARG_STAR2], - [None, None], - ret_type=AnyType(), - fallback=fallback, - is_ellipsis_args=True) - elif len(t.args) == 2: - ret_type = t.args[1].accept(self) - if isinstance(t.args[0], TypeList): - # Callable[[ARG, ...], RET] (ordinary callable type) - args = t.args[0].items - return CallableType(self.anal_array(args), - [nodes.ARG_POS] * len(args), - [None] * len(args), - ret_type=ret_type, - fallback=fallback) - elif isinstance(t.args[0], EllipsisType): - # Callable[..., RET] (with literal ellipsis; accept arbitrary arguments) - return CallableType([AnyType(), AnyType()], - [nodes.ARG_STAR, nodes.ARG_STAR2], - [None, None], - ret_type=ret_type, - fallback=fallback, - is_ellipsis_args=True) - else: - self.fail('The first argument to Callable must be a list of types or "..."', t) - return AnyType() - - self.fail('Invalid function type', t) - return AnyType() - - def anal_array(self, a: List[Type]) -> List[Type]: - res = [] # type: List[Type] - for t in a: - res.append(t.accept(self)) - return res - - def anal_var_defs(self, var_defs: List[TypeVarDef]) -> List[TypeVarDef]: - a = [] # type: List[TypeVarDef] - for vd in var_defs: - a.append(TypeVarDef(vd.name, vd.id.raw_id, self.anal_array(vd.values), - vd.upper_bound.accept(self), - vd.variance, - vd.line)) - return a - - def builtin_type(self, fully_qualified_name: str, args: List[Type] = None) -> Instance: - node = self.lookup_fqn_func(fully_qualified_name) - info = cast(TypeInfo, node.node) - return Instance(info, args or []) - - def tuple_type(self, items: List[Type]) -> TupleType: - return TupleType(items, fallback=self.builtin_type('builtins.tuple', [AnyType()])) - - -class TypeAnalyserPass3(TypeVisitor[None]): - """Analyze type argument counts and values of generic types. - - This is semantic analysis pass 3 for types. - - Perform these operations: - - * Report error for invalid type argument counts, such as List[x, y]. - * Make implicit Any type arguments explicit my modifying types - in-place. For example, modify Foo into Foo[Any] if Foo expects a single - type argument. - * If a type variable has a value restriction, ensure that the value is - valid. For example, reject IO[int] if the type argument must be str - or bytes. - - We can't do this earlier than the third pass, since type argument counts - are only determined in pass 2, and we have to support forward references - to types. - """ - - def __init__(self, fail_func: Callable[[str, Context], None]) -> None: - self.fail = fail_func - - def visit_instance(self, t: Instance) -> None: - info = t.type - # Check type argument count. - if len(t.args) != len(info.type_vars): - if len(t.args) == 0: - # Insert implicit 'Any' type arguments. - t.args = [AnyType()] * len(info.type_vars) - return - # Invalid number of type parameters. - n = len(info.type_vars) - s = '{} type arguments'.format(n) - if n == 0: - s = 'no type arguments' - elif n == 1: - s = '1 type argument' - act = str(len(t.args)) - if act == '0': - act = 'none' - self.fail('"{}" expects {}, but {} given'.format( - info.name(), s, act), t) - # Construct the correct number of type arguments, as - # otherwise the type checker may crash as it expects - # things to be right. - t.args = [AnyType() for _ in info.type_vars] - elif info.defn.type_vars: - # Check type argument values. - for arg, TypeVar in zip(t.args, info.defn.type_vars): - if TypeVar.values: - if isinstance(arg, TypeVarType): - arg_values = arg.values - if not arg_values: - self.fail('Type variable "{}" not valid as type ' - 'argument value for "{}"'.format( - arg.name, info.name()), t) - continue - else: - arg_values = [arg] - self.check_type_var_values(info, arg_values, - TypeVar.values, t) - if not satisfies_upper_bound(arg, TypeVar.upper_bound): - self.fail('Type argument "{}" of "{}" must be ' - 'a subtype of "{}"'.format( - arg, info.name(), TypeVar.upper_bound), t) - for arg in t.args: - arg.accept(self) - - def check_type_var_values(self, type: TypeInfo, actuals: List[Type], - valids: List[Type], context: Context) -> None: - for actual in actuals: - if (not isinstance(actual, AnyType) and - not any(is_same_type(actual, value) for value in valids)): - self.fail('Invalid type argument value for "{}"'.format( - type.name()), context) - - def visit_callable_type(self, t: CallableType) -> None: - t.ret_type.accept(self) - for arg_type in t.arg_types: - arg_type.accept(self) - - def visit_tuple_type(self, t: TupleType) -> None: - for item in t.items: - item.accept(self) - - def visit_union_type(self, t: UnionType) -> None: - for item in t.items: - item.accept(self) - - def visit_star_type(self, t: StarType) -> None: - t.type.accept(self) - - # Other kinds of type are trivial, since they are atomic (or invalid). - - def visit_unbound_type(self, t: UnboundType) -> None: - pass - - def visit_any(self, t: AnyType) -> None: - pass - - def visit_void(self, t: Void) -> None: - pass - - def visit_none_type(self, t: NoneTyp) -> None: - pass - - def visit_uninhabited_type(self, t: UninhabitedType) -> None: - pass - - def visit_deleted_type(self, t: DeletedType) -> None: - pass - - def visit_type_list(self, t: TypeList) -> None: - self.fail('Invalid type', t) - - def visit_type_var(self, t: TypeVarType) -> None: - pass - - def visit_partial_type(self, t: PartialType) -> None: - pass - - def visit_type_type(self, t: TypeType) -> None: - pass +"""Semantic analysis of types""" + +from typing import Callable, cast, List, Tuple + +from mypy.types import ( + Type, UnboundType, TypeVarType, TupleType, NamedTupleType, UnionType, Instance, AnyType, CallableType, + Void, NoneTyp, DeletedType, TypeList, TypeVarDef, TypeVisitor, StarType, PartialType, + EllipsisType, UninhabitedType, TypeType +) +from mypy.nodes import ( + BOUND_TVAR, TYPE_ALIAS, UNBOUND_IMPORTED, + TypeInfo, Context, SymbolTableNode, TypeVarExpr, Var, Node, + IndexExpr, RefExpr +) +from mypy.sametypes import is_same_type +from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError +from mypy.subtypes import satisfies_upper_bound +from mypy import nodes +from mypy import experiments + + +type_constructors = {'typing.Tuple', 'typing.Union', 'typing.Callable', 'typing.Type'} + + +def analyze_type_alias(node: Node, + lookup_func: Callable[[str, Context], SymbolTableNode], + lookup_fqn_func: Callable[[str], SymbolTableNode], + fail_func: Callable[[str, Context], None]) -> Type: + """Return type if node is valid as a type alias rvalue. + + Return None otherwise. 'node' must have been semantically analyzed. + """ + # Quickly return None if the expression doesn't look like a type. Note + # that we don't support straight string literals as type aliases + # (only string literals within index expressions). + if isinstance(node, RefExpr): + if not (isinstance(node.node, TypeInfo) or + node.fullname == 'typing.Any' or + node.kind == TYPE_ALIAS): + return None + elif isinstance(node, IndexExpr): + base = node.base + if isinstance(base, RefExpr): + if not (isinstance(base.node, TypeInfo) or + base.fullname in type_constructors): + return None + else: + return None + else: + return None + + # It's a type alias (though it may be an invalid one). + try: + type = expr_to_unanalyzed_type(node) + except TypeTranslationError: + fail_func('Invalid type alias', node) + return None + analyzer = TypeAnalyser(lookup_func, lookup_fqn_func, fail_func) + return type.accept(analyzer) + + +class TypeAnalyser(TypeVisitor[Type]): + """Semantic analyzer for types (semantic analysis pass 2). + + Converts unbound types into bound types. + """ + + def __init__(self, + lookup_func: Callable[[str, Context], SymbolTableNode], + lookup_fqn_func: Callable[[str], SymbolTableNode], + fail_func: Callable[[str, Context], None]) -> None: + self.lookup = lookup_func + self.lookup_fqn_func = lookup_fqn_func + self.fail = fail_func + + def visit_unbound_type(self, t: UnboundType) -> Type: + if t.optional: + t.optional = False + # We don't need to worry about double-wrapping Optionals or + # wrapping Anys: Union simplification will take care of that. + return UnionType.make_simplified_union([self.visit_unbound_type(t), NoneTyp()]) + sym = self.lookup(t.name, t) + if sym is not None: + if sym.node is None: + # UNBOUND_IMPORTED can happen if an unknown name was imported. + if sym.kind != UNBOUND_IMPORTED: + self.fail('Internal error (node is None, kind={})'.format(sym.kind), t) + return AnyType() + fullname = sym.node.fullname() + if sym.kind == BOUND_TVAR: + if len(t.args) > 0: + self.fail('Type variable "{}" used with arguments'.format( + t.name), t) + assert sym.tvar_def is not None + return TypeVarType(sym.tvar_def, t.line) + elif fullname == 'builtins.None': + if experiments.STRICT_OPTIONAL: + if t.is_ret_type: + return Void() + else: + return NoneTyp() + else: + return Void() + elif fullname == 'typing.Any': + return AnyType() + elif fullname == 'typing.Tuple': + if len(t.args) == 2 and isinstance(t.args[1], EllipsisType): + # Tuple[T, ...] (uniform, variable-length tuple) + node = self.lookup_fqn_func('builtins.tuple') + tuple_info = cast(TypeInfo, node.node) + return Instance(tuple_info, [t.args[0].accept(self)], t.line) + return self.tuple_type(self.anal_array(t.args)) + elif fullname == 'typing.Union': + items = self.anal_array(t.args) + items = [item for item in items if not isinstance(item, Void)] + return UnionType.make_union(items) + elif fullname == 'typing.Optional': + if len(t.args) != 1: + self.fail('Optional[...] must have exactly one type argument', t) + return AnyType() + items = self.anal_array(t.args) + if experiments.STRICT_OPTIONAL: + return UnionType.make_simplified_union([items[0], NoneTyp()]) + else: + # Without strict Optional checking Optional[t] is just an alias for t. + return items[0] + elif fullname == 'typing.Callable': + return self.analyze_callable_type(t) + elif fullname == 'typing.Type': + if len(t.args) == 0: + return TypeType(AnyType(), line=t.line) + if len(t.args) != 1: + self.fail('Type[...] must have exactly one type argument', t) + items = self.anal_array(t.args) + item = items[0] + return TypeType(item, line=t.line) + elif sym.kind == TYPE_ALIAS: + # TODO: Generic type aliases. + return sym.type_override + elif not isinstance(sym.node, TypeInfo): + name = sym.fullname + if name is None: + name = sym.node.name() + if isinstance(sym.node, Var) and isinstance(sym.node.type, AnyType): + # Something with an Any type -- make it an alias for Any in a type + # context. This is slightly problematic as it allows using the type 'Any' + # as a base class -- however, this will fail soon at runtime so the problem + # is pretty minor. + return AnyType() + self.fail('Invalid type "{}"'.format(name), t) + return t + info = sym.node # type: TypeInfo + if len(t.args) > 0 and info.fullname() == 'builtins.tuple': + return TupleType(self.anal_array(t.args), + Instance(info, [AnyType()], t.line), + t.line) + else: + # Analyze arguments and construct Instance type. The + # number of type arguments and their values are + # checked only later, since we do not always know the + # valid count at this point. Thus we may construct an + # Instance with an invalid number of type arguments. + instance = Instance(info, self.anal_array(t.args), t.line) + if info.tuple_type is None: + return instance + else: + # The class has a Tuple[...] base class so it will be + # represented as a tuple type. + if t.args: + self.fail('Generic tuple types not supported', t) + return AnyType() + # import pdb; pdb.set_trace() + if info.is_named_tuple: + return NamedTupleType(info.name(), info.attrs, + self.anal_array(info.tuple_type.items), + fallback=instance, + line=t.line) + return TupleType(self.anal_array(info.tuple_type.items), + fallback=instance, + line=t.line) + else: + return AnyType() + + def visit_any(self, t: AnyType) -> Type: + return t + + def visit_void(self, t: Void) -> Type: + return t + + def visit_none_type(self, t: NoneTyp) -> Type: + return t + + def visit_uninhabited_type(self, t: UninhabitedType) -> Type: + return t + + def visit_deleted_type(self, t: DeletedType) -> Type: + return t + + def visit_type_list(self, t: TypeList) -> Type: + self.fail('Invalid type', t) + return AnyType() + + def visit_instance(self, t: Instance) -> Type: + return t + + def visit_type_var(self, t: TypeVarType) -> Type: + raise RuntimeError('TypeVarType is already analyzed') + + def visit_callable_type(self, t: CallableType) -> Type: + return t.copy_modified(arg_types=self.anal_array(t.arg_types), + ret_type=t.ret_type.accept(self), + fallback=t.fallback or self.builtin_type('builtins.function'), + variables=self.anal_var_defs(t.variables)) + + def visit_tuple_type(self, t: TupleType) -> Type: + if t.implicit: + self.fail('Invalid tuple literal type', t) + return AnyType() + star_count = sum(1 for item in t.items if isinstance(item, StarType)) + if star_count > 1: + self.fail('At most one star type allowed in a tuple', t) + return AnyType() + fallback = t.fallback if t.fallback else self.builtin_type('builtins.tuple', [AnyType()]) + return TupleType(self.anal_array(t.items), fallback, t.line) + + def visit_namedtuple_type(self, t: NamedTupleType) -> Type: + return NamedTupleType(t.name, t.attrs, + self.anal_array(t.items), + fallback=t.fallback, + line=t.line) + + def visit_star_type(self, t: StarType) -> Type: + return StarType(t.type.accept(self), t.line) + + def visit_union_type(self, t: UnionType) -> Type: + return UnionType(self.anal_array(t.items), t.line) + + def visit_partial_type(self, t: PartialType) -> Type: + assert False, "Internal error: Unexpected partial type" + + def visit_ellipsis_type(self, t: EllipsisType) -> Type: + self.fail("Unexpected '...'", t) + return AnyType() + + def visit_type_type(self, t: TypeType) -> Type: + return TypeType(t.item.accept(self), line=t.line) + + def analyze_callable_type(self, t: UnboundType) -> Type: + fallback = self.builtin_type('builtins.function') + if len(t.args) == 0: + # Callable (bare). Treat as Callable[..., Any]. + return CallableType([AnyType(), AnyType()], + [nodes.ARG_STAR, nodes.ARG_STAR2], + [None, None], + ret_type=AnyType(), + fallback=fallback, + is_ellipsis_args=True) + elif len(t.args) == 2: + ret_type = t.args[1].accept(self) + if isinstance(t.args[0], TypeList): + # Callable[[ARG, ...], RET] (ordinary callable type) + args = t.args[0].items + return CallableType(self.anal_array(args), + [nodes.ARG_POS] * len(args), + [None] * len(args), + ret_type=ret_type, + fallback=fallback) + elif isinstance(t.args[0], EllipsisType): + # Callable[..., RET] (with literal ellipsis; accept arbitrary arguments) + return CallableType([AnyType(), AnyType()], + [nodes.ARG_STAR, nodes.ARG_STAR2], + [None, None], + ret_type=ret_type, + fallback=fallback, + is_ellipsis_args=True) + else: + self.fail('The first argument to Callable must be a list of types or "..."', t) + return AnyType() + + self.fail('Invalid function type', t) + return AnyType() + + def anal_array(self, a: List[Type]) -> List[Type]: + res = [] # type: List[Type] + for t in a: + res.append(t.accept(self)) + return res + + def anal_var_defs(self, var_defs: List[TypeVarDef]) -> List[TypeVarDef]: + a = [] # type: List[TypeVarDef] + for vd in var_defs: + a.append(TypeVarDef(vd.name, vd.id.raw_id, self.anal_array(vd.values), + vd.upper_bound.accept(self), + vd.variance, + vd.line)) + return a + + def builtin_type(self, fully_qualified_name: str, args: List[Type] = None) -> Instance: + node = self.lookup_fqn_func(fully_qualified_name) + info = cast(TypeInfo, node.node) + return Instance(info, args or []) + + def tuple_type(self, items: List[Type]) -> TupleType: + return TupleType(items, fallback=self.builtin_type('builtins.tuple', [AnyType()])) + + +class TypeAnalyserPass3(TypeVisitor[None]): + """Analyze type argument counts and values of generic types. + + This is semantic analysis pass 3 for types. + + Perform these operations: + + * Report error for invalid type argument counts, such as List[x, y]. + * Make implicit Any type arguments explicit my modifying types + in-place. For example, modify Foo into Foo[Any] if Foo expects a single + type argument. + * If a type variable has a value restriction, ensure that the value is + valid. For example, reject IO[int] if the type argument must be str + or bytes. + + We can't do this earlier than the third pass, since type argument counts + are only determined in pass 2, and we have to support forward references + to types. + """ + + def __init__(self, fail_func: Callable[[str, Context], None]) -> None: + self.fail = fail_func + + def visit_instance(self, t: Instance) -> None: + info = t.type + # Check type argument count. + if len(t.args) != len(info.type_vars): + if len(t.args) == 0: + # Insert implicit 'Any' type arguments. + t.args = [AnyType()] * len(info.type_vars) + return + # Invalid number of type parameters. + n = len(info.type_vars) + s = '{} type arguments'.format(n) + if n == 0: + s = 'no type arguments' + elif n == 1: + s = '1 type argument' + act = str(len(t.args)) + if act == '0': + act = 'none' + self.fail('"{}" expects {}, but {} given'.format( + info.name(), s, act), t) + # Construct the correct number of type arguments, as + # otherwise the type checker may crash as it expects + # things to be right. + t.args = [AnyType() for _ in info.type_vars] + elif info.defn.type_vars: + # Check type argument values. + for arg, TypeVar in zip(t.args, info.defn.type_vars): + if TypeVar.values: + if isinstance(arg, TypeVarType): + arg_values = arg.values + if not arg_values: + self.fail('Type variable "{}" not valid as type ' + 'argument value for "{}"'.format( + arg.name, info.name()), t) + continue + else: + arg_values = [arg] + self.check_type_var_values(info, arg_values, + TypeVar.values, t) + if not satisfies_upper_bound(arg, TypeVar.upper_bound): + self.fail('Type argument "{}" of "{}" must be ' + 'a subtype of "{}"'.format( + arg, info.name(), TypeVar.upper_bound), t) + for arg in t.args: + arg.accept(self) + + def check_type_var_values(self, type: TypeInfo, actuals: List[Type], + valids: List[Type], context: Context) -> None: + for actual in actuals: + if (not isinstance(actual, AnyType) and + not any(is_same_type(actual, value) for value in valids)): + self.fail('Invalid type argument value for "{}"'.format( + type.name()), context) + + def visit_callable_type(self, t: CallableType) -> None: + t.ret_type.accept(self) + for arg_type in t.arg_types: + arg_type.accept(self) + + def visit_tuple_type(self, t: TupleType) -> None: + for item in t.items: + item.accept(self) + + def visit_union_type(self, t: UnionType) -> None: + for item in t.items: + item.accept(self) + + def visit_star_type(self, t: StarType) -> None: + t.type.accept(self) + + # Other kinds of type are trivial, since they are atomic (or invalid). + + def visit_unbound_type(self, t: UnboundType) -> None: + pass + + def visit_any(self, t: AnyType) -> None: + pass + + def visit_void(self, t: Void) -> None: + pass + + def visit_none_type(self, t: NoneTyp) -> None: + pass + + def visit_uninhabited_type(self, t: UninhabitedType) -> None: + pass + + def visit_deleted_type(self, t: DeletedType) -> None: + pass + + def visit_type_list(self, t: TypeList) -> None: + self.fail('Invalid type', t) + + def visit_type_var(self, t: TypeVarType) -> None: + pass + + def visit_partial_type(self, t: PartialType) -> None: + pass + + def visit_type_type(self, t: TypeType) -> None: + pass diff --git a/mypy/types.py b/mypy/types.py index 57bc7bd6b26f7..7d8dcb1c1a603 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1,1451 +1,1451 @@ -"""Classes for representing mypy types.""" - -from abc import abstractmethod -from typing import ( - Any, TypeVar, Dict, List, Tuple, cast, Generic, Set, Sequence, Optional, Union -) - -import mypy.nodes -from mypy.nodes import INVARIANT, SymbolNode - -from mypy import experiments - - -T = TypeVar('T') - -JsonDict = Dict[str, Any] - - -class Type(mypy.nodes.Context): - """Abstract base class for all types.""" - - line = 0 - - def __init__(self, line: int = -1) -> None: - self.line = line - - def get_line(self) -> int: - return self.line - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - raise RuntimeError('Not implemented') - - def __repr__(self) -> str: - return self.accept(TypeStrVisitor()) - - def serialize(self) -> JsonDict: - raise NotImplementedError('Cannot serialize {} instance'.format(self.__class__.__name__)) - - @classmethod - def deserialize(cls, data: JsonDict) -> 'Type': - classname = data['.class'] - glo = globals() - if classname in glo: - cl = glo[classname] - if 'deserialize' in cl.__dict__: - return cl.deserialize(data) - raise NotImplementedError('unexpected .class {}'.format(classname)) - - -class TypeVarId: - # A type variable is uniquely identified by its raw id and meta level. - - # For plain variables (type parameters of generic classes and - # functions) raw ids are allocated by semantic analysis, using - # positive ids 1, 2, ... for generic class parameters and negative - # ids -1, ... for generic function type arguments. This convention - # is only used to keep type variable ids distinct when allocating - # them; the type checker makes no distinction between class and - # function type variables. - - # Metavariables are allocated unique ids starting from 1. - raw_id = 0 # type: int - - # Level of the variable in type inference. Currently either 0 for - # declared types, or 1 for type inference metavariables. - meta_level = 0 # type: int - - # Class variable used for allocating fresh ids for metavariables. - next_raw_id = 1 # type: int - - def __init__(self, raw_id: int, meta_level: int = 0) -> None: - self.raw_id = raw_id - self.meta_level = meta_level - - @staticmethod - def new(meta_level: int) -> 'TypeVarId': - raw_id = TypeVarId.next_raw_id - TypeVarId.next_raw_id += 1 - return TypeVarId(raw_id, meta_level) - - def __repr__(self) -> str: - return self.raw_id.__repr__() - - def __eq__(self, other: object) -> bool: - if isinstance(other, TypeVarId): - return (self.raw_id == other.raw_id and - self.meta_level == other.meta_level) - else: - return False - - def __ne__(self, other: object) -> bool: - return not (self == other) - - def __hash__(self) -> int: - return hash((self.raw_id, self.meta_level)) - - def is_meta_var(self) -> bool: - return self.meta_level > 0 - - -class TypeVarDef(mypy.nodes.Context): - """Definition of a single type variable.""" - - name = '' - id = None # type: TypeVarId - values = None # type: List[Type] # Value restriction, empty list if no restriction - upper_bound = None # type: Type - variance = INVARIANT # type: int - line = 0 - - def __init__(self, name: str, id: Union[TypeVarId, int], values: Optional[List[Type]], - upper_bound: Type, variance: int = INVARIANT, line: int = -1) -> None: - self.name = name - if isinstance(id, int): - id = TypeVarId(id) - self.id = id - self.values = values - self.upper_bound = upper_bound - self.variance = variance - self.line = line - - @staticmethod - def new_unification_variable(old: 'TypeVarDef') -> 'TypeVarDef': - new_id = TypeVarId.new(meta_level=1) - return TypeVarDef(old.name, new_id, old.values, - old.upper_bound, old.variance, old.line) - - def get_line(self) -> int: - return self.line - - def __repr__(self) -> str: - if self.values: - return '{} in {}'.format(self.name, tuple(self.values)) - elif not is_named_instance(self.upper_bound, 'builtins.object'): - return '{} <: {}'.format(self.name, self.upper_bound) - else: - return self.name - - def serialize(self) -> JsonDict: - assert not self.id.is_meta_var() - return {'.class': 'TypeVarDef', - 'name': self.name, - 'id': self.id.raw_id, - 'values': None if self.values is None else [v.serialize() for v in self.values], - 'upper_bound': self.upper_bound.serialize(), - 'variance': self.variance, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'TypeVarDef': - assert data['.class'] == 'TypeVarDef' - return TypeVarDef(data['name'], - data['id'], - None if data['values'] is None - else [Type.deserialize(v) for v in data['values']], - Type.deserialize(data['upper_bound']), - data['variance'], - ) - - -class UnboundType(Type): - """Instance type that has not been bound during semantic analysis.""" - - name = '' - args = None # type: List[Type] - # should this type be wrapped in an Optional? - optional = False - # is this type a return type? - is_ret_type = False - - def __init__(self, - name: str, - args: List[Type] = None, - line: int = -1, - optional: bool = False, - is_ret_type: bool = False) -> None: - if not args: - args = [] - self.name = name - self.args = args - self.optional = optional - self.is_ret_type = is_ret_type - super().__init__(line) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_unbound_type(self) - - def serialize(self) -> JsonDict: - return {'.class': 'UnboundType', - 'name': self.name, - 'args': [a.serialize() for a in self.args], - } - - @classmethod - def deserialize(self, data: JsonDict) -> 'UnboundType': - assert data['.class'] == 'UnboundType' - return UnboundType(data['name'], - [Type.deserialize(a) for a in data['args']]) - - -class ErrorType(Type): - """The error type is used as the result of failed type operations.""" - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_error_type(self) - - -class TypeList(Type): - """A list of types [...]. - - This is only used for the arguments of a Callable type, i.e. for - [arg, ...] in Callable[[arg, ...], ret]. This is not a real type - but a syntactic AST construct. - """ - - items = None # type: List[Type] - - def __init__(self, items: List[Type], line: int = -1) -> None: - super().__init__(line) - self.items = items - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_type_list(self) - - def serialize(self) -> JsonDict: - return {'.class': 'TypeList', - 'items': [t.serialize() for t in self.items], - } - - @classmethod - def deserialize(self, data: JsonDict) -> 'TypeList': - assert data['.class'] == 'TypeList' - return TypeList([Type.deserialize(t) for t in data['items']]) - - -class AnyType(Type): - """The type 'Any'.""" - - def __init__(self, implicit=False, line: int = -1) -> None: - super().__init__(line) - self.implicit = implicit - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_any(self) - - def serialize(self) -> JsonDict: - return {'.class': 'AnyType'} - - @classmethod - def deserialize(cls, data: JsonDict) -> 'AnyType': - assert data['.class'] == 'AnyType' - return AnyType() - - -class Void(Type): - """The return type 'None'. - - This can only be used as the return type in a callable type and as - the result type of calling such callable. - """ - - source = '' # May be None; function that generated this value - - def __init__(self, source: str = None, line: int = -1) -> None: - self.source = source - super().__init__(line) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_void(self) - - def with_source(self, source: str) -> 'Void': - return Void(source, self.line) - - def serialize(self) -> JsonDict: - return {'.class': 'Void'} - - @classmethod - def deserialize(cls, data: JsonDict) -> 'Void': - assert data['.class'] == 'Void' - return Void() - - -class UninhabitedType(Type): - """This type has no members. - - This type is almost the bottom type, except it is not a subtype of Void. - With strict Optional checking, it is the only common subtype between all - other types, which allows `meet` to be well defined. Without strict - Optional checking, NoneTyp fills this role. - - In general, for any type T that isn't Void: - join(UninhabitedType, T) = T - meet(UninhabitedType, T) = UninhabitedType - is_subtype(UninhabitedType, T) = True - """ - - def __init__(self, line: int = -1) -> None: - super().__init__(line) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_uninhabited_type(self) - - def serialize(self) -> JsonDict: - return {'.class': 'UninhabitedType'} - - @classmethod - def deserialize(cls, data: JsonDict) -> 'UninhabitedType': - assert data['.class'] == 'UninhabitedType' - return UninhabitedType() - - -class NoneTyp(Type): - """The type of 'None'. - - Without strict Optional checking: - This is only used internally during type inference. Programs - cannot declare a variable of this type, and the type checker - refuses to infer this type for a variable. However, subexpressions - often have this type. Note that this is not used as the result - type when calling a function with a void type, even though - semantically such a function returns a None value; the void type - is used instead so that we can report an error if the caller tries - to do anything with the return value. - - With strict Optional checking: - This type can be written by users as 'None', except as the return value - of a function, where 'None' means Void. - """ - - def __init__(self, line: int = -1) -> None: - super().__init__(line) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_none_type(self) - - def serialize(self) -> JsonDict: - return {'.class': 'NoneTyp'} - - @classmethod - def deserialize(self, data: JsonDict) -> 'NoneTyp': - assert data['.class'] == 'NoneTyp' - return NoneTyp() - - -class ErasedType(Type): - """Placeholder for an erased type. - - This is used during type inference. This has the special property that - it is ignored during type inference. - """ - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_erased_type(self) - - -class DeletedType(Type): - """Type of deleted variables. - - These can be used as lvalues but not rvalues. - """ - - source = '' # May be None; name that generated this value - - def __init__(self, source: str = None, line: int = -1) -> None: - self.source = source - super().__init__(line) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_deleted_type(self) - - def serialize(self) -> JsonDict: - return {'.class': 'DeletedType', - 'source': self.source} - - @classmethod - def deserialize(self, data: JsonDict) -> 'DeletedType': - assert data['.class'] == 'DeletedType' - return DeletedType(data['source']) - - -class Instance(Type): - """An instance type of form C[T1, ..., Tn]. - - The list of type variables may be empty. - """ - - type = None # type: mypy.nodes.TypeInfo - args = None # type: List[Type] - erased = False # True if result of type variable substitution - - def __init__(self, typ: mypy.nodes.TypeInfo, args: List[Type], - line: int = -1, erased: bool = False) -> None: - self.type = typ - self.args = args - self.erased = erased - super().__init__(line) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_instance(self) - - type_ref = None # type: str - - def serialize(self) -> JsonDict: - data = {'.class': 'Instance', - } # type: JsonDict - assert self.type is not None - data['type_ref'] = self.type.alt_fullname or self.type.fullname() - if self.args: - data['args'] = [arg.serialize() for arg in self.args] - return data - - @classmethod - def deserialize(cls, data: JsonDict) -> 'Instance': - assert data['.class'] == 'Instance' - args = [] # type: List[Type] - if 'args' in data: - args_list = data['args'] - assert isinstance(args_list, list) - args = [Type.deserialize(arg) for arg in args_list] - inst = Instance(None, args) - inst.type_ref = data['type_ref'] # Will be fixed up by fixup.py later. - return inst - - -class TypeVarType(Type): - """A type variable type. - - This refers to either a class type variable (id > 0) or a function - type variable (id < 0). - """ - - name = '' # Name of the type variable (for messages and debugging) - id = None # type: TypeVarId - values = None # type: List[Type] # Value restriction, empty list if no restriction - upper_bound = None # type: Type # Upper bound for values - # See comments in TypeVarDef for more about variance. - variance = INVARIANT # type: int - - def __init__(self, binder: TypeVarDef, line: int = -1) -> None: - self.name = binder.name - self.id = binder.id - self.values = binder.values - self.upper_bound = binder.upper_bound - self.variance = binder.variance - super().__init__(line) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_type_var(self) - - def erase_to_union_or_bound(self) -> Type: - if self.values: - return UnionType.make_simplified_union(self.values) - else: - return self.upper_bound - - def serialize(self) -> JsonDict: - assert not self.id.is_meta_var() - return {'.class': 'TypeVarType', - 'name': self.name, - 'id': self.id.raw_id, - 'values': [v.serialize() for v in self.values], - 'upper_bound': self.upper_bound.serialize(), - 'variance': self.variance, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'TypeVarType': - assert data['.class'] == 'TypeVarType' - tvdef = TypeVarDef(data['name'], - data['id'], - [Type.deserialize(v) for v in data['values']], - Type.deserialize(data['upper_bound']), - data['variance']) - return TypeVarType(tvdef) - - -class FunctionLike(Type): - """Abstract base class for function types.""" - - @abstractmethod - def is_type_obj(self) -> bool: pass - - def is_concrete_type_obj(self) -> bool: - return self.is_type_obj() - - @abstractmethod - def type_object(self) -> mypy.nodes.TypeInfo: pass - - @abstractmethod - def items(self) -> List['CallableType']: pass - - @abstractmethod - def with_name(self, name: str) -> 'FunctionLike': pass - - # Corresponding instance type (e.g. builtins.type) - fallback = None # type: Instance - - @classmethod - def deserialize(cls, data: JsonDict) -> 'FunctionLike': - return cast(FunctionLike, super().deserialize(data)) - - -_dummy = object() # type: Any - - -class CallableType(FunctionLike): - """Type of a non-overloaded callable object (function).""" - - arg_types = None # type: List[Type] # Types of function arguments - arg_kinds = None # type: List[int] # mypy.nodes.ARG_ constants - arg_names = None # type: List[str] # None if not a keyword argument - min_args = 0 # Minimum number of arguments; derived from arg_kinds - is_var_arg = False # Is it a varargs function? derived from arg_kinds - ret_type = None # type: Type # Return value type - name = '' # Name (may be None; for error messages) - definition = None # type: SymbolNode # For error messages. May be None. - # Type variables for a generic function - variables = None # type: List[TypeVarDef] - - # Is this Callable[..., t] (with literal '...')? - is_ellipsis_args = False - # Is this callable constructed for the benefit of a classmethod's 'cls' argument? - is_classmethod_class = False - # Was this type implicitly generated instead of explicitly specified by the user? - implicit = False - # Defined for signatures that require special handling (currently only value is 'dict' - # for a signature similar to 'dict') - special_sig = None # type: Optional[str] - - def __init__(self, - arg_types: List[Type], - arg_kinds: List[int], - arg_names: List[str], - ret_type: Type, - fallback: Instance, - name: str = None, - definition: SymbolNode = None, - variables: List[TypeVarDef] = None, - line: int = -1, - is_ellipsis_args: bool = False, - implicit=False, - is_classmethod_class=False, - special_sig=None, - ) -> None: - if variables is None: - variables = [] - self.arg_types = arg_types - self.arg_kinds = arg_kinds - self.arg_names = arg_names - self.min_args = arg_kinds.count(mypy.nodes.ARG_POS) - self.is_var_arg = mypy.nodes.ARG_STAR in arg_kinds - self.ret_type = ret_type - self.fallback = fallback - assert not name or ' 'CallableType': - return CallableType( - arg_types=arg_types if arg_types is not _dummy else self.arg_types, - arg_kinds=arg_kinds if arg_kinds is not _dummy else self.arg_kinds, - arg_names=arg_names if arg_names is not _dummy else self.arg_names, - ret_type=ret_type if ret_type is not _dummy else self.ret_type, - fallback=fallback if fallback is not _dummy else self.fallback, - name=name if name is not _dummy else self.name, - definition=definition if definition is not _dummy else self.definition, - variables=variables if variables is not _dummy else self.variables, - line=line if line is not _dummy else self.line, - is_ellipsis_args=( - is_ellipsis_args if is_ellipsis_args is not _dummy else self.is_ellipsis_args), - implicit=self.implicit, - is_classmethod_class=self.is_classmethod_class, - special_sig=special_sig if special_sig is not _dummy else self.special_sig, - ) - - def is_type_obj(self) -> bool: - return self.fallback.type is not None and self.fallback.type.fullname() == 'builtins.type' - - def is_concrete_type_obj(self) -> bool: - return self.is_type_obj() and self.is_classmethod_class - - def type_object(self) -> mypy.nodes.TypeInfo: - assert self.is_type_obj() - ret = self.ret_type - if isinstance(ret, TupleType): - ret = ret.fallback - return cast(Instance, ret).type - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_callable_type(self) - - def with_name(self, name: str) -> 'CallableType': - """Return a copy of this type with the specified name.""" - ret = self.ret_type - if isinstance(ret, Void): - ret = ret.with_source(name) - return self.copy_modified(ret_type=ret, name=name) - - def max_fixed_args(self) -> int: - n = len(self.arg_types) - if self.is_var_arg: - n -= 1 - return n - - def items(self) -> List['CallableType']: - return [self] - - def is_generic(self) -> bool: - return bool(self.variables) - - def type_var_ids(self) -> List[TypeVarId]: - a = [] # type: List[TypeVarId] - for tv in self.variables: - a.append(tv.id) - return a - - def serialize(self) -> JsonDict: - # TODO: As an optimization, leave out everything related to - # generic functions for non-generic functions. - return {'.class': 'CallableType', - 'arg_types': [(None if t is None else t.serialize()) - for t in self.arg_types], - 'arg_kinds': self.arg_kinds, - 'arg_names': self.arg_names, - 'ret_type': self.ret_type.serialize(), - 'fallback': self.fallback.serialize(), - 'name': self.name, - # We don't serialize the definition (only used for error messages). - 'variables': [v.serialize() for v in self.variables], - 'is_ellipsis_args': self.is_ellipsis_args, - 'implicit': self.implicit, - 'is_classmethod_class': self.is_classmethod_class, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'CallableType': - assert data['.class'] == 'CallableType' - # TODO: Set definition to the containing SymbolNode? - return CallableType([(None if t is None else Type.deserialize(t)) - for t in data['arg_types']], - data['arg_kinds'], - data['arg_names'], - Type.deserialize(data['ret_type']), - Instance.deserialize(data['fallback']), - name=data['name'], - variables=[TypeVarDef.deserialize(v) for v in data['variables']], - is_ellipsis_args=data['is_ellipsis_args'], - implicit=data['implicit'], - is_classmethod_class=data['is_classmethod_class'], - ) - - -class Overloaded(FunctionLike): - """Overloaded function type T1, ... Tn, where each Ti is CallableType. - - The variant to call is chosen based on static argument - types. Overloaded function types can only be defined in stub - files, and thus there is no explicit runtime dispatch - implementation. - """ - - _items = None # type: List[CallableType] # Must not be empty - - def __init__(self, items: List[CallableType]) -> None: - self._items = items - self.fallback = items[0].fallback - super().__init__(items[0].line) - - def items(self) -> List[CallableType]: - return self._items - - def name(self) -> str: - return self._items[0].name - - def is_type_obj(self) -> bool: - # All the items must have the same type object status, so it's - # sufficient to query only (any) one of them. - return self._items[0].is_type_obj() - - def type_object(self) -> mypy.nodes.TypeInfo: - # All the items must have the same type object, so it's sufficient to - # query only (any) one of them. - return self._items[0].type_object() - - def with_name(self, name: str) -> 'Overloaded': - ni = [] # type: List[CallableType] - for it in self._items: - ni.append(it.with_name(name)) - return Overloaded(ni) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_overloaded(self) - - def serialize(self) -> JsonDict: - return {'.class': 'Overloaded', - 'items': [t.serialize() for t in self.items()], - } - - @classmethod - def deserialize(self, data: JsonDict) -> 'Overloaded': - assert data['.class'] == 'Overloaded' - return Overloaded([CallableType.deserialize(t) for t in data['items']]) - - -class TupleType(Type): - """The tuple type Tuple[T1, ..., Tn] (at least one type argument). - - Instance variables: - items: tuple item types - fallback: the underlying instance type that is used for non-tuple methods - (this is currently always builtins.tuple, but it could be different for named - tuples, for example) - implicit: if True, derived from a tuple expression (t,....) instead of Tuple[t, ...] - """ - - items = None # type: List[Type] - fallback = None # type: Instance - implicit = False - - def __init__(self, items: List[Type], fallback: Instance, line: int = -1, - implicit: bool = False) -> None: - #import pdb; pdb.set_trace() - self.items = items - self.fallback = fallback - self.implicit = implicit - super().__init__(line) - - def length(self) -> int: - return len(self.items) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_tuple_type(self) - - def serialize(self) -> JsonDict: - return {'.class': 'TupleType', - 'items': [t.serialize() for t in self.items], - 'fallback': self.fallback.serialize(), - 'implicit': self.implicit, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'TupleType': - assert data['.class'] == 'TupleType' - return TupleType([Type.deserialize(t) for t in data['items']], - Instance.deserialize(data['fallback']), - implicit=data['implicit']) - - def self_type(self, inst: Instance) -> 'TupleType': - return TupleType(self.items, inst, self.line, self.implicit) - - def with_types(self, items: List[Type]) -> 'TupleType': - return TupleType(items, self.fallback, self.line, self.implicit) - - def slice(self, begin: int, stride: int, end: int) -> 'TupleType': - return TupleType(self.items[begin:end:stride], self.fallback, - self.line, self.implicit) - - -class NamedTupleType(TupleType): - attrs = None # type: List[str] - name = None # type: str - - def __init__(self, name: str, attrs: List[str], *args, **kwargs) -> None: - self.attrs = attrs - self.name = name - super().__init__(*args, **kwargs) - - def self_type(self, inst: Instance) -> 'NamedTupleType': - return NamedTupleType(self.name, self.attrs, self.items, inst, self.line) - - def with_types(self, items: List[Type]) -> 'NamedTupleType': - return NamedTupleType(self.name, self.attrs, items, - self.fallback, self.line, self.implicit) - - def __eq__(self, other) -> bool: - return self.serialize() == other.serialize() - - def serialize(self) -> JsonDict: - d = super().serialize() - d.update({'.class': 'NamedTupleType', - 'name': self.name, - 'attrs': self.attrs}) - return d - - @classmethod - def deserialize(cls, data: JsonDict) -> 'NamedTupleType': - assert data['.class'] == 'NamedTupleType' - data['.class'] = 'TupleType' - tup = TupleType.deserialize(data) - return NamedTupleType(data['name'], - data['attrs'], - tup.items, - tup.fallback, - implicit=tup.implicit) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - try: - return visitor.visit_namedtuple_type(self) - except: - return super().accept(visitor) - - -class StarType(Type): - """The star type *type_parameter. - - This is not a real type but a syntactic AST construct. - """ - - type = None # type: Type - - def __init__(self, type: Type, line: int = -1) -> None: - self.type = type - super().__init__(line) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_star_type(self) - - -class UnionType(Type): - """The union type Union[T1, ..., Tn] (at least one type argument).""" - - items = None # type: List[Type] - - def __init__(self, items: List[Type], line: int = -1) -> None: - self.items = items - super().__init__(line) - - @staticmethod - def make_union(items: List[Type], line: int = -1) -> Type: - if len(items) > 1: - return UnionType(items, line) - elif len(items) == 1: - return items[0] - else: - if experiments.STRICT_OPTIONAL: - return UninhabitedType() - else: - return Void() - - @staticmethod - def make_simplified_union(items: List[Type], line: int = -1) -> Type: - while any(isinstance(typ, UnionType) for typ in items): - all_items = [] # type: List[Type] - for typ in items: - if isinstance(typ, UnionType): - all_items.extend(typ.items) - else: - all_items.append(typ) - items = all_items - - if any(isinstance(typ, AnyType) for typ in items): - return AnyType() - - from mypy.subtypes import is_subtype - removed = set() # type: Set[int] - for i in range(len(items)): - if any(is_subtype(items[i], items[j]) for j in range(len(items)) - if j not in removed and j != i): - removed.add(i) - - simplified_set = [items[i] for i in range(len(items)) if i not in removed] - return UnionType.make_union(simplified_set) - - def length(self) -> int: - return len(self.items) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_union_type(self) - - def has_readable_member(self, name: str) -> bool: - """For a tree of unions of instances, check whether all instances have a given member. - - TODO: Deal with attributes of TupleType etc. - TODO: This should probably be refactored to go elsewhere. - """ - return all((isinstance(x, UnionType) and x.has_readable_member(name)) or - (isinstance(x, Instance) and x.type.has_readable_member(name)) - for x in self.items) - - def serialize(self) -> JsonDict: - return {'.class': 'UnionType', - 'items': [t.serialize() for t in self.items], - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'UnionType': - assert data['.class'] == 'UnionType' - return UnionType([Type.deserialize(t) for t in data['items']]) - - -class PartialType(Type): - """Type such as List[?] where type arguments are unknown, or partial None type. - - These are used for inferring types in multiphase initialization such as this: - - x = [] # x gets a partial type List[?], as item type is unknown - x.append(1) # partial type gets replaced with normal type List[int] - - Or with None: - - x = None # x gets a partial type None - if c: - x = 1 # Infer actual type int for x - """ - - # None for the 'None' partial type; otherwise a generic class - type = None # type: Optional[mypy.nodes.TypeInfo] - var = None # type: mypy.nodes.Var - inner_types = None # type: List[Type] - - def __init__(self, - type: Optional['mypy.nodes.TypeInfo'], - var: 'mypy.nodes.Var', - inner_types: List[Type]) -> None: - self.type = type - self.var = var - self.inner_types = inner_types - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_partial_type(self) - - -class EllipsisType(Type): - """The type ... (ellipsis). - - This is not a real type but a syntactic AST construct, used in Callable[..., T], for example. - - A semantically analyzed type will never have ellipsis types. - """ - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_ellipsis_type(self) - - def serialize(self) -> JsonDict: - return {'.class': 'EllipsisType'} - - @classmethod - def deserialize(self, data: JsonDict) -> 'EllipsisType': - assert data['.class'] == 'EllipsisType' - return EllipsisType() - - -class TypeType(Type): - """For types like Type[User]. - - This annotates variables that are class objects, constrained by - the type argument. See PEP 484 for more details. - - We may encounter expressions whose values are specific classes; - those are represented as callables (possibly overloaded) - corresponding to the class's constructor's signature and returning - an instance of that class. The difference with Type[C] is that - those callables always represent the exact class given as the - return type; Type[C] represents any class that's a subclass of C, - and C may also be a type variable or a union (or Any). - - Many questions around subtype relationships between Type[C1] and - def(...) -> C2 are answered by looking at the subtype - relationships between C1 and C2, since Type[] is considered - covariant. - - There's an unsolved problem with constructor signatures (also - unsolved in PEP 484): calling a variable whose type is Type[C] - assumes the constructor signature for C, even though a subclass of - C might completely change the constructor signature. For now we - just assume that users of Type[C] are careful not to do that (in - the future we might detect when they are violating that - assumption). - """ - - # This can't be everything, but it can be a class reference, - # a generic class instance, a union, Any, a type variable... - item = None # type: Type - - def __init__(self, item: Type, *, line: int = -1) -> None: - super().__init__(line) - self.item = item - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_type_type(self) - - def serialize(self) -> JsonDict: - return {'.class': 'TypeType', 'item': self.item.serialize()} - - def deserialize(self, data: JsonDict) -> 'TypeType': - assert data['.class'] == 'TypeType' - return TypeType(Type.deserialize(data['item'])) - - -# -# Visitor-related classes -# - - -class TypeVisitor(Generic[T]): - """Visitor class for types (Type subclasses). - - The parameter T is the return type of the visit methods. - """ - - def _notimplemented_helper(self, name: str) -> NotImplementedError: - return NotImplementedError("Method {}.visit_{}() not implemented\n" - .format(type(self).__name__, name) - + "This is a known bug, track development in " - + "'https://github.com/JukkaL/mypy/issues/730'") - - @abstractmethod - def visit_unbound_type(self, t: UnboundType) -> T: - pass - - def visit_type_list(self, t: TypeList) -> T: - raise self._notimplemented_helper('type_list') - - def visit_error_type(self, t: ErrorType) -> T: - raise self._notimplemented_helper('error_type') - - @abstractmethod - def visit_any(self, t: AnyType) -> T: - pass - - @abstractmethod - def visit_void(self, t: Void) -> T: - pass - - @abstractmethod - def visit_none_type(self, t: NoneTyp) -> T: - pass - - @abstractmethod - def visit_uninhabited_type(self, t: UninhabitedType) -> T: - pass - - def visit_erased_type(self, t: ErasedType) -> T: - raise self._notimplemented_helper('erased_type') - - @abstractmethod - def visit_deleted_type(self, t: DeletedType) -> T: - pass - - @abstractmethod - def visit_type_var(self, t: TypeVarType) -> T: - pass - - @abstractmethod - def visit_instance(self, t: Instance) -> T: - pass - - @abstractmethod - def visit_callable_type(self, t: CallableType) -> T: - pass - - def visit_overloaded(self, t: Overloaded) -> T: - raise self._notimplemented_helper('overloaded') - - @abstractmethod - def visit_tuple_type(self, t: TupleType) -> T: - pass - - def visit_star_type(self, t: StarType) -> T: - raise self._notimplemented_helper('star_type') - - @abstractmethod - def visit_union_type(self, t: UnionType) -> T: - pass - - @abstractmethod - def visit_partial_type(self, t: PartialType) -> T: - pass - - def visit_ellipsis_type(self, t: EllipsisType) -> T: - raise self._notimplemented_helper('ellipsis_type') - - @abstractmethod - def visit_type_type(self, t: TypeType) -> T: - pass - - -class TypeTranslator(TypeVisitor[Type]): - """Identity type transformation. - - Subclass this and override some methods to implement a non-trivial - transformation. - """ - - def visit_unbound_type(self, t: UnboundType) -> Type: - return t - - def visit_type_list(self, t: TypeList) -> Type: - return t - - def visit_error_type(self, t: ErrorType) -> Type: - return t - - def visit_any(self, t: AnyType) -> Type: - return t - - def visit_void(self, t: Void) -> Type: - return t - - def visit_none_type(self, t: NoneTyp) -> Type: - return t - - def visit_uninhabited_type(self, t: UninhabitedType) -> Type: - return t - - def visit_erased_type(self, t: ErasedType) -> Type: - return t - - def visit_deleted_type(self, t: DeletedType) -> Type: - return t - - def visit_instance(self, t: Instance) -> Type: - return Instance(t.type, self.translate_types(t.args), t.line) - - def visit_type_var(self, t: TypeVarType) -> Type: - return t - - def visit_partial_type(self, t: PartialType) -> Type: - return t - - def visit_callable_type(self, t: CallableType) -> Type: - return t.copy_modified(arg_types=self.translate_types(t.arg_types), - ret_type=t.ret_type.accept(self), - variables=self.translate_variables(t.variables)) - - def visit_tuple_type(self, t: TupleType) -> Type: - return TupleType(self.translate_types(t.items), - cast(Any, t.fallback.accept(self)), - t.line) - - def visit_star_type(self, t: StarType) -> Type: - return StarType(t.type.accept(self), t.line) - - def visit_union_type(self, t: UnionType) -> Type: - return UnionType(self.translate_types(t.items), t.line) - - def visit_ellipsis_type(self, t: EllipsisType) -> Type: - return t - - def translate_types(self, types: List[Type]) -> List[Type]: - return [t.accept(self) for t in types] - - def translate_variables(self, - variables: List[TypeVarDef]) -> List[TypeVarDef]: - return variables - - def visit_overloaded(self, t: Overloaded) -> Type: - items = [] # type: List[CallableType] - for item in t.items(): - new = item.accept(self) - if isinstance(new, CallableType): - items.append(new) - else: - raise RuntimeError('CallableType expectected, but got {}'.format(type(new))) - return Overloaded(items=items) - - def visit_type_type(self, t: TypeType) -> Type: - return TypeType(t.item.accept(self), line=t.line) - - -class TypeStrVisitor(TypeVisitor[str]): - """Visitor for pretty-printing types into strings. - - This is mostly for debugging/testing. - - Do not preserve original formatting. - - Notes: - - Represent unbound types as Foo? or Foo?[...]. - - Represent the NoneTyp type as None. - """ - - def visit_unbound_type(self, t): - s = t.name + '?' - if t.args != []: - s += '[{}]'.format(self.list_str(t.args)) - return s - - def visit_type_list(self, t): - return ''.format(self.list_str(t.items)) - - def visit_error_type(self, t): - return '' - - def visit_any(self, t): - return 'Any' - - def visit_void(self, t): - return 'void' - - def visit_none_type(self, t): - # Fully qualify to make this distinct from the None value. - return "builtins.None" - - def visit_uninhabited_type(self, t): - return "" - - def visit_erased_type(self, t): - return "" - - def visit_deleted_type(self, t): - if t.source is None: - return "" - else: - return "".format(t.source) - - def visit_instance(self, t): - s = t.type.fullname() if t.type is not None else '' - if t.erased: - s += '*' - if t.args != []: - s += '[{}]'.format(self.list_str(t.args)) - return s - - def visit_type_var(self, t): - if t.name is None: - # Anonymous type variable type (only numeric id). - return '`{}'.format(t.id) - else: - # Named type variable type. - return '{}`{}'.format(t.name, t.id) - - def visit_callable_type(self, t): - s = '' - bare_asterisk = False - for i in range(len(t.arg_types)): - if s != '': - s += ', ' - if t.arg_kinds[i] == mypy.nodes.ARG_NAMED and not bare_asterisk: - s += '*, ' - bare_asterisk = True - if t.arg_kinds[i] == mypy.nodes.ARG_STAR: - s += '*' - if t.arg_kinds[i] == mypy.nodes.ARG_STAR2: - s += '**' - if t.arg_names[i]: - s += t.arg_names[i] + ': ' - s += str(t.arg_types[i]) - if t.arg_kinds[i] == mypy.nodes.ARG_OPT: - s += ' =' - - s = '({})'.format(s) - - if not isinstance(t.ret_type, Void): - s += ' -> {}'.format(t.ret_type) - - if t.variables: - s = '{} {}'.format(t.variables, s) - - return 'def {}'.format(s) - - def visit_overloaded(self, t): - a = [] - for i in t.items(): - a.append(i.accept(self)) - return 'Overload({})'.format(', '.join(a)) - - def visit_tuple_type(self, t): - s = self.list_str(t.items) - if t.fallback and t.fallback.type: - fallback_name = t.fallback.type.fullname() - if fallback_name != 'builtins.tuple': - return 'Tuple[{}, fallback={}]'.format(s, t.fallback.accept(self)) - return 'Tuple[{}]'.format(s) - - def visit_namedtuple_type(self, t): - s = self.list_str(zip(t.attrs, t.items)) - return '{}[{}]'.format(t.name, s) - - def visit_star_type(self, t): - s = t.type.accept(self) - return '*{}'.format(s) - - def visit_union_type(self, t): - s = self.list_str(t.items) - return 'Union[{}]'.format(s) - - def visit_partial_type(self, t: PartialType) -> str: - if t.type is None: - return '' - else: - return ''.format(t.type.name(), - ', '.join(['?'] * len(t.type.type_vars))) - - def visit_ellipsis_type(self, t): - return '...' - - def visit_type_type(self, t): - return 'Type[{}]'.format(t.item.accept(self)) - - def list_str(self, a): - """Convert items of an array to strings (pretty-print types) - and join the results with commas. - """ - res = [] - for t in a: - if isinstance(t, Type): - res.append(t.accept(self)) - else: - res.append(str(t)) - return ', '.join(res) - - -# These constants define the method used by TypeQuery to combine multiple -# query results, e.g. for tuple types. The strategy is not used for empty -# result lists; in that case the default value takes precedence. -ANY_TYPE_STRATEGY = 0 # Return True if any of the results are True. -ALL_TYPES_STRATEGY = 1 # Return True if all of the results are True. - - -class TypeQuery(TypeVisitor[bool]): - """Visitor for performing simple boolean queries of types. - - This class allows defining the default value for leafs to simplify the - implementation of many queries. - """ - - default = False # Default result - strategy = 0 # Strategy for combining multiple values (ANY_TYPE_STRATEGY or ALL_TYPES_...). - - def __init__(self, default: bool, strategy: int) -> None: - """Construct a query visitor. - - Use the given default result and strategy for combining - multiple results. The strategy must be either - ANY_TYPE_STRATEGY or ALL_TYPES_STRATEGY. - """ - self.default = default - self.strategy = strategy - - def visit_unbound_type(self, t: UnboundType) -> bool: - return self.default - - def visit_type_list(self, t: TypeList) -> bool: - return self.default - - def visit_error_type(self, t: ErrorType) -> bool: - return self.default - - def visit_any(self, t: AnyType) -> bool: - return self.default - - def visit_void(self, t: Void) -> bool: - return self.default - - def visit_uninhabited_type(self, t: UninhabitedType) -> bool: - return self.default - - def visit_none_type(self, t: NoneTyp) -> bool: - return self.default - - def visit_erased_type(self, t: ErasedType) -> bool: - return self.default - - def visit_deleted_type(self, t: DeletedType) -> bool: - return self.default - - def visit_type_var(self, t: TypeVarType) -> bool: - return self.default - - def visit_partial_type(self, t: PartialType) -> bool: - return self.default - - def visit_instance(self, t: Instance) -> bool: - return self.query_types(t.args) - - def visit_callable_type(self, t: CallableType) -> bool: - # FIX generics - return self.query_types(t.arg_types + [t.ret_type]) - - def visit_tuple_type(self, t: TupleType) -> bool: - return self.query_types(t.items) - - def visit_star_type(self, t: StarType) -> bool: - return t.type.accept(self) - - def visit_union_type(self, t: UnionType) -> bool: - return self.query_types(t.items) - - def visit_overloaded(self, t: Overloaded) -> bool: - return self.query_types(t.items()) - - def visit_type_type(self, t: TypeType) -> bool: - return t.item.accept(self) - - def query_types(self, types: Sequence[Type]) -> bool: - """Perform a query for a list of types. - - Use the strategy constant to combine the results. - """ - if not types: - # Use default result for empty list. - return self.default - if self.strategy == ANY_TYPE_STRATEGY: - # Return True if at least one component is true. - res = False - for t in types: - res = res or t.accept(self) - if res: - break - return res - else: - # Return True if all components are true. - res = True - for t in types: - res = res and t.accept(self) - if not res: - break - return res - - -def strip_type(typ: Type) -> Type: - """Make a copy of type without 'debugging info' (function name).""" - - if isinstance(typ, CallableType): - return typ.copy_modified(name=None) - elif isinstance(typ, Overloaded): - return Overloaded([cast(CallableType, strip_type(item)) - for item in typ.items()]) - else: - return typ - - -def replace_leading_arg_type(t: CallableType, self_type: Type) -> CallableType: - """Return a copy of a callable type with a different self argument type. - - Assume that the callable is the signature of a method. - """ - return t.copy_modified(arg_types=[self_type] + t.arg_types[1:]) - - -def is_named_instance(t: Type, fullname: str) -> bool: - return (isinstance(t, Instance) and - t.type is not None and - t.type.fullname() == fullname) +"""Classes for representing mypy types.""" + +from abc import abstractmethod +from typing import ( + Any, TypeVar, Dict, List, Tuple, cast, Generic, Set, Sequence, Optional, Union +) + +import mypy.nodes +from mypy.nodes import INVARIANT, SymbolNode + +from mypy import experiments + + +T = TypeVar('T') + +JsonDict = Dict[str, Any] + + +class Type(mypy.nodes.Context): + """Abstract base class for all types.""" + + line = 0 + + def __init__(self, line: int = -1) -> None: + self.line = line + + def get_line(self) -> int: + return self.line + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + raise RuntimeError('Not implemented') + + def __repr__(self) -> str: + return self.accept(TypeStrVisitor()) + + def serialize(self) -> JsonDict: + raise NotImplementedError('Cannot serialize {} instance'.format(self.__class__.__name__)) + + @classmethod + def deserialize(cls, data: JsonDict) -> 'Type': + classname = data['.class'] + glo = globals() + if classname in glo: + cl = glo[classname] + if 'deserialize' in cl.__dict__: + return cl.deserialize(data) + raise NotImplementedError('unexpected .class {}'.format(classname)) + + +class TypeVarId: + # A type variable is uniquely identified by its raw id and meta level. + + # For plain variables (type parameters of generic classes and + # functions) raw ids are allocated by semantic analysis, using + # positive ids 1, 2, ... for generic class parameters and negative + # ids -1, ... for generic function type arguments. This convention + # is only used to keep type variable ids distinct when allocating + # them; the type checker makes no distinction between class and + # function type variables. + + # Metavariables are allocated unique ids starting from 1. + raw_id = 0 # type: int + + # Level of the variable in type inference. Currently either 0 for + # declared types, or 1 for type inference metavariables. + meta_level = 0 # type: int + + # Class variable used for allocating fresh ids for metavariables. + next_raw_id = 1 # type: int + + def __init__(self, raw_id: int, meta_level: int = 0) -> None: + self.raw_id = raw_id + self.meta_level = meta_level + + @staticmethod + def new(meta_level: int) -> 'TypeVarId': + raw_id = TypeVarId.next_raw_id + TypeVarId.next_raw_id += 1 + return TypeVarId(raw_id, meta_level) + + def __repr__(self) -> str: + return self.raw_id.__repr__() + + def __eq__(self, other: object) -> bool: + if isinstance(other, TypeVarId): + return (self.raw_id == other.raw_id and + self.meta_level == other.meta_level) + else: + return False + + def __ne__(self, other: object) -> bool: + return not (self == other) + + def __hash__(self) -> int: + return hash((self.raw_id, self.meta_level)) + + def is_meta_var(self) -> bool: + return self.meta_level > 0 + + +class TypeVarDef(mypy.nodes.Context): + """Definition of a single type variable.""" + + name = '' + id = None # type: TypeVarId + values = None # type: List[Type] # Value restriction, empty list if no restriction + upper_bound = None # type: Type + variance = INVARIANT # type: int + line = 0 + + def __init__(self, name: str, id: Union[TypeVarId, int], values: Optional[List[Type]], + upper_bound: Type, variance: int = INVARIANT, line: int = -1) -> None: + self.name = name + if isinstance(id, int): + id = TypeVarId(id) + self.id = id + self.values = values + self.upper_bound = upper_bound + self.variance = variance + self.line = line + + @staticmethod + def new_unification_variable(old: 'TypeVarDef') -> 'TypeVarDef': + new_id = TypeVarId.new(meta_level=1) + return TypeVarDef(old.name, new_id, old.values, + old.upper_bound, old.variance, old.line) + + def get_line(self) -> int: + return self.line + + def __repr__(self) -> str: + if self.values: + return '{} in {}'.format(self.name, tuple(self.values)) + elif not is_named_instance(self.upper_bound, 'builtins.object'): + return '{} <: {}'.format(self.name, self.upper_bound) + else: + return self.name + + def serialize(self) -> JsonDict: + assert not self.id.is_meta_var() + return {'.class': 'TypeVarDef', + 'name': self.name, + 'id': self.id.raw_id, + 'values': None if self.values is None else [v.serialize() for v in self.values], + 'upper_bound': self.upper_bound.serialize(), + 'variance': self.variance, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'TypeVarDef': + assert data['.class'] == 'TypeVarDef' + return TypeVarDef(data['name'], + data['id'], + None if data['values'] is None + else [Type.deserialize(v) for v in data['values']], + Type.deserialize(data['upper_bound']), + data['variance'], + ) + + +class UnboundType(Type): + """Instance type that has not been bound during semantic analysis.""" + + name = '' + args = None # type: List[Type] + # should this type be wrapped in an Optional? + optional = False + # is this type a return type? + is_ret_type = False + + def __init__(self, + name: str, + args: List[Type] = None, + line: int = -1, + optional: bool = False, + is_ret_type: bool = False) -> None: + if not args: + args = [] + self.name = name + self.args = args + self.optional = optional + self.is_ret_type = is_ret_type + super().__init__(line) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_unbound_type(self) + + def serialize(self) -> JsonDict: + return {'.class': 'UnboundType', + 'name': self.name, + 'args': [a.serialize() for a in self.args], + } + + @classmethod + def deserialize(self, data: JsonDict) -> 'UnboundType': + assert data['.class'] == 'UnboundType' + return UnboundType(data['name'], + [Type.deserialize(a) for a in data['args']]) + + +class ErrorType(Type): + """The error type is used as the result of failed type operations.""" + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_error_type(self) + + +class TypeList(Type): + """A list of types [...]. + + This is only used for the arguments of a Callable type, i.e. for + [arg, ...] in Callable[[arg, ...], ret]. This is not a real type + but a syntactic AST construct. + """ + + items = None # type: List[Type] + + def __init__(self, items: List[Type], line: int = -1) -> None: + super().__init__(line) + self.items = items + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_type_list(self) + + def serialize(self) -> JsonDict: + return {'.class': 'TypeList', + 'items': [t.serialize() for t in self.items], + } + + @classmethod + def deserialize(self, data: JsonDict) -> 'TypeList': + assert data['.class'] == 'TypeList' + return TypeList([Type.deserialize(t) for t in data['items']]) + + +class AnyType(Type): + """The type 'Any'.""" + + def __init__(self, implicit=False, line: int = -1) -> None: + super().__init__(line) + self.implicit = implicit + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_any(self) + + def serialize(self) -> JsonDict: + return {'.class': 'AnyType'} + + @classmethod + def deserialize(cls, data: JsonDict) -> 'AnyType': + assert data['.class'] == 'AnyType' + return AnyType() + + +class Void(Type): + """The return type 'None'. + + This can only be used as the return type in a callable type and as + the result type of calling such callable. + """ + + source = '' # May be None; function that generated this value + + def __init__(self, source: str = None, line: int = -1) -> None: + self.source = source + super().__init__(line) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_void(self) + + def with_source(self, source: str) -> 'Void': + return Void(source, self.line) + + def serialize(self) -> JsonDict: + return {'.class': 'Void'} + + @classmethod + def deserialize(cls, data: JsonDict) -> 'Void': + assert data['.class'] == 'Void' + return Void() + + +class UninhabitedType(Type): + """This type has no members. + + This type is almost the bottom type, except it is not a subtype of Void. + With strict Optional checking, it is the only common subtype between all + other types, which allows `meet` to be well defined. Without strict + Optional checking, NoneTyp fills this role. + + In general, for any type T that isn't Void: + join(UninhabitedType, T) = T + meet(UninhabitedType, T) = UninhabitedType + is_subtype(UninhabitedType, T) = True + """ + + def __init__(self, line: int = -1) -> None: + super().__init__(line) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_uninhabited_type(self) + + def serialize(self) -> JsonDict: + return {'.class': 'UninhabitedType'} + + @classmethod + def deserialize(cls, data: JsonDict) -> 'UninhabitedType': + assert data['.class'] == 'UninhabitedType' + return UninhabitedType() + + +class NoneTyp(Type): + """The type of 'None'. + + Without strict Optional checking: + This is only used internally during type inference. Programs + cannot declare a variable of this type, and the type checker + refuses to infer this type for a variable. However, subexpressions + often have this type. Note that this is not used as the result + type when calling a function with a void type, even though + semantically such a function returns a None value; the void type + is used instead so that we can report an error if the caller tries + to do anything with the return value. + + With strict Optional checking: + This type can be written by users as 'None', except as the return value + of a function, where 'None' means Void. + """ + + def __init__(self, line: int = -1) -> None: + super().__init__(line) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_none_type(self) + + def serialize(self) -> JsonDict: + return {'.class': 'NoneTyp'} + + @classmethod + def deserialize(self, data: JsonDict) -> 'NoneTyp': + assert data['.class'] == 'NoneTyp' + return NoneTyp() + + +class ErasedType(Type): + """Placeholder for an erased type. + + This is used during type inference. This has the special property that + it is ignored during type inference. + """ + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_erased_type(self) + + +class DeletedType(Type): + """Type of deleted variables. + + These can be used as lvalues but not rvalues. + """ + + source = '' # May be None; name that generated this value + + def __init__(self, source: str = None, line: int = -1) -> None: + self.source = source + super().__init__(line) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_deleted_type(self) + + def serialize(self) -> JsonDict: + return {'.class': 'DeletedType', + 'source': self.source} + + @classmethod + def deserialize(self, data: JsonDict) -> 'DeletedType': + assert data['.class'] == 'DeletedType' + return DeletedType(data['source']) + + +class Instance(Type): + """An instance type of form C[T1, ..., Tn]. + + The list of type variables may be empty. + """ + + type = None # type: mypy.nodes.TypeInfo + args = None # type: List[Type] + erased = False # True if result of type variable substitution + + def __init__(self, typ: mypy.nodes.TypeInfo, args: List[Type], + line: int = -1, erased: bool = False) -> None: + self.type = typ + self.args = args + self.erased = erased + super().__init__(line) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_instance(self) + + type_ref = None # type: str + + def serialize(self) -> JsonDict: + data = {'.class': 'Instance', + } # type: JsonDict + assert self.type is not None + data['type_ref'] = self.type.alt_fullname or self.type.fullname() + if self.args: + data['args'] = [arg.serialize() for arg in self.args] + return data + + @classmethod + def deserialize(cls, data: JsonDict) -> 'Instance': + assert data['.class'] == 'Instance' + args = [] # type: List[Type] + if 'args' in data: + args_list = data['args'] + assert isinstance(args_list, list) + args = [Type.deserialize(arg) for arg in args_list] + inst = Instance(None, args) + inst.type_ref = data['type_ref'] # Will be fixed up by fixup.py later. + return inst + + +class TypeVarType(Type): + """A type variable type. + + This refers to either a class type variable (id > 0) or a function + type variable (id < 0). + """ + + name = '' # Name of the type variable (for messages and debugging) + id = None # type: TypeVarId + values = None # type: List[Type] # Value restriction, empty list if no restriction + upper_bound = None # type: Type # Upper bound for values + # See comments in TypeVarDef for more about variance. + variance = INVARIANT # type: int + + def __init__(self, binder: TypeVarDef, line: int = -1) -> None: + self.name = binder.name + self.id = binder.id + self.values = binder.values + self.upper_bound = binder.upper_bound + self.variance = binder.variance + super().__init__(line) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_type_var(self) + + def erase_to_union_or_bound(self) -> Type: + if self.values: + return UnionType.make_simplified_union(self.values) + else: + return self.upper_bound + + def serialize(self) -> JsonDict: + assert not self.id.is_meta_var() + return {'.class': 'TypeVarType', + 'name': self.name, + 'id': self.id.raw_id, + 'values': [v.serialize() for v in self.values], + 'upper_bound': self.upper_bound.serialize(), + 'variance': self.variance, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'TypeVarType': + assert data['.class'] == 'TypeVarType' + tvdef = TypeVarDef(data['name'], + data['id'], + [Type.deserialize(v) for v in data['values']], + Type.deserialize(data['upper_bound']), + data['variance']) + return TypeVarType(tvdef) + + +class FunctionLike(Type): + """Abstract base class for function types.""" + + @abstractmethod + def is_type_obj(self) -> bool: pass + + def is_concrete_type_obj(self) -> bool: + return self.is_type_obj() + + @abstractmethod + def type_object(self) -> mypy.nodes.TypeInfo: pass + + @abstractmethod + def items(self) -> List['CallableType']: pass + + @abstractmethod + def with_name(self, name: str) -> 'FunctionLike': pass + + # Corresponding instance type (e.g. builtins.type) + fallback = None # type: Instance + + @classmethod + def deserialize(cls, data: JsonDict) -> 'FunctionLike': + return cast(FunctionLike, super().deserialize(data)) + + +_dummy = object() # type: Any + + +class CallableType(FunctionLike): + """Type of a non-overloaded callable object (function).""" + + arg_types = None # type: List[Type] # Types of function arguments + arg_kinds = None # type: List[int] # mypy.nodes.ARG_ constants + arg_names = None # type: List[str] # None if not a keyword argument + min_args = 0 # Minimum number of arguments; derived from arg_kinds + is_var_arg = False # Is it a varargs function? derived from arg_kinds + ret_type = None # type: Type # Return value type + name = '' # Name (may be None; for error messages) + definition = None # type: SymbolNode # For error messages. May be None. + # Type variables for a generic function + variables = None # type: List[TypeVarDef] + + # Is this Callable[..., t] (with literal '...')? + is_ellipsis_args = False + # Is this callable constructed for the benefit of a classmethod's 'cls' argument? + is_classmethod_class = False + # Was this type implicitly generated instead of explicitly specified by the user? + implicit = False + # Defined for signatures that require special handling (currently only value is 'dict' + # for a signature similar to 'dict') + special_sig = None # type: Optional[str] + + def __init__(self, + arg_types: List[Type], + arg_kinds: List[int], + arg_names: List[str], + ret_type: Type, + fallback: Instance, + name: str = None, + definition: SymbolNode = None, + variables: List[TypeVarDef] = None, + line: int = -1, + is_ellipsis_args: bool = False, + implicit=False, + is_classmethod_class=False, + special_sig=None, + ) -> None: + if variables is None: + variables = [] + self.arg_types = arg_types + self.arg_kinds = arg_kinds + self.arg_names = arg_names + self.min_args = arg_kinds.count(mypy.nodes.ARG_POS) + self.is_var_arg = mypy.nodes.ARG_STAR in arg_kinds + self.ret_type = ret_type + self.fallback = fallback + assert not name or ' 'CallableType': + return CallableType( + arg_types=arg_types if arg_types is not _dummy else self.arg_types, + arg_kinds=arg_kinds if arg_kinds is not _dummy else self.arg_kinds, + arg_names=arg_names if arg_names is not _dummy else self.arg_names, + ret_type=ret_type if ret_type is not _dummy else self.ret_type, + fallback=fallback if fallback is not _dummy else self.fallback, + name=name if name is not _dummy else self.name, + definition=definition if definition is not _dummy else self.definition, + variables=variables if variables is not _dummy else self.variables, + line=line if line is not _dummy else self.line, + is_ellipsis_args=( + is_ellipsis_args if is_ellipsis_args is not _dummy else self.is_ellipsis_args), + implicit=self.implicit, + is_classmethod_class=self.is_classmethod_class, + special_sig=special_sig if special_sig is not _dummy else self.special_sig, + ) + + def is_type_obj(self) -> bool: + return self.fallback.type is not None and self.fallback.type.fullname() == 'builtins.type' + + def is_concrete_type_obj(self) -> bool: + return self.is_type_obj() and self.is_classmethod_class + + def type_object(self) -> mypy.nodes.TypeInfo: + assert self.is_type_obj() + ret = self.ret_type + if isinstance(ret, TupleType): + ret = ret.fallback + return cast(Instance, ret).type + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_callable_type(self) + + def with_name(self, name: str) -> 'CallableType': + """Return a copy of this type with the specified name.""" + ret = self.ret_type + if isinstance(ret, Void): + ret = ret.with_source(name) + return self.copy_modified(ret_type=ret, name=name) + + def max_fixed_args(self) -> int: + n = len(self.arg_types) + if self.is_var_arg: + n -= 1 + return n + + def items(self) -> List['CallableType']: + return [self] + + def is_generic(self) -> bool: + return bool(self.variables) + + def type_var_ids(self) -> List[TypeVarId]: + a = [] # type: List[TypeVarId] + for tv in self.variables: + a.append(tv.id) + return a + + def serialize(self) -> JsonDict: + # TODO: As an optimization, leave out everything related to + # generic functions for non-generic functions. + return {'.class': 'CallableType', + 'arg_types': [(None if t is None else t.serialize()) + for t in self.arg_types], + 'arg_kinds': self.arg_kinds, + 'arg_names': self.arg_names, + 'ret_type': self.ret_type.serialize(), + 'fallback': self.fallback.serialize(), + 'name': self.name, + # We don't serialize the definition (only used for error messages). + 'variables': [v.serialize() for v in self.variables], + 'is_ellipsis_args': self.is_ellipsis_args, + 'implicit': self.implicit, + 'is_classmethod_class': self.is_classmethod_class, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'CallableType': + assert data['.class'] == 'CallableType' + # TODO: Set definition to the containing SymbolNode? + return CallableType([(None if t is None else Type.deserialize(t)) + for t in data['arg_types']], + data['arg_kinds'], + data['arg_names'], + Type.deserialize(data['ret_type']), + Instance.deserialize(data['fallback']), + name=data['name'], + variables=[TypeVarDef.deserialize(v) for v in data['variables']], + is_ellipsis_args=data['is_ellipsis_args'], + implicit=data['implicit'], + is_classmethod_class=data['is_classmethod_class'], + ) + + +class Overloaded(FunctionLike): + """Overloaded function type T1, ... Tn, where each Ti is CallableType. + + The variant to call is chosen based on static argument + types. Overloaded function types can only be defined in stub + files, and thus there is no explicit runtime dispatch + implementation. + """ + + _items = None # type: List[CallableType] # Must not be empty + + def __init__(self, items: List[CallableType]) -> None: + self._items = items + self.fallback = items[0].fallback + super().__init__(items[0].line) + + def items(self) -> List[CallableType]: + return self._items + + def name(self) -> str: + return self._items[0].name + + def is_type_obj(self) -> bool: + # All the items must have the same type object status, so it's + # sufficient to query only (any) one of them. + return self._items[0].is_type_obj() + + def type_object(self) -> mypy.nodes.TypeInfo: + # All the items must have the same type object, so it's sufficient to + # query only (any) one of them. + return self._items[0].type_object() + + def with_name(self, name: str) -> 'Overloaded': + ni = [] # type: List[CallableType] + for it in self._items: + ni.append(it.with_name(name)) + return Overloaded(ni) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_overloaded(self) + + def serialize(self) -> JsonDict: + return {'.class': 'Overloaded', + 'items': [t.serialize() for t in self.items()], + } + + @classmethod + def deserialize(self, data: JsonDict) -> 'Overloaded': + assert data['.class'] == 'Overloaded' + return Overloaded([CallableType.deserialize(t) for t in data['items']]) + + +class TupleType(Type): + """The tuple type Tuple[T1, ..., Tn] (at least one type argument). + + Instance variables: + items: tuple item types + fallback: the underlying instance type that is used for non-tuple methods + (this is currently always builtins.tuple, but it could be different for named + tuples, for example) + implicit: if True, derived from a tuple expression (t,....) instead of Tuple[t, ...] + """ + + items = None # type: List[Type] + fallback = None # type: Instance + implicit = False + + def __init__(self, items: List[Type], fallback: Instance, line: int = -1, + implicit: bool = False) -> None: + #import pdb; pdb.set_trace() + self.items = items + self.fallback = fallback + self.implicit = implicit + super().__init__(line) + + def length(self) -> int: + return len(self.items) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_tuple_type(self) + + def serialize(self) -> JsonDict: + return {'.class': 'TupleType', + 'items': [t.serialize() for t in self.items], + 'fallback': self.fallback.serialize(), + 'implicit': self.implicit, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'TupleType': + assert data['.class'] == 'TupleType' + return TupleType([Type.deserialize(t) for t in data['items']], + Instance.deserialize(data['fallback']), + implicit=data['implicit']) + + def self_type(self, inst: Instance) -> 'TupleType': + return TupleType(self.items, inst, self.line, self.implicit) + + def with_types(self, items: List[Type]) -> 'TupleType': + return TupleType(items, self.fallback, self.line, self.implicit) + + def slice(self, begin: int, stride: int, end: int) -> 'TupleType': + return TupleType(self.items[begin:end:stride], self.fallback, + self.line, self.implicit) + + +class NamedTupleType(TupleType): + attrs = None # type: List[str] + name = None # type: str + + def __init__(self, name: str, attrs: List[str], *args, **kwargs) -> None: + self.attrs = attrs + self.name = name + super().__init__(*args, **kwargs) + + def self_type(self, inst: Instance) -> 'NamedTupleType': + return NamedTupleType(self.name, self.attrs, self.items, inst, self.line) + + def with_types(self, items: List[Type]) -> 'NamedTupleType': + return NamedTupleType(self.name, self.attrs, items, + self.fallback, self.line, self.implicit) + + def __eq__(self, other) -> bool: + return self.serialize() == other.serialize() + + def serialize(self) -> JsonDict: + d = super().serialize() + d.update({'.class': 'NamedTupleType', + 'name': self.name, + 'attrs': self.attrs}) + return d + + @classmethod + def deserialize(cls, data: JsonDict) -> 'NamedTupleType': + assert data['.class'] == 'NamedTupleType' + data['.class'] = 'TupleType' + tup = TupleType.deserialize(data) + return NamedTupleType(data['name'], + data['attrs'], + tup.items, + tup.fallback, + implicit=tup.implicit) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + try: + return visitor.visit_namedtuple_type(self) + except: + return super().accept(visitor) + + +class StarType(Type): + """The star type *type_parameter. + + This is not a real type but a syntactic AST construct. + """ + + type = None # type: Type + + def __init__(self, type: Type, line: int = -1) -> None: + self.type = type + super().__init__(line) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_star_type(self) + + +class UnionType(Type): + """The union type Union[T1, ..., Tn] (at least one type argument).""" + + items = None # type: List[Type] + + def __init__(self, items: List[Type], line: int = -1) -> None: + self.items = items + super().__init__(line) + + @staticmethod + def make_union(items: List[Type], line: int = -1) -> Type: + if len(items) > 1: + return UnionType(items, line) + elif len(items) == 1: + return items[0] + else: + if experiments.STRICT_OPTIONAL: + return UninhabitedType() + else: + return Void() + + @staticmethod + def make_simplified_union(items: List[Type], line: int = -1) -> Type: + while any(isinstance(typ, UnionType) for typ in items): + all_items = [] # type: List[Type] + for typ in items: + if isinstance(typ, UnionType): + all_items.extend(typ.items) + else: + all_items.append(typ) + items = all_items + + if any(isinstance(typ, AnyType) for typ in items): + return AnyType() + + from mypy.subtypes import is_subtype + removed = set() # type: Set[int] + for i in range(len(items)): + if any(is_subtype(items[i], items[j]) for j in range(len(items)) + if j not in removed and j != i): + removed.add(i) + + simplified_set = [items[i] for i in range(len(items)) if i not in removed] + return UnionType.make_union(simplified_set) + + def length(self) -> int: + return len(self.items) + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_union_type(self) + + def has_readable_member(self, name: str) -> bool: + """For a tree of unions of instances, check whether all instances have a given member. + + TODO: Deal with attributes of TupleType etc. + TODO: This should probably be refactored to go elsewhere. + """ + return all((isinstance(x, UnionType) and x.has_readable_member(name)) or + (isinstance(x, Instance) and x.type.has_readable_member(name)) + for x in self.items) + + def serialize(self) -> JsonDict: + return {'.class': 'UnionType', + 'items': [t.serialize() for t in self.items], + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'UnionType': + assert data['.class'] == 'UnionType' + return UnionType([Type.deserialize(t) for t in data['items']]) + + +class PartialType(Type): + """Type such as List[?] where type arguments are unknown, or partial None type. + + These are used for inferring types in multiphase initialization such as this: + + x = [] # x gets a partial type List[?], as item type is unknown + x.append(1) # partial type gets replaced with normal type List[int] + + Or with None: + + x = None # x gets a partial type None + if c: + x = 1 # Infer actual type int for x + """ + + # None for the 'None' partial type; otherwise a generic class + type = None # type: Optional[mypy.nodes.TypeInfo] + var = None # type: mypy.nodes.Var + inner_types = None # type: List[Type] + + def __init__(self, + type: Optional['mypy.nodes.TypeInfo'], + var: 'mypy.nodes.Var', + inner_types: List[Type]) -> None: + self.type = type + self.var = var + self.inner_types = inner_types + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_partial_type(self) + + +class EllipsisType(Type): + """The type ... (ellipsis). + + This is not a real type but a syntactic AST construct, used in Callable[..., T], for example. + + A semantically analyzed type will never have ellipsis types. + """ + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_ellipsis_type(self) + + def serialize(self) -> JsonDict: + return {'.class': 'EllipsisType'} + + @classmethod + def deserialize(self, data: JsonDict) -> 'EllipsisType': + assert data['.class'] == 'EllipsisType' + return EllipsisType() + + +class TypeType(Type): + """For types like Type[User]. + + This annotates variables that are class objects, constrained by + the type argument. See PEP 484 for more details. + + We may encounter expressions whose values are specific classes; + those are represented as callables (possibly overloaded) + corresponding to the class's constructor's signature and returning + an instance of that class. The difference with Type[C] is that + those callables always represent the exact class given as the + return type; Type[C] represents any class that's a subclass of C, + and C may also be a type variable or a union (or Any). + + Many questions around subtype relationships between Type[C1] and + def(...) -> C2 are answered by looking at the subtype + relationships between C1 and C2, since Type[] is considered + covariant. + + There's an unsolved problem with constructor signatures (also + unsolved in PEP 484): calling a variable whose type is Type[C] + assumes the constructor signature for C, even though a subclass of + C might completely change the constructor signature. For now we + just assume that users of Type[C] are careful not to do that (in + the future we might detect when they are violating that + assumption). + """ + + # This can't be everything, but it can be a class reference, + # a generic class instance, a union, Any, a type variable... + item = None # type: Type + + def __init__(self, item: Type, *, line: int = -1) -> None: + super().__init__(line) + self.item = item + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_type_type(self) + + def serialize(self) -> JsonDict: + return {'.class': 'TypeType', 'item': self.item.serialize()} + + def deserialize(self, data: JsonDict) -> 'TypeType': + assert data['.class'] == 'TypeType' + return TypeType(Type.deserialize(data['item'])) + + +# +# Visitor-related classes +# + + +class TypeVisitor(Generic[T]): + """Visitor class for types (Type subclasses). + + The parameter T is the return type of the visit methods. + """ + + def _notimplemented_helper(self, name: str) -> NotImplementedError: + return NotImplementedError("Method {}.visit_{}() not implemented\n" + .format(type(self).__name__, name) + + "This is a known bug, track development in " + + "'https://github.com/JukkaL/mypy/issues/730'") + + @abstractmethod + def visit_unbound_type(self, t: UnboundType) -> T: + pass + + def visit_type_list(self, t: TypeList) -> T: + raise self._notimplemented_helper('type_list') + + def visit_error_type(self, t: ErrorType) -> T: + raise self._notimplemented_helper('error_type') + + @abstractmethod + def visit_any(self, t: AnyType) -> T: + pass + + @abstractmethod + def visit_void(self, t: Void) -> T: + pass + + @abstractmethod + def visit_none_type(self, t: NoneTyp) -> T: + pass + + @abstractmethod + def visit_uninhabited_type(self, t: UninhabitedType) -> T: + pass + + def visit_erased_type(self, t: ErasedType) -> T: + raise self._notimplemented_helper('erased_type') + + @abstractmethod + def visit_deleted_type(self, t: DeletedType) -> T: + pass + + @abstractmethod + def visit_type_var(self, t: TypeVarType) -> T: + pass + + @abstractmethod + def visit_instance(self, t: Instance) -> T: + pass + + @abstractmethod + def visit_callable_type(self, t: CallableType) -> T: + pass + + def visit_overloaded(self, t: Overloaded) -> T: + raise self._notimplemented_helper('overloaded') + + @abstractmethod + def visit_tuple_type(self, t: TupleType) -> T: + pass + + def visit_star_type(self, t: StarType) -> T: + raise self._notimplemented_helper('star_type') + + @abstractmethod + def visit_union_type(self, t: UnionType) -> T: + pass + + @abstractmethod + def visit_partial_type(self, t: PartialType) -> T: + pass + + def visit_ellipsis_type(self, t: EllipsisType) -> T: + raise self._notimplemented_helper('ellipsis_type') + + @abstractmethod + def visit_type_type(self, t: TypeType) -> T: + pass + + +class TypeTranslator(TypeVisitor[Type]): + """Identity type transformation. + + Subclass this and override some methods to implement a non-trivial + transformation. + """ + + def visit_unbound_type(self, t: UnboundType) -> Type: + return t + + def visit_type_list(self, t: TypeList) -> Type: + return t + + def visit_error_type(self, t: ErrorType) -> Type: + return t + + def visit_any(self, t: AnyType) -> Type: + return t + + def visit_void(self, t: Void) -> Type: + return t + + def visit_none_type(self, t: NoneTyp) -> Type: + return t + + def visit_uninhabited_type(self, t: UninhabitedType) -> Type: + return t + + def visit_erased_type(self, t: ErasedType) -> Type: + return t + + def visit_deleted_type(self, t: DeletedType) -> Type: + return t + + def visit_instance(self, t: Instance) -> Type: + return Instance(t.type, self.translate_types(t.args), t.line) + + def visit_type_var(self, t: TypeVarType) -> Type: + return t + + def visit_partial_type(self, t: PartialType) -> Type: + return t + + def visit_callable_type(self, t: CallableType) -> Type: + return t.copy_modified(arg_types=self.translate_types(t.arg_types), + ret_type=t.ret_type.accept(self), + variables=self.translate_variables(t.variables)) + + def visit_tuple_type(self, t: TupleType) -> Type: + return TupleType(self.translate_types(t.items), + cast(Any, t.fallback.accept(self)), + t.line) + + def visit_star_type(self, t: StarType) -> Type: + return StarType(t.type.accept(self), t.line) + + def visit_union_type(self, t: UnionType) -> Type: + return UnionType(self.translate_types(t.items), t.line) + + def visit_ellipsis_type(self, t: EllipsisType) -> Type: + return t + + def translate_types(self, types: List[Type]) -> List[Type]: + return [t.accept(self) for t in types] + + def translate_variables(self, + variables: List[TypeVarDef]) -> List[TypeVarDef]: + return variables + + def visit_overloaded(self, t: Overloaded) -> Type: + items = [] # type: List[CallableType] + for item in t.items(): + new = item.accept(self) + if isinstance(new, CallableType): + items.append(new) + else: + raise RuntimeError('CallableType expectected, but got {}'.format(type(new))) + return Overloaded(items=items) + + def visit_type_type(self, t: TypeType) -> Type: + return TypeType(t.item.accept(self), line=t.line) + + +class TypeStrVisitor(TypeVisitor[str]): + """Visitor for pretty-printing types into strings. + + This is mostly for debugging/testing. + + Do not preserve original formatting. + + Notes: + - Represent unbound types as Foo? or Foo?[...]. + - Represent the NoneTyp type as None. + """ + + def visit_unbound_type(self, t): + s = t.name + '?' + if t.args != []: + s += '[{}]'.format(self.list_str(t.args)) + return s + + def visit_type_list(self, t): + return ''.format(self.list_str(t.items)) + + def visit_error_type(self, t): + return '' + + def visit_any(self, t): + return 'Any' + + def visit_void(self, t): + return 'void' + + def visit_none_type(self, t): + # Fully qualify to make this distinct from the None value. + return "builtins.None" + + def visit_uninhabited_type(self, t): + return "" + + def visit_erased_type(self, t): + return "" + + def visit_deleted_type(self, t): + if t.source is None: + return "" + else: + return "".format(t.source) + + def visit_instance(self, t): + s = t.type.fullname() if t.type is not None else '' + if t.erased: + s += '*' + if t.args != []: + s += '[{}]'.format(self.list_str(t.args)) + return s + + def visit_type_var(self, t): + if t.name is None: + # Anonymous type variable type (only numeric id). + return '`{}'.format(t.id) + else: + # Named type variable type. + return '{}`{}'.format(t.name, t.id) + + def visit_callable_type(self, t): + s = '' + bare_asterisk = False + for i in range(len(t.arg_types)): + if s != '': + s += ', ' + if t.arg_kinds[i] == mypy.nodes.ARG_NAMED and not bare_asterisk: + s += '*, ' + bare_asterisk = True + if t.arg_kinds[i] == mypy.nodes.ARG_STAR: + s += '*' + if t.arg_kinds[i] == mypy.nodes.ARG_STAR2: + s += '**' + if t.arg_names[i]: + s += t.arg_names[i] + ': ' + s += str(t.arg_types[i]) + if t.arg_kinds[i] == mypy.nodes.ARG_OPT: + s += ' =' + + s = '({})'.format(s) + + if not isinstance(t.ret_type, Void): + s += ' -> {}'.format(t.ret_type) + + if t.variables: + s = '{} {}'.format(t.variables, s) + + return 'def {}'.format(s) + + def visit_overloaded(self, t): + a = [] + for i in t.items(): + a.append(i.accept(self)) + return 'Overload({})'.format(', '.join(a)) + + def visit_tuple_type(self, t): + s = self.list_str(t.items) + if t.fallback and t.fallback.type: + fallback_name = t.fallback.type.fullname() + if fallback_name != 'builtins.tuple': + return 'Tuple[{}, fallback={}]'.format(s, t.fallback.accept(self)) + return 'Tuple[{}]'.format(s) + + def visit_namedtuple_type(self, t): + s = self.list_str(zip(t.attrs, t.items)) + return '{}[{}]'.format(t.name, s) + + def visit_star_type(self, t): + s = t.type.accept(self) + return '*{}'.format(s) + + def visit_union_type(self, t): + s = self.list_str(t.items) + return 'Union[{}]'.format(s) + + def visit_partial_type(self, t: PartialType) -> str: + if t.type is None: + return '' + else: + return ''.format(t.type.name(), + ', '.join(['?'] * len(t.type.type_vars))) + + def visit_ellipsis_type(self, t): + return '...' + + def visit_type_type(self, t): + return 'Type[{}]'.format(t.item.accept(self)) + + def list_str(self, a): + """Convert items of an array to strings (pretty-print types) + and join the results with commas. + """ + res = [] + for t in a: + if isinstance(t, Type): + res.append(t.accept(self)) + else: + res.append(str(t)) + return ', '.join(res) + + +# These constants define the method used by TypeQuery to combine multiple +# query results, e.g. for tuple types. The strategy is not used for empty +# result lists; in that case the default value takes precedence. +ANY_TYPE_STRATEGY = 0 # Return True if any of the results are True. +ALL_TYPES_STRATEGY = 1 # Return True if all of the results are True. + + +class TypeQuery(TypeVisitor[bool]): + """Visitor for performing simple boolean queries of types. + + This class allows defining the default value for leafs to simplify the + implementation of many queries. + """ + + default = False # Default result + strategy = 0 # Strategy for combining multiple values (ANY_TYPE_STRATEGY or ALL_TYPES_...). + + def __init__(self, default: bool, strategy: int) -> None: + """Construct a query visitor. + + Use the given default result and strategy for combining + multiple results. The strategy must be either + ANY_TYPE_STRATEGY or ALL_TYPES_STRATEGY. + """ + self.default = default + self.strategy = strategy + + def visit_unbound_type(self, t: UnboundType) -> bool: + return self.default + + def visit_type_list(self, t: TypeList) -> bool: + return self.default + + def visit_error_type(self, t: ErrorType) -> bool: + return self.default + + def visit_any(self, t: AnyType) -> bool: + return self.default + + def visit_void(self, t: Void) -> bool: + return self.default + + def visit_uninhabited_type(self, t: UninhabitedType) -> bool: + return self.default + + def visit_none_type(self, t: NoneTyp) -> bool: + return self.default + + def visit_erased_type(self, t: ErasedType) -> bool: + return self.default + + def visit_deleted_type(self, t: DeletedType) -> bool: + return self.default + + def visit_type_var(self, t: TypeVarType) -> bool: + return self.default + + def visit_partial_type(self, t: PartialType) -> bool: + return self.default + + def visit_instance(self, t: Instance) -> bool: + return self.query_types(t.args) + + def visit_callable_type(self, t: CallableType) -> bool: + # FIX generics + return self.query_types(t.arg_types + [t.ret_type]) + + def visit_tuple_type(self, t: TupleType) -> bool: + return self.query_types(t.items) + + def visit_star_type(self, t: StarType) -> bool: + return t.type.accept(self) + + def visit_union_type(self, t: UnionType) -> bool: + return self.query_types(t.items) + + def visit_overloaded(self, t: Overloaded) -> bool: + return self.query_types(t.items()) + + def visit_type_type(self, t: TypeType) -> bool: + return t.item.accept(self) + + def query_types(self, types: Sequence[Type]) -> bool: + """Perform a query for a list of types. + + Use the strategy constant to combine the results. + """ + if not types: + # Use default result for empty list. + return self.default + if self.strategy == ANY_TYPE_STRATEGY: + # Return True if at least one component is true. + res = False + for t in types: + res = res or t.accept(self) + if res: + break + return res + else: + # Return True if all components are true. + res = True + for t in types: + res = res and t.accept(self) + if not res: + break + return res + + +def strip_type(typ: Type) -> Type: + """Make a copy of type without 'debugging info' (function name).""" + + if isinstance(typ, CallableType): + return typ.copy_modified(name=None) + elif isinstance(typ, Overloaded): + return Overloaded([cast(CallableType, strip_type(item)) + for item in typ.items()]) + else: + return typ + + +def replace_leading_arg_type(t: CallableType, self_type: Type) -> CallableType: + """Return a copy of a callable type with a different self argument type. + + Assume that the callable is the signature of a method. + """ + return t.copy_modified(arg_types=[self_type] + t.arg_types[1:]) + + +def is_named_instance(t: Type, fullname: str) -> bool: + return (isinstance(t, Instance) and + t.type is not None and + t.type.fullname() == fullname) diff --git a/test-data/unit/check-lists.test b/test-data/unit/check-lists.test index a6df5ae740c61..9db5fb018abfa 100644 --- a/test-data/unit/check-lists.test +++ b/test-data/unit/check-lists.test @@ -1,63 +1,63 @@ --- Nested list assignment --- ----------------------------- - -[case testNestedListAssignment] -from typing import List -a1, b1, c1 = None, None, None # type: (A, B, C) -a2, b2, c2 = None, None, None # type: (A, B, C) - -a1, [b1, c1] = a2, [b2, c2] -a1, [a1, [b1, c1]] = a2, [a2, [b2, c2]] -a1, [a1, [a1, b1]] = a1, [a1, [a1, c1]] # E: Incompatible types in assignment (expression has type "C", variable has type "B") - -class A: pass -class B: pass -class C: pass -[builtins fixtures/list.py] -[out] - -[case testNestedListAssignmentToTuple] -from typing import List -a, b, c = None, None, None # type: (A, B, C) - -a, b = [a, b] -a, b = [a] # E: Need more than 1 value to unpack (2 expected) -a, b = [a, b, c] # E: Too many values to unpack (2 expected, 3 provided) - -class A: pass -class B: pass -class C: pass -[builtins fixtures/list.py] -[out] - -[case testListAssignmentFromTuple] -from typing import List -a, b, c = None, None, None # type: (A, B, C) -t = a, b - -[a, b], c = t, c -[a, c], c = t, c # E: Incompatible types in assignment (expression has type "B", variable has type "C") -[a, a, a], c = t, c # E: Need more than 2 values to unpack (3 expected) -[a], c = t, c # E: Too many values to unpack (1 expected, 2 provided) - -class A: pass -class B: pass -class C: pass -[builtins fixtures/list.py] -[out] - -[case testListAssignmentUnequalAmountToUnpack] -from typing import List -a, b, c = None, None, None # type: (A, B, C) - -def f() -> None: # needed because test parser tries to parse [a, b] as section header - [a, b] = [a, b] - [a, b] = [a] # E: Need more than 1 value to unpack (2 expected) - [a, b] = [a, b, c] # E: Too many values to unpack (2 expected, 3 provided) - -class A: pass -class B: pass -class C: pass -[builtins fixtures/list.py] -[out] -main: note: In function "f": +-- Nested list assignment +-- ----------------------------- + +[case testNestedListAssignment] +from typing import List +a1, b1, c1 = None, None, None # type: (A, B, C) +a2, b2, c2 = None, None, None # type: (A, B, C) + +a1, [b1, c1] = a2, [b2, c2] +a1, [a1, [b1, c1]] = a2, [a2, [b2, c2]] +a1, [a1, [a1, b1]] = a1, [a1, [a1, c1]] # E: Incompatible types in assignment (expression has type "C", variable has type "B") + +class A: pass +class B: pass +class C: pass +[builtins fixtures/list.py] +[out] + +[case testNestedListAssignmentToTuple] +from typing import List +a, b, c = None, None, None # type: (A, B, C) + +a, b = [a, b] +a, b = [a] # E: Need more than 1 value to unpack (2 expected) +a, b = [a, b, c] # E: Too many values to unpack (2 expected, 3 provided) + +class A: pass +class B: pass +class C: pass +[builtins fixtures/list.py] +[out] + +[case testListAssignmentFromTuple] +from typing import List +a, b, c = None, None, None # type: (A, B, C) +t = a, b + +[a, b], c = t, c +[a, c], c = t, c # E: Incompatible types in assignment (expression has type "B", variable has type "C") +[a, a, a], c = t, c # E: Need more than 2 values to unpack (3 expected) +[a], c = t, c # E: Too many values to unpack (1 expected, 2 provided) + +class A: pass +class B: pass +class C: pass +[builtins fixtures/list.py] +[out] + +[case testListAssignmentUnequalAmountToUnpack] +from typing import List +a, b, c = None, None, None # type: (A, B, C) + +def f() -> None: # needed because test parser tries to parse [a, b] as section header + [a, b] = [a, b] + [a, b] = [a] # E: Need more than 1 value to unpack (2 expected) + [a, b] = [a, b, c] # E: Too many values to unpack (2 expected, 3 provided) + +class A: pass +class B: pass +class C: pass +[builtins fixtures/list.py] +[out] +main: note: In function "f": diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 86c19956ca634..adb07ec522705 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -1,223 +1,223 @@ -[case testNamedTupleDeclaration] -from typing import NamedTuple - -A = NamedTuple('A', [('x', str), ('y', int)]) - -[case testNamedTupleUsedAsTuple] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -a, b = x -b = x[0] -a = x[1] -a, b, c = x # E: Need more than 2 values to unpack (3 expected) -x[2] # E: Tuple index out of range - -[case testAccessingNamedTupleAttributes] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -x.x -x.y -x.z # E: "X" has no attribute "z" - -[case testCreateNamedTupleWithPositionalArguments] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = X(1, 'x') -x.x -x.z # E: "X" has no attribute "z" -x = X(1) # E: Too few arguments for "X" -x = X(1, 2, 3) # E: Too many arguments for "X" - -[case testCreateNamedTupleWithKeywordArguments] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = X(x=1, y='x') -x = X(1, y='x') -x = X(x=1, z=1) # E: Unexpected keyword argument "z" for "X" -x = X(y=1) # E: Missing positional argument "x" in call to "X" - -[case testCreateNamedTupleAndUseAsTuple] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = X(1, 'x') -a, b = x -a, b, c = x # E: Need more than 2 values to unpack (3 expected) - -[case testNamedTupleWithItemTypes] -from typing import NamedTuple -N = NamedTuple('N', [('a', int), - ('b', str)]) -n = N(1, 'x') -s = n.a # type: str # E: Incompatible types in assignment (expression has type "int", \ - variable has type "str") -i = n.b # type: int # E: Incompatible types in assignment (expression has type "str", \ - variable has type "int") -x, y = n -x = y # E: Incompatible types in assignment (expression has type "str", variable has type "int") - -[case testNamedTupleConstructorArgumentTypes] -from typing import NamedTuple -N = NamedTuple('N', [('a', int), - ('b', str)]) -n = N('x', 'x') # E: Argument 1 to "N" has incompatible type "str"; expected "int" -n = N(1, b=2) # E: Argument 2 to "N" has incompatible type "int"; expected "str" -N(1, 'x') -N(b='x', a=1) - -[case testNamedTupleAsBaseClass] -from typing import NamedTuple -N = NamedTuple('N', [('a', int), - ('b', str)]) -class X(N): - pass -x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" -s = '' -i = 0 -s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") -i, s = x -s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") - -[case testNamedTupleAsBaseClass2] -from typing import NamedTuple -class X(NamedTuple('N', [('a', int), - ('b', str)])): - pass -x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" -s = '' -i = 0 -s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") -i, s = x -s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") - -[case testTwoNamedTuplesAsBaseClasses] -from typing import NamedTuple -A = NamedTuple('A', [('a', int)]) -B = NamedTuple('B', [('a', int)]) -class X(A, B): # E: Class has two incompatible bases derived from tuple - pass - -[case testTwoNamedTuplesAsBaseClasses2] -from typing import NamedTuple -A = NamedTuple('A', [('a', int)]) -class X(A, NamedTuple('B', [('a', int)])): # E: Class has two incompatible bases derived from tuple - pass - -[case testSelfTypeWithNamedTupleAsBase] -from typing import NamedTuple -A = NamedTuple('A', [('a', int), ('b', str)]) -class B(A): - def f(self, x: int) -> None: - self.f(self.a) - self.f(self.b) # E: Argument 1 to "f" of "B" has incompatible type "str"; expected "int" - i = 0 - s = '' - i, s = self - i, i = self # E: Incompatible types in assignment (expression has type "str", \ - variable has type "int") -[out] -main: note: In member "f" of class "B": - -[case testTypeReferenceToClassDerivedFromNamedTuple] -from typing import NamedTuple -A = NamedTuple('A', [('a', int), ('b', str)]) -class B(A): - def f(self, x: 'B') -> None: - i = 0 - s = '' - self = x - i, s = x - i, s = x.a, x.b - i, s = x.a, x.a # E: Incompatible types in assignment (expression has type "int", \ - variable has type "str") - i, i = self # E: Incompatible types in assignment (expression has type "str", \ - variable has type "int") -[out] -main: note: In member "f" of class "B": - -[case testNamedTupleSubtyping] -from typing import NamedTuple, Tuple -A = NamedTuple('A', [('a', int), ('b', str)]) -class B(A): pass -a = A(1, '') -b = B(1, '') -t = None # type: Tuple[int, str] -b = a # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "A") -b = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "B") -t = a -t = (1, '') -t = b -a = b - -[case testNamedTupleSimpleTypeInference] -from typing import NamedTuple, Tuple -A = NamedTuple('A', [('a', int)]) -l = [A(1), A(2)] -a = A(1) -a = l[0] -(i,) = l[0] -i, i = l[0] # E: Need more than 1 value to unpack (2 expected) -a = (1,) # E: Incompatible types in assignment (expression has type "Tuple[int]", \ - variable has type "A") -[builtins fixtures/list.py] - -[case testNamedTupleMissingClassAttribute] -import collections -MyNamedTuple = collections.namedtuple('MyNamedTuple', ['spam', 'eggs']) -MyNamedTuple.x # E: "MyNamedTuple" has no attribute "x" - -[case testNamedTupleEmptyItems] -from typing import NamedTuple -A = NamedTuple('A', []) -[builtins fixtures/list.py] - -[case testNamedTupleProperty] -from typing import NamedTuple -A = NamedTuple('A', [('a', int)]) -class B(A): - @property - def b(self) -> int: - return self.a -class C(B): pass -B(1).b -C(2).b -[builtins fixtures/property.py] - -[case testNamedTupleAsDict] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -a = x._asdict() - -[case testNamedTupleReplace] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -x = x._replace(y=5) -x = x._replace(z=5) # E: Unexpected keyword argument "z" for X._replace -x = x._replace(5) # E: Too many positional arguments for X._replace - -[case testNamedTupleReplaceAsClass] -from collections import namedtuple - -X = namedtuple('X', ['x', 'y']) -x = None # type: X -X._replace(x, x=1, y=2) -X._replace(x=1, y=2) # E: Missing positional argument "self" in call to X._replace - -[case testNamedTupleReplaceTyped] -from typing import NamedTuple - -X = NamedTuple('X', [('x', int), ('y', str)]) -x = None # type: X -x = x._replace(x=5) -x = x._replace(y=5) # E: Argument 1 to X._replace has incompatible type "int"; expected "str" +[case testNamedTupleDeclaration] +from typing import NamedTuple + +A = NamedTuple('A', [('x', str), ('y', int)]) + +[case testNamedTupleUsedAsTuple] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +a, b = x +b = x[0] +a = x[1] +a, b, c = x # E: Need more than 2 values to unpack (3 expected) +x[2] # E: Tuple index out of range + +[case testAccessingNamedTupleAttributes] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +x.x +x.y +x.z # E: "X" has no attribute "z" + +[case testCreateNamedTupleWithPositionalArguments] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = X(1, 'x') +x.x +x.z # E: "X" has no attribute "z" +x = X(1) # E: Too few arguments for "X" +x = X(1, 2, 3) # E: Too many arguments for "X" + +[case testCreateNamedTupleWithKeywordArguments] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = X(x=1, y='x') +x = X(1, y='x') +x = X(x=1, z=1) # E: Unexpected keyword argument "z" for "X" +x = X(y=1) # E: Missing positional argument "x" in call to "X" + +[case testCreateNamedTupleAndUseAsTuple] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = X(1, 'x') +a, b = x +a, b, c = x # E: Need more than 2 values to unpack (3 expected) + +[case testNamedTupleWithItemTypes] +from typing import NamedTuple +N = NamedTuple('N', [('a', int), + ('b', str)]) +n = N(1, 'x') +s = n.a # type: str # E: Incompatible types in assignment (expression has type "int", \ + variable has type "str") +i = n.b # type: int # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +x, y = n +x = y # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testNamedTupleConstructorArgumentTypes] +from typing import NamedTuple +N = NamedTuple('N', [('a', int), + ('b', str)]) +n = N('x', 'x') # E: Argument 1 to "N" has incompatible type "str"; expected "int" +n = N(1, b=2) # E: Argument 2 to "N" has incompatible type "int"; expected "str" +N(1, 'x') +N(b='x', a=1) + +[case testNamedTupleAsBaseClass] +from typing import NamedTuple +N = NamedTuple('N', [('a', int), + ('b', str)]) +class X(N): + pass +x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" +s = '' +i = 0 +s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") +i, s = x +s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testNamedTupleAsBaseClass2] +from typing import NamedTuple +class X(NamedTuple('N', [('a', int), + ('b', str)])): + pass +x = X(1, 2) # E: Argument 2 to "X" has incompatible type "int"; expected "str" +s = '' +i = 0 +s = x.a # E: Incompatible types in assignment (expression has type "int", variable has type "str") +i, s = x +s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testTwoNamedTuplesAsBaseClasses] +from typing import NamedTuple +A = NamedTuple('A', [('a', int)]) +B = NamedTuple('B', [('a', int)]) +class X(A, B): # E: Class has two incompatible bases derived from tuple + pass + +[case testTwoNamedTuplesAsBaseClasses2] +from typing import NamedTuple +A = NamedTuple('A', [('a', int)]) +class X(A, NamedTuple('B', [('a', int)])): # E: Class has two incompatible bases derived from tuple + pass + +[case testSelfTypeWithNamedTupleAsBase] +from typing import NamedTuple +A = NamedTuple('A', [('a', int), ('b', str)]) +class B(A): + def f(self, x: int) -> None: + self.f(self.a) + self.f(self.b) # E: Argument 1 to "f" of "B" has incompatible type "str"; expected "int" + i = 0 + s = '' + i, s = self + i, i = self # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +[out] +main: note: In member "f" of class "B": + +[case testTypeReferenceToClassDerivedFromNamedTuple] +from typing import NamedTuple +A = NamedTuple('A', [('a', int), ('b', str)]) +class B(A): + def f(self, x: 'B') -> None: + i = 0 + s = '' + self = x + i, s = x + i, s = x.a, x.b + i, s = x.a, x.a # E: Incompatible types in assignment (expression has type "int", \ + variable has type "str") + i, i = self # E: Incompatible types in assignment (expression has type "str", \ + variable has type "int") +[out] +main: note: In member "f" of class "B": + +[case testNamedTupleSubtyping] +from typing import NamedTuple, Tuple +A = NamedTuple('A', [('a', int), ('b', str)]) +class B(A): pass +a = A(1, '') +b = B(1, '') +t = None # type: Tuple[int, str] +b = a # E: Incompatible types in assignment (expression has type "A", variable has type "B") +a = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "A") +b = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type "B") +t = a +t = (1, '') +t = b +a = b + +[case testNamedTupleSimpleTypeInference] +from typing import NamedTuple, Tuple +A = NamedTuple('A', [('a', int)]) +l = [A(1), A(2)] +a = A(1) +a = l[0] +(i,) = l[0] +i, i = l[0] # E: Need more than 1 value to unpack (2 expected) +a = (1,) # E: Incompatible types in assignment (expression has type "Tuple[int]", \ + variable has type "A") +[builtins fixtures/list.py] + +[case testNamedTupleMissingClassAttribute] +import collections +MyNamedTuple = collections.namedtuple('MyNamedTuple', ['spam', 'eggs']) +MyNamedTuple.x # E: "MyNamedTuple" has no attribute "x" + +[case testNamedTupleEmptyItems] +from typing import NamedTuple +A = NamedTuple('A', []) +[builtins fixtures/list.py] + +[case testNamedTupleProperty] +from typing import NamedTuple +A = NamedTuple('A', [('a', int)]) +class B(A): + @property + def b(self) -> int: + return self.a +class C(B): pass +B(1).b +C(2).b +[builtins fixtures/property.py] + +[case testNamedTupleAsDict] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +a = x._asdict() + +[case testNamedTupleReplace] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +x = x._replace(y=5) +x = x._replace(z=5) # E: Unexpected keyword argument "z" for X._replace +x = x._replace(5) # E: Too many positional arguments for X._replace + +[case testNamedTupleReplaceAsClass] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +X._replace(x, x=1, y=2) +X._replace(x=1, y=2) # E: Missing positional argument "self" in call to X._replace + +[case testNamedTupleReplaceTyped] +from typing import NamedTuple + +X = NamedTuple('X', [('x', int), ('y', str)]) +x = None # type: X +x = x._replace(x=5) +x = x._replace(y=5) # E: Argument 1 to X._replace has incompatible type "int"; expected "str" diff --git a/test-data/unit/check-tuples.test b/test-data/unit/check-tuples.test index f946c451123fd..50d1b3e64598d 100644 --- a/test-data/unit/check-tuples.test +++ b/test-data/unit/check-tuples.test @@ -1,770 +1,770 @@ --- Normal assignment and subtyping --- ------------------------------- - - -[case testTupleAssignmentWithTupleTypes] -from typing import Tuple -t1 = None # type: Tuple[A] -t2 = None # type: Tuple[B] -t3 = None # type: Tuple[A, A] -t4 = None # type: Tuple[A, B] -t5 = None # type: Tuple[B, A] - -t1 = t2 # E: Incompatible types in assignment (expression has type "Tuple[B]", variable has type "Tuple[A]") -t1 = t3 # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A]") -t3 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A]", variable has type "Tuple[A, A]") -t3 = t4 # E: Incompatible types in assignment (expression has type "Tuple[A, B]", variable has type "Tuple[A, A]") -t3 = t5 # E: Incompatible types in assignment (expression has type "Tuple[B, A]", variable has type "Tuple[A, A]") - -# Ok -t1 = t1 -t2 = t2 -t3 = t3 -t4 = t4 -t5 = t5 - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - -[case testTupleSubtyping] -from typing import Tuple -t1 = None # type: Tuple[A, A] -t2 = None # type: Tuple[A, B] -t3 = None # type: Tuple[B, A] - -t2 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A, B]") -t2 = t3 # E: Incompatible types in assignment (expression has type "Tuple[B, A]", variable has type "Tuple[A, B]") -t3 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[B, A]") -t3 = t2 # E: Incompatible types in assignment (expression has type "Tuple[A, B]", variable has type "Tuple[B, A]") - -t1 = t2 -t1 = t3 - -class A: pass -class B(A): pass -[builtins fixtures/tuple.py] - -[case testTupleCompatibilityWithOtherTypes] -from typing import Tuple -a, o = None, None # type: (A, object) -t = None # type: Tuple[A, A] - -a = t # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "A") -t = o # E: Incompatible types in assignment (expression has type "object", variable has type "Tuple[A, A]") -t = a # E: Incompatible types in assignment (expression has type "A", variable has type "Tuple[A, A]") -# TODO: callable types + tuples - -# Ok -o = t -t = None - -class A: pass -[builtins fixtures/tuple.py] - -[case testNestedTupleTypes] -from typing import Tuple -t1 = None # type: Tuple[A, Tuple[A, A]] -t2 = None # type: Tuple[B, Tuple[B, B]] - -t2 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, Tuple[A, A]]", variable has type "Tuple[B, Tuple[B, B]]") -t1 = t2 - -class A: pass -class B(A): pass -[builtins fixtures/tuple.py] - -[case testNestedTupleTypes2] -from typing import Tuple -t1 = None # type: Tuple[A, Tuple[A, A]] -t2 = None # type: Tuple[B, Tuple[B, B]] - -t2 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, Tuple[A, A]]", variable has type "Tuple[B, Tuple[B, B]]") -t1 = t2 - -class A: pass -class B(A): pass -[builtins fixtures/tuple.py] - -[case testSubtypingWithNamedTupleType] -from typing import Tuple -t1 = None # type: Tuple[A, A] -t2 = None # type: tuple - -t1 = t2 # E: Incompatible types in assignment (expression has type Tuple[Any, ...], variable has type "Tuple[A, A]") -t2 = t1 - -class A: pass -[builtins fixtures/tuple.py] - -[case testTupleInitializationWithNone] -from typing import Tuple -t = None # type: Tuple[A, A] -t = None -class A: pass -[builtins fixtures/tuple.py] - - --- Tuple expressions --- ----------------- - - -[case testTupleExpressions] -from typing import Tuple -t1 = None # type: tuple -t2 = None # type: Tuple[A] -t3 = None # type: Tuple[A, B] - -a, b, c = None, None, None # type: (A, B, C) - -t2 = () # E: Incompatible types in assignment (expression has type "Tuple[]", variable has type "Tuple[A]") -t2 = (a, a) # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A]") -t3 = (a, a) # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A, B]") -t3 = (b, b) # E: Incompatible types in assignment (expression has type "Tuple[B, B]", variable has type "Tuple[A, B]") -t3 = (a, b, a) # E: Incompatible types in assignment (expression has type "Tuple[A, B, A]", variable has type "Tuple[A, B]") - -t1 = () -t1 = (a,) -t2 = (a,) -t3 = (a, b) -t3 = (a, c) -t3 = (None, None) - -class A: pass -class B: pass -class C(B): pass -[builtins fixtures/tuple.py] - -[case testVoidValueInTuple] -import typing -(None, f()) # E: "f" does not return a value -(f(), None) # E: "f" does not return a value - -def f() -> None: pass -[builtins fixtures/tuple.py] - - --- Indexing --- -------- - - -[case testIndexingTuples] -from typing import Tuple -t1 = None # type: Tuple[A, B] -t2 = None # type: Tuple[A] -t3 = None # type: Tuple[A, B, C, D, E] -a, b = None, None # type: (A, B) -x = None # type: Tuple[A, B, C] -y = None # type: Tuple[A, C, E] -n = 0 - -a = t1[1] # E: Incompatible types in assignment (expression has type "B", variable has type "A") -b = t1[0] # E: Incompatible types in assignment (expression has type "A", variable has type "B") -t1[2] # E: Tuple index out of range -t1[3] # E: Tuple index out of range -t2[1] # E: Tuple index out of range -t1[n] # E: Tuple index must be an integer literal -t3[n:] # E: Tuple slice must be an integer literal -b = t1[(0)] # E: Incompatible types in assignment (expression has type "A", variable has type "B") - -a = t1[0] -b = t1[1] -b = t1[-1] -a = t1[(0)] -x = t3[0:3] # type (A, B, C) -y = t3[0:5:2] # type (A, C, E) -x = t3[:-2] # type (A, B, C) - -class A: pass -class B: pass -class C: pass -class D: pass -class E: pass -[builtins fixtures/tuple.py] - -[case testIndexingTuplesWithNegativeIntegers] -from typing import Tuple -t1 = None # type: Tuple[A, B] -t2 = None # type: Tuple[A] -a, b = None, None # type: A, B - -a = t1[-1] # E: Incompatible types in assignment (expression has type "B", variable has type "A") -b = t1[-2] # E: Incompatible types in assignment (expression has type "A", variable has type "B") -t1[-3] # E: Tuple index out of range -t1[-4] # E: Tuple index out of range -b = t2[(-1)] # E: Incompatible types in assignment (expression has type "A", variable has type "B") - -a = t1[-2] -b = t1[-1] -a = t2[(-1)] - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - -[case testAssigningToTupleItems] -from typing import Tuple -t = None # type: Tuple[A, B] -n = 0 - -t[0] = A() # E: Unsupported target for indexed assignment -t[2] = A() # E: Unsupported target for indexed assignment -t[n] = A() # E: Unsupported target for indexed assignment - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - - --- Multiple assignment --- ------------------- - - -[case testMultipleAssignmentWithTuples] -from typing import Tuple -t1 = None # type: Tuple[A, B] -t2 = None # type: Tuple[A, B, A] -a, b = None, None # type: (A, B) - -a, a = t1 # E: Incompatible types in assignment (expression has type "B", variable has type "A") -b, b = t1 # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a, b, b = t2 # E: Incompatible types in assignment (expression has type "A", variable has type "B") - -a, b = t1 -a, b, a = t2 - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - -[case testMultipleAssignmentWithInvalidNumberOfValues] -from typing import Tuple -t1 = None # type: Tuple[A, A, A] -a = None # type: A - -a, a = t1 # E: Too many values to unpack (2 expected, 3 provided) -a, a, a, a = t1 # E: Need more than 3 values to unpack (4 expected) - -a, a, a = t1 - -class A: pass -[builtins fixtures/tuple.py] - -[case testMultipleAssignmentWithTupleExpressionRvalue] - -a, b = None, None # type: (A, B) - -a, b = a, a # Fail -a, b = b, a # Fail - -a, b = a, b -a, a = a, a - -class A: pass -class B: pass -[builtins fixtures/tuple.py] -[out] -main:4: error: Incompatible types in assignment (expression has type "A", variable has type "B") -main:5: error: Incompatible types in assignment (expression has type "B", variable has type "A") -main:5: error: Incompatible types in assignment (expression has type "A", variable has type "B") - -[case testSubtypingInMultipleAssignment] - -a, b = None, None # type: (A, B) - -b, b = a, b # E: Incompatible types in assignment (expression has type "A", variable has type "B") -b, b = b, a # E: Incompatible types in assignment (expression has type "A", variable has type "B") - -a, b = b, b -b, a = b, b - -class A: pass -class B(A): pass -[builtins fixtures/tuple.py] - -[case testInitializationWithMultipleValues] - -a, b = None, None # type: (A, B) - -a1, b1 = a, a # type: (A, B) # E: Incompatible types in assignment (expression has type "A", variable has type "B") -a2, b2 = b, b # type: (A, B) # E: Incompatible types in assignment (expression has type "B", variable has type "A") -a3, b3 = a # type: (A, B) # E: '__main__.A' object is not iterable -a4, b4 = None # type: (A, B) # E: 'builtins.None' object is not iterable -a5, b5 = a, b, a # type: (A, B) # E: Too many values to unpack (2 expected, 3 provided) - -ax, bx = a, b # type: (A, B) - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - -[case testMultipleAssignmentWithNonTupleRvalue] - -a, b = None, None # type: (A, B) -def f(): pass - -a, b = None # E: 'builtins.None' object is not iterable -a, b = a # E: '__main__.A' object is not iterable -a, b = f # E: 'def () -> Any' object is not iterable - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - -[case testMultipleAssignmentWithIndexedLvalues] - -a, b = None, None # type: (A, B) -aa, bb = None, None # type: (AA, BB) - -a[a], b[b] = a, bb # E: Incompatible types in assignment (expression has type "A", target has type "AA") -a[a], b[b] = aa, b # E: Incompatible types in assignment (expression has type "B", target has type "BB") -a[aa], b[b] = aa, bb # E: Invalid index type "AA" for "A" -a[a], b[bb] = aa, bb # E: Invalid index type "BB" for "B" -a[a], b[b] = aa, bb - -class A: - def __setitem__(self, x: 'A', y: 'AA') -> None: pass -class B: - def __setitem__(self, x: 'B', y: 'BB') -> None: pass - -class AA: pass -class BB: pass -[builtins fixtures/tuple.py] - -[case testMultipleDeclarationWithParentheses] - -(a, b) = (None, None) # type: int, str -a = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -b = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") -a = 1 -b = '' - -[case testMultipleAssignmentWithExtraParentheses] - -a, b = None, None # type: (A, B) - -(a, b) = (a, a) # E: Incompatible types in assignment (expression has type "A", variable has type "B") -(a, b) = (b, b) # E: Incompatible types in assignment (expression has type "B", variable has type "A") -((a), (b)) = ((a), (a)) # E: Incompatible types in assignment (expression has type "A", variable has type "B") -((a), (b)) = ((b), (b)) # E: Incompatible types in assignment (expression has type "B", variable has type "A") -[a, b] = a, a # E: Incompatible types in assignment (expression has type "A", variable has type "B") -[a, b] = b, b # E: Incompatible types in assignment (expression has type "B", variable has type "A") - -(a, b) = (a, b) -((a), (b)) = ((a), (b)) -[a, b] = a, b - -class A: pass -class B: pass -[builtins fixtures/tuple.py] - -[case testMultipleAssignmentUsingSingleTupleType] -from typing import Tuple -a, b = None, None # type: Tuple[int, str] -a = 1 -b = '' -a = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -b = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") - -[case testMultipleAssignmentWithMixedVariables] -a = b, c = 1, 1 -x, y = p, q = 1, 1 -u, v, w = r, s = 1, 1 # E: Need more than 2 values to unpack (3 expected) -d, e = f, g, h = 1, 1 # E: Need more than 2 values to unpack (3 expected) - - --- Assignment to starred expressions --- --------------------------------- - - -[case testAssignmentToStarMissingAnnotation] -from typing import List -t = 1, 2 -a, b, *c = 1, 2 # E: Need type annotation for variable -aa, bb, *cc = t # E: Need type annotation for variable -[builtins fixtures/list.py] - -[case testAssignmentToStarAnnotation] -from typing import List -li, lo = None, None # type: List[int], List[object] -a, b, *c = 1, 2 # type: int, int, *List[int] -c = lo # E: Incompatible types in assignment (expression has type List[object], variable has type List[int]) -c = li -[builtins fixtures/list.py] - -[case testAssignmentToStarCount1] -from typing import List -ca = None # type: List[int] -c = [1] -a, b, *c = 1, # E: Need more than 1 value to unpack (2 expected) -a, b, *c = 1, 2 -a, b, *c = 1, 2, 3 -a, b, *c = 1, 2, 3, 4 -[builtins fixtures/list.py] - -[case testAssignmentToStarCount2] -from typing import List -ca = None # type: List[int] -t1 = 1, -t2 = 1, 2 -t3 = 1, 2, 3 -t4 = 1, 2, 3, 4 -c = [1] -a, b, *c = t1 # E: Need more than 1 value to unpack (2 expected) -a, b, *c = t2 -a, b, *c = t3 -a, b, *c = t4 -[builtins fixtures/list.py] - -[case testAssignmentToStarFromAny] -from typing import Any -a, c = Any(1), C() -p, *q = a -c = a -c = q - -class C: pass - -[case testAssignmentToComplexStar] -from typing import List -li = None # type: List[int] -a, *(li) = 1, -a, *(b, c) = 1, 2 # E: Need more than 1 value to unpack (2 expected) -a, *(b, c) = 1, 2, 3 -a, *(b, c) = 1, 2, 3, 4 # E: Too many values to unpack (2 expected, 3 provided) -[builtins fixtures/list.py] - -[case testAssignmentToStarFromTupleType] -from typing import List, Tuple -li = None # type: List[int] -la = None # type: List[A] -ta = None # type: Tuple[A, A, A] -a, *la = ta -a, *li = ta # E -a, *na = ta -na = la -na = a # E - -class A: pass -[builtins fixtures/list.py] -[out] -main:6: error: List item 0 has incompatible type "A" -main:6: error: List item 1 has incompatible type "A" -main:9: error: Incompatible types in assignment (expression has type "A", variable has type List[A]) - -[case testAssignmentToStarFromTupleInference] -from typing import List -li = None # type: List[int] -la = None # type: List[A] -a, *l = A(), A() -l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) -l = la - -class A: pass -[builtins fixtures/list.py] -[out] - -[case testAssignmentToStarFromListInference] -from typing import List -li = None # type: List[int] -la = None # type: List[A] -a, *l = [A(), A()] -l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) -l = la - -class A: pass -[builtins fixtures/list.py] -[out] - -[case testAssignmentToStarFromTupleTypeInference] -from typing import List, Tuple -li = None # type: List[int] -la = None # type: List[A] -ta = None # type: Tuple[A, A, A] -a, *l = ta -l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) -l = la - -class A: pass -[builtins fixtures/list.py] -[out] - -[case testAssignmentToStarFromListTypeInference] -from typing import List -li = None # type: List[int] -la = None # type: List[A] -a, *l = la -l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) -l = la - -class A: pass -[builtins fixtures/list.py] -[out] - - --- Nested tuple assignment --- ---------------------------- - - -[case testNestedTupleAssignment1] - -a1, b1, c1 = None, None, None # type: (A, B, C) -a2, b2, c2 = None, None, None # type: (A, B, C) - -a1, (b1, c1) = a2, (b2, c2) -a1, (a1, (b1, c1)) = a2, (a2, (b2, c2)) -a1, (a1, (a1, b1)) = a1, (a1, (a1, c1)) # Fail - -class A: pass -class B: pass -class C: pass -[out] -main:7: error: Incompatible types in assignment (expression has type "C", variable has type "B") - -[case testNestedTupleAssignment2] - -a1, b1, c1 = None, None, None # type: (A, B, C) -a2, b2, c2 = None, None, None # type: (A, B, C) -t = a1, b1 - -a2, b2 = t -(a2, b2), c2 = t, c1 -(a2, c2), c2 = t, c1 # Fail -t, c2 = (a2, b2), c2 -t, c2 = (a2, a2), c2 # Fail -t = a1, a1, a1 # Fail -t = a1 # Fail -a2, a2, a2 = t # Fail -a2, = t # Fail -a2 = t # Fail - -class A: pass -class B: pass -class C: pass -[out] -main:8: error: Incompatible types in assignment (expression has type "B", variable has type "C") -main:10: error: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A, B]") -main:11: error: Incompatible types in assignment (expression has type "Tuple[A, A, A]", variable has type "Tuple[A, B]") -main:12: error: Incompatible types in assignment (expression has type "A", variable has type "Tuple[A, B]") -main:13: error: Need more than 2 values to unpack (3 expected) -main:14: error: Too many values to unpack (1 expected, 2 provided) -main:15: error: Incompatible types in assignment (expression has type "Tuple[A, B]", variable has type "A") - - --- Error messages --- -------------- - - -[case testTupleErrorMessages] - -a = None # type: A - -(a, a) + a # E: Unsupported left operand type for + (Tuple[A, ...]) -a + (a, a) # E: Unsupported operand types for + ("A" and "Tuple[A, A]") -f((a, a)) # E: Argument 1 to "f" has incompatible type "Tuple[A, A]"; expected "A" -(a, a).foo # E: Tuple[A, ...] has no attribute "foo" - -def f(x: 'A') -> None: pass - -class A: - def __add__(self, x: 'A') -> 'A': pass -[builtins fixtures/tuple.py] - -[case testLargeTuplesInErrorMessages] - -a = None # type: LongTypeName -a + (a, a, a, a, a, a, a) # Fail - -class LongTypeName: - def __add__(self, x: 'LongTypeName') -> 'LongTypeName': pass -[builtins fixtures/tuple.py] -[out] -main:3: error: Unsupported operand types for + ("LongTypeName" and tuple(length 7)) - - --- Tuple methods --- ------------- - - -[case testTupleMethods] -from typing import Tuple -t = None # type: Tuple[int, str] -i = 0 -s = '' -b = bool() - -s = t.__len__() # E: Incompatible types in assignment (expression has type "int", variable has type "str") -i = t.__str__() # E: Incompatible types in assignment (expression has type "str", variable has type "int") -i = s in t # E: Incompatible types in assignment (expression has type "bool", variable has type "int") -t.foo # E: Tuple[Any, ...] has no attribute "foo" - -i = t.__len__() -s = t.__str__() -b = s in t - -[file builtins.py] -from typing import TypeVar, Generic -_T = TypeVar('_T') -class object: - def __init__(self) -> None: pass -class tuple(Generic[_T]): - def __len__(self) -> int: pass - def __str__(self) -> str: pass - def __contains__(self, o: object) -> bool: pass -class int: pass -class str: pass -class bool: pass -class type: pass -class function: pass - - --- For loop over tuple --- ------------------- - - -[case testForLoopOverTuple] -import typing -t = 1, 2 -for x in t: - x = 1 - x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") -[builtins fixtures/for.py] - -[case testForLoopOverEmptyTuple] -import typing -t = () -for x in t: pass # E: Need type annotation for variable -[builtins fixtures/for.py] - -[case testForLoopOverNoneValuedTuple] -import typing -t = () -for x in None, None: pass # E: Need type annotation for variable -[builtins fixtures/for.py] - -[case testForLoopOverTupleAndSubtyping] -import typing -class A: pass -class B(A): pass -for x in B(), A(): - x = A() - x = B() - x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "A") -[builtins fixtures/for.py] - -[case testTupleIterable] -y = 'a' -x = sum((1,2)) -y = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") -[builtins fixtures/tuple.py] - - --- Tuple as a base type --- -------------------- - - -[case testTupleBaseClass] -import m -[file m.pyi] -from typing import Tuple -class A(Tuple[int, str]): - def f(self, x: int) -> None: - a, b = 1, '' - a, b = self - b, a = self # Error - self.f('') # Error -[builtins fixtures/tuple.py] -[out] -main:1: note: In module imported here: -tmp/m.pyi: note: In member "f" of class "A": -tmp/m.pyi:6: error: Incompatible types in assignment (expression has type "int", variable has type "str") -tmp/m.pyi:6: error: Incompatible types in assignment (expression has type "str", variable has type "int") -tmp/m.pyi:7: error: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" - -[case testInvalidTupleBaseClass] -from typing import Tuple -class A(Tuple[int, str]): pass # E: Tuple[...] not supported as a base class outside a stub file -[builtins fixtures/tuple.py] -[out] - -[case testValidTupleBaseClass] -from typing import Tuple -class A(tuple): pass -[out] - -[case testTupleBaseClass2-skip] -import m -[file m.pyi] -# This doesn't work correctly -- no errors are reported (#867) -from typing import Tuple -a = None # type: A -class A(Tuple[int, str]): pass -x, y = a -x() # Expected: "int" not callable -y() # Expected: "str" not callable -[out] -(should fail) - -[case testGenericClassWithTupleBaseClass] -from typing import TypeVar, Generic, Tuple -T = TypeVar('T') -class Test(Generic[T], Tuple[T]): pass -x = Test() # type: Test[int] -[builtins fixtures/tuple.py] -[out] -main:3: error: Tuple[...] not supported as a base class outside a stub file -main:4: error: Generic tuple types not supported - - --- Variable-length tuples (Tuple[t, ...] with literal '...') --- --------------------------------------------------------- - - -[case testIndexingVariableLengthTuple] -from typing import Tuple -x = () # type: Tuple[str, ...] -n = 5 -x[n]() # E: "str" not callable -x[3]() # E: "str" not callable -[builtins fixtures/tuple.py] - -[case testSubtypingVariableLengthTuple] -from typing import Tuple -class A: pass -class B(A): pass -def fa(t: Tuple[A, ...]) -> None: pass -def fb(t: Tuple[B, ...]) -> None: pass -ta = () # type: Tuple[A, ...] -tb = () # type: Tuple[B, ...] -fa(ta) -fa(tb) -fb(tb) -fb(ta) # E: Argument 1 to "fb" has incompatible type Tuple[A, ...]; expected Tuple[B, ...] -[builtins fixtures/tuple.py] - -[case testSubtypingFixedAndVariableLengthTuples] -from typing import Tuple -class A: pass -class B(A): pass -def fa(t: Tuple[A, ...]) -> None: pass -def fb(t: Tuple[B, ...]) -> None: pass -aa = (A(), A()) -ab = (A(), B()) -bb = (B(), B()) -fa(aa) -fa(ab) -fa(bb) -fb(bb) -fb(ab) # E: Argument 1 to "fb" has incompatible type "Tuple[A, B]"; expected Tuple[B, ...] -fb(aa) # E: Argument 1 to "fb" has incompatible type "Tuple[A, A]"; expected Tuple[B, ...] -[builtins fixtures/tuple.py] - -[case testSubtypingTupleIsContainer] -from typing import Container -a = None # type: Container[str] -a = () - -[case testSubtypingTupleIsSized] -from typing import Sized -a = None # type: Sized -a = () +-- Normal assignment and subtyping +-- ------------------------------- + + +[case testTupleAssignmentWithTupleTypes] +from typing import Tuple +t1 = None # type: Tuple[A] +t2 = None # type: Tuple[B] +t3 = None # type: Tuple[A, A] +t4 = None # type: Tuple[A, B] +t5 = None # type: Tuple[B, A] + +t1 = t2 # E: Incompatible types in assignment (expression has type "Tuple[B]", variable has type "Tuple[A]") +t1 = t3 # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A]") +t3 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A]", variable has type "Tuple[A, A]") +t3 = t4 # E: Incompatible types in assignment (expression has type "Tuple[A, B]", variable has type "Tuple[A, A]") +t3 = t5 # E: Incompatible types in assignment (expression has type "Tuple[B, A]", variable has type "Tuple[A, A]") + +# Ok +t1 = t1 +t2 = t2 +t3 = t3 +t4 = t4 +t5 = t5 + +class A: pass +class B: pass +[builtins fixtures/tuple.py] + +[case testTupleSubtyping] +from typing import Tuple +t1 = None # type: Tuple[A, A] +t2 = None # type: Tuple[A, B] +t3 = None # type: Tuple[B, A] + +t2 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A, B]") +t2 = t3 # E: Incompatible types in assignment (expression has type "Tuple[B, A]", variable has type "Tuple[A, B]") +t3 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[B, A]") +t3 = t2 # E: Incompatible types in assignment (expression has type "Tuple[A, B]", variable has type "Tuple[B, A]") + +t1 = t2 +t1 = t3 + +class A: pass +class B(A): pass +[builtins fixtures/tuple.py] + +[case testTupleCompatibilityWithOtherTypes] +from typing import Tuple +a, o = None, None # type: (A, object) +t = None # type: Tuple[A, A] + +a = t # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "A") +t = o # E: Incompatible types in assignment (expression has type "object", variable has type "Tuple[A, A]") +t = a # E: Incompatible types in assignment (expression has type "A", variable has type "Tuple[A, A]") +# TODO: callable types + tuples + +# Ok +o = t +t = None + +class A: pass +[builtins fixtures/tuple.py] + +[case testNestedTupleTypes] +from typing import Tuple +t1 = None # type: Tuple[A, Tuple[A, A]] +t2 = None # type: Tuple[B, Tuple[B, B]] + +t2 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, Tuple[A, A]]", variable has type "Tuple[B, Tuple[B, B]]") +t1 = t2 + +class A: pass +class B(A): pass +[builtins fixtures/tuple.py] + +[case testNestedTupleTypes2] +from typing import Tuple +t1 = None # type: Tuple[A, Tuple[A, A]] +t2 = None # type: Tuple[B, Tuple[B, B]] + +t2 = t1 # E: Incompatible types in assignment (expression has type "Tuple[A, Tuple[A, A]]", variable has type "Tuple[B, Tuple[B, B]]") +t1 = t2 + +class A: pass +class B(A): pass +[builtins fixtures/tuple.py] + +[case testSubtypingWithNamedTupleType] +from typing import Tuple +t1 = None # type: Tuple[A, A] +t2 = None # type: tuple + +t1 = t2 # E: Incompatible types in assignment (expression has type Tuple[Any, ...], variable has type "Tuple[A, A]") +t2 = t1 + +class A: pass +[builtins fixtures/tuple.py] + +[case testTupleInitializationWithNone] +from typing import Tuple +t = None # type: Tuple[A, A] +t = None +class A: pass +[builtins fixtures/tuple.py] + + +-- Tuple expressions +-- ----------------- + + +[case testTupleExpressions] +from typing import Tuple +t1 = None # type: tuple +t2 = None # type: Tuple[A] +t3 = None # type: Tuple[A, B] + +a, b, c = None, None, None # type: (A, B, C) + +t2 = () # E: Incompatible types in assignment (expression has type "Tuple[]", variable has type "Tuple[A]") +t2 = (a, a) # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A]") +t3 = (a, a) # E: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A, B]") +t3 = (b, b) # E: Incompatible types in assignment (expression has type "Tuple[B, B]", variable has type "Tuple[A, B]") +t3 = (a, b, a) # E: Incompatible types in assignment (expression has type "Tuple[A, B, A]", variable has type "Tuple[A, B]") + +t1 = () +t1 = (a,) +t2 = (a,) +t3 = (a, b) +t3 = (a, c) +t3 = (None, None) + +class A: pass +class B: pass +class C(B): pass +[builtins fixtures/tuple.py] + +[case testVoidValueInTuple] +import typing +(None, f()) # E: "f" does not return a value +(f(), None) # E: "f" does not return a value + +def f() -> None: pass +[builtins fixtures/tuple.py] + + +-- Indexing +-- -------- + + +[case testIndexingTuples] +from typing import Tuple +t1 = None # type: Tuple[A, B] +t2 = None # type: Tuple[A] +t3 = None # type: Tuple[A, B, C, D, E] +a, b = None, None # type: (A, B) +x = None # type: Tuple[A, B, C] +y = None # type: Tuple[A, C, E] +n = 0 + +a = t1[1] # E: Incompatible types in assignment (expression has type "B", variable has type "A") +b = t1[0] # E: Incompatible types in assignment (expression has type "A", variable has type "B") +t1[2] # E: Tuple index out of range +t1[3] # E: Tuple index out of range +t2[1] # E: Tuple index out of range +t1[n] # E: Tuple index must be an integer literal +t3[n:] # E: Tuple slice must be an integer literal +b = t1[(0)] # E: Incompatible types in assignment (expression has type "A", variable has type "B") + +a = t1[0] +b = t1[1] +b = t1[-1] +a = t1[(0)] +x = t3[0:3] # type (A, B, C) +y = t3[0:5:2] # type (A, C, E) +x = t3[:-2] # type (A, B, C) + +class A: pass +class B: pass +class C: pass +class D: pass +class E: pass +[builtins fixtures/tuple.py] + +[case testIndexingTuplesWithNegativeIntegers] +from typing import Tuple +t1 = None # type: Tuple[A, B] +t2 = None # type: Tuple[A] +a, b = None, None # type: A, B + +a = t1[-1] # E: Incompatible types in assignment (expression has type "B", variable has type "A") +b = t1[-2] # E: Incompatible types in assignment (expression has type "A", variable has type "B") +t1[-3] # E: Tuple index out of range +t1[-4] # E: Tuple index out of range +b = t2[(-1)] # E: Incompatible types in assignment (expression has type "A", variable has type "B") + +a = t1[-2] +b = t1[-1] +a = t2[(-1)] + +class A: pass +class B: pass +[builtins fixtures/tuple.py] + +[case testAssigningToTupleItems] +from typing import Tuple +t = None # type: Tuple[A, B] +n = 0 + +t[0] = A() # E: Unsupported target for indexed assignment +t[2] = A() # E: Unsupported target for indexed assignment +t[n] = A() # E: Unsupported target for indexed assignment + +class A: pass +class B: pass +[builtins fixtures/tuple.py] + + +-- Multiple assignment +-- ------------------- + + +[case testMultipleAssignmentWithTuples] +from typing import Tuple +t1 = None # type: Tuple[A, B] +t2 = None # type: Tuple[A, B, A] +a, b = None, None # type: (A, B) + +a, a = t1 # E: Incompatible types in assignment (expression has type "B", variable has type "A") +b, b = t1 # E: Incompatible types in assignment (expression has type "A", variable has type "B") +a, b, b = t2 # E: Incompatible types in assignment (expression has type "A", variable has type "B") + +a, b = t1 +a, b, a = t2 + +class A: pass +class B: pass +[builtins fixtures/tuple.py] + +[case testMultipleAssignmentWithInvalidNumberOfValues] +from typing import Tuple +t1 = None # type: Tuple[A, A, A] +a = None # type: A + +a, a = t1 # E: Too many values to unpack (2 expected, 3 provided) +a, a, a, a = t1 # E: Need more than 3 values to unpack (4 expected) + +a, a, a = t1 + +class A: pass +[builtins fixtures/tuple.py] + +[case testMultipleAssignmentWithTupleExpressionRvalue] + +a, b = None, None # type: (A, B) + +a, b = a, a # Fail +a, b = b, a # Fail + +a, b = a, b +a, a = a, a + +class A: pass +class B: pass +[builtins fixtures/tuple.py] +[out] +main:4: error: Incompatible types in assignment (expression has type "A", variable has type "B") +main:5: error: Incompatible types in assignment (expression has type "B", variable has type "A") +main:5: error: Incompatible types in assignment (expression has type "A", variable has type "B") + +[case testSubtypingInMultipleAssignment] + +a, b = None, None # type: (A, B) + +b, b = a, b # E: Incompatible types in assignment (expression has type "A", variable has type "B") +b, b = b, a # E: Incompatible types in assignment (expression has type "A", variable has type "B") + +a, b = b, b +b, a = b, b + +class A: pass +class B(A): pass +[builtins fixtures/tuple.py] + +[case testInitializationWithMultipleValues] + +a, b = None, None # type: (A, B) + +a1, b1 = a, a # type: (A, B) # E: Incompatible types in assignment (expression has type "A", variable has type "B") +a2, b2 = b, b # type: (A, B) # E: Incompatible types in assignment (expression has type "B", variable has type "A") +a3, b3 = a # type: (A, B) # E: '__main__.A' object is not iterable +a4, b4 = None # type: (A, B) # E: 'builtins.None' object is not iterable +a5, b5 = a, b, a # type: (A, B) # E: Too many values to unpack (2 expected, 3 provided) + +ax, bx = a, b # type: (A, B) + +class A: pass +class B: pass +[builtins fixtures/tuple.py] + +[case testMultipleAssignmentWithNonTupleRvalue] + +a, b = None, None # type: (A, B) +def f(): pass + +a, b = None # E: 'builtins.None' object is not iterable +a, b = a # E: '__main__.A' object is not iterable +a, b = f # E: 'def () -> Any' object is not iterable + +class A: pass +class B: pass +[builtins fixtures/tuple.py] + +[case testMultipleAssignmentWithIndexedLvalues] + +a, b = None, None # type: (A, B) +aa, bb = None, None # type: (AA, BB) + +a[a], b[b] = a, bb # E: Incompatible types in assignment (expression has type "A", target has type "AA") +a[a], b[b] = aa, b # E: Incompatible types in assignment (expression has type "B", target has type "BB") +a[aa], b[b] = aa, bb # E: Invalid index type "AA" for "A" +a[a], b[bb] = aa, bb # E: Invalid index type "BB" for "B" +a[a], b[b] = aa, bb + +class A: + def __setitem__(self, x: 'A', y: 'AA') -> None: pass +class B: + def __setitem__(self, x: 'B', y: 'BB') -> None: pass + +class AA: pass +class BB: pass +[builtins fixtures/tuple.py] + +[case testMultipleDeclarationWithParentheses] + +(a, b) = (None, None) # type: int, str +a = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +b = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") +a = 1 +b = '' + +[case testMultipleAssignmentWithExtraParentheses] + +a, b = None, None # type: (A, B) + +(a, b) = (a, a) # E: Incompatible types in assignment (expression has type "A", variable has type "B") +(a, b) = (b, b) # E: Incompatible types in assignment (expression has type "B", variable has type "A") +((a), (b)) = ((a), (a)) # E: Incompatible types in assignment (expression has type "A", variable has type "B") +((a), (b)) = ((b), (b)) # E: Incompatible types in assignment (expression has type "B", variable has type "A") +[a, b] = a, a # E: Incompatible types in assignment (expression has type "A", variable has type "B") +[a, b] = b, b # E: Incompatible types in assignment (expression has type "B", variable has type "A") + +(a, b) = (a, b) +((a), (b)) = ((a), (b)) +[a, b] = a, b + +class A: pass +class B: pass +[builtins fixtures/tuple.py] + +[case testMultipleAssignmentUsingSingleTupleType] +from typing import Tuple +a, b = None, None # type: Tuple[int, str] +a = 1 +b = '' +a = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +b = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testMultipleAssignmentWithMixedVariables] +a = b, c = 1, 1 +x, y = p, q = 1, 1 +u, v, w = r, s = 1, 1 # E: Need more than 2 values to unpack (3 expected) +d, e = f, g, h = 1, 1 # E: Need more than 2 values to unpack (3 expected) + + +-- Assignment to starred expressions +-- --------------------------------- + + +[case testAssignmentToStarMissingAnnotation] +from typing import List +t = 1, 2 +a, b, *c = 1, 2 # E: Need type annotation for variable +aa, bb, *cc = t # E: Need type annotation for variable +[builtins fixtures/list.py] + +[case testAssignmentToStarAnnotation] +from typing import List +li, lo = None, None # type: List[int], List[object] +a, b, *c = 1, 2 # type: int, int, *List[int] +c = lo # E: Incompatible types in assignment (expression has type List[object], variable has type List[int]) +c = li +[builtins fixtures/list.py] + +[case testAssignmentToStarCount1] +from typing import List +ca = None # type: List[int] +c = [1] +a, b, *c = 1, # E: Need more than 1 value to unpack (2 expected) +a, b, *c = 1, 2 +a, b, *c = 1, 2, 3 +a, b, *c = 1, 2, 3, 4 +[builtins fixtures/list.py] + +[case testAssignmentToStarCount2] +from typing import List +ca = None # type: List[int] +t1 = 1, +t2 = 1, 2 +t3 = 1, 2, 3 +t4 = 1, 2, 3, 4 +c = [1] +a, b, *c = t1 # E: Need more than 1 value to unpack (2 expected) +a, b, *c = t2 +a, b, *c = t3 +a, b, *c = t4 +[builtins fixtures/list.py] + +[case testAssignmentToStarFromAny] +from typing import Any +a, c = Any(1), C() +p, *q = a +c = a +c = q + +class C: pass + +[case testAssignmentToComplexStar] +from typing import List +li = None # type: List[int] +a, *(li) = 1, +a, *(b, c) = 1, 2 # E: Need more than 1 value to unpack (2 expected) +a, *(b, c) = 1, 2, 3 +a, *(b, c) = 1, 2, 3, 4 # E: Too many values to unpack (2 expected, 3 provided) +[builtins fixtures/list.py] + +[case testAssignmentToStarFromTupleType] +from typing import List, Tuple +li = None # type: List[int] +la = None # type: List[A] +ta = None # type: Tuple[A, A, A] +a, *la = ta +a, *li = ta # E +a, *na = ta +na = la +na = a # E + +class A: pass +[builtins fixtures/list.py] +[out] +main:6: error: List item 0 has incompatible type "A" +main:6: error: List item 1 has incompatible type "A" +main:9: error: Incompatible types in assignment (expression has type "A", variable has type List[A]) + +[case testAssignmentToStarFromTupleInference] +from typing import List +li = None # type: List[int] +la = None # type: List[A] +a, *l = A(), A() +l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) +l = la + +class A: pass +[builtins fixtures/list.py] +[out] + +[case testAssignmentToStarFromListInference] +from typing import List +li = None # type: List[int] +la = None # type: List[A] +a, *l = [A(), A()] +l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) +l = la + +class A: pass +[builtins fixtures/list.py] +[out] + +[case testAssignmentToStarFromTupleTypeInference] +from typing import List, Tuple +li = None # type: List[int] +la = None # type: List[A] +ta = None # type: Tuple[A, A, A] +a, *l = ta +l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) +l = la + +class A: pass +[builtins fixtures/list.py] +[out] + +[case testAssignmentToStarFromListTypeInference] +from typing import List +li = None # type: List[int] +la = None # type: List[A] +a, *l = la +l = li # E: Incompatible types in assignment (expression has type List[int], variable has type List[A]) +l = la + +class A: pass +[builtins fixtures/list.py] +[out] + + +-- Nested tuple assignment +-- ---------------------------- + + +[case testNestedTupleAssignment1] + +a1, b1, c1 = None, None, None # type: (A, B, C) +a2, b2, c2 = None, None, None # type: (A, B, C) + +a1, (b1, c1) = a2, (b2, c2) +a1, (a1, (b1, c1)) = a2, (a2, (b2, c2)) +a1, (a1, (a1, b1)) = a1, (a1, (a1, c1)) # Fail + +class A: pass +class B: pass +class C: pass +[out] +main:7: error: Incompatible types in assignment (expression has type "C", variable has type "B") + +[case testNestedTupleAssignment2] + +a1, b1, c1 = None, None, None # type: (A, B, C) +a2, b2, c2 = None, None, None # type: (A, B, C) +t = a1, b1 + +a2, b2 = t +(a2, b2), c2 = t, c1 +(a2, c2), c2 = t, c1 # Fail +t, c2 = (a2, b2), c2 +t, c2 = (a2, a2), c2 # Fail +t = a1, a1, a1 # Fail +t = a1 # Fail +a2, a2, a2 = t # Fail +a2, = t # Fail +a2 = t # Fail + +class A: pass +class B: pass +class C: pass +[out] +main:8: error: Incompatible types in assignment (expression has type "B", variable has type "C") +main:10: error: Incompatible types in assignment (expression has type "Tuple[A, A]", variable has type "Tuple[A, B]") +main:11: error: Incompatible types in assignment (expression has type "Tuple[A, A, A]", variable has type "Tuple[A, B]") +main:12: error: Incompatible types in assignment (expression has type "A", variable has type "Tuple[A, B]") +main:13: error: Need more than 2 values to unpack (3 expected) +main:14: error: Too many values to unpack (1 expected, 2 provided) +main:15: error: Incompatible types in assignment (expression has type "Tuple[A, B]", variable has type "A") + + +-- Error messages +-- -------------- + + +[case testTupleErrorMessages] + +a = None # type: A + +(a, a) + a # E: Unsupported left operand type for + (Tuple[A, ...]) +a + (a, a) # E: Unsupported operand types for + ("A" and "Tuple[A, A]") +f((a, a)) # E: Argument 1 to "f" has incompatible type "Tuple[A, A]"; expected "A" +(a, a).foo # E: Tuple[A, ...] has no attribute "foo" + +def f(x: 'A') -> None: pass + +class A: + def __add__(self, x: 'A') -> 'A': pass +[builtins fixtures/tuple.py] + +[case testLargeTuplesInErrorMessages] + +a = None # type: LongTypeName +a + (a, a, a, a, a, a, a) # Fail + +class LongTypeName: + def __add__(self, x: 'LongTypeName') -> 'LongTypeName': pass +[builtins fixtures/tuple.py] +[out] +main:3: error: Unsupported operand types for + ("LongTypeName" and tuple(length 7)) + + +-- Tuple methods +-- ------------- + + +[case testTupleMethods] +from typing import Tuple +t = None # type: Tuple[int, str] +i = 0 +s = '' +b = bool() + +s = t.__len__() # E: Incompatible types in assignment (expression has type "int", variable has type "str") +i = t.__str__() # E: Incompatible types in assignment (expression has type "str", variable has type "int") +i = s in t # E: Incompatible types in assignment (expression has type "bool", variable has type "int") +t.foo # E: Tuple[Any, ...] has no attribute "foo" + +i = t.__len__() +s = t.__str__() +b = s in t + +[file builtins.py] +from typing import TypeVar, Generic +_T = TypeVar('_T') +class object: + def __init__(self) -> None: pass +class tuple(Generic[_T]): + def __len__(self) -> int: pass + def __str__(self) -> str: pass + def __contains__(self, o: object) -> bool: pass +class int: pass +class str: pass +class bool: pass +class type: pass +class function: pass + + +-- For loop over tuple +-- ------------------- + + +[case testForLoopOverTuple] +import typing +t = 1, 2 +for x in t: + x = 1 + x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +[builtins fixtures/for.py] + +[case testForLoopOverEmptyTuple] +import typing +t = () +for x in t: pass # E: Need type annotation for variable +[builtins fixtures/for.py] + +[case testForLoopOverNoneValuedTuple] +import typing +t = () +for x in None, None: pass # E: Need type annotation for variable +[builtins fixtures/for.py] + +[case testForLoopOverTupleAndSubtyping] +import typing +class A: pass +class B(A): pass +for x in B(), A(): + x = A() + x = B() + x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "A") +[builtins fixtures/for.py] + +[case testTupleIterable] +y = 'a' +x = sum((1,2)) +y = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") +[builtins fixtures/tuple.py] + + +-- Tuple as a base type +-- -------------------- + + +[case testTupleBaseClass] +import m +[file m.pyi] +from typing import Tuple +class A(Tuple[int, str]): + def f(self, x: int) -> None: + a, b = 1, '' + a, b = self + b, a = self # Error + self.f('') # Error +[builtins fixtures/tuple.py] +[out] +main:1: note: In module imported here: +tmp/m.pyi: note: In member "f" of class "A": +tmp/m.pyi:6: error: Incompatible types in assignment (expression has type "int", variable has type "str") +tmp/m.pyi:6: error: Incompatible types in assignment (expression has type "str", variable has type "int") +tmp/m.pyi:7: error: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" + +[case testInvalidTupleBaseClass] +from typing import Tuple +class A(Tuple[int, str]): pass # E: Tuple[...] not supported as a base class outside a stub file +[builtins fixtures/tuple.py] +[out] + +[case testValidTupleBaseClass] +from typing import Tuple +class A(tuple): pass +[out] + +[case testTupleBaseClass2-skip] +import m +[file m.pyi] +# This doesn't work correctly -- no errors are reported (#867) +from typing import Tuple +a = None # type: A +class A(Tuple[int, str]): pass +x, y = a +x() # Expected: "int" not callable +y() # Expected: "str" not callable +[out] +(should fail) + +[case testGenericClassWithTupleBaseClass] +from typing import TypeVar, Generic, Tuple +T = TypeVar('T') +class Test(Generic[T], Tuple[T]): pass +x = Test() # type: Test[int] +[builtins fixtures/tuple.py] +[out] +main:3: error: Tuple[...] not supported as a base class outside a stub file +main:4: error: Generic tuple types not supported + + +-- Variable-length tuples (Tuple[t, ...] with literal '...') +-- --------------------------------------------------------- + + +[case testIndexingVariableLengthTuple] +from typing import Tuple +x = () # type: Tuple[str, ...] +n = 5 +x[n]() # E: "str" not callable +x[3]() # E: "str" not callable +[builtins fixtures/tuple.py] + +[case testSubtypingVariableLengthTuple] +from typing import Tuple +class A: pass +class B(A): pass +def fa(t: Tuple[A, ...]) -> None: pass +def fb(t: Tuple[B, ...]) -> None: pass +ta = () # type: Tuple[A, ...] +tb = () # type: Tuple[B, ...] +fa(ta) +fa(tb) +fb(tb) +fb(ta) # E: Argument 1 to "fb" has incompatible type Tuple[A, ...]; expected Tuple[B, ...] +[builtins fixtures/tuple.py] + +[case testSubtypingFixedAndVariableLengthTuples] +from typing import Tuple +class A: pass +class B(A): pass +def fa(t: Tuple[A, ...]) -> None: pass +def fb(t: Tuple[B, ...]) -> None: pass +aa = (A(), A()) +ab = (A(), B()) +bb = (B(), B()) +fa(aa) +fa(ab) +fa(bb) +fb(bb) +fb(ab) # E: Argument 1 to "fb" has incompatible type "Tuple[A, B]"; expected Tuple[B, ...] +fb(aa) # E: Argument 1 to "fb" has incompatible type "Tuple[A, A]"; expected Tuple[B, ...] +[builtins fixtures/tuple.py] + +[case testSubtypingTupleIsContainer] +from typing import Container +a = None # type: Container[str] +a = () + +[case testSubtypingTupleIsSized] +from typing import Sized +a = None # type: Sized +a = () From 8f61c13f85ce39e187d984e56e648e582463a307 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Sun, 14 Aug 2016 19:22:01 +0300 Subject: [PATCH 51/74] flake8-ing --- mypy/join.py | 2 +- mypy/messages.py | 1 - mypy/nodes.py | 3 ++- mypy/semanal.py | 8 ++++---- mypy/subtypes.py | 4 ++-- mypy/typeanal.py | 18 ++++++++---------- mypy/types.py | 10 ++++------ 7 files changed, 21 insertions(+), 25 deletions(-) diff --git a/mypy/join.py b/mypy/join.py index 684d62cfeaebd..061052e6d87f8 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -217,7 +217,7 @@ def visit_tuple_type(self, t: TupleType) -> Type: for i in range(t.length()): items.append(self.join(t.items[i], self.s.items[i])) if (not (isinstance(t, NamedTupleType) and isinstance(self.s, NamedTupleType)) - or self.s.attrs == t.attrs): + or self.s.attrs == t.attrs): return self.s.with_types(items) # TODO: What if the fallback types are different? return TupleType(items, t.fallback) diff --git a/mypy/messages.py b/mypy/messages.py index 5d82c4845782a..d8aed9626a49d 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -487,7 +487,6 @@ def incompatible_argument(self, n: int, m: int, callee: CallableType, arg_type: except IndexError: # Varargs callees expected_type = callee.arg_types[-1] arg_type_str, expected_type_str = self.format_distinctly(arg_type, expected_type) - #import pdb; pdb.set_trace() if arg_kind == ARG_STAR: arg_type_str = '*' + arg_type_str elif arg_kind == ARG_STAR2: diff --git a/mypy/nodes.py b/mypy/nodes.py index 60d7b5d18944f..3e4f3f4fda129 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1179,7 +1179,7 @@ def __init__(self, callee: Expression, args: List[Expression], arg_kinds: List[i arg_names: List[str] = None, analyzed: Expression = None) -> None: if not arg_names: arg_names = [None] * len(args) - + self.callee = callee self.args = args self.arg_kinds = arg_kinds @@ -1985,6 +1985,7 @@ def NamedTupleTypeInfo(tup: 'mypy.types.NamedTupleType', *args) -> TypeInfo: info.mro = [info] + tup.fallback.type.mro return info + class SymbolTableNode: # Kind of node. Possible values: # - LDEF: local definition (of any kind) diff --git a/mypy/semanal.py b/mypy/semanal.py index 0f4331499d7cc..d7fe54678b70d 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1541,15 +1541,15 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], this_type = self_type(info) # Add __init__, _replace, _asdict methods and _fields static field self.add_namedtuple_method('_replace', symbols, info, this_type, this_type, - self.make_factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) + self.factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) self.add_namedtuple_method('__init__', symbols, info, this_type, NoneTyp(), - self.make_factory_args(vars, ARG_POS), info.name()) + self.factory_args(vars, ARG_POS), info.name()) # TODO: refine to OrderedDict[Union[types]] self.add_namedtuple_method('_asdict', symbols, info, this_type, AnyType(), []) return info - def make_factory_args(self, vars: List[Var], kind: int, - initializer: Expression = None) -> List[Argument]: + def factory_args(self, vars: List[Var], kind: int, + initializer: Expression = None) -> List[Argument]: return [Argument(var, var.type, initializer, kind) for var in vars] def add_namedtuple_method(self, funcname: str, symbols: SymbolTable, info: TypeInfo, diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 6872e297c3a64..32b44c36e89f4 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -2,8 +2,8 @@ from mypy.types import ( Type, AnyType, UnboundType, TypeVisitor, ErrorType, Void, NoneTyp, - Instance, TypeVarType, CallableType, TupleType, NamedTupleType, UnionType, Overloaded, ErasedType, TypeList, - PartialType, DeletedType, UninhabitedType, TypeType, is_named_instance + Instance, TypeVarType, CallableType, TupleType, NamedTupleType, UnionType, Overloaded, + ErasedType, TypeList, PartialType, DeletedType, UninhabitedType, TypeType, is_named_instance ) import mypy.applytype import mypy.constraints diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 1841f283a19e1..092849567c425 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -1,15 +1,15 @@ """Semantic analysis of types""" -from typing import Callable, cast, List, Tuple +from typing import Callable, cast, List from mypy.types import ( - Type, UnboundType, TypeVarType, TupleType, NamedTupleType, UnionType, Instance, AnyType, CallableType, - Void, NoneTyp, DeletedType, TypeList, TypeVarDef, TypeVisitor, StarType, PartialType, - EllipsisType, UninhabitedType, TypeType + Type, UnboundType, TypeVarType, TupleType, NamedTupleType, UnionType, Instance, + AnyType, CallableType, Void, NoneTyp, DeletedType, TypeList, TypeVarDef, TypeVisitor, + StarType, PartialType, EllipsisType, UninhabitedType, TypeType ) from mypy.nodes import ( BOUND_TVAR, TYPE_ALIAS, UNBOUND_IMPORTED, - TypeInfo, Context, SymbolTableNode, TypeVarExpr, Var, Node, + TypeInfo, Context, SymbolTableNode, Var, Node, IndexExpr, RefExpr ) from mypy.sametypes import is_same_type @@ -171,7 +171,7 @@ def visit_unbound_type(self, t: UnboundType) -> Type: return AnyType() # import pdb; pdb.set_trace() if info.is_named_tuple: - return NamedTupleType(info.name(), info.attrs, + return NamedTupleType(info.name(), info.attrs, self.anal_array(info.tuple_type.items), fallback=instance, line=t.line) @@ -224,10 +224,8 @@ def visit_tuple_type(self, t: TupleType) -> Type: return TupleType(self.anal_array(t.items), fallback, t.line) def visit_namedtuple_type(self, t: NamedTupleType) -> Type: - return NamedTupleType(t.name, t.attrs, - self.anal_array(t.items), - fallback=t.fallback, - line=t.line) + return NamedTupleType(t.name, t.attrs, self.anal_array(t.items), + fallback=t.fallback, line=t.line) def visit_star_type(self, t: StarType) -> Type: return StarType(t.type.accept(self), t.line) diff --git a/mypy/types.py b/mypy/types.py index 7d8dcb1c1a603..b23e84333be5d 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -735,7 +735,6 @@ class TupleType(Type): def __init__(self, items: List[Type], fallback: Instance, line: int = -1, implicit: bool = False) -> None: - #import pdb; pdb.set_trace() self.items = items self.fallback = fallback self.implicit = implicit @@ -790,7 +789,7 @@ def with_types(self, items: List[Type]) -> 'NamedTupleType': def __eq__(self, other) -> bool: return self.serialize() == other.serialize() - + def serialize(self) -> JsonDict: d = super().serialize() d.update({'.class': 'NamedTupleType', @@ -808,12 +807,11 @@ def deserialize(cls, data: JsonDict) -> 'NamedTupleType': tup.items, tup.fallback, implicit=tup.implicit) - + def accept(self, visitor: 'TypeVisitor[T]') -> T: - try: + if hasattr(visitor, "visit_namedtuple_type"): return visitor.visit_namedtuple_type(self) - except: - return super().accept(visitor) + return super().accept(visitor) class StarType(Type): From ff21b0cee2cf6ceca48da6dbc0d7d92bc2dd0792 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Sun, 14 Aug 2016 19:45:59 +0300 Subject: [PATCH 52/74] s/self_type/with_fallback. Fix visitor type --- mypy/semanal.py | 2 +- mypy/types.py | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index d7fe54678b70d..bc27841d2d327 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -2638,7 +2638,7 @@ def self_type(typ: TypeInfo) -> Union[Instance, TupleType]: inst = Instance(typ, tv) if typ.tuple_type is None: return inst - return typ.tuple_type.self_type(inst) + return typ.tuple_type.with_fallback(inst) def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: diff --git a/mypy/types.py b/mypy/types.py index b23e84333be5d..090f01ed6d001 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -760,7 +760,7 @@ def deserialize(cls, data: JsonDict) -> 'TupleType': Instance.deserialize(data['fallback']), implicit=data['implicit']) - def self_type(self, inst: Instance) -> 'TupleType': + def with_fallback(self, inst: Instance) -> 'TupleType': return TupleType(self.items, inst, self.line, self.implicit) def with_types(self, items: List[Type]) -> 'TupleType': @@ -780,7 +780,7 @@ def __init__(self, name: str, attrs: List[str], *args, **kwargs) -> None: self.name = name super().__init__(*args, **kwargs) - def self_type(self, inst: Instance) -> 'NamedTupleType': + def with_fallback(self, inst: Instance) -> 'NamedTupleType': return NamedTupleType(self.name, self.attrs, self.items, inst, self.line) def with_types(self, items: List[Type]) -> 'NamedTupleType': @@ -809,9 +809,7 @@ def deserialize(cls, data: JsonDict) -> 'NamedTupleType': implicit=tup.implicit) def accept(self, visitor: 'TypeVisitor[T]') -> T: - if hasattr(visitor, "visit_namedtuple_type"): - return visitor.visit_namedtuple_type(self) - return super().accept(visitor) + return visitor.visit_namedtuple_type(self) class StarType(Type): @@ -1070,6 +1068,9 @@ def visit_overloaded(self, t: Overloaded) -> T: def visit_tuple_type(self, t: TupleType) -> T: pass + def visit_namedtuple_type(self, t: NamedTupleType) -> T: + return self.visit_tuple_type(t) + def visit_star_type(self, t: StarType) -> T: raise self._notimplemented_helper('star_type') @@ -1142,6 +1143,10 @@ def visit_tuple_type(self, t: TupleType) -> Type: cast(Any, t.fallback.accept(self)), t.line) + def visit_namedtuple_type(self, t: NamedTupleType) -> Type: + t = t.with_types(self.translate_types(t.items)) + return t.with_fallback(cast(Any, t.fallback.accept(self))) + def visit_star_type(self, t: StarType) -> Type: return StarType(t.type.accept(self), t.line) From 4c1bfe473e6707d6e76d35ab3b7b71ad08bc8ebf Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Sun, 14 Aug 2016 20:44:48 +0300 Subject: [PATCH 53/74] uniform copy_with. typechecks. --- mypy/checker.py | 2 +- mypy/expandtype.py | 2 +- mypy/join.py | 2 +- mypy/nodes.py | 1 + mypy/semanal.py | 5 ++--- mypy/typeanal.py | 25 +++++++++---------------- mypy/types.py | 35 ++++++++++++++++++----------------- 7 files changed, 33 insertions(+), 39 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 7063758096dfa..c457ca712ad30 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -801,7 +801,7 @@ def visit_class_def(self, defn: ClassDef) -> Type: with self.binder.frame_context(): self.accept(defn.defs) self.binder = old_binder - if not hasattr(defn, 'incompatible_baseclass'): + if not defn.has_incompatible_baseclass: # Otherwise we've already found errors; more errors are not useful self.check_multiple_inheritance(typ) self.leave_partial_types() diff --git a/mypy/expandtype.py b/mypy/expandtype.py index ce6d5190fac6a..25a34ea06b2a3 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -88,7 +88,7 @@ def visit_overloaded(self, t: Overloaded) -> Type: return Overloaded(items) def visit_tuple_type(self, t: TupleType) -> Type: - return t.with_types(self.expand_types(t.items)) + return t.copy_with(items=self.expand_types(t.items)) def visit_union_type(self, t: UnionType) -> Type: # After substituting for type variables in t.items, diff --git a/mypy/join.py b/mypy/join.py index 061052e6d87f8..f9ae697cfba39 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -218,7 +218,7 @@ def visit_tuple_type(self, t: TupleType) -> Type: items.append(self.join(t.items[i], self.s.items[i])) if (not (isinstance(t, NamedTupleType) and isinstance(self.s, NamedTupleType)) or self.s.attrs == t.attrs): - return self.s.with_types(items) + return self.s.copy_with(items=items) # TODO: What if the fallback types are different? return TupleType(items, t.fallback) else: diff --git a/mypy/nodes.py b/mypy/nodes.py index 3e4f3f4fda129..e0e9b080d5371 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -636,6 +636,7 @@ class ClassDef(Statement): decorators = None # type: List[Expression] # Built-in/extension class? (single implementation inheritance only) is_builtinclass = False + has_incompatible_baseclass = False def __init__(self, name: str, diff --git a/mypy/semanal.py b/mypy/semanal.py index bc27841d2d327..fb7981e1fee88 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -756,7 +756,7 @@ def analyze_base_classes(self, defn: ClassDef) -> None: if isinstance(base, TupleType): if info.tuple_type: self.fail("Class has two incompatible bases derived from tuple", defn) - defn.incompatible_baseclass = True + defn.has_incompatible_baseclass = True if (not self.is_stub_file and not info.is_named_tuple and base.fallback.type.fullname() == 'builtins.tuple'): @@ -1531,7 +1531,6 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], class_def = ClassDef(name, Block([])) class_def.fullname = self.qualified_name(name) info = NamedTupleTypeInfo(tup, symbols, class_def) - info.attrs = items vars = [Var(item, typ) for item, typ in zip(items, types)] # Add named tuple items as attributes. # TODO: Make them read-only. @@ -2638,7 +2637,7 @@ def self_type(typ: TypeInfo) -> Union[Instance, TupleType]: inst = Instance(typ, tv) if typ.tuple_type is None: return inst - return typ.tuple_type.with_fallback(inst) + return typ.tuple_type.copy_with(fallback=inst) def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 092849567c425..241febab24a95 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -161,23 +161,16 @@ def visit_unbound_type(self, t: UnboundType) -> Type: # valid count at this point. Thus we may construct an # Instance with an invalid number of type arguments. instance = Instance(info, self.anal_array(t.args), t.line) - if info.tuple_type is None: + tup = info.tuple_type + if tup is None: return instance - else: - # The class has a Tuple[...] base class so it will be - # represented as a tuple type. - if t.args: - self.fail('Generic tuple types not supported', t) - return AnyType() - # import pdb; pdb.set_trace() - if info.is_named_tuple: - return NamedTupleType(info.name(), info.attrs, - self.anal_array(info.tuple_type.items), - fallback=instance, - line=t.line) - return TupleType(self.anal_array(info.tuple_type.items), - fallback=instance, - line=t.line) + # The class has a Tuple[...] base class so it will be + # represented as a tuple type. + if t.args: + self.fail('Generic tuple types not supported', t) + return AnyType() + return tup.copy_with(items=self.anal_array(tup.items), + fallback=instance) else: return AnyType() diff --git a/mypy/types.py b/mypy/types.py index 090f01ed6d001..bc8eec17e27ff 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -760,11 +760,13 @@ def deserialize(cls, data: JsonDict) -> 'TupleType': Instance.deserialize(data['fallback']), implicit=data['implicit']) - def with_fallback(self, inst: Instance) -> 'TupleType': - return TupleType(self.items, inst, self.line, self.implicit) - - def with_types(self, items: List[Type]) -> 'TupleType': - return TupleType(items, self.fallback, self.line, self.implicit) + def copy_with(self, *, fallback: Instance = None, + items: List[Type] = None) -> 'TupleType': + if fallback is None: + fallback = self.fallback + if items is None: + items = self.items + return TupleType(items, fallback, self.line) def slice(self, begin: int, stride: int, end: int) -> 'TupleType': return TupleType(self.items[begin:end:stride], self.fallback, @@ -776,19 +778,18 @@ class NamedTupleType(TupleType): name = None # type: str def __init__(self, name: str, attrs: List[str], *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + assert len(self.items) == len(attrs) self.attrs = attrs self.name = name - super().__init__(*args, **kwargs) - - def with_fallback(self, inst: Instance) -> 'NamedTupleType': - return NamedTupleType(self.name, self.attrs, self.items, inst, self.line) - - def with_types(self, items: List[Type]) -> 'NamedTupleType': - return NamedTupleType(self.name, self.attrs, items, - self.fallback, self.line, self.implicit) - def __eq__(self, other) -> bool: - return self.serialize() == other.serialize() + def copy_with(self, *, fallback: Instance = None, + items: List[Type] = None) -> 'NamedTupleType': + if fallback is None: + fallback = self.fallback + if items is None: + items = self.items + return NamedTupleType(self.name, self.attrs, items, fallback, self.line) def serialize(self) -> JsonDict: d = super().serialize() @@ -1144,8 +1145,8 @@ def visit_tuple_type(self, t: TupleType) -> Type: t.line) def visit_namedtuple_type(self, t: NamedTupleType) -> Type: - t = t.with_types(self.translate_types(t.items)) - return t.with_fallback(cast(Any, t.fallback.accept(self))) + return t.copy_with(items=self.translate_types(t.items), + fallback=cast(Any, t.fallback.accept(self))) def visit_star_type(self, t: StarType) -> Type: return StarType(t.type.accept(self), t.line) From b0d78bac405914e6e21c55a70c98a2ffc91a171b Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Sun, 14 Aug 2016 20:54:08 +0300 Subject: [PATCH 54/74] uniform copy_with. typechecks. --- mypy/semanal.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mypy/semanal.py b/mypy/semanal.py index fb7981e1fee88..28ee0ed69cf62 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1466,6 +1466,7 @@ def parse_namedtuple_args(self, call: CallExpr, if len(args) < 2: return self.fail_namedtuple_arg("Too few arguments for namedtuple()", call) if len(args) > 2: + # FIX incorrect. There are two additional parameters return self.fail_namedtuple_arg("Too many arguments for namedtuple()", call) if call.arg_kinds != [ARG_POS, ARG_POS]: return self.fail_namedtuple_arg("Unexpected arguments to namedtuple()", call) @@ -1536,6 +1537,7 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], # TODO: Make them read-only. for var in vars: var.info = info + var.is_property = True symbols[var.name()] = SymbolTableNode(MDEF, var) this_type = self_type(info) # Add __init__, _replace, _asdict methods and _fields static field From 87c804cfa0d40af326445156394b002e45e2369b Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Mon, 15 Aug 2016 22:33:15 +0300 Subject: [PATCH 55/74] rename tests to startwith NamedTuple --- test-data/unit/check-namedtuple.test | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index adb07ec522705..74e5872aed256 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -14,7 +14,7 @@ a = x[1] a, b, c = x # E: Need more than 2 values to unpack (3 expected) x[2] # E: Tuple index out of range -[case testAccessingNamedTupleAttributes] +[case testNamedTupleAccessingAttributes] from collections import namedtuple X = namedtuple('X', ['x', 'y']) @@ -23,7 +23,7 @@ x.x x.y x.z # E: "X" has no attribute "z" -[case testCreateNamedTupleWithPositionalArguments] +[case testNamedTupleCreateWithPositionalArguments] from collections import namedtuple X = namedtuple('X', ['x', 'y']) @@ -42,7 +42,7 @@ x = X(1, y='x') x = X(x=1, z=1) # E: Unexpected keyword argument "z" for "X" x = X(y=1) # E: Missing positional argument "x" in call to "X" -[case testCreateNamedTupleAndUseAsTuple] +[case testNamedTupleCreateAndUseAsTuple] from collections import namedtuple X = namedtuple('X', ['x', 'y']) @@ -96,20 +96,20 @@ s = x.a # E: Incompatible types in assignment (expression has type "int", varia i, s = x s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") -[case testTwoNamedTuplesAsBaseClasses] +[case testNamedTuplesTwoAsBaseClasses] from typing import NamedTuple A = NamedTuple('A', [('a', int)]) B = NamedTuple('B', [('a', int)]) class X(A, B): # E: Class has two incompatible bases derived from tuple pass -[case testTwoNamedTuplesAsBaseClasses2] +[case testNamedTuplesTwoAsBaseClasses2] from typing import NamedTuple A = NamedTuple('A', [('a', int)]) class X(A, NamedTuple('B', [('a', int)])): # E: Class has two incompatible bases derived from tuple pass -[case testSelfTypeWithNamedTupleAsBase] +[case testNamedTupleSelfTypeWithNamedTupleAsBase] from typing import NamedTuple A = NamedTuple('A', [('a', int), ('b', str)]) class B(A): @@ -124,7 +124,7 @@ class B(A): [out] main: note: In member "f" of class "B": -[case testTypeReferenceToClassDerivedFromNamedTuple] +[case testNamedTupleTypeReferenceToClassDerivedFrom] from typing import NamedTuple A = NamedTuple('A', [('a', int), ('b', str)]) class B(A): From 6adc2f91640d06049f843849986eb27b99f42ce3 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Mon, 15 Aug 2016 23:26:34 +0300 Subject: [PATCH 56/74] Read-only attributes. Can't set new attributes in subclasses yet --- mypy/checkmember.py | 4 +++- mypy/messages.py | 1 - mypy/semanal.py | 3 +-- test-data/unit/check-namedtuple.test | 22 +++++++++++++++++----- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/mypy/checkmember.py b/mypy/checkmember.py index cc0be2058fbf9..b5f5ae1f2e3cb 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -151,7 +151,6 @@ def analyze_member_var_access(name: str, itype: Instance, info: TypeInfo, if isinstance(vv, Decorator): # The associated Var node of a decorator contains the type. v = vv.var - if isinstance(v, Var): return analyze_var(name, v, itype, info, node, is_lvalue, msg, not_ready_callback) elif isinstance(v, FuncDef): @@ -191,6 +190,9 @@ def analyze_var(name: str, var: Var, itype: Instance, info: TypeInfo, node: Cont if isinstance(typ, PartialType): return handle_partial_attribute_type(typ, is_lvalue, msg, var) t = expand_type_by_instance(typ, itype) + if is_lvalue and var.is_property and not var.is_settable_property: + # TODO allow setting attributes in subclass (although it is probably an error) + msg.read_only_property(name, info, node) if var.is_initialized_in_class and isinstance(t, FunctionLike): if is_lvalue: if var.is_property: diff --git a/mypy/messages.py b/mypy/messages.py index d8aed9626a49d..209010764eca3 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -146,7 +146,6 @@ def report(self, msg: str, context: Context, severity: str, file: str = None) -> def fail(self, msg: str, context: Context, file: str = None) -> None: """Report an error message (unless disabled).""" - # import pdb; pdb.set_trace() self.report(msg, context, 'error', file=file) def note(self, msg: str, context: Context, file: str = None) -> None: diff --git a/mypy/semanal.py b/mypy/semanal.py index 28ee0ed69cf62..27a17b6277a8b 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1545,7 +1545,7 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], self.factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) self.add_namedtuple_method('__init__', symbols, info, this_type, NoneTyp(), self.factory_args(vars, ARG_POS), info.name()) - # TODO: refine to OrderedDict[Union[types]] + # TODO: refine to OrderedDict[str, Union[types]] self.add_namedtuple_method('_asdict', symbols, info, this_type, AnyType(), []) return info @@ -1559,7 +1559,6 @@ def add_namedtuple_method(self, funcname: str, symbols: SymbolTable, info: TypeI types = [arg.type_annotation for arg in args] items = [arg.variable.name() for arg in args] arg_kinds = [arg.kind for arg in args] - # TODO: Make sure that the self argument name is not visible? signature = CallableType(types, arg_kinds, items, ret, self.named_type('__builtins__.function'), name=name or info.name() + '.' + funcname) diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 74e5872aed256..ea12d058b8204 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -1,8 +1,3 @@ -[case testNamedTupleDeclaration] -from typing import NamedTuple - -A = NamedTuple('A', [('x', str), ('y', int)]) - [case testNamedTupleUsedAsTuple] from collections import namedtuple @@ -23,6 +18,21 @@ x.x x.y x.z # E: "X" has no attribute "z" +[case testNamedTupleAttributesAreReadOnly] +from collections import namedtuple + +X = namedtuple('X', ['x', 'y']) +x = None # type: X +x.x = 5 # E: Property "x" defined in "X" is read-only +x.y = 5 # E: Property "y" defined in "X" is read-only +x.z = 5 # E: "X" has no attribute "z" + +class A(X): pass +a = None # type: A +a.x = 5 # E: Property "x" defined in "A" is read-only +a.y = 5 # E: Property "y" defined in "A" is read-only +-- a.z = 5 # not supported yet + [case testNamedTupleCreateWithPositionalArguments] from collections import namedtuple @@ -164,6 +174,7 @@ a = A(1) a = l[0] (i,) = l[0] i, i = l[0] # E: Need more than 1 value to unpack (2 expected) +l = [A(1)] a = (1,) # E: Incompatible types in assignment (expression has type "Tuple[int]", \ variable has type "A") [builtins fixtures/list.py] @@ -203,6 +214,7 @@ from collections import namedtuple X = namedtuple('X', ['x', 'y']) x = None # type: X x = x._replace(y=5) +x = x._replace(x=3, y=5) x = x._replace(z=5) # E: Unexpected keyword argument "z" for X._replace x = x._replace(5) # E: Too many positional arguments for X._replace From 7ed7b0e10800cf639d60f093f8bad4cab393fd45 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Tue, 16 Aug 2016 00:28:24 +0300 Subject: [PATCH 57/74] update semantic namedtuple repr name --- test-data/unit/semanal-namedtuple.test | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test-data/unit/semanal-namedtuple.test b/test-data/unit/semanal-namedtuple.test index 9ff9f6af6e8a1..78a678b974adf 100644 --- a/test-data/unit/semanal-namedtuple.test +++ b/test-data/unit/semanal-namedtuple.test @@ -1,6 +1,6 @@ -- Semantic analysis of named tuples -[case testSimpleNamedtuple] +[case testSimpleNamedTuple] from collections import namedtuple N = namedtuple('N', ['a']) def f() -> N: pass @@ -9,14 +9,14 @@ MypyFile:1( ImportFrom:1(collections, [namedtuple]) AssignmentStmt:2( NameExpr(N* [__main__.N]) - NamedTupleExpr:2(N, Tuple[Any])) + NamedTupleExpr:2(N, N[('a', Any)])) FuncDef:3( f - def () -> Tuple[Any, fallback=__main__.N] + def () -> N[('a', Any)] Block:3( PassStmt:3()))) -[case testTwoItemNamedtuple] +[case testTwoItemNamedTuple] from collections import namedtuple N = namedtuple('N', ['a', 'xyz']) def f() -> N: pass @@ -25,14 +25,14 @@ MypyFile:1( ImportFrom:1(collections, [namedtuple]) AssignmentStmt:2( NameExpr(N* [__main__.N]) - NamedTupleExpr:2(N, Tuple[Any, Any])) + NamedTupleExpr:2(N, N[('a', Any), ('xyz', Any)])) FuncDef:3( f - def () -> Tuple[Any, Any, fallback=__main__.N] + def () -> N[('a', Any), ('xyz', Any)] Block:3( PassStmt:3()))) -[case testTwoItemNamedtupleWithShorthandSyntax] +[case testTwoItemNamedTupleWithShorthandSyntax] from collections import namedtuple N = namedtuple('N', ' a xyz ') def f() -> N: pass @@ -41,10 +41,10 @@ MypyFile:1( ImportFrom:1(collections, [namedtuple]) AssignmentStmt:2( NameExpr(N* [__main__.N]) - NamedTupleExpr:2(N, Tuple[Any, Any])) + NamedTupleExpr:2(N, N[('a', Any), ('xyz', Any)])) FuncDef:3( f - def () -> Tuple[Any, Any, fallback=__main__.N] + def () -> N[('a', Any), ('xyz', Any)] Block:3( PassStmt:3()))) @@ -57,7 +57,7 @@ MypyFile:1( ImportFrom:1(typing, [NamedTuple]) AssignmentStmt:2( NameExpr(N* [__main__.N]) - NamedTupleExpr:2(N, Tuple[builtins.int, builtins.str]))) + NamedTupleExpr:2(N, N[('a', builtins.int), ('b', builtins.str)]))) [case testNamedTupleBaseClass] from collections import namedtuple @@ -68,11 +68,11 @@ MypyFile:1( ImportFrom:1(collections, [namedtuple]) AssignmentStmt:2( NameExpr(N* [__main__.N]) - NamedTupleExpr:2(N, Tuple[Any])) + NamedTupleExpr:2(N, N[('x', Any)])) ClassDef:3( A TupleType( - Tuple[Any, fallback=__main__.N]) + N[('x', Any)]) BaseType( __main__.N) PassStmt:3())) From c7dc9a7e8311582d3c44f6e33faa5b6f786d46b9 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Tue, 16 Aug 2016 01:37:56 +0300 Subject: [PATCH 58/74] support _fields --- mypy/semanal.py | 42 +++++++++++++++------------- mypy/types.py | 10 +++++++ test-data/unit/check-namedtuple.test | 12 ++++++++ 3 files changed, 45 insertions(+), 19 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 27a17b6277a8b..6fea9755edc23 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -43,8 +43,10 @@ traverse the entire AST. """ +from functools import partial + from typing import ( - List, Dict, Set, Tuple, cast, Any, overload, TypeVar, Union, Optional, Callable + List, Dict, Set, Tuple, cast, Any, TypeVar, Union, Optional, Callable ) from mypy.nodes import ( @@ -71,18 +73,15 @@ from mypy.errors import Errors, report_internal_error from mypy.types import ( NoneTyp, CallableType, Overloaded, Instance, Type, TypeVarType, AnyType, - FunctionLike, UnboundType, TypeList, ErrorType, TypeVarDef, + FunctionLike, UnboundType, TypeList, TypeVarDef, replace_leading_arg_type, TupleType, UnionType, StarType, EllipsisType, NamedTupleType ) from mypy.nodes import function_type, implicit_module_attrs from mypy.typeanal import TypeAnalyser, TypeAnalyserPass3, analyze_type_alias from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError -from mypy.lex import lex -from mypy.parsetype import parse_type from mypy.sametypes import is_same_type from mypy.erasetype import erase_typevars -from mypy import defaults from mypy.options import Options @@ -1533,28 +1532,36 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], class_def.fullname = self.qualified_name(name) info = NamedTupleTypeInfo(tup, symbols, class_def) vars = [Var(item, typ) for item, typ in zip(items, types)] - # Add named tuple items as attributes. - # TODO: Make them read-only. + # Add namedtuple items as attributes. for var in vars: var.info = info var.is_property = True symbols[var.name()] = SymbolTableNode(MDEF, var) + + typ = TupleType([self.named_type('__builtins__.str') for _ in items], + self.named_type('__builtins__.tuple', [AnyType()])) + _fields = Var('_fields', typ) + _fields.info = info + _fields.is_initialized_in_class = True + symbols[_fields.name()] = SymbolTableNode(MDEF, _fields) + this_type = self_type(info) - # Add __init__, _replace, _asdict methods and _fields static field - self.add_namedtuple_method('_replace', symbols, info, this_type, this_type, - self.factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) - self.add_namedtuple_method('__init__', symbols, info, this_type, NoneTyp(), - self.factory_args(vars, ARG_POS), info.name()) + add_method = partial(self.add_namedtuple_method, symbols, info, this_type) + add_method('_replace', this_type, + self.factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) + add_method('__init__', NoneTyp(), + self.factory_args(vars, ARG_POS), info.name()) # TODO: refine to OrderedDict[str, Union[types]] - self.add_namedtuple_method('_asdict', symbols, info, this_type, AnyType(), []) + add_method('_asdict', AnyType(), []) + # self.add_namedtuple_method('_source', self.named_type_or_none('builtins.str'), []) return info def factory_args(self, vars: List[Var], kind: int, initializer: Expression = None) -> List[Argument]: return [Argument(var, var.type, initializer, kind) for var in vars] - def add_namedtuple_method(self, funcname: str, symbols: SymbolTable, info: TypeInfo, - this_type: Type, ret: Type, args: List[Argument], name=None) -> None: + def add_namedtuple_method(self, symbols: SymbolTable, info: TypeInfo, this_type: Type, + funcname: str, ret: Type, args: List[Argument], name=None) -> None: args = [Argument(Var('self'), this_type, None, ARG_POS)] + args types = [arg.type_annotation for arg in args] items = [arg.variable.name() for arg in args] @@ -1562,10 +1569,7 @@ def add_namedtuple_method(self, funcname: str, symbols: SymbolTable, info: TypeI signature = CallableType(types, arg_kinds, items, ret, self.named_type('__builtins__.function'), name=name or info.name() + '.' + funcname) - func = FuncDef(funcname, - args, - Block([]), - typ=signature) + func = FuncDef(funcname, args, Block([]), typ=signature) func.info = info symbols[funcname] = SymbolTableNode(MDEF, func) diff --git a/mypy/types.py b/mypy/types.py index bc8eec17e27ff..c900ac1644bae 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -774,6 +774,16 @@ def slice(self, begin: int, stride: int, end: int) -> 'TupleType': class NamedTupleType(TupleType): + """The namedtuple type NamedTuple[name, (a1, T1), ..., (an, Tn)]. + Subtype of Tuple[T1, ..., Tn] + + Instance variables: + name: type name + attrs: names of the attributes + + Uses the default TupleType.fallback: builtins.tuple + """ + attrs = None # type: List[str] name = None # type: str diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index ea12d058b8204..ad9e3b95b7a98 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -233,3 +233,15 @@ X = NamedTuple('X', [('x', int), ('y', str)]) x = None # type: X x = x._replace(x=5) x = x._replace(y=5) # E: Argument 1 to X._replace has incompatible type "int"; expected "str" + +[case testNamedTupleFields] +from typing import NamedTuple + +X = NamedTuple('X', [('x', int), ('y', str)]) +a = None # type: str +b = None # type: str +a, b = X._fields +x = None # type: X +a, b = x._fields +c = None # type: int +a, c = X._fields # E: Incompatible types in assignment (expression has type "str", variable has type "int") From b39925b69afc5fda69b1c576b76794bbefd0e725 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Tue, 16 Aug 2016 01:57:12 +0300 Subject: [PATCH 59/74] support _source. test for unit namedtuple --- mypy/semanal.py | 5 +++++ test-data/unit/check-namedtuple.test | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/mypy/semanal.py b/mypy/semanal.py index 6fea9755edc23..a6a8c9348c821 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1545,6 +1545,11 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], _fields.is_initialized_in_class = True symbols[_fields.name()] = SymbolTableNode(MDEF, _fields) + _source = Var('_source', self.named_type('__builtins__.str')) + _source.info = info + _source.is_initialized_in_class = True + symbols[_source.name()] = SymbolTableNode(MDEF, _source) + this_type = self_type(info) add_method = partial(self.add_namedtuple_method, symbols, info, this_type) add_method('_replace', this_type, diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index ad9e3b95b7a98..daf547ec8f8d5 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -245,3 +245,23 @@ x = None # type: X a, b = x._fields c = None # type: int a, c = X._fields # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testNamedTupleSource] +from typing import NamedTuple + +X = NamedTuple('X', [('x', int), ('y', str)]) +x = None # type: X +a = None # type: str +b = None # type: int +a = X._source +a = x._source +b = X._source # E: Incompatible types in assignment (expression has type "str", variable has type "int") +b = x._source # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testNamedTupleUnit] +from typing import NamedTuple + +X = NamedTuple('X', []) +x = X() # type: X +x = x._replace() +_ = x._fields[0] # E: Tuple index out of range From 030307ebd88efa0359e8a1257e0825679e5d4be2 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Tue, 16 Aug 2016 02:30:39 +0300 Subject: [PATCH 60/74] support _asdict()->OrderedDict as an UnboundType --- mypy/semanal.py | 40 +++++++++++++------------- mypy/types.py | 2 +- test-data/unit/check-namedtuple.test | 3 +- test-data/unit/lib-stub/collections.py | 3 +- 4 files changed, 25 insertions(+), 23 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index a6a8c9348c821..44964c28afc98 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1531,36 +1531,36 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], class_def = ClassDef(name, Block([])) class_def.fullname = self.qualified_name(name) info = NamedTupleTypeInfo(tup, symbols, class_def) + vars = [Var(item, typ) for item, typ in zip(items, types)] - # Add namedtuple items as attributes. - for var in vars: - var.info = info - var.is_property = True - symbols[var.name()] = SymbolTableNode(MDEF, var) - - typ = TupleType([self.named_type('__builtins__.str') for _ in items], - self.named_type('__builtins__.tuple', [AnyType()])) - _fields = Var('_fields', typ) - _fields.info = info - _fields.is_initialized_in_class = True - symbols[_fields.name()] = SymbolTableNode(MDEF, _fields) - - _source = Var('_source', self.named_type('__builtins__.str')) - _source.info = info - _source.is_initialized_in_class = True - symbols[_source.name()] = SymbolTableNode(MDEF, _source) - this_type = self_type(info) + add_field = partial(self.add_namedtuple_field, symbols, info) add_method = partial(self.add_namedtuple_method, symbols, info, this_type) + strtype = self.named_type('__builtins__.str') + + for var in vars: + add_field(var, is_property=True) + + tuple_of_strins = TupleType([strtype for _ in items], + self.named_type('__builtins__.tuple', [AnyType()])) + add_field(Var('_fields', tuple_of_strins), is_initialized_in_class=True) + add_field(Var('_source', strtype), is_initialized_in_class=True) + add_method('_replace', this_type, self.factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) add_method('__init__', NoneTyp(), self.factory_args(vars, ARG_POS), info.name()) # TODO: refine to OrderedDict[str, Union[types]] - add_method('_asdict', AnyType(), []) - # self.add_namedtuple_method('_source', self.named_type_or_none('builtins.str'), []) + add_method('_asdict', UnboundType('OrderedDict', is_ret_type=True), []) return info + def add_namedtuple_field(self, symbols, info, var, + is_initialized_in_class=False, is_property=False): + var.info = info + var.is_initialized_in_class = is_initialized_in_class + var.is_property = is_property + symbols[var.name()] = SymbolTableNode(MDEF, var) + def factory_args(self, vars: List[Var], kind: int, initializer: Expression = None) -> List[Argument]: return [Argument(var, var.type, initializer, kind) for var in vars] diff --git a/mypy/types.py b/mypy/types.py index c900ac1644bae..373bff4154662 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -780,7 +780,7 @@ class NamedTupleType(TupleType): Instance variables: name: type name attrs: names of the attributes - + Uses the default TupleType.fallback: builtins.tuple """ diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index daf547ec8f8d5..b4dbe7c640393 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -202,10 +202,11 @@ C(2).b [builtins fixtures/property.py] [case testNamedTupleAsDict] -from collections import namedtuple +from collections import namedtuple, OrderedDict X = namedtuple('X', ['x', 'y']) x = None # type: X +a = None # type: OrderedDict a = x._asdict() [case testNamedTupleReplace] diff --git a/test-data/unit/lib-stub/collections.py b/test-data/unit/lib-stub/collections.py index fa6f4ee58bcc6..29e4b8d37f16d 100644 --- a/test-data/unit/lib-stub/collections.py +++ b/test-data/unit/lib-stub/collections.py @@ -1 +1,2 @@ -namedtuple = object() +namedtuple = object() +class OrderedDict: pass \ No newline at end of file From 12d34077640d4bdc7e8772579d4afd5351c11bd2 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Tue, 16 Aug 2016 02:40:30 +0300 Subject: [PATCH 61/74] minor cleanup --- mypy/nodes.py | 2 +- mypy/semanal.py | 5 +++-- mypy/typeanal.py | 15 ++++++++------- test-data/unit/lib-stub/collections.py | 6 ++++-- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index e0e9b080d5371..49d7e821f9393 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1978,7 +1978,7 @@ def deserialize(cls, data: JsonDict) -> 'TypeInfo': return ti -def NamedTupleTypeInfo(tup: 'mypy.types.NamedTupleType', *args) -> TypeInfo: +def namedtuple_type_info(tup: 'mypy.types.NamedTupleType', *args) -> TypeInfo: info = TypeInfo(*args) info.tuple_type = tup info.bases = [tup.fallback] diff --git a/mypy/semanal.py b/mypy/semanal.py index 44964c28afc98..d78894512e461 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -66,7 +66,8 @@ SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr, YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, COVARIANT, CONTRAVARIANT, IntExpr, FloatExpr, UnicodeExpr, - INVARIANT, UNBOUND_IMPORTED, Expression, EllipsisExpr, NamedTupleTypeInfo + INVARIANT, UNBOUND_IMPORTED, Expression, EllipsisExpr, + namedtuple_type_info ) from mypy.visitor import NodeVisitor from mypy.traverser import TraverserVisitor @@ -1530,7 +1531,7 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], tup = NamedTupleType(name, items, types, self.named_type('__builtins__.tuple', types)) class_def = ClassDef(name, Block([])) class_def.fullname = self.qualified_name(name) - info = NamedTupleTypeInfo(tup, symbols, class_def) + info = namedtuple_type_info(tup, symbols, class_def) vars = [Var(item, typ) for item, typ in zip(items, types)] this_type = self_type(info) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 241febab24a95..b7f0e84e0a839 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -164,13 +164,14 @@ def visit_unbound_type(self, t: UnboundType) -> Type: tup = info.tuple_type if tup is None: return instance - # The class has a Tuple[...] base class so it will be - # represented as a tuple type. - if t.args: - self.fail('Generic tuple types not supported', t) - return AnyType() - return tup.copy_with(items=self.anal_array(tup.items), - fallback=instance) + else: + # The class has a Tuple[...] base class so it will be + # represented as a tuple type. + if t.args: + self.fail('Generic tuple types not supported', t) + return AnyType() + return tup.copy_with(items=self.anal_array(tup.items), + fallback=instance) else: return AnyType() diff --git a/test-data/unit/lib-stub/collections.py b/test-data/unit/lib-stub/collections.py index 29e4b8d37f16d..fabe356675b4a 100644 --- a/test-data/unit/lib-stub/collections.py +++ b/test-data/unit/lib-stub/collections.py @@ -1,2 +1,4 @@ -namedtuple = object() -class OrderedDict: pass \ No newline at end of file +namedtuple = object() + +class OrderedDict: + pass From 1f944fcaf0275a3f0a81cce3912e4e6d3bd7a4e0 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Tue, 16 Aug 2016 05:55:59 +0300 Subject: [PATCH 62/74] support _make and _field_types --- mypy/semanal.py | 51 +++++++++++++++++++--------- test-data/unit/check-namedtuple.test | 10 ++++++ 2 files changed, 45 insertions(+), 16 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index d78894512e461..8c02f94c1bd25 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -61,7 +61,7 @@ SymbolTableNode, BOUND_TVAR, UNBOUND_TVAR, ListComprehension, GeneratorExpr, FuncExpr, MDEF, FuncBase, Decorator, SetExpr, TypeVarExpr, StrExpr, BytesExpr, PrintStmt, ConditionalExpr, PromoteExpr, - ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, MroError, type_aliases, + ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, ARG_OPT, MroError, type_aliases, YieldFromExpr, NamedTupleExpr, NonlocalDecl, SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr, YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, COVARIANT, CONTRAVARIANT, @@ -76,7 +76,7 @@ NoneTyp, CallableType, Overloaded, Instance, Type, TypeVarType, AnyType, FunctionLike, UnboundType, TypeList, TypeVarDef, replace_leading_arg_type, TupleType, UnionType, StarType, EllipsisType, - NamedTupleType + NamedTupleType, TypeType ) from mypy.nodes import function_type, implicit_module_attrs from mypy.typeanal import TypeAnalyser, TypeAnalyserPass3, analyze_type_alias @@ -1531,32 +1531,47 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], tup = NamedTupleType(name, items, types, self.named_type('__builtins__.tuple', types)) class_def = ClassDef(name, Block([])) class_def.fullname = self.qualified_name(name) + self.enter_class(class_def) info = namedtuple_type_info(tup, symbols, class_def) vars = [Var(item, typ) for item, typ in zip(items, types)] this_type = self_type(info) add_field = partial(self.add_namedtuple_field, symbols, info) - add_method = partial(self.add_namedtuple_method, symbols, info, this_type) strtype = self.named_type('__builtins__.str') for var in vars: add_field(var, is_property=True) - tuple_of_strins = TupleType([strtype for _ in items], - self.named_type('__builtins__.tuple', [AnyType()])) - add_field(Var('_fields', tuple_of_strins), is_initialized_in_class=True) - add_field(Var('_source', strtype), is_initialized_in_class=True) + tuple_of_strings = TupleType([strtype for _ in items], + self.named_type('__builtins__.tuple', [AnyType()])) + add_field(Var('_fields', tuple_of_strings), + is_initialized_in_class=True) + add_field(Var('_field_types', UnboundType('Dict', [strtype, AnyType()])), + is_initialized_in_class=True) + add_field(Var('_source', strtype), + is_initialized_in_class=True) - add_method('_replace', this_type, - self.factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) - add_method('__init__', NoneTyp(), - self.factory_args(vars, ARG_POS), info.name()) + add_method = partial(self.add_namedtuple_method, symbols, info, this_type) + add_method('_replace', ret=this_type, + args=self.factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) + add_method('__init__', ret=NoneTyp(), + args=self.factory_args(vars, ARG_POS), name=info.name()) # TODO: refine to OrderedDict[str, Union[types]] - add_method('_asdict', UnboundType('OrderedDict', is_ret_type=True), []) + add_method('_asdict', ret=UnboundType('collections.OrderedDict', is_ret_type=True), + args=[]) + + # TODO: refine signature + union = UnboundType('Iterable', [UnionType(types)]) + add_method('_make', ret=this_type, is_classmethod=True, + args=[Argument(Var('iterable', union), union, None, ARG_POS), + Argument(Var('new'), AnyType(), EllipsisExpr(), ARG_NAMED), + Argument(Var('len'), AnyType(), EllipsisExpr(), ARG_NAMED)]) + self.leave_class() return info - def add_namedtuple_field(self, symbols, info, var, - is_initialized_in_class=False, is_property=False): + def add_namedtuple_field(self, symbols: SymbolTable, info: TypeInfo, var: Var, + is_initialized_in_class: bool = False, + is_property: bool = False) -> None: var.info = info var.is_initialized_in_class = is_initialized_in_class var.is_property = is_property @@ -1567,16 +1582,20 @@ def factory_args(self, vars: List[Var], kind: int, return [Argument(var, var.type, initializer, kind) for var in vars] def add_namedtuple_method(self, symbols: SymbolTable, info: TypeInfo, this_type: Type, - funcname: str, ret: Type, args: List[Argument], name=None) -> None: - args = [Argument(Var('self'), this_type, None, ARG_POS)] + args + funcname: str, ret: Type, args: List[Argument], name=None, + is_classmethod=False) -> None: + if not is_classmethod: + args = [Argument(Var('self'), this_type, None, ARG_POS)] + args types = [arg.type_annotation for arg in args] items = [arg.variable.name() for arg in args] arg_kinds = [arg.kind for arg in args] signature = CallableType(types, arg_kinds, items, ret, self.named_type('__builtins__.function'), name=name or info.name() + '.' + funcname) + signature.is_classmethod_class = is_classmethod func = FuncDef(funcname, args, Block([]), typ=signature) func.info = info + func.is_class = is_classmethod symbols[funcname] = SymbolTableNode(MDEF, func) def analyze_types(self, items: List[Node]) -> List[Type]: diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index b4dbe7c640393..f339525cfb392 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -235,6 +235,16 @@ x = None # type: X x = x._replace(x=5) x = x._replace(y=5) # E: Argument 1 to X._replace has incompatible type "int"; expected "str" + +[case testNamedTupleMake] +from typing import NamedTuple + +X = NamedTuple('X', [('x', int), ('y', str)]) +x = None # type: X +x = X._make([5, 'a']) + +[builtins fixtures/list.py] + [case testNamedTupleFields] from typing import NamedTuple From 797835765b04796f2bf11ccbb48f60f55a4db34e Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Wed, 17 Aug 2016 02:03:41 +0300 Subject: [PATCH 63/74] merge fix except Awaitable lookup --- mypy/semanal.py | 5 ++++- mypy/types.py | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index ef51099b15502..dc2f28f6d4e86 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -75,7 +75,7 @@ from mypy.errors import Errors, report_internal_error from mypy.types import ( NoneTyp, CallableType, Overloaded, Instance, Type, TypeVarType, AnyType, - FunctionLike, UnboundType, TypeList, TypeVarDef, TypeVarDef, Void, + FunctionLike, UnboundType, TypeList, TypeVarDef, Void, replace_leading_arg_type, TupleType, UnionType, StarType, EllipsisType, NamedTupleType, TypeType ) @@ -1728,6 +1728,9 @@ def add_namedtuple_method(self, symbols: SymbolTable, info: TypeInfo, this_type: func.is_class = is_classmethod symbols[funcname] = SymbolTableNode(MDEF, func) + def make_argument(self, name: str, type: Type) -> Argument: + return Argument(Var(name), type, None, ARG_POS) + def analyze_types(self, items: List[Node]) -> List[Type]: result = [] # type: List[Type] for node in items: diff --git a/mypy/types.py b/mypy/types.py index a827b236ea200..cb82d85fd84bf 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1298,7 +1298,8 @@ def visit_tuple_type(self, t: TupleType) -> str: return 'Tuple[{}]'.format(s) def visit_namedtuple_type(self, t: NamedTupleType) -> str: - s = self.list_str(zip(t.attrs, t.items)) + s = ', '.join('({}, {})'.format(repr(attr), item.accept(self)) + for attr, item in zip(t.attrs, t.items)) return '{}[{}]'.format(t.name, s) def visit_star_type(self, t: StarType) -> str: From badc3f9874eb76b789bb808e8fe3943c6e04d513 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Wed, 17 Aug 2016 02:03:41 +0300 Subject: [PATCH 64/74] merge fix --- mypy/semanal.py | 5 ++++- mypy/types.py | 3 ++- typeshed | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index ef51099b15502..dc2f28f6d4e86 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -75,7 +75,7 @@ from mypy.errors import Errors, report_internal_error from mypy.types import ( NoneTyp, CallableType, Overloaded, Instance, Type, TypeVarType, AnyType, - FunctionLike, UnboundType, TypeList, TypeVarDef, TypeVarDef, Void, + FunctionLike, UnboundType, TypeList, TypeVarDef, Void, replace_leading_arg_type, TupleType, UnionType, StarType, EllipsisType, NamedTupleType, TypeType ) @@ -1728,6 +1728,9 @@ def add_namedtuple_method(self, symbols: SymbolTable, info: TypeInfo, this_type: func.is_class = is_classmethod symbols[funcname] = SymbolTableNode(MDEF, func) + def make_argument(self, name: str, type: Type) -> Argument: + return Argument(Var(name), type, None, ARG_POS) + def analyze_types(self, items: List[Node]) -> List[Type]: result = [] # type: List[Type] for node in items: diff --git a/mypy/types.py b/mypy/types.py index a827b236ea200..cb82d85fd84bf 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1298,7 +1298,8 @@ def visit_tuple_type(self, t: TupleType) -> str: return 'Tuple[{}]'.format(s) def visit_namedtuple_type(self, t: NamedTupleType) -> str: - s = self.list_str(zip(t.attrs, t.items)) + s = ', '.join('({}, {})'.format(repr(attr), item.accept(self)) + for attr, item in zip(t.attrs, t.items)) return '{}[{}]'.format(t.name, s) def visit_star_type(self, t: StarType) -> str: diff --git a/typeshed b/typeshed index cc1f92103d965..3d32e2e661796 160000 --- a/typeshed +++ b/typeshed @@ -1 +1 @@ -Subproject commit cc1f92103d965eab6dc3878afb683ed71bbb56d3 +Subproject commit 3d32e2e6617968d21db8c195edf103af4b2d4905 From d067c9e90377075c26a080bd37336dc8988a4181 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Wed, 17 Aug 2016 23:52:14 +0300 Subject: [PATCH 65/74] fix syntax error --- mypy/join.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/join.py b/mypy/join.py index d8873346fc2fc..a2fe7fd60ee3d 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -6,7 +6,7 @@ Type, AnyType, NoneTyp, Void, TypeVisitor, Instance, UnboundType, ErrorType, TypeVarType, CallableType, TupleType, ErasedType, TypeList, UnionType, FunctionLike, Overloaded, PartialType, DeletedType, - UninhabitedType, TypeType, NamedTupleType + UninhabitedType, TypeType, NamedTupleType, true_or_false ) from mypy.maptype import map_instance_to_supertype From a5ccc5016b68406250b320f55e534a28592a5645 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Sat, 20 Aug 2016 23:36:26 +0300 Subject: [PATCH 66/74] join/meet namedtuples as namedtuple only if attrs are equal --- mypy/join.py | 16 +++++++++++----- mypy/meet.py | 13 ++++++++++++- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/mypy/join.py b/mypy/join.py index a2fe7fd60ee3d..5aef64d6ec4ae 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -228,20 +228,26 @@ def visit_overloaded(self, t: Overloaded) -> Type: return join_types(t.fallback, s) def visit_tuple_type(self, t: TupleType) -> Type: - if self.s == t: - return self.s if isinstance(self.s, TupleType) and self.s.length() == t.length(): items = [] # type: List[Type] for i in range(t.length()): items.append(self.join(t.items[i], self.s.items[i])) - if (not (isinstance(t, NamedTupleType) and isinstance(self.s, NamedTupleType)) - or self.s.attrs == t.attrs): - return self.s.copy_with(items=items) # TODO: What if the fallback types are different? return TupleType(items, t.fallback) else: return self.default(self.s) + def visit_namedtuple_type(self, t: NamedTupleType) -> Type: + tuple_type = self.visit_tuple_type(t) + if isinstance(tuple_type, TupleType): + if isinstance(self.s, NamedTupleType) and self.s.attrs == t.attrs: + # we forget about t.name + return self.s.copy_with(items=tuple_type.items) + else: + return tuple_type + else: + return self.default(self.s) + def visit_partial_type(self, t: PartialType) -> Type: # We only have partial information so we can't decide the join result. We should # never get here. diff --git a/mypy/meet.py b/mypy/meet.py index 63e3aaee8bce2..23bd3b2a58b8c 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -4,7 +4,7 @@ from mypy.types import ( Type, AnyType, TypeVisitor, UnboundType, Void, ErrorType, NoneTyp, TypeVarType, Instance, CallableType, TupleType, ErasedType, TypeList, UnionType, PartialType, - DeletedType, UninhabitedType, TypeType + DeletedType, UninhabitedType, TypeType, NamedTupleType ) from mypy.subtypes import is_subtype from mypy.nodes import TypeInfo @@ -255,6 +255,17 @@ def visit_tuple_type(self, t: TupleType) -> Type: else: return self.default(self.s) + def visit_namedtuple_type(self, t: NamedTupleType) -> Type: + tuple_type = self.visit_tuple_type(t) + if isinstance(tuple_type, TupleType): + if isinstance(self.s, NamedTupleType) and self.s.attrs == t.attrs: + # we forget about t.name + return self.s.copy_with(items=tuple_type.items) + else: + return tuple_type + else: + return self.default(self.s) + def visit_partial_type(self, t: PartialType) -> Type: # We can't determine the meet of partial types. We should never get here. assert False, 'Internal error' From eaf4b99ba195d07eb01d04cd8da96e230f0db54b Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Sat, 20 Aug 2016 23:36:26 +0300 Subject: [PATCH 67/74] join/meet namedtuples as namedtuple only if attrs are equal. add tests. --- mypy/join.py | 18 +++++++++++++----- mypy/meet.py | 13 ++++++++++++- test-data/unit/check-namedtuple.test | 28 ++++++++++++++++++++++++++-- 3 files changed, 51 insertions(+), 8 deletions(-) diff --git a/mypy/join.py b/mypy/join.py index a2fe7fd60ee3d..42f46f962bb89 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -228,20 +228,28 @@ def visit_overloaded(self, t: Overloaded) -> Type: return join_types(t.fallback, s) def visit_tuple_type(self, t: TupleType) -> Type: - if self.s == t: - return self.s if isinstance(self.s, TupleType) and self.s.length() == t.length(): items = [] # type: List[Type] for i in range(t.length()): items.append(self.join(t.items[i], self.s.items[i])) - if (not (isinstance(t, NamedTupleType) and isinstance(self.s, NamedTupleType)) - or self.s.attrs == t.attrs): - return self.s.copy_with(items=items) # TODO: What if the fallback types are different? return TupleType(items, t.fallback) else: return self.default(self.s) + def visit_namedtuple_type(self, t: NamedTupleType) -> Type: + tuple_type = self.visit_tuple_type(t) + if not isinstance(self.s, NamedTupleType): + return tuple_type + if isinstance(tuple_type, TupleType): + if isinstance(self.s, NamedTupleType) and self.s.attrs == t.attrs: + # we forget about t.name + return self.s.copy_with(items=tuple_type.items) + else: + return tuple_type + else: + return self.default(self.s) + def visit_partial_type(self, t: PartialType) -> Type: # We only have partial information so we can't decide the join result. We should # never get here. diff --git a/mypy/meet.py b/mypy/meet.py index 63e3aaee8bce2..23bd3b2a58b8c 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -4,7 +4,7 @@ from mypy.types import ( Type, AnyType, TypeVisitor, UnboundType, Void, ErrorType, NoneTyp, TypeVarType, Instance, CallableType, TupleType, ErasedType, TypeList, UnionType, PartialType, - DeletedType, UninhabitedType, TypeType + DeletedType, UninhabitedType, TypeType, NamedTupleType ) from mypy.subtypes import is_subtype from mypy.nodes import TypeInfo @@ -255,6 +255,17 @@ def visit_tuple_type(self, t: TupleType) -> Type: else: return self.default(self.s) + def visit_namedtuple_type(self, t: NamedTupleType) -> Type: + tuple_type = self.visit_tuple_type(t) + if isinstance(tuple_type, TupleType): + if isinstance(self.s, NamedTupleType) and self.s.attrs == t.attrs: + # we forget about t.name + return self.s.copy_with(items=tuple_type.items) + else: + return tuple_type + else: + return self.default(self.s) + def visit_partial_type(self, t: PartialType) -> Type: # We can't determine the meet of partial types. We should never get here. assert False, 'Internal error' diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 972ca3b0ee2c1..f4a5d2d35b71c 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -214,7 +214,9 @@ from collections import namedtuple X = namedtuple('X', ['x', 'y']) x = None # type: X +x = x._replace() x = x._replace(y=5) +x = x._replace(x=3) x = x._replace(x=3, y=5) x = x._replace(z=5) # E: Unexpected keyword argument "z" for X._replace x = x._replace(5) # E: Too many positional arguments for X._replace @@ -265,9 +267,7 @@ x = None # type: X a = None # type: str b = None # type: int a = X._source -a = x._source b = X._source # E: Incompatible types in assignment (expression has type "str", variable has type "int") -b = x._source # E: Incompatible types in assignment (expression has type "str", variable has type "int") [case testNamedTupleUnit] from typing import NamedTuple @@ -277,3 +277,27 @@ x = X() # type: X x = x._replace() _ = x._fields[0] # E: Tuple index out of range +[case testNamedTupleJoinNamedTuple] +from typing import NamedTuple + +X = NamedTuple('X', [('x', int), ('y', str)]) +Y = NamedTuple('Y', [('x', int), ('y', str)]) +xy = None +if None: + xy = X(3, 'b') +else: + xy = Y(1, 'a') +xy.x +xy.y + +[case testNamedTupleJoinTuple] +from typing import NamedTuple, Tuple + +X = NamedTuple('X', [('x', int), ('y', str)]) +y = None # type: Tuple[int, str] +if (lambda : ...)(): + y = (3, 'b') +else: + y = X(1, 'a') + +reveal_type(y) # E: Revealed type is 'Tuple[builtins.int, builtins.str]' From 053ee56d3bc1c5863c1a5e26a8c04e8589de7438 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Wed, 31 Aug 2016 02:41:44 +0300 Subject: [PATCH 68/74] remove NamedTupleType --- mypy/expandtype.py | 2 +- mypy/join.py | 16 +----------- mypy/meet.py | 13 +--------- mypy/nodes.py | 2 +- mypy/semanal.py | 10 +++----- mypy/subtypes.py | 10 +------- mypy/typeanal.py | 10 +++----- mypy/types.py | 64 +--------------------------------------------- 8 files changed, 13 insertions(+), 114 deletions(-) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 25a34ea06b2a3..3c608c4e1ad72 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -88,7 +88,7 @@ def visit_overloaded(self, t: Overloaded) -> Type: return Overloaded(items) def visit_tuple_type(self, t: TupleType) -> Type: - return t.copy_with(items=self.expand_types(t.items)) + return t.copy_modified(items=self.expand_types(t.items)) def visit_union_type(self, t: UnionType) -> Type: # After substituting for type variables in t.items, diff --git a/mypy/join.py b/mypy/join.py index 42f46f962bb89..c639b3648984a 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -6,8 +6,7 @@ Type, AnyType, NoneTyp, Void, TypeVisitor, Instance, UnboundType, ErrorType, TypeVarType, CallableType, TupleType, ErasedType, TypeList, UnionType, FunctionLike, Overloaded, PartialType, DeletedType, - UninhabitedType, TypeType, NamedTupleType, - true_or_false + UninhabitedType, TypeType, true_or_false ) from mypy.maptype import map_instance_to_supertype from mypy.subtypes import is_subtype, is_equivalent, is_subtype_ignoring_tvars @@ -237,19 +236,6 @@ def visit_tuple_type(self, t: TupleType) -> Type: else: return self.default(self.s) - def visit_namedtuple_type(self, t: NamedTupleType) -> Type: - tuple_type = self.visit_tuple_type(t) - if not isinstance(self.s, NamedTupleType): - return tuple_type - if isinstance(tuple_type, TupleType): - if isinstance(self.s, NamedTupleType) and self.s.attrs == t.attrs: - # we forget about t.name - return self.s.copy_with(items=tuple_type.items) - else: - return tuple_type - else: - return self.default(self.s) - def visit_partial_type(self, t: PartialType) -> Type: # We only have partial information so we can't decide the join result. We should # never get here. diff --git a/mypy/meet.py b/mypy/meet.py index 23bd3b2a58b8c..63e3aaee8bce2 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -4,7 +4,7 @@ from mypy.types import ( Type, AnyType, TypeVisitor, UnboundType, Void, ErrorType, NoneTyp, TypeVarType, Instance, CallableType, TupleType, ErasedType, TypeList, UnionType, PartialType, - DeletedType, UninhabitedType, TypeType, NamedTupleType + DeletedType, UninhabitedType, TypeType ) from mypy.subtypes import is_subtype from mypy.nodes import TypeInfo @@ -255,17 +255,6 @@ def visit_tuple_type(self, t: TupleType) -> Type: else: return self.default(self.s) - def visit_namedtuple_type(self, t: NamedTupleType) -> Type: - tuple_type = self.visit_tuple_type(t) - if isinstance(tuple_type, TupleType): - if isinstance(self.s, NamedTupleType) and self.s.attrs == t.attrs: - # we forget about t.name - return self.s.copy_with(items=tuple_type.items) - else: - return tuple_type - else: - return self.default(self.s) - def visit_partial_type(self, t: PartialType) -> Type: # We can't determine the meet of partial types. We should never get here. assert False, 'Internal error' diff --git a/mypy/nodes.py b/mypy/nodes.py index 29b7d69b3b725..ed9c3ddc817a6 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -2027,7 +2027,7 @@ def deserialize(cls, data: JsonDict) -> 'TypeInfo': return ti -def namedtuple_type_info(tup: 'mypy.types.NamedTupleType', *args) -> TypeInfo: +def namedtuple_type_info(tup: 'mypy.types.TupleType', *args) -> TypeInfo: info = TypeInfo(*args) info.tuple_type = tup info.bases = [tup.fallback] diff --git a/mypy/semanal.py b/mypy/semanal.py index 5a040c94fe951..8c51d8a1c732b 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -76,8 +76,7 @@ from mypy.types import ( NoneTyp, CallableType, Overloaded, Instance, Type, TypeVarType, AnyType, FunctionLike, UnboundType, TypeList, TypeVarDef, Void, - replace_leading_arg_type, TupleType, UnionType, StarType, EllipsisType, - NamedTupleType, TypeType + replace_leading_arg_type, TupleType, UnionType, StarType, EllipsisType ) from mypy.nodes import function_type, implicit_module_attrs from mypy.typeanal import TypeAnalyser, TypeAnalyserPass3, analyze_type_alias @@ -1658,10 +1657,10 @@ def fail_namedtuple_arg(self, message: str, def build_namedtuple_typeinfo(self, name: str, items: List[str], types: List[Type]) -> TypeInfo: symbols = SymbolTable() - tup = NamedTupleType(name, items, types, self.named_type('__builtins__.tuple', types)) + tup = TupleType(types, self.named_type('__builtins__.tuple', types)) class_def = ClassDef(name, Block([])) class_def.fullname = self.qualified_name(name) - self.enter_class(class_def) + info = namedtuple_type_info(tup, symbols, class_def) vars = [Var(item, typ) for item, typ in zip(items, types)] @@ -1696,7 +1695,6 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], args=[Argument(Var('iterable', union), union, None, ARG_POS), Argument(Var('new'), AnyType(), EllipsisExpr(), ARG_NAMED), Argument(Var('len'), AnyType(), EllipsisExpr(), ARG_NAMED)]) - self.leave_class() return info def add_namedtuple_field(self, symbols: SymbolTable, info: TypeInfo, var: Var, @@ -2832,7 +2830,7 @@ def self_type(typ: TypeInfo) -> Union[Instance, TupleType]: inst = Instance(typ, tv) if typ.tuple_type is None: return inst - return typ.tuple_type.copy_with(fallback=inst) + return typ.tuple_type.copy_modified(fallback=inst) def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: diff --git a/mypy/subtypes.py b/mypy/subtypes.py index da34d9ef49541..32b8d4d3bcd46 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -2,7 +2,7 @@ from mypy.types import ( Type, AnyType, UnboundType, TypeVisitor, ErrorType, Void, NoneTyp, - Instance, TypeVarType, CallableType, TupleType, NamedTupleType, UnionType, Overloaded, + Instance, TypeVarType, CallableType, TupleType, UnionType, Overloaded, ErasedType, TypeList, PartialType, DeletedType, UninhabitedType, TypeType, is_named_instance ) import mypy.applytype @@ -190,14 +190,6 @@ def visit_tuple_type(self, left: TupleType) -> bool: else: return False - def visit_namedtuple_type(self, left: NamedTupleType) -> bool: - if isinstance(self.right, NamedTupleType): - # left.attrs.issubset(right.attrs) is insufficient - # (a: int, b: str) is not subtype of (b: int, a: str) - if any(l != r for l, r in zip(left.attrs, self.right.attrs)): - return False - return self.visit_tuple_type(left) - def visit_overloaded(self, left: Overloaded) -> bool: right = self.right if isinstance(right, Instance): diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 6de1f42d4a50c..0fe5c451f2b2d 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -3,7 +3,7 @@ from typing import Callable, cast, List from mypy.types import ( - Type, UnboundType, TypeVarType, TupleType, NamedTupleType, UnionType, Instance, + Type, UnboundType, TypeVarType, TupleType, UnionType, Instance, AnyType, CallableType, Void, NoneTyp, DeletedType, TypeList, TypeVarDef, TypeVisitor, StarType, PartialType, EllipsisType, UninhabitedType, TypeType ) @@ -173,8 +173,8 @@ def visit_unbound_type(self, t: UnboundType) -> Type: if t.args: self.fail('Generic tuple types not supported', t) return AnyType() - return tup.copy_with(items=self.anal_array(tup.items), - fallback=instance) + return tup.copy_modified(items=self.anal_array(tup.items), + fallback=instance) else: return AnyType() @@ -220,10 +220,6 @@ def visit_tuple_type(self, t: TupleType) -> Type: fallback = t.fallback if t.fallback else self.builtin_type('builtins.tuple', [AnyType()]) return TupleType(self.anal_array(t.items), fallback, t.line) - def visit_namedtuple_type(self, t: NamedTupleType) -> Type: - return NamedTupleType(t.name, t.attrs, self.anal_array(t.items), - fallback=t.fallback, line=t.line) - def visit_star_type(self, t: StarType) -> Type: return StarType(t.type.accept(self), t.line) diff --git a/mypy/types.py b/mypy/types.py index 9ee790cbfe507..0b6c46970e7fa 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -776,7 +776,7 @@ def deserialize(cls, data: JsonDict) -> 'TupleType': Instance.deserialize(data['fallback']), implicit=data['implicit']) - def copy_with(self, *, fallback: Instance = None, + def copy_modified(self, *, fallback: Instance = None, items: List[Type] = None) -> 'TupleType': if fallback is None: fallback = self.fallback @@ -789,56 +789,6 @@ def slice(self, begin: int, stride: int, end: int) -> 'TupleType': self.line, self.implicit) -class NamedTupleType(TupleType): - """The namedtuple type NamedTuple[name, (a1, T1), ..., (an, Tn)]. - Subtype of Tuple[T1, ..., Tn] - - Instance variables: - name: type name - attrs: names of the attributes - - Uses the default TupleType.fallback: builtins.tuple - """ - - attrs = None # type: List[str] - name = None # type: str - - def __init__(self, name: str, attrs: List[str], *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - assert len(self.items) == len(attrs) - self.attrs = attrs - self.name = name - - def copy_with(self, *, fallback: Instance = None, - items: List[Type] = None) -> 'NamedTupleType': - if fallback is None: - fallback = self.fallback - if items is None: - items = self.items - return NamedTupleType(self.name, self.attrs, items, fallback, self.line) - - def serialize(self) -> JsonDict: - d = super().serialize() - d.update({'.class': 'NamedTupleType', - 'name': self.name, - 'attrs': self.attrs}) - return d - - @classmethod - def deserialize(cls, data: JsonDict) -> 'NamedTupleType': - assert data['.class'] == 'NamedTupleType' - data['.class'] = 'TupleType' - tup = TupleType.deserialize(data) - return NamedTupleType(data['name'], - data['attrs'], - tup.items, - tup.fallback, - implicit=tup.implicit) - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - return visitor.visit_namedtuple_type(self) - - class StarType(Type): """The star type *type_parameter. @@ -1108,9 +1058,6 @@ def visit_overloaded(self, t: Overloaded) -> T: def visit_tuple_type(self, t: TupleType) -> T: pass - def visit_namedtuple_type(self, t: NamedTupleType) -> T: - return self.visit_tuple_type(t) - def visit_star_type(self, t: StarType) -> T: raise self._notimplemented_helper('star_type') @@ -1183,10 +1130,6 @@ def visit_tuple_type(self, t: TupleType) -> Type: cast(Any, t.fallback.accept(self)), t.line) - def visit_namedtuple_type(self, t: NamedTupleType) -> Type: - return t.copy_with(items=self.translate_types(t.items), - fallback=cast(Any, t.fallback.accept(self))) - def visit_star_type(self, t: StarType) -> Type: return StarType(t.type.accept(self), t.line) @@ -1322,11 +1265,6 @@ def visit_tuple_type(self, t: TupleType) -> str: return 'Tuple[{}, fallback={}]'.format(s, t.fallback.accept(self)) return 'Tuple[{}]'.format(s) - def visit_namedtuple_type(self, t: NamedTupleType) -> str: - s = ', '.join('({}, {})'.format(repr(attr), item.accept(self)) - for attr, item in zip(t.attrs, t.items)) - return '{}[{}]'.format(t.name, s) - def visit_star_type(self, t: StarType) -> str: s = t.type.accept(self) return '*{}'.format(s) From 71f0d781fcfaf01c2dd51185d016f17a0643e7a4 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Wed, 31 Aug 2016 02:53:28 +0300 Subject: [PATCH 69/74] restore semanal test --- test-data/unit/semanal-namedtuple.test | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test-data/unit/semanal-namedtuple.test b/test-data/unit/semanal-namedtuple.test index 78a678b974adf..9ff9f6af6e8a1 100644 --- a/test-data/unit/semanal-namedtuple.test +++ b/test-data/unit/semanal-namedtuple.test @@ -1,6 +1,6 @@ -- Semantic analysis of named tuples -[case testSimpleNamedTuple] +[case testSimpleNamedtuple] from collections import namedtuple N = namedtuple('N', ['a']) def f() -> N: pass @@ -9,14 +9,14 @@ MypyFile:1( ImportFrom:1(collections, [namedtuple]) AssignmentStmt:2( NameExpr(N* [__main__.N]) - NamedTupleExpr:2(N, N[('a', Any)])) + NamedTupleExpr:2(N, Tuple[Any])) FuncDef:3( f - def () -> N[('a', Any)] + def () -> Tuple[Any, fallback=__main__.N] Block:3( PassStmt:3()))) -[case testTwoItemNamedTuple] +[case testTwoItemNamedtuple] from collections import namedtuple N = namedtuple('N', ['a', 'xyz']) def f() -> N: pass @@ -25,14 +25,14 @@ MypyFile:1( ImportFrom:1(collections, [namedtuple]) AssignmentStmt:2( NameExpr(N* [__main__.N]) - NamedTupleExpr:2(N, N[('a', Any), ('xyz', Any)])) + NamedTupleExpr:2(N, Tuple[Any, Any])) FuncDef:3( f - def () -> N[('a', Any), ('xyz', Any)] + def () -> Tuple[Any, Any, fallback=__main__.N] Block:3( PassStmt:3()))) -[case testTwoItemNamedTupleWithShorthandSyntax] +[case testTwoItemNamedtupleWithShorthandSyntax] from collections import namedtuple N = namedtuple('N', ' a xyz ') def f() -> N: pass @@ -41,10 +41,10 @@ MypyFile:1( ImportFrom:1(collections, [namedtuple]) AssignmentStmt:2( NameExpr(N* [__main__.N]) - NamedTupleExpr:2(N, N[('a', Any), ('xyz', Any)])) + NamedTupleExpr:2(N, Tuple[Any, Any])) FuncDef:3( f - def () -> N[('a', Any), ('xyz', Any)] + def () -> Tuple[Any, Any, fallback=__main__.N] Block:3( PassStmt:3()))) @@ -57,7 +57,7 @@ MypyFile:1( ImportFrom:1(typing, [NamedTuple]) AssignmentStmt:2( NameExpr(N* [__main__.N]) - NamedTupleExpr:2(N, N[('a', builtins.int), ('b', builtins.str)]))) + NamedTupleExpr:2(N, Tuple[builtins.int, builtins.str]))) [case testNamedTupleBaseClass] from collections import namedtuple @@ -68,11 +68,11 @@ MypyFile:1( ImportFrom:1(collections, [namedtuple]) AssignmentStmt:2( NameExpr(N* [__main__.N]) - NamedTupleExpr:2(N, N[('x', Any)])) + NamedTupleExpr:2(N, Tuple[Any])) ClassDef:3( A TupleType( - N[('x', Any)]) + Tuple[Any, fallback=__main__.N]) BaseType( __main__.N) PassStmt:3())) From 66a77fba8602494c0117928047040a5f99a2bf13 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Thu, 1 Sep 2016 04:38:52 +0300 Subject: [PATCH 70/74] fix signatures --- mypy/join.py | 5 +- mypy/nodes.py | 4 +- mypy/semanal.py | 76 ++++++++++---------- test-data/unit/check-namedtuple.test | 92 ++++++++++++++----------- test-data/unit/check-python2.test | 2 + test-data/unit/fixtures/list.pyi | 1 + test-data/unit/fixtures/property.pyi | 1 + test-data/unit/lib-stub/collections.pyi | 6 +- test-data/unit/lib-stub/typing.pyi | 1 + 9 files changed, 107 insertions(+), 81 deletions(-) diff --git a/mypy/join.py b/mypy/join.py index c639b3648984a..c6d63331b233d 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -231,8 +231,9 @@ def visit_tuple_type(self, t: TupleType) -> Type: items = [] # type: List[Type] for i in range(t.length()): items.append(self.join(t.items[i], self.s.items[i])) - # TODO: What if the fallback types are different? - return TupleType(items, t.fallback) + # join fallback types if they are different + from typing import cast + return TupleType(items, cast(Instance, join_instances(self.s.fallback, t.fallback))) else: return self.default(self.s) diff --git a/mypy/nodes.py b/mypy/nodes.py index ed9c3ddc817a6..46ff236e62476 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -2027,8 +2027,8 @@ def deserialize(cls, data: JsonDict) -> 'TypeInfo': return ti -def namedtuple_type_info(tup: 'mypy.types.TupleType', *args) -> TypeInfo: - info = TypeInfo(*args) +def namedtuple_type_info(tup: 'mypy.types.TupleType', names: 'SymbolTable', defn: ClassDef) -> TypeInfo: + info = TypeInfo(names, defn) info.tuple_type = tup info.bases = [tup.fallback] info.is_named_tuple = True diff --git a/mypy/semanal.py b/mypy/semanal.py index e988ddd564402..27880e4348eb4 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -43,9 +43,6 @@ traverse the entire AST. """ -from functools import partial - -import sys from typing import ( List, Dict, Set, Tuple, cast, Any, TypeVar, Union, Optional, Callable ) @@ -62,7 +59,7 @@ SymbolTableNode, BOUND_TVAR, UNBOUND_TVAR, ListComprehension, GeneratorExpr, FuncExpr, MDEF, FuncBase, Decorator, SetExpr, TypeVarExpr, NewTypeExpr, StrExpr, BytesExpr, PrintStmt, ConditionalExpr, PromoteExpr, - ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, ARG_OPT, MroError, type_aliases, + ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, MroError, type_aliases, YieldFromExpr, NamedTupleExpr, NonlocalDecl, SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr, YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, AwaitExpr, @@ -75,9 +72,8 @@ from mypy.errors import Errors, report_internal_error from mypy.types import ( NoneTyp, CallableType, Overloaded, Instance, Type, TypeVarType, AnyType, - FunctionLike, UnboundType, TypeList, TypeVarDef, Void, - replace_leading_arg_type, TupleType, UnionType, StarType, EllipsisType -) + FunctionLike, UnboundType, TypeList, TypeVarDef, + replace_leading_arg_type, TupleType, UnionType, StarType, EllipsisType, TypeType) from mypy.nodes import function_type, implicit_module_attrs from mypy.typeanal import TypeAnalyser, TypeAnalyserPass3, analyze_type_alias from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError @@ -319,9 +315,8 @@ def visit_func_def(self, defn: FuncDef) -> None: # A coroutine defined as `async def foo(...) -> T: ...` # has external return type `Awaitable[T]`. defn.type = defn.type.copy_modified( - ret_type=Instance( - self.named_type_or_none('typing.Awaitable').type, - [defn.type.ret_type])) + ret_type=self.external_instance('typing.Awaitable', + [defn.type.ret_type])) self.errors.pop_function() def prepare_method_signature(self, func: FuncDef) -> None: @@ -874,6 +869,9 @@ def named_type_or_none(self, qualified_name: str) -> Instance: return None return Instance(cast(TypeInfo, sym.node), []) + def external_instance(self, qualified_name: str, args: List[Type] = None) -> Instance: + return Instance(self.named_type_or_none(qualified_name).type, args or None) + def is_instance_type(self, t: Type) -> bool: return isinstance(t, Instance) @@ -1642,7 +1640,7 @@ def parse_namedtuple_args(self, call: CallExpr, if (fullname == 'collections.namedtuple' and isinstance(args[1], (StrExpr, BytesExpr, UnicodeExpr))): str_expr = cast(StrExpr, args[1]) - items = str_expr.value.split() + items = str_expr.value.replace(',', ' ').split() else: return self.fail_namedtuple_arg( "List literal expected as the second argument to namedtuple()", call) @@ -1696,51 +1694,53 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], tup = TupleType(types, self.named_type('__builtins__.tuple', types)) class_def = ClassDef(name, Block([])) class_def.fullname = self.qualified_name(name) - info = namedtuple_type_info(tup, symbols, class_def) vars = [Var(item, typ) for item, typ in zip(items, types)] this_type = self_type(info) - add_field = partial(self.add_namedtuple_field, symbols, info) - strtype = self.named_type('__builtins__.str') + strtype = self.named_type('__builtins__.str') # type: Type + + def add_field(var: Var, is_initialized_in_class: bool = False, + is_property: bool = False) -> None: + var.info = info + var.is_initialized_in_class = is_initialized_in_class + var.is_property = is_property + symbols[var.name()] = SymbolTableNode(MDEF, var) for var in vars: add_field(var, is_property=True) tuple_of_strings = TupleType([strtype for _ in items], self.named_type('__builtins__.tuple', [AnyType()])) - add_field(Var('_fields', tuple_of_strings), - is_initialized_in_class=True) - add_field(Var('_field_types', UnboundType('Dict', [strtype, AnyType()])), - is_initialized_in_class=True) - add_field(Var('_source', strtype), - is_initialized_in_class=True) - - add_method = partial(self.add_namedtuple_method, symbols, info, this_type) + # TODO: refine type of values + dictype = self.external_instance('typing.Dict', [strtype, AnyType()]) + add_field(Var('_fields', tuple_of_strings), is_initialized_in_class=True) + add_field(Var('_field_types', dictype), is_initialized_in_class=True) + add_field(Var('_source', strtype), is_initialized_in_class=True) + + def add_method(funcname: str, ret: Type, args: List[Argument], name=None, + is_classmethod=False) -> None: + self.add_namedtuple_method(symbols, info, this_type, + funcname, ret, args, name, is_classmethod) + add_method('_replace', ret=this_type, args=self.factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) add_method('__init__', ret=NoneTyp(), args=self.factory_args(vars, ARG_POS), name=info.name()) - # TODO: refine to OrderedDict[str, Union[types]] - add_method('_asdict', ret=UnboundType('collections.OrderedDict', is_ret_type=True), - args=[]) - - # TODO: refine signature - union = UnboundType('Iterable', [UnionType(types)]) + # actual signature should return OrderedDict[str, Union[types]] + dictype = self.external_instance('typing.OrderedDict', [strtype, AnyType()]) + add_method('_asdict', args=[], ret=dictype) + + # note: actual signature should accept an invariant version of Iterable[UnionType[types]]. + # but it can't be expressed + # 'new' and 'len' as function types. + iterable_type = self.external_instance('typing.Iterable', [AnyType()]) add_method('_make', ret=this_type, is_classmethod=True, - args=[Argument(Var('iterable', union), union, None, ARG_POS), + args=[Argument(Var('iterable', iterable_type), iterable_type, None, ARG_POS), Argument(Var('new'), AnyType(), EllipsisExpr(), ARG_NAMED), Argument(Var('len'), AnyType(), EllipsisExpr(), ARG_NAMED)]) return info - def add_namedtuple_field(self, symbols: SymbolTable, info: TypeInfo, var: Var, - is_initialized_in_class: bool = False, - is_property: bool = False) -> None: - var.info = info - var.is_initialized_in_class = is_initialized_in_class - var.is_property = is_property - symbols[var.name()] = SymbolTableNode(MDEF, var) - def factory_args(self, vars: List[Var], kind: int, initializer: Expression = None) -> List[Argument]: return [Argument(var, var.type, initializer, kind) for var in vars] @@ -2511,6 +2511,8 @@ def fail(self, msg: str, ctx: Context, serious: bool = False, *, self.function_stack and self.function_stack[-1].is_dynamic()): return + # In case it's a bug and we don't really have context + assert ctx is not None, msg self.errors.report(ctx.get_line(), msg, blocker=blocker) def fail_blocker(self, msg: str, ctx: Context) -> None: diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index f4a5d2d35b71c..525bed76d8322 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -9,6 +9,7 @@ a = x[1] a, b, c = x # E: Need more than 2 values to unpack (3 expected) x[2] # E: Tuple index out of range + [case testNamedTupleAccessingAttributes] from collections import namedtuple @@ -18,6 +19,7 @@ x.x x.y x.z # E: "X" has no attribute "z" + [case testNamedTupleAttributesAreReadOnly] from collections import namedtuple @@ -33,6 +35,7 @@ a.x = 5 # E: Property "x" defined in "A" is read-only a.y = 5 # E: Property "y" defined in "A" is read-only -- a.z = 5 # not supported yet + [case testNamedTupleCreateWithPositionalArguments] from collections import namedtuple @@ -52,6 +55,7 @@ x = X(1, y='x') x = X(x=1, z=1) # E: Unexpected keyword argument "z" for "X" x = X(y=1) # E: Missing positional argument "x" in call to "X" + [case testNamedTupleCreateAndUseAsTuple] from collections import namedtuple @@ -60,6 +64,7 @@ x = X(1, 'x') a, b = x a, b, c = x # E: Need more than 2 values to unpack (3 expected) + [case testNamedTupleWithItemTypes] from typing import NamedTuple N = NamedTuple('N', [('a', int), @@ -72,6 +77,7 @@ i = n.b # type: int # E: Incompatible types in assignment (expression has type x, y = n x = y # E: Incompatible types in assignment (expression has type "str", variable has type "int") + [case testNamedTupleConstructorArgumentTypes] from typing import NamedTuple N = NamedTuple('N', [('a', int), @@ -106,6 +112,7 @@ s = x.a # E: Incompatible types in assignment (expression has type "int", varia i, s = x s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") + [case testNamedTuplesTwoAsBaseClasses] from typing import NamedTuple A = NamedTuple('A', [('a', int)]) @@ -113,12 +120,14 @@ B = NamedTuple('B', [('a', int)]) class X(A, B): # E: Class has two incompatible bases derived from tuple pass + [case testNamedTuplesTwoAsBaseClasses2] from typing import NamedTuple A = NamedTuple('A', [('a', int)]) class X(A, NamedTuple('B', [('a', int)])): # E: Class has two incompatible bases derived from tuple pass + [case testNamedTupleSelfTypeWithNamedTupleAsBase] from typing import NamedTuple A = NamedTuple('A', [('a', int), ('b', str)]) @@ -131,6 +140,8 @@ class B(A): i, s = self i, i = self # E: Incompatible types in assignment (expression has type "str", \ variable has type "int") + + [out] main: note: In member "f" of class "B": @@ -148,6 +159,7 @@ class B(A): variable has type "str") i, i = self # E: Incompatible types in assignment (expression has type "str", \ variable has type "int") + [out] main: note: In member "f" of class "B": @@ -166,6 +178,7 @@ t = (1, '') t = b a = b + [case testNamedTupleSimpleTypeInference] from typing import NamedTuple, Tuple A = NamedTuple('A', [('a', int)]) @@ -184,10 +197,11 @@ import collections MyNamedTuple = collections.namedtuple('MyNamedTuple', ['spam', 'eggs']) MyNamedTuple.x # E: "MyNamedTuple" has no attribute "x" + [case testNamedTupleEmptyItems] from typing import NamedTuple A = NamedTuple('A', []) -[builtins fixtures/list.pyi] + [case testNamedTupleProperty] from typing import NamedTuple @@ -199,6 +213,7 @@ class B(A): class C(B): pass B(1).b C(2).b + [builtins fixtures/property.pyi] [case testNamedTupleAsDict] @@ -206,20 +221,19 @@ from collections import namedtuple, OrderedDict X = namedtuple('X', ['x', 'y']) x = None # type: X -a = None # type: OrderedDict -a = x._asdict() +reveal_type(x._asdict()) # E: Revealed type is 'typing.OrderedDict[builtins.str, Any]' [case testNamedTupleReplace] from collections import namedtuple X = namedtuple('X', ['x', 'y']) x = None # type: X -x = x._replace() -x = x._replace(y=5) -x = x._replace(x=3) -x = x._replace(x=3, y=5) -x = x._replace(z=5) # E: Unexpected keyword argument "z" for X._replace -x = x._replace(5) # E: Too many positional arguments for X._replace +reveal_type(x._replace()) # E: Revealed type is 'Tuple[Any, Any, fallback=__main__.X]' +x._replace(y=5) +x._replace(x=3) +x._replace(x=3, y=5) +x._replace(z=5) # E: Unexpected keyword argument "z" for X._replace +x._replace(5) # E: Too many positional arguments for X._replace [case testNamedTupleReplaceAsClass] from collections import namedtuple @@ -229,21 +243,28 @@ x = None # type: X X._replace(x, x=1, y=2) X._replace(x=1, y=2) # E: Missing positional argument "self" in call to X._replace + [case testNamedTupleReplaceTyped] from typing import NamedTuple X = NamedTuple('X', [('x', int), ('y', str)]) x = None # type: X -x = x._replace(x=5) -x = x._replace(y=5) # E: Argument 1 to X._replace has incompatible type "int"; expected "str" +reveal_type(x._replace()) # E: Revealed type is 'Tuple[builtins.int, builtins.str, fallback=__main__.X]' +x._replace(x=5) +x._replace(y=5) # E: Argument 1 to X._replace has incompatible type "int"; expected "str" [case testNamedTupleMake] from typing import NamedTuple X = NamedTuple('X', [('x', int), ('y', str)]) -x = None # type: X -x = X._make([5, 'a']) +reveal_type(X._make([5, 'a'])) # E: Revealed type is 'Tuple[builtins.int, builtins.str, fallback=__main__.X]' +X._make('a b') # E: Argument 1 to X._make has incompatible type "str"; expected Iterable[Any] + +-- # FIX: not a proper class method +-- x = None # type: X +-- reveal_type(x._make([5, 'a'])) # E: Revealed type is 'Tuple[builtins.int, builtins.str, fallback=__main__.X]' +-- x._make('a b') # E: Argument 1 to X._make has incompatible type "str"; expected Iterable[Any] [builtins fixtures/list.pyi] @@ -251,53 +272,46 @@ x = X._make([5, 'a']) from typing import NamedTuple X = NamedTuple('X', [('x', int), ('y', str)]) -a = None # type: str -b = None # type: str -a, b = X._fields -x = None # type: X -a, b = x._fields -c = None # type: int -a, c = X._fields # E: Incompatible types in assignment (expression has type "str", variable has type "int") +reveal_type(X._fields) # E: Revealed type is 'Tuple[builtins.str, builtins.str]' [case testNamedTupleSource] from typing import NamedTuple X = NamedTuple('X', [('x', int), ('y', str)]) +reveal_type(X._source) # E: Revealed type is 'builtins.str' x = None # type: X -a = None # type: str -b = None # type: int -a = X._source -b = X._source # E: Incompatible types in assignment (expression has type "str", variable has type "int") +reveal_type(x._source) # E: Revealed type is 'builtins.str' [case testNamedTupleUnit] from typing import NamedTuple X = NamedTuple('X', []) x = X() # type: X -x = x._replace() -_ = x._fields[0] # E: Tuple index out of range +x._replace() +x._fields[0] # E: Tuple index out of range [case testNamedTupleJoinNamedTuple] from typing import NamedTuple X = NamedTuple('X', [('x', int), ('y', str)]) Y = NamedTuple('Y', [('x', int), ('y', str)]) -xy = None -if None: - xy = X(3, 'b') -else: - xy = Y(1, 'a') -xy.x -xy.y +reveal_type([X(3, 'b'), Y(1, 'a')]) # E: Revealed type is 'builtins.list[Tuple[builtins.int, builtins.str]]' + +[builtins fixtures/list.pyi] [case testNamedTupleJoinTuple] from typing import NamedTuple, Tuple X = NamedTuple('X', [('x', int), ('y', str)]) -y = None # type: Tuple[int, str] -if (lambda : ...)(): - y = (3, 'b') -else: - y = X(1, 'a') +reveal_type([(3, 'b'), X(1, 'a')]) # E: Revealed type is 'builtins.list[Tuple[builtins.int, builtins.str]]' +reveal_type([X(1, 'a'), (3, 'b')]) # E: Revealed type is 'builtins.list[Tuple[builtins.int, builtins.str]]' + +[builtins fixtures/list.pyi] -reveal_type(y) # E: Revealed type is 'Tuple[builtins.int, builtins.str]' +[case testNamedTupleFieldTypes] +from typing import NamedTuple + +X = NamedTuple('X', [('x', int), ('y', str)]) +reveal_type(X._field_types) # E: Revealed type is 'typing.Dict[builtins.str, Any]' +x = None # type: X +reveal_type(x._field_types) # E: Revealed type is 'typing.Dict[builtins.str, Any]' diff --git a/test-data/unit/check-python2.test b/test-data/unit/check-python2.test index 4980d47210d26..f8d42f97f5f54 100644 --- a/test-data/unit/check-python2.test +++ b/test-data/unit/check-python2.test @@ -19,6 +19,8 @@ from collections import namedtuple N = NamedTuple(u'N', [(u'x', int)]) n = namedtuple(u'n', u'x y') +[builtins fixtures/dict.pyi] + [case testPrintStatement] print ''() # E: "str" not callable print 1, 1() # E: "int" not callable diff --git a/test-data/unit/fixtures/list.pyi b/test-data/unit/fixtures/list.pyi index 220ab529b8186..ae05dd99cc83b 100644 --- a/test-data/unit/fixtures/list.pyi +++ b/test-data/unit/fixtures/list.pyi @@ -26,3 +26,4 @@ class tuple: pass class function: pass class int: pass class str: pass +class dict: pass \ No newline at end of file diff --git a/test-data/unit/fixtures/property.pyi b/test-data/unit/fixtures/property.pyi index 503bc1aba915c..f704c96b85d20 100644 --- a/test-data/unit/fixtures/property.pyi +++ b/test-data/unit/fixtures/property.pyi @@ -14,3 +14,4 @@ class int: pass class str: pass class bytes: pass class tuple: pass +class dict: pass diff --git a/test-data/unit/lib-stub/collections.pyi b/test-data/unit/lib-stub/collections.pyi index fabe356675b4a..d3022bc571cb7 100644 --- a/test-data/unit/lib-stub/collections.pyi +++ b/test-data/unit/lib-stub/collections.pyi @@ -1,4 +1,8 @@ +from typing import Generic, TypeVar + namedtuple = object() +KT = TypeVar('KT') +KV = TypeVar('KV') -class OrderedDict: +class OrderedDict(Generic[KT, KV]): pass diff --git a/test-data/unit/lib-stub/typing.pyi b/test-data/unit/lib-stub/typing.pyi index 73abb3d3ebf4c..dd9df4cd41b8c 100644 --- a/test-data/unit/lib-stub/typing.pyi +++ b/test-data/unit/lib-stub/typing.pyi @@ -21,6 +21,7 @@ no_type_check = 0 # Type aliases. List = 0 Dict = 0 +OrderedDict = 0 Set = 0 T = TypeVar('T') From 922dd912dee46e406adaf77ef070526ac7e23da3 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Thu, 1 Sep 2016 05:24:31 +0300 Subject: [PATCH 71/74] _asdict() -> Dict --- mypy/nodes.py | 3 ++- mypy/semanal.py | 2 +- test-data/unit/check-namedtuple.test | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index 46ff236e62476..259d8f0bd0a9b 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -2027,7 +2027,8 @@ def deserialize(cls, data: JsonDict) -> 'TypeInfo': return ti -def namedtuple_type_info(tup: 'mypy.types.TupleType', names: 'SymbolTable', defn: ClassDef) -> TypeInfo: +def namedtuple_type_info(tup: 'mypy.types.TupleType', + names: 'SymbolTable', defn: ClassDef) -> TypeInfo: info = TypeInfo(names, defn) info.tuple_type = tup info.bases = [tup.fallback] diff --git a/mypy/semanal.py b/mypy/semanal.py index 27880e4348eb4..e80033a19250b 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1728,7 +1728,7 @@ def add_method(funcname: str, ret: Type, args: List[Argument], name=None, add_method('__init__', ret=NoneTyp(), args=self.factory_args(vars, ARG_POS), name=info.name()) # actual signature should return OrderedDict[str, Union[types]] - dictype = self.external_instance('typing.OrderedDict', [strtype, AnyType()]) + dictype = self.external_instance('typing.Dict', [strtype, AnyType()]) add_method('_asdict', args=[], ret=dictype) # note: actual signature should accept an invariant version of Iterable[UnionType[types]]. diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 525bed76d8322..75e4af262453d 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -217,11 +217,11 @@ C(2).b [builtins fixtures/property.pyi] [case testNamedTupleAsDict] -from collections import namedtuple, OrderedDict +from collections import namedtuple X = namedtuple('X', ['x', 'y']) x = None # type: X -reveal_type(x._asdict()) # E: Revealed type is 'typing.OrderedDict[builtins.str, Any]' +reveal_type(x._asdict()) # E: Revealed type is 'typing.Dict[builtins.str, Any]' [case testNamedTupleReplace] from collections import namedtuple From 6460839c7dcb0802bcb4acfca7fa1b7d7fb86227 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Thu, 1 Sep 2016 05:26:44 +0300 Subject: [PATCH 72/74] _asdict() -> OrderedDict --- mypy/semanal.py | 2 +- test-data/unit/check-namedtuple.test | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index e80033a19250b..27880e4348eb4 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1728,7 +1728,7 @@ def add_method(funcname: str, ret: Type, args: List[Argument], name=None, add_method('__init__', ret=NoneTyp(), args=self.factory_args(vars, ARG_POS), name=info.name()) # actual signature should return OrderedDict[str, Union[types]] - dictype = self.external_instance('typing.Dict', [strtype, AnyType()]) + dictype = self.external_instance('typing.OrderedDict', [strtype, AnyType()]) add_method('_asdict', args=[], ret=dictype) # note: actual signature should accept an invariant version of Iterable[UnionType[types]]. diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 75e4af262453d..4e3eb07cf838c 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -221,7 +221,7 @@ from collections import namedtuple X = namedtuple('X', ['x', 'y']) x = None # type: X -reveal_type(x._asdict()) # E: Revealed type is 'typing.Dict[builtins.str, Any]' +reveal_type(x._asdict()) # E: Revealed type is 'typing.OrderedDict[builtins.str, Any]' [case testNamedTupleReplace] from collections import namedtuple From cd5b93336a8254c5bc477ddd44efa8690ca3af99 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Thu, 1 Sep 2016 20:45:26 +0300 Subject: [PATCH 73/74] use buitlins.dict; rearrange build_namedtuple_typeinfo --- mypy/semanal.py | 94 +++++++++++-------------- test-data/unit/check-namedtuple.test | 10 ++- test-data/unit/fixtures/list.pyi | 1 - test-data/unit/lib-stub/collections.pyi | 7 +- test-data/unit/lib-stub/typing.pyi | 1 - 5 files changed, 50 insertions(+), 63 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 27880e4348eb4..7d496676fde7b 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -315,8 +315,8 @@ def visit_func_def(self, defn: FuncDef) -> None: # A coroutine defined as `async def foo(...) -> T: ...` # has external return type `Awaitable[T]`. defn.type = defn.type.copy_modified( - ret_type=self.external_instance('typing.Awaitable', - [defn.type.ret_type])) + ret_type = self.named_type_or_none('typing.Awaitable', + [defn.type.ret_type])) self.errors.pop_function() def prepare_method_signature(self, func: FuncDef) -> None: @@ -863,14 +863,11 @@ def named_type(self, qualified_name: str, args: List[Type] = None) -> Instance: sym = self.lookup_qualified(qualified_name, None) return Instance(cast(TypeInfo, sym.node), args or []) - def named_type_or_none(self, qualified_name: str) -> Instance: + def named_type_or_none(self, qualified_name: str, args: List[Type] = None) -> Instance: sym = self.lookup_fully_qualified_or_none(qualified_name) if not sym: return None - return Instance(cast(TypeInfo, sym.node), []) - - def external_instance(self, qualified_name: str, args: List[Type] = None) -> Instance: - return Instance(self.named_type_or_none(qualified_name).type, args or None) + return Instance(cast(TypeInfo, sym.node), args or []) def is_instance_type(self, t: Type) -> bool: return isinstance(t, Instance) @@ -1690,15 +1687,24 @@ def fail_namedtuple_arg(self, message: str, def build_namedtuple_typeinfo(self, name: str, items: List[str], types: List[Type]) -> TypeInfo: + strtype = self.named_type('__builtins__.str') # type: Type + basetuple_type = self.named_type('__builtins__.tuple', [AnyType()]) + dictype = (self.named_type_or_none('builtins.dict', [strtype, AnyType()]) + or self.object_type()) + # Actual signature should return OrderedDict[str, Union[types]] + ordereddictype = (self.named_type_or_none('builtins.dict', [strtype, AnyType()]) + or self.object_type()) + fallback = self.named_type('__builtins__.tuple', types) + # Note: actual signature should accept an invariant version of Iterable[UnionType[types]]. + # but it can't be expressed. 'new' and 'len' should be callable types. + iterable_type = self.named_type_or_none('typing.Iterable', [AnyType()]) + function_type = self.named_type('__builtins__.function') + fullname = self.qualified_name(name) + symbols = SymbolTable() - tup = TupleType(types, self.named_type('__builtins__.tuple', types)) class_def = ClassDef(name, Block([])) - class_def.fullname = self.qualified_name(name) - info = namedtuple_type_info(tup, symbols, class_def) - - vars = [Var(item, typ) for item, typ in zip(items, types)] - this_type = self_type(info) - strtype = self.named_type('__builtins__.str') # type: Type + class_def.fullname = fullname + info = namedtuple_type_info(TupleType(types, fallback), symbols, class_def) def add_field(var: Var, is_initialized_in_class: bool = False, is_property: bool = False) -> None: @@ -1707,61 +1713,45 @@ def add_field(var: Var, is_initialized_in_class: bool = False, var.is_property = is_property symbols[var.name()] = SymbolTableNode(MDEF, var) + vars = [Var(item, typ) for item, typ in zip(items, types)] for var in vars: add_field(var, is_property=True) - tuple_of_strings = TupleType([strtype for _ in items], - self.named_type('__builtins__.tuple', [AnyType()])) - # TODO: refine type of values - dictype = self.external_instance('typing.Dict', [strtype, AnyType()]) + tuple_of_strings = TupleType([strtype for _ in items], basetuple_type) add_field(Var('_fields', tuple_of_strings), is_initialized_in_class=True) add_field(Var('_field_types', dictype), is_initialized_in_class=True) add_field(Var('_source', strtype), is_initialized_in_class=True) + # TODO: SelfType should be bind to actual 'self' + this_type = self_type(info) + def add_method(funcname: str, ret: Type, args: List[Argument], name=None, is_classmethod=False) -> None: - self.add_namedtuple_method(symbols, info, this_type, - funcname, ret, args, name, is_classmethod) + if not is_classmethod: + args = [Argument(Var('self'), this_type, None, ARG_POS)] + args + types = [arg.type_annotation for arg in args] + items = [arg.variable.name() for arg in args] + arg_kinds = [arg.kind for arg in args] + signature = CallableType(types, arg_kinds, items, ret, function_type, + name=name or info.name() + '.' + funcname) + signature.is_classmethod_class = is_classmethod + func = FuncDef(funcname, args, Block([]), typ=signature) + func.info = info + func.is_class = is_classmethod + symbols[funcname] = SymbolTableNode(MDEF, func) add_method('_replace', ret=this_type, - args=self.factory_args(vars, ARG_NAMED, initializer=EllipsisExpr())) - add_method('__init__', ret=NoneTyp(), - args=self.factory_args(vars, ARG_POS), name=info.name()) - # actual signature should return OrderedDict[str, Union[types]] - dictype = self.external_instance('typing.OrderedDict', [strtype, AnyType()]) - add_method('_asdict', args=[], ret=dictype) - - # note: actual signature should accept an invariant version of Iterable[UnionType[types]]. - # but it can't be expressed - # 'new' and 'len' as function types. - iterable_type = self.external_instance('typing.Iterable', [AnyType()]) + args=[Argument(var, var.type, EllipsisExpr(), ARG_NAMED) for var in vars]) + add_method('__init__', ret=NoneTyp(), name=info.name(), + args=[Argument(var, var.type, None, ARG_POS) for var in vars]) + add_method('_asdict', args=[], ret=ordereddictype) + # FIX: make it actual class method add_method('_make', ret=this_type, is_classmethod=True, args=[Argument(Var('iterable', iterable_type), iterable_type, None, ARG_POS), Argument(Var('new'), AnyType(), EllipsisExpr(), ARG_NAMED), Argument(Var('len'), AnyType(), EllipsisExpr(), ARG_NAMED)]) return info - def factory_args(self, vars: List[Var], kind: int, - initializer: Expression = None) -> List[Argument]: - return [Argument(var, var.type, initializer, kind) for var in vars] - - def add_namedtuple_method(self, symbols: SymbolTable, info: TypeInfo, this_type: Type, - funcname: str, ret: Type, args: List[Argument], name=None, - is_classmethod=False) -> None: - if not is_classmethod: - args = [Argument(Var('self'), this_type, None, ARG_POS)] + args - types = [arg.type_annotation for arg in args] - items = [arg.variable.name() for arg in args] - arg_kinds = [arg.kind for arg in args] - signature = CallableType(types, arg_kinds, items, ret, - self.named_type('__builtins__.function'), - name=name or info.name() + '.' + funcname) - signature.is_classmethod_class = is_classmethod - func = FuncDef(funcname, args, Block([]), typ=signature) - func.info = info - func.is_class = is_classmethod - symbols[funcname] = SymbolTableNode(MDEF, func) - def make_argument(self, name: str, type: Type) -> Argument: return Argument(Var(name), type, None, ARG_POS) diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 4e3eb07cf838c..82cebd1cb6d80 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -221,7 +221,9 @@ from collections import namedtuple X = namedtuple('X', ['x', 'y']) x = None # type: X -reveal_type(x._asdict()) # E: Revealed type is 'typing.OrderedDict[builtins.str, Any]' +reveal_type(x._asdict()) # E: Revealed type is 'builtins.dict[builtins.str, Any]' + +[builtins fixtures/dict.pyi] [case testNamedTupleReplace] from collections import namedtuple @@ -312,6 +314,8 @@ reveal_type([X(1, 'a'), (3, 'b')]) # E: Revealed type is 'builtins.list[Tuple[b from typing import NamedTuple X = NamedTuple('X', [('x', int), ('y', str)]) -reveal_type(X._field_types) # E: Revealed type is 'typing.Dict[builtins.str, Any]' +reveal_type(X._field_types) # E: Revealed type is 'builtins.dict[builtins.str, Any]' x = None # type: X -reveal_type(x._field_types) # E: Revealed type is 'typing.Dict[builtins.str, Any]' +reveal_type(x._field_types) # E: Revealed type is 'builtins.dict[builtins.str, Any]' + +[builtins fixtures/dict.pyi] \ No newline at end of file diff --git a/test-data/unit/fixtures/list.pyi b/test-data/unit/fixtures/list.pyi index ae05dd99cc83b..220ab529b8186 100644 --- a/test-data/unit/fixtures/list.pyi +++ b/test-data/unit/fixtures/list.pyi @@ -26,4 +26,3 @@ class tuple: pass class function: pass class int: pass class str: pass -class dict: pass \ No newline at end of file diff --git a/test-data/unit/lib-stub/collections.pyi b/test-data/unit/lib-stub/collections.pyi index d3022bc571cb7..00b7cea64bebd 100644 --- a/test-data/unit/lib-stub/collections.pyi +++ b/test-data/unit/lib-stub/collections.pyi @@ -1,8 +1,3 @@ -from typing import Generic, TypeVar +import typing namedtuple = object() -KT = TypeVar('KT') -KV = TypeVar('KV') - -class OrderedDict(Generic[KT, KV]): - pass diff --git a/test-data/unit/lib-stub/typing.pyi b/test-data/unit/lib-stub/typing.pyi index dd9df4cd41b8c..73abb3d3ebf4c 100644 --- a/test-data/unit/lib-stub/typing.pyi +++ b/test-data/unit/lib-stub/typing.pyi @@ -21,7 +21,6 @@ no_type_check = 0 # Type aliases. List = 0 Dict = 0 -OrderedDict = 0 Set = 0 T = TypeVar('T') From 500a847adbe13df027472c919e4b4451ee02dbaa Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Thu, 1 Sep 2016 20:52:24 +0300 Subject: [PATCH 74/74] remove unneeded 'class dict' from proprty.pyi --- test-data/unit/fixtures/property.pyi | 1 - 1 file changed, 1 deletion(-) diff --git a/test-data/unit/fixtures/property.pyi b/test-data/unit/fixtures/property.pyi index f704c96b85d20..503bc1aba915c 100644 --- a/test-data/unit/fixtures/property.pyi +++ b/test-data/unit/fixtures/property.pyi @@ -14,4 +14,3 @@ class int: pass class str: pass class bytes: pass class tuple: pass -class dict: pass