From 8e8f0d51e52ec19d2cd13b80fcec847194c64a35 Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Tue, 18 Sep 2018 17:38:46 -0700 Subject: [PATCH 1/3] Make NamedTuple provide __new__ instead of __init__ Closes #1279. --- mypy/checker.py | 4 ++-- mypy/semanal.py | 2 +- mypy/semanal_namedtuple.py | 8 ++++--- test-data/unit/check-class-namedtuple.test | 2 +- test-data/unit/check-namedtuple.test | 12 +++++++++++ test-data/unit/cmdline.test | 4 ++-- test-data/unit/diff.test | 2 +- test-data/unit/merge.test | 4 ++-- test-data/unit/pythoneval.test | 25 +++++++++++++++++++++- 9 files changed, 50 insertions(+), 13 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index adb1e4ab0b726..3f7bc3fc0b03e 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -1563,8 +1563,8 @@ def check_compatibility(self, name: str, base1: TypeInfo, a direct subclass relationship (i.e., the compatibility requirement only derives from multiple inheritance). """ - if name == '__init__': - # __init__ can be incompatible -- it's a special case. + if name in ('__init__', '__new__', '__init_subclass__'): + # __init__ and friends can be incompatible -- it's a special case. return first = base1[name] second = base2[name] diff --git a/mypy/semanal.py b/mypy/semanal.py index 26cdf024f18fd..cf431ed55bfb3 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -776,7 +776,7 @@ def analyze_class_body(self, defn: ClassDef) -> Iterator[bool]: nt_names = named_tuple_info.names named_tuple_info.names = SymbolTable() # This is needed for the cls argument to classmethods to get bound correctly. - named_tuple_info.names['__init__'] = nt_names['__init__'] + named_tuple_info.names['__new__'] = nt_names['__new__'] self.enter_class(named_tuple_info) diff --git a/mypy/semanal_namedtuple.py b/mypy/semanal_namedtuple.py index 0e9088c3fe8cf..115b6253fd51f 100644 --- a/mypy/semanal_namedtuple.py +++ b/mypy/semanal_namedtuple.py @@ -347,8 +347,9 @@ def add_method(funcname: str, args: List[Argument], name: Optional[str] = None, is_classmethod: bool = False, + is_new: bool = False, ) -> None: - if is_classmethod: + if is_classmethod or is_new: first = [Argument(Var('cls'), TypeType.make_normalized(selftype), None, ARG_POS)] else: first = [Argument(Var('self'), selftype, None, ARG_POS)] @@ -384,8 +385,9 @@ def make_init_arg(var: Var) -> Argument: kind = ARG_POS if default is None else ARG_OPT return Argument(var, var.type, default, kind) - add_method('__init__', ret=NoneTyp(), name=info.name(), - args=[make_init_arg(var) for var in vars]) + add_method('__new__', ret=selftype, name=info.name(), + args=[make_init_arg(var) for var in vars], + is_new=True) add_method('_asdict', args=[], ret=ordereddictype) special_form_any = AnyType(TypeOfAny.special_form) add_method('_make', ret=selftype, is_classmethod=True, diff --git a/test-data/unit/check-class-namedtuple.test b/test-data/unit/check-class-namedtuple.test index cf5bb46fc647f..990cba4e6f011 100644 --- a/test-data/unit/check-class-namedtuple.test +++ b/test-data/unit/check-class-namedtuple.test @@ -587,7 +587,7 @@ class XMethBad(NamedTuple): class MagicalFields(NamedTuple): x: int def __slots__(self) -> None: pass # E: Cannot overwrite NamedTuple attribute "__slots__" - def __new__(cls) -> None: pass # E: Cannot overwrite NamedTuple attribute "__new__" + def __new__(cls) -> None: pass # E: Name '__new__' already defined (possibly by an import) def _source(self) -> int: pass # E: Cannot overwrite NamedTuple attribute "_source" __annotations__ = {'x': float} # E: NamedTuple field name cannot start with an underscore: __annotations__ \ # E: Invalid statement in NamedTuple definition; expected "field_name: field_type [= default]" \ diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 8d2aa9abef4d2..3247d6c3cc1a6 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -748,3 +748,15 @@ def f(o: Base) -> None: reveal_type(o.value) # E: Revealed type is 'builtins.float' [builtins fixtures/isinstance.pyi] [out] + +[case testNamedTupleNew] +from typing import NamedTuple + +Base = NamedTuple('Base', [('param', int)]) + +class Child(Base): + def __new__(cls, param: int = 1) -> 'Child': + return Base.__new__(cls, param) + +Base(param=10) +Child(param=10) diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test index 57d4915e6f15a..5dcd1d0bb45c6 100644 --- a/test-data/unit/cmdline.test +++ b/test-data/unit/cmdline.test @@ -1182,7 +1182,7 @@ warn_unused_configs = True [[mypy-emarg.hatch] -- Currently we don't treat an unstructured pattern like a.*.b as unused -- if it matches another section (like a.x.b). This would be reasonable --- to change. +-- to change. ' [[mypy-a.*.b] [[mypy-a.*.c] [[mypy-a.x.b] @@ -1257,6 +1257,6 @@ import d [case testCacheMap] -- This just checks that a valid --cache-map triple is accepted. -- (Errors are too verbose to check.) -# cmd: mypy a.py --cache-map a.py a.meta.json a.data.json +# cmd: mypy a.py --cache-map a.py a.meta.json a.data.json [file a.py] [out] diff --git a/test-data/unit/diff.test b/test-data/unit/diff.test index 0c618d1ed07ec..d5531118de656 100644 --- a/test-data/unit/diff.test +++ b/test-data/unit/diff.test @@ -275,7 +275,7 @@ M = NamedTuple('M', [('x', int), ('y', str)]) [out] __main__.A __main__.N -__main__.N.__init__ +__main__.N.__new__ __main__.N._asdict __main__.N._make __main__.N._replace diff --git a/test-data/unit/merge.test b/test-data/unit/merge.test index 736fa15bfef73..281006557083d 100644 --- a/test-data/unit/merge.test +++ b/test-data/unit/merge.test @@ -668,7 +668,7 @@ TypeInfo<2>( Names( __annotations__<4> (builtins.object<1>) __doc__<5> (builtins.str<6>) - __init__<7> + __new__<7> _asdict<8> _field_defaults<9> (builtins.object<1>) _field_types<10> (builtins.object<1>) @@ -690,7 +690,7 @@ TypeInfo<2>( Names( __annotations__<4> (builtins.object<1>) __doc__<5> (builtins.str<6>) - __init__<7> + __new__<7> _asdict<8> _field_defaults<9> (builtins.object<1>) _field_types<10> (builtins.object<1>) diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index 446cb0f697fd8..bcd30c20f158f 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -26,6 +26,7 @@ print(list(reversed(A()))) [[3, 2, 1] [['c', 'b', 'a'] [['f', 'o', 'o'] +-- ]]]]] [case testIntAndFloatConversion] from typing import SupportsInt, SupportsFloat @@ -90,6 +91,7 @@ import typing print(list.__add__([1, 2], [3, 4])) [out] [[1, 2, 3, 4] +-- ] [case testInheritedClassAttribute] import typing @@ -1047,7 +1049,7 @@ _testTypedDictGet.py:9: error: TypedDict "D" has no key 'z' _testTypedDictGet.py:10: error: All overload variants of "get" of "Mapping" require at least one argument _testTypedDictGet.py:10: note: Possible overload variants: _testTypedDictGet.py:10: note: def get(self, k: str) -> object -_testTypedDictGet.py:10: note: def [_T] get(self, k: str, default: object) -> object +_testTypedDictGet.py:10: note: def [_T] get(self, k: str, default: object) -> object _testTypedDictGet.py:12: error: Revealed type is 'builtins.object*' [case testTypedDictMappingMethods] @@ -1317,3 +1319,24 @@ def g(ms: 'T[M]') -> None: reduce(f, ms) T = Iterable [out] + +[case testNamedTupleNew] +# This is an eval test because there was a snag found only with full stubs +from typing import NamedTuple + +Base = NamedTuple('Base', [('param', int)]) + +class Child(Base): + def __new__(cls, param: int = 1) -> 'Child': + return Base.__new__(cls, param) + +Base(param=10) +Child(param=10) +reveal_type(Child()) + +from collections import namedtuple +X = namedtuple('X', ['a', 'b']) +x = X(a=1, b='s') + +[out] +_testNamedTupleNew.py:12: error: Revealed type is 'Tuple[builtins.int, fallback=_testNamedTupleNew.Child]' From 0f17104b72df23fe68f5dad5c1c7d5217d3cbbfc Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Wed, 19 Sep 2018 11:48:36 -0700 Subject: [PATCH 2/3] Improve the __new__ error message to have a line number --- mypy/semanal_namedtuple.py | 10 ++++++---- test-data/unit/check-class-namedtuple.test | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/mypy/semanal_namedtuple.py b/mypy/semanal_namedtuple.py index 115b6253fd51f..212e7d77254f0 100644 --- a/mypy/semanal_namedtuple.py +++ b/mypy/semanal_namedtuple.py @@ -51,7 +51,7 @@ def analyze_namedtuple_classdef(self, defn: ClassDef) -> Optional[TypeInfo]: node.kind = GDEF # TODO in process_namedtuple_definition also applies here items, types, default_items = self.check_namedtuple_classdef(defn) info = self.build_namedtuple_typeinfo( - defn.name, items, types, default_items) + defn.name, items, types, default_items, defn.line) node.node = info defn.info.replaced = info defn.info = info @@ -160,7 +160,7 @@ def check_namedtuple(self, name = var_name else: name = 'namedtuple@' + str(call.line) - info = self.build_namedtuple_typeinfo(name, [], [], {}) + info = self.build_namedtuple_typeinfo(name, [], [], {}, node.line) self.store_namedtuple_info(info, name, call, is_typed) return info name = cast(Union[StrExpr, BytesExpr, UnicodeExpr], call.args[0]).value @@ -174,7 +174,7 @@ def check_namedtuple(self, } else: default_items = {} - info = self.build_namedtuple_typeinfo(name, items, types, default_items) + info = self.build_namedtuple_typeinfo(name, items, types, default_items, node.line) # Store it as a global just in case it would remain anonymous. # (Or in the nearest class if there is one.) self.store_namedtuple_info(info, name, call, is_typed) @@ -290,7 +290,8 @@ def fail_namedtuple_arg(self, message: str, context: Context return [], [], [], False def build_namedtuple_typeinfo(self, name: str, items: List[str], types: List[Type], - default_items: Mapping[str, Expression]) -> TypeInfo: + default_items: Mapping[str, Expression], + line: int) -> TypeInfo: strtype = self.api.named_type('__builtins__.str') implicit_any = AnyType(TypeOfAny.special_form) basetuple_type = self.api.named_type('__builtins__.tuple', [implicit_any]) @@ -363,6 +364,7 @@ def add_method(funcname: str, function_type) signature.variables = [tvd] func = FuncDef(funcname, args, Block([])) + func.line = line func.info = info func.is_class = is_classmethod func.type = set_callable_name(signature, func) diff --git a/test-data/unit/check-class-namedtuple.test b/test-data/unit/check-class-namedtuple.test index 990cba4e6f011..d310f1b6c9d00 100644 --- a/test-data/unit/check-class-namedtuple.test +++ b/test-data/unit/check-class-namedtuple.test @@ -587,7 +587,7 @@ class XMethBad(NamedTuple): class MagicalFields(NamedTuple): x: int def __slots__(self) -> None: pass # E: Cannot overwrite NamedTuple attribute "__slots__" - def __new__(cls) -> None: pass # E: Name '__new__' already defined (possibly by an import) + def __new__(cls) -> None: pass # E: Name '__new__' already defined on line 8 def _source(self) -> int: pass # E: Cannot overwrite NamedTuple attribute "_source" __annotations__ = {'x': float} # E: NamedTuple field name cannot start with an underscore: __annotations__ \ # E: Invalid statement in NamedTuple definition; expected "field_name: field_type [= default]" \ From 6c836245813abefc8f3e5348d847b977c76b0bc3 Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Wed, 19 Sep 2018 15:00:16 -0700 Subject: [PATCH 3/3] Revert "Improve the __new__ error message to have a line number" This reverts commit 0f17104b72df23fe68f5dad5c1c7d5217d3cbbfc. --- mypy/semanal_namedtuple.py | 10 ++++------ test-data/unit/check-class-namedtuple.test | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/mypy/semanal_namedtuple.py b/mypy/semanal_namedtuple.py index 212e7d77254f0..115b6253fd51f 100644 --- a/mypy/semanal_namedtuple.py +++ b/mypy/semanal_namedtuple.py @@ -51,7 +51,7 @@ def analyze_namedtuple_classdef(self, defn: ClassDef) -> Optional[TypeInfo]: node.kind = GDEF # TODO in process_namedtuple_definition also applies here items, types, default_items = self.check_namedtuple_classdef(defn) info = self.build_namedtuple_typeinfo( - defn.name, items, types, default_items, defn.line) + defn.name, items, types, default_items) node.node = info defn.info.replaced = info defn.info = info @@ -160,7 +160,7 @@ def check_namedtuple(self, name = var_name else: name = 'namedtuple@' + str(call.line) - info = self.build_namedtuple_typeinfo(name, [], [], {}, node.line) + info = self.build_namedtuple_typeinfo(name, [], [], {}) self.store_namedtuple_info(info, name, call, is_typed) return info name = cast(Union[StrExpr, BytesExpr, UnicodeExpr], call.args[0]).value @@ -174,7 +174,7 @@ def check_namedtuple(self, } else: default_items = {} - info = self.build_namedtuple_typeinfo(name, items, types, default_items, node.line) + info = self.build_namedtuple_typeinfo(name, items, types, default_items) # Store it as a global just in case it would remain anonymous. # (Or in the nearest class if there is one.) self.store_namedtuple_info(info, name, call, is_typed) @@ -290,8 +290,7 @@ def fail_namedtuple_arg(self, message: str, context: Context return [], [], [], False def build_namedtuple_typeinfo(self, name: str, items: List[str], types: List[Type], - default_items: Mapping[str, Expression], - line: int) -> TypeInfo: + default_items: Mapping[str, Expression]) -> TypeInfo: strtype = self.api.named_type('__builtins__.str') implicit_any = AnyType(TypeOfAny.special_form) basetuple_type = self.api.named_type('__builtins__.tuple', [implicit_any]) @@ -364,7 +363,6 @@ def add_method(funcname: str, function_type) signature.variables = [tvd] func = FuncDef(funcname, args, Block([])) - func.line = line func.info = info func.is_class = is_classmethod func.type = set_callable_name(signature, func) diff --git a/test-data/unit/check-class-namedtuple.test b/test-data/unit/check-class-namedtuple.test index d310f1b6c9d00..990cba4e6f011 100644 --- a/test-data/unit/check-class-namedtuple.test +++ b/test-data/unit/check-class-namedtuple.test @@ -587,7 +587,7 @@ class XMethBad(NamedTuple): class MagicalFields(NamedTuple): x: int def __slots__(self) -> None: pass # E: Cannot overwrite NamedTuple attribute "__slots__" - def __new__(cls) -> None: pass # E: Name '__new__' already defined on line 8 + def __new__(cls) -> None: pass # E: Name '__new__' already defined (possibly by an import) def _source(self) -> int: pass # E: Cannot overwrite NamedTuple attribute "_source" __annotations__ = {'x': float} # E: NamedTuple field name cannot start with an underscore: __annotations__ \ # E: Invalid statement in NamedTuple definition; expected "field_name: field_type [= default]" \