From ea85fcc9cb4fda05292e57e69dd681a01355f6e8 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Fri, 7 Jan 2022 10:23:56 +0300 Subject: [PATCH 1/7] `Literal[b]` was not properly checked by `stubtest` --- mypy/test/teststubtest.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/mypy/test/teststubtest.py b/mypy/test/teststubtest.py index e6eb8465c6658..10aa68ed7c01c 100644 --- a/mypy/test/teststubtest.py +++ b/mypy/test/teststubtest.py @@ -750,6 +750,30 @@ def __init__(self, x): pass error="X.__init__" ) + @collect_cases + def test_literal(self) -> Iterator[Case]: + yield Case( + stub=r""" + from typing_extensions import Literal + + NUM: Literal[1] + CHAR: Literal['a'] + FLAG: Literal[True] + NON: Literal[None] + BYT1: Literal[b'abc'] + BYT2: Literal[b'\x90'] + """, + runtime=r""" + NUM = 1 + CHAR = 'a' + NON = None + FLAG = True + BYT1 = b"abc" + BYT2 = b'\x90' + """, + error=None, + ) + def remove_color_code(s: str) -> str: return re.sub("\\x1b.*?m", "", s) # this works! From 653bab0bec27bfbce95f319412846eac7198d866 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Fri, 7 Jan 2022 11:10:09 +0300 Subject: [PATCH 2/7] Now `Literal[bytes]` compares correctly --- mypy/fastparse.py | 15 +------ mypy/fastparse2.py | 3 +- mypy/stubtest.py | 11 ++++- mypy/test/teststubtest.py | 84 ++++++++++++++++++++++++++++++++++++++- mypy/util.py | 14 +++++++ 5 files changed, 110 insertions(+), 17 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 34fe2c0da32d3..f3dd1f0a8fefb 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -40,6 +40,7 @@ from mypy.errors import Errors from mypy.options import Options from mypy.reachability import mark_block_unreachable +from mypy.util import bytes_to_human_readable_repr try: # pull this into a final variable to make mypyc be quiet about the @@ -1638,17 +1639,3 @@ def stringify_name(n: AST) -> Optional[str]: if sv is not None: return "{}.{}".format(sv, n.attr) return None # Can't do it. - - -def bytes_to_human_readable_repr(b: bytes) -> str: - """Converts bytes into some human-readable representation. Unprintable - bytes such as the nul byte are escaped. For example: - - >>> b = bytes([102, 111, 111, 10, 0]) - >>> s = bytes_to_human_readable_repr(b) - >>> print(s) - foo\n\x00 - >>> print(repr(s)) - 'foo\\n\\x00' - """ - return repr(b)[2:-1] diff --git a/mypy/fastparse2.py b/mypy/fastparse2.py index 2d288bf158e5c..bf3c09453ec0a 100644 --- a/mypy/fastparse2.py +++ b/mypy/fastparse2.py @@ -47,10 +47,11 @@ from mypy import message_registry, errorcodes as codes from mypy.errors import Errors from mypy.fastparse import ( - TypeConverter, parse_type_comment, bytes_to_human_readable_repr, parse_type_ignore_tag, + TypeConverter, parse_type_comment, parse_type_ignore_tag, TYPE_IGNORE_PATTERN, INVALID_TYPE_IGNORE ) from mypy.options import Options +from mypy.util import bytes_to_human_readable_repr from mypy.reachability import mark_block_unreachable try: diff --git a/mypy/stubtest.py b/mypy/stubtest.py index 7228afaed446f..4ebfa3248071e 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -25,7 +25,7 @@ from mypy import nodes from mypy.config_parser import parse_config_file from mypy.options import Options -from mypy.util import FancyFormatter +from mypy.util import FancyFormatter, bytes_to_human_readable_repr class Missing: @@ -942,6 +942,15 @@ def is_subtype_helper(left: mypy.types.Type, right: mypy.types.Type) -> bool: ): # Pretend Literal[0, 1] is a subtype of bool to avoid unhelpful errors. return True + if ( + isinstance(left, mypy.types.LiteralType) + and isinstance(left.value, bytes) + and isinstance(right, mypy.types.LiteralType) + and isinstance(right.value, str) + and right.fallback.type.fullname == 'builtins.bytes' + ): + # This is a representation mismatch. + return bytes_to_human_readable_repr(left.value) == right.value with mypy.state.strict_optional_set(True): return mypy.subtypes.is_subtype(left, right) diff --git a/mypy/test/teststubtest.py b/mypy/test/teststubtest.py index 10aa68ed7c01c..4fb97076340ea 100644 --- a/mypy/test/teststubtest.py +++ b/mypy/test/teststubtest.py @@ -751,7 +751,7 @@ def __init__(self, x): pass ) @collect_cases - def test_literal(self) -> Iterator[Case]: + def test_correct_literal(self) -> Iterator[Case]: yield Case( stub=r""" from typing_extensions import Literal @@ -774,6 +774,88 @@ def test_literal(self) -> Iterator[Case]: error=None, ) + @collect_cases + def test_invalid_int_float_literal(self) -> Iterator[Case]: + yield Case( + stub=""" + from typing_extensions import Literal + + WRONG_NUM: Literal[1] + """, + runtime='WRONG_NUM = 1.0', + error='WRONG_NUM', + ) + + @collect_cases + def test_invalid_int_literal(self) -> Iterator[Case]: + yield Case( + stub=""" + from typing_extensions import Literal + + WRONG_NUM: Literal[1] + """, + runtime='WRONG_NUM = 2', + error='WRONG_NUM', + ) + + @collect_cases + def test_invalid_str_literal(self) -> Iterator[Case]: + yield Case( + stub=""" + from typing_extensions import Literal + + WRONG_STR: Literal['a'] + """, + runtime="WRONG_STR = 'b'", + error='WRONG_STR', + ) + + @collect_cases + def test_invalid_str_bytes_literal(self) -> Iterator[Case]: + yield Case( + stub=""" + from typing_extensions import Literal + + WRONG_STR1: Literal[b'a'] + """, + runtime="WRONG_STR1 = 'a'", + error='WRONG_STR1', + ) + yield Case( + stub="WRONG_STR2: Literal['b']", + runtime="WRONG_STR2 = b'b'", + error='WRONG_STR2', + ) + + @collect_cases + def test_invalid_bytes_literal(self) -> Iterator[Case]: + yield Case( + stub=""" + from typing_extensions import Literal + + WRONG_BYTES1: Literal[b'a'] + """, + runtime="WRONG_BYTES1 = b'b'", + error='WRONG_BYTES1', + ) + + @collect_cases + def test_invalid_bool_literal(self) -> Iterator[Case]: + yield Case( + stub=""" + from typing_extensions import Literal + + WRONG_BOOL1: Literal[True] + """, + runtime="WRONG_BOOL1 = False", + error='WRONG_BOOL1', + ) + yield Case( + stub="WRONG_BOOL2: Literal[False]", + runtime="WRONG_BOOL2 = True", + error='WRONG_BOOL2', + ) + def remove_color_code(s: str) -> str: return re.sub("\\x1b.*?m", "", s) # this works! diff --git a/mypy/util.py b/mypy/util.py index 533e9c6bee6e9..9f620e823e919 100644 --- a/mypy/util.py +++ b/mypy/util.py @@ -105,6 +105,20 @@ def find_python_encoding(text: bytes, pyversion: Tuple[int, int]) -> Tuple[str, return default_encoding, -1 +def bytes_to_human_readable_repr(b: bytes) -> str: + """Converts bytes into some human-readable representation. Unprintable + bytes such as the nul byte are escaped. For example: + + >>> b = bytes([102, 111, 111, 10, 0]) + >>> s = bytes_to_human_readable_repr(b) + >>> print(s) + foo\n\x00 + >>> print(repr(s)) + 'foo\\n\\x00' + """ + return repr(b)[2:-1] + + class DecodeError(Exception): """Exception raised when a file cannot be decoded due to an unknown encoding type. From a1a4c96df5540648e1793932293775f31c59996c Mon Sep 17 00:00:00 2001 From: sobolevn Date: Fri, 7 Jan 2022 12:03:26 +0300 Subject: [PATCH 3/7] Lets try another solution --- mypy/stubtest.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index 4ebfa3248071e..ac579d96826ed 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -942,15 +942,6 @@ def is_subtype_helper(left: mypy.types.Type, right: mypy.types.Type) -> bool: ): # Pretend Literal[0, 1] is a subtype of bool to avoid unhelpful errors. return True - if ( - isinstance(left, mypy.types.LiteralType) - and isinstance(left.value, bytes) - and isinstance(right, mypy.types.LiteralType) - and isinstance(right.value, str) - and right.fallback.type.fullname == 'builtins.bytes' - ): - # This is a representation mismatch. - return bytes_to_human_readable_repr(left.value) == right.value with mypy.state.strict_optional_set(True): return mypy.subtypes.is_subtype(left, right) @@ -1038,6 +1029,8 @@ def anytype() -> mypy.types.AnyType: return mypy.types.TupleType(items, fallback) fallback = mypy.types.Instance(type_info, [anytype() for _ in type_info.type_vars]) + if isinstance(runtime, bytes): + runtime = bytes_to_human_readable_repr(runtime) try: # Literals are supposed to be only bool, int, str, bytes or enums, but this seems to work # well (when not using mypyc, for which bytes and enums are also problematic). From 6c2deea30faf35c5d6255dd99c41ea17d178a243 Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Fri, 7 Jan 2022 01:14:54 -0800 Subject: [PATCH 4/7] fix bytes, enums, collate tests --- mypy/stubtest.py | 11 ++++- mypy/test/teststubtest.py | 100 ++++++++++++++------------------------ 2 files changed, 45 insertions(+), 66 deletions(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index ac579d96826ed..c50f024428ab8 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -713,6 +713,7 @@ def verify_var( and not is_subtype_helper(runtime_type, stub.type) ): should_error = True + # TODO: is this still necessary # Avoid errors when defining enums, since runtime_type is the enum itself, but we'd # annotate it with the type of runtime.value if isinstance(runtime, enum.Enum): @@ -942,6 +943,7 @@ def is_subtype_helper(left: mypy.types.Type, right: mypy.types.Type) -> bool: ): # Pretend Literal[0, 1] is a subtype of bool to avoid unhelpful errors. return True + with mypy.state.strict_optional_set(True): return mypy.subtypes.is_subtype(left, right) @@ -1029,13 +1031,18 @@ def anytype() -> mypy.types.AnyType: return mypy.types.TupleType(items, fallback) fallback = mypy.types.Instance(type_info, [anytype() for _ in type_info.type_vars]) + if isinstance(runtime, bytes): - runtime = bytes_to_human_readable_repr(runtime) + value = bytes_to_human_readable_repr(runtime) + elif isinstance(runtime, enum.Enum): + value = runtime.name + else: + value = runtime try: # Literals are supposed to be only bool, int, str, bytes or enums, but this seems to work # well (when not using mypyc, for which bytes and enums are also problematic). return mypy.types.LiteralType( - value=runtime, + value=value, fallback=fallback, ) except TypeError: diff --git a/mypy/test/teststubtest.py b/mypy/test/teststubtest.py index 4fb97076340ea..6cc5560043376 100644 --- a/mypy/test/teststubtest.py +++ b/mypy/test/teststubtest.py @@ -751,109 +751,81 @@ def __init__(self, x): pass ) @collect_cases - def test_correct_literal(self) -> Iterator[Case]: + def test_good_literal(self) -> Iterator[Case]: yield Case( stub=r""" from typing_extensions import Literal + import enum + class Color(enum.Enum): + RED: int + NUM: Literal[1] CHAR: Literal['a'] FLAG: Literal[True] NON: Literal[None] BYT1: Literal[b'abc'] BYT2: Literal[b'\x90'] + ENUM: Literal[Color.RED] """, runtime=r""" + import enum + class Color(enum.Enum): + RED = 3 + NUM = 1 CHAR = 'a' NON = None FLAG = True BYT1 = b"abc" BYT2 = b'\x90' + ENUM = Color.RED """, error=None, ) @collect_cases - def test_invalid_int_float_literal(self) -> Iterator[Case]: + def test_bad_literal(self) -> Iterator[Case]: + yield Case("from typing_extensions import Literal", "", None) # dummy case yield Case( - stub=""" - from typing_extensions import Literal - - WRONG_NUM: Literal[1] - """, - runtime='WRONG_NUM = 1.0', - error='WRONG_NUM', + stub="INT_FLOAT_MISMATCH: Literal[1]", + runtime="INT_FLOAT_MISMATCH = 1.0", + error="INT_FLOAT_MISMATCH", ) - - @collect_cases - def test_invalid_int_literal(self) -> Iterator[Case]: yield Case( - stub=""" - from typing_extensions import Literal - - WRONG_NUM: Literal[1] - """, - runtime='WRONG_NUM = 2', - error='WRONG_NUM', + stub="WRONG_INT: Literal[1]", + runtime="WRONG_INT = 2", + error="WRONG_INT", ) - - @collect_cases - def test_invalid_str_literal(self) -> Iterator[Case]: yield Case( - stub=""" - from typing_extensions import Literal - - WRONG_STR: Literal['a'] - """, + stub="WRONG_STR: Literal['a']", runtime="WRONG_STR = 'b'", - error='WRONG_STR', + error="WRONG_STR", ) - - @collect_cases - def test_invalid_str_bytes_literal(self) -> Iterator[Case]: yield Case( - stub=""" - from typing_extensions import Literal - - WRONG_STR1: Literal[b'a'] - """, - runtime="WRONG_STR1 = 'a'", - error='WRONG_STR1', + stub="BYTES_STR_MISMATCH: Literal[b'value']", + runtime="BYTES_STR_MISMATCH = 'value'", + error="BYTES_STR_MISMATCH", ) yield Case( - stub="WRONG_STR2: Literal['b']", - runtime="WRONG_STR2 = b'b'", - error='WRONG_STR2', + stub="STR_BYTES_MISMATCH: Literal['value']", + runtime="STR_BYTES_MISMATCH = b'value'", + error="STR_BYTES_MISMATCH", ) - - @collect_cases - def test_invalid_bytes_literal(self) -> Iterator[Case]: yield Case( - stub=""" - from typing_extensions import Literal - - WRONG_BYTES1: Literal[b'a'] - """, - runtime="WRONG_BYTES1 = b'b'", - error='WRONG_BYTES1', + stub="WRONG_BYTES: Literal[b'abc']", + runtime="WRONG_BYTES = b'xyz'", + error="WRONG_BYTES", ) - - @collect_cases - def test_invalid_bool_literal(self) -> Iterator[Case]: yield Case( - stub=""" - from typing_extensions import Literal - - WRONG_BOOL1: Literal[True] - """, - runtime="WRONG_BOOL1 = False", - error='WRONG_BOOL1', + stub="WRONG_BOOL_1: Literal[True]", + runtime="WRONG_BOOL_1 = False", + error='WRONG_BOOL_1', ) yield Case( - stub="WRONG_BOOL2: Literal[False]", - runtime="WRONG_BOOL2 = True", - error='WRONG_BOOL2', + stub="WRONG_BOOL_2: Literal[False]", + runtime="WRONG_BOOL_2 = True", + error='WRONG_BOOL_2', ) From 3fddd32e18bf54ebb9625a415fe10093ab54c369 Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Fri, 7 Jan 2022 01:39:23 -0800 Subject: [PATCH 5/7] remove fallback --- mypy/stubtest.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index c50f024428ab8..615ec2f91cf69 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -713,7 +713,6 @@ def verify_var( and not is_subtype_helper(runtime_type, stub.type) ): should_error = True - # TODO: is this still necessary # Avoid errors when defining enums, since runtime_type is the enum itself, but we'd # annotate it with the type of runtime.value if isinstance(runtime, enum.Enum): @@ -1038,16 +1037,8 @@ def anytype() -> mypy.types.AnyType: value = runtime.name else: value = runtime - try: - # Literals are supposed to be only bool, int, str, bytes or enums, but this seems to work - # well (when not using mypyc, for which bytes and enums are also problematic). - return mypy.types.LiteralType( - value=value, - fallback=fallback, - ) - except TypeError: - # Ask for forgiveness if we're using mypyc. - return fallback + + return mypy.types.LiteralType(value=value, fallback=fallback) _all_stubs: Dict[str, nodes.MypyFile] = {} From 9b976e2bdd153820aa057816cdd87cb072037e3f Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Fri, 7 Jan 2022 02:02:06 -0800 Subject: [PATCH 6/7] more fallback --- mypy/stubtest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index 615ec2f91cf69..409a2b4dfa3a5 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -1035,8 +1035,10 @@ def anytype() -> mypy.types.AnyType: value = bytes_to_human_readable_repr(runtime) elif isinstance(runtime, enum.Enum): value = runtime.name - else: + elif isinstance(runtime, (int, str, bool)): value = runtime + else: + return fallback return mypy.types.LiteralType(value=value, fallback=fallback) From a45c579f7ada8e6a08d54019d639acacd25bb1c3 Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Fri, 7 Jan 2022 02:07:35 -0800 Subject: [PATCH 7/7] fix type check --- mypy/stubtest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index 409a2b4dfa3a5..0a05cf2104099 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -1031,11 +1031,12 @@ def anytype() -> mypy.types.AnyType: fallback = mypy.types.Instance(type_info, [anytype() for _ in type_info.type_vars]) + value: Union[bool, int, str] if isinstance(runtime, bytes): value = bytes_to_human_readable_repr(runtime) elif isinstance(runtime, enum.Enum): value = runtime.name - elif isinstance(runtime, (int, str, bool)): + elif isinstance(runtime, (bool, int, str)): value = runtime else: return fallback