From 1983a03dca3cf1b0024583d02edb0f08134da9a6 Mon Sep 17 00:00:00 2001 From: Elazar Gershuni Date: Mon, 12 Sep 2016 03:15:34 +0300 Subject: [PATCH 1/2] Make Statement a proper subclass of Node --- mypy/fastparse.py | 18 ++++++++++-------- mypy/fastparse2.py | 22 +++++++++++----------- mypy/nodes.py | 21 ++++++++++++++------- mypy/parse.py | 31 ++++++++++++++++--------------- mypy/treetransform.py | 11 +++++++---- 5 files changed, 58 insertions(+), 45 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 7ad71be62462a..112b89d88544c 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -14,11 +14,9 @@ UnaryExpr, FuncExpr, ComparisonExpr, StarExpr, YieldFromExpr, NonlocalDecl, DictionaryComprehension, SetComprehension, ComplexExpr, EllipsisExpr, YieldExpr, Argument, - AwaitExpr, - ARG_POS, ARG_OPT, ARG_STAR, ARG_NAMED, ARG_STAR2 -) + AwaitExpr, ARG_POS, ARG_OPT, ARG_STAR, ARG_NAMED, ARG_STAR2, Statement) from mypy.types import ( - Type, CallableType, FunctionLike, AnyType, UnboundType, TupleType, TypeList, EllipsisType, + Type, CallableType, AnyType, UnboundType, TupleType, TypeList, EllipsisType, ) from mypy import defaults from mypy import experiments @@ -128,6 +126,9 @@ def visit_NoneType(self, n: Any) -> Optional[Node]: def visit_list(self, l: Sequence[ast35.AST]) -> List[Node]: return [self.visit(e) for e in l] + def visit_statement_list(self, l: Sequence[ast35.AST]) -> List[Statement]: + return [self.visit(e) for e in l] + op_map = { ast35.Add: '+', ast35.Sub: '-', @@ -176,12 +177,13 @@ def from_comp_operator(self, op: ast35.cmpop) -> str: def as_block(self, stmts: List[ast35.stmt], lineno: int) -> Block: b = None if stmts: - b = Block(self.fix_function_overloads(self.visit_list(stmts))) + stmt_list = self.visit_statement_list(stmts) + b = Block(self.fix_function_overloads(stmt_list)) b.set_line(lineno) return b - def fix_function_overloads(self, stmts: List[Node]) -> List[Node]: - ret = [] # type: List[Node] + def fix_function_overloads(self, stmts: List[Statement]) -> List[Statement]: + ret = [] # type: List[Statement] current_overload = [] current_overload_name = None # mypy doesn't actually check that the decorator is literally @overload @@ -225,7 +227,7 @@ def translate_module_id(self, id: str) -> str: return id def visit_Module(self, mod: ast35.Module) -> Node: - body = self.fix_function_overloads(self.visit_list(mod.body)) + body = self.fix_function_overloads(self.visit_statement_list(mod.body)) return MypyFile(body, self.imports, diff --git a/mypy/fastparse2.py b/mypy/fastparse2.py index 25e5bcb78fbcb..216dfa2d53524 100644 --- a/mypy/fastparse2.py +++ b/mypy/fastparse2.py @@ -27,14 +27,12 @@ TupleExpr, GeneratorExpr, ListComprehension, ListExpr, ConditionalExpr, DictExpr, SetExpr, NameExpr, IntExpr, StrExpr, BytesExpr, UnicodeExpr, FloatExpr, CallExpr, SuperExpr, MemberExpr, IndexExpr, SliceExpr, OpExpr, - UnaryExpr, FuncExpr, ComparisonExpr, - StarExpr, YieldFromExpr, NonlocalDecl, DictionaryComprehension, - SetComprehension, ComplexExpr, EllipsisExpr, YieldExpr, Argument, - AwaitExpr, Expression, - ARG_POS, ARG_OPT, ARG_STAR, ARG_NAMED, ARG_STAR2 + UnaryExpr, FuncExpr, ComparisonExpr, DictionaryComprehension, + SetComprehension, ComplexExpr, EllipsisExpr, YieldExpr, Argument, Expression, + ARG_POS, ARG_OPT, ARG_STAR, ARG_NAMED, ARG_STAR2, Statement ) from mypy.types import ( - Type, CallableType, FunctionLike, AnyType, UnboundType, TupleType, TypeList, EllipsisType, + Type, CallableType, AnyType, UnboundType, ) from mypy import defaults from mypy import experiments @@ -44,7 +42,6 @@ try: from typed_ast import ast27 from typed_ast import ast35 - from typed_ast import conversions except ImportError: if sys.version_info.minor > 2: print('You must install the typed_ast package before you can run mypy' @@ -147,6 +144,9 @@ def visit_NoneType(self, n: Any) -> Optional[Node]: def visit_list(self, l: Sequence[ast27.AST]) -> List[Node]: return [self.visit(e) for e in l] + def visit_statement_list(self, l: Sequence[ast27.AST]) -> List[Statement]: + return [self.visit(e) for e in l] + op_map = { ast27.Add: '+', ast27.Sub: '-', @@ -194,12 +194,12 @@ def from_comp_operator(self, op: ast27.cmpop) -> str: def as_block(self, stmts: List[ast27.stmt], lineno: int) -> Block: b = None if stmts: - b = Block(self.fix_function_overloads(self.visit_list(stmts))) + b = Block(self.fix_function_overloads(self.visit_statement_list(stmts))) b.set_line(lineno) return b - def fix_function_overloads(self, stmts: List[Node]) -> List[Node]: - ret = [] # type: List[Node] + def fix_function_overloads(self, stmts: List[Statement]) -> List[Statement]: + ret = [] # type: List[Statement] current_overload = [] current_overload_name = None # mypy doesn't actually check that the decorator is literally @overload @@ -243,7 +243,7 @@ def translate_module_id(self, id: str) -> str: return id def visit_Module(self, mod: ast27.Module) -> Node: - body = self.fix_function_overloads(self.visit_list(mod.body)) + body = self.fix_function_overloads(self.visit_statement_list(mod.body)) return MypyFile(body, self.imports, diff --git a/mypy/nodes.py b/mypy/nodes.py index d989cf680f9cb..27aa749df9ae5 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1,11 +1,10 @@ """Abstract syntax tree node classes (i.e. parse tree).""" import os -import re -from abc import abstractmethod, ABCMeta +from abc import abstractmethod from typing import ( - Any, TypeVar, List, Tuple, cast, Set, Dict, Union, Optional + Any, TypeVar, List, Tuple, cast, Set, Dict, Union, Optional, NewType ) from mypy.lex import Token @@ -117,13 +116,21 @@ def accept(self, visitor: NodeVisitor[T]) -> T: raise RuntimeError('Not implemented') -# These are placeholders for a future refactoring; see #1783. -# For now they serve as (unchecked) documentation of what various +# This is a placeholder for a future refactoring; see #1783. +# For now it serves as (unchecked) documentation of what various # fields of Node subtypes are expected to contain. -Statement = Node Expression = Node +class Statement(Node): + pass + + +# For now, only Var is a declaration +class Declaration(Node): + pass + + class SymbolNode(Node): # Nodes that can be stored in a symbol table. @@ -572,7 +579,7 @@ def deserialize(cls, data: JsonDict) -> 'Decorator': return dec -class Var(SymbolNode, Statement): +class Var(SymbolNode, Declaration): """A variable. It can refer to global/local variable or a data attribute. diff --git a/mypy/parse.py b/mypy/parse.py index 39979d7126b88..c6638e6cb2a71 100644 --- a/mypy/parse.py +++ b/mypy/parse.py @@ -14,7 +14,6 @@ UnicodeLit, FloatLit, Op, Indent, Keyword, Punct, LexError, ComplexLit, EllipsisToken ) -import mypy.types from mypy.nodes import ( MypyFile, Import, Node, ImportAll, ImportFrom, FuncDef, OverloadedFuncDef, ClassDef, Decorator, Block, Var, OperatorAssignmentStmt, @@ -27,14 +26,14 @@ UnaryExpr, FuncExpr, PrintStmt, ImportBase, ComparisonExpr, StarExpr, YieldFromExpr, NonlocalDecl, DictionaryComprehension, SetComprehension, ComplexExpr, EllipsisExpr, YieldExpr, ExecStmt, Argument, - BackquoteExpr + BackquoteExpr, Statement ) from mypy import defaults from mypy import nodes from mypy.errors import Errors, CompileError from mypy.types import Type, CallableType, AnyType, UnboundType from mypy.parsetype import ( - parse_type, parse_types, parse_signature, TypeParseError, parse_str_as_signature + parse_type, parse_types, parse_signature, TypeParseError ) from mypy.options import Options @@ -235,7 +234,7 @@ def translate_module_id(self, id: str) -> str: return 'builtins' return id - def parse_import_from(self) -> Node: + def parse_import_from(self) -> Statement: self.expect('from') # Build the list of beginning relative tokens. @@ -318,8 +317,8 @@ def parse_qualified_name(self) -> str: # Parsing global definitions - def parse_defs(self) -> List[Node]: - defs = [] # type: List[Node] + def parse_defs(self) -> List[Statement]: + defs = [] # type: List[Statement] while not self.eof(): try: defn, is_simple = self.parse_statement() @@ -394,7 +393,7 @@ def parse_class_keywords(self) -> Optional[str]: def parse_super_type(self) -> Node: return self.parse_expression(precedence[',']) - def parse_decorated_function_or_class(self) -> Node: + def parse_decorated_function_or_class(self) -> Statement: decorators = [] no_type_checks = False while self.current_str() == '@': @@ -868,7 +867,7 @@ def parse_block(self, allow_type: bool = False) -> Tuple[Block, Type]: brk = self.expect_break() type = self.parse_type_comment(brk, signature=True) self.expect_indent() - stmt_list = [] # type: List[Node] + stmt_list = [] # type: List[Statement] while (not isinstance(self.current(), Dedent) and not isinstance(self.current(), Eof)): try: @@ -886,7 +885,7 @@ def parse_block(self, allow_type: bool = False) -> Tuple[Block, Type]: node.set_line(colon) return node, type - def try_combine_overloads(self, s: Node, stmt: List[Node]) -> bool: + def try_combine_overloads(self, s: Node, stmt: List[Statement]) -> bool: if isinstance(s, Decorator) and stmt: fdef = s n = fdef.func.name() @@ -898,8 +897,8 @@ def try_combine_overloads(self, s: Node, stmt: List[Node]) -> bool: return True return False - def parse_statement(self) -> Tuple[Node, bool]: - stmt = None # type: Node + def parse_statement(self) -> Tuple[Statement, bool]: + stmt = None # type: Statement t = self.current() ts = self.current_str() is_simple = True # Is this a non-block statement? @@ -964,7 +963,7 @@ def parse_statement(self) -> Tuple[Node, bool]: stmt.set_line(t) return stmt, is_simple - def parse_expression_or_assignment(self) -> Node: + def parse_expression_or_assignment(self) -> Statement: expr = self.parse_expression(star_expr_allowed=True) if self.current_str() == '=': return self.parse_assignment(expr) @@ -978,7 +977,7 @@ def parse_expression_or_assignment(self) -> Node: # Expression statement. return ExpressionStmt(expr) - def parse_assignment(self, lvalue: Any) -> Node: + def parse_assignment(self, lvalue: Any) -> AssignmentStmt: """Parse an assignment statement. Assume that lvalue has been parsed already, and the current token is '='. @@ -1184,7 +1183,7 @@ def parse_if_stmt(self) -> IfStmt: else: return None - def parse_try_stmt(self) -> Node: + def parse_try_stmt(self) -> Statement: self.expect('try') body, _ = self.parse_block() is_error = False @@ -1813,7 +1812,9 @@ def parse_lambda_expr(self) -> FuncExpr: expr = self.parse_expression(precedence[',']) - nodes = [ReturnStmt(expr).set_line(lambda_tok)] + ret = ReturnStmt(expr) + ret.set_line(lambda_tok) + nodes = [ret] # type: List[Statement] # Potentially insert extra assignment statements to the beginning of the # body, used to decompose Python 2 tuple arguments. nodes[:0] = extra_stmts diff --git a/mypy/treetransform.py b/mypy/treetransform.py index b6143d4a7fefb..02a0310538708 100644 --- a/mypy/treetransform.py +++ b/mypy/treetransform.py @@ -19,9 +19,9 @@ ComparisonExpr, TempNode, StarExpr, YieldFromExpr, NamedTupleExpr, NonlocalDecl, SetComprehension, DictionaryComprehension, ComplexExpr, TypeAliasExpr, EllipsisExpr, - YieldExpr, ExecStmt, Argument, BackquoteExpr, AwaitExpr, + YieldExpr, ExecStmt, Argument, BackquoteExpr, AwaitExpr, Statement ) -from mypy.types import Type, FunctionLike, Instance +from mypy.types import Type, FunctionLike from mypy.traverser import TraverserVisitor from mypy.visitor import NodeVisitor @@ -57,7 +57,7 @@ def __init__(self) -> None: def visit_mypy_file(self, node: MypyFile) -> Node: # NOTE: The 'names' and 'imports' instance variables will be empty! - new = MypyFile(self.nodes(node.defs), [], node.is_bom, + new = MypyFile(self.statements(node.defs), [], node.is_bom, ignored_lines=set(node.ignored_lines)) new._name = node._name new._fullname = node._fullname @@ -200,7 +200,7 @@ def visit_nonlocal_decl(self, node: NonlocalDecl) -> Node: return NonlocalDecl(node.names[:]) def visit_block(self, node: Block) -> Block: - return Block(self.nodes(node.body)) + return Block(self.statements(node.body)) def visit_decorator(self, node: Decorator) -> Decorator: # Note that a Decorator must be transformed to a Decorator. @@ -524,6 +524,9 @@ def optional_block(self, block: Block) -> Block: def nodes(self, nodes: List[Node]) -> List[Node]: return [self.node(node) for node in nodes] + def statements(self, nodes: List[Statement]) -> List[Statement]: + return [cast(Statement, self.node(node)) for node in nodes] + def optional_nodes(self, nodes: List[Node]) -> List[Node]: return [self.optional_node(node) for node in nodes] From 9c59230d46301980a945e1d1fc5fd42697ee3869 Mon Sep 17 00:00:00 2001 From: Elazar Date: Mon, 12 Sep 2016 03:36:06 +0300 Subject: [PATCH 2/2] remove NewType from import list --- mypy/nodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index 27aa749df9ae5..a310ddadd603b 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -4,7 +4,7 @@ from abc import abstractmethod from typing import ( - Any, TypeVar, List, Tuple, cast, Set, Dict, Union, Optional, NewType + Any, TypeVar, List, Tuple, cast, Set, Dict, Union, Optional ) from mypy.lex import Token