From 292a7fc3b0ac1fbd79526caa91f7de4c83a8e00b Mon Sep 17 00:00:00 2001 From: Cory Jones Date: Tue, 4 Aug 2015 12:41:44 -0400 Subject: [PATCH 1/5] Add support for Python3 parameter annotations --- grammars/python.cson | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grammars/python.cson b/grammars/python.cson index 8fa6e9c..65659b1 100644 --- a/grammars/python.cson +++ b/grammars/python.cson @@ -279,7 +279,7 @@ 'name': 'variable.parameter.function.python' '2': 'name': 'punctuation.separator.parameters.python' - 'match': '\\b([a-zA-Z_][a-zA-Z_0-9]*)\\s*(?:(,)|(?=[\\n\\)]))' + 'match': '\\b([a-zA-Z_][a-zA-Z_0-9]*)\\s*(?:(,)|\\:\\s*\\w*(,*))' } ] } From 3389fafa148580e9b4944f7e51d94c497665720d Mon Sep 17 00:00:00 2001 From: Dario Bertini Date: Sat, 19 Sep 2015 23:21:18 +0100 Subject: [PATCH 2/5] Add function annotations --- grammars/python.cson | 14 +++++++++++--- spec/python-spec.coffee | 14 ++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/grammars/python.cson b/grammars/python.cson index 65659b1..25c7d3b 100644 --- a/grammars/python.cson +++ b/grammars/python.cson @@ -239,13 +239,17 @@ 'beginCaptures': '1': 'name': 'storage.type.function.python' - 'end': '(\\))\\s*(?:(\\:)|(.*$\\n?))' + 'end': '(\\))\\s*(?:(->)\\s*([A-Za-z_][A-Za-z0-9_]*)\\s*)?(?:(\\:)|(.*$\\n?))' 'endCaptures': '1': 'name': 'punctuation.definition.parameters.end.python' '2': - 'name': 'punctuation.section.function.begin.python' + 'name': 'punctuation.definition.annotation.return.python' '3': + 'name': 'variable.annotation.function.python' + '4': + 'name': 'punctuation.section.function.begin.python' + '5': 'name': 'invalid.illegal.missing-section-begin.python' 'name': 'meta.function.python' 'patterns': [ @@ -278,8 +282,12 @@ '1': 'name': 'variable.parameter.function.python' '2': + 'name': 'punctuation.definition.annotation.parameter.python' + '3': + 'name': 'variable.annotation.function.python' + '4': 'name': 'punctuation.separator.parameters.python' - 'match': '\\b([a-zA-Z_][a-zA-Z_0-9]*)\\s*(?:(,)|\\:\\s*\\w*(,*))' + 'match': '\\b([a-zA-Z_][a-zA-Z_0-9]*)\\s*(?:(:)\\s*([a-zA-Z_][a-zA-Z_0-9]*))?(?:(,)|(?=[\\n\\)]))' } ] } diff --git a/spec/python-spec.coffee b/spec/python-spec.coffee index 86bc0ba..37d0f3c 100644 --- a/spec/python-spec.coffee +++ b/spec/python-spec.coffee @@ -724,3 +724,17 @@ describe "Python grammar", -> expect(tokens[0]).toEqual value: '\'', scopes: ['source.python', 'string.quoted.single.single-line.python', 'punctuation.definition.string.begin.python'] expect(tokens[1]).toEqual value: 'WITH example_cte AS (SELECT bar FROM foo) SELECT COUNT(*) FROM example_cte', scopes: ['source.python', 'string.quoted.single.single-line.python'] expect(tokens[2]).toEqual value: '\'', scopes: ['source.python', 'string.quoted.single.single-line.python', 'punctuation.definition.string.end.python'] + + it "tokenizes a function definition with annotations", -> + tokens = grammar.tokenizeLines('def f(a: int) -> int:') + + expect(tokens[0][0]).toEqual value: 'def', scopes: ['source.python', 'meta.function.python', 'storage.type.function.python'] + expect(tokens[0][2]).toEqual value: 'f', scopes: ['source.python', 'meta.function.python', 'entity.name.function.python', 'support.function.magic.python'] + expect(tokens[0][3]).toEqual value: '(', scopes: ['source.python', 'meta.function.python', 'punctuation.definition.parameters.begin.python'] + expect(tokens[1][1]).toEqual value: 'a', scopes: ['source.python', 'meta.function.python', 'meta.function.parameters.python', 'variable.parameter.function.python'] + expect(tokens[1][2]).toEqual value: ':', scopes: ['source.python', 'meta.function.python', 'meta.function.parameters.python', 'punctuation.definition.annotation.parameter.python'] + expect(tokens[2][1]).toEqual value: 'int', scopes: ['source.python', 'meta.function.python', 'meta.function.parameters.python', 'variable.annotation.function.python'] + expect(tokens[4][0]).toEqual value: ')', scopes: ['source.python', 'meta.function.python', 'punctuation.definition.parameters.end.python'] + expect(tokens[4][0]).toEqual value: '->', scopes: ['source.python', 'meta.function.python', 'punctuation.definition.annotation.return.python'] + expect(tokens[4][0]).toEqual value: 'int', scopes: ['source.python', 'meta.function.python', 'variable.annotation.function.python'] + expect(tokens[4][1]).toEqual value: ':', scopes: ['source.python', 'meta.function.python', 'punctuation.section.function.begin.python'] From 40d397b6264c0a48acdb257d02e0cb09d592198b Mon Sep 17 00:00:00 2001 From: Dario Bertini Date: Sat, 17 Oct 2015 13:02:14 +0100 Subject: [PATCH 3/5] Apply some fixes --- grammars/python.cson | 2 +- spec/python-spec.coffee | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/grammars/python.cson b/grammars/python.cson index 25c7d3b..996e41f 100644 --- a/grammars/python.cson +++ b/grammars/python.cson @@ -246,7 +246,7 @@ '2': 'name': 'punctuation.definition.annotation.return.python' '3': - 'name': 'variable.annotation.function.python' + 'name': 'variable.other.annotation' '4': 'name': 'punctuation.section.function.begin.python' '5': diff --git a/spec/python-spec.coffee b/spec/python-spec.coffee index 37d0f3c..d7f1de5 100644 --- a/spec/python-spec.coffee +++ b/spec/python-spec.coffee @@ -729,7 +729,7 @@ describe "Python grammar", -> tokens = grammar.tokenizeLines('def f(a: int) -> int:') expect(tokens[0][0]).toEqual value: 'def', scopes: ['source.python', 'meta.function.python', 'storage.type.function.python'] - expect(tokens[0][2]).toEqual value: 'f', scopes: ['source.python', 'meta.function.python', 'entity.name.function.python', 'support.function.magic.python'] + expect(tokens[0][2]).toEqual value: 'f', scopes: ['source.python', 'meta.function.python', 'entity.name.function.python'] expect(tokens[0][3]).toEqual value: '(', scopes: ['source.python', 'meta.function.python', 'punctuation.definition.parameters.begin.python'] expect(tokens[1][1]).toEqual value: 'a', scopes: ['source.python', 'meta.function.python', 'meta.function.parameters.python', 'variable.parameter.function.python'] expect(tokens[1][2]).toEqual value: ':', scopes: ['source.python', 'meta.function.python', 'meta.function.parameters.python', 'punctuation.definition.annotation.parameter.python'] From d5482d3d4f1964be56299ee80808ef75c6023240 Mon Sep 17 00:00:00 2001 From: Wliu <50Wliu@users.noreply.github.com> Date: Sun, 21 Jan 2018 22:11:35 -0500 Subject: [PATCH 4/5] Tokenize function and parameter annotations --- grammars/python.cson | 66 ++++++++++++++++++++++++++++------------- spec/python-spec.coffee | 37 +++++++++++++---------- 2 files changed, 66 insertions(+), 37 deletions(-) diff --git a/grammars/python.cson b/grammars/python.cson index 996e41f..ac650ac 100644 --- a/grammars/python.cson +++ b/grammars/python.cson @@ -235,22 +235,14 @@ ] } { - 'begin': '^\\s*(def)\\s+(?=[A-Za-z_][A-Za-z0-9_]*\\s*\\()' + 'begin': '^\\s*(def)\\s+(?=[A-Za-z_][\\w_]*\\s*\\()' 'beginCaptures': '1': 'name': 'storage.type.function.python' - 'end': '(\\))\\s*(?:(->)\\s*([A-Za-z_][A-Za-z0-9_]*)\\s*)?(?:(\\:)|(.*$\\n?))' + 'end': ':' 'endCaptures': - '1': - 'name': 'punctuation.definition.parameters.end.python' - '2': - 'name': 'punctuation.definition.annotation.return.python' - '3': - 'name': 'variable.other.annotation' - '4': - 'name': 'punctuation.section.function.begin.python' - '5': - 'name': 'invalid.illegal.missing-section-begin.python' + '0': + 'name': 'punctuation.definition.function.begin.python' 'name': 'meta.function.python' 'patterns': [ { @@ -264,33 +256,65 @@ ] } { - 'begin': '(\\()' + 'begin': '\\(' 'beginCaptures': - '1': + '0': 'name': 'punctuation.definition.parameters.begin.python' + 'end': '\\)' + 'endCaptures': + '0': + 'name': 'punctuation.definition.parameters.end.python' 'contentName': 'meta.function.parameters.python' - 'end': '(?=\\)\\s*\\:)' 'patterns': [ { 'include': '#line_comments' } { - 'include': '#keyword_arguments' + # param = 3 + # param: int = 3 + 'begin': '\\b([a-zA-Z_][\\w_]*)\\s*(?:(:)\\s*([a-zA-Z_][\\w_]*))?\\s*(=)\\s*' + 'beginCaptures': + '1': + 'name': 'variable.parameter.function.python' + '2': + 'name': 'punctuation.separator.python' + '3': + 'name': 'storage.type.python' + '4': + 'name': 'keyword.operator.assignment.python' + 'end': '(?!\\G)' + 'patterns': [ + { + 'include': '$self' + } + ] } { + # param + # param: int + 'match': '\\b([a-zA-Z_][\\w_]*)\\s*(?:(:)\\s*([a-zA-Z_][\\w_]*))?' 'captures': '1': 'name': 'variable.parameter.function.python' '2': - 'name': 'punctuation.definition.annotation.parameter.python' + 'name': 'punctuation.separator.python' '3': - 'name': 'variable.annotation.function.python' - '4': - 'name': 'punctuation.separator.parameters.python' - 'match': '\\b([a-zA-Z_][a-zA-Z_0-9]*)\\s*(?:(:)\\s*([a-zA-Z_][a-zA-Z_0-9]*))?(?:(,)|(?=[\\n\\)]))' + 'name': 'storage.type.python' + } + { + 'match': ',' + 'name': 'punctuation.separator.parameters.python' } ] } + { + 'match': '(->)\\s*([A-Za-z_][\\w_]*)(?=\\s*:)' + 'captures': + '1': + 'name': 'keyword.operator.function-annotation.python' + '2': + 'name': 'storage.type.python' + } ] } { diff --git a/spec/python-spec.coffee b/spec/python-spec.coffee index d7f1de5..8e1f8a0 100644 --- a/spec/python-spec.coffee +++ b/spec/python-spec.coffee @@ -650,7 +650,27 @@ describe "Python grammar", -> expect(tokens[2][5]).toEqual value: ' comment', scopes: ['source.python', 'meta.function.python', 'meta.function.parameters.python', 'comment.line.number-sign.python'] expect(tokens[3][1]).toEqual value: 'config', scopes: ['source.python', 'meta.function.python', 'meta.function.parameters.python', 'variable.parameter.function.python'] expect(tokens[4][0]).toEqual value: ')', scopes: ['source.python', 'meta.function.python', 'punctuation.definition.parameters.end.python'] - expect(tokens[4][1]).toEqual value: ':', scopes: ['source.python', 'meta.function.python', 'punctuation.section.function.begin.python'] + expect(tokens[4][1]).toEqual value: ':', scopes: ['source.python', 'meta.function.python', 'punctuation.definition.function.begin.python'] + + it "tokenizes a function definition with annotations", -> + {tokens} = grammar.tokenizeLine 'def f(a: None, b: int = 3) -> int:' + + expect(tokens[0]).toEqual value: 'def', scopes: ['source.python', 'meta.function.python', 'storage.type.function.python'] + expect(tokens[2]).toEqual value: 'f', scopes: ['source.python', 'meta.function.python', 'entity.name.function.python'] + expect(tokens[3]).toEqual value: '(', scopes: ['source.python', 'meta.function.python', 'punctuation.definition.parameters.begin.python'] + expect(tokens[4]).toEqual value: 'a', scopes: ['source.python', 'meta.function.python', 'meta.function.parameters.python', 'variable.parameter.function.python'] + expect(tokens[5]).toEqual value: ':', scopes: ['source.python', 'meta.function.python', 'meta.function.parameters.python', 'punctuation.separator.python'] + expect(tokens[7]).toEqual value: 'None', scopes: ['source.python', 'meta.function.python', 'meta.function.parameters.python', 'storage.type.python'] + expect(tokens[8]).toEqual value: ',', scopes: ['source.python', 'meta.function.python', 'meta.function.parameters.python', 'punctuation.separator.parameters.python'] + expect(tokens[10]).toEqual value: 'b', scopes: ['source.python', 'meta.function.python', 'meta.function.parameters.python', 'variable.parameter.function.python'] + expect(tokens[11]).toEqual value: ':', scopes: ['source.python', 'meta.function.python', 'meta.function.parameters.python', 'punctuation.separator.python'] + expect(tokens[13]).toEqual value: 'int', scopes: ['source.python', 'meta.function.python', 'meta.function.parameters.python', 'storage.type.python'] + expect(tokens[15]).toEqual value: '=', scopes: ['source.python', 'meta.function.python', 'meta.function.parameters.python', 'keyword.operator.assignment.python'] + expect(tokens[17]).toEqual value: '3', scopes: ['source.python', 'meta.function.python', 'meta.function.parameters.python', 'constant.numeric.integer.decimal.python'] + expect(tokens[18]).toEqual value: ')', scopes: ['source.python', 'meta.function.python', 'punctuation.definition.parameters.end.python'] + expect(tokens[20]).toEqual value: '->', scopes: ['source.python', 'meta.function.python', 'keyword.operator.function-annotation.python'] + expect(tokens[22]).toEqual value: 'int', scopes: ['source.python', 'meta.function.python', 'storage.type.python'] + expect(tokens[23]).toEqual value: ':', scopes: ['source.python', 'meta.function.python', 'punctuation.definition.function.begin.python'] it "tokenizes complex function calls", -> {tokens} = grammar.tokenizeLine "torch.nn.BCELoss()(Variable(bayes_optimal_prob, 1, requires_grad=False), Yvar).data[0]" @@ -718,23 +738,8 @@ describe "Python grammar", -> expect(tokens[9][0]).toEqual value: delim, scopes: ['source.python', scope, 'punctuation.definition.string.end.python'] it "tokenizes SQL inline highlighting on single line with a CTE", -> - {tokens} = grammar.tokenizeLine('\'WITH example_cte AS (SELECT bar FROM foo) SELECT COUNT(*) FROM example_cte\'') expect(tokens[0]).toEqual value: '\'', scopes: ['source.python', 'string.quoted.single.single-line.python', 'punctuation.definition.string.begin.python'] expect(tokens[1]).toEqual value: 'WITH example_cte AS (SELECT bar FROM foo) SELECT COUNT(*) FROM example_cte', scopes: ['source.python', 'string.quoted.single.single-line.python'] expect(tokens[2]).toEqual value: '\'', scopes: ['source.python', 'string.quoted.single.single-line.python', 'punctuation.definition.string.end.python'] - - it "tokenizes a function definition with annotations", -> - tokens = grammar.tokenizeLines('def f(a: int) -> int:') - - expect(tokens[0][0]).toEqual value: 'def', scopes: ['source.python', 'meta.function.python', 'storage.type.function.python'] - expect(tokens[0][2]).toEqual value: 'f', scopes: ['source.python', 'meta.function.python', 'entity.name.function.python'] - expect(tokens[0][3]).toEqual value: '(', scopes: ['source.python', 'meta.function.python', 'punctuation.definition.parameters.begin.python'] - expect(tokens[1][1]).toEqual value: 'a', scopes: ['source.python', 'meta.function.python', 'meta.function.parameters.python', 'variable.parameter.function.python'] - expect(tokens[1][2]).toEqual value: ':', scopes: ['source.python', 'meta.function.python', 'meta.function.parameters.python', 'punctuation.definition.annotation.parameter.python'] - expect(tokens[2][1]).toEqual value: 'int', scopes: ['source.python', 'meta.function.python', 'meta.function.parameters.python', 'variable.annotation.function.python'] - expect(tokens[4][0]).toEqual value: ')', scopes: ['source.python', 'meta.function.python', 'punctuation.definition.parameters.end.python'] - expect(tokens[4][0]).toEqual value: '->', scopes: ['source.python', 'meta.function.python', 'punctuation.definition.annotation.return.python'] - expect(tokens[4][0]).toEqual value: 'int', scopes: ['source.python', 'meta.function.python', 'variable.annotation.function.python'] - expect(tokens[4][1]).toEqual value: ':', scopes: ['source.python', 'meta.function.python', 'punctuation.section.function.begin.python'] From ecaf5645c7a58840694037deb858dd0dc129d578 Mon Sep 17 00:00:00 2001 From: Wliu <50Wliu@users.noreply.github.com> Date: Mon, 22 Jan 2018 20:25:33 -0500 Subject: [PATCH 5/5] Remove highlighting of missing parameters It's a bit too complicated to do now with annotations in the mix. --- grammars/python.cson | 6 ------ spec/python-spec.coffee | 13 ------------- 2 files changed, 19 deletions(-) diff --git a/grammars/python.cson b/grammars/python.cson index ed7ee5b..ca7e36b 100644 --- a/grammars/python.cson +++ b/grammars/python.cson @@ -318,12 +318,6 @@ '2': 'name': 'storage.type.python' } - { - # No match, not at the end of the line, and no opening parentheses - 'begin': '(?!\\G)(?!\\s*$)(?!.*\\()' - 'end': '$' - 'name': 'invalid.illegal.missing-parameters.python' - } ] } { diff --git a/spec/python-spec.coffee b/spec/python-spec.coffee index 4fbf2dd..978fab7 100644 --- a/spec/python-spec.coffee +++ b/spec/python-spec.coffee @@ -634,19 +634,6 @@ describe "Python grammar", -> expect(tokens[2]).toEqual value: 'def', scopes: ['source.python', 'meta.function.python', 'storage.type.function.python'] expect(tokens[4]).toEqual value: 'test', scopes: ['source.python', 'meta.function.python', 'entity.name.function.python'] - it "tokenizes functions that are missing parameters", -> - {tokens} = grammar.tokenizeLine 'def test # whoops' - - expect(tokens[0]).toEqual value: 'def', scopes: ['source.python', 'meta.function.python', 'storage.type.function.python'] - expect(tokens[2]).toEqual value: 'test', scopes: ['source.python', 'meta.function.python', 'entity.name.function.python'] - expect(tokens[3]).toEqual value: ' # whoops', scopes: ['source.python', 'meta.function.python', 'invalid.illegal.missing-parameters.python'] - - {tokens} = grammar.tokenizeLine 'def test:' - - expect(tokens[0]).toEqual value: 'def', scopes: ['source.python', 'meta.function.python', 'storage.type.function.python'] - expect(tokens[2]).toEqual value: 'test', scopes: ['source.python', 'meta.function.python', 'entity.name.function.python'] - expect(tokens[3]).toEqual value: ':', scopes: ['source.python', 'meta.function.python', 'invalid.illegal.missing-parameters.python'] - it "tokenizes comments inside function parameters", -> {tokens} = grammar.tokenizeLine('def test(arg, # comment')